1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2025 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_uio_control.h" 12 #include "adf_uio_cleanup.h" 13 #include "adf_uio.h" 14 #include "adf_transport_access_macros.h" 15 #include "adf_transport_internal.h" 16 17 #define ADF_DEV_PROCESSES_NAME "qat_dev_processes" 18 #define ADF_DEV_STATE_NAME "qat_dev_state" 19 20 #define ADF_STATE_CALLOUT_TIME 10 21 22 static const char *mtx_name = "state_mtx"; 23 static const char *mtx_callout_name = "callout_mtx"; 24 25 static d_open_t adf_processes_open; 26 static void adf_processes_release(void *data); 27 static d_read_t adf_processes_read; 28 static d_write_t adf_processes_write; 29 30 static d_open_t adf_state_open; 31 static void adf_state_release(void *data); 32 static d_read_t adf_state_read; 33 static int adf_state_kqfilter(struct cdev *dev, struct knote *kn); 34 static int adf_state_kqread_event(struct knote *kn, long hint); 35 static void adf_state_kqread_detach(struct knote *kn); 36 37 static struct callout callout; 38 static struct mtx mtx; 39 static struct mtx callout_mtx; 40 static struct service_hndl adf_state_hndl; 41 42 struct entry_proc_events { 43 struct adf_state_priv_data *proc_events; 44 45 SLIST_ENTRY(entry_proc_events) entries_proc_events; 46 }; 47 48 struct entry_state { 49 struct adf_state state; 50 51 STAILQ_ENTRY(entry_state) entries_state; 52 }; 53 54 SLIST_HEAD(proc_events_head, entry_proc_events); 55 STAILQ_HEAD(state_head, entry_state); 56 57 static struct proc_events_head proc_events_head; 58 59 struct adf_processes_priv_data { 60 char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; 61 int read_flag; 62 struct list_head list; 63 }; 64 65 struct adf_state_priv_data { 66 struct cdev *cdev; 67 struct selinfo rsel; 68 struct state_head state_head; 69 }; 70 71 static struct cdevsw adf_processes_cdevsw = { 72 .d_version = D_VERSION, 73 .d_open = adf_processes_open, 74 .d_read = adf_processes_read, 75 .d_write = adf_processes_write, 76 .d_name = ADF_DEV_PROCESSES_NAME, 77 }; 78 79 static struct cdevsw adf_state_cdevsw = { 80 .d_version = D_VERSION, 81 .d_open = adf_state_open, 82 .d_read = adf_state_read, 83 .d_kqfilter = adf_state_kqfilter, 84 .d_name = ADF_DEV_STATE_NAME, 85 }; 86 87 static struct filterops adf_state_read_filterops = { 88 .f_isfd = 1, 89 .f_attach = NULL, 90 .f_detach = adf_state_kqread_detach, 91 .f_event = adf_state_kqread_event, 92 .f_copy = knote_triv_copy, 93 }; 94 95 static struct cdev *adf_processes_dev; 96 static struct cdev *adf_state_dev; 97 98 static LINUX_LIST_HEAD(processes_list); 99 100 struct sx processes_list_sema; 101 SX_SYSINIT(processes_list_sema, &processes_list_sema, "adf proc list"); 102 103 static void 104 adf_chr_drv_destroy(void) 105 { 106 destroy_dev(adf_processes_dev); 107 } 108 109 static int 110 adf_chr_drv_create(void) 111 { 112 113 adf_processes_dev = make_dev(&adf_processes_cdevsw, 114 0, 115 UID_ROOT, 116 GID_WHEEL, 117 0600, 118 ADF_DEV_PROCESSES_NAME); 119 if (adf_processes_dev == NULL) { 120 printf("QAT: failed to create device\n"); 121 goto err_cdev_del; 122 } 123 return 0; 124 err_cdev_del: 125 return EFAULT; 126 } 127 128 static int 129 adf_processes_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 130 { 131 int i = 0, devices = 0; 132 struct adf_accel_dev *accel_dev = NULL; 133 struct adf_processes_priv_data *prv_data = NULL; 134 int error = 0; 135 136 for (i = 0; i < ADF_MAX_DEVICES; i++) { 137 accel_dev = adf_devmgr_get_dev_by_id(i); 138 if (!accel_dev) 139 continue; 140 if (!adf_dev_started(accel_dev)) 141 continue; 142 devices++; 143 } 144 if (!devices) { 145 printf("QAT: No active devices found.\n"); 146 return ENXIO; 147 } 148 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO); 149 INIT_LIST_HEAD(&prv_data->list); 150 error = devfs_set_cdevpriv(prv_data, adf_processes_release); 151 if (error) { 152 free(prv_data, M_QAT); 153 return error; 154 } 155 156 return 0; 157 } 158 159 static int 160 adf_get_first_started_dev(void) 161 { 162 int i = 0; 163 struct adf_accel_dev *accel_dev = NULL; 164 165 for (i = 0; i < ADF_MAX_DEVICES; i++) { 166 accel_dev = adf_devmgr_get_dev_by_id(i); 167 if (!accel_dev) 168 continue; 169 if (adf_dev_started(accel_dev)) 170 return i; 171 } 172 173 return -1; 174 } 175 176 static int 177 adf_processes_write(struct cdev *dev, struct uio *uio, int ioflag) 178 { 179 struct adf_processes_priv_data *prv_data = NULL; 180 struct adf_processes_priv_data *pdata = NULL; 181 int dev_num = 0, pr_num = 0; 182 struct list_head *lpos = NULL; 183 char usr_name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES] = { 0 }; 184 struct adf_accel_dev *accel_dev = NULL; 185 struct adf_cfg_section *section_ptr = NULL; 186 bool pr_name_available = 1; 187 uint32_t num_accel_devs = 0; 188 int error = 0; 189 ssize_t count; 190 int dev_id; 191 192 error = devfs_get_cdevpriv((void **)&prv_data); 193 if (error) { 194 printf("QAT: invalid file descriptor\n"); 195 return error; 196 } 197 198 if (prv_data->read_flag == 1) { 199 printf("QAT: can only write once\n"); 200 return EBADF; 201 } 202 count = uio->uio_resid; 203 if ((count <= 0) || (count > ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) { 204 printf("QAT: wrong size %d\n", (int)count); 205 return EIO; 206 } 207 208 error = uiomove(usr_name, count, uio); 209 if (error) { 210 printf("QAT: can't copy data\n"); 211 return error; 212 } 213 214 /* Lock other processes and try to find out the process name */ 215 if (sx_xlock_sig(&processes_list_sema)) { 216 printf("QAT: can't aquire process info lock\n"); 217 return EBADF; 218 } 219 220 dev_id = adf_get_first_started_dev(); 221 if (-1 == dev_id) { 222 pr_err("QAT: could not find started device\n"); 223 sx_xunlock(&processes_list_sema); 224 return -EIO; 225 } 226 227 accel_dev = adf_devmgr_get_dev_by_id(dev_id); 228 if (!accel_dev) { 229 pr_err("QAT: could not find started device\n"); 230 sx_xunlock(&processes_list_sema); 231 return -EIO; 232 } 233 234 /* If there is nothing there then take the first name and return */ 235 if (list_empty(&processes_list)) { 236 snprintf(prv_data->name, 237 ADF_CFG_MAX_SECTION_LEN_IN_BYTES, 238 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d", 239 usr_name, 240 0); 241 list_add(&prv_data->list, &processes_list); 242 sx_xunlock(&processes_list_sema); 243 prv_data->read_flag = 1; 244 return 0; 245 } 246 247 /* If there are processes running then search for a first free name */ 248 adf_devmgr_get_num_dev(&num_accel_devs); 249 for (dev_num = 0; dev_num < num_accel_devs; dev_num++) { 250 accel_dev = adf_devmgr_get_dev_by_id(dev_num); 251 if (!accel_dev) 252 continue; 253 254 if (!adf_dev_started(accel_dev)) 255 continue; /* to next device */ 256 257 for (pr_num = 0; pr_num < GET_MAX_PROCESSES(accel_dev); 258 pr_num++) { 259 snprintf(prv_data->name, 260 ADF_CFG_MAX_SECTION_LEN_IN_BYTES, 261 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d", 262 usr_name, 263 pr_num); 264 pr_name_available = 1; 265 /* Figure out if section exists in the config table */ 266 section_ptr = 267 adf_cfg_sec_find(accel_dev, prv_data->name); 268 if (NULL == section_ptr) { 269 /* This section name doesn't exist */ 270 pr_name_available = 0; 271 /* As process_num enumerates from 0, once we get 272 * to one which doesn't exist no further ones 273 * will exist. On to next device 274 */ 275 break; 276 } 277 /* Figure out if it's been taken already */ 278 list_for_each(lpos, &processes_list) 279 { 280 pdata = 281 list_entry(lpos, 282 struct adf_processes_priv_data, 283 list); 284 if (!strncmp( 285 pdata->name, 286 prv_data->name, 287 ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) { 288 pr_name_available = 0; 289 break; 290 } 291 } 292 if (pr_name_available) 293 break; 294 } 295 if (pr_name_available) 296 break; 297 } 298 /* 299 * If we have a valid name that is not on 300 * the list take it and add to the list 301 */ 302 if (pr_name_available) { 303 list_add(&prv_data->list, &processes_list); 304 sx_xunlock(&processes_list_sema); 305 prv_data->read_flag = 1; 306 return 0; 307 } 308 /* If not then the process needs to wait */ 309 sx_xunlock(&processes_list_sema); 310 explicit_bzero(prv_data->name, ADF_CFG_MAX_SECTION_LEN_IN_BYTES); 311 prv_data->read_flag = 0; 312 return 1; 313 } 314 315 static int 316 adf_processes_read(struct cdev *dev, struct uio *uio, int ioflag) 317 { 318 struct adf_processes_priv_data *prv_data = NULL; 319 int error = 0; 320 321 error = devfs_get_cdevpriv((void **)&prv_data); 322 if (error) { 323 printf("QAT: invalid file descriptor\n"); 324 return error; 325 } 326 327 /* 328 * If there is a name that the process can use then give it 329 * to the proocess. 330 */ 331 if (prv_data->read_flag) { 332 error = uiomove(prv_data->name, 333 strnlen(prv_data->name, 334 ADF_CFG_MAX_SECTION_LEN_IN_BYTES), 335 uio); 336 if (error) { 337 printf("QAT: failed to copy data to user\n"); 338 return error; 339 } 340 return 0; 341 } 342 343 return EIO; 344 } 345 346 static void 347 adf_processes_release(void *data) 348 { 349 struct adf_processes_priv_data *prv_data = NULL; 350 351 prv_data = (struct adf_processes_priv_data *)data; 352 sx_xlock(&processes_list_sema); 353 list_del(&prv_data->list); 354 sx_xunlock(&processes_list_sema); 355 free(prv_data, M_QAT); 356 } 357 358 int 359 adf_processes_dev_register(void) 360 { 361 return adf_chr_drv_create(); 362 } 363 364 void 365 adf_processes_dev_unregister(void) 366 { 367 adf_chr_drv_destroy(); 368 } 369 370 static void 371 adf_state_callout_notify_ev(void *arg) 372 { 373 int notified = 0; 374 struct adf_state_priv_data *priv = NULL; 375 struct entry_proc_events *proc_events = NULL; 376 377 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 378 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) { 379 notified = 1; 380 priv = proc_events->proc_events; 381 wakeup(priv); 382 selwakeup(&priv->rsel); 383 KNOTE_UNLOCKED(&priv->rsel.si_note, 0); 384 } 385 } 386 if (notified) 387 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 388 } 389 390 static void 391 adf_state_set(int dev, enum adf_event event) 392 { 393 struct adf_accel_dev *accel_dev = NULL; 394 struct state_head *head = NULL; 395 struct entry_proc_events *proc_events = NULL; 396 struct entry_state *state = NULL; 397 398 accel_dev = adf_devmgr_get_dev_by_id(dev); 399 if (!accel_dev) 400 return; 401 mtx_lock(&mtx); 402 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 403 state = NULL; 404 head = &proc_events->proc_events->state_head; 405 state = malloc(sizeof(struct entry_state), 406 M_QAT, 407 M_NOWAIT | M_ZERO); 408 if (!state) 409 continue; 410 state->state.dev_state = event; 411 state->state.dev_id = dev; 412 STAILQ_INSERT_TAIL(head, state, entries_state); 413 } 414 mtx_unlock(&mtx); 415 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 416 } 417 418 static int 419 adf_state_event_handler(struct adf_accel_dev *accel_dev, enum adf_event event) 420 { 421 int ret = 0; 422 423 #if defined(QAT_UIO) && defined(QAT_DBG) 424 if (event > ADF_EVENT_DBG_SHUTDOWN) 425 return -EINVAL; 426 #else 427 if (event > ADF_EVENT_ERROR) 428 return -EINVAL; 429 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */ 430 431 switch (event) { 432 case ADF_EVENT_INIT: 433 return ret; 434 case ADF_EVENT_SHUTDOWN: 435 return ret; 436 case ADF_EVENT_RESTARTING: 437 break; 438 case ADF_EVENT_RESTARTED: 439 break; 440 case ADF_EVENT_START: 441 return ret; 442 case ADF_EVENT_STOP: 443 return ret; 444 case ADF_EVENT_ERROR: 445 break; 446 #if defined(QAT_UIO) && defined(QAT_DBG) 447 case ADF_EVENT_PROC_CRASH: 448 break; 449 case ADF_EVENT_MANUAL_DUMP: 450 break; 451 case ADF_EVENT_SLICE_HANG: 452 break; 453 case ADF_EVENT_DBG_SHUTDOWN: 454 break; 455 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */ 456 default: 457 return -1; 458 } 459 460 adf_state_set(accel_dev->accel_id, event); 461 462 return 0; 463 } 464 465 static int 466 adf_state_kqfilter(struct cdev *dev, struct knote *kn) 467 { 468 struct adf_state_priv_data *priv; 469 470 mtx_lock(&mtx); 471 priv = dev->si_drv1; 472 switch (kn->kn_filter) { 473 case EVFILT_READ: 474 kn->kn_fop = &adf_state_read_filterops; 475 kn->kn_hook = priv; 476 knlist_add(&priv->rsel.si_note, kn, 1); 477 mtx_unlock(&mtx); 478 return 0; 479 default: 480 mtx_unlock(&mtx); 481 return -EINVAL; 482 } 483 } 484 485 static int 486 adf_state_kqread_event(struct knote *kn, long hint) 487 { 488 return 1; 489 } 490 491 static void 492 adf_state_kqread_detach(struct knote *kn) 493 { 494 struct adf_state_priv_data *priv = NULL; 495 496 mtx_lock(&mtx); 497 if (!kn) { 498 mtx_unlock(&mtx); 499 return; 500 } 501 priv = kn->kn_hook; 502 if (!priv) { 503 mtx_unlock(&mtx); 504 return; 505 } 506 knlist_remove(&priv->rsel.si_note, kn, 1); 507 mtx_unlock(&mtx); 508 } 509 510 void 511 adf_state_init(void) 512 { 513 adf_state_dev = make_dev(&adf_state_cdevsw, 514 0, 515 UID_ROOT, 516 GID_WHEEL, 517 0600, 518 "%s", 519 ADF_DEV_STATE_NAME); 520 SLIST_INIT(&proc_events_head); 521 mtx_init(&mtx, mtx_name, NULL, MTX_DEF); 522 mtx_init(&callout_mtx, mtx_callout_name, NULL, MTX_DEF); 523 callout_init_mtx(&callout, &callout_mtx, 0); 524 explicit_bzero(&adf_state_hndl, sizeof(adf_state_hndl)); 525 adf_state_hndl.event_hld = adf_state_event_handler; 526 adf_state_hndl.name = "adf_state_event_handler"; 527 adf_service_register(&adf_state_hndl); 528 callout_reset(&callout, 529 ADF_STATE_CALLOUT_TIME, 530 adf_state_callout_notify_ev, 531 NULL); 532 } 533 534 void 535 adf_state_destroy(void) 536 { 537 struct entry_proc_events *proc_events = NULL; 538 539 adf_service_unregister(&adf_state_hndl); 540 destroy_dev(adf_state_dev); 541 mtx_lock(&callout_mtx); 542 callout_stop(&callout); 543 mtx_unlock(&callout_mtx); 544 mtx_destroy(&callout_mtx); 545 mtx_lock(&mtx); 546 while (!SLIST_EMPTY(&proc_events_head)) { 547 proc_events = SLIST_FIRST(&proc_events_head); 548 SLIST_REMOVE_HEAD(&proc_events_head, entries_proc_events); 549 free(proc_events, M_QAT); 550 } 551 mtx_unlock(&mtx); 552 mtx_destroy(&mtx); 553 } 554 555 static int 556 adf_state_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 557 { 558 struct adf_state_priv_data *prv_data = NULL; 559 struct entry_proc_events *entry_proc_events = NULL; 560 int ret = 0; 561 562 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO); 563 entry_proc_events = 564 malloc(sizeof(struct entry_proc_events), M_QAT, M_WAITOK | M_ZERO); 565 mtx_lock(&mtx); 566 prv_data->cdev = dev; 567 prv_data->cdev->si_drv1 = prv_data; 568 knlist_init_mtx(&prv_data->rsel.si_note, &mtx); 569 STAILQ_INIT(&prv_data->state_head); 570 entry_proc_events->proc_events = prv_data; 571 SLIST_INSERT_HEAD(&proc_events_head, 572 entry_proc_events, 573 entries_proc_events); 574 mtx_unlock(&mtx); 575 ret = devfs_set_cdevpriv(prv_data, adf_state_release); 576 if (ret) { 577 SLIST_REMOVE(&proc_events_head, 578 entry_proc_events, 579 entry_proc_events, 580 entries_proc_events); 581 free(entry_proc_events, M_QAT); 582 free(prv_data, M_QAT); 583 } 584 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 585 return ret; 586 } 587 588 static int 589 adf_state_read(struct cdev *dev, struct uio *uio, int ioflag) 590 { 591 int ret = 0; 592 struct adf_state_priv_data *prv_data = NULL; 593 struct state_head *state_head = NULL; 594 struct entry_state *entry_state = NULL; 595 struct adf_state *state = NULL; 596 struct entry_proc_events *proc_events = NULL; 597 598 mtx_lock(&mtx); 599 ret = devfs_get_cdevpriv((void **)&prv_data); 600 if (ret) { 601 mtx_unlock(&mtx); 602 return 0; 603 } 604 state_head = &prv_data->state_head; 605 if (STAILQ_EMPTY(state_head)) { 606 mtx_unlock(&mtx); 607 return 0; 608 } 609 entry_state = STAILQ_FIRST(state_head); 610 state = &entry_state->state; 611 ret = uiomove(state, sizeof(struct adf_state), uio); 612 if (!ret && !STAILQ_EMPTY(state_head)) { 613 STAILQ_REMOVE_HEAD(state_head, entries_state); 614 free(entry_state, M_QAT); 615 } 616 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 617 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) { 618 prv_data = proc_events->proc_events; 619 wakeup(prv_data); 620 selwakeup(&prv_data->rsel); 621 KNOTE_UNLOCKED(&prv_data->rsel.si_note, 0); 622 } 623 } 624 mtx_unlock(&mtx); 625 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 626 return ret; 627 } 628 629 static void 630 adf_state_release(void *data) 631 { 632 struct adf_state_priv_data *prv_data = NULL; 633 struct entry_state *entry_state = NULL; 634 struct entry_proc_events *entry_proc_events = NULL; 635 struct entry_proc_events *tmp = NULL; 636 637 mtx_lock(&mtx); 638 prv_data = (struct adf_state_priv_data *)data; 639 knlist_delete(&prv_data->rsel.si_note, curthread, 1); 640 knlist_destroy(&prv_data->rsel.si_note); 641 seldrain(&prv_data->rsel); 642 while (!STAILQ_EMPTY(&prv_data->state_head)) { 643 entry_state = STAILQ_FIRST(&prv_data->state_head); 644 STAILQ_REMOVE_HEAD(&prv_data->state_head, entries_state); 645 free(entry_state, M_QAT); 646 } 647 SLIST_FOREACH_SAFE (entry_proc_events, 648 &proc_events_head, 649 entries_proc_events, 650 tmp) { 651 if (entry_proc_events->proc_events == prv_data) { 652 SLIST_REMOVE(&proc_events_head, 653 entry_proc_events, 654 entry_proc_events, 655 entries_proc_events); 656 free(entry_proc_events, M_QAT); 657 } 658 } 659 free(prv_data, M_QAT); 660 mtx_unlock(&mtx); 661 } 662