1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2025 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_uio_control.h" 12 #include "adf_uio_cleanup.h" 13 #include "adf_uio.h" 14 #include "adf_transport_access_macros.h" 15 #include "adf_transport_internal.h" 16 17 #define ADF_DEV_PROCESSES_NAME "qat_dev_processes" 18 #define ADF_DEV_STATE_NAME "qat_dev_state" 19 20 #define ADF_STATE_CALLOUT_TIME 10 21 22 static const char *mtx_name = "state_mtx"; 23 static const char *mtx_callout_name = "callout_mtx"; 24 25 static d_open_t adf_processes_open; 26 static void adf_processes_release(void *data); 27 static d_read_t adf_processes_read; 28 static d_write_t adf_processes_write; 29 30 static d_open_t adf_state_open; 31 static void adf_state_release(void *data); 32 static d_read_t adf_state_read; 33 static int adf_state_kqfilter(struct cdev *dev, struct knote *kn); 34 static int adf_state_kqread_event(struct knote *kn, long hint); 35 static void adf_state_kqread_detach(struct knote *kn); 36 37 static struct callout callout; 38 static struct mtx mtx; 39 static struct mtx callout_mtx; 40 static struct service_hndl adf_state_hndl; 41 42 struct entry_proc_events { 43 struct adf_state_priv_data *proc_events; 44 45 SLIST_ENTRY(entry_proc_events) entries_proc_events; 46 }; 47 48 struct entry_state { 49 struct adf_state state; 50 51 STAILQ_ENTRY(entry_state) entries_state; 52 }; 53 54 SLIST_HEAD(proc_events_head, entry_proc_events); 55 STAILQ_HEAD(state_head, entry_state); 56 57 static struct proc_events_head proc_events_head; 58 59 struct adf_processes_priv_data { 60 char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; 61 int read_flag; 62 struct list_head list; 63 }; 64 65 struct adf_state_priv_data { 66 struct cdev *cdev; 67 struct selinfo rsel; 68 struct state_head state_head; 69 }; 70 71 static struct cdevsw adf_processes_cdevsw = { 72 .d_version = D_VERSION, 73 .d_open = adf_processes_open, 74 .d_read = adf_processes_read, 75 .d_write = adf_processes_write, 76 .d_name = ADF_DEV_PROCESSES_NAME, 77 }; 78 79 static struct cdevsw adf_state_cdevsw = { 80 .d_version = D_VERSION, 81 .d_open = adf_state_open, 82 .d_read = adf_state_read, 83 .d_kqfilter = adf_state_kqfilter, 84 .d_name = ADF_DEV_STATE_NAME, 85 }; 86 87 static struct filterops adf_state_read_filterops = { 88 .f_isfd = 1, 89 .f_attach = NULL, 90 .f_detach = adf_state_kqread_detach, 91 .f_event = adf_state_kqread_event, 92 }; 93 94 static struct cdev *adf_processes_dev; 95 static struct cdev *adf_state_dev; 96 97 static LINUX_LIST_HEAD(processes_list); 98 99 struct sx processes_list_sema; 100 SX_SYSINIT(processes_list_sema, &processes_list_sema, "adf proc list"); 101 102 static void 103 adf_chr_drv_destroy(void) 104 { 105 destroy_dev(adf_processes_dev); 106 } 107 108 static int 109 adf_chr_drv_create(void) 110 { 111 112 adf_processes_dev = make_dev(&adf_processes_cdevsw, 113 0, 114 UID_ROOT, 115 GID_WHEEL, 116 0600, 117 ADF_DEV_PROCESSES_NAME); 118 if (adf_processes_dev == NULL) { 119 printf("QAT: failed to create device\n"); 120 goto err_cdev_del; 121 } 122 return 0; 123 err_cdev_del: 124 return EFAULT; 125 } 126 127 static int 128 adf_processes_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 129 { 130 int i = 0, devices = 0; 131 struct adf_accel_dev *accel_dev = NULL; 132 struct adf_processes_priv_data *prv_data = NULL; 133 int error = 0; 134 135 for (i = 0; i < ADF_MAX_DEVICES; i++) { 136 accel_dev = adf_devmgr_get_dev_by_id(i); 137 if (!accel_dev) 138 continue; 139 if (!adf_dev_started(accel_dev)) 140 continue; 141 devices++; 142 } 143 if (!devices) { 144 printf("QAT: No active devices found.\n"); 145 return ENXIO; 146 } 147 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO); 148 INIT_LIST_HEAD(&prv_data->list); 149 error = devfs_set_cdevpriv(prv_data, adf_processes_release); 150 if (error) { 151 free(prv_data, M_QAT); 152 return error; 153 } 154 155 return 0; 156 } 157 158 static int 159 adf_get_first_started_dev(void) 160 { 161 int i = 0; 162 struct adf_accel_dev *accel_dev = NULL; 163 164 for (i = 0; i < ADF_MAX_DEVICES; i++) { 165 accel_dev = adf_devmgr_get_dev_by_id(i); 166 if (!accel_dev) 167 continue; 168 if (adf_dev_started(accel_dev)) 169 return i; 170 } 171 172 return -1; 173 } 174 175 static int 176 adf_processes_write(struct cdev *dev, struct uio *uio, int ioflag) 177 { 178 struct adf_processes_priv_data *prv_data = NULL; 179 struct adf_processes_priv_data *pdata = NULL; 180 int dev_num = 0, pr_num = 0; 181 struct list_head *lpos = NULL; 182 char usr_name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES] = { 0 }; 183 struct adf_accel_dev *accel_dev = NULL; 184 struct adf_cfg_section *section_ptr = NULL; 185 bool pr_name_available = 1; 186 uint32_t num_accel_devs = 0; 187 int error = 0; 188 ssize_t count; 189 int dev_id; 190 191 error = devfs_get_cdevpriv((void **)&prv_data); 192 if (error) { 193 printf("QAT: invalid file descriptor\n"); 194 return error; 195 } 196 197 if (prv_data->read_flag == 1) { 198 printf("QAT: can only write once\n"); 199 return EBADF; 200 } 201 count = uio->uio_resid; 202 if ((count <= 0) || (count > ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) { 203 printf("QAT: wrong size %d\n", (int)count); 204 return EIO; 205 } 206 207 error = uiomove(usr_name, count, uio); 208 if (error) { 209 printf("QAT: can't copy data\n"); 210 return error; 211 } 212 213 /* Lock other processes and try to find out the process name */ 214 if (sx_xlock_sig(&processes_list_sema)) { 215 printf("QAT: can't aquire process info lock\n"); 216 return EBADF; 217 } 218 219 dev_id = adf_get_first_started_dev(); 220 if (-1 == dev_id) { 221 pr_err("QAT: could not find started device\n"); 222 sx_xunlock(&processes_list_sema); 223 return -EIO; 224 } 225 226 accel_dev = adf_devmgr_get_dev_by_id(dev_id); 227 if (!accel_dev) { 228 pr_err("QAT: could not find started device\n"); 229 sx_xunlock(&processes_list_sema); 230 return -EIO; 231 } 232 233 /* If there is nothing there then take the first name and return */ 234 if (list_empty(&processes_list)) { 235 snprintf(prv_data->name, 236 ADF_CFG_MAX_SECTION_LEN_IN_BYTES, 237 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d", 238 usr_name, 239 0); 240 list_add(&prv_data->list, &processes_list); 241 sx_xunlock(&processes_list_sema); 242 prv_data->read_flag = 1; 243 return 0; 244 } 245 246 /* If there are processes running then search for a first free name */ 247 adf_devmgr_get_num_dev(&num_accel_devs); 248 for (dev_num = 0; dev_num < num_accel_devs; dev_num++) { 249 accel_dev = adf_devmgr_get_dev_by_id(dev_num); 250 if (!accel_dev) 251 continue; 252 253 if (!adf_dev_started(accel_dev)) 254 continue; /* to next device */ 255 256 for (pr_num = 0; pr_num < GET_MAX_PROCESSES(accel_dev); 257 pr_num++) { 258 snprintf(prv_data->name, 259 ADF_CFG_MAX_SECTION_LEN_IN_BYTES, 260 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d", 261 usr_name, 262 pr_num); 263 pr_name_available = 1; 264 /* Figure out if section exists in the config table */ 265 section_ptr = 266 adf_cfg_sec_find(accel_dev, prv_data->name); 267 if (NULL == section_ptr) { 268 /* This section name doesn't exist */ 269 pr_name_available = 0; 270 /* As process_num enumerates from 0, once we get 271 * to one which doesn't exist no further ones 272 * will exist. On to next device 273 */ 274 break; 275 } 276 /* Figure out if it's been taken already */ 277 list_for_each(lpos, &processes_list) 278 { 279 pdata = 280 list_entry(lpos, 281 struct adf_processes_priv_data, 282 list); 283 if (!strncmp( 284 pdata->name, 285 prv_data->name, 286 ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) { 287 pr_name_available = 0; 288 break; 289 } 290 } 291 if (pr_name_available) 292 break; 293 } 294 if (pr_name_available) 295 break; 296 } 297 /* 298 * If we have a valid name that is not on 299 * the list take it and add to the list 300 */ 301 if (pr_name_available) { 302 list_add(&prv_data->list, &processes_list); 303 sx_xunlock(&processes_list_sema); 304 prv_data->read_flag = 1; 305 return 0; 306 } 307 /* If not then the process needs to wait */ 308 sx_xunlock(&processes_list_sema); 309 explicit_bzero(prv_data->name, ADF_CFG_MAX_SECTION_LEN_IN_BYTES); 310 prv_data->read_flag = 0; 311 return 1; 312 } 313 314 static int 315 adf_processes_read(struct cdev *dev, struct uio *uio, int ioflag) 316 { 317 struct adf_processes_priv_data *prv_data = NULL; 318 int error = 0; 319 320 error = devfs_get_cdevpriv((void **)&prv_data); 321 if (error) { 322 printf("QAT: invalid file descriptor\n"); 323 return error; 324 } 325 326 /* 327 * If there is a name that the process can use then give it 328 * to the proocess. 329 */ 330 if (prv_data->read_flag) { 331 error = uiomove(prv_data->name, 332 strnlen(prv_data->name, 333 ADF_CFG_MAX_SECTION_LEN_IN_BYTES), 334 uio); 335 if (error) { 336 printf("QAT: failed to copy data to user\n"); 337 return error; 338 } 339 return 0; 340 } 341 342 return EIO; 343 } 344 345 static void 346 adf_processes_release(void *data) 347 { 348 struct adf_processes_priv_data *prv_data = NULL; 349 350 prv_data = (struct adf_processes_priv_data *)data; 351 sx_xlock(&processes_list_sema); 352 list_del(&prv_data->list); 353 sx_xunlock(&processes_list_sema); 354 free(prv_data, M_QAT); 355 } 356 357 int 358 adf_processes_dev_register(void) 359 { 360 return adf_chr_drv_create(); 361 } 362 363 void 364 adf_processes_dev_unregister(void) 365 { 366 adf_chr_drv_destroy(); 367 } 368 369 static void 370 adf_state_callout_notify_ev(void *arg) 371 { 372 int notified = 0; 373 struct adf_state_priv_data *priv = NULL; 374 struct entry_proc_events *proc_events = NULL; 375 376 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 377 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) { 378 notified = 1; 379 priv = proc_events->proc_events; 380 wakeup(priv); 381 selwakeup(&priv->rsel); 382 KNOTE_UNLOCKED(&priv->rsel.si_note, 0); 383 } 384 } 385 if (notified) 386 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 387 } 388 389 static void 390 adf_state_set(int dev, enum adf_event event) 391 { 392 struct adf_accel_dev *accel_dev = NULL; 393 struct state_head *head = NULL; 394 struct entry_proc_events *proc_events = NULL; 395 struct entry_state *state = NULL; 396 397 accel_dev = adf_devmgr_get_dev_by_id(dev); 398 if (!accel_dev) 399 return; 400 mtx_lock(&mtx); 401 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 402 state = NULL; 403 head = &proc_events->proc_events->state_head; 404 state = malloc(sizeof(struct entry_state), 405 M_QAT, 406 M_NOWAIT | M_ZERO); 407 if (!state) 408 continue; 409 state->state.dev_state = event; 410 state->state.dev_id = dev; 411 STAILQ_INSERT_TAIL(head, state, entries_state); 412 } 413 mtx_unlock(&mtx); 414 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 415 } 416 417 static int 418 adf_state_event_handler(struct adf_accel_dev *accel_dev, enum adf_event event) 419 { 420 int ret = 0; 421 422 #if defined(QAT_UIO) && defined(QAT_DBG) 423 if (event > ADF_EVENT_DBG_SHUTDOWN) 424 return -EINVAL; 425 #else 426 if (event > ADF_EVENT_ERROR) 427 return -EINVAL; 428 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */ 429 430 switch (event) { 431 case ADF_EVENT_INIT: 432 return ret; 433 case ADF_EVENT_SHUTDOWN: 434 return ret; 435 case ADF_EVENT_RESTARTING: 436 break; 437 case ADF_EVENT_RESTARTED: 438 break; 439 case ADF_EVENT_START: 440 return ret; 441 case ADF_EVENT_STOP: 442 return ret; 443 case ADF_EVENT_ERROR: 444 break; 445 #if defined(QAT_UIO) && defined(QAT_DBG) 446 case ADF_EVENT_PROC_CRASH: 447 break; 448 case ADF_EVENT_MANUAL_DUMP: 449 break; 450 case ADF_EVENT_SLICE_HANG: 451 break; 452 case ADF_EVENT_DBG_SHUTDOWN: 453 break; 454 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */ 455 default: 456 return -1; 457 } 458 459 adf_state_set(accel_dev->accel_id, event); 460 461 return 0; 462 } 463 464 static int 465 adf_state_kqfilter(struct cdev *dev, struct knote *kn) 466 { 467 struct adf_state_priv_data *priv; 468 469 mtx_lock(&mtx); 470 priv = dev->si_drv1; 471 switch (kn->kn_filter) { 472 case EVFILT_READ: 473 kn->kn_fop = &adf_state_read_filterops; 474 kn->kn_hook = priv; 475 knlist_add(&priv->rsel.si_note, kn, 1); 476 mtx_unlock(&mtx); 477 return 0; 478 default: 479 mtx_unlock(&mtx); 480 return -EINVAL; 481 } 482 } 483 484 static int 485 adf_state_kqread_event(struct knote *kn, long hint) 486 { 487 return 1; 488 } 489 490 static void 491 adf_state_kqread_detach(struct knote *kn) 492 { 493 struct adf_state_priv_data *priv = NULL; 494 495 mtx_lock(&mtx); 496 if (!kn) { 497 mtx_unlock(&mtx); 498 return; 499 } 500 priv = kn->kn_hook; 501 if (!priv) { 502 mtx_unlock(&mtx); 503 return; 504 } 505 knlist_remove(&priv->rsel.si_note, kn, 1); 506 mtx_unlock(&mtx); 507 } 508 509 void 510 adf_state_init(void) 511 { 512 adf_state_dev = make_dev(&adf_state_cdevsw, 513 0, 514 UID_ROOT, 515 GID_WHEEL, 516 0600, 517 "%s", 518 ADF_DEV_STATE_NAME); 519 SLIST_INIT(&proc_events_head); 520 mtx_init(&mtx, mtx_name, NULL, MTX_DEF); 521 mtx_init(&callout_mtx, mtx_callout_name, NULL, MTX_DEF); 522 callout_init_mtx(&callout, &callout_mtx, 0); 523 explicit_bzero(&adf_state_hndl, sizeof(adf_state_hndl)); 524 adf_state_hndl.event_hld = adf_state_event_handler; 525 adf_state_hndl.name = "adf_state_event_handler"; 526 adf_service_register(&adf_state_hndl); 527 callout_reset(&callout, 528 ADF_STATE_CALLOUT_TIME, 529 adf_state_callout_notify_ev, 530 NULL); 531 } 532 533 void 534 adf_state_destroy(void) 535 { 536 struct entry_proc_events *proc_events = NULL; 537 538 adf_service_unregister(&adf_state_hndl); 539 destroy_dev(adf_state_dev); 540 mtx_lock(&callout_mtx); 541 callout_stop(&callout); 542 mtx_unlock(&callout_mtx); 543 mtx_destroy(&callout_mtx); 544 mtx_lock(&mtx); 545 while (!SLIST_EMPTY(&proc_events_head)) { 546 proc_events = SLIST_FIRST(&proc_events_head); 547 SLIST_REMOVE_HEAD(&proc_events_head, entries_proc_events); 548 free(proc_events, M_QAT); 549 } 550 mtx_unlock(&mtx); 551 mtx_destroy(&mtx); 552 } 553 554 static int 555 adf_state_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 556 { 557 struct adf_state_priv_data *prv_data = NULL; 558 struct entry_proc_events *entry_proc_events = NULL; 559 int ret = 0; 560 561 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO); 562 entry_proc_events = 563 malloc(sizeof(struct entry_proc_events), M_QAT, M_WAITOK | M_ZERO); 564 mtx_lock(&mtx); 565 prv_data->cdev = dev; 566 prv_data->cdev->si_drv1 = prv_data; 567 knlist_init_mtx(&prv_data->rsel.si_note, &mtx); 568 STAILQ_INIT(&prv_data->state_head); 569 entry_proc_events->proc_events = prv_data; 570 SLIST_INSERT_HEAD(&proc_events_head, 571 entry_proc_events, 572 entries_proc_events); 573 mtx_unlock(&mtx); 574 ret = devfs_set_cdevpriv(prv_data, adf_state_release); 575 if (ret) { 576 SLIST_REMOVE(&proc_events_head, 577 entry_proc_events, 578 entry_proc_events, 579 entries_proc_events); 580 free(entry_proc_events, M_QAT); 581 free(prv_data, M_QAT); 582 } 583 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 584 return ret; 585 } 586 587 static int 588 adf_state_read(struct cdev *dev, struct uio *uio, int ioflag) 589 { 590 int ret = 0; 591 struct adf_state_priv_data *prv_data = NULL; 592 struct state_head *state_head = NULL; 593 struct entry_state *entry_state = NULL; 594 struct adf_state *state = NULL; 595 struct entry_proc_events *proc_events = NULL; 596 597 mtx_lock(&mtx); 598 ret = devfs_get_cdevpriv((void **)&prv_data); 599 if (ret) { 600 mtx_unlock(&mtx); 601 return 0; 602 } 603 state_head = &prv_data->state_head; 604 if (STAILQ_EMPTY(state_head)) { 605 mtx_unlock(&mtx); 606 return 0; 607 } 608 entry_state = STAILQ_FIRST(state_head); 609 state = &entry_state->state; 610 ret = uiomove(state, sizeof(struct adf_state), uio); 611 if (!ret && !STAILQ_EMPTY(state_head)) { 612 STAILQ_REMOVE_HEAD(state_head, entries_state); 613 free(entry_state, M_QAT); 614 } 615 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) { 616 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) { 617 prv_data = proc_events->proc_events; 618 wakeup(prv_data); 619 selwakeup(&prv_data->rsel); 620 KNOTE_UNLOCKED(&prv_data->rsel.si_note, 0); 621 } 622 } 623 mtx_unlock(&mtx); 624 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); 625 return ret; 626 } 627 628 static void 629 adf_state_release(void *data) 630 { 631 struct adf_state_priv_data *prv_data = NULL; 632 struct entry_state *entry_state = NULL; 633 struct entry_proc_events *entry_proc_events = NULL; 634 struct entry_proc_events *tmp = NULL; 635 636 mtx_lock(&mtx); 637 prv_data = (struct adf_state_priv_data *)data; 638 knlist_delete(&prv_data->rsel.si_note, curthread, 1); 639 knlist_destroy(&prv_data->rsel.si_note); 640 seldrain(&prv_data->rsel); 641 while (!STAILQ_EMPTY(&prv_data->state_head)) { 642 entry_state = STAILQ_FIRST(&prv_data->state_head); 643 STAILQ_REMOVE_HEAD(&prv_data->state_head, entries_state); 644 free(entry_state, M_QAT); 645 } 646 SLIST_FOREACH_SAFE (entry_proc_events, 647 &proc_events_head, 648 entries_proc_events, 649 tmp) { 650 if (entry_proc_events->proc_events == prv_data) { 651 SLIST_REMOVE(&proc_events_head, 652 entry_proc_events, 653 entry_proc_events, 654 entries_proc_events); 655 free(entry_proc_events, M_QAT); 656 } 657 } 658 free(prv_data, M_QAT); 659 mtx_unlock(&mtx); 660 } 661