1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3 #include "qat_freebsd.h"
4 #include "adf_cfg.h"
5 #include "adf_common_drv.h"
6 #include "adf_accel_devices.h"
7 #include "icp_qat_uclo.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_init_admin.h"
10 #include "adf_cfg_strings.h"
11 #include "adf_uio_control.h"
12 #include "adf_uio_cleanup.h"
13 #include "adf_uio.h"
14 #include "adf_transport_access_macros.h"
15 #include "adf_transport_internal.h"
16
17 #define ADF_DEV_PROCESSES_NAME "qat_dev_processes"
18 #define ADF_DEV_STATE_NAME "qat_dev_state"
19
20 #define ADF_STATE_CALLOUT_TIME 10
21
22 static const char *mtx_name = "state_mtx";
23 static const char *mtx_callout_name = "callout_mtx";
24
25 static d_open_t adf_processes_open;
26 static void adf_processes_release(void *data);
27 static d_read_t adf_processes_read;
28 static d_write_t adf_processes_write;
29
30 static d_open_t adf_state_open;
31 static void adf_state_release(void *data);
32 static d_read_t adf_state_read;
33 static int adf_state_kqfilter(struct cdev *dev, struct knote *kn);
34 static int adf_state_kqread_event(struct knote *kn, long hint);
35 static void adf_state_kqread_detach(struct knote *kn);
36
37 static struct callout callout;
38 static struct mtx mtx;
39 static struct mtx callout_mtx;
40 static struct service_hndl adf_state_hndl;
41
42 struct entry_proc_events {
43 struct adf_state_priv_data *proc_events;
44
45 SLIST_ENTRY(entry_proc_events) entries_proc_events;
46 };
47
48 struct entry_state {
49 struct adf_state state;
50
51 STAILQ_ENTRY(entry_state) entries_state;
52 };
53
54 SLIST_HEAD(proc_events_head, entry_proc_events);
55 STAILQ_HEAD(state_head, entry_state);
56
57 static struct proc_events_head proc_events_head;
58
59 struct adf_processes_priv_data {
60 char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES];
61 int read_flag;
62 struct list_head list;
63 };
64
65 struct adf_state_priv_data {
66 struct cdev *cdev;
67 struct selinfo rsel;
68 struct state_head state_head;
69 };
70
71 static struct cdevsw adf_processes_cdevsw = {
72 .d_version = D_VERSION,
73 .d_open = adf_processes_open,
74 .d_read = adf_processes_read,
75 .d_write = adf_processes_write,
76 .d_name = ADF_DEV_PROCESSES_NAME,
77 };
78
79 static struct cdevsw adf_state_cdevsw = {
80 .d_version = D_VERSION,
81 .d_open = adf_state_open,
82 .d_read = adf_state_read,
83 .d_kqfilter = adf_state_kqfilter,
84 .d_name = ADF_DEV_STATE_NAME,
85 };
86
87 static struct filterops adf_state_read_filterops = {
88 .f_isfd = 1,
89 .f_attach = NULL,
90 .f_detach = adf_state_kqread_detach,
91 .f_event = adf_state_kqread_event,
92 };
93
94 static struct cdev *adf_processes_dev;
95 static struct cdev *adf_state_dev;
96
97 static LINUX_LIST_HEAD(processes_list);
98
99 struct sx processes_list_sema;
100 SX_SYSINIT(processes_list_sema, &processes_list_sema, "adf proc list");
101
102 static void
adf_chr_drv_destroy(void)103 adf_chr_drv_destroy(void)
104 {
105 destroy_dev(adf_processes_dev);
106 }
107
108 static int
adf_chr_drv_create(void)109 adf_chr_drv_create(void)
110 {
111
112 adf_processes_dev = make_dev(&adf_processes_cdevsw,
113 0,
114 UID_ROOT,
115 GID_WHEEL,
116 0600,
117 ADF_DEV_PROCESSES_NAME);
118 if (adf_processes_dev == NULL) {
119 printf("QAT: failed to create device\n");
120 goto err_cdev_del;
121 }
122 return 0;
123 err_cdev_del:
124 return EFAULT;
125 }
126
127 static int
adf_processes_open(struct cdev * dev,int oflags,int devtype,struct thread * td)128 adf_processes_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
129 {
130 int i = 0, devices = 0;
131 struct adf_accel_dev *accel_dev = NULL;
132 struct adf_processes_priv_data *prv_data = NULL;
133 int error = 0;
134
135 for (i = 0; i < ADF_MAX_DEVICES; i++) {
136 accel_dev = adf_devmgr_get_dev_by_id(i);
137 if (!accel_dev)
138 continue;
139 if (!adf_dev_started(accel_dev))
140 continue;
141 devices++;
142 }
143 if (!devices) {
144 printf("QAT: No active devices found.\n");
145 return ENXIO;
146 }
147 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO);
148 INIT_LIST_HEAD(&prv_data->list);
149 error = devfs_set_cdevpriv(prv_data, adf_processes_release);
150 if (error) {
151 free(prv_data, M_QAT);
152 return error;
153 }
154
155 return 0;
156 }
157
158 static int
adf_get_first_started_dev(void)159 adf_get_first_started_dev(void)
160 {
161 int i = 0;
162 struct adf_accel_dev *accel_dev = NULL;
163
164 for (i = 0; i < ADF_MAX_DEVICES; i++) {
165 accel_dev = adf_devmgr_get_dev_by_id(i);
166 if (!accel_dev)
167 continue;
168 if (adf_dev_started(accel_dev))
169 return i;
170 }
171
172 return -1;
173 }
174
175 static int
adf_processes_write(struct cdev * dev,struct uio * uio,int ioflag)176 adf_processes_write(struct cdev *dev, struct uio *uio, int ioflag)
177 {
178 struct adf_processes_priv_data *prv_data = NULL;
179 struct adf_processes_priv_data *pdata = NULL;
180 int dev_num = 0, pr_num = 0;
181 struct list_head *lpos = NULL;
182 char usr_name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES] = { 0 };
183 struct adf_accel_dev *accel_dev = NULL;
184 struct adf_cfg_section *section_ptr = NULL;
185 bool pr_name_available = 1;
186 uint32_t num_accel_devs = 0;
187 int error = 0;
188 ssize_t count;
189 int dev_id;
190
191 error = devfs_get_cdevpriv((void **)&prv_data);
192 if (error) {
193 printf("QAT: invalid file descriptor\n");
194 return error;
195 }
196
197 if (prv_data->read_flag == 1) {
198 printf("QAT: can only write once\n");
199 return EBADF;
200 }
201 count = uio->uio_resid;
202 if ((count <= 0) || (count > ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) {
203 printf("QAT: wrong size %d\n", (int)count);
204 return EIO;
205 }
206
207 error = uiomove(usr_name, count, uio);
208 if (error) {
209 printf("QAT: can't copy data\n");
210 return error;
211 }
212
213 /* Lock other processes and try to find out the process name */
214 if (sx_xlock_sig(&processes_list_sema)) {
215 printf("QAT: can't aquire process info lock\n");
216 return EBADF;
217 }
218
219 dev_id = adf_get_first_started_dev();
220 if (-1 == dev_id) {
221 pr_err("QAT: could not find started device\n");
222 sx_xunlock(&processes_list_sema);
223 return -EIO;
224 }
225
226 accel_dev = adf_devmgr_get_dev_by_id(dev_id);
227 if (!accel_dev) {
228 pr_err("QAT: could not find started device\n");
229 sx_xunlock(&processes_list_sema);
230 return -EIO;
231 }
232
233 /* If there is nothing there then take the first name and return */
234 if (list_empty(&processes_list)) {
235 snprintf(prv_data->name,
236 ADF_CFG_MAX_SECTION_LEN_IN_BYTES,
237 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d",
238 usr_name,
239 0);
240 list_add(&prv_data->list, &processes_list);
241 sx_xunlock(&processes_list_sema);
242 prv_data->read_flag = 1;
243 return 0;
244 }
245
246 /* If there are processes running then search for a first free name */
247 adf_devmgr_get_num_dev(&num_accel_devs);
248 for (dev_num = 0; dev_num < num_accel_devs; dev_num++) {
249 accel_dev = adf_devmgr_get_dev_by_id(dev_num);
250 if (!accel_dev)
251 continue;
252
253 if (!adf_dev_started(accel_dev))
254 continue; /* to next device */
255
256 for (pr_num = 0; pr_num < GET_MAX_PROCESSES(accel_dev);
257 pr_num++) {
258 snprintf(prv_data->name,
259 ADF_CFG_MAX_SECTION_LEN_IN_BYTES,
260 "%s" ADF_INTERNAL_USERSPACE_SEC_SUFF "%d",
261 usr_name,
262 pr_num);
263 pr_name_available = 1;
264 /* Figure out if section exists in the config table */
265 section_ptr =
266 adf_cfg_sec_find(accel_dev, prv_data->name);
267 if (NULL == section_ptr) {
268 /* This section name doesn't exist */
269 pr_name_available = 0;
270 /* As process_num enumerates from 0, once we get
271 * to one which doesn't exist no further ones
272 * will exist. On to next device
273 */
274 break;
275 }
276 /* Figure out if it's been taken already */
277 list_for_each(lpos, &processes_list)
278 {
279 pdata =
280 list_entry(lpos,
281 struct adf_processes_priv_data,
282 list);
283 if (!strncmp(
284 pdata->name,
285 prv_data->name,
286 ADF_CFG_MAX_SECTION_LEN_IN_BYTES)) {
287 pr_name_available = 0;
288 break;
289 }
290 }
291 if (pr_name_available)
292 break;
293 }
294 if (pr_name_available)
295 break;
296 }
297 /*
298 * If we have a valid name that is not on
299 * the list take it and add to the list
300 */
301 if (pr_name_available) {
302 list_add(&prv_data->list, &processes_list);
303 sx_xunlock(&processes_list_sema);
304 prv_data->read_flag = 1;
305 return 0;
306 }
307 /* If not then the process needs to wait */
308 sx_xunlock(&processes_list_sema);
309 explicit_bzero(prv_data->name, ADF_CFG_MAX_SECTION_LEN_IN_BYTES);
310 prv_data->read_flag = 0;
311 return 1;
312 }
313
314 static int
adf_processes_read(struct cdev * dev,struct uio * uio,int ioflag)315 adf_processes_read(struct cdev *dev, struct uio *uio, int ioflag)
316 {
317 struct adf_processes_priv_data *prv_data = NULL;
318 int error = 0;
319
320 error = devfs_get_cdevpriv((void **)&prv_data);
321 if (error) {
322 printf("QAT: invalid file descriptor\n");
323 return error;
324 }
325
326 /*
327 * If there is a name that the process can use then give it
328 * to the proocess.
329 */
330 if (prv_data->read_flag) {
331 error = uiomove(prv_data->name,
332 strnlen(prv_data->name,
333 ADF_CFG_MAX_SECTION_LEN_IN_BYTES),
334 uio);
335 if (error) {
336 printf("QAT: failed to copy data to user\n");
337 return error;
338 }
339 return 0;
340 }
341
342 return EIO;
343 }
344
345 static void
adf_processes_release(void * data)346 adf_processes_release(void *data)
347 {
348 struct adf_processes_priv_data *prv_data = NULL;
349
350 prv_data = (struct adf_processes_priv_data *)data;
351 sx_xlock(&processes_list_sema);
352 list_del(&prv_data->list);
353 sx_xunlock(&processes_list_sema);
354 free(prv_data, M_QAT);
355 }
356
357 int
adf_processes_dev_register(void)358 adf_processes_dev_register(void)
359 {
360 return adf_chr_drv_create();
361 }
362
363 void
adf_processes_dev_unregister(void)364 adf_processes_dev_unregister(void)
365 {
366 adf_chr_drv_destroy();
367 }
368
369 static void
adf_state_callout_notify_ev(void * arg)370 adf_state_callout_notify_ev(void *arg)
371 {
372 int notified = 0;
373 struct adf_state_priv_data *priv = NULL;
374 struct entry_proc_events *proc_events = NULL;
375
376 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) {
377 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) {
378 notified = 1;
379 priv = proc_events->proc_events;
380 wakeup(priv);
381 selwakeup(&priv->rsel);
382 KNOTE_UNLOCKED(&priv->rsel.si_note, 0);
383 }
384 }
385 if (notified)
386 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME);
387 }
388
389 static void
adf_state_set(int dev,enum adf_event event)390 adf_state_set(int dev, enum adf_event event)
391 {
392 struct adf_accel_dev *accel_dev = NULL;
393 struct state_head *head = NULL;
394 struct entry_proc_events *proc_events = NULL;
395 struct entry_state *state = NULL;
396
397 accel_dev = adf_devmgr_get_dev_by_id(dev);
398 if (!accel_dev)
399 return;
400 mtx_lock(&mtx);
401 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) {
402 state = NULL;
403 head = &proc_events->proc_events->state_head;
404 state = malloc(sizeof(struct entry_state),
405 M_QAT,
406 M_NOWAIT | M_ZERO);
407 if (!state)
408 continue;
409 state->state.dev_state = event;
410 state->state.dev_id = dev;
411 STAILQ_INSERT_TAIL(head, state, entries_state);
412 }
413 mtx_unlock(&mtx);
414 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME);
415 }
416
417 static int
adf_state_event_handler(struct adf_accel_dev * accel_dev,enum adf_event event)418 adf_state_event_handler(struct adf_accel_dev *accel_dev, enum adf_event event)
419 {
420 int ret = 0;
421
422 #if defined(QAT_UIO) && defined(QAT_DBG)
423 if (event > ADF_EVENT_DBG_SHUTDOWN)
424 return -EINVAL;
425 #else
426 if (event > ADF_EVENT_ERROR)
427 return -EINVAL;
428 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */
429
430 switch (event) {
431 case ADF_EVENT_INIT:
432 return ret;
433 case ADF_EVENT_SHUTDOWN:
434 return ret;
435 case ADF_EVENT_RESTARTING:
436 break;
437 case ADF_EVENT_RESTARTED:
438 break;
439 case ADF_EVENT_START:
440 return ret;
441 case ADF_EVENT_STOP:
442 return ret;
443 case ADF_EVENT_ERROR:
444 break;
445 #if defined(QAT_UIO) && defined(QAT_DBG)
446 case ADF_EVENT_PROC_CRASH:
447 break;
448 case ADF_EVENT_MANUAL_DUMP:
449 break;
450 case ADF_EVENT_SLICE_HANG:
451 break;
452 case ADF_EVENT_DBG_SHUTDOWN:
453 break;
454 #endif /* defined(QAT_UIO) && defined(QAT_DBG) */
455 default:
456 return -1;
457 }
458
459 adf_state_set(accel_dev->accel_id, event);
460
461 return 0;
462 }
463
464 static int
adf_state_kqfilter(struct cdev * dev,struct knote * kn)465 adf_state_kqfilter(struct cdev *dev, struct knote *kn)
466 {
467 struct adf_state_priv_data *priv;
468
469 mtx_lock(&mtx);
470 priv = dev->si_drv1;
471 switch (kn->kn_filter) {
472 case EVFILT_READ:
473 kn->kn_fop = &adf_state_read_filterops;
474 kn->kn_hook = priv;
475 knlist_add(&priv->rsel.si_note, kn, 1);
476 mtx_unlock(&mtx);
477 return 0;
478 default:
479 mtx_unlock(&mtx);
480 return -EINVAL;
481 }
482 }
483
484 static int
adf_state_kqread_event(struct knote * kn,long hint)485 adf_state_kqread_event(struct knote *kn, long hint)
486 {
487 return 1;
488 }
489
490 static void
adf_state_kqread_detach(struct knote * kn)491 adf_state_kqread_detach(struct knote *kn)
492 {
493 struct adf_state_priv_data *priv = NULL;
494
495 mtx_lock(&mtx);
496 if (!kn) {
497 mtx_unlock(&mtx);
498 return;
499 }
500 priv = kn->kn_hook;
501 if (!priv) {
502 mtx_unlock(&mtx);
503 return;
504 }
505 knlist_remove(&priv->rsel.si_note, kn, 1);
506 mtx_unlock(&mtx);
507 }
508
509 void
adf_state_init(void)510 adf_state_init(void)
511 {
512 adf_state_dev = make_dev(&adf_state_cdevsw,
513 0,
514 UID_ROOT,
515 GID_WHEEL,
516 0600,
517 "%s",
518 ADF_DEV_STATE_NAME);
519 SLIST_INIT(&proc_events_head);
520 mtx_init(&mtx, mtx_name, NULL, MTX_DEF);
521 mtx_init(&callout_mtx, mtx_callout_name, NULL, MTX_DEF);
522 callout_init_mtx(&callout, &callout_mtx, 0);
523 explicit_bzero(&adf_state_hndl, sizeof(adf_state_hndl));
524 adf_state_hndl.event_hld = adf_state_event_handler;
525 adf_state_hndl.name = "adf_state_event_handler";
526 adf_service_register(&adf_state_hndl);
527 callout_reset(&callout,
528 ADF_STATE_CALLOUT_TIME,
529 adf_state_callout_notify_ev,
530 NULL);
531 }
532
533 void
adf_state_destroy(void)534 adf_state_destroy(void)
535 {
536 struct entry_proc_events *proc_events = NULL;
537
538 adf_service_unregister(&adf_state_hndl);
539 destroy_dev(adf_state_dev);
540 mtx_lock(&callout_mtx);
541 callout_stop(&callout);
542 mtx_unlock(&callout_mtx);
543 mtx_destroy(&callout_mtx);
544 mtx_lock(&mtx);
545 while (!SLIST_EMPTY(&proc_events_head)) {
546 proc_events = SLIST_FIRST(&proc_events_head);
547 SLIST_REMOVE_HEAD(&proc_events_head, entries_proc_events);
548 free(proc_events, M_QAT);
549 }
550 mtx_unlock(&mtx);
551 mtx_destroy(&mtx);
552 }
553
554 static int
adf_state_open(struct cdev * dev,int oflags,int devtype,struct thread * td)555 adf_state_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
556 {
557 struct adf_state_priv_data *prv_data = NULL;
558 struct entry_proc_events *entry_proc_events = NULL;
559 int ret = 0;
560
561 prv_data = malloc(sizeof(*prv_data), M_QAT, M_WAITOK | M_ZERO);
562 entry_proc_events =
563 malloc(sizeof(struct entry_proc_events), M_QAT, M_WAITOK | M_ZERO);
564 mtx_lock(&mtx);
565 prv_data->cdev = dev;
566 prv_data->cdev->si_drv1 = prv_data;
567 knlist_init_mtx(&prv_data->rsel.si_note, &mtx);
568 STAILQ_INIT(&prv_data->state_head);
569 entry_proc_events->proc_events = prv_data;
570 SLIST_INSERT_HEAD(&proc_events_head,
571 entry_proc_events,
572 entries_proc_events);
573 mtx_unlock(&mtx);
574 ret = devfs_set_cdevpriv(prv_data, adf_state_release);
575 if (ret) {
576 SLIST_REMOVE(&proc_events_head,
577 entry_proc_events,
578 entry_proc_events,
579 entries_proc_events);
580 free(entry_proc_events, M_QAT);
581 free(prv_data, M_QAT);
582 }
583 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME);
584 return ret;
585 }
586
587 static int
adf_state_read(struct cdev * dev,struct uio * uio,int ioflag)588 adf_state_read(struct cdev *dev, struct uio *uio, int ioflag)
589 {
590 int ret = 0;
591 struct adf_state_priv_data *prv_data = NULL;
592 struct state_head *state_head = NULL;
593 struct entry_state *entry_state = NULL;
594 struct adf_state *state = NULL;
595 struct entry_proc_events *proc_events = NULL;
596
597 mtx_lock(&mtx);
598 ret = devfs_get_cdevpriv((void **)&prv_data);
599 if (ret) {
600 mtx_unlock(&mtx);
601 return 0;
602 }
603 state_head = &prv_data->state_head;
604 if (STAILQ_EMPTY(state_head)) {
605 mtx_unlock(&mtx);
606 return 0;
607 }
608 entry_state = STAILQ_FIRST(state_head);
609 state = &entry_state->state;
610 ret = uiomove(state, sizeof(struct adf_state), uio);
611 if (!ret && !STAILQ_EMPTY(state_head)) {
612 STAILQ_REMOVE_HEAD(state_head, entries_state);
613 free(entry_state, M_QAT);
614 }
615 SLIST_FOREACH (proc_events, &proc_events_head, entries_proc_events) {
616 if (!STAILQ_EMPTY(&proc_events->proc_events->state_head)) {
617 prv_data = proc_events->proc_events;
618 wakeup(prv_data);
619 selwakeup(&prv_data->rsel);
620 KNOTE_UNLOCKED(&prv_data->rsel.si_note, 0);
621 }
622 }
623 mtx_unlock(&mtx);
624 callout_schedule(&callout, ADF_STATE_CALLOUT_TIME);
625 return ret;
626 }
627
628 static void
adf_state_release(void * data)629 adf_state_release(void *data)
630 {
631 struct adf_state_priv_data *prv_data = NULL;
632 struct entry_state *entry_state = NULL;
633 struct entry_proc_events *entry_proc_events = NULL;
634 struct entry_proc_events *tmp = NULL;
635
636 mtx_lock(&mtx);
637 prv_data = (struct adf_state_priv_data *)data;
638 knlist_delete(&prv_data->rsel.si_note, curthread, 1);
639 knlist_destroy(&prv_data->rsel.si_note);
640 seldrain(&prv_data->rsel);
641 while (!STAILQ_EMPTY(&prv_data->state_head)) {
642 entry_state = STAILQ_FIRST(&prv_data->state_head);
643 STAILQ_REMOVE_HEAD(&prv_data->state_head, entries_state);
644 free(entry_state, M_QAT);
645 }
646 SLIST_FOREACH_SAFE (entry_proc_events,
647 &proc_events_head,
648 entries_proc_events,
649 tmp) {
650 if (entry_proc_events->proc_events == prv_data) {
651 SLIST_REMOVE(&proc_events_head,
652 entry_proc_events,
653 entry_proc_events,
654 entries_proc_events);
655 free(entry_proc_events, M_QAT);
656 }
657 }
658 free(prv_data, M_QAT);
659 mtx_unlock(&mtx);
660 }
661