xref: /linux/drivers/crypto/intel/iaa/iaa_crypto_main.c (revision a0efa2f362a69e47b9d8b48f770ef3a0249a7911)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
3 
4 #include <linux/init.h>
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/device.h>
9 #include <linux/iommu.h>
10 #include <uapi/linux/idxd.h>
11 #include <linux/highmem.h>
12 #include <linux/sched/smt.h>
13 #include <crypto/internal/acompress.h>
14 
15 #include "idxd.h"
16 #include "iaa_crypto.h"
17 #include "iaa_crypto_stats.h"
18 
19 #ifdef pr_fmt
20 #undef pr_fmt
21 #endif
22 
23 #define pr_fmt(fmt)			"idxd: " IDXD_SUBDRIVER_NAME ": " fmt
24 
25 #define IAA_ALG_PRIORITY               300
26 
27 /* number of iaa instances probed */
28 static unsigned int nr_iaa;
29 static unsigned int nr_cpus;
30 static unsigned int nr_nodes;
31 static unsigned int nr_cpus_per_node;
32 
33 /* Number of physical cpus sharing each iaa instance */
34 static unsigned int cpus_per_iaa;
35 
36 static struct crypto_comp *deflate_generic_tfm;
37 
38 /* Per-cpu lookup table for balanced wqs */
39 static struct wq_table_entry __percpu *wq_table;
40 
41 static struct idxd_wq *wq_table_next_wq(int cpu)
42 {
43 	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
44 
45 	if (++entry->cur_wq >= entry->n_wqs)
46 		entry->cur_wq = 0;
47 
48 	if (!entry->wqs[entry->cur_wq])
49 		return NULL;
50 
51 	pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
52 		 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
53 		 entry->wqs[entry->cur_wq]->id, cpu);
54 
55 	return entry->wqs[entry->cur_wq];
56 }
57 
58 static void wq_table_add(int cpu, struct idxd_wq *wq)
59 {
60 	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
61 
62 	if (WARN_ON(entry->n_wqs == entry->max_wqs))
63 		return;
64 
65 	entry->wqs[entry->n_wqs++] = wq;
66 
67 	pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,
68 		 entry->wqs[entry->n_wqs - 1]->idxd->id,
69 		 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);
70 }
71 
72 static void wq_table_free_entry(int cpu)
73 {
74 	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
75 
76 	kfree(entry->wqs);
77 	memset(entry, 0, sizeof(*entry));
78 }
79 
80 static void wq_table_clear_entry(int cpu)
81 {
82 	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
83 
84 	entry->n_wqs = 0;
85 	entry->cur_wq = 0;
86 	memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
87 }
88 
89 LIST_HEAD(iaa_devices);
90 DEFINE_MUTEX(iaa_devices_lock);
91 
92 /* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
93 static bool iaa_crypto_enabled;
94 static bool iaa_crypto_registered;
95 
96 /* Verify results of IAA compress or not */
97 static bool iaa_verify_compress = true;
98 
99 static ssize_t verify_compress_show(struct device_driver *driver, char *buf)
100 {
101 	return sprintf(buf, "%d\n", iaa_verify_compress);
102 }
103 
104 static ssize_t verify_compress_store(struct device_driver *driver,
105 				     const char *buf, size_t count)
106 {
107 	int ret = -EBUSY;
108 
109 	mutex_lock(&iaa_devices_lock);
110 
111 	if (iaa_crypto_enabled)
112 		goto out;
113 
114 	ret = kstrtobool(buf, &iaa_verify_compress);
115 	if (ret)
116 		goto out;
117 
118 	ret = count;
119 out:
120 	mutex_unlock(&iaa_devices_lock);
121 
122 	return ret;
123 }
124 static DRIVER_ATTR_RW(verify_compress);
125 
126 /*
127  * The iaa crypto driver supports three 'sync' methods determining how
128  * compressions and decompressions are performed:
129  *
130  * - sync:      the compression or decompression completes before
131  *              returning.  This is the mode used by the async crypto
132  *              interface when the sync mode is set to 'sync' and by
133  *              the sync crypto interface regardless of setting.
134  *
135  * - async:     the compression or decompression is submitted and returns
136  *              immediately.  Completion interrupts are not used so
137  *              the caller is responsible for polling the descriptor
138  *              for completion.  This mode is applicable to only the
139  *              async crypto interface and is ignored for anything
140  *              else.
141  *
142  * - async_irq: the compression or decompression is submitted and
143  *              returns immediately.  Completion interrupts are
144  *              enabled so the caller can wait for the completion and
145  *              yield to other threads.  When the compression or
146  *              decompression completes, the completion is signaled
147  *              and the caller awakened.  This mode is applicable to
148  *              only the async crypto interface and is ignored for
149  *              anything else.
150  *
151  * These modes can be set using the iaa_crypto sync_mode driver
152  * attribute.
153  */
154 
155 /* Use async mode */
156 static bool async_mode;
157 /* Use interrupts */
158 static bool use_irq;
159 
160 /**
161  * set_iaa_sync_mode - Set IAA sync mode
162  * @name: The name of the sync mode
163  *
164  * Make the IAA sync mode named @name the current sync mode used by
165  * compression/decompression.
166  */
167 
168 static int set_iaa_sync_mode(const char *name)
169 {
170 	int ret = 0;
171 
172 	if (sysfs_streq(name, "sync")) {
173 		async_mode = false;
174 		use_irq = false;
175 	} else if (sysfs_streq(name, "async")) {
176 		async_mode = true;
177 		use_irq = false;
178 	} else if (sysfs_streq(name, "async_irq")) {
179 		async_mode = true;
180 		use_irq = true;
181 	} else {
182 		ret = -EINVAL;
183 	}
184 
185 	return ret;
186 }
187 
188 static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
189 {
190 	int ret = 0;
191 
192 	if (!async_mode && !use_irq)
193 		ret = sprintf(buf, "%s\n", "sync");
194 	else if (async_mode && !use_irq)
195 		ret = sprintf(buf, "%s\n", "async");
196 	else if (async_mode && use_irq)
197 		ret = sprintf(buf, "%s\n", "async_irq");
198 
199 	return ret;
200 }
201 
202 static ssize_t sync_mode_store(struct device_driver *driver,
203 			       const char *buf, size_t count)
204 {
205 	int ret = -EBUSY;
206 
207 	mutex_lock(&iaa_devices_lock);
208 
209 	if (iaa_crypto_enabled)
210 		goto out;
211 
212 	ret = set_iaa_sync_mode(buf);
213 	if (ret == 0)
214 		ret = count;
215 out:
216 	mutex_unlock(&iaa_devices_lock);
217 
218 	return ret;
219 }
220 static DRIVER_ATTR_RW(sync_mode);
221 
222 static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
223 
224 static int find_empty_iaa_compression_mode(void)
225 {
226 	int i = -EINVAL;
227 
228 	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
229 		if (iaa_compression_modes[i])
230 			continue;
231 		break;
232 	}
233 
234 	return i;
235 }
236 
237 static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)
238 {
239 	struct iaa_compression_mode *mode;
240 	int i;
241 
242 	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
243 		mode = iaa_compression_modes[i];
244 		if (!mode)
245 			continue;
246 
247 		if (!strcmp(mode->name, name)) {
248 			*idx = i;
249 			return iaa_compression_modes[i];
250 		}
251 	}
252 
253 	return NULL;
254 }
255 
256 static void free_iaa_compression_mode(struct iaa_compression_mode *mode)
257 {
258 	kfree(mode->name);
259 	kfree(mode->ll_table);
260 	kfree(mode->d_table);
261 
262 	kfree(mode);
263 }
264 
265 /*
266  * IAA Compression modes are defined by an ll_table and a d_table.
267  * These tables are typically generated and captured using statistics
268  * collected from running actual compress/decompress workloads.
269  *
270  * A module or other kernel code can add and remove compression modes
271  * with a given name using the exported @add_iaa_compression_mode()
272  * and @remove_iaa_compression_mode functions.
273  *
274  * When a new compression mode is added, the tables are saved in a
275  * global compression mode list.  When IAA devices are added, a
276  * per-IAA device dma mapping is created for each IAA device, for each
277  * compression mode.  These are the tables used to do the actual
278  * compression/deccompression and are unmapped if/when the devices are
279  * removed.  Currently, compression modes must be added before any
280  * device is added, and removed after all devices have been removed.
281  */
282 
283 /**
284  * remove_iaa_compression_mode - Remove an IAA compression mode
285  * @name: The name the compression mode will be known as
286  *
287  * Remove the IAA compression mode named @name.
288  */
289 void remove_iaa_compression_mode(const char *name)
290 {
291 	struct iaa_compression_mode *mode;
292 	int idx;
293 
294 	mutex_lock(&iaa_devices_lock);
295 
296 	if (!list_empty(&iaa_devices))
297 		goto out;
298 
299 	mode = find_iaa_compression_mode(name, &idx);
300 	if (mode) {
301 		free_iaa_compression_mode(mode);
302 		iaa_compression_modes[idx] = NULL;
303 	}
304 out:
305 	mutex_unlock(&iaa_devices_lock);
306 }
307 EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);
308 
309 /**
310  * add_iaa_compression_mode - Add an IAA compression mode
311  * @name: The name the compression mode will be known as
312  * @ll_table: The ll table
313  * @ll_table_size: The ll table size in bytes
314  * @d_table: The d table
315  * @d_table_size: The d table size in bytes
316  * @init: Optional callback function to init the compression mode data
317  * @free: Optional callback function to free the compression mode data
318  *
319  * Add a new IAA compression mode named @name.
320  *
321  * Returns 0 if successful, errcode otherwise.
322  */
323 int add_iaa_compression_mode(const char *name,
324 			     const u32 *ll_table,
325 			     int ll_table_size,
326 			     const u32 *d_table,
327 			     int d_table_size,
328 			     iaa_dev_comp_init_fn_t init,
329 			     iaa_dev_comp_free_fn_t free)
330 {
331 	struct iaa_compression_mode *mode;
332 	int idx, ret = -ENOMEM;
333 
334 	mutex_lock(&iaa_devices_lock);
335 
336 	if (!list_empty(&iaa_devices)) {
337 		ret = -EBUSY;
338 		goto out;
339 	}
340 
341 	mode = kzalloc(sizeof(*mode), GFP_KERNEL);
342 	if (!mode)
343 		goto out;
344 
345 	mode->name = kstrdup(name, GFP_KERNEL);
346 	if (!mode->name)
347 		goto free;
348 
349 	if (ll_table) {
350 		mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL);
351 		if (!mode->ll_table)
352 			goto free;
353 		mode->ll_table_size = ll_table_size;
354 	}
355 
356 	if (d_table) {
357 		mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL);
358 		if (!mode->d_table)
359 			goto free;
360 		mode->d_table_size = d_table_size;
361 	}
362 
363 	mode->init = init;
364 	mode->free = free;
365 
366 	idx = find_empty_iaa_compression_mode();
367 	if (idx < 0)
368 		goto free;
369 
370 	pr_debug("IAA compression mode %s added at idx %d\n",
371 		 mode->name, idx);
372 
373 	iaa_compression_modes[idx] = mode;
374 
375 	ret = 0;
376 out:
377 	mutex_unlock(&iaa_devices_lock);
378 
379 	return ret;
380 free:
381 	free_iaa_compression_mode(mode);
382 	goto out;
383 }
384 EXPORT_SYMBOL_GPL(add_iaa_compression_mode);
385 
386 static struct iaa_device_compression_mode *
387 get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)
388 {
389 	return iaa_device->compression_modes[idx];
390 }
391 
392 static void free_device_compression_mode(struct iaa_device *iaa_device,
393 					 struct iaa_device_compression_mode *device_mode)
394 {
395 	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
396 	struct device *dev = &iaa_device->idxd->pdev->dev;
397 
398 	kfree(device_mode->name);
399 
400 	if (device_mode->aecs_comp_table)
401 		dma_free_coherent(dev, size, device_mode->aecs_comp_table,
402 				  device_mode->aecs_comp_table_dma_addr);
403 	kfree(device_mode);
404 }
405 
406 #define IDXD_OP_FLAG_AECS_RW_TGLS       0x400000
407 #define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)
408 #define IAX_AECS_COMPRESS_FLAG	(IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
409 #define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
410 #define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \
411 						IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \
412 						IDXD_OP_FLAG_AECS_RW_TGLS)
413 
414 static int check_completion(struct device *dev,
415 			    struct iax_completion_record *comp,
416 			    bool compress,
417 			    bool only_once);
418 
419 static int init_device_compression_mode(struct iaa_device *iaa_device,
420 					struct iaa_compression_mode *mode,
421 					int idx, struct idxd_wq *wq)
422 {
423 	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
424 	struct device *dev = &iaa_device->idxd->pdev->dev;
425 	struct iaa_device_compression_mode *device_mode;
426 	int ret = -ENOMEM;
427 
428 	device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);
429 	if (!device_mode)
430 		return -ENOMEM;
431 
432 	device_mode->name = kstrdup(mode->name, GFP_KERNEL);
433 	if (!device_mode->name)
434 		goto free;
435 
436 	device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,
437 							  &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);
438 	if (!device_mode->aecs_comp_table)
439 		goto free;
440 
441 	/* Add Huffman table to aecs */
442 	memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
443 	memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);
444 	memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);
445 
446 	if (mode->init) {
447 		ret = mode->init(device_mode);
448 		if (ret)
449 			goto free;
450 	}
451 
452 	/* mode index should match iaa_compression_modes idx */
453 	iaa_device->compression_modes[idx] = device_mode;
454 
455 	pr_debug("IAA %s compression mode initialized for iaa device %d\n",
456 		 mode->name, iaa_device->idxd->id);
457 
458 	ret = 0;
459 out:
460 	return ret;
461 free:
462 	pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",
463 		 mode->name, iaa_device->idxd->id);
464 
465 	free_device_compression_mode(iaa_device, device_mode);
466 	goto out;
467 }
468 
469 static int init_device_compression_modes(struct iaa_device *iaa_device,
470 					 struct idxd_wq *wq)
471 {
472 	struct iaa_compression_mode *mode;
473 	int i, ret = 0;
474 
475 	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
476 		mode = iaa_compression_modes[i];
477 		if (!mode)
478 			continue;
479 
480 		ret = init_device_compression_mode(iaa_device, mode, i, wq);
481 		if (ret)
482 			break;
483 	}
484 
485 	return ret;
486 }
487 
488 static void remove_device_compression_modes(struct iaa_device *iaa_device)
489 {
490 	struct iaa_device_compression_mode *device_mode;
491 	int i;
492 
493 	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
494 		device_mode = iaa_device->compression_modes[i];
495 		if (!device_mode)
496 			continue;
497 
498 		if (iaa_compression_modes[i]->free)
499 			iaa_compression_modes[i]->free(device_mode);
500 		free_device_compression_mode(iaa_device, device_mode);
501 		iaa_device->compression_modes[i] = NULL;
502 	}
503 }
504 
505 static struct iaa_device *iaa_device_alloc(void)
506 {
507 	struct iaa_device *iaa_device;
508 
509 	iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);
510 	if (!iaa_device)
511 		return NULL;
512 
513 	INIT_LIST_HEAD(&iaa_device->wqs);
514 
515 	return iaa_device;
516 }
517 
518 static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
519 {
520 	struct iaa_wq *iaa_wq;
521 
522 	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
523 		if (iaa_wq->wq == wq)
524 			return true;
525 	}
526 
527 	return false;
528 }
529 
530 static struct iaa_device *add_iaa_device(struct idxd_device *idxd)
531 {
532 	struct iaa_device *iaa_device;
533 
534 	iaa_device = iaa_device_alloc();
535 	if (!iaa_device)
536 		return NULL;
537 
538 	iaa_device->idxd = idxd;
539 
540 	list_add_tail(&iaa_device->list, &iaa_devices);
541 
542 	nr_iaa++;
543 
544 	return iaa_device;
545 }
546 
547 static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)
548 {
549 	int ret = 0;
550 
551 	ret = init_device_compression_modes(iaa_device, iaa_wq->wq);
552 	if (ret)
553 		return ret;
554 
555 	return ret;
556 }
557 
558 static void del_iaa_device(struct iaa_device *iaa_device)
559 {
560 	list_del(&iaa_device->list);
561 
562 	nr_iaa--;
563 }
564 
565 static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,
566 		      struct iaa_wq **new_wq)
567 {
568 	struct idxd_device *idxd = iaa_device->idxd;
569 	struct pci_dev *pdev = idxd->pdev;
570 	struct device *dev = &pdev->dev;
571 	struct iaa_wq *iaa_wq;
572 
573 	iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);
574 	if (!iaa_wq)
575 		return -ENOMEM;
576 
577 	iaa_wq->wq = wq;
578 	iaa_wq->iaa_device = iaa_device;
579 	idxd_wq_set_private(wq, iaa_wq);
580 
581 	list_add_tail(&iaa_wq->list, &iaa_device->wqs);
582 
583 	iaa_device->n_wq++;
584 
585 	if (new_wq)
586 		*new_wq = iaa_wq;
587 
588 	dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",
589 		wq->id, iaa_device->idxd->id, iaa_device->n_wq);
590 
591 	return 0;
592 }
593 
594 static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
595 {
596 	struct idxd_device *idxd = iaa_device->idxd;
597 	struct pci_dev *pdev = idxd->pdev;
598 	struct device *dev = &pdev->dev;
599 	struct iaa_wq *iaa_wq;
600 
601 	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
602 		if (iaa_wq->wq == wq) {
603 			list_del(&iaa_wq->list);
604 			iaa_device->n_wq--;
605 
606 			dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",
607 				wq->id, iaa_device->idxd->id,
608 				iaa_device->n_wq, nr_iaa);
609 
610 			if (iaa_device->n_wq == 0)
611 				del_iaa_device(iaa_device);
612 			break;
613 		}
614 	}
615 }
616 
617 static void clear_wq_table(void)
618 {
619 	int cpu;
620 
621 	for (cpu = 0; cpu < nr_cpus; cpu++)
622 		wq_table_clear_entry(cpu);
623 
624 	pr_debug("cleared wq table\n");
625 }
626 
627 static void free_iaa_device(struct iaa_device *iaa_device)
628 {
629 	if (!iaa_device)
630 		return;
631 
632 	remove_device_compression_modes(iaa_device);
633 	kfree(iaa_device);
634 }
635 
636 static void __free_iaa_wq(struct iaa_wq *iaa_wq)
637 {
638 	struct iaa_device *iaa_device;
639 
640 	if (!iaa_wq)
641 		return;
642 
643 	iaa_device = iaa_wq->iaa_device;
644 	if (iaa_device->n_wq == 0)
645 		free_iaa_device(iaa_wq->iaa_device);
646 }
647 
648 static void free_iaa_wq(struct iaa_wq *iaa_wq)
649 {
650 	struct idxd_wq *wq;
651 
652 	__free_iaa_wq(iaa_wq);
653 
654 	wq = iaa_wq->wq;
655 
656 	kfree(iaa_wq);
657 	idxd_wq_set_private(wq, NULL);
658 }
659 
660 static int iaa_wq_get(struct idxd_wq *wq)
661 {
662 	struct idxd_device *idxd = wq->idxd;
663 	struct iaa_wq *iaa_wq;
664 	int ret = 0;
665 
666 	spin_lock(&idxd->dev_lock);
667 	iaa_wq = idxd_wq_get_private(wq);
668 	if (iaa_wq && !iaa_wq->remove) {
669 		iaa_wq->ref++;
670 		idxd_wq_get(wq);
671 	} else {
672 		ret = -ENODEV;
673 	}
674 	spin_unlock(&idxd->dev_lock);
675 
676 	return ret;
677 }
678 
679 static int iaa_wq_put(struct idxd_wq *wq)
680 {
681 	struct idxd_device *idxd = wq->idxd;
682 	struct iaa_wq *iaa_wq;
683 	bool free = false;
684 	int ret = 0;
685 
686 	spin_lock(&idxd->dev_lock);
687 	iaa_wq = idxd_wq_get_private(wq);
688 	if (iaa_wq) {
689 		iaa_wq->ref--;
690 		if (iaa_wq->ref == 0 && iaa_wq->remove) {
691 			idxd_wq_set_private(wq, NULL);
692 			free = true;
693 		}
694 		idxd_wq_put(wq);
695 	} else {
696 		ret = -ENODEV;
697 	}
698 	spin_unlock(&idxd->dev_lock);
699 	if (free) {
700 		__free_iaa_wq(iaa_wq);
701 		kfree(iaa_wq);
702 	}
703 
704 	return ret;
705 }
706 
707 static void free_wq_table(void)
708 {
709 	int cpu;
710 
711 	for (cpu = 0; cpu < nr_cpus; cpu++)
712 		wq_table_free_entry(cpu);
713 
714 	free_percpu(wq_table);
715 
716 	pr_debug("freed wq table\n");
717 }
718 
719 static int alloc_wq_table(int max_wqs)
720 {
721 	struct wq_table_entry *entry;
722 	int cpu;
723 
724 	wq_table = alloc_percpu(struct wq_table_entry);
725 	if (!wq_table)
726 		return -ENOMEM;
727 
728 	for (cpu = 0; cpu < nr_cpus; cpu++) {
729 		entry = per_cpu_ptr(wq_table, cpu);
730 		entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL);
731 		if (!entry->wqs) {
732 			free_wq_table();
733 			return -ENOMEM;
734 		}
735 
736 		entry->max_wqs = max_wqs;
737 	}
738 
739 	pr_debug("initialized wq table\n");
740 
741 	return 0;
742 }
743 
744 static int save_iaa_wq(struct idxd_wq *wq)
745 {
746 	struct iaa_device *iaa_device, *found = NULL;
747 	struct idxd_device *idxd;
748 	struct pci_dev *pdev;
749 	struct device *dev;
750 	int ret = 0;
751 
752 	list_for_each_entry(iaa_device, &iaa_devices, list) {
753 		if (iaa_device->idxd == wq->idxd) {
754 			idxd = iaa_device->idxd;
755 			pdev = idxd->pdev;
756 			dev = &pdev->dev;
757 			/*
758 			 * Check to see that we don't already have this wq.
759 			 * Shouldn't happen but we don't control probing.
760 			 */
761 			if (iaa_has_wq(iaa_device, wq)) {
762 				dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",
763 					iaa_device);
764 				goto out;
765 			}
766 
767 			found = iaa_device;
768 
769 			ret = add_iaa_wq(iaa_device, wq, NULL);
770 			if (ret)
771 				goto out;
772 
773 			break;
774 		}
775 	}
776 
777 	if (!found) {
778 		struct iaa_device *new_device;
779 		struct iaa_wq *new_wq;
780 
781 		new_device = add_iaa_device(wq->idxd);
782 		if (!new_device) {
783 			ret = -ENOMEM;
784 			goto out;
785 		}
786 
787 		ret = add_iaa_wq(new_device, wq, &new_wq);
788 		if (ret) {
789 			del_iaa_device(new_device);
790 			free_iaa_device(new_device);
791 			goto out;
792 		}
793 
794 		ret = init_iaa_device(new_device, new_wq);
795 		if (ret) {
796 			del_iaa_wq(new_device, new_wq->wq);
797 			del_iaa_device(new_device);
798 			free_iaa_wq(new_wq);
799 			goto out;
800 		}
801 	}
802 
803 	if (WARN_ON(nr_iaa == 0))
804 		return -EINVAL;
805 
806 	cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
807 	if (!cpus_per_iaa)
808 		cpus_per_iaa = 1;
809 out:
810 	return 0;
811 }
812 
813 static void remove_iaa_wq(struct idxd_wq *wq)
814 {
815 	struct iaa_device *iaa_device;
816 
817 	list_for_each_entry(iaa_device, &iaa_devices, list) {
818 		if (iaa_has_wq(iaa_device, wq)) {
819 			del_iaa_wq(iaa_device, wq);
820 			break;
821 		}
822 	}
823 
824 	if (nr_iaa) {
825 		cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
826 		if (!cpus_per_iaa)
827 			cpus_per_iaa = 1;
828 	} else
829 		cpus_per_iaa = 1;
830 }
831 
832 static int wq_table_add_wqs(int iaa, int cpu)
833 {
834 	struct iaa_device *iaa_device, *found_device = NULL;
835 	int ret = 0, cur_iaa = 0, n_wqs_added = 0;
836 	struct idxd_device *idxd;
837 	struct iaa_wq *iaa_wq;
838 	struct pci_dev *pdev;
839 	struct device *dev;
840 
841 	list_for_each_entry(iaa_device, &iaa_devices, list) {
842 		idxd = iaa_device->idxd;
843 		pdev = idxd->pdev;
844 		dev = &pdev->dev;
845 
846 		if (cur_iaa != iaa) {
847 			cur_iaa++;
848 			continue;
849 		}
850 
851 		found_device = iaa_device;
852 		dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",
853 			found_device->idxd->id, cur_iaa);
854 		break;
855 	}
856 
857 	if (!found_device) {
858 		found_device = list_first_entry_or_null(&iaa_devices,
859 							struct iaa_device, list);
860 		if (!found_device) {
861 			pr_debug("couldn't find any iaa devices with wqs!\n");
862 			ret = -EINVAL;
863 			goto out;
864 		}
865 		cur_iaa = 0;
866 
867 		idxd = found_device->idxd;
868 		pdev = idxd->pdev;
869 		dev = &pdev->dev;
870 		dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",
871 			found_device->idxd->id, cur_iaa);
872 	}
873 
874 	list_for_each_entry(iaa_wq, &found_device->wqs, list) {
875 		wq_table_add(cpu, iaa_wq->wq);
876 		pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
877 			 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);
878 		n_wqs_added++;
879 	}
880 
881 	if (!n_wqs_added) {
882 		pr_debug("couldn't find any iaa wqs!\n");
883 		ret = -EINVAL;
884 		goto out;
885 	}
886 out:
887 	return ret;
888 }
889 
890 /*
891  * Rebalance the wq table so that given a cpu, it's easy to find the
892  * closest IAA instance.  The idea is to try to choose the most
893  * appropriate IAA instance for a caller and spread available
894  * workqueues around to clients.
895  */
896 static void rebalance_wq_table(void)
897 {
898 	const struct cpumask *node_cpus;
899 	int node, cpu, iaa = -1;
900 
901 	if (nr_iaa == 0)
902 		return;
903 
904 	pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",
905 		 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);
906 
907 	clear_wq_table();
908 
909 	if (nr_iaa == 1) {
910 		for (cpu = 0; cpu < nr_cpus; cpu++) {
911 			if (WARN_ON(wq_table_add_wqs(0, cpu))) {
912 				pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu);
913 				return;
914 			}
915 		}
916 
917 		return;
918 	}
919 
920 	for_each_node_with_cpus(node) {
921 		node_cpus = cpumask_of_node(node);
922 
923 		for (cpu = 0; cpu <  cpumask_weight(node_cpus); cpu++) {
924 			int node_cpu = cpumask_nth(cpu, node_cpus);
925 
926 			if (WARN_ON(node_cpu >= nr_cpu_ids)) {
927 				pr_debug("node_cpu %d doesn't exist!\n", node_cpu);
928 				return;
929 			}
930 
931 			if ((cpu % cpus_per_iaa) == 0)
932 				iaa++;
933 
934 			if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) {
935 				pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);
936 				return;
937 			}
938 		}
939 	}
940 }
941 
942 static inline int check_completion(struct device *dev,
943 				   struct iax_completion_record *comp,
944 				   bool compress,
945 				   bool only_once)
946 {
947 	char *op_str = compress ? "compress" : "decompress";
948 	int ret = 0;
949 
950 	while (!comp->status) {
951 		if (only_once)
952 			return -EAGAIN;
953 		cpu_relax();
954 	}
955 
956 	if (comp->status != IAX_COMP_SUCCESS) {
957 		if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {
958 			ret = -ETIMEDOUT;
959 			dev_dbg(dev, "%s timed out, size=0x%x\n",
960 				op_str, comp->output_size);
961 			update_completion_timeout_errs();
962 			goto out;
963 		}
964 
965 		if (comp->status == IAA_ANALYTICS_ERROR &&
966 		    comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {
967 			ret = -E2BIG;
968 			dev_dbg(dev, "compressed > uncompressed size,"
969 				" not compressing, size=0x%x\n",
970 				comp->output_size);
971 			update_completion_comp_buf_overflow_errs();
972 			goto out;
973 		}
974 
975 		if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {
976 			ret = -EOVERFLOW;
977 			goto out;
978 		}
979 
980 		ret = -EINVAL;
981 		dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
982 			op_str, comp->status, comp->error_code, comp->output_size);
983 		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0);
984 		update_completion_einval_errs();
985 
986 		goto out;
987 	}
988 out:
989 	return ret;
990 }
991 
992 static int deflate_generic_decompress(struct acomp_req *req)
993 {
994 	void *src, *dst;
995 	int ret;
996 
997 	src = kmap_local_page(sg_page(req->src)) + req->src->offset;
998 	dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset;
999 
1000 	ret = crypto_comp_decompress(deflate_generic_tfm,
1001 				     src, req->slen, dst, &req->dlen);
1002 
1003 	kunmap_local(src);
1004 	kunmap_local(dst);
1005 
1006 	update_total_sw_decomp_calls();
1007 
1008 	return ret;
1009 }
1010 
1011 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1012 				struct acomp_req *req,
1013 				dma_addr_t *src_addr, dma_addr_t *dst_addr);
1014 
1015 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1016 			       struct idxd_wq *wq,
1017 			       dma_addr_t src_addr, unsigned int slen,
1018 			       dma_addr_t dst_addr, unsigned int *dlen,
1019 			       u32 compression_crc);
1020 
1021 static void iaa_desc_complete(struct idxd_desc *idxd_desc,
1022 			      enum idxd_complete_type comp_type,
1023 			      bool free_desc, void *__ctx,
1024 			      u32 *status)
1025 {
1026 	struct iaa_device_compression_mode *active_compression_mode;
1027 	struct iaa_compression_ctx *compression_ctx;
1028 	struct crypto_ctx *ctx = __ctx;
1029 	struct iaa_device *iaa_device;
1030 	struct idxd_device *idxd;
1031 	struct iaa_wq *iaa_wq;
1032 	struct pci_dev *pdev;
1033 	struct device *dev;
1034 	int ret, err = 0;
1035 
1036 	compression_ctx = crypto_tfm_ctx(ctx->tfm);
1037 
1038 	iaa_wq = idxd_wq_get_private(idxd_desc->wq);
1039 	iaa_device = iaa_wq->iaa_device;
1040 	idxd = iaa_device->idxd;
1041 	pdev = idxd->pdev;
1042 	dev = &pdev->dev;
1043 
1044 	active_compression_mode = get_iaa_device_compression_mode(iaa_device,
1045 								  compression_ctx->mode);
1046 	dev_dbg(dev, "%s: compression mode %s,"
1047 		" ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
1048 		active_compression_mode->name,
1049 		ctx->src_addr, ctx->dst_addr);
1050 
1051 	ret = check_completion(dev, idxd_desc->iax_completion,
1052 			       ctx->compress, false);
1053 	if (ret) {
1054 		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1055 		if (!ctx->compress &&
1056 		    idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1057 			pr_warn("%s: falling back to deflate-generic decompress, "
1058 				"analytics error code %x\n", __func__,
1059 				idxd_desc->iax_completion->error_code);
1060 			ret = deflate_generic_decompress(ctx->req);
1061 			if (ret) {
1062 				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1063 					__func__, ret);
1064 				err = -EIO;
1065 				goto err;
1066 			}
1067 		} else {
1068 			err = -EIO;
1069 			goto err;
1070 		}
1071 	} else {
1072 		ctx->req->dlen = idxd_desc->iax_completion->output_size;
1073 	}
1074 
1075 	/* Update stats */
1076 	if (ctx->compress) {
1077 		update_total_comp_bytes_out(ctx->req->dlen);
1078 		update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);
1079 	} else {
1080 		update_total_decomp_bytes_in(ctx->req->slen);
1081 		update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen);
1082 	}
1083 
1084 	if (ctx->compress && compression_ctx->verify_compress) {
1085 		dma_addr_t src_addr, dst_addr;
1086 		u32 compression_crc;
1087 
1088 		compression_crc = idxd_desc->iax_completion->crc;
1089 
1090 		ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
1091 		if (ret) {
1092 			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1093 			err = -EIO;
1094 			goto out;
1095 		}
1096 
1097 		ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
1098 					  ctx->req->slen, dst_addr, &ctx->req->dlen,
1099 					  compression_crc);
1100 		if (ret) {
1101 			dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
1102 			err = -EIO;
1103 		}
1104 
1105 		dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
1106 		dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
1107 
1108 		goto out;
1109 	}
1110 err:
1111 	dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
1112 	dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
1113 out:
1114 	if (ret != 0)
1115 		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1116 
1117 	if (ctx->req->base.complete)
1118 		acomp_request_complete(ctx->req, err);
1119 
1120 	if (free_desc)
1121 		idxd_free_desc(idxd_desc->wq, idxd_desc);
1122 	iaa_wq_put(idxd_desc->wq);
1123 }
1124 
1125 static int iaa_compress(struct crypto_tfm *tfm,	struct acomp_req *req,
1126 			struct idxd_wq *wq,
1127 			dma_addr_t src_addr, unsigned int slen,
1128 			dma_addr_t dst_addr, unsigned int *dlen,
1129 			u32 *compression_crc,
1130 			bool disable_async)
1131 {
1132 	struct iaa_device_compression_mode *active_compression_mode;
1133 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1134 	struct iaa_device *iaa_device;
1135 	struct idxd_desc *idxd_desc;
1136 	struct iax_hw_desc *desc;
1137 	struct idxd_device *idxd;
1138 	struct iaa_wq *iaa_wq;
1139 	struct pci_dev *pdev;
1140 	struct device *dev;
1141 	int ret = 0;
1142 
1143 	iaa_wq = idxd_wq_get_private(wq);
1144 	iaa_device = iaa_wq->iaa_device;
1145 	idxd = iaa_device->idxd;
1146 	pdev = idxd->pdev;
1147 	dev = &pdev->dev;
1148 
1149 	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1150 
1151 	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1152 	if (IS_ERR(idxd_desc)) {
1153 		dev_dbg(dev, "idxd descriptor allocation failed\n");
1154 		dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
1155 		return PTR_ERR(idxd_desc);
1156 	}
1157 	desc = idxd_desc->iax_hw;
1158 
1159 	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
1160 		IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
1161 	desc->opcode = IAX_OPCODE_COMPRESS;
1162 	desc->compr_flags = IAA_COMP_FLAGS;
1163 	desc->priv = 0;
1164 
1165 	desc->src1_addr = (u64)src_addr;
1166 	desc->src1_size = slen;
1167 	desc->dst_addr = (u64)dst_addr;
1168 	desc->max_dst_size = *dlen;
1169 	desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
1170 	desc->src2_size = sizeof(struct aecs_comp_table_record);
1171 	desc->completion_addr = idxd_desc->compl_dma;
1172 
1173 	if (ctx->use_irq && !disable_async) {
1174 		desc->flags |= IDXD_OP_FLAG_RCI;
1175 
1176 		idxd_desc->crypto.req = req;
1177 		idxd_desc->crypto.tfm = tfm;
1178 		idxd_desc->crypto.src_addr = src_addr;
1179 		idxd_desc->crypto.dst_addr = dst_addr;
1180 		idxd_desc->crypto.compress = true;
1181 
1182 		dev_dbg(dev, "%s use_async_irq: compression mode %s,"
1183 			" src_addr %llx, dst_addr %llx\n", __func__,
1184 			active_compression_mode->name,
1185 			src_addr, dst_addr);
1186 	} else if (ctx->async_mode && !disable_async)
1187 		req->base.data = idxd_desc;
1188 
1189 	dev_dbg(dev, "%s: compression mode %s,"
1190 		" desc->src1_addr %llx, desc->src1_size %d,"
1191 		" desc->dst_addr %llx, desc->max_dst_size %d,"
1192 		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1193 		active_compression_mode->name,
1194 		desc->src1_addr, desc->src1_size, desc->dst_addr,
1195 		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1196 
1197 	ret = idxd_submit_desc(wq, idxd_desc);
1198 	if (ret) {
1199 		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1200 		goto err;
1201 	}
1202 
1203 	/* Update stats */
1204 	update_total_comp_calls();
1205 	update_wq_comp_calls(wq);
1206 
1207 	if (ctx->async_mode && !disable_async) {
1208 		ret = -EINPROGRESS;
1209 		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1210 		goto out;
1211 	}
1212 
1213 	ret = check_completion(dev, idxd_desc->iax_completion, true, false);
1214 	if (ret) {
1215 		dev_dbg(dev, "check_completion failed ret=%d\n", ret);
1216 		goto err;
1217 	}
1218 
1219 	*dlen = idxd_desc->iax_completion->output_size;
1220 
1221 	/* Update stats */
1222 	update_total_comp_bytes_out(*dlen);
1223 	update_wq_comp_bytes(wq, *dlen);
1224 
1225 	*compression_crc = idxd_desc->iax_completion->crc;
1226 
1227 	if (!ctx->async_mode || disable_async)
1228 		idxd_free_desc(wq, idxd_desc);
1229 out:
1230 	return ret;
1231 err:
1232 	idxd_free_desc(wq, idxd_desc);
1233 	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1234 
1235 	goto out;
1236 }
1237 
1238 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1239 				struct acomp_req *req,
1240 				dma_addr_t *src_addr, dma_addr_t *dst_addr)
1241 {
1242 	int ret = 0;
1243 	int nr_sgs;
1244 
1245 	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1246 	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1247 
1248 	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1249 	if (nr_sgs <= 0 || nr_sgs > 1) {
1250 		dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"
1251 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1252 			iaa_wq->wq->id, ret);
1253 		ret = -EIO;
1254 		goto out;
1255 	}
1256 	*src_addr = sg_dma_address(req->src);
1257 	dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1258 		" req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
1259 		req->src, req->slen, sg_dma_len(req->src));
1260 
1261 	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1262 	if (nr_sgs <= 0 || nr_sgs > 1) {
1263 		dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"
1264 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1265 			iaa_wq->wq->id, ret);
1266 		ret = -EIO;
1267 		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1268 		goto out;
1269 	}
1270 	*dst_addr = sg_dma_address(req->dst);
1271 	dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1272 		" req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
1273 		req->dst, req->dlen, sg_dma_len(req->dst));
1274 out:
1275 	return ret;
1276 }
1277 
1278 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1279 			       struct idxd_wq *wq,
1280 			       dma_addr_t src_addr, unsigned int slen,
1281 			       dma_addr_t dst_addr, unsigned int *dlen,
1282 			       u32 compression_crc)
1283 {
1284 	struct iaa_device_compression_mode *active_compression_mode;
1285 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1286 	struct iaa_device *iaa_device;
1287 	struct idxd_desc *idxd_desc;
1288 	struct iax_hw_desc *desc;
1289 	struct idxd_device *idxd;
1290 	struct iaa_wq *iaa_wq;
1291 	struct pci_dev *pdev;
1292 	struct device *dev;
1293 	int ret = 0;
1294 
1295 	iaa_wq = idxd_wq_get_private(wq);
1296 	iaa_device = iaa_wq->iaa_device;
1297 	idxd = iaa_device->idxd;
1298 	pdev = idxd->pdev;
1299 	dev = &pdev->dev;
1300 
1301 	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1302 
1303 	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1304 	if (IS_ERR(idxd_desc)) {
1305 		dev_dbg(dev, "idxd descriptor allocation failed\n");
1306 		dev_dbg(dev, "iaa compress failed: ret=%ld\n",
1307 			PTR_ERR(idxd_desc));
1308 		return PTR_ERR(idxd_desc);
1309 	}
1310 	desc = idxd_desc->iax_hw;
1311 
1312 	/* Verify (optional) - decompress and check crc, suppress dest write */
1313 
1314 	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1315 	desc->opcode = IAX_OPCODE_DECOMPRESS;
1316 	desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;
1317 	desc->priv = 0;
1318 
1319 	desc->src1_addr = (u64)dst_addr;
1320 	desc->src1_size = *dlen;
1321 	desc->dst_addr = (u64)src_addr;
1322 	desc->max_dst_size = slen;
1323 	desc->completion_addr = idxd_desc->compl_dma;
1324 
1325 	dev_dbg(dev, "(verify) compression mode %s,"
1326 		" desc->src1_addr %llx, desc->src1_size %d,"
1327 		" desc->dst_addr %llx, desc->max_dst_size %d,"
1328 		" desc->src2_addr %llx, desc->src2_size %d\n",
1329 		active_compression_mode->name,
1330 		desc->src1_addr, desc->src1_size, desc->dst_addr,
1331 		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1332 
1333 	ret = idxd_submit_desc(wq, idxd_desc);
1334 	if (ret) {
1335 		dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);
1336 		goto err;
1337 	}
1338 
1339 	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1340 	if (ret) {
1341 		dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);
1342 		goto err;
1343 	}
1344 
1345 	if (compression_crc != idxd_desc->iax_completion->crc) {
1346 		ret = -EINVAL;
1347 		dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
1348 			" comp=0x%x, decomp=0x%x\n", compression_crc,
1349 			idxd_desc->iax_completion->crc);
1350 		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
1351 			       8, 1, idxd_desc->iax_completion, 64, 0);
1352 		goto err;
1353 	}
1354 
1355 	idxd_free_desc(wq, idxd_desc);
1356 out:
1357 	return ret;
1358 err:
1359 	idxd_free_desc(wq, idxd_desc);
1360 	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1361 
1362 	goto out;
1363 }
1364 
1365 static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
1366 			  struct idxd_wq *wq,
1367 			  dma_addr_t src_addr, unsigned int slen,
1368 			  dma_addr_t dst_addr, unsigned int *dlen,
1369 			  bool disable_async)
1370 {
1371 	struct iaa_device_compression_mode *active_compression_mode;
1372 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1373 	struct iaa_device *iaa_device;
1374 	struct idxd_desc *idxd_desc;
1375 	struct iax_hw_desc *desc;
1376 	struct idxd_device *idxd;
1377 	struct iaa_wq *iaa_wq;
1378 	struct pci_dev *pdev;
1379 	struct device *dev;
1380 	int ret = 0;
1381 
1382 	iaa_wq = idxd_wq_get_private(wq);
1383 	iaa_device = iaa_wq->iaa_device;
1384 	idxd = iaa_device->idxd;
1385 	pdev = idxd->pdev;
1386 	dev = &pdev->dev;
1387 
1388 	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1389 
1390 	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1391 	if (IS_ERR(idxd_desc)) {
1392 		dev_dbg(dev, "idxd descriptor allocation failed\n");
1393 		dev_dbg(dev, "iaa decompress failed: ret=%ld\n",
1394 			PTR_ERR(idxd_desc));
1395 		return PTR_ERR(idxd_desc);
1396 	}
1397 	desc = idxd_desc->iax_hw;
1398 
1399 	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1400 	desc->opcode = IAX_OPCODE_DECOMPRESS;
1401 	desc->max_dst_size = PAGE_SIZE;
1402 	desc->decompr_flags = IAA_DECOMP_FLAGS;
1403 	desc->priv = 0;
1404 
1405 	desc->src1_addr = (u64)src_addr;
1406 	desc->dst_addr = (u64)dst_addr;
1407 	desc->max_dst_size = *dlen;
1408 	desc->src1_size = slen;
1409 	desc->completion_addr = idxd_desc->compl_dma;
1410 
1411 	if (ctx->use_irq && !disable_async) {
1412 		desc->flags |= IDXD_OP_FLAG_RCI;
1413 
1414 		idxd_desc->crypto.req = req;
1415 		idxd_desc->crypto.tfm = tfm;
1416 		idxd_desc->crypto.src_addr = src_addr;
1417 		idxd_desc->crypto.dst_addr = dst_addr;
1418 		idxd_desc->crypto.compress = false;
1419 
1420 		dev_dbg(dev, "%s: use_async_irq compression mode %s,"
1421 			" src_addr %llx, dst_addr %llx\n", __func__,
1422 			active_compression_mode->name,
1423 			src_addr, dst_addr);
1424 	} else if (ctx->async_mode && !disable_async)
1425 		req->base.data = idxd_desc;
1426 
1427 	dev_dbg(dev, "%s: decompression mode %s,"
1428 		" desc->src1_addr %llx, desc->src1_size %d,"
1429 		" desc->dst_addr %llx, desc->max_dst_size %d,"
1430 		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1431 		active_compression_mode->name,
1432 		desc->src1_addr, desc->src1_size, desc->dst_addr,
1433 		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1434 
1435 	ret = idxd_submit_desc(wq, idxd_desc);
1436 	if (ret) {
1437 		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1438 		goto err;
1439 	}
1440 
1441 	/* Update stats */
1442 	update_total_decomp_calls();
1443 	update_wq_decomp_calls(wq);
1444 
1445 	if (ctx->async_mode && !disable_async) {
1446 		ret = -EINPROGRESS;
1447 		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1448 		goto out;
1449 	}
1450 
1451 	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1452 	if (ret) {
1453 		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1454 		if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1455 			pr_warn("%s: falling back to deflate-generic decompress, "
1456 				"analytics error code %x\n", __func__,
1457 				idxd_desc->iax_completion->error_code);
1458 			ret = deflate_generic_decompress(req);
1459 			if (ret) {
1460 				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1461 					__func__, ret);
1462 				goto err;
1463 			}
1464 		} else {
1465 			goto err;
1466 		}
1467 	} else {
1468 		req->dlen = idxd_desc->iax_completion->output_size;
1469 	}
1470 
1471 	*dlen = req->dlen;
1472 
1473 	if (!ctx->async_mode || disable_async)
1474 		idxd_free_desc(wq, idxd_desc);
1475 
1476 	/* Update stats */
1477 	update_total_decomp_bytes_in(slen);
1478 	update_wq_decomp_bytes(wq, slen);
1479 out:
1480 	return ret;
1481 err:
1482 	idxd_free_desc(wq, idxd_desc);
1483 	dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);
1484 
1485 	goto out;
1486 }
1487 
1488 static int iaa_comp_acompress(struct acomp_req *req)
1489 {
1490 	struct iaa_compression_ctx *compression_ctx;
1491 	struct crypto_tfm *tfm = req->base.tfm;
1492 	dma_addr_t src_addr, dst_addr;
1493 	bool disable_async = false;
1494 	int nr_sgs, cpu, ret = 0;
1495 	struct iaa_wq *iaa_wq;
1496 	u32 compression_crc;
1497 	struct idxd_wq *wq;
1498 	struct device *dev;
1499 	int order = -1;
1500 
1501 	compression_ctx = crypto_tfm_ctx(tfm);
1502 
1503 	if (!iaa_crypto_enabled) {
1504 		pr_debug("iaa_crypto disabled, not compressing\n");
1505 		return -ENODEV;
1506 	}
1507 
1508 	if (!req->src || !req->slen) {
1509 		pr_debug("invalid src, not compressing\n");
1510 		return -EINVAL;
1511 	}
1512 
1513 	cpu = get_cpu();
1514 	wq = wq_table_next_wq(cpu);
1515 	put_cpu();
1516 	if (!wq) {
1517 		pr_debug("no wq configured for cpu=%d\n", cpu);
1518 		return -ENODEV;
1519 	}
1520 
1521 	ret = iaa_wq_get(wq);
1522 	if (ret) {
1523 		pr_debug("no wq available for cpu=%d\n", cpu);
1524 		return -ENODEV;
1525 	}
1526 
1527 	iaa_wq = idxd_wq_get_private(wq);
1528 
1529 	if (!req->dst) {
1530 		gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
1531 
1532 		/* incompressible data will always be < 2 * slen */
1533 		req->dlen = 2 * req->slen;
1534 		order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);
1535 		req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);
1536 		if (!req->dst) {
1537 			ret = -ENOMEM;
1538 			order = -1;
1539 			goto out;
1540 		}
1541 		disable_async = true;
1542 	}
1543 
1544 	dev = &wq->idxd->pdev->dev;
1545 
1546 	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1547 	if (nr_sgs <= 0 || nr_sgs > 1) {
1548 		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1549 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1550 			iaa_wq->wq->id, ret);
1551 		ret = -EIO;
1552 		goto out;
1553 	}
1554 	src_addr = sg_dma_address(req->src);
1555 	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1556 		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1557 		req->src, req->slen, sg_dma_len(req->src));
1558 
1559 	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1560 	if (nr_sgs <= 0 || nr_sgs > 1) {
1561 		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1562 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1563 			iaa_wq->wq->id, ret);
1564 		ret = -EIO;
1565 		goto err_map_dst;
1566 	}
1567 	dst_addr = sg_dma_address(req->dst);
1568 	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1569 		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1570 		req->dst, req->dlen, sg_dma_len(req->dst));
1571 
1572 	ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr,
1573 			   &req->dlen, &compression_crc, disable_async);
1574 	if (ret == -EINPROGRESS)
1575 		return ret;
1576 
1577 	if (!ret && compression_ctx->verify_compress) {
1578 		ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr);
1579 		if (ret) {
1580 			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1581 			goto out;
1582 		}
1583 
1584 		ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen,
1585 					  dst_addr, &req->dlen, compression_crc);
1586 		if (ret)
1587 			dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);
1588 
1589 		dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1590 		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1591 
1592 		goto out;
1593 	}
1594 
1595 	if (ret)
1596 		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1597 
1598 	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1599 err_map_dst:
1600 	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1601 out:
1602 	iaa_wq_put(wq);
1603 
1604 	if (order >= 0)
1605 		sgl_free_order(req->dst, order);
1606 
1607 	return ret;
1608 }
1609 
1610 static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req)
1611 {
1612 	gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1613 		GFP_KERNEL : GFP_ATOMIC;
1614 	struct crypto_tfm *tfm = req->base.tfm;
1615 	dma_addr_t src_addr, dst_addr;
1616 	int nr_sgs, cpu, ret = 0;
1617 	struct iaa_wq *iaa_wq;
1618 	struct device *dev;
1619 	struct idxd_wq *wq;
1620 	int order = -1;
1621 
1622 	cpu = get_cpu();
1623 	wq = wq_table_next_wq(cpu);
1624 	put_cpu();
1625 	if (!wq) {
1626 		pr_debug("no wq configured for cpu=%d\n", cpu);
1627 		return -ENODEV;
1628 	}
1629 
1630 	ret = iaa_wq_get(wq);
1631 	if (ret) {
1632 		pr_debug("no wq available for cpu=%d\n", cpu);
1633 		return -ENODEV;
1634 	}
1635 
1636 	iaa_wq = idxd_wq_get_private(wq);
1637 
1638 	dev = &wq->idxd->pdev->dev;
1639 
1640 	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1641 	if (nr_sgs <= 0 || nr_sgs > 1) {
1642 		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1643 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1644 			iaa_wq->wq->id, ret);
1645 		ret = -EIO;
1646 		goto out;
1647 	}
1648 	src_addr = sg_dma_address(req->src);
1649 	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1650 		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1651 		req->src, req->slen, sg_dma_len(req->src));
1652 
1653 	req->dlen = 4 * req->slen; /* start with ~avg comp rato */
1654 alloc_dest:
1655 	order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);
1656 	req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);
1657 	if (!req->dst) {
1658 		ret = -ENOMEM;
1659 		order = -1;
1660 		goto out;
1661 	}
1662 
1663 	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1664 	if (nr_sgs <= 0 || nr_sgs > 1) {
1665 		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1666 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1667 			iaa_wq->wq->id, ret);
1668 		ret = -EIO;
1669 		goto err_map_dst;
1670 	}
1671 
1672 	dst_addr = sg_dma_address(req->dst);
1673 	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1674 		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1675 		req->dst, req->dlen, sg_dma_len(req->dst));
1676 	ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1677 			     dst_addr, &req->dlen, true);
1678 	if (ret == -EOVERFLOW) {
1679 		dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1680 		req->dlen *= 2;
1681 		if (req->dlen > CRYPTO_ACOMP_DST_MAX)
1682 			goto err_map_dst;
1683 		goto alloc_dest;
1684 	}
1685 
1686 	if (ret != 0)
1687 		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1688 
1689 	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1690 err_map_dst:
1691 	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1692 out:
1693 	iaa_wq_put(wq);
1694 
1695 	if (order >= 0)
1696 		sgl_free_order(req->dst, order);
1697 
1698 	return ret;
1699 }
1700 
1701 static int iaa_comp_adecompress(struct acomp_req *req)
1702 {
1703 	struct crypto_tfm *tfm = req->base.tfm;
1704 	dma_addr_t src_addr, dst_addr;
1705 	int nr_sgs, cpu, ret = 0;
1706 	struct iaa_wq *iaa_wq;
1707 	struct device *dev;
1708 	struct idxd_wq *wq;
1709 
1710 	if (!iaa_crypto_enabled) {
1711 		pr_debug("iaa_crypto disabled, not decompressing\n");
1712 		return -ENODEV;
1713 	}
1714 
1715 	if (!req->src || !req->slen) {
1716 		pr_debug("invalid src, not decompressing\n");
1717 		return -EINVAL;
1718 	}
1719 
1720 	if (!req->dst)
1721 		return iaa_comp_adecompress_alloc_dest(req);
1722 
1723 	cpu = get_cpu();
1724 	wq = wq_table_next_wq(cpu);
1725 	put_cpu();
1726 	if (!wq) {
1727 		pr_debug("no wq configured for cpu=%d\n", cpu);
1728 		return -ENODEV;
1729 	}
1730 
1731 	ret = iaa_wq_get(wq);
1732 	if (ret) {
1733 		pr_debug("no wq available for cpu=%d\n", cpu);
1734 		return -ENODEV;
1735 	}
1736 
1737 	iaa_wq = idxd_wq_get_private(wq);
1738 
1739 	dev = &wq->idxd->pdev->dev;
1740 
1741 	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1742 	if (nr_sgs <= 0 || nr_sgs > 1) {
1743 		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1744 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1745 			iaa_wq->wq->id, ret);
1746 		ret = -EIO;
1747 		goto out;
1748 	}
1749 	src_addr = sg_dma_address(req->src);
1750 	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1751 		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1752 		req->src, req->slen, sg_dma_len(req->src));
1753 
1754 	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1755 	if (nr_sgs <= 0 || nr_sgs > 1) {
1756 		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1757 			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1758 			iaa_wq->wq->id, ret);
1759 		ret = -EIO;
1760 		goto err_map_dst;
1761 	}
1762 	dst_addr = sg_dma_address(req->dst);
1763 	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1764 		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1765 		req->dst, req->dlen, sg_dma_len(req->dst));
1766 
1767 	ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1768 			     dst_addr, &req->dlen, false);
1769 	if (ret == -EINPROGRESS)
1770 		return ret;
1771 
1772 	if (ret != 0)
1773 		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1774 
1775 	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1776 err_map_dst:
1777 	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1778 out:
1779 	iaa_wq_put(wq);
1780 
1781 	return ret;
1782 }
1783 
1784 static void compression_ctx_init(struct iaa_compression_ctx *ctx)
1785 {
1786 	ctx->verify_compress = iaa_verify_compress;
1787 	ctx->async_mode = async_mode;
1788 	ctx->use_irq = use_irq;
1789 }
1790 
1791 static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
1792 {
1793 	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
1794 	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1795 
1796 	compression_ctx_init(ctx);
1797 
1798 	ctx->mode = IAA_MODE_FIXED;
1799 
1800 	return 0;
1801 }
1802 
1803 static void dst_free(struct scatterlist *sgl)
1804 {
1805 	/*
1806 	 * Called for req->dst = NULL cases but we free elsewhere
1807 	 * using sgl_free_order().
1808 	 */
1809 }
1810 
1811 static struct acomp_alg iaa_acomp_fixed_deflate = {
1812 	.init			= iaa_comp_init_fixed,
1813 	.compress		= iaa_comp_acompress,
1814 	.decompress		= iaa_comp_adecompress,
1815 	.dst_free               = dst_free,
1816 	.base			= {
1817 		.cra_name		= "deflate",
1818 		.cra_driver_name	= "deflate-iaa",
1819 		.cra_flags		= CRYPTO_ALG_ASYNC,
1820 		.cra_ctxsize		= sizeof(struct iaa_compression_ctx),
1821 		.cra_module		= THIS_MODULE,
1822 		.cra_priority		= IAA_ALG_PRIORITY,
1823 	}
1824 };
1825 
1826 static int iaa_register_compression_device(void)
1827 {
1828 	int ret;
1829 
1830 	ret = crypto_register_acomp(&iaa_acomp_fixed_deflate);
1831 	if (ret) {
1832 		pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
1833 		goto out;
1834 	}
1835 
1836 	iaa_crypto_registered = true;
1837 out:
1838 	return ret;
1839 }
1840 
1841 static int iaa_unregister_compression_device(void)
1842 {
1843 	if (iaa_crypto_registered)
1844 		crypto_unregister_acomp(&iaa_acomp_fixed_deflate);
1845 
1846 	return 0;
1847 }
1848 
1849 static int iaa_crypto_probe(struct idxd_dev *idxd_dev)
1850 {
1851 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1852 	struct idxd_device *idxd = wq->idxd;
1853 	struct idxd_driver_data *data = idxd->data;
1854 	struct device *dev = &idxd_dev->conf_dev;
1855 	bool first_wq = false;
1856 	int ret = 0;
1857 
1858 	if (idxd->state != IDXD_DEV_ENABLED)
1859 		return -ENXIO;
1860 
1861 	if (data->type != IDXD_TYPE_IAX)
1862 		return -ENODEV;
1863 
1864 	mutex_lock(&wq->wq_lock);
1865 
1866 	if (idxd_wq_get_private(wq)) {
1867 		mutex_unlock(&wq->wq_lock);
1868 		return -EBUSY;
1869 	}
1870 
1871 	if (!idxd_wq_driver_name_match(wq, dev)) {
1872 		dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",
1873 			idxd->id, wq->id, wq->driver_name, dev->driver->name);
1874 		idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
1875 		ret = -ENODEV;
1876 		goto err;
1877 	}
1878 
1879 	wq->type = IDXD_WQT_KERNEL;
1880 
1881 	ret = idxd_drv_enable_wq(wq);
1882 	if (ret < 0) {
1883 		dev_dbg(dev, "enable wq %d.%d failed: %d\n",
1884 			idxd->id, wq->id, ret);
1885 		ret = -ENXIO;
1886 		goto err;
1887 	}
1888 
1889 	mutex_lock(&iaa_devices_lock);
1890 
1891 	if (list_empty(&iaa_devices)) {
1892 		ret = alloc_wq_table(wq->idxd->max_wqs);
1893 		if (ret)
1894 			goto err_alloc;
1895 		first_wq = true;
1896 	}
1897 
1898 	ret = save_iaa_wq(wq);
1899 	if (ret)
1900 		goto err_save;
1901 
1902 	rebalance_wq_table();
1903 
1904 	if (first_wq) {
1905 		iaa_crypto_enabled = true;
1906 		ret = iaa_register_compression_device();
1907 		if (ret != 0) {
1908 			iaa_crypto_enabled = false;
1909 			dev_dbg(dev, "IAA compression device registration failed\n");
1910 			goto err_register;
1911 		}
1912 		try_module_get(THIS_MODULE);
1913 
1914 		pr_info("iaa_crypto now ENABLED\n");
1915 	}
1916 
1917 	mutex_unlock(&iaa_devices_lock);
1918 out:
1919 	mutex_unlock(&wq->wq_lock);
1920 
1921 	return ret;
1922 
1923 err_register:
1924 	remove_iaa_wq(wq);
1925 	free_iaa_wq(idxd_wq_get_private(wq));
1926 err_save:
1927 	if (first_wq)
1928 		free_wq_table();
1929 err_alloc:
1930 	mutex_unlock(&iaa_devices_lock);
1931 	idxd_drv_disable_wq(wq);
1932 err:
1933 	wq->type = IDXD_WQT_NONE;
1934 
1935 	goto out;
1936 }
1937 
1938 static void iaa_crypto_remove(struct idxd_dev *idxd_dev)
1939 {
1940 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1941 	struct idxd_device *idxd = wq->idxd;
1942 	struct iaa_wq *iaa_wq;
1943 	bool free = false;
1944 
1945 	idxd_wq_quiesce(wq);
1946 
1947 	mutex_lock(&wq->wq_lock);
1948 	mutex_lock(&iaa_devices_lock);
1949 
1950 	remove_iaa_wq(wq);
1951 
1952 	spin_lock(&idxd->dev_lock);
1953 	iaa_wq = idxd_wq_get_private(wq);
1954 	if (!iaa_wq) {
1955 		spin_unlock(&idxd->dev_lock);
1956 		pr_err("%s: no iaa_wq available to remove\n", __func__);
1957 		goto out;
1958 	}
1959 
1960 	if (iaa_wq->ref) {
1961 		iaa_wq->remove = true;
1962 	} else {
1963 		wq = iaa_wq->wq;
1964 		idxd_wq_set_private(wq, NULL);
1965 		free = true;
1966 	}
1967 	spin_unlock(&idxd->dev_lock);
1968 	if (free) {
1969 		__free_iaa_wq(iaa_wq);
1970 		kfree(iaa_wq);
1971 	}
1972 
1973 	idxd_drv_disable_wq(wq);
1974 	rebalance_wq_table();
1975 
1976 	if (nr_iaa == 0) {
1977 		iaa_crypto_enabled = false;
1978 		free_wq_table();
1979 		module_put(THIS_MODULE);
1980 
1981 		pr_info("iaa_crypto now DISABLED\n");
1982 	}
1983 out:
1984 	mutex_unlock(&iaa_devices_lock);
1985 	mutex_unlock(&wq->wq_lock);
1986 }
1987 
1988 static enum idxd_dev_type dev_types[] = {
1989 	IDXD_DEV_WQ,
1990 	IDXD_DEV_NONE,
1991 };
1992 
1993 static struct idxd_device_driver iaa_crypto_driver = {
1994 	.probe = iaa_crypto_probe,
1995 	.remove = iaa_crypto_remove,
1996 	.name = IDXD_SUBDRIVER_NAME,
1997 	.type = dev_types,
1998 	.desc_complete = iaa_desc_complete,
1999 };
2000 
2001 static int __init iaa_crypto_init_module(void)
2002 {
2003 	int ret = 0;
2004 	int node;
2005 
2006 	nr_cpus = num_possible_cpus();
2007 	for_each_node_with_cpus(node)
2008 		nr_nodes++;
2009 	if (!nr_nodes) {
2010 		pr_err("IAA couldn't find any nodes with cpus\n");
2011 		return -ENODEV;
2012 	}
2013 	nr_cpus_per_node = nr_cpus / nr_nodes;
2014 
2015 	if (crypto_has_comp("deflate-generic", 0, 0))
2016 		deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0);
2017 
2018 	if (IS_ERR_OR_NULL(deflate_generic_tfm)) {
2019 		pr_err("IAA could not alloc %s tfm: errcode = %ld\n",
2020 		       "deflate-generic", PTR_ERR(deflate_generic_tfm));
2021 		return -ENOMEM;
2022 	}
2023 
2024 	ret = iaa_aecs_init_fixed();
2025 	if (ret < 0) {
2026 		pr_debug("IAA fixed compression mode init failed\n");
2027 		goto err_aecs_init;
2028 	}
2029 
2030 	ret = idxd_driver_register(&iaa_crypto_driver);
2031 	if (ret) {
2032 		pr_debug("IAA wq sub-driver registration failed\n");
2033 		goto err_driver_reg;
2034 	}
2035 
2036 	ret = driver_create_file(&iaa_crypto_driver.drv,
2037 				 &driver_attr_verify_compress);
2038 	if (ret) {
2039 		pr_debug("IAA verify_compress attr creation failed\n");
2040 		goto err_verify_attr_create;
2041 	}
2042 
2043 	ret = driver_create_file(&iaa_crypto_driver.drv,
2044 				 &driver_attr_sync_mode);
2045 	if (ret) {
2046 		pr_debug("IAA sync mode attr creation failed\n");
2047 		goto err_sync_attr_create;
2048 	}
2049 
2050 	if (iaa_crypto_debugfs_init())
2051 		pr_warn("debugfs init failed, stats not available\n");
2052 
2053 	pr_debug("initialized\n");
2054 out:
2055 	return ret;
2056 
2057 err_sync_attr_create:
2058 	driver_remove_file(&iaa_crypto_driver.drv,
2059 			   &driver_attr_verify_compress);
2060 err_verify_attr_create:
2061 	idxd_driver_unregister(&iaa_crypto_driver);
2062 err_driver_reg:
2063 	iaa_aecs_cleanup_fixed();
2064 err_aecs_init:
2065 	crypto_free_comp(deflate_generic_tfm);
2066 
2067 	goto out;
2068 }
2069 
2070 static void __exit iaa_crypto_cleanup_module(void)
2071 {
2072 	if (iaa_unregister_compression_device())
2073 		pr_debug("IAA compression device unregister failed\n");
2074 
2075 	iaa_crypto_debugfs_cleanup();
2076 	driver_remove_file(&iaa_crypto_driver.drv,
2077 			   &driver_attr_sync_mode);
2078 	driver_remove_file(&iaa_crypto_driver.drv,
2079 			   &driver_attr_verify_compress);
2080 	idxd_driver_unregister(&iaa_crypto_driver);
2081 	iaa_aecs_cleanup_fixed();
2082 	crypto_free_comp(deflate_generic_tfm);
2083 
2084 	pr_debug("cleaned up\n");
2085 }
2086 
2087 MODULE_IMPORT_NS(IDXD);
2088 MODULE_LICENSE("GPL");
2089 MODULE_ALIAS_IDXD_DEVICE(0);
2090 MODULE_AUTHOR("Intel Corporation");
2091 MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");
2092 
2093 module_init(iaa_crypto_init_module);
2094 module_exit(iaa_crypto_cleanup_module);
2095