1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ 3 4 #include <linux/init.h> 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/pci.h> 8 #include <linux/device.h> 9 #include <linux/iommu.h> 10 #include <uapi/linux/idxd.h> 11 #include <linux/highmem.h> 12 #include <linux/sched/smt.h> 13 #include <crypto/internal/acompress.h> 14 15 #include "idxd.h" 16 #include "iaa_crypto.h" 17 #include "iaa_crypto_stats.h" 18 19 #ifdef pr_fmt 20 #undef pr_fmt 21 #endif 22 23 #define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt 24 25 #define IAA_ALG_PRIORITY 300 26 27 /* number of iaa instances probed */ 28 static unsigned int nr_iaa; 29 static unsigned int nr_cpus; 30 static unsigned int nr_nodes; 31 static unsigned int nr_cpus_per_node; 32 33 /* Number of physical cpus sharing each iaa instance */ 34 static unsigned int cpus_per_iaa; 35 36 static struct crypto_comp *deflate_generic_tfm; 37 38 /* Per-cpu lookup table for balanced wqs */ 39 static struct wq_table_entry __percpu *wq_table; 40 41 static struct idxd_wq *wq_table_next_wq(int cpu) 42 { 43 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); 44 45 if (++entry->cur_wq >= entry->n_wqs) 46 entry->cur_wq = 0; 47 48 if (!entry->wqs[entry->cur_wq]) 49 return NULL; 50 51 pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__, 52 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id, 53 entry->wqs[entry->cur_wq]->id, cpu); 54 55 return entry->wqs[entry->cur_wq]; 56 } 57 58 static void wq_table_add(int cpu, struct idxd_wq *wq) 59 { 60 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); 61 62 if (WARN_ON(entry->n_wqs == entry->max_wqs)) 63 return; 64 65 entry->wqs[entry->n_wqs++] = wq; 66 67 pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__, 68 entry->wqs[entry->n_wqs - 1]->idxd->id, 69 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu); 70 } 71 72 static void wq_table_free_entry(int cpu) 73 { 74 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); 75 76 kfree(entry->wqs); 77 memset(entry, 0, sizeof(*entry)); 78 } 79 80 static void wq_table_clear_entry(int cpu) 81 { 82 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); 83 84 entry->n_wqs = 0; 85 entry->cur_wq = 0; 86 memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *)); 87 } 88 89 LIST_HEAD(iaa_devices); 90 DEFINE_MUTEX(iaa_devices_lock); 91 92 /* If enabled, IAA hw crypto algos are registered, unavailable otherwise */ 93 static bool iaa_crypto_enabled; 94 static bool iaa_crypto_registered; 95 96 /* Verify results of IAA compress or not */ 97 static bool iaa_verify_compress = true; 98 99 static ssize_t verify_compress_show(struct device_driver *driver, char *buf) 100 { 101 return sprintf(buf, "%d\n", iaa_verify_compress); 102 } 103 104 static ssize_t verify_compress_store(struct device_driver *driver, 105 const char *buf, size_t count) 106 { 107 int ret = -EBUSY; 108 109 mutex_lock(&iaa_devices_lock); 110 111 if (iaa_crypto_enabled) 112 goto out; 113 114 ret = kstrtobool(buf, &iaa_verify_compress); 115 if (ret) 116 goto out; 117 118 ret = count; 119 out: 120 mutex_unlock(&iaa_devices_lock); 121 122 return ret; 123 } 124 static DRIVER_ATTR_RW(verify_compress); 125 126 /* 127 * The iaa crypto driver supports three 'sync' methods determining how 128 * compressions and decompressions are performed: 129 * 130 * - sync: the compression or decompression completes before 131 * returning. This is the mode used by the async crypto 132 * interface when the sync mode is set to 'sync' and by 133 * the sync crypto interface regardless of setting. 134 * 135 * - async: the compression or decompression is submitted and returns 136 * immediately. Completion interrupts are not used so 137 * the caller is responsible for polling the descriptor 138 * for completion. This mode is applicable to only the 139 * async crypto interface and is ignored for anything 140 * else. 141 * 142 * - async_irq: the compression or decompression is submitted and 143 * returns immediately. Completion interrupts are 144 * enabled so the caller can wait for the completion and 145 * yield to other threads. When the compression or 146 * decompression completes, the completion is signaled 147 * and the caller awakened. This mode is applicable to 148 * only the async crypto interface and is ignored for 149 * anything else. 150 * 151 * These modes can be set using the iaa_crypto sync_mode driver 152 * attribute. 153 */ 154 155 /* Use async mode */ 156 static bool async_mode; 157 /* Use interrupts */ 158 static bool use_irq; 159 160 /** 161 * set_iaa_sync_mode - Set IAA sync mode 162 * @name: The name of the sync mode 163 * 164 * Make the IAA sync mode named @name the current sync mode used by 165 * compression/decompression. 166 */ 167 168 static int set_iaa_sync_mode(const char *name) 169 { 170 int ret = 0; 171 172 if (sysfs_streq(name, "sync")) { 173 async_mode = false; 174 use_irq = false; 175 } else if (sysfs_streq(name, "async")) { 176 async_mode = true; 177 use_irq = false; 178 } else if (sysfs_streq(name, "async_irq")) { 179 async_mode = true; 180 use_irq = true; 181 } else { 182 ret = -EINVAL; 183 } 184 185 return ret; 186 } 187 188 static ssize_t sync_mode_show(struct device_driver *driver, char *buf) 189 { 190 int ret = 0; 191 192 if (!async_mode && !use_irq) 193 ret = sprintf(buf, "%s\n", "sync"); 194 else if (async_mode && !use_irq) 195 ret = sprintf(buf, "%s\n", "async"); 196 else if (async_mode && use_irq) 197 ret = sprintf(buf, "%s\n", "async_irq"); 198 199 return ret; 200 } 201 202 static ssize_t sync_mode_store(struct device_driver *driver, 203 const char *buf, size_t count) 204 { 205 int ret = -EBUSY; 206 207 mutex_lock(&iaa_devices_lock); 208 209 if (iaa_crypto_enabled) 210 goto out; 211 212 ret = set_iaa_sync_mode(buf); 213 if (ret == 0) 214 ret = count; 215 out: 216 mutex_unlock(&iaa_devices_lock); 217 218 return ret; 219 } 220 static DRIVER_ATTR_RW(sync_mode); 221 222 static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; 223 224 static int find_empty_iaa_compression_mode(void) 225 { 226 int i = -EINVAL; 227 228 for (i = 0; i < IAA_COMP_MODES_MAX; i++) { 229 if (iaa_compression_modes[i]) 230 continue; 231 break; 232 } 233 234 return i; 235 } 236 237 static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) 238 { 239 struct iaa_compression_mode *mode; 240 int i; 241 242 for (i = 0; i < IAA_COMP_MODES_MAX; i++) { 243 mode = iaa_compression_modes[i]; 244 if (!mode) 245 continue; 246 247 if (!strcmp(mode->name, name)) { 248 *idx = i; 249 return iaa_compression_modes[i]; 250 } 251 } 252 253 return NULL; 254 } 255 256 static void free_iaa_compression_mode(struct iaa_compression_mode *mode) 257 { 258 kfree(mode->name); 259 kfree(mode->ll_table); 260 kfree(mode->d_table); 261 262 kfree(mode); 263 } 264 265 /* 266 * IAA Compression modes are defined by an ll_table and a d_table. 267 * These tables are typically generated and captured using statistics 268 * collected from running actual compress/decompress workloads. 269 * 270 * A module or other kernel code can add and remove compression modes 271 * with a given name using the exported @add_iaa_compression_mode() 272 * and @remove_iaa_compression_mode functions. 273 * 274 * When a new compression mode is added, the tables are saved in a 275 * global compression mode list. When IAA devices are added, a 276 * per-IAA device dma mapping is created for each IAA device, for each 277 * compression mode. These are the tables used to do the actual 278 * compression/deccompression and are unmapped if/when the devices are 279 * removed. Currently, compression modes must be added before any 280 * device is added, and removed after all devices have been removed. 281 */ 282 283 /** 284 * remove_iaa_compression_mode - Remove an IAA compression mode 285 * @name: The name the compression mode will be known as 286 * 287 * Remove the IAA compression mode named @name. 288 */ 289 void remove_iaa_compression_mode(const char *name) 290 { 291 struct iaa_compression_mode *mode; 292 int idx; 293 294 mutex_lock(&iaa_devices_lock); 295 296 if (!list_empty(&iaa_devices)) 297 goto out; 298 299 mode = find_iaa_compression_mode(name, &idx); 300 if (mode) { 301 free_iaa_compression_mode(mode); 302 iaa_compression_modes[idx] = NULL; 303 } 304 out: 305 mutex_unlock(&iaa_devices_lock); 306 } 307 EXPORT_SYMBOL_GPL(remove_iaa_compression_mode); 308 309 /** 310 * add_iaa_compression_mode - Add an IAA compression mode 311 * @name: The name the compression mode will be known as 312 * @ll_table: The ll table 313 * @ll_table_size: The ll table size in bytes 314 * @d_table: The d table 315 * @d_table_size: The d table size in bytes 316 * @init: Optional callback function to init the compression mode data 317 * @free: Optional callback function to free the compression mode data 318 * 319 * Add a new IAA compression mode named @name. 320 * 321 * Returns 0 if successful, errcode otherwise. 322 */ 323 int add_iaa_compression_mode(const char *name, 324 const u32 *ll_table, 325 int ll_table_size, 326 const u32 *d_table, 327 int d_table_size, 328 iaa_dev_comp_init_fn_t init, 329 iaa_dev_comp_free_fn_t free) 330 { 331 struct iaa_compression_mode *mode; 332 int idx, ret = -ENOMEM; 333 334 mutex_lock(&iaa_devices_lock); 335 336 if (!list_empty(&iaa_devices)) { 337 ret = -EBUSY; 338 goto out; 339 } 340 341 mode = kzalloc(sizeof(*mode), GFP_KERNEL); 342 if (!mode) 343 goto out; 344 345 mode->name = kstrdup(name, GFP_KERNEL); 346 if (!mode->name) 347 goto free; 348 349 if (ll_table) { 350 mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL); 351 if (!mode->ll_table) 352 goto free; 353 mode->ll_table_size = ll_table_size; 354 } 355 356 if (d_table) { 357 mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL); 358 if (!mode->d_table) 359 goto free; 360 mode->d_table_size = d_table_size; 361 } 362 363 mode->init = init; 364 mode->free = free; 365 366 idx = find_empty_iaa_compression_mode(); 367 if (idx < 0) 368 goto free; 369 370 pr_debug("IAA compression mode %s added at idx %d\n", 371 mode->name, idx); 372 373 iaa_compression_modes[idx] = mode; 374 375 ret = 0; 376 out: 377 mutex_unlock(&iaa_devices_lock); 378 379 return ret; 380 free: 381 free_iaa_compression_mode(mode); 382 goto out; 383 } 384 EXPORT_SYMBOL_GPL(add_iaa_compression_mode); 385 386 static struct iaa_device_compression_mode * 387 get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx) 388 { 389 return iaa_device->compression_modes[idx]; 390 } 391 392 static void free_device_compression_mode(struct iaa_device *iaa_device, 393 struct iaa_device_compression_mode *device_mode) 394 { 395 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; 396 struct device *dev = &iaa_device->idxd->pdev->dev; 397 398 kfree(device_mode->name); 399 400 if (device_mode->aecs_comp_table) 401 dma_free_coherent(dev, size, device_mode->aecs_comp_table, 402 device_mode->aecs_comp_table_dma_addr); 403 kfree(device_mode); 404 } 405 406 #define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000 407 #define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC) 408 #define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) 409 #define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) 410 #define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \ 411 IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \ 412 IDXD_OP_FLAG_AECS_RW_TGLS) 413 414 static int check_completion(struct device *dev, 415 struct iax_completion_record *comp, 416 bool compress, 417 bool only_once); 418 419 static int init_device_compression_mode(struct iaa_device *iaa_device, 420 struct iaa_compression_mode *mode, 421 int idx, struct idxd_wq *wq) 422 { 423 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; 424 struct device *dev = &iaa_device->idxd->pdev->dev; 425 struct iaa_device_compression_mode *device_mode; 426 int ret = -ENOMEM; 427 428 device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL); 429 if (!device_mode) 430 return -ENOMEM; 431 432 device_mode->name = kstrdup(mode->name, GFP_KERNEL); 433 if (!device_mode->name) 434 goto free; 435 436 device_mode->aecs_comp_table = dma_alloc_coherent(dev, size, 437 &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL); 438 if (!device_mode->aecs_comp_table) 439 goto free; 440 441 /* Add Huffman table to aecs */ 442 memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table)); 443 memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size); 444 memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size); 445 446 if (mode->init) { 447 ret = mode->init(device_mode); 448 if (ret) 449 goto free; 450 } 451 452 /* mode index should match iaa_compression_modes idx */ 453 iaa_device->compression_modes[idx] = device_mode; 454 455 pr_debug("IAA %s compression mode initialized for iaa device %d\n", 456 mode->name, iaa_device->idxd->id); 457 458 ret = 0; 459 out: 460 return ret; 461 free: 462 pr_debug("IAA %s compression mode initialization failed for iaa device %d\n", 463 mode->name, iaa_device->idxd->id); 464 465 free_device_compression_mode(iaa_device, device_mode); 466 goto out; 467 } 468 469 static int init_device_compression_modes(struct iaa_device *iaa_device, 470 struct idxd_wq *wq) 471 { 472 struct iaa_compression_mode *mode; 473 int i, ret = 0; 474 475 for (i = 0; i < IAA_COMP_MODES_MAX; i++) { 476 mode = iaa_compression_modes[i]; 477 if (!mode) 478 continue; 479 480 ret = init_device_compression_mode(iaa_device, mode, i, wq); 481 if (ret) 482 break; 483 } 484 485 return ret; 486 } 487 488 static void remove_device_compression_modes(struct iaa_device *iaa_device) 489 { 490 struct iaa_device_compression_mode *device_mode; 491 int i; 492 493 for (i = 0; i < IAA_COMP_MODES_MAX; i++) { 494 device_mode = iaa_device->compression_modes[i]; 495 if (!device_mode) 496 continue; 497 498 if (iaa_compression_modes[i]->free) 499 iaa_compression_modes[i]->free(device_mode); 500 free_device_compression_mode(iaa_device, device_mode); 501 iaa_device->compression_modes[i] = NULL; 502 } 503 } 504 505 static struct iaa_device *iaa_device_alloc(void) 506 { 507 struct iaa_device *iaa_device; 508 509 iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL); 510 if (!iaa_device) 511 return NULL; 512 513 INIT_LIST_HEAD(&iaa_device->wqs); 514 515 return iaa_device; 516 } 517 518 static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) 519 { 520 struct iaa_wq *iaa_wq; 521 522 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { 523 if (iaa_wq->wq == wq) 524 return true; 525 } 526 527 return false; 528 } 529 530 static struct iaa_device *add_iaa_device(struct idxd_device *idxd) 531 { 532 struct iaa_device *iaa_device; 533 534 iaa_device = iaa_device_alloc(); 535 if (!iaa_device) 536 return NULL; 537 538 iaa_device->idxd = idxd; 539 540 list_add_tail(&iaa_device->list, &iaa_devices); 541 542 nr_iaa++; 543 544 return iaa_device; 545 } 546 547 static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq) 548 { 549 int ret = 0; 550 551 ret = init_device_compression_modes(iaa_device, iaa_wq->wq); 552 if (ret) 553 return ret; 554 555 return ret; 556 } 557 558 static void del_iaa_device(struct iaa_device *iaa_device) 559 { 560 list_del(&iaa_device->list); 561 562 nr_iaa--; 563 } 564 565 static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq, 566 struct iaa_wq **new_wq) 567 { 568 struct idxd_device *idxd = iaa_device->idxd; 569 struct pci_dev *pdev = idxd->pdev; 570 struct device *dev = &pdev->dev; 571 struct iaa_wq *iaa_wq; 572 573 iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL); 574 if (!iaa_wq) 575 return -ENOMEM; 576 577 iaa_wq->wq = wq; 578 iaa_wq->iaa_device = iaa_device; 579 idxd_wq_set_private(wq, iaa_wq); 580 581 list_add_tail(&iaa_wq->list, &iaa_device->wqs); 582 583 iaa_device->n_wq++; 584 585 if (new_wq) 586 *new_wq = iaa_wq; 587 588 dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n", 589 wq->id, iaa_device->idxd->id, iaa_device->n_wq); 590 591 return 0; 592 } 593 594 static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) 595 { 596 struct idxd_device *idxd = iaa_device->idxd; 597 struct pci_dev *pdev = idxd->pdev; 598 struct device *dev = &pdev->dev; 599 struct iaa_wq *iaa_wq; 600 601 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { 602 if (iaa_wq->wq == wq) { 603 list_del(&iaa_wq->list); 604 iaa_device->n_wq--; 605 606 dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n", 607 wq->id, iaa_device->idxd->id, 608 iaa_device->n_wq, nr_iaa); 609 610 if (iaa_device->n_wq == 0) 611 del_iaa_device(iaa_device); 612 break; 613 } 614 } 615 } 616 617 static void clear_wq_table(void) 618 { 619 int cpu; 620 621 for (cpu = 0; cpu < nr_cpus; cpu++) 622 wq_table_clear_entry(cpu); 623 624 pr_debug("cleared wq table\n"); 625 } 626 627 static void free_iaa_device(struct iaa_device *iaa_device) 628 { 629 if (!iaa_device) 630 return; 631 632 remove_device_compression_modes(iaa_device); 633 kfree(iaa_device); 634 } 635 636 static void __free_iaa_wq(struct iaa_wq *iaa_wq) 637 { 638 struct iaa_device *iaa_device; 639 640 if (!iaa_wq) 641 return; 642 643 iaa_device = iaa_wq->iaa_device; 644 if (iaa_device->n_wq == 0) 645 free_iaa_device(iaa_wq->iaa_device); 646 } 647 648 static void free_iaa_wq(struct iaa_wq *iaa_wq) 649 { 650 struct idxd_wq *wq; 651 652 __free_iaa_wq(iaa_wq); 653 654 wq = iaa_wq->wq; 655 656 kfree(iaa_wq); 657 idxd_wq_set_private(wq, NULL); 658 } 659 660 static int iaa_wq_get(struct idxd_wq *wq) 661 { 662 struct idxd_device *idxd = wq->idxd; 663 struct iaa_wq *iaa_wq; 664 int ret = 0; 665 666 spin_lock(&idxd->dev_lock); 667 iaa_wq = idxd_wq_get_private(wq); 668 if (iaa_wq && !iaa_wq->remove) { 669 iaa_wq->ref++; 670 idxd_wq_get(wq); 671 } else { 672 ret = -ENODEV; 673 } 674 spin_unlock(&idxd->dev_lock); 675 676 return ret; 677 } 678 679 static int iaa_wq_put(struct idxd_wq *wq) 680 { 681 struct idxd_device *idxd = wq->idxd; 682 struct iaa_wq *iaa_wq; 683 bool free = false; 684 int ret = 0; 685 686 spin_lock(&idxd->dev_lock); 687 iaa_wq = idxd_wq_get_private(wq); 688 if (iaa_wq) { 689 iaa_wq->ref--; 690 if (iaa_wq->ref == 0 && iaa_wq->remove) { 691 idxd_wq_set_private(wq, NULL); 692 free = true; 693 } 694 idxd_wq_put(wq); 695 } else { 696 ret = -ENODEV; 697 } 698 spin_unlock(&idxd->dev_lock); 699 if (free) { 700 __free_iaa_wq(iaa_wq); 701 kfree(iaa_wq); 702 } 703 704 return ret; 705 } 706 707 static void free_wq_table(void) 708 { 709 int cpu; 710 711 for (cpu = 0; cpu < nr_cpus; cpu++) 712 wq_table_free_entry(cpu); 713 714 free_percpu(wq_table); 715 716 pr_debug("freed wq table\n"); 717 } 718 719 static int alloc_wq_table(int max_wqs) 720 { 721 struct wq_table_entry *entry; 722 int cpu; 723 724 wq_table = alloc_percpu(struct wq_table_entry); 725 if (!wq_table) 726 return -ENOMEM; 727 728 for (cpu = 0; cpu < nr_cpus; cpu++) { 729 entry = per_cpu_ptr(wq_table, cpu); 730 entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL); 731 if (!entry->wqs) { 732 free_wq_table(); 733 return -ENOMEM; 734 } 735 736 entry->max_wqs = max_wqs; 737 } 738 739 pr_debug("initialized wq table\n"); 740 741 return 0; 742 } 743 744 static int save_iaa_wq(struct idxd_wq *wq) 745 { 746 struct iaa_device *iaa_device, *found = NULL; 747 struct idxd_device *idxd; 748 struct pci_dev *pdev; 749 struct device *dev; 750 int ret = 0; 751 752 list_for_each_entry(iaa_device, &iaa_devices, list) { 753 if (iaa_device->idxd == wq->idxd) { 754 idxd = iaa_device->idxd; 755 pdev = idxd->pdev; 756 dev = &pdev->dev; 757 /* 758 * Check to see that we don't already have this wq. 759 * Shouldn't happen but we don't control probing. 760 */ 761 if (iaa_has_wq(iaa_device, wq)) { 762 dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n", 763 iaa_device); 764 goto out; 765 } 766 767 found = iaa_device; 768 769 ret = add_iaa_wq(iaa_device, wq, NULL); 770 if (ret) 771 goto out; 772 773 break; 774 } 775 } 776 777 if (!found) { 778 struct iaa_device *new_device; 779 struct iaa_wq *new_wq; 780 781 new_device = add_iaa_device(wq->idxd); 782 if (!new_device) { 783 ret = -ENOMEM; 784 goto out; 785 } 786 787 ret = add_iaa_wq(new_device, wq, &new_wq); 788 if (ret) { 789 del_iaa_device(new_device); 790 free_iaa_device(new_device); 791 goto out; 792 } 793 794 ret = init_iaa_device(new_device, new_wq); 795 if (ret) { 796 del_iaa_wq(new_device, new_wq->wq); 797 del_iaa_device(new_device); 798 free_iaa_wq(new_wq); 799 goto out; 800 } 801 } 802 803 if (WARN_ON(nr_iaa == 0)) 804 return -EINVAL; 805 806 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; 807 if (!cpus_per_iaa) 808 cpus_per_iaa = 1; 809 out: 810 return 0; 811 } 812 813 static void remove_iaa_wq(struct idxd_wq *wq) 814 { 815 struct iaa_device *iaa_device; 816 817 list_for_each_entry(iaa_device, &iaa_devices, list) { 818 if (iaa_has_wq(iaa_device, wq)) { 819 del_iaa_wq(iaa_device, wq); 820 break; 821 } 822 } 823 824 if (nr_iaa) { 825 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; 826 if (!cpus_per_iaa) 827 cpus_per_iaa = 1; 828 } else 829 cpus_per_iaa = 1; 830 } 831 832 static int wq_table_add_wqs(int iaa, int cpu) 833 { 834 struct iaa_device *iaa_device, *found_device = NULL; 835 int ret = 0, cur_iaa = 0, n_wqs_added = 0; 836 struct idxd_device *idxd; 837 struct iaa_wq *iaa_wq; 838 struct pci_dev *pdev; 839 struct device *dev; 840 841 list_for_each_entry(iaa_device, &iaa_devices, list) { 842 idxd = iaa_device->idxd; 843 pdev = idxd->pdev; 844 dev = &pdev->dev; 845 846 if (cur_iaa != iaa) { 847 cur_iaa++; 848 continue; 849 } 850 851 found_device = iaa_device; 852 dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n", 853 found_device->idxd->id, cur_iaa); 854 break; 855 } 856 857 if (!found_device) { 858 found_device = list_first_entry_or_null(&iaa_devices, 859 struct iaa_device, list); 860 if (!found_device) { 861 pr_debug("couldn't find any iaa devices with wqs!\n"); 862 ret = -EINVAL; 863 goto out; 864 } 865 cur_iaa = 0; 866 867 idxd = found_device->idxd; 868 pdev = idxd->pdev; 869 dev = &pdev->dev; 870 dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n", 871 found_device->idxd->id, cur_iaa); 872 } 873 874 list_for_each_entry(iaa_wq, &found_device->wqs, list) { 875 wq_table_add(cpu, iaa_wq->wq); 876 pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n", 877 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id); 878 n_wqs_added++; 879 } 880 881 if (!n_wqs_added) { 882 pr_debug("couldn't find any iaa wqs!\n"); 883 ret = -EINVAL; 884 goto out; 885 } 886 out: 887 return ret; 888 } 889 890 /* 891 * Rebalance the wq table so that given a cpu, it's easy to find the 892 * closest IAA instance. The idea is to try to choose the most 893 * appropriate IAA instance for a caller and spread available 894 * workqueues around to clients. 895 */ 896 static void rebalance_wq_table(void) 897 { 898 const struct cpumask *node_cpus; 899 int node, cpu, iaa = -1; 900 901 if (nr_iaa == 0) 902 return; 903 904 pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n", 905 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa); 906 907 clear_wq_table(); 908 909 if (nr_iaa == 1) { 910 for (cpu = 0; cpu < nr_cpus; cpu++) { 911 if (WARN_ON(wq_table_add_wqs(0, cpu))) { 912 pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu); 913 return; 914 } 915 } 916 917 return; 918 } 919 920 for_each_node_with_cpus(node) { 921 node_cpus = cpumask_of_node(node); 922 923 for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) { 924 int node_cpu = cpumask_nth(cpu, node_cpus); 925 926 if (WARN_ON(node_cpu >= nr_cpu_ids)) { 927 pr_debug("node_cpu %d doesn't exist!\n", node_cpu); 928 return; 929 } 930 931 if ((cpu % cpus_per_iaa) == 0) 932 iaa++; 933 934 if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) { 935 pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); 936 return; 937 } 938 } 939 } 940 } 941 942 static inline int check_completion(struct device *dev, 943 struct iax_completion_record *comp, 944 bool compress, 945 bool only_once) 946 { 947 char *op_str = compress ? "compress" : "decompress"; 948 int ret = 0; 949 950 while (!comp->status) { 951 if (only_once) 952 return -EAGAIN; 953 cpu_relax(); 954 } 955 956 if (comp->status != IAX_COMP_SUCCESS) { 957 if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) { 958 ret = -ETIMEDOUT; 959 dev_dbg(dev, "%s timed out, size=0x%x\n", 960 op_str, comp->output_size); 961 update_completion_timeout_errs(); 962 goto out; 963 } 964 965 if (comp->status == IAA_ANALYTICS_ERROR && 966 comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) { 967 ret = -E2BIG; 968 dev_dbg(dev, "compressed > uncompressed size," 969 " not compressing, size=0x%x\n", 970 comp->output_size); 971 update_completion_comp_buf_overflow_errs(); 972 goto out; 973 } 974 975 if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) { 976 ret = -EOVERFLOW; 977 goto out; 978 } 979 980 ret = -EINVAL; 981 dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n", 982 op_str, comp->status, comp->error_code, comp->output_size); 983 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0); 984 update_completion_einval_errs(); 985 986 goto out; 987 } 988 out: 989 return ret; 990 } 991 992 static int deflate_generic_decompress(struct acomp_req *req) 993 { 994 void *src, *dst; 995 int ret; 996 997 src = kmap_local_page(sg_page(req->src)) + req->src->offset; 998 dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset; 999 1000 ret = crypto_comp_decompress(deflate_generic_tfm, 1001 src, req->slen, dst, &req->dlen); 1002 1003 kunmap_local(src); 1004 kunmap_local(dst); 1005 1006 update_total_sw_decomp_calls(); 1007 1008 return ret; 1009 } 1010 1011 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, 1012 struct acomp_req *req, 1013 dma_addr_t *src_addr, dma_addr_t *dst_addr); 1014 1015 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, 1016 struct idxd_wq *wq, 1017 dma_addr_t src_addr, unsigned int slen, 1018 dma_addr_t dst_addr, unsigned int *dlen, 1019 u32 compression_crc); 1020 1021 static void iaa_desc_complete(struct idxd_desc *idxd_desc, 1022 enum idxd_complete_type comp_type, 1023 bool free_desc, void *__ctx, 1024 u32 *status) 1025 { 1026 struct iaa_device_compression_mode *active_compression_mode; 1027 struct iaa_compression_ctx *compression_ctx; 1028 struct crypto_ctx *ctx = __ctx; 1029 struct iaa_device *iaa_device; 1030 struct idxd_device *idxd; 1031 struct iaa_wq *iaa_wq; 1032 struct pci_dev *pdev; 1033 struct device *dev; 1034 int ret, err = 0; 1035 1036 compression_ctx = crypto_tfm_ctx(ctx->tfm); 1037 1038 iaa_wq = idxd_wq_get_private(idxd_desc->wq); 1039 iaa_device = iaa_wq->iaa_device; 1040 idxd = iaa_device->idxd; 1041 pdev = idxd->pdev; 1042 dev = &pdev->dev; 1043 1044 active_compression_mode = get_iaa_device_compression_mode(iaa_device, 1045 compression_ctx->mode); 1046 dev_dbg(dev, "%s: compression mode %s," 1047 " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__, 1048 active_compression_mode->name, 1049 ctx->src_addr, ctx->dst_addr); 1050 1051 ret = check_completion(dev, idxd_desc->iax_completion, 1052 ctx->compress, false); 1053 if (ret) { 1054 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); 1055 if (!ctx->compress && 1056 idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { 1057 pr_warn("%s: falling back to deflate-generic decompress, " 1058 "analytics error code %x\n", __func__, 1059 idxd_desc->iax_completion->error_code); 1060 ret = deflate_generic_decompress(ctx->req); 1061 if (ret) { 1062 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", 1063 __func__, ret); 1064 err = -EIO; 1065 goto err; 1066 } 1067 } else { 1068 err = -EIO; 1069 goto err; 1070 } 1071 } else { 1072 ctx->req->dlen = idxd_desc->iax_completion->output_size; 1073 } 1074 1075 /* Update stats */ 1076 if (ctx->compress) { 1077 update_total_comp_bytes_out(ctx->req->dlen); 1078 update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen); 1079 } else { 1080 update_total_decomp_bytes_in(ctx->req->slen); 1081 update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen); 1082 } 1083 1084 if (ctx->compress && compression_ctx->verify_compress) { 1085 dma_addr_t src_addr, dst_addr; 1086 u32 compression_crc; 1087 1088 compression_crc = idxd_desc->iax_completion->crc; 1089 1090 ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); 1091 if (ret) { 1092 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); 1093 err = -EIO; 1094 goto out; 1095 } 1096 1097 ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, 1098 ctx->req->slen, dst_addr, &ctx->req->dlen, 1099 compression_crc); 1100 if (ret) { 1101 dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); 1102 err = -EIO; 1103 } 1104 1105 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE); 1106 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE); 1107 1108 goto out; 1109 } 1110 err: 1111 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE); 1112 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE); 1113 out: 1114 if (ret != 0) 1115 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); 1116 1117 if (ctx->req->base.complete) 1118 acomp_request_complete(ctx->req, err); 1119 1120 if (free_desc) 1121 idxd_free_desc(idxd_desc->wq, idxd_desc); 1122 iaa_wq_put(idxd_desc->wq); 1123 } 1124 1125 static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, 1126 struct idxd_wq *wq, 1127 dma_addr_t src_addr, unsigned int slen, 1128 dma_addr_t dst_addr, unsigned int *dlen, 1129 u32 *compression_crc, 1130 bool disable_async) 1131 { 1132 struct iaa_device_compression_mode *active_compression_mode; 1133 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); 1134 struct iaa_device *iaa_device; 1135 struct idxd_desc *idxd_desc; 1136 struct iax_hw_desc *desc; 1137 struct idxd_device *idxd; 1138 struct iaa_wq *iaa_wq; 1139 struct pci_dev *pdev; 1140 struct device *dev; 1141 int ret = 0; 1142 1143 iaa_wq = idxd_wq_get_private(wq); 1144 iaa_device = iaa_wq->iaa_device; 1145 idxd = iaa_device->idxd; 1146 pdev = idxd->pdev; 1147 dev = &pdev->dev; 1148 1149 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); 1150 1151 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); 1152 if (IS_ERR(idxd_desc)) { 1153 dev_dbg(dev, "idxd descriptor allocation failed\n"); 1154 dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc)); 1155 return PTR_ERR(idxd_desc); 1156 } 1157 desc = idxd_desc->iax_hw; 1158 1159 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | 1160 IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC; 1161 desc->opcode = IAX_OPCODE_COMPRESS; 1162 desc->compr_flags = IAA_COMP_FLAGS; 1163 desc->priv = 0; 1164 1165 desc->src1_addr = (u64)src_addr; 1166 desc->src1_size = slen; 1167 desc->dst_addr = (u64)dst_addr; 1168 desc->max_dst_size = *dlen; 1169 desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr; 1170 desc->src2_size = sizeof(struct aecs_comp_table_record); 1171 desc->completion_addr = idxd_desc->compl_dma; 1172 1173 if (ctx->use_irq && !disable_async) { 1174 desc->flags |= IDXD_OP_FLAG_RCI; 1175 1176 idxd_desc->crypto.req = req; 1177 idxd_desc->crypto.tfm = tfm; 1178 idxd_desc->crypto.src_addr = src_addr; 1179 idxd_desc->crypto.dst_addr = dst_addr; 1180 idxd_desc->crypto.compress = true; 1181 1182 dev_dbg(dev, "%s use_async_irq: compression mode %s," 1183 " src_addr %llx, dst_addr %llx\n", __func__, 1184 active_compression_mode->name, 1185 src_addr, dst_addr); 1186 } else if (ctx->async_mode && !disable_async) 1187 req->base.data = idxd_desc; 1188 1189 dev_dbg(dev, "%s: compression mode %s," 1190 " desc->src1_addr %llx, desc->src1_size %d," 1191 " desc->dst_addr %llx, desc->max_dst_size %d," 1192 " desc->src2_addr %llx, desc->src2_size %d\n", __func__, 1193 active_compression_mode->name, 1194 desc->src1_addr, desc->src1_size, desc->dst_addr, 1195 desc->max_dst_size, desc->src2_addr, desc->src2_size); 1196 1197 ret = idxd_submit_desc(wq, idxd_desc); 1198 if (ret) { 1199 dev_dbg(dev, "submit_desc failed ret=%d\n", ret); 1200 goto err; 1201 } 1202 1203 /* Update stats */ 1204 update_total_comp_calls(); 1205 update_wq_comp_calls(wq); 1206 1207 if (ctx->async_mode && !disable_async) { 1208 ret = -EINPROGRESS; 1209 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); 1210 goto out; 1211 } 1212 1213 ret = check_completion(dev, idxd_desc->iax_completion, true, false); 1214 if (ret) { 1215 dev_dbg(dev, "check_completion failed ret=%d\n", ret); 1216 goto err; 1217 } 1218 1219 *dlen = idxd_desc->iax_completion->output_size; 1220 1221 /* Update stats */ 1222 update_total_comp_bytes_out(*dlen); 1223 update_wq_comp_bytes(wq, *dlen); 1224 1225 *compression_crc = idxd_desc->iax_completion->crc; 1226 1227 if (!ctx->async_mode || disable_async) 1228 idxd_free_desc(wq, idxd_desc); 1229 out: 1230 return ret; 1231 err: 1232 idxd_free_desc(wq, idxd_desc); 1233 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); 1234 1235 goto out; 1236 } 1237 1238 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, 1239 struct acomp_req *req, 1240 dma_addr_t *src_addr, dma_addr_t *dst_addr) 1241 { 1242 int ret = 0; 1243 int nr_sgs; 1244 1245 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1246 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1247 1248 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); 1249 if (nr_sgs <= 0 || nr_sgs > 1) { 1250 dev_dbg(dev, "verify: couldn't map src sg for iaa device %d," 1251 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1252 iaa_wq->wq->id, ret); 1253 ret = -EIO; 1254 goto out; 1255 } 1256 *src_addr = sg_dma_address(req->src); 1257 dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," 1258 " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs, 1259 req->src, req->slen, sg_dma_len(req->src)); 1260 1261 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); 1262 if (nr_sgs <= 0 || nr_sgs > 1) { 1263 dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d," 1264 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1265 iaa_wq->wq->id, ret); 1266 ret = -EIO; 1267 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); 1268 goto out; 1269 } 1270 *dst_addr = sg_dma_address(req->dst); 1271 dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," 1272 " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs, 1273 req->dst, req->dlen, sg_dma_len(req->dst)); 1274 out: 1275 return ret; 1276 } 1277 1278 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, 1279 struct idxd_wq *wq, 1280 dma_addr_t src_addr, unsigned int slen, 1281 dma_addr_t dst_addr, unsigned int *dlen, 1282 u32 compression_crc) 1283 { 1284 struct iaa_device_compression_mode *active_compression_mode; 1285 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); 1286 struct iaa_device *iaa_device; 1287 struct idxd_desc *idxd_desc; 1288 struct iax_hw_desc *desc; 1289 struct idxd_device *idxd; 1290 struct iaa_wq *iaa_wq; 1291 struct pci_dev *pdev; 1292 struct device *dev; 1293 int ret = 0; 1294 1295 iaa_wq = idxd_wq_get_private(wq); 1296 iaa_device = iaa_wq->iaa_device; 1297 idxd = iaa_device->idxd; 1298 pdev = idxd->pdev; 1299 dev = &pdev->dev; 1300 1301 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); 1302 1303 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); 1304 if (IS_ERR(idxd_desc)) { 1305 dev_dbg(dev, "idxd descriptor allocation failed\n"); 1306 dev_dbg(dev, "iaa compress failed: ret=%ld\n", 1307 PTR_ERR(idxd_desc)); 1308 return PTR_ERR(idxd_desc); 1309 } 1310 desc = idxd_desc->iax_hw; 1311 1312 /* Verify (optional) - decompress and check crc, suppress dest write */ 1313 1314 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; 1315 desc->opcode = IAX_OPCODE_DECOMPRESS; 1316 desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT; 1317 desc->priv = 0; 1318 1319 desc->src1_addr = (u64)dst_addr; 1320 desc->src1_size = *dlen; 1321 desc->dst_addr = (u64)src_addr; 1322 desc->max_dst_size = slen; 1323 desc->completion_addr = idxd_desc->compl_dma; 1324 1325 dev_dbg(dev, "(verify) compression mode %s," 1326 " desc->src1_addr %llx, desc->src1_size %d," 1327 " desc->dst_addr %llx, desc->max_dst_size %d," 1328 " desc->src2_addr %llx, desc->src2_size %d\n", 1329 active_compression_mode->name, 1330 desc->src1_addr, desc->src1_size, desc->dst_addr, 1331 desc->max_dst_size, desc->src2_addr, desc->src2_size); 1332 1333 ret = idxd_submit_desc(wq, idxd_desc); 1334 if (ret) { 1335 dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret); 1336 goto err; 1337 } 1338 1339 ret = check_completion(dev, idxd_desc->iax_completion, false, false); 1340 if (ret) { 1341 dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret); 1342 goto err; 1343 } 1344 1345 if (compression_crc != idxd_desc->iax_completion->crc) { 1346 ret = -EINVAL; 1347 dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" 1348 " comp=0x%x, decomp=0x%x\n", compression_crc, 1349 idxd_desc->iax_completion->crc); 1350 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 1351 8, 1, idxd_desc->iax_completion, 64, 0); 1352 goto err; 1353 } 1354 1355 idxd_free_desc(wq, idxd_desc); 1356 out: 1357 return ret; 1358 err: 1359 idxd_free_desc(wq, idxd_desc); 1360 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); 1361 1362 goto out; 1363 } 1364 1365 static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, 1366 struct idxd_wq *wq, 1367 dma_addr_t src_addr, unsigned int slen, 1368 dma_addr_t dst_addr, unsigned int *dlen, 1369 bool disable_async) 1370 { 1371 struct iaa_device_compression_mode *active_compression_mode; 1372 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); 1373 struct iaa_device *iaa_device; 1374 struct idxd_desc *idxd_desc; 1375 struct iax_hw_desc *desc; 1376 struct idxd_device *idxd; 1377 struct iaa_wq *iaa_wq; 1378 struct pci_dev *pdev; 1379 struct device *dev; 1380 int ret = 0; 1381 1382 iaa_wq = idxd_wq_get_private(wq); 1383 iaa_device = iaa_wq->iaa_device; 1384 idxd = iaa_device->idxd; 1385 pdev = idxd->pdev; 1386 dev = &pdev->dev; 1387 1388 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); 1389 1390 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); 1391 if (IS_ERR(idxd_desc)) { 1392 dev_dbg(dev, "idxd descriptor allocation failed\n"); 1393 dev_dbg(dev, "iaa decompress failed: ret=%ld\n", 1394 PTR_ERR(idxd_desc)); 1395 return PTR_ERR(idxd_desc); 1396 } 1397 desc = idxd_desc->iax_hw; 1398 1399 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; 1400 desc->opcode = IAX_OPCODE_DECOMPRESS; 1401 desc->max_dst_size = PAGE_SIZE; 1402 desc->decompr_flags = IAA_DECOMP_FLAGS; 1403 desc->priv = 0; 1404 1405 desc->src1_addr = (u64)src_addr; 1406 desc->dst_addr = (u64)dst_addr; 1407 desc->max_dst_size = *dlen; 1408 desc->src1_size = slen; 1409 desc->completion_addr = idxd_desc->compl_dma; 1410 1411 if (ctx->use_irq && !disable_async) { 1412 desc->flags |= IDXD_OP_FLAG_RCI; 1413 1414 idxd_desc->crypto.req = req; 1415 idxd_desc->crypto.tfm = tfm; 1416 idxd_desc->crypto.src_addr = src_addr; 1417 idxd_desc->crypto.dst_addr = dst_addr; 1418 idxd_desc->crypto.compress = false; 1419 1420 dev_dbg(dev, "%s: use_async_irq compression mode %s," 1421 " src_addr %llx, dst_addr %llx\n", __func__, 1422 active_compression_mode->name, 1423 src_addr, dst_addr); 1424 } else if (ctx->async_mode && !disable_async) 1425 req->base.data = idxd_desc; 1426 1427 dev_dbg(dev, "%s: decompression mode %s," 1428 " desc->src1_addr %llx, desc->src1_size %d," 1429 " desc->dst_addr %llx, desc->max_dst_size %d," 1430 " desc->src2_addr %llx, desc->src2_size %d\n", __func__, 1431 active_compression_mode->name, 1432 desc->src1_addr, desc->src1_size, desc->dst_addr, 1433 desc->max_dst_size, desc->src2_addr, desc->src2_size); 1434 1435 ret = idxd_submit_desc(wq, idxd_desc); 1436 if (ret) { 1437 dev_dbg(dev, "submit_desc failed ret=%d\n", ret); 1438 goto err; 1439 } 1440 1441 /* Update stats */ 1442 update_total_decomp_calls(); 1443 update_wq_decomp_calls(wq); 1444 1445 if (ctx->async_mode && !disable_async) { 1446 ret = -EINPROGRESS; 1447 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); 1448 goto out; 1449 } 1450 1451 ret = check_completion(dev, idxd_desc->iax_completion, false, false); 1452 if (ret) { 1453 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); 1454 if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { 1455 pr_warn("%s: falling back to deflate-generic decompress, " 1456 "analytics error code %x\n", __func__, 1457 idxd_desc->iax_completion->error_code); 1458 ret = deflate_generic_decompress(req); 1459 if (ret) { 1460 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", 1461 __func__, ret); 1462 goto err; 1463 } 1464 } else { 1465 goto err; 1466 } 1467 } else { 1468 req->dlen = idxd_desc->iax_completion->output_size; 1469 } 1470 1471 *dlen = req->dlen; 1472 1473 if (!ctx->async_mode || disable_async) 1474 idxd_free_desc(wq, idxd_desc); 1475 1476 /* Update stats */ 1477 update_total_decomp_bytes_in(slen); 1478 update_wq_decomp_bytes(wq, slen); 1479 out: 1480 return ret; 1481 err: 1482 idxd_free_desc(wq, idxd_desc); 1483 dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret); 1484 1485 goto out; 1486 } 1487 1488 static int iaa_comp_acompress(struct acomp_req *req) 1489 { 1490 struct iaa_compression_ctx *compression_ctx; 1491 struct crypto_tfm *tfm = req->base.tfm; 1492 dma_addr_t src_addr, dst_addr; 1493 bool disable_async = false; 1494 int nr_sgs, cpu, ret = 0; 1495 struct iaa_wq *iaa_wq; 1496 u32 compression_crc; 1497 struct idxd_wq *wq; 1498 struct device *dev; 1499 int order = -1; 1500 1501 compression_ctx = crypto_tfm_ctx(tfm); 1502 1503 if (!iaa_crypto_enabled) { 1504 pr_debug("iaa_crypto disabled, not compressing\n"); 1505 return -ENODEV; 1506 } 1507 1508 if (!req->src || !req->slen) { 1509 pr_debug("invalid src, not compressing\n"); 1510 return -EINVAL; 1511 } 1512 1513 cpu = get_cpu(); 1514 wq = wq_table_next_wq(cpu); 1515 put_cpu(); 1516 if (!wq) { 1517 pr_debug("no wq configured for cpu=%d\n", cpu); 1518 return -ENODEV; 1519 } 1520 1521 ret = iaa_wq_get(wq); 1522 if (ret) { 1523 pr_debug("no wq available for cpu=%d\n", cpu); 1524 return -ENODEV; 1525 } 1526 1527 iaa_wq = idxd_wq_get_private(wq); 1528 1529 if (!req->dst) { 1530 gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; 1531 1532 /* incompressible data will always be < 2 * slen */ 1533 req->dlen = 2 * req->slen; 1534 order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); 1535 req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); 1536 if (!req->dst) { 1537 ret = -ENOMEM; 1538 order = -1; 1539 goto out; 1540 } 1541 disable_async = true; 1542 } 1543 1544 dev = &wq->idxd->pdev->dev; 1545 1546 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1547 if (nr_sgs <= 0 || nr_sgs > 1) { 1548 dev_dbg(dev, "couldn't map src sg for iaa device %d," 1549 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1550 iaa_wq->wq->id, ret); 1551 ret = -EIO; 1552 goto out; 1553 } 1554 src_addr = sg_dma_address(req->src); 1555 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," 1556 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, 1557 req->src, req->slen, sg_dma_len(req->src)); 1558 1559 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1560 if (nr_sgs <= 0 || nr_sgs > 1) { 1561 dev_dbg(dev, "couldn't map dst sg for iaa device %d," 1562 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1563 iaa_wq->wq->id, ret); 1564 ret = -EIO; 1565 goto err_map_dst; 1566 } 1567 dst_addr = sg_dma_address(req->dst); 1568 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," 1569 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, 1570 req->dst, req->dlen, sg_dma_len(req->dst)); 1571 1572 ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, 1573 &req->dlen, &compression_crc, disable_async); 1574 if (ret == -EINPROGRESS) 1575 return ret; 1576 1577 if (!ret && compression_ctx->verify_compress) { 1578 ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr); 1579 if (ret) { 1580 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); 1581 goto out; 1582 } 1583 1584 ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, 1585 dst_addr, &req->dlen, compression_crc); 1586 if (ret) 1587 dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); 1588 1589 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); 1590 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); 1591 1592 goto out; 1593 } 1594 1595 if (ret) 1596 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); 1597 1598 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1599 err_map_dst: 1600 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1601 out: 1602 iaa_wq_put(wq); 1603 1604 if (order >= 0) 1605 sgl_free_order(req->dst, order); 1606 1607 return ret; 1608 } 1609 1610 static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) 1611 { 1612 gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 1613 GFP_KERNEL : GFP_ATOMIC; 1614 struct crypto_tfm *tfm = req->base.tfm; 1615 dma_addr_t src_addr, dst_addr; 1616 int nr_sgs, cpu, ret = 0; 1617 struct iaa_wq *iaa_wq; 1618 struct device *dev; 1619 struct idxd_wq *wq; 1620 int order = -1; 1621 1622 cpu = get_cpu(); 1623 wq = wq_table_next_wq(cpu); 1624 put_cpu(); 1625 if (!wq) { 1626 pr_debug("no wq configured for cpu=%d\n", cpu); 1627 return -ENODEV; 1628 } 1629 1630 ret = iaa_wq_get(wq); 1631 if (ret) { 1632 pr_debug("no wq available for cpu=%d\n", cpu); 1633 return -ENODEV; 1634 } 1635 1636 iaa_wq = idxd_wq_get_private(wq); 1637 1638 dev = &wq->idxd->pdev->dev; 1639 1640 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1641 if (nr_sgs <= 0 || nr_sgs > 1) { 1642 dev_dbg(dev, "couldn't map src sg for iaa device %d," 1643 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1644 iaa_wq->wq->id, ret); 1645 ret = -EIO; 1646 goto out; 1647 } 1648 src_addr = sg_dma_address(req->src); 1649 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," 1650 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, 1651 req->src, req->slen, sg_dma_len(req->src)); 1652 1653 req->dlen = 4 * req->slen; /* start with ~avg comp rato */ 1654 alloc_dest: 1655 order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); 1656 req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); 1657 if (!req->dst) { 1658 ret = -ENOMEM; 1659 order = -1; 1660 goto out; 1661 } 1662 1663 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1664 if (nr_sgs <= 0 || nr_sgs > 1) { 1665 dev_dbg(dev, "couldn't map dst sg for iaa device %d," 1666 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1667 iaa_wq->wq->id, ret); 1668 ret = -EIO; 1669 goto err_map_dst; 1670 } 1671 1672 dst_addr = sg_dma_address(req->dst); 1673 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," 1674 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, 1675 req->dst, req->dlen, sg_dma_len(req->dst)); 1676 ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, 1677 dst_addr, &req->dlen, true); 1678 if (ret == -EOVERFLOW) { 1679 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1680 req->dlen *= 2; 1681 if (req->dlen > CRYPTO_ACOMP_DST_MAX) 1682 goto err_map_dst; 1683 goto alloc_dest; 1684 } 1685 1686 if (ret != 0) 1687 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); 1688 1689 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1690 err_map_dst: 1691 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1692 out: 1693 iaa_wq_put(wq); 1694 1695 if (order >= 0) 1696 sgl_free_order(req->dst, order); 1697 1698 return ret; 1699 } 1700 1701 static int iaa_comp_adecompress(struct acomp_req *req) 1702 { 1703 struct crypto_tfm *tfm = req->base.tfm; 1704 dma_addr_t src_addr, dst_addr; 1705 int nr_sgs, cpu, ret = 0; 1706 struct iaa_wq *iaa_wq; 1707 struct device *dev; 1708 struct idxd_wq *wq; 1709 1710 if (!iaa_crypto_enabled) { 1711 pr_debug("iaa_crypto disabled, not decompressing\n"); 1712 return -ENODEV; 1713 } 1714 1715 if (!req->src || !req->slen) { 1716 pr_debug("invalid src, not decompressing\n"); 1717 return -EINVAL; 1718 } 1719 1720 if (!req->dst) 1721 return iaa_comp_adecompress_alloc_dest(req); 1722 1723 cpu = get_cpu(); 1724 wq = wq_table_next_wq(cpu); 1725 put_cpu(); 1726 if (!wq) { 1727 pr_debug("no wq configured for cpu=%d\n", cpu); 1728 return -ENODEV; 1729 } 1730 1731 ret = iaa_wq_get(wq); 1732 if (ret) { 1733 pr_debug("no wq available for cpu=%d\n", cpu); 1734 return -ENODEV; 1735 } 1736 1737 iaa_wq = idxd_wq_get_private(wq); 1738 1739 dev = &wq->idxd->pdev->dev; 1740 1741 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1742 if (nr_sgs <= 0 || nr_sgs > 1) { 1743 dev_dbg(dev, "couldn't map src sg for iaa device %d," 1744 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1745 iaa_wq->wq->id, ret); 1746 ret = -EIO; 1747 goto out; 1748 } 1749 src_addr = sg_dma_address(req->src); 1750 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," 1751 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, 1752 req->src, req->slen, sg_dma_len(req->src)); 1753 1754 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1755 if (nr_sgs <= 0 || nr_sgs > 1) { 1756 dev_dbg(dev, "couldn't map dst sg for iaa device %d," 1757 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, 1758 iaa_wq->wq->id, ret); 1759 ret = -EIO; 1760 goto err_map_dst; 1761 } 1762 dst_addr = sg_dma_address(req->dst); 1763 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," 1764 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, 1765 req->dst, req->dlen, sg_dma_len(req->dst)); 1766 1767 ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, 1768 dst_addr, &req->dlen, false); 1769 if (ret == -EINPROGRESS) 1770 return ret; 1771 1772 if (ret != 0) 1773 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); 1774 1775 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); 1776 err_map_dst: 1777 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); 1778 out: 1779 iaa_wq_put(wq); 1780 1781 return ret; 1782 } 1783 1784 static void compression_ctx_init(struct iaa_compression_ctx *ctx) 1785 { 1786 ctx->verify_compress = iaa_verify_compress; 1787 ctx->async_mode = async_mode; 1788 ctx->use_irq = use_irq; 1789 } 1790 1791 static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) 1792 { 1793 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); 1794 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); 1795 1796 compression_ctx_init(ctx); 1797 1798 ctx->mode = IAA_MODE_FIXED; 1799 1800 return 0; 1801 } 1802 1803 static void dst_free(struct scatterlist *sgl) 1804 { 1805 /* 1806 * Called for req->dst = NULL cases but we free elsewhere 1807 * using sgl_free_order(). 1808 */ 1809 } 1810 1811 static struct acomp_alg iaa_acomp_fixed_deflate = { 1812 .init = iaa_comp_init_fixed, 1813 .compress = iaa_comp_acompress, 1814 .decompress = iaa_comp_adecompress, 1815 .dst_free = dst_free, 1816 .base = { 1817 .cra_name = "deflate", 1818 .cra_driver_name = "deflate-iaa", 1819 .cra_flags = CRYPTO_ALG_ASYNC, 1820 .cra_ctxsize = sizeof(struct iaa_compression_ctx), 1821 .cra_module = THIS_MODULE, 1822 .cra_priority = IAA_ALG_PRIORITY, 1823 } 1824 }; 1825 1826 static int iaa_register_compression_device(void) 1827 { 1828 int ret; 1829 1830 ret = crypto_register_acomp(&iaa_acomp_fixed_deflate); 1831 if (ret) { 1832 pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret); 1833 goto out; 1834 } 1835 1836 iaa_crypto_registered = true; 1837 out: 1838 return ret; 1839 } 1840 1841 static int iaa_unregister_compression_device(void) 1842 { 1843 if (iaa_crypto_registered) 1844 crypto_unregister_acomp(&iaa_acomp_fixed_deflate); 1845 1846 return 0; 1847 } 1848 1849 static int iaa_crypto_probe(struct idxd_dev *idxd_dev) 1850 { 1851 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 1852 struct idxd_device *idxd = wq->idxd; 1853 struct idxd_driver_data *data = idxd->data; 1854 struct device *dev = &idxd_dev->conf_dev; 1855 bool first_wq = false; 1856 int ret = 0; 1857 1858 if (idxd->state != IDXD_DEV_ENABLED) 1859 return -ENXIO; 1860 1861 if (data->type != IDXD_TYPE_IAX) 1862 return -ENODEV; 1863 1864 mutex_lock(&wq->wq_lock); 1865 1866 if (idxd_wq_get_private(wq)) { 1867 mutex_unlock(&wq->wq_lock); 1868 return -EBUSY; 1869 } 1870 1871 if (!idxd_wq_driver_name_match(wq, dev)) { 1872 dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n", 1873 idxd->id, wq->id, wq->driver_name, dev->driver->name); 1874 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; 1875 ret = -ENODEV; 1876 goto err; 1877 } 1878 1879 wq->type = IDXD_WQT_KERNEL; 1880 1881 ret = idxd_drv_enable_wq(wq); 1882 if (ret < 0) { 1883 dev_dbg(dev, "enable wq %d.%d failed: %d\n", 1884 idxd->id, wq->id, ret); 1885 ret = -ENXIO; 1886 goto err; 1887 } 1888 1889 mutex_lock(&iaa_devices_lock); 1890 1891 if (list_empty(&iaa_devices)) { 1892 ret = alloc_wq_table(wq->idxd->max_wqs); 1893 if (ret) 1894 goto err_alloc; 1895 first_wq = true; 1896 } 1897 1898 ret = save_iaa_wq(wq); 1899 if (ret) 1900 goto err_save; 1901 1902 rebalance_wq_table(); 1903 1904 if (first_wq) { 1905 iaa_crypto_enabled = true; 1906 ret = iaa_register_compression_device(); 1907 if (ret != 0) { 1908 iaa_crypto_enabled = false; 1909 dev_dbg(dev, "IAA compression device registration failed\n"); 1910 goto err_register; 1911 } 1912 try_module_get(THIS_MODULE); 1913 1914 pr_info("iaa_crypto now ENABLED\n"); 1915 } 1916 1917 mutex_unlock(&iaa_devices_lock); 1918 out: 1919 mutex_unlock(&wq->wq_lock); 1920 1921 return ret; 1922 1923 err_register: 1924 remove_iaa_wq(wq); 1925 free_iaa_wq(idxd_wq_get_private(wq)); 1926 err_save: 1927 if (first_wq) 1928 free_wq_table(); 1929 err_alloc: 1930 mutex_unlock(&iaa_devices_lock); 1931 idxd_drv_disable_wq(wq); 1932 err: 1933 wq->type = IDXD_WQT_NONE; 1934 1935 goto out; 1936 } 1937 1938 static void iaa_crypto_remove(struct idxd_dev *idxd_dev) 1939 { 1940 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 1941 struct idxd_device *idxd = wq->idxd; 1942 struct iaa_wq *iaa_wq; 1943 bool free = false; 1944 1945 idxd_wq_quiesce(wq); 1946 1947 mutex_lock(&wq->wq_lock); 1948 mutex_lock(&iaa_devices_lock); 1949 1950 remove_iaa_wq(wq); 1951 1952 spin_lock(&idxd->dev_lock); 1953 iaa_wq = idxd_wq_get_private(wq); 1954 if (!iaa_wq) { 1955 spin_unlock(&idxd->dev_lock); 1956 pr_err("%s: no iaa_wq available to remove\n", __func__); 1957 goto out; 1958 } 1959 1960 if (iaa_wq->ref) { 1961 iaa_wq->remove = true; 1962 } else { 1963 wq = iaa_wq->wq; 1964 idxd_wq_set_private(wq, NULL); 1965 free = true; 1966 } 1967 spin_unlock(&idxd->dev_lock); 1968 if (free) { 1969 __free_iaa_wq(iaa_wq); 1970 kfree(iaa_wq); 1971 } 1972 1973 idxd_drv_disable_wq(wq); 1974 rebalance_wq_table(); 1975 1976 if (nr_iaa == 0) { 1977 iaa_crypto_enabled = false; 1978 free_wq_table(); 1979 module_put(THIS_MODULE); 1980 1981 pr_info("iaa_crypto now DISABLED\n"); 1982 } 1983 out: 1984 mutex_unlock(&iaa_devices_lock); 1985 mutex_unlock(&wq->wq_lock); 1986 } 1987 1988 static enum idxd_dev_type dev_types[] = { 1989 IDXD_DEV_WQ, 1990 IDXD_DEV_NONE, 1991 }; 1992 1993 static struct idxd_device_driver iaa_crypto_driver = { 1994 .probe = iaa_crypto_probe, 1995 .remove = iaa_crypto_remove, 1996 .name = IDXD_SUBDRIVER_NAME, 1997 .type = dev_types, 1998 .desc_complete = iaa_desc_complete, 1999 }; 2000 2001 static int __init iaa_crypto_init_module(void) 2002 { 2003 int ret = 0; 2004 int node; 2005 2006 nr_cpus = num_possible_cpus(); 2007 for_each_node_with_cpus(node) 2008 nr_nodes++; 2009 if (!nr_nodes) { 2010 pr_err("IAA couldn't find any nodes with cpus\n"); 2011 return -ENODEV; 2012 } 2013 nr_cpus_per_node = nr_cpus / nr_nodes; 2014 2015 if (crypto_has_comp("deflate-generic", 0, 0)) 2016 deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0); 2017 2018 if (IS_ERR_OR_NULL(deflate_generic_tfm)) { 2019 pr_err("IAA could not alloc %s tfm: errcode = %ld\n", 2020 "deflate-generic", PTR_ERR(deflate_generic_tfm)); 2021 return -ENOMEM; 2022 } 2023 2024 ret = iaa_aecs_init_fixed(); 2025 if (ret < 0) { 2026 pr_debug("IAA fixed compression mode init failed\n"); 2027 goto err_aecs_init; 2028 } 2029 2030 ret = idxd_driver_register(&iaa_crypto_driver); 2031 if (ret) { 2032 pr_debug("IAA wq sub-driver registration failed\n"); 2033 goto err_driver_reg; 2034 } 2035 2036 ret = driver_create_file(&iaa_crypto_driver.drv, 2037 &driver_attr_verify_compress); 2038 if (ret) { 2039 pr_debug("IAA verify_compress attr creation failed\n"); 2040 goto err_verify_attr_create; 2041 } 2042 2043 ret = driver_create_file(&iaa_crypto_driver.drv, 2044 &driver_attr_sync_mode); 2045 if (ret) { 2046 pr_debug("IAA sync mode attr creation failed\n"); 2047 goto err_sync_attr_create; 2048 } 2049 2050 if (iaa_crypto_debugfs_init()) 2051 pr_warn("debugfs init failed, stats not available\n"); 2052 2053 pr_debug("initialized\n"); 2054 out: 2055 return ret; 2056 2057 err_sync_attr_create: 2058 driver_remove_file(&iaa_crypto_driver.drv, 2059 &driver_attr_verify_compress); 2060 err_verify_attr_create: 2061 idxd_driver_unregister(&iaa_crypto_driver); 2062 err_driver_reg: 2063 iaa_aecs_cleanup_fixed(); 2064 err_aecs_init: 2065 crypto_free_comp(deflate_generic_tfm); 2066 2067 goto out; 2068 } 2069 2070 static void __exit iaa_crypto_cleanup_module(void) 2071 { 2072 if (iaa_unregister_compression_device()) 2073 pr_debug("IAA compression device unregister failed\n"); 2074 2075 iaa_crypto_debugfs_cleanup(); 2076 driver_remove_file(&iaa_crypto_driver.drv, 2077 &driver_attr_sync_mode); 2078 driver_remove_file(&iaa_crypto_driver.drv, 2079 &driver_attr_verify_compress); 2080 idxd_driver_unregister(&iaa_crypto_driver); 2081 iaa_aecs_cleanup_fixed(); 2082 crypto_free_comp(deflate_generic_tfm); 2083 2084 pr_debug("cleaned up\n"); 2085 } 2086 2087 MODULE_IMPORT_NS(IDXD); 2088 MODULE_LICENSE("GPL"); 2089 MODULE_ALIAS_IDXD_DEVICE(0); 2090 MODULE_AUTHOR("Intel Corporation"); 2091 MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver"); 2092 2093 module_init(iaa_crypto_init_module); 2094 module_exit(iaa_crypto_cleanup_module); 2095