1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_transport_access_macros.h" 12 #include "adf_transport_internal.h" 13 #include <linux/delay.h> 14 #include "adf_accel_devices.h" 15 #include "adf_transport_internal.h" 16 #include "adf_transport_access_macros.h" 17 #include "adf_cfg.h" 18 #include "adf_common_drv.h" 19 20 #define QAT_RING_ALIGNMENT 64 21 22 static inline u32 23 adf_modulo(u32 data, u32 shift) 24 { 25 u32 div = data >> shift; 26 u32 mult = div << shift; 27 28 return data - mult; 29 } 30 31 static inline int 32 adf_check_ring_alignment(u64 addr, u64 size) 33 { 34 if (((size - 1) & addr) != 0) 35 return EFAULT; 36 return 0; 37 } 38 39 static int 40 adf_verify_ring_size(u32 msg_size, u32 msg_num) 41 { 42 int i = ADF_MIN_RING_SIZE; 43 44 for (; i <= ADF_MAX_RING_SIZE; i++) 45 if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) 46 return i; 47 48 return ADF_DEFAULT_RING_SIZE; 49 } 50 51 static int 52 adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring) 53 { 54 mtx_lock(&bank->lock); 55 if (bank->ring_mask & (1 << ring)) { 56 mtx_unlock(&bank->lock); 57 return EFAULT; 58 } 59 bank->ring_mask |= (1 << ring); 60 mtx_unlock(&bank->lock); 61 return 0; 62 } 63 64 static void 65 adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) 66 { 67 mtx_lock(&bank->lock); 68 bank->ring_mask &= ~(1 << ring); 69 mtx_unlock(&bank->lock); 70 } 71 72 static void 73 adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) 74 { 75 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 76 u32 enable_int_col_mask = 0; 77 78 if (csr_ops->get_int_col_ctl_enable_mask) 79 enable_int_col_mask = csr_ops->get_int_col_ctl_enable_mask(); 80 81 mtx_lock(&bank->lock); 82 bank->irq_mask |= (1 << ring); 83 mtx_unlock(&bank->lock); 84 csr_ops->write_csr_int_col_en(bank->csr_addr, 85 bank->bank_number, 86 bank->irq_mask); 87 csr_ops->write_csr_int_col_ctl(bank->csr_addr, 88 bank->bank_number, 89 bank->irq_coalesc_timer | 90 enable_int_col_mask); 91 } 92 93 static void 94 adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) 95 { 96 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 97 98 mtx_lock(&bank->lock); 99 bank->irq_mask &= ~(1 << ring); 100 mtx_unlock(&bank->lock); 101 csr_ops->write_csr_int_col_en(bank->csr_addr, 102 bank->bank_number, 103 bank->irq_mask); 104 } 105 106 int 107 adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) 108 { 109 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 110 u32 msg_size = 0; 111 112 if (atomic_add_return(1, ring->inflights) > ring->max_inflights) { 113 atomic_dec(ring->inflights); 114 return EAGAIN; 115 } 116 117 msg_size = ADF_MSG_SIZE_TO_BYTES(ring->msg_size); 118 mtx_lock(&ring->lock); 119 memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), 120 msg, 121 msg_size); 122 123 ring->tail = adf_modulo(ring->tail + msg_size, 124 ADF_RING_SIZE_MODULO(ring->ring_size)); 125 126 csr_ops->write_csr_ring_tail(ring->bank->csr_addr, 127 ring->bank->bank_number, 128 ring->ring_number, 129 ring->tail); 130 ring->csr_tail_offset = ring->tail; 131 mtx_unlock(&ring->lock); 132 return 0; 133 } 134 135 int 136 adf_handle_response(struct adf_etr_ring_data *ring, u32 quota) 137 { 138 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 139 u32 msg_counter = 0; 140 u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); 141 142 if (!quota) 143 quota = ADF_NO_RESPONSE_QUOTA; 144 145 while ((*msg != ADF_RING_EMPTY_SIG) && (msg_counter < quota)) { 146 ring->callback((u32 *)msg); 147 atomic_dec(ring->inflights); 148 *msg = ADF_RING_EMPTY_SIG; 149 ring->head = 150 adf_modulo(ring->head + 151 ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 152 ADF_RING_SIZE_MODULO(ring->ring_size)); 153 msg_counter++; 154 msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); 155 } 156 if (msg_counter > 0) 157 csr_ops->write_csr_ring_head(ring->bank->csr_addr, 158 ring->bank->bank_number, 159 ring->ring_number, 160 ring->head); 161 return msg_counter; 162 } 163 164 int 165 adf_poll_bank(u32 accel_id, u32 bank_num, u32 quota) 166 { 167 int num_resp; 168 struct adf_accel_dev *accel_dev; 169 struct adf_etr_data *trans_data; 170 struct adf_etr_bank_data *bank; 171 struct adf_etr_ring_data *ring; 172 struct adf_hw_csr_ops *csr_ops; 173 u32 rings_not_empty; 174 u32 ring_num; 175 u32 resp_total = 0; 176 u32 num_rings_per_bank; 177 178 /* Find the accel device associated with the accelId 179 * passed in. 180 */ 181 accel_dev = adf_devmgr_get_dev_by_id(accel_id); 182 if (!accel_dev) { 183 pr_err("There is no device with id: %d\n", accel_id); 184 return EINVAL; 185 } 186 187 csr_ops = GET_CSR_OPS(accel_dev); 188 trans_data = accel_dev->transport; 189 bank = &trans_data->banks[bank_num]; 190 mtx_lock(&bank->lock); 191 192 /* Read the ring status CSR to determine which rings are empty. */ 193 rings_not_empty = 194 csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number); 195 /* Complement to find which rings have data to be processed. */ 196 rings_not_empty = (~rings_not_empty) & bank->ring_mask; 197 198 /* Return RETRY if the bank polling rings 199 * are all empty. 200 */ 201 if (!(rings_not_empty & bank->ring_mask)) { 202 mtx_unlock(&bank->lock); 203 return EAGAIN; 204 } 205 206 /* 207 * Loop over all rings within this bank. 208 * The ring structure is global to all 209 * rings hence while we loop over all rings in the 210 * bank we use ring_number to get the global ring. 211 */ 212 num_rings_per_bank = accel_dev->hw_device->num_rings_per_bank; 213 for (ring_num = 0; ring_num < num_rings_per_bank; ring_num++) { 214 ring = &bank->rings[ring_num]; 215 216 /* And with polling ring mask. 217 * If the there is no data on this ring 218 * move to the next one. 219 */ 220 if (!(rings_not_empty & (1 << ring->ring_number))) 221 continue; 222 223 /* Poll the ring. */ 224 num_resp = adf_handle_response(ring, quota); 225 resp_total += num_resp; 226 } 227 228 mtx_unlock(&bank->lock); 229 /* Return SUCCESS if there's any response message 230 * returned. 231 */ 232 if (resp_total) 233 return 0; 234 return EAGAIN; 235 } 236 237 int 238 adf_poll_all_banks(u32 accel_id, u32 quota) 239 { 240 int status = EAGAIN; 241 struct adf_accel_dev *accel_dev; 242 struct adf_etr_data *trans_data; 243 struct adf_etr_bank_data *bank; 244 u32 bank_num; 245 u32 stat_total = 0; 246 247 /* Find the accel device associated with the accelId 248 * passed in. 249 */ 250 accel_dev = adf_devmgr_get_dev_by_id(accel_id); 251 if (!accel_dev) { 252 pr_err("There is no device with id: %d\n", accel_id); 253 return EINVAL; 254 } 255 256 /* Loop over banks and call adf_poll_bank */ 257 trans_data = accel_dev->transport; 258 for (bank_num = 0; bank_num < GET_MAX_BANKS(accel_dev); bank_num++) { 259 bank = &trans_data->banks[bank_num]; 260 /* if there are no polling rings on this bank 261 * continue to the next bank number. 262 */ 263 if (bank->ring_mask == 0) 264 continue; 265 status = adf_poll_bank(accel_id, bank_num, quota); 266 /* The successful status should be AGAIN or 0 */ 267 if (status == 0) 268 stat_total++; 269 else if (status != EAGAIN) 270 return status; 271 } 272 273 /* Return SUCCESS if adf_poll_bank returned SUCCESS 274 * at any stage. adf_poll_bank cannot 275 * return fail in the above case. 276 */ 277 if (stat_total) 278 return 0; 279 280 return EAGAIN; 281 } 282 283 static void 284 adf_configure_tx_ring(struct adf_etr_ring_data *ring) 285 { 286 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 287 u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); 288 289 csr_ops->write_csr_ring_config(ring->bank->csr_addr, 290 ring->bank->bank_number, 291 ring->ring_number, 292 ring_config); 293 } 294 295 static void 296 adf_configure_rx_ring(struct adf_etr_ring_data *ring) 297 { 298 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 299 u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size, 300 ADF_RING_NEAR_WATERMARK_512, 301 ADF_RING_NEAR_WATERMARK_0); 302 303 csr_ops->write_csr_ring_config(ring->bank->csr_addr, 304 ring->bank->bank_number, 305 ring->ring_number, 306 ring_config); 307 } 308 309 static int 310 adf_init_ring(struct adf_etr_ring_data *ring) 311 { 312 struct adf_etr_bank_data *bank = ring->bank; 313 struct adf_accel_dev *accel_dev = bank->accel_dev; 314 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 315 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 316 u64 ring_base; 317 u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); 318 319 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); 320 int ret; 321 322 ret = bus_dma_mem_create(&ring->dma_mem, 323 accel_dev->dma_tag, 324 ring_size_bytes, 325 BUS_SPACE_MAXADDR, 326 ring_size_bytes, 327 M_WAITOK | M_ZERO); 328 if (ret) 329 return ret; 330 ring->base_addr = ring->dma_mem.dma_vaddr; 331 ring->dma_addr = ring->dma_mem.dma_baddr; 332 333 memset(ring->base_addr, 0x7F, ring_size_bytes); 334 /* The base_addr has to be aligned to the size of the buffer */ 335 if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { 336 device_printf(GET_DEV(accel_dev), "Ring address not aligned\n"); 337 bus_dma_mem_free(&ring->dma_mem); 338 ring->base_addr = NULL; 339 return EFAULT; 340 } 341 342 if (hw_data->tx_rings_mask & (1 << ring->ring_number)) 343 adf_configure_tx_ring(ring); 344 else 345 adf_configure_rx_ring(ring); 346 347 ring_base = 348 csr_ops->build_csr_ring_base_addr(ring->dma_addr, ring->ring_size); 349 csr_ops->write_csr_ring_base(ring->bank->csr_addr, 350 ring->bank->bank_number, 351 ring->ring_number, 352 ring_base); 353 mtx_init(&ring->lock, "adf bank", NULL, MTX_DEF); 354 return 0; 355 } 356 357 static void 358 adf_cleanup_ring(struct adf_etr_ring_data *ring) 359 { 360 u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); 361 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); 362 363 if (ring->base_addr) { 364 explicit_bzero(ring->base_addr, ring_size_bytes); 365 bus_dma_mem_free(&ring->dma_mem); 366 } 367 mtx_destroy(&ring->lock); 368 } 369 370 int 371 adf_create_ring(struct adf_accel_dev *accel_dev, 372 const char *section, 373 u32 bank_num, 374 u32 num_msgs, 375 u32 msg_size, 376 const char *ring_name, 377 adf_callback_fn callback, 378 int poll_mode, 379 struct adf_etr_ring_data **ring_ptr) 380 { 381 struct adf_etr_data *transport_data = accel_dev->transport; 382 struct adf_etr_bank_data *bank; 383 struct adf_etr_ring_data *ring; 384 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 385 u32 ring_num; 386 int ret; 387 u8 num_rings_per_bank = accel_dev->hw_device->num_rings_per_bank; 388 389 if (bank_num >= GET_MAX_BANKS(accel_dev)) { 390 device_printf(GET_DEV(accel_dev), "Invalid bank number\n"); 391 return EFAULT; 392 } 393 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { 394 device_printf(GET_DEV(accel_dev), "Invalid msg size\n"); 395 return EFAULT; 396 } 397 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), 398 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { 399 device_printf(GET_DEV(accel_dev), 400 "Invalid ring size for given msg size\n"); 401 return EFAULT; 402 } 403 if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { 404 device_printf(GET_DEV(accel_dev), 405 "Section %s, no such entry : %s\n", 406 section, 407 ring_name); 408 return EFAULT; 409 } 410 if (compat_strtouint(val, 10, &ring_num)) { 411 device_printf(GET_DEV(accel_dev), "Can't get ring number\n"); 412 return EFAULT; 413 } 414 if (ring_num >= num_rings_per_bank) { 415 device_printf(GET_DEV(accel_dev), "Invalid ring number\n"); 416 return EFAULT; 417 } 418 419 bank = &transport_data->banks[bank_num]; 420 if (adf_reserve_ring(bank, ring_num)) { 421 device_printf(GET_DEV(accel_dev), 422 "Ring %d, %s already exists.\n", 423 ring_num, 424 ring_name); 425 return EFAULT; 426 } 427 ring = &bank->rings[ring_num]; 428 ring->ring_number = ring_num; 429 ring->bank = bank; 430 ring->callback = callback; 431 ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); 432 ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); 433 ring->max_inflights = 434 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); 435 ring->head = 0; 436 ring->tail = 0; 437 ring->csr_tail_offset = 0; 438 ret = adf_init_ring(ring); 439 if (ret) 440 goto err; 441 442 /* Enable HW arbitration for the given ring */ 443 adf_update_ring_arb(ring); 444 445 if (adf_ring_debugfs_add(ring, ring_name)) { 446 device_printf(GET_DEV(accel_dev), 447 "Couldn't add ring debugfs entry\n"); 448 ret = EFAULT; 449 goto err; 450 } 451 452 /* Enable interrupts if needed */ 453 if (callback && !poll_mode) 454 adf_enable_ring_irq(bank, ring->ring_number); 455 *ring_ptr = ring; 456 return 0; 457 err: 458 adf_cleanup_ring(ring); 459 adf_unreserve_ring(bank, ring_num); 460 adf_update_ring_arb(ring); 461 return ret; 462 } 463 464 void 465 adf_remove_ring(struct adf_etr_ring_data *ring) 466 { 467 struct adf_etr_bank_data *bank = ring->bank; 468 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 469 470 /* Disable interrupts for the given ring */ 471 adf_disable_ring_irq(bank, ring->ring_number); 472 473 /* Clear PCI config space */ 474 csr_ops->write_csr_ring_config(bank->csr_addr, 475 bank->bank_number, 476 ring->ring_number, 477 0); 478 csr_ops->write_csr_ring_base(bank->csr_addr, 479 bank->bank_number, 480 ring->ring_number, 481 0); 482 adf_ring_debugfs_rm(ring); 483 adf_unreserve_ring(bank, ring->ring_number); 484 /* Disable HW arbitration for the given ring */ 485 adf_update_ring_arb(ring); 486 adf_cleanup_ring(ring); 487 } 488 489 static void 490 adf_ring_response_handler(struct adf_etr_bank_data *bank) 491 { 492 struct adf_accel_dev *accel_dev = bank->accel_dev; 493 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 494 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 495 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 496 u32 empty_rings, i; 497 498 empty_rings = 499 csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number); 500 empty_rings = ~empty_rings & bank->irq_mask; 501 502 for (i = 0; i < num_rings_per_bank; ++i) { 503 if (empty_rings & (1 << i)) 504 adf_handle_response(&bank->rings[i], 0); 505 } 506 } 507 508 void 509 adf_response_handler(uintptr_t bank_addr) 510 { 511 struct adf_etr_bank_data *bank = (void *)bank_addr; 512 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 513 514 /* Handle all the responses and re-enable IRQs */ 515 adf_ring_response_handler(bank); 516 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, 517 bank->bank_number, 518 bank->irq_mask); 519 } 520 521 static inline int 522 adf_get_cfg_int(struct adf_accel_dev *accel_dev, 523 const char *section, 524 const char *format, 525 u32 key, 526 u32 *value) 527 { 528 char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 529 char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 530 531 snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); 532 533 if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) 534 return EFAULT; 535 536 if (compat_strtouint(val_buf, 10, value)) 537 return EFAULT; 538 return 0; 539 } 540 541 static void 542 adf_get_coalesc_timer(struct adf_etr_bank_data *bank, 543 const char *section, 544 u32 bank_num_in_accel) 545 { 546 struct adf_accel_dev *accel_dev = bank->accel_dev; 547 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 548 u32 coalesc_timer = ADF_COALESCING_DEF_TIME; 549 550 adf_get_cfg_int(accel_dev, 551 section, 552 ADF_ETRMGR_COALESCE_TIMER_FORMAT, 553 bank_num_in_accel, 554 &coalesc_timer); 555 556 if (hw_data->get_clock_speed) 557 bank->irq_coalesc_timer = 558 (coalesc_timer * 559 (hw_data->get_clock_speed(hw_data) / USEC_PER_SEC)) / 560 NSEC_PER_USEC; 561 else 562 bank->irq_coalesc_timer = coalesc_timer; 563 564 if (bank->irq_coalesc_timer > ADF_COALESCING_MAX_TIME) 565 bank->irq_coalesc_timer = ADF_COALESCING_MAX_TIME; 566 else if (bank->irq_coalesc_timer < ADF_COALESCING_MIN_TIME) 567 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; 568 } 569 570 static int 571 adf_init_bank(struct adf_accel_dev *accel_dev, 572 struct adf_etr_bank_data *bank, 573 u32 bank_num, 574 struct resource *csr_addr) 575 { 576 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 577 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; 578 struct adf_etr_ring_data *ring; 579 struct adf_etr_ring_data *tx_ring; 580 u32 i, coalesc_enabled = 0; 581 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 582 u32 irq_mask = BIT(num_rings_per_bank) - 1; 583 u32 size = 0; 584 585 explicit_bzero(bank, sizeof(*bank)); 586 bank->bank_number = bank_num; 587 bank->csr_addr = csr_addr; 588 bank->accel_dev = accel_dev; 589 mtx_init(&bank->lock, "adf bank", NULL, MTX_DEF); 590 591 /* Allocate the rings in the bank */ 592 size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); 593 bank->rings = kzalloc_node(size, 594 M_WAITOK | M_ZERO, 595 dev_to_node(GET_DEV(accel_dev))); 596 597 /* Enable IRQ coalescing always. This will allow to use 598 * the optimised flag and coalesc register. 599 * If it is disabled in the config file just use min time value */ 600 if ((adf_get_cfg_int(accel_dev, 601 "Accelerator0", 602 ADF_ETRMGR_COALESCING_ENABLED_FORMAT, 603 bank_num, 604 &coalesc_enabled) == 0) && 605 coalesc_enabled) 606 adf_get_coalesc_timer(bank, "Accelerator0", bank_num); 607 else 608 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; 609 610 for (i = 0; i < num_rings_per_bank; i++) { 611 csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); 612 csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); 613 ring = &bank->rings[i]; 614 if (hw_data->tx_rings_mask & (1 << i)) { 615 ring->inflights = 616 kzalloc_node(sizeof(atomic_t), 617 M_WAITOK | M_ZERO, 618 dev_to_node(GET_DEV(accel_dev))); 619 } else { 620 if (i < hw_data->tx_rx_gap) { 621 device_printf(GET_DEV(accel_dev), 622 "Invalid tx rings mask config\n"); 623 goto err; 624 } 625 tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; 626 ring->inflights = tx_ring->inflights; 627 } 628 } 629 630 if (adf_bank_debugfs_add(bank)) { 631 device_printf(GET_DEV(accel_dev), 632 "Failed to add bank debugfs entry\n"); 633 goto err; 634 } 635 636 csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); 637 csr_ops->write_csr_int_srcsel(csr_addr, bank_num); 638 return 0; 639 err: 640 for (i = 0; i < num_rings_per_bank; i++) { 641 ring = &bank->rings[i]; 642 if (hw_data->tx_rings_mask & (1 << i)) { 643 kfree(ring->inflights); 644 ring->inflights = NULL; 645 } 646 } 647 kfree(bank->rings); 648 return ENOMEM; 649 } 650 651 /** 652 * adf_init_etr_data() - Initialize transport rings for acceleration device 653 * @accel_dev: Pointer to acceleration device. 654 * 655 * Function initializes the communications channels (rings) to the 656 * acceleration device accel_dev. 657 * To be used by QAT device specific drivers. 658 * 659 * Return: 0 on success, error code otherwise. 660 */ 661 int 662 adf_init_etr_data(struct adf_accel_dev *accel_dev) 663 { 664 struct adf_etr_data *etr_data; 665 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 666 struct resource *csr_addr; 667 u32 size; 668 u32 num_banks = 0; 669 int i, ret; 670 671 etr_data = kzalloc_node(sizeof(*etr_data), 672 M_WAITOK | M_ZERO, 673 dev_to_node(GET_DEV(accel_dev))); 674 675 num_banks = GET_MAX_BANKS(accel_dev); 676 size = num_banks * sizeof(struct adf_etr_bank_data); 677 etr_data->banks = kzalloc_node(size, 678 M_WAITOK | M_ZERO, 679 dev_to_node(GET_DEV(accel_dev))); 680 681 accel_dev->transport = etr_data; 682 i = hw_data->get_etr_bar_id(hw_data); 683 csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; 684 685 etr_data->debug = 686 SYSCTL_ADD_NODE(&accel_dev->sysctl_ctx, 687 SYSCTL_CHILDREN( 688 device_get_sysctl_tree(GET_DEV(accel_dev))), 689 OID_AUTO, 690 "transport", 691 CTLFLAG_RD, 692 NULL, 693 "Transport parameters"); 694 if (!etr_data->debug) { 695 device_printf(GET_DEV(accel_dev), 696 "Unable to create transport debugfs entry\n"); 697 ret = ENOENT; 698 goto err_bank_all; 699 } 700 701 for (i = 0; i < num_banks; i++) { 702 ret = 703 adf_init_bank(accel_dev, &etr_data->banks[i], i, csr_addr); 704 if (ret) 705 goto err_bank_all; 706 } 707 708 return 0; 709 710 err_bank_all: 711 kfree(etr_data->banks); 712 kfree(etr_data); 713 accel_dev->transport = NULL; 714 return ret; 715 } 716 717 static void 718 cleanup_bank(struct adf_etr_bank_data *bank) 719 { 720 u32 i; 721 struct adf_accel_dev *accel_dev = bank->accel_dev; 722 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 723 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 724 725 for (i = 0; i < num_rings_per_bank; i++) { 726 struct adf_accel_dev *accel_dev = bank->accel_dev; 727 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 728 struct adf_etr_ring_data *ring = &bank->rings[i]; 729 730 if (bank->ring_mask & (1 << i)) 731 adf_cleanup_ring(ring); 732 733 if (hw_data->tx_rings_mask & (1 << i)) { 734 kfree(ring->inflights); 735 ring->inflights = NULL; 736 } 737 } 738 kfree(bank->rings); 739 adf_bank_debugfs_rm(bank); 740 mtx_destroy(&bank->lock); 741 explicit_bzero(bank, sizeof(*bank)); 742 } 743 744 static void 745 adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) 746 { 747 struct adf_etr_data *etr_data = accel_dev->transport; 748 u32 i, num_banks = GET_MAX_BANKS(accel_dev); 749 750 for (i = 0; i < num_banks; i++) 751 cleanup_bank(&etr_data->banks[i]); 752 } 753 754 /** 755 * adf_cleanup_etr_data() - Clear transport rings for acceleration device 756 * @accel_dev: Pointer to acceleration device. 757 * 758 * Function is the clears the communications channels (rings) of the 759 * acceleration device accel_dev. 760 * To be used by QAT device specific drivers. 761 * 762 * Return: void 763 */ 764 void 765 adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) 766 { 767 struct adf_etr_data *etr_data = accel_dev->transport; 768 769 if (etr_data) { 770 adf_cleanup_etr_handles(accel_dev); 771 kfree(etr_data->banks); 772 kfree(etr_data); 773 accel_dev->transport = NULL; 774 } 775 } 776