1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/delay.h> 4 #include <linux/nospec.h> 5 #include "adf_accel_devices.h" 6 #include "adf_transport_internal.h" 7 #include "adf_transport_access_macros.h" 8 #include "adf_cfg.h" 9 #include "adf_common_drv.h" 10 11 #define ADF_MAX_RING_THRESHOLD 80 12 #define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100) 13 14 static inline u32 adf_modulo(u32 data, u32 shift) 15 { 16 u32 div = data >> shift; 17 u32 mult = div << shift; 18 19 return data - mult; 20 } 21 22 static inline int adf_check_ring_alignment(u64 addr, u64 size) 23 { 24 if (((size - 1) & addr) != 0) 25 return -EFAULT; 26 return 0; 27 } 28 29 static int adf_verify_ring_size(u32 msg_size, u32 msg_num) 30 { 31 int i = ADF_MIN_RING_SIZE; 32 33 for (; i <= ADF_MAX_RING_SIZE; i++) 34 if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) 35 return i; 36 37 return ADF_DEFAULT_RING_SIZE; 38 } 39 40 static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring) 41 { 42 spin_lock(&bank->lock); 43 if (bank->ring_mask & (1 << ring)) { 44 spin_unlock(&bank->lock); 45 return -EFAULT; 46 } 47 bank->ring_mask |= (1 << ring); 48 spin_unlock(&bank->lock); 49 return 0; 50 } 51 52 static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) 53 { 54 spin_lock(&bank->lock); 55 bank->ring_mask &= ~(1 << ring); 56 spin_unlock(&bank->lock); 57 } 58 59 static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) 60 { 61 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 62 63 spin_lock_bh(&bank->lock); 64 bank->irq_mask |= (1 << ring); 65 spin_unlock_bh(&bank->lock); 66 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, 67 bank->irq_mask); 68 csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, 69 bank->irq_coalesc_timer); 70 } 71 72 static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) 73 { 74 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 75 76 spin_lock_bh(&bank->lock); 77 bank->irq_mask &= ~(1 << ring); 78 spin_unlock_bh(&bank->lock); 79 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, 80 bank->irq_mask); 81 } 82 83 bool adf_ring_nearly_full(struct adf_etr_ring_data *ring) 84 { 85 return atomic_read(ring->inflights) > ring->threshold; 86 } 87 88 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) 89 { 90 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 91 92 if (atomic_add_return(1, ring->inflights) > 93 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { 94 atomic_dec(ring->inflights); 95 return -EAGAIN; 96 } 97 spin_lock_bh(&ring->lock); 98 memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg, 99 ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); 100 101 ring->tail = adf_modulo(ring->tail + 102 ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 103 ADF_RING_SIZE_MODULO(ring->ring_size)); 104 csr_ops->write_csr_ring_tail(ring->bank->csr_addr, 105 ring->bank->bank_number, ring->ring_number, 106 ring->tail); 107 spin_unlock_bh(&ring->lock); 108 109 return 0; 110 } 111 112 static int adf_handle_response(struct adf_etr_ring_data *ring) 113 { 114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 115 u32 msg_counter = 0; 116 u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); 117 118 while (*msg != ADF_RING_EMPTY_SIG) { 119 ring->callback((u32 *)msg); 120 atomic_dec(ring->inflights); 121 *msg = ADF_RING_EMPTY_SIG; 122 ring->head = adf_modulo(ring->head + 123 ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 124 ADF_RING_SIZE_MODULO(ring->ring_size)); 125 msg_counter++; 126 msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); 127 } 128 if (msg_counter > 0) { 129 csr_ops->write_csr_ring_head(ring->bank->csr_addr, 130 ring->bank->bank_number, 131 ring->ring_number, ring->head); 132 } 133 return 0; 134 } 135 136 static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) 137 { 138 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 139 u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); 140 141 csr_ops->write_csr_ring_config(ring->bank->csr_addr, 142 ring->bank->bank_number, 143 ring->ring_number, ring_config); 144 145 } 146 147 static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) 148 { 149 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 150 u32 ring_config = 151 BUILD_RESP_RING_CONFIG(ring->ring_size, 152 ADF_RING_NEAR_WATERMARK_512, 153 ADF_RING_NEAR_WATERMARK_0); 154 155 csr_ops->write_csr_ring_config(ring->bank->csr_addr, 156 ring->bank->bank_number, 157 ring->ring_number, ring_config); 158 } 159 160 static int adf_init_ring(struct adf_etr_ring_data *ring) 161 { 162 struct adf_etr_bank_data *bank = ring->bank; 163 struct adf_accel_dev *accel_dev = bank->accel_dev; 164 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 165 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 166 u64 ring_base; 167 u32 ring_size_bytes = 168 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); 169 170 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); 171 ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev), 172 ring_size_bytes, &ring->dma_addr, 173 GFP_KERNEL); 174 if (!ring->base_addr) 175 return -ENOMEM; 176 177 memset(ring->base_addr, 0x7F, ring_size_bytes); 178 /* The base_addr has to be aligned to the size of the buffer */ 179 if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { 180 dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n"); 181 dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, 182 ring->base_addr, ring->dma_addr); 183 ring->base_addr = NULL; 184 return -EFAULT; 185 } 186 187 if (hw_data->tx_rings_mask & (1 << ring->ring_number)) 188 adf_configure_tx_ring(ring); 189 190 else 191 adf_configure_rx_ring(ring); 192 193 ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, 194 ring->ring_size); 195 196 csr_ops->write_csr_ring_base(ring->bank->csr_addr, 197 ring->bank->bank_number, ring->ring_number, 198 ring_base); 199 spin_lock_init(&ring->lock); 200 return 0; 201 } 202 203 static void adf_cleanup_ring(struct adf_etr_ring_data *ring) 204 { 205 u32 ring_size_bytes = 206 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); 207 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); 208 209 if (ring->base_addr) { 210 memset(ring->base_addr, 0x7F, ring_size_bytes); 211 dma_free_coherent(&GET_DEV(ring->bank->accel_dev), 212 ring_size_bytes, ring->base_addr, 213 ring->dma_addr); 214 } 215 } 216 217 int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, 218 u32 bank_num, u32 num_msgs, 219 u32 msg_size, const char *ring_name, 220 adf_callback_fn callback, int poll_mode, 221 struct adf_etr_ring_data **ring_ptr) 222 { 223 struct adf_etr_data *transport_data = accel_dev->transport; 224 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); 225 struct adf_etr_bank_data *bank; 226 struct adf_etr_ring_data *ring; 227 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 228 int max_inflights; 229 u32 ring_num; 230 int ret; 231 232 if (bank_num >= GET_MAX_BANKS(accel_dev)) { 233 dev_err(&GET_DEV(accel_dev), "Invalid bank number\n"); 234 return -EFAULT; 235 } 236 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { 237 dev_err(&GET_DEV(accel_dev), "Invalid msg size\n"); 238 return -EFAULT; 239 } 240 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), 241 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { 242 dev_err(&GET_DEV(accel_dev), 243 "Invalid ring size for given msg size\n"); 244 return -EFAULT; 245 } 246 if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { 247 dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n", 248 section, ring_name); 249 return -EFAULT; 250 } 251 if (kstrtouint(val, 10, &ring_num)) { 252 dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); 253 return -EFAULT; 254 } 255 if (ring_num >= num_rings_per_bank) { 256 dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); 257 return -EFAULT; 258 } 259 260 ring_num = array_index_nospec(ring_num, num_rings_per_bank); 261 bank = &transport_data->banks[bank_num]; 262 if (adf_reserve_ring(bank, ring_num)) { 263 dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", 264 ring_num, ring_name); 265 return -EFAULT; 266 } 267 ring = &bank->rings[ring_num]; 268 ring->ring_number = ring_num; 269 ring->bank = bank; 270 ring->callback = callback; 271 ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); 272 ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); 273 ring->head = 0; 274 ring->tail = 0; 275 max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); 276 ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD); 277 atomic_set(ring->inflights, 0); 278 ret = adf_init_ring(ring); 279 if (ret) 280 goto err; 281 282 /* Enable HW arbitration for the given ring */ 283 adf_update_ring_arb(ring); 284 285 if (adf_ring_debugfs_add(ring, ring_name)) { 286 dev_err(&GET_DEV(accel_dev), 287 "Couldn't add ring debugfs entry\n"); 288 ret = -EFAULT; 289 goto err; 290 } 291 292 /* Enable interrupts if needed */ 293 if (callback && (!poll_mode)) 294 adf_enable_ring_irq(bank, ring->ring_number); 295 *ring_ptr = ring; 296 return 0; 297 err: 298 adf_cleanup_ring(ring); 299 adf_unreserve_ring(bank, ring_num); 300 adf_update_ring_arb(ring); 301 return ret; 302 } 303 304 void adf_remove_ring(struct adf_etr_ring_data *ring) 305 { 306 struct adf_etr_bank_data *bank = ring->bank; 307 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 308 309 /* Disable interrupts for the given ring */ 310 adf_disable_ring_irq(bank, ring->ring_number); 311 312 /* Clear PCI config space */ 313 314 csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, 315 ring->ring_number, 0); 316 csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, 317 ring->ring_number, 0); 318 adf_ring_debugfs_rm(ring); 319 adf_unreserve_ring(bank, ring->ring_number); 320 /* Disable HW arbitration for the given ring */ 321 adf_update_ring_arb(ring); 322 adf_cleanup_ring(ring); 323 } 324 325 static void adf_ring_response_handler(struct adf_etr_bank_data *bank) 326 { 327 struct adf_accel_dev *accel_dev = bank->accel_dev; 328 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); 329 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 330 unsigned long empty_rings; 331 int i; 332 333 empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, 334 bank->bank_number); 335 empty_rings = ~empty_rings & bank->irq_mask; 336 337 for_each_set_bit(i, &empty_rings, num_rings_per_bank) 338 adf_handle_response(&bank->rings[i]); 339 } 340 341 void adf_response_handler(uintptr_t bank_addr) 342 { 343 struct adf_etr_bank_data *bank = (void *)bank_addr; 344 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 345 346 /* Handle all the responses and reenable IRQs */ 347 adf_ring_response_handler(bank); 348 349 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, 350 bank->irq_mask); 351 } 352 353 static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, 354 const char *section, const char *format, 355 u32 key, u32 *value) 356 { 357 char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 358 char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 359 360 snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); 361 362 if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) 363 return -EFAULT; 364 365 if (kstrtouint(val_buf, 10, value)) 366 return -EFAULT; 367 return 0; 368 } 369 370 static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank, 371 const char *section, 372 u32 bank_num_in_accel) 373 { 374 if (adf_get_cfg_int(bank->accel_dev, section, 375 ADF_ETRMGR_COALESCE_TIMER_FORMAT, 376 bank_num_in_accel, &bank->irq_coalesc_timer)) 377 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; 378 379 if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer || 380 ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer) 381 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; 382 } 383 384 static int adf_init_bank(struct adf_accel_dev *accel_dev, 385 struct adf_etr_bank_data *bank, 386 u32 bank_num, void __iomem *csr_addr) 387 { 388 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 389 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 390 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; 391 u32 irq_mask = BIT(num_rings_per_bank) - 1; 392 struct adf_etr_ring_data *ring; 393 struct adf_etr_ring_data *tx_ring; 394 u32 i, coalesc_enabled = 0; 395 unsigned long ring_mask; 396 int size; 397 398 memset(bank, 0, sizeof(*bank)); 399 bank->bank_number = bank_num; 400 bank->csr_addr = csr_addr; 401 bank->accel_dev = accel_dev; 402 spin_lock_init(&bank->lock); 403 404 /* Allocate the rings in the bank */ 405 size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); 406 bank->rings = kzalloc_node(size, GFP_KERNEL, 407 dev_to_node(&GET_DEV(accel_dev))); 408 if (!bank->rings) 409 return -ENOMEM; 410 411 /* Enable IRQ coalescing always. This will allow to use 412 * the optimised flag and coalesc register. 413 * If it is disabled in the config file just use min time value */ 414 if ((adf_get_cfg_int(accel_dev, "Accelerator0", 415 ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num, 416 &coalesc_enabled) == 0) && coalesc_enabled) 417 adf_get_coalesc_timer(bank, "Accelerator0", bank_num); 418 else 419 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; 420 421 for (i = 0; i < num_rings_per_bank; i++) { 422 csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); 423 csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); 424 425 ring = &bank->rings[i]; 426 if (hw_data->tx_rings_mask & (1 << i)) { 427 ring->inflights = 428 kzalloc_node(sizeof(atomic_t), 429 GFP_KERNEL, 430 dev_to_node(&GET_DEV(accel_dev))); 431 if (!ring->inflights) 432 goto err; 433 } else { 434 if (i < hw_data->tx_rx_gap) { 435 dev_err(&GET_DEV(accel_dev), 436 "Invalid tx rings mask config\n"); 437 goto err; 438 } 439 tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; 440 ring->inflights = tx_ring->inflights; 441 } 442 } 443 if (adf_bank_debugfs_add(bank)) { 444 dev_err(&GET_DEV(accel_dev), 445 "Failed to add bank debugfs entry\n"); 446 goto err; 447 } 448 449 csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); 450 csr_ops->write_csr_int_srcsel(csr_addr, bank_num); 451 452 return 0; 453 err: 454 ring_mask = hw_data->tx_rings_mask; 455 for_each_set_bit(i, &ring_mask, num_rings_per_bank) { 456 ring = &bank->rings[i]; 457 kfree(ring->inflights); 458 ring->inflights = NULL; 459 } 460 kfree(bank->rings); 461 return -ENOMEM; 462 } 463 464 /** 465 * adf_init_etr_data() - Initialize transport rings for acceleration device 466 * @accel_dev: Pointer to acceleration device. 467 * 468 * Function is the initializes the communications channels (rings) to the 469 * acceleration device accel_dev. 470 * To be used by QAT device specific drivers. 471 * 472 * Return: 0 on success, error code otherwise. 473 */ 474 int adf_init_etr_data(struct adf_accel_dev *accel_dev) 475 { 476 struct adf_etr_data *etr_data; 477 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 478 void __iomem *csr_addr; 479 u32 size; 480 u32 num_banks = 0; 481 int i, ret; 482 483 etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL, 484 dev_to_node(&GET_DEV(accel_dev))); 485 if (!etr_data) 486 return -ENOMEM; 487 488 num_banks = GET_MAX_BANKS(accel_dev); 489 size = num_banks * sizeof(struct adf_etr_bank_data); 490 etr_data->banks = kzalloc_node(size, GFP_KERNEL, 491 dev_to_node(&GET_DEV(accel_dev))); 492 if (!etr_data->banks) { 493 ret = -ENOMEM; 494 goto err_bank; 495 } 496 497 accel_dev->transport = etr_data; 498 i = hw_data->get_etr_bar_id(hw_data); 499 csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; 500 501 /* accel_dev->debugfs_dir should always be non-NULL here */ 502 etr_data->debug = debugfs_create_dir("transport", 503 accel_dev->debugfs_dir); 504 505 for (i = 0; i < num_banks; i++) { 506 ret = adf_init_bank(accel_dev, &etr_data->banks[i], i, 507 csr_addr); 508 if (ret) 509 goto err_bank_all; 510 } 511 512 return 0; 513 514 err_bank_all: 515 debugfs_remove(etr_data->debug); 516 kfree(etr_data->banks); 517 err_bank: 518 kfree(etr_data); 519 accel_dev->transport = NULL; 520 return ret; 521 } 522 EXPORT_SYMBOL_GPL(adf_init_etr_data); 523 524 static void cleanup_bank(struct adf_etr_bank_data *bank) 525 { 526 struct adf_accel_dev *accel_dev = bank->accel_dev; 527 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 528 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 529 u32 i; 530 531 for (i = 0; i < num_rings_per_bank; i++) { 532 struct adf_etr_ring_data *ring = &bank->rings[i]; 533 534 if (bank->ring_mask & (1 << i)) 535 adf_cleanup_ring(ring); 536 537 if (hw_data->tx_rings_mask & (1 << i)) 538 kfree(ring->inflights); 539 } 540 kfree(bank->rings); 541 adf_bank_debugfs_rm(bank); 542 memset(bank, 0, sizeof(*bank)); 543 } 544 545 static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) 546 { 547 struct adf_etr_data *etr_data = accel_dev->transport; 548 u32 i, num_banks = GET_MAX_BANKS(accel_dev); 549 550 for (i = 0; i < num_banks; i++) 551 cleanup_bank(&etr_data->banks[i]); 552 } 553 554 /** 555 * adf_cleanup_etr_data() - Clear transport rings for acceleration device 556 * @accel_dev: Pointer to acceleration device. 557 * 558 * Function is the clears the communications channels (rings) of the 559 * acceleration device accel_dev. 560 * To be used by QAT device specific drivers. 561 * 562 * Return: void 563 */ 564 void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) 565 { 566 struct adf_etr_data *etr_data = accel_dev->transport; 567 568 if (etr_data) { 569 adf_cleanup_etr_handles(accel_dev); 570 debugfs_remove(etr_data->debug); 571 kfree(etr_data->banks->rings); 572 kfree(etr_data->banks); 573 kfree(etr_data); 574 accel_dev->transport = NULL; 575 } 576 } 577 EXPORT_SYMBOL_GPL(adf_cleanup_etr_data); 578