1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/kernel.h> 4 #include <linux/init.h> 5 #include <linux/types.h> 6 #include <linux/pci.h> 7 #include <linux/slab.h> 8 #include <linux/errno.h> 9 #include <linux/interrupt.h> 10 #include "adf_accel_devices.h" 11 #include "adf_common_drv.h" 12 #include "adf_cfg.h" 13 #include "adf_cfg_strings.h" 14 #include "adf_cfg_common.h" 15 #include "adf_transport_access_macros.h" 16 #include "adf_transport_internal.h" 17 18 #define ADF_MAX_NUM_VFS 32 19 static struct workqueue_struct *adf_misc_wq; 20 21 static int adf_enable_msix(struct adf_accel_dev *accel_dev) 22 { 23 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 24 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 25 u32 msix_num_entries = hw_data->num_banks + 1; 26 int ret; 27 28 if (hw_data->set_msix_rttable) 29 hw_data->set_msix_rttable(accel_dev); 30 31 ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries, 32 msix_num_entries, PCI_IRQ_MSIX); 33 if (unlikely(ret < 0)) { 34 dev_err(&GET_DEV(accel_dev), 35 "Failed to allocate %d MSI-X vectors\n", 36 msix_num_entries); 37 return ret; 38 } 39 return 0; 40 } 41 42 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) 43 { 44 pci_free_irq_vectors(pci_dev_info->pci_dev); 45 } 46 47 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) 48 { 49 struct adf_etr_bank_data *bank = bank_ptr; 50 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); 51 52 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, 53 0); 54 tasklet_hi_schedule(&bank->resp_handler); 55 return IRQ_HANDLED; 56 } 57 58 #ifdef CONFIG_PCI_IOV 59 void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) 60 { 61 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 62 unsigned long flags; 63 64 spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); 65 if (!READ_ONCE(accel_dev->pf.vf2pf_disabled)) 66 GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); 67 spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); 68 } 69 70 void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs) 71 { 72 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 73 unsigned long flags; 74 u32 vf_mask; 75 76 vf_mask = BIT_ULL(num_vfs) - 1; 77 if (!vf_mask) 78 return; 79 80 spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); 81 WRITE_ONCE(accel_dev->pf.vf2pf_disabled, false); 82 GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); 83 spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); 84 } 85 86 void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev) 87 { 88 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 89 unsigned long flags; 90 91 spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); 92 WRITE_ONCE(accel_dev->pf.vf2pf_disabled, true); 93 GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr); 94 spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); 95 } 96 97 static u32 adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev *accel_dev) 98 { 99 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 100 u32 pending; 101 102 spin_lock(&accel_dev->pf.vf2pf_ints_lock); 103 pending = GET_PFVF_OPS(accel_dev)->disable_pending_vf2pf_interrupts(pmisc_addr); 104 spin_unlock(&accel_dev->pf.vf2pf_ints_lock); 105 106 return pending; 107 } 108 109 static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) 110 { 111 bool irq_handled = false; 112 unsigned long vf_mask; 113 114 /* Get the interrupt sources triggered by VFs, except for those already disabled */ 115 vf_mask = adf_disable_pending_vf2pf_interrupts(accel_dev); 116 if (vf_mask) { 117 struct adf_accel_vf_info *vf_info; 118 int i; 119 120 /* 121 * Handle VF2PF interrupt unless the VF is malicious and 122 * is attempting to flood the host OS with VF2PF interrupts. 123 */ 124 for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { 125 vf_info = accel_dev->pf.vf_info + i; 126 127 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { 128 dev_info(&GET_DEV(accel_dev), 129 "Too many ints from VF%d\n", 130 vf_info->vf_nr); 131 continue; 132 } 133 134 adf_schedule_vf2pf_handler(vf_info); 135 irq_handled = true; 136 } 137 } 138 return irq_handled; 139 } 140 #endif /* CONFIG_PCI_IOV */ 141 142 static bool adf_handle_pm_int(struct adf_accel_dev *accel_dev) 143 { 144 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 145 146 if (hw_data->handle_pm_interrupt && 147 hw_data->handle_pm_interrupt(accel_dev)) 148 return true; 149 150 return false; 151 } 152 153 static bool adf_handle_ras_int(struct adf_accel_dev *accel_dev) 154 { 155 struct adf_ras_ops *ras_ops = &accel_dev->hw_device->ras_ops; 156 bool reset_required; 157 158 if (ras_ops->handle_interrupt && 159 ras_ops->handle_interrupt(accel_dev, &reset_required)) { 160 if (reset_required) { 161 dev_err(&GET_DEV(accel_dev), "Fatal error, reset required\n"); 162 if (adf_notify_fatal_error(accel_dev)) 163 dev_err(&GET_DEV(accel_dev), 164 "Failed to notify fatal error\n"); 165 } 166 167 return true; 168 } 169 170 return false; 171 } 172 173 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) 174 { 175 struct adf_accel_dev *accel_dev = dev_ptr; 176 177 #ifdef CONFIG_PCI_IOV 178 /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ 179 if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev)) 180 return IRQ_HANDLED; 181 #endif /* CONFIG_PCI_IOV */ 182 183 if (adf_handle_pm_int(accel_dev)) 184 return IRQ_HANDLED; 185 186 if (adf_handle_ras_int(accel_dev)) 187 return IRQ_HANDLED; 188 189 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", 190 accel_dev->accel_id); 191 192 return IRQ_NONE; 193 } 194 195 void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev) 196 { 197 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 198 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 199 u32 num_entries = pci_dev_info->msix_entries.num_entries; 200 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; 201 u32 irq_idx; 202 int irq; 203 204 if (!test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status) || !irqs) 205 return; 206 207 irq_idx = num_entries > 1 ? hw_data->num_banks : 0; 208 if (irq_idx >= num_entries || !irqs[irq_idx].enabled) 209 return; 210 211 irq = pci_irq_vector(pci_dev_info->pci_dev, hw_data->num_banks); 212 if (irq > 0) 213 synchronize_irq(irq); 214 } 215 216 static void adf_free_irqs(struct adf_accel_dev *accel_dev) 217 { 218 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 219 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 220 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; 221 struct adf_etr_data *etr_data = accel_dev->transport; 222 int clust_irq = hw_data->num_banks; 223 int irq, i = 0; 224 225 if (pci_dev_info->msix_entries.num_entries > 1) { 226 for (i = 0; i < hw_data->num_banks; i++) { 227 if (irqs[i].enabled) { 228 irq = pci_irq_vector(pci_dev_info->pci_dev, i); 229 irq_set_affinity_hint(irq, NULL); 230 free_irq(irq, &etr_data->banks[i]); 231 } 232 } 233 } 234 235 if (irqs[i].enabled) { 236 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); 237 free_irq(irq, accel_dev); 238 } 239 } 240 241 static int adf_request_irqs(struct adf_accel_dev *accel_dev) 242 { 243 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 244 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 245 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; 246 struct adf_etr_data *etr_data = accel_dev->transport; 247 int clust_irq = hw_data->num_banks; 248 int ret, irq, i = 0; 249 char *name; 250 251 /* Request msix irq for all banks unless SR-IOV enabled */ 252 if (!accel_dev->pf.vf_info) { 253 for (i = 0; i < hw_data->num_banks; i++) { 254 struct adf_etr_bank_data *bank = &etr_data->banks[i]; 255 unsigned int cpu, cpus = num_online_cpus(); 256 257 name = irqs[i].name; 258 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, 259 "qat%d-bundle%d", accel_dev->accel_id, i); 260 irq = pci_irq_vector(pci_dev_info->pci_dev, i); 261 if (unlikely(irq < 0)) { 262 dev_err(&GET_DEV(accel_dev), 263 "Failed to get IRQ number of device vector %d - %s\n", 264 i, name); 265 ret = irq; 266 goto err; 267 } 268 ret = request_irq(irq, adf_msix_isr_bundle, 0, 269 &name[0], bank); 270 if (ret) { 271 dev_err(&GET_DEV(accel_dev), 272 "Failed to allocate IRQ %d for %s\n", 273 irq, name); 274 goto err; 275 } 276 277 cpu = ((accel_dev->accel_id * hw_data->num_banks) + 278 i) % cpus; 279 irq_set_affinity_hint(irq, get_cpu_mask(cpu)); 280 irqs[i].enabled = true; 281 } 282 } 283 284 /* Request msix irq for AE */ 285 name = irqs[i].name; 286 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, 287 "qat%d-ae-cluster", accel_dev->accel_id); 288 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); 289 if (unlikely(irq < 0)) { 290 dev_err(&GET_DEV(accel_dev), 291 "Failed to get IRQ number of device vector %d - %s\n", 292 i, name); 293 ret = irq; 294 goto err; 295 } 296 ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev); 297 if (ret) { 298 dev_err(&GET_DEV(accel_dev), 299 "Failed to allocate IRQ %d for %s\n", irq, name); 300 goto err; 301 } 302 irqs[i].enabled = true; 303 return ret; 304 err: 305 adf_free_irqs(accel_dev); 306 return ret; 307 } 308 309 static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) 310 { 311 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 312 u32 msix_num_entries = 1; 313 struct adf_irq *irqs; 314 315 /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ 316 if (!accel_dev->pf.vf_info) 317 msix_num_entries += hw_data->num_banks; 318 319 irqs = kcalloc_node(msix_num_entries, sizeof(*irqs), 320 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); 321 if (!irqs) 322 return -ENOMEM; 323 324 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; 325 accel_dev->accel_pci_dev.msix_entries.irqs = irqs; 326 return 0; 327 } 328 329 static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev) 330 { 331 kfree(accel_dev->accel_pci_dev.msix_entries.irqs); 332 accel_dev->accel_pci_dev.msix_entries.irqs = NULL; 333 } 334 335 static int adf_setup_bh(struct adf_accel_dev *accel_dev) 336 { 337 struct adf_etr_data *priv_data = accel_dev->transport; 338 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 339 int i; 340 341 for (i = 0; i < hw_data->num_banks; i++) 342 tasklet_init(&priv_data->banks[i].resp_handler, 343 adf_response_handler, 344 (unsigned long)&priv_data->banks[i]); 345 return 0; 346 } 347 348 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) 349 { 350 struct adf_etr_data *priv_data = accel_dev->transport; 351 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 352 int i; 353 354 for (i = 0; i < hw_data->num_banks; i++) { 355 tasklet_disable(&priv_data->banks[i].resp_handler); 356 tasklet_kill(&priv_data->banks[i].resp_handler); 357 } 358 } 359 360 /** 361 * adf_isr_resource_free() - Free IRQ for acceleration device 362 * @accel_dev: Pointer to acceleration device. 363 * 364 * Function frees interrupts for acceleration device. 365 */ 366 void adf_isr_resource_free(struct adf_accel_dev *accel_dev) 367 { 368 adf_free_irqs(accel_dev); 369 adf_cleanup_bh(accel_dev); 370 adf_disable_msix(&accel_dev->accel_pci_dev); 371 adf_isr_free_msix_vectors_data(accel_dev); 372 } 373 EXPORT_SYMBOL_GPL(adf_isr_resource_free); 374 375 /** 376 * adf_isr_resource_alloc() - Allocate IRQ for acceleration device 377 * @accel_dev: Pointer to acceleration device. 378 * 379 * Function allocates interrupts for acceleration device. 380 * 381 * Return: 0 on success, error code otherwise. 382 */ 383 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) 384 { 385 int ret; 386 387 ret = adf_isr_alloc_msix_vectors_data(accel_dev); 388 if (ret) 389 goto err_out; 390 391 ret = adf_enable_msix(accel_dev); 392 if (ret) 393 goto err_free_msix_table; 394 395 ret = adf_setup_bh(accel_dev); 396 if (ret) 397 goto err_disable_msix; 398 399 ret = adf_request_irqs(accel_dev); 400 if (ret) 401 goto err_cleanup_bh; 402 403 return 0; 404 405 err_cleanup_bh: 406 adf_cleanup_bh(accel_dev); 407 408 err_disable_msix: 409 adf_disable_msix(&accel_dev->accel_pci_dev); 410 411 err_free_msix_table: 412 adf_isr_free_msix_vectors_data(accel_dev); 413 414 err_out: 415 return ret; 416 } 417 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); 418 419 /** 420 * adf_init_misc_wq() - Init misc workqueue 421 * 422 * Return: 0 on success, error code otherwise. 423 */ 424 int __init adf_init_misc_wq(void) 425 { 426 adf_misc_wq = alloc_workqueue("qat_misc_wq", 427 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 428 429 return !adf_misc_wq ? -ENOMEM : 0; 430 } 431 432 void adf_exit_misc_wq(void) 433 { 434 if (adf_misc_wq) 435 destroy_workqueue(adf_misc_wq); 436 437 adf_misc_wq = NULL; 438 } 439 440 bool adf_misc_wq_queue_work(struct work_struct *work) 441 { 442 return queue_work(adf_misc_wq, work); 443 } 444 445 bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, 446 unsigned long delay) 447 { 448 return queue_delayed_work(adf_misc_wq, work, delay); 449 } 450 451 void adf_misc_wq_flush(void) 452 { 453 flush_workqueue(adf_misc_wq); 454 } 455