1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2025 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include <sys/kernel.h> 5 #include <sys/systm.h> 6 7 #include <sys/types.h> 8 #include <sys/interrupt.h> 9 #include <dev/pci/pcivar.h> 10 #include <sys/param.h> 11 #include <linux/workqueue.h> 12 #include "adf_accel_devices.h" 13 #include "adf_common_drv.h" 14 #include "adf_cfg.h" 15 #include "adf_cfg_strings.h" 16 #include "adf_cfg_common.h" 17 #include "adf_transport_access_macros.h" 18 #include "adf_transport_internal.h" 19 #include "adf_pfvf_utils.h" 20 #include "adf_pfvf_vf_msg.h" 21 22 static TASKQUEUE_DEFINE_THREAD(qat_vf); 23 static TASKQUEUE_DEFINE_THREAD(qat_bank_handler); 24 25 static struct workqueue_struct *adf_vf_stop_wq; 26 static DEFINE_MUTEX(vf_stop_wq_lock); 27 28 struct adf_vf_stop_data { 29 struct adf_accel_dev *accel_dev; 30 struct work_struct work; 31 }; 32 33 static int 34 adf_enable_msi(struct adf_accel_dev *accel_dev) 35 { 36 int stat; 37 int count = 1; 38 stat = pci_alloc_msi(accel_to_pci_dev(accel_dev), &count); 39 if (stat) { 40 device_printf(GET_DEV(accel_dev), 41 "Failed to enable MSI interrupts\n"); 42 return stat; 43 } 44 45 return stat; 46 } 47 48 static void 49 adf_disable_msi(struct adf_accel_dev *accel_dev) 50 { 51 device_t pdev = accel_to_pci_dev(accel_dev); 52 pci_release_msi(pdev); 53 } 54 55 static void 56 adf_dev_stop_async(struct work_struct *work) 57 { 58 struct adf_vf_stop_data *stop_data = 59 container_of(work, struct adf_vf_stop_data, work); 60 struct adf_accel_dev *accel_dev = stop_data->accel_dev; 61 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 62 63 adf_dev_restarting_notify(accel_dev); 64 adf_dev_stop(accel_dev); 65 adf_dev_shutdown(accel_dev); 66 67 /* Re-enable PF2VF interrupts */ 68 hw_data->enable_pf2vf_interrupt(accel_dev); 69 adf_vf2pf_restarting_complete(accel_dev); 70 kfree(stop_data); 71 } 72 73 int 74 adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev) 75 { 76 struct adf_vf_stop_data *stop_data; 77 78 clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); 79 stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC); 80 if (!stop_data) { 81 device_printf(GET_DEV(accel_dev), 82 "Couldn't schedule stop for vf_%d\n", 83 accel_dev->accel_id); 84 return -ENOMEM; 85 } 86 stop_data->accel_dev = accel_dev; 87 INIT_WORK(&stop_data->work, adf_dev_stop_async); 88 queue_work(adf_vf_stop_wq, &stop_data->work); 89 90 return 0; 91 } 92 93 int 94 adf_pf2vf_handle_pf_rp_reset(struct adf_accel_dev *accel_dev, 95 struct pfvf_message msg) 96 { 97 accel_dev->u1.vf.rpreset_sts = msg.data; 98 if (accel_dev->u1.vf.rpreset_sts == RPRESET_SUCCESS) 99 device_printf( 100 GET_DEV(accel_dev), 101 "rpreset resp(success) from PF type:0x%x data:0x%x\n", 102 msg.type, 103 msg.data); 104 else if (accel_dev->u1.vf.rpreset_sts == RPRESET_NOT_SUPPORTED) 105 device_printf( 106 GET_DEV(accel_dev), 107 "rpreset resp(not supported) from PF type:0x%x data:0x%x\n", 108 msg.type, 109 msg.data); 110 else if (accel_dev->u1.vf.rpreset_sts == RPRESET_INVAL_BANK) 111 device_printf( 112 GET_DEV(accel_dev), 113 "rpreset resp(invalid bank) from PF type:0x%x data:0x%x\n", 114 msg.type, 115 msg.data); 116 else 117 device_printf( 118 GET_DEV(accel_dev), 119 "rpreset resp(timeout) from PF type:0x%x data:0x%x\nn", 120 msg.type, 121 msg.data); 122 123 complete(&accel_dev->u1.vf.msg_received); 124 125 return 0; 126 } 127 128 int 129 adf_pf2vf_handle_pf_error(struct adf_accel_dev *accel_dev) 130 { 131 device_printf(GET_DEV(accel_dev), "Fatal error received from PF\n"); 132 133 if (adf_notify_fatal_error(accel_dev)) 134 device_printf(GET_DEV(accel_dev), "Couldn't notify fatal error\n"); 135 136 return 0; 137 } 138 139 static void 140 adf_pf2vf_bh_handler(void *data, int pending) 141 { 142 struct adf_accel_dev *accel_dev = data; 143 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 144 145 if (adf_recv_and_handle_pf2vf_msg(accel_dev)) 146 /* Re-enable PF2VF interrupts */ 147 hw_data->enable_pf2vf_interrupt(accel_dev); 148 149 return; 150 } 151 152 static int 153 adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) 154 { 155 TASK_INIT(&accel_dev->u1.vf.pf2vf_bh_tasklet, 156 0, 157 adf_pf2vf_bh_handler, 158 accel_dev); 159 mutex_init(&accel_dev->u1.vf.vf2pf_lock); 160 161 return 0; 162 } 163 164 static void 165 adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) 166 { 167 taskqueue_cancel(taskqueue_qat_vf, 168 &accel_dev->u1.vf.pf2vf_bh_tasklet, 169 NULL); 170 taskqueue_drain(taskqueue_qat_vf, &accel_dev->u1.vf.pf2vf_bh_tasklet); 171 mutex_destroy(&accel_dev->u1.vf.vf2pf_lock); 172 } 173 174 static void 175 adf_bh_handler(void *data, int pending) 176 { 177 struct adf_etr_bank_data *bank = (void *)data; 178 179 adf_response_handler((uintptr_t)bank); 180 181 return; 182 } 183 184 static int 185 adf_setup_bh(struct adf_accel_dev *accel_dev) 186 { 187 int i = 0; 188 struct adf_etr_data *priv_data = accel_dev->transport; 189 190 for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) { 191 TASK_INIT(&priv_data->banks[i].resp_handler, 192 0, 193 adf_bh_handler, 194 &priv_data->banks[i]); 195 } 196 197 return 0; 198 } 199 200 static void 201 adf_cleanup_bh(struct adf_accel_dev *accel_dev) 202 { 203 int i = 0; 204 struct adf_etr_data *transport; 205 206 if (!accel_dev || !accel_dev->transport) 207 return; 208 209 transport = accel_dev->transport; 210 for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) { 211 taskqueue_cancel(taskqueue_qat_bank_handler, 212 &transport->banks[i].resp_handler, 213 NULL); 214 taskqueue_drain(taskqueue_qat_bank_handler, 215 &transport->banks[i].resp_handler); 216 } 217 } 218 219 static void 220 adf_isr(void *privdata) 221 { 222 struct adf_accel_dev *accel_dev = privdata; 223 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 224 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; 225 int int_active_bundles = 0; 226 int i = 0; 227 228 /* Check for PF2VF interrupt */ 229 if (hw_data->interrupt_active_pf2vf(accel_dev)) { 230 /* Disable PF to VF interrupt */ 231 hw_data->disable_pf2vf_interrupt(accel_dev); 232 /* Schedule tasklet to handle interrupt BH */ 233 taskqueue_enqueue(taskqueue_qat_vf, 234 &accel_dev->u1.vf.pf2vf_bh_tasklet); 235 } 236 237 if (hw_data->get_int_active_bundles) 238 int_active_bundles = hw_data->get_int_active_bundles(accel_dev); 239 240 for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) { 241 if (int_active_bundles & BIT(i)) { 242 struct adf_etr_data *etr_data = accel_dev->transport; 243 struct adf_etr_bank_data *bank = &etr_data->banks[i]; 244 245 /* Disable Flag and Coalesce Ring Interrupts */ 246 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, 247 bank->bank_number, 248 0); 249 /* Schedule tasklet to handle interrupt BH */ 250 taskqueue_enqueue(taskqueue_qat_bank_handler, 251 &bank->resp_handler); 252 } 253 } 254 } 255 256 static int 257 adf_request_msi_irq(struct adf_accel_dev *accel_dev) 258 { 259 device_t pdev = accel_to_pci_dev(accel_dev); 260 int ret; 261 int rid = 1; 262 int cpu; 263 264 accel_dev->u1.vf.irq = 265 bus_alloc_resource_any(pdev, SYS_RES_IRQ, &rid, RF_ACTIVE); 266 if (accel_dev->u1.vf.irq == NULL) { 267 device_printf(GET_DEV(accel_dev), "failed to allocate IRQ\n"); 268 return ENXIO; 269 } 270 ret = bus_setup_intr(pdev, 271 accel_dev->u1.vf.irq, 272 INTR_TYPE_MISC | INTR_MPSAFE, 273 NULL, 274 adf_isr, 275 accel_dev, 276 &accel_dev->u1.vf.cookie); 277 if (ret) { 278 device_printf(GET_DEV(accel_dev), "failed to enable irq\n"); 279 goto errout; 280 } 281 282 cpu = accel_dev->accel_id % num_online_cpus(); 283 ret = bus_bind_intr(pdev, accel_dev->u1.vf.irq, cpu); 284 if (ret) { 285 device_printf(GET_DEV(accel_dev), 286 "failed to bind IRQ handler to cpu core\n"); 287 goto errout; 288 } 289 accel_dev->u1.vf.irq_enabled = true; 290 291 return ret; 292 errout: 293 bus_free_resource(pdev, SYS_RES_IRQ, accel_dev->u1.vf.irq); 294 295 return ret; 296 } 297 298 /** 299 * adf_vf_isr_resource_free() - Free IRQ for acceleration device 300 * @accel_dev: Pointer to acceleration device. 301 * 302 * Function frees interrupts for acceleration device virtual function. 303 */ 304 void 305 adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) 306 { 307 device_t pdev = accel_to_pci_dev(accel_dev); 308 309 if (accel_dev->u1.vf.irq_enabled) { 310 bus_teardown_intr(pdev, 311 accel_dev->u1.vf.irq, 312 accel_dev->u1.vf.cookie); 313 bus_free_resource(pdev, SYS_RES_IRQ, accel_dev->u1.vf.irq); 314 } 315 adf_cleanup_bh(accel_dev); 316 adf_cleanup_pf2vf_bh(accel_dev); 317 adf_disable_msi(accel_dev); 318 } 319 320 /** 321 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device 322 * @accel_dev: Pointer to acceleration device. 323 * 324 * Function allocates interrupts for acceleration device virtual function. 325 * 326 * Return: 0 on success, error code otherwise. 327 */ 328 int 329 adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) 330 { 331 if (adf_enable_msi(accel_dev)) 332 goto err_out; 333 334 if (adf_setup_pf2vf_bh(accel_dev)) 335 goto err_disable_msi; 336 337 if (adf_setup_bh(accel_dev)) 338 goto err_out; 339 340 if (adf_request_msi_irq(accel_dev)) 341 goto err_disable_msi; 342 343 return 0; 344 345 err_disable_msi: 346 adf_disable_msi(accel_dev); 347 348 err_out: 349 return -EFAULT; 350 } 351 352 /** 353 * adf_flush_vf_wq() - Flush workqueue for VF 354 * @accel_dev: Pointer to acceleration device. 355 * 356 * Function disables the PF/VF interrupts on the VF so that no new messages 357 * are received and flushes the workqueue 'adf_vf_stop_wq'. 358 * 359 * Return: void. 360 */ 361 void 362 adf_flush_vf_wq(struct adf_accel_dev *accel_dev) 363 { 364 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 365 366 hw_data->disable_pf2vf_interrupt(accel_dev); 367 368 if (adf_vf_stop_wq) 369 flush_workqueue(adf_vf_stop_wq); 370 } 371 372 /** 373 * adf_init_vf_wq() - Init workqueue for VF 374 * 375 * Function init workqueue 'adf_vf_stop_wq' for VF. 376 * 377 * Return: 0 on success, error code otherwise. 378 */ 379 int 380 adf_init_vf_wq(void) 381 { 382 int ret = 0; 383 384 mutex_lock(&vf_stop_wq_lock); 385 if (!adf_vf_stop_wq) 386 adf_vf_stop_wq = 387 alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); 388 389 if (!adf_vf_stop_wq) 390 ret = ENOMEM; 391 392 mutex_unlock(&vf_stop_wq_lock); 393 return ret; 394 } 395 396 void 397 adf_exit_vf_wq(void) 398 { 399 if (adf_vf_stop_wq) 400 destroy_workqueue(adf_vf_stop_wq); 401 402 adf_vf_stop_wq = NULL; 403 } 404