1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_transport_access_macros.h" 12 #include "adf_transport_internal.h" 13 #include <sys/bus.h> 14 #include <dev/pci/pcireg.h> 15 #include <dev/pci/pcivar.h> 16 #include <sys/systm.h> 17 18 #define ADF_PPAERUCM_MASK (BIT(14) | BIT(20) | BIT(22)) 19 20 static struct workqueue_struct *fatal_error_wq; 21 struct adf_fatal_error_data { 22 struct adf_accel_dev *accel_dev; 23 struct work_struct work; 24 }; 25 26 static struct workqueue_struct *device_reset_wq; 27 28 void 29 linux_complete_common(struct completion *c, int all) 30 { 31 sleepq_lock(c); 32 c->done++; 33 if (all) 34 sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 35 else 36 sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 37 sleepq_release(c); 38 } 39 40 /* reset dev data */ 41 struct adf_reset_dev_data { 42 int mode; 43 struct adf_accel_dev *accel_dev; 44 struct completion compl; 45 struct work_struct reset_work; 46 }; 47 48 int 49 adf_aer_store_ppaerucm_reg(device_t dev, struct adf_hw_device_data *hw_data) 50 { 51 unsigned int aer_offset, reg_val = 0; 52 53 if (!hw_data) 54 return -EINVAL; 55 56 if (pci_find_extcap(dev, PCIZ_AER, &aer_offset) == 0) { 57 reg_val = 58 pci_read_config(dev, aer_offset + PCIR_AER_UC_MASK, 4); 59 60 hw_data->aerucm_mask = reg_val; 61 } else { 62 device_printf(dev, 63 "Unable to find AER capability of the device\n"); 64 return -ENODEV; 65 } 66 67 return 0; 68 } 69 70 void 71 adf_reset_sbr(struct adf_accel_dev *accel_dev) 72 { 73 device_t pdev = accel_to_pci_dev(accel_dev); 74 device_t parent = device_get_parent(device_get_parent(pdev)); 75 uint16_t bridge_ctl = 0; 76 77 if (accel_dev->is_vf) 78 return; 79 80 if (!parent) 81 parent = pdev; 82 83 if (!pcie_wait_for_pending_transactions(pdev, 0)) 84 device_printf(GET_DEV(accel_dev), 85 "Transaction still in progress. Proceeding\n"); 86 87 device_printf(GET_DEV(accel_dev), "Secondary bus reset\n"); 88 89 pci_save_state(pdev); 90 bridge_ctl = pci_read_config(parent, PCIR_BRIDGECTL_1, 2); 91 bridge_ctl |= PCIB_BCR_SECBUS_RESET; 92 pci_write_config(parent, PCIR_BRIDGECTL_1, bridge_ctl, 2); 93 pause_ms("adfrst", 100); 94 bridge_ctl &= ~PCIB_BCR_SECBUS_RESET; 95 pci_write_config(parent, PCIR_BRIDGECTL_1, bridge_ctl, 2); 96 pause_ms("adfrst", 100); 97 pci_restore_state(pdev); 98 } 99 100 void 101 adf_reset_flr(struct adf_accel_dev *accel_dev) 102 { 103 device_t pdev = accel_to_pci_dev(accel_dev); 104 105 pci_save_state(pdev); 106 if (pcie_flr(pdev, 107 max(pcie_get_max_completion_timeout(pdev) / 1000, 10), 108 true)) { 109 pci_restore_state(pdev); 110 return; 111 } 112 pci_restore_state(pdev); 113 device_printf(GET_DEV(accel_dev), 114 "FLR qat_dev%d failed trying secondary bus reset\n", 115 accel_dev->accel_id); 116 adf_reset_sbr(accel_dev); 117 } 118 119 void 120 adf_dev_pre_reset(struct adf_accel_dev *accel_dev) 121 { 122 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 123 device_t pdev = accel_to_pci_dev(accel_dev); 124 u32 aer_offset, reg_val = 0; 125 126 if (pci_find_extcap(pdev, PCIZ_AER, &aer_offset) == 0) { 127 reg_val = 128 pci_read_config(pdev, aer_offset + PCIR_AER_UC_MASK, 4); 129 reg_val |= ADF_PPAERUCM_MASK; 130 pci_write_config(pdev, 131 aer_offset + PCIR_AER_UC_MASK, 132 reg_val, 133 4); 134 } else { 135 device_printf(pdev, 136 "Unable to find AER capability of the device\n"); 137 } 138 139 if (hw_device->disable_arb) { 140 device_printf(GET_DEV(accel_dev), "Disable arbiter.\n"); 141 hw_device->disable_arb(accel_dev); 142 } 143 } 144 145 void 146 adf_dev_post_reset(struct adf_accel_dev *accel_dev) 147 { 148 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 149 device_t pdev = accel_to_pci_dev(accel_dev); 150 u32 aer_offset; 151 152 if (pci_find_extcap(pdev, PCIZ_AER, &aer_offset) == 0) { 153 pci_write_config(pdev, 154 aer_offset + PCIR_AER_UC_MASK, 155 hw_device->aerucm_mask, 156 4); 157 } else { 158 device_printf(pdev, 159 "Unable to find AER capability of the device\n"); 160 } 161 } 162 163 void 164 adf_dev_restore(struct adf_accel_dev *accel_dev) 165 { 166 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 167 device_t pdev = accel_to_pci_dev(accel_dev); 168 169 if (hw_device->pre_reset) { 170 dev_dbg(GET_DEV(accel_dev), "Performing pre reset save\n"); 171 hw_device->pre_reset(accel_dev); 172 } 173 174 if (hw_device->reset_device) { 175 device_printf(GET_DEV(accel_dev), 176 "Resetting device qat_dev%d\n", 177 accel_dev->accel_id); 178 hw_device->reset_device(accel_dev); 179 pci_restore_state(pdev); 180 pci_save_state(pdev); 181 } 182 183 if (hw_device->post_reset) { 184 dev_dbg(GET_DEV(accel_dev), "Performing post reset restore\n"); 185 hw_device->post_reset(accel_dev); 186 } 187 } 188 189 static void 190 adf_device_reset_worker(struct work_struct *work) 191 { 192 struct adf_reset_dev_data *reset_data = 193 container_of(work, struct adf_reset_dev_data, reset_work); 194 struct adf_accel_dev *accel_dev = reset_data->accel_dev; 195 196 if (adf_dev_restarting_notify(accel_dev)) { 197 device_printf(GET_DEV(accel_dev), 198 "Unable to send RESTARTING notification.\n"); 199 return; 200 } 201 202 if (adf_dev_stop(accel_dev)) { 203 device_printf(GET_DEV(accel_dev), "Stopping device failed.\n"); 204 return; 205 } 206 207 adf_dev_shutdown(accel_dev); 208 209 if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) { 210 /* The device hanged and we can't restart it */ 211 /* so stop here */ 212 device_printf(GET_DEV(accel_dev), "Restart device failed\n"); 213 if (reset_data->mode == ADF_DEV_RESET_ASYNC) 214 kfree(reset_data); 215 WARN(1, "QAT: device restart failed. Device is unusable\n"); 216 return; 217 } 218 219 adf_dev_restarted_notify(accel_dev); 220 clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); 221 222 /* The dev is back alive. Notify the caller if in sync mode */ 223 if (reset_data->mode == ADF_DEV_RESET_SYNC) 224 complete(&reset_data->compl); 225 else 226 kfree(reset_data); 227 } 228 229 int 230 adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, 231 enum adf_dev_reset_mode mode) 232 { 233 struct adf_reset_dev_data *reset_data; 234 if (!adf_dev_started(accel_dev) || 235 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) 236 return 0; 237 set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); 238 reset_data = kzalloc(sizeof(*reset_data), GFP_ATOMIC); 239 if (!reset_data) 240 return -ENOMEM; 241 reset_data->accel_dev = accel_dev; 242 init_completion(&reset_data->compl); 243 reset_data->mode = mode; 244 INIT_WORK(&reset_data->reset_work, adf_device_reset_worker); 245 queue_work(device_reset_wq, &reset_data->reset_work); 246 /* If in sync mode wait for the result */ 247 if (mode == ADF_DEV_RESET_SYNC) { 248 int ret = 0; 249 /* Maximum device reset time is 10 seconds */ 250 unsigned long wait_jiffies = msecs_to_jiffies(10000); 251 unsigned long timeout = 252 wait_for_completion_timeout(&reset_data->compl, 253 wait_jiffies); 254 if (!timeout) { 255 device_printf(GET_DEV(accel_dev), 256 "Reset device timeout expired\n"); 257 ret = -EFAULT; 258 } 259 kfree(reset_data); 260 return ret; 261 } 262 return 0; 263 } 264 265 int 266 adf_dev_autoreset(struct adf_accel_dev *accel_dev) 267 { 268 if (accel_dev->autoreset_on_error) 269 return adf_dev_reset(accel_dev, ADF_DEV_RESET_ASYNC); 270 return 0; 271 } 272 273 static void 274 adf_notify_fatal_error_work(struct work_struct *work) 275 { 276 struct adf_fatal_error_data *wq_data = 277 container_of(work, struct adf_fatal_error_data, work); 278 struct adf_accel_dev *accel_dev = wq_data->accel_dev; 279 280 adf_error_notifier((uintptr_t)accel_dev); 281 if (!accel_dev->is_vf) { 282 adf_dev_autoreset(accel_dev); 283 } 284 285 kfree(wq_data); 286 } 287 288 int 289 adf_notify_fatal_error(struct adf_accel_dev *accel_dev) 290 { 291 struct adf_fatal_error_data *wq_data; 292 293 wq_data = kzalloc(sizeof(*wq_data), GFP_ATOMIC); 294 if (!wq_data) { 295 device_printf(GET_DEV(accel_dev), 296 "Failed to allocate memory\n"); 297 return ENOMEM; 298 } 299 wq_data->accel_dev = accel_dev; 300 301 INIT_WORK(&wq_data->work, adf_notify_fatal_error_work); 302 queue_work(fatal_error_wq, &wq_data->work); 303 304 return 0; 305 } 306 307 int __init 308 adf_init_fatal_error_wq(void) 309 { 310 fatal_error_wq = create_workqueue("qat_fatal_error_wq"); 311 return !fatal_error_wq ? EFAULT : 0; 312 } 313 314 void 315 adf_exit_fatal_error_wq(void) 316 { 317 if (fatal_error_wq) 318 destroy_workqueue(fatal_error_wq); 319 fatal_error_wq = NULL; 320 } 321 322 int 323 adf_init_aer(void) 324 { 325 device_reset_wq = create_workqueue("qat_device_reset_wq"); 326 return !device_reset_wq ? -EFAULT : 0; 327 } 328 329 void 330 adf_exit_aer(void) 331 { 332 if (device_reset_wq) 333 destroy_workqueue(device_reset_wq); 334 device_reset_wq = NULL; 335 } 336