1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3 #include "qat_freebsd.h"
4 #include "adf_cfg.h"
5 #include "adf_common_drv.h"
6 #include "adf_accel_devices.h"
7 #include "icp_qat_uclo.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_init_admin.h"
10 #include "adf_cfg_strings.h"
11 #include "adf_transport_access_macros.h"
12 #include "adf_transport_internal.h"
13 #include <sys/bus.h>
14 #include <dev/pci/pcireg.h>
15 #include <dev/pci/pcivar.h>
16 #include <sys/systm.h>
17
18 #define ADF_PPAERUCM_MASK (BIT(14) | BIT(20) | BIT(22))
19
20 static struct workqueue_struct *fatal_error_wq;
21 struct adf_fatal_error_data {
22 struct adf_accel_dev *accel_dev;
23 struct work_struct work;
24 };
25
26 static struct workqueue_struct *device_reset_wq;
27
28 void
linux_complete_common(struct completion * c,int all)29 linux_complete_common(struct completion *c, int all)
30 {
31 sleepq_lock(c);
32 c->done++;
33 if (all)
34 sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0);
35 else
36 sleepq_signal(c, SLEEPQ_SLEEP, 0, 0);
37 sleepq_release(c);
38 }
39
40 /* reset dev data */
41 struct adf_reset_dev_data {
42 int mode;
43 struct adf_accel_dev *accel_dev;
44 struct completion compl;
45 struct work_struct reset_work;
46 };
47
48 int
adf_aer_store_ppaerucm_reg(device_t dev,struct adf_hw_device_data * hw_data)49 adf_aer_store_ppaerucm_reg(device_t dev, struct adf_hw_device_data *hw_data)
50 {
51 unsigned int aer_offset, reg_val = 0;
52
53 if (!hw_data)
54 return -EINVAL;
55
56 if (pci_find_extcap(dev, PCIZ_AER, &aer_offset) == 0) {
57 reg_val =
58 pci_read_config(dev, aer_offset + PCIR_AER_UC_MASK, 4);
59
60 hw_data->aerucm_mask = reg_val;
61 } else {
62 device_printf(dev,
63 "Unable to find AER capability of the device\n");
64 return -ENODEV;
65 }
66
67 return 0;
68 }
69
70 void
adf_reset_sbr(struct adf_accel_dev * accel_dev)71 adf_reset_sbr(struct adf_accel_dev *accel_dev)
72 {
73 device_t pdev = accel_to_pci_dev(accel_dev);
74 device_t parent = device_get_parent(device_get_parent(pdev));
75 uint16_t bridge_ctl = 0;
76
77 if (accel_dev->is_vf)
78 return;
79
80 if (!parent)
81 parent = pdev;
82
83 if (!pcie_wait_for_pending_transactions(pdev, 0))
84 device_printf(GET_DEV(accel_dev),
85 "Transaction still in progress. Proceeding\n");
86
87 device_printf(GET_DEV(accel_dev), "Secondary bus reset\n");
88
89 pci_save_state(pdev);
90 bridge_ctl = pci_read_config(parent, PCIR_BRIDGECTL_1, 2);
91 bridge_ctl |= PCIB_BCR_SECBUS_RESET;
92 pci_write_config(parent, PCIR_BRIDGECTL_1, bridge_ctl, 2);
93 pause_ms("adfrst", 100);
94 bridge_ctl &= ~PCIB_BCR_SECBUS_RESET;
95 pci_write_config(parent, PCIR_BRIDGECTL_1, bridge_ctl, 2);
96 pause_ms("adfrst", 100);
97 pci_restore_state(pdev);
98 }
99
100 void
adf_reset_flr(struct adf_accel_dev * accel_dev)101 adf_reset_flr(struct adf_accel_dev *accel_dev)
102 {
103 device_t pdev = accel_to_pci_dev(accel_dev);
104
105 pci_save_state(pdev);
106 if (pcie_flr(pdev,
107 max(pcie_get_max_completion_timeout(pdev) / 1000, 10),
108 true)) {
109 pci_restore_state(pdev);
110 return;
111 }
112 pci_restore_state(pdev);
113 device_printf(GET_DEV(accel_dev),
114 "FLR qat_dev%d failed trying secondary bus reset\n",
115 accel_dev->accel_id);
116 adf_reset_sbr(accel_dev);
117 }
118
119 void
adf_dev_pre_reset(struct adf_accel_dev * accel_dev)120 adf_dev_pre_reset(struct adf_accel_dev *accel_dev)
121 {
122 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
123 device_t pdev = accel_to_pci_dev(accel_dev);
124 u32 aer_offset, reg_val = 0;
125
126 if (pci_find_extcap(pdev, PCIZ_AER, &aer_offset) == 0) {
127 reg_val =
128 pci_read_config(pdev, aer_offset + PCIR_AER_UC_MASK, 4);
129 reg_val |= ADF_PPAERUCM_MASK;
130 pci_write_config(pdev,
131 aer_offset + PCIR_AER_UC_MASK,
132 reg_val,
133 4);
134 } else {
135 device_printf(pdev,
136 "Unable to find AER capability of the device\n");
137 }
138
139 if (hw_device->disable_arb) {
140 device_printf(GET_DEV(accel_dev), "Disable arbiter.\n");
141 hw_device->disable_arb(accel_dev);
142 }
143 }
144
145 void
adf_dev_post_reset(struct adf_accel_dev * accel_dev)146 adf_dev_post_reset(struct adf_accel_dev *accel_dev)
147 {
148 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
149 device_t pdev = accel_to_pci_dev(accel_dev);
150 u32 aer_offset;
151
152 if (pci_find_extcap(pdev, PCIZ_AER, &aer_offset) == 0) {
153 pci_write_config(pdev,
154 aer_offset + PCIR_AER_UC_MASK,
155 hw_device->aerucm_mask,
156 4);
157 } else {
158 device_printf(pdev,
159 "Unable to find AER capability of the device\n");
160 }
161 }
162
163 void
adf_dev_restore(struct adf_accel_dev * accel_dev)164 adf_dev_restore(struct adf_accel_dev *accel_dev)
165 {
166 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
167 device_t pdev = accel_to_pci_dev(accel_dev);
168
169 if (hw_device->pre_reset) {
170 dev_dbg(GET_DEV(accel_dev), "Performing pre reset save\n");
171 hw_device->pre_reset(accel_dev);
172 }
173
174 if (hw_device->reset_device) {
175 device_printf(GET_DEV(accel_dev),
176 "Resetting device qat_dev%d\n",
177 accel_dev->accel_id);
178 hw_device->reset_device(accel_dev);
179 pci_restore_state(pdev);
180 pci_save_state(pdev);
181 }
182
183 if (hw_device->post_reset) {
184 dev_dbg(GET_DEV(accel_dev), "Performing post reset restore\n");
185 hw_device->post_reset(accel_dev);
186 }
187 }
188
189 static void
adf_device_reset_worker(struct work_struct * work)190 adf_device_reset_worker(struct work_struct *work)
191 {
192 struct adf_reset_dev_data *reset_data =
193 container_of(work, struct adf_reset_dev_data, reset_work);
194 struct adf_accel_dev *accel_dev = reset_data->accel_dev;
195
196 if (adf_dev_restarting_notify(accel_dev)) {
197 device_printf(GET_DEV(accel_dev),
198 "Unable to send RESTARTING notification.\n");
199 return;
200 }
201
202 if (adf_dev_stop(accel_dev)) {
203 device_printf(GET_DEV(accel_dev), "Stopping device failed.\n");
204 return;
205 }
206
207 adf_dev_shutdown(accel_dev);
208
209 if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) {
210 /* The device hanged and we can't restart it */
211 /* so stop here */
212 device_printf(GET_DEV(accel_dev), "Restart device failed\n");
213 if (reset_data->mode == ADF_DEV_RESET_ASYNC)
214 kfree(reset_data);
215 WARN(1, "QAT: device restart failed. Device is unusable\n");
216 return;
217 }
218
219 adf_dev_restarted_notify(accel_dev);
220 clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
221
222 /* The dev is back alive. Notify the caller if in sync mode */
223 if (reset_data->mode == ADF_DEV_RESET_SYNC)
224 complete(&reset_data->compl);
225 else
226 kfree(reset_data);
227 }
228
229 int
adf_dev_aer_schedule_reset(struct adf_accel_dev * accel_dev,enum adf_dev_reset_mode mode)230 adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev,
231 enum adf_dev_reset_mode mode)
232 {
233 struct adf_reset_dev_data *reset_data;
234 if (!adf_dev_started(accel_dev) ||
235 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
236 return 0;
237 set_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
238 reset_data = kzalloc(sizeof(*reset_data), GFP_ATOMIC);
239 if (!reset_data)
240 return -ENOMEM;
241 reset_data->accel_dev = accel_dev;
242 init_completion(&reset_data->compl);
243 reset_data->mode = mode;
244 INIT_WORK(&reset_data->reset_work, adf_device_reset_worker);
245 queue_work(device_reset_wq, &reset_data->reset_work);
246 /* If in sync mode wait for the result */
247 if (mode == ADF_DEV_RESET_SYNC) {
248 int ret = 0;
249 /* Maximum device reset time is 10 seconds */
250 unsigned long wait_jiffies = msecs_to_jiffies(10000);
251 unsigned long timeout =
252 wait_for_completion_timeout(&reset_data->compl,
253 wait_jiffies);
254 if (!timeout) {
255 device_printf(GET_DEV(accel_dev),
256 "Reset device timeout expired\n");
257 ret = -EFAULT;
258 }
259 kfree(reset_data);
260 return ret;
261 }
262 return 0;
263 }
264
265 int
adf_dev_autoreset(struct adf_accel_dev * accel_dev)266 adf_dev_autoreset(struct adf_accel_dev *accel_dev)
267 {
268 if (accel_dev->autoreset_on_error)
269 return adf_dev_reset(accel_dev, ADF_DEV_RESET_ASYNC);
270 return 0;
271 }
272
273 static void
adf_notify_fatal_error_work(struct work_struct * work)274 adf_notify_fatal_error_work(struct work_struct *work)
275 {
276 struct adf_fatal_error_data *wq_data =
277 container_of(work, struct adf_fatal_error_data, work);
278 struct adf_accel_dev *accel_dev = wq_data->accel_dev;
279 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
280
281 if (adf_dev_in_use(accel_dev)) {
282 if (hw_device->pre_reset) {
283 device_printf(GET_DEV(accel_dev),
284 "Performing pre reset save\n");
285 hw_device->pre_reset(accel_dev);
286 }
287 }
288
289 adf_error_notifier((uintptr_t)accel_dev);
290 if (!accel_dev->is_vf) {
291 adf_dev_autoreset(accel_dev);
292 }
293
294 kfree(wq_data);
295 }
296
297 int
adf_notify_fatal_error(struct adf_accel_dev * accel_dev)298 adf_notify_fatal_error(struct adf_accel_dev *accel_dev)
299 {
300 struct adf_fatal_error_data *wq_data;
301
302 wq_data = kzalloc(sizeof(*wq_data), GFP_ATOMIC);
303 if (!wq_data) {
304 device_printf(GET_DEV(accel_dev),
305 "Failed to allocate memory\n");
306 return ENOMEM;
307 }
308 wq_data->accel_dev = accel_dev;
309
310 INIT_WORK(&wq_data->work, adf_notify_fatal_error_work);
311 queue_work(fatal_error_wq, &wq_data->work);
312
313 return 0;
314 }
315
316 int __init
adf_init_fatal_error_wq(void)317 adf_init_fatal_error_wq(void)
318 {
319 fatal_error_wq = create_workqueue("qat_fatal_error_wq");
320 return !fatal_error_wq ? EFAULT : 0;
321 }
322
323 void
adf_exit_fatal_error_wq(void)324 adf_exit_fatal_error_wq(void)
325 {
326 if (fatal_error_wq)
327 destroy_workqueue(fatal_error_wq);
328 fatal_error_wq = NULL;
329 }
330
331 int
adf_init_aer(void)332 adf_init_aer(void)
333 {
334 device_reset_wq = create_workqueue("qat_device_reset_wq");
335 return !device_reset_wq ? -EFAULT : 0;
336 }
337
338 void
adf_exit_aer(void)339 adf_exit_aer(void)
340 {
341 if (device_reset_wq)
342 destroy_workqueue(device_reset_wq);
343 device_reset_wq = NULL;
344 }
345