1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
3
4 #include <devlink.h>
5
6 #include "fw_reset.h"
7 #include "diag/fw_tracer.h"
8 #include "lib/tout.h"
9 #include "sf/sf.h"
10
11 enum {
12 MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
13 MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
14 MLX5_FW_RESET_FLAGS_PENDING_COMP,
15 MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
16 MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED,
17 MLX5_FW_RESET_FLAGS_UNLOAD_EVENT,
18 };
19
20 struct mlx5_fw_reset {
21 struct mlx5_core_dev *dev;
22 struct mlx5_nb nb;
23 struct workqueue_struct *wq;
24 struct work_struct fw_live_patch_work;
25 struct work_struct reset_request_work;
26 struct work_struct reset_unload_work;
27 struct work_struct reset_reload_work;
28 struct work_struct reset_now_work;
29 struct work_struct reset_abort_work;
30 unsigned long reset_flags;
31 u8 reset_method;
32 struct timer_list timer;
33 struct completion done;
34 int ret;
35 };
36
37 enum {
38 MLX5_FW_RST_STATE_IDLE = 0,
39 MLX5_FW_RST_STATE_TOGGLE_REQ = 4,
40 MLX5_FW_RST_STATE_DROP_MODE = 5,
41 };
42
43 enum {
44 MLX5_RST_STATE_BIT_NUM = 12,
45 MLX5_RST_ACK_BIT_NUM = 22,
46 };
47
mlx5_get_fw_rst_state(struct mlx5_core_dev * dev)48 static u8 mlx5_get_fw_rst_state(struct mlx5_core_dev *dev)
49 {
50 return (ioread32be(&dev->iseg->initializing) >> MLX5_RST_STATE_BIT_NUM) & 0xF;
51 }
52
mlx5_set_fw_rst_ack(struct mlx5_core_dev * dev)53 static void mlx5_set_fw_rst_ack(struct mlx5_core_dev *dev)
54 {
55 iowrite32be(BIT(MLX5_RST_ACK_BIT_NUM), &dev->iseg->initializing);
56 }
57
mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)58 static int mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink *devlink, u32 id,
59 struct devlink_param_gset_ctx *ctx,
60 struct netlink_ext_ack *extack)
61 {
62 struct mlx5_core_dev *dev = devlink_priv(devlink);
63 struct mlx5_fw_reset *fw_reset;
64
65 fw_reset = dev->priv.fw_reset;
66
67 if (ctx->val.vbool)
68 clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
69 else
70 set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
71 return 0;
72 }
73
mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)74 static int mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink *devlink, u32 id,
75 struct devlink_param_gset_ctx *ctx)
76 {
77 struct mlx5_core_dev *dev = devlink_priv(devlink);
78 struct mlx5_fw_reset *fw_reset;
79
80 fw_reset = dev->priv.fw_reset;
81
82 ctx->val.vbool = !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
83 &fw_reset->reset_flags);
84 return 0;
85 }
86
mlx5_reg_mfrl_set(struct mlx5_core_dev * dev,u8 reset_level,u8 reset_type_sel,u8 sync_resp,bool sync_start)87 static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
88 u8 reset_type_sel, u8 sync_resp, bool sync_start)
89 {
90 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
91 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
92
93 MLX5_SET(mfrl_reg, in, reset_level, reset_level);
94 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
95 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
96 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);
97
98 return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
99 }
100
mlx5_reg_mfrl_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type,u8 * reset_state,u8 * reset_method)101 static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level,
102 u8 *reset_type, u8 *reset_state, u8 *reset_method)
103 {
104 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
105 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
106 int err;
107
108 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
109 if (err)
110 return err;
111
112 if (reset_level)
113 *reset_level = MLX5_GET(mfrl_reg, out, reset_level);
114 if (reset_type)
115 *reset_type = MLX5_GET(mfrl_reg, out, reset_type);
116 if (reset_state)
117 *reset_state = MLX5_GET(mfrl_reg, out, reset_state);
118 if (reset_method)
119 *reset_method = MLX5_GET(mfrl_reg, out, pci_reset_req_method);
120
121 return 0;
122 }
123
mlx5_fw_reset_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type)124 int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
125 {
126 return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL);
127 }
128
mlx5_fw_reset_get_reset_method(struct mlx5_core_dev * dev,u8 * reset_method)129 static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev,
130 u8 *reset_method)
131 {
132 if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method)) {
133 *reset_method = MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE;
134 return 0;
135 }
136
137 return mlx5_reg_mfrl_query(dev, NULL, NULL, NULL, reset_method);
138 }
139
mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev * dev,struct netlink_ext_ack * extack)140 static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev,
141 struct netlink_ext_ack *extack)
142 {
143 u8 reset_state;
144
145 if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state, NULL))
146 goto out;
147
148 if (!reset_state)
149 return 0;
150
151 switch (reset_state) {
152 case MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION:
153 case MLX5_MFRL_REG_RESET_STATE_RESET_IN_PROGRESS:
154 NL_SET_ERR_MSG_MOD(extack, "Sync reset still in progress");
155 return -EBUSY;
156 case MLX5_MFRL_REG_RESET_STATE_NEG_TIMEOUT:
157 NL_SET_ERR_MSG_MOD(extack, "Sync reset negotiation timeout");
158 return -ETIMEDOUT;
159 case MLX5_MFRL_REG_RESET_STATE_NACK:
160 NL_SET_ERR_MSG_MOD(extack, "One of the hosts disabled reset");
161 return -EPERM;
162 case MLX5_MFRL_REG_RESET_STATE_UNLOAD_TIMEOUT:
163 NL_SET_ERR_MSG_MOD(extack, "Sync reset unload timeout");
164 return -ETIMEDOUT;
165 }
166
167 out:
168 NL_SET_ERR_MSG_MOD(extack, "Sync reset failed");
169 return -EIO;
170 }
171
mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev * dev,u8 reset_type_sel,struct netlink_ext_ack * extack)172 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
173 struct netlink_ext_ack *extack)
174 {
175 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
176 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
177 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
178 int err, rst_res;
179
180 set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
181
182 MLX5_SET(mfrl_reg, in, reset_level, MLX5_MFRL_REG_RESET_LEVEL3);
183 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
184 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, 1);
185 err = mlx5_access_reg(dev, in, sizeof(in), out, sizeof(out),
186 MLX5_REG_MFRL, 0, 1, false);
187 if (!err)
188 return 0;
189
190 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
191 if (err == -EREMOTEIO && MLX5_CAP_MCAM_FEATURE(dev, reset_state)) {
192 rst_res = mlx5_fw_reset_get_reset_state_err(dev, extack);
193 return rst_res ? rst_res : err;
194 }
195
196 NL_SET_ERR_MSG_MOD(extack, "Sync reset command failed");
197 return mlx5_cmd_check(dev, err, in, out);
198 }
199
mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev * dev,struct netlink_ext_ack * extack)200 int mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev *dev,
201 struct netlink_ext_ack *extack)
202 {
203 u8 rst_state;
204 int err;
205
206 err = mlx5_fw_reset_get_reset_state_err(dev, extack);
207 if (err)
208 return err;
209
210 rst_state = mlx5_get_fw_rst_state(dev);
211 if (!rst_state)
212 return 0;
213
214 mlx5_core_err(dev, "Sync reset did not complete, state=%d\n", rst_state);
215 NL_SET_ERR_MSG_MOD(extack, "Sync reset did not complete successfully");
216 return rst_state;
217 }
218
mlx5_fw_reset_set_live_patch(struct mlx5_core_dev * dev)219 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
220 {
221 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
222 }
223
mlx5_fw_reset_complete_reload(struct mlx5_core_dev * dev)224 static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
225 {
226 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
227 struct devlink *devlink = priv_to_devlink(dev);
228
229 /* if this is the driver that initiated the fw reset, devlink completed the reload */
230 if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
231 complete(&fw_reset->done);
232 } else {
233 mlx5_sync_reset_unload_flow(dev, false);
234 if (mlx5_health_wait_pci_up(dev))
235 mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
236 else
237 mlx5_load_one(dev, true);
238 devl_lock(devlink);
239 devlink_remote_reload_actions_performed(devlink, 0,
240 BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
241 BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
242 devl_unlock(devlink);
243 }
244 }
245
mlx5_stop_sync_reset_poll(struct mlx5_core_dev * dev)246 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
247 {
248 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
249
250 timer_delete_sync(&fw_reset->timer);
251 }
252
mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev * dev,bool poll_health)253 static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
254 {
255 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
256
257 if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
258 mlx5_core_warn(dev, "Reset request was already cleared\n");
259 return -EALREADY;
260 }
261
262 mlx5_stop_sync_reset_poll(dev);
263 if (poll_health)
264 mlx5_start_health_poll(dev);
265 return 0;
266 }
267
mlx5_sync_reset_reload_work(struct work_struct * work)268 static void mlx5_sync_reset_reload_work(struct work_struct *work)
269 {
270 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
271 reset_reload_work);
272 struct mlx5_core_dev *dev = fw_reset->dev;
273
274 mlx5_sync_reset_clear_reset_requested(dev, false);
275 mlx5_enter_error_state(dev, true);
276 mlx5_fw_reset_complete_reload(dev);
277 }
278
279 #define MLX5_RESET_POLL_INTERVAL (HZ / 10)
poll_sync_reset(struct timer_list * t)280 static void poll_sync_reset(struct timer_list *t)
281 {
282 struct mlx5_fw_reset *fw_reset = timer_container_of(fw_reset, t,
283 timer);
284 struct mlx5_core_dev *dev = fw_reset->dev;
285 u32 fatal_error;
286
287 if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
288 return;
289
290 fatal_error = mlx5_health_check_fatal_sensors(dev);
291
292 if (fatal_error) {
293 mlx5_core_warn(dev, "Got Device Reset\n");
294 if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
295 queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
296 else
297 mlx5_core_err(dev, "Device is being removed, Drop new reset work\n");
298 return;
299 }
300
301 mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
302 }
303
mlx5_start_sync_reset_poll(struct mlx5_core_dev * dev)304 static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
305 {
306 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
307
308 timer_setup(&fw_reset->timer, poll_sync_reset, 0);
309 fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
310 add_timer(&fw_reset->timer);
311 }
312
mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev * dev)313 static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
314 {
315 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
316 }
317
mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev * dev)318 static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
319 {
320 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
321 }
322
mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev * dev)323 static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
324 {
325 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
326
327 if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
328 mlx5_core_warn(dev, "Reset request was already set\n");
329 return -EALREADY;
330 }
331 mlx5_stop_health_poll(dev, true);
332 mlx5_start_sync_reset_poll(dev);
333 return 0;
334 }
335
mlx5_fw_live_patch_event(struct work_struct * work)336 static void mlx5_fw_live_patch_event(struct work_struct *work)
337 {
338 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
339 fw_live_patch_work);
340 struct mlx5_core_dev *dev = fw_reset->dev;
341
342 mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
343 fw_rev_min(dev), fw_rev_sub(dev));
344
345 if (mlx5_fw_tracer_reload(dev->tracer))
346 mlx5_core_err(dev, "Failed to reload FW tracer\n");
347 }
348
349 #if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
mlx5_check_hotplug_interrupt(struct mlx5_core_dev * dev,struct pci_dev * bridge)350 static int mlx5_check_hotplug_interrupt(struct mlx5_core_dev *dev,
351 struct pci_dev *bridge)
352 {
353 u16 reg16;
354 int err;
355
356 err = pcie_capability_read_word(bridge, PCI_EXP_SLTCTL, ®16);
357 if (err)
358 return err;
359
360 if ((reg16 & PCI_EXP_SLTCTL_HPIE) && (reg16 & PCI_EXP_SLTCTL_DLLSCE)) {
361 mlx5_core_warn(dev, "FW reset is not supported as HotPlug is enabled\n");
362 return -EOPNOTSUPP;
363 }
364
365 return 0;
366 }
367 #endif
368
369 static const struct pci_device_id mgt_ifc_device_ids[] = {
370 { PCI_VDEVICE(MELLANOX, 0xc2d2) }, /* BlueField1 MGT interface device ID */
371 { PCI_VDEVICE(MELLANOX, 0xc2d3) }, /* BlueField2 MGT interface device ID */
372 { PCI_VDEVICE(MELLANOX, 0xc2d4) }, /* BlueField3-Lx MGT interface device ID */
373 { PCI_VDEVICE(MELLANOX, 0xc2d5) }, /* BlueField3 MGT interface device ID */
374 { PCI_VDEVICE(MELLANOX, 0xc2d6) }, /* BlueField4 MGT interface device ID */
375 };
376
mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev * dev,u16 dev_id)377 static bool mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev *dev, u16 dev_id)
378 {
379 int i;
380
381 for (i = 0; i < ARRAY_SIZE(mgt_ifc_device_ids); ++i)
382 if (mgt_ifc_device_ids[i].device == dev_id)
383 return true;
384
385 return false;
386 }
387
mlx5_check_dev_ids(struct mlx5_core_dev * dev,u16 dev_id)388 static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
389 {
390 struct pci_bus *bridge_bus = dev->pdev->bus;
391 struct pci_dev *sdev;
392 u16 sdev_id;
393 int err;
394
395 /* Check that all functions under the pci bridge are PFs of
396 * this device otherwise fail this function.
397 */
398 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
399 err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
400 if (err)
401 return pcibios_err_to_errno(err);
402
403 if (sdev_id == dev_id)
404 continue;
405
406 if (mlx5_is_mgt_ifc_pci_device(dev, sdev_id))
407 continue;
408
409 mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
410 return -EPERM;
411 }
412 return 0;
413 }
414
mlx5_is_reset_now_capable(struct mlx5_core_dev * dev,u8 reset_method)415 static bool mlx5_is_reset_now_capable(struct mlx5_core_dev *dev,
416 u8 reset_method)
417 {
418 struct pci_dev *bridge = dev->pdev->bus->self;
419 u16 dev_id;
420 int err;
421
422 if (!bridge) {
423 mlx5_core_warn(dev, "PCI bus bridge is not accessible\n");
424 return false;
425 }
426
427 if (!MLX5_CAP_GEN(dev, fast_teardown)) {
428 mlx5_core_warn(dev, "fast teardown is not supported by firmware\n");
429 return false;
430 }
431
432 if (!mlx5_core_is_ecpf(dev) && !mlx5_sf_table_empty(dev)) {
433 mlx5_core_warn(dev, "SFs should be removed before reset\n");
434 return false;
435 }
436
437 #if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
438 if (reset_method != MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET) {
439 err = mlx5_check_hotplug_interrupt(dev, bridge);
440 if (err)
441 return false;
442 }
443 #endif
444
445 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
446 if (err)
447 return false;
448 return (!mlx5_check_dev_ids(dev, dev_id));
449 }
450
mlx5_sync_reset_request_event(struct work_struct * work)451 static void mlx5_sync_reset_request_event(struct work_struct *work)
452 {
453 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
454 reset_request_work);
455 struct mlx5_core_dev *dev = fw_reset->dev;
456 int err;
457
458 err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method);
459 if (err)
460 mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err);
461
462 if (err || test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) ||
463 !mlx5_is_reset_now_capable(dev, fw_reset->reset_method)) {
464 err = mlx5_fw_reset_set_reset_sync_nack(dev);
465 mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
466 err ? "Failed" : "Sent");
467 return;
468 }
469 if (mlx5_sync_reset_set_reset_requested(dev))
470 return;
471
472 err = mlx5_fw_reset_set_reset_sync_ack(dev);
473 if (err)
474 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
475 else
476 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
477 }
478
mlx5_pci_link_toggle(struct mlx5_core_dev * dev,u16 dev_id)479 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id)
480 {
481 struct pci_bus *bridge_bus = dev->pdev->bus;
482 struct pci_dev *bridge = bridge_bus->self;
483 unsigned long timeout;
484 struct pci_dev *sdev;
485 int cap, err;
486 u16 reg16;
487
488 cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
489 if (!cap)
490 return -EOPNOTSUPP;
491
492 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
493 pci_save_state(sdev);
494 pci_cfg_access_lock(sdev);
495 }
496 /* PCI link toggle */
497 err = pcie_capability_set_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
498 if (err)
499 return pcibios_err_to_errno(err);
500 msleep(500);
501 err = pcie_capability_clear_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
502 if (err)
503 return pcibios_err_to_errno(err);
504
505 /* Check link */
506 if (!bridge->link_active_reporting) {
507 mlx5_core_warn(dev, "No PCI link reporting capability\n");
508 msleep(1000);
509 goto restore;
510 }
511
512 timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE));
513 do {
514 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, ®16);
515 if (err)
516 return pcibios_err_to_errno(err);
517 if (reg16 & PCI_EXP_LNKSTA_DLLLA)
518 break;
519 msleep(20);
520 } while (!time_after(jiffies, timeout));
521
522 if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
523 mlx5_core_info(dev, "PCI Link up\n");
524 } else {
525 mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n",
526 reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
527 err = -ETIMEDOUT;
528 goto restore;
529 }
530
531 do {
532 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, ®16);
533 if (err)
534 return pcibios_err_to_errno(err);
535 if (reg16 == dev_id)
536 break;
537 msleep(20);
538 } while (!time_after(jiffies, timeout));
539
540 if (reg16 == dev_id) {
541 mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n");
542 } else {
543 mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n",
544 reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
545 err = -ETIMEDOUT;
546 }
547
548 restore:
549 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
550 pci_cfg_access_unlock(sdev);
551 pci_restore_state(sdev);
552 }
553
554 return err;
555 }
556
mlx5_pci_reset_bus(struct mlx5_core_dev * dev)557 static int mlx5_pci_reset_bus(struct mlx5_core_dev *dev)
558 {
559 if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method))
560 return -EOPNOTSUPP;
561
562 return pci_reset_bus(dev->pdev);
563 }
564
mlx5_sync_pci_reset(struct mlx5_core_dev * dev,u8 reset_method)565 static int mlx5_sync_pci_reset(struct mlx5_core_dev *dev, u8 reset_method)
566 {
567 u16 dev_id;
568 int err;
569
570 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
571 if (err)
572 return pcibios_err_to_errno(err);
573 err = mlx5_check_dev_ids(dev, dev_id);
574 if (err)
575 return err;
576
577 switch (reset_method) {
578 case MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE:
579 err = mlx5_pci_link_toggle(dev, dev_id);
580 if (err)
581 mlx5_core_warn(dev, "mlx5_pci_link_toggle failed\n");
582 break;
583 case MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET:
584 err = mlx5_pci_reset_bus(dev);
585 if (err)
586 mlx5_core_warn(dev, "mlx5_pci_reset_bus failed\n");
587 break;
588 default:
589 return -EOPNOTSUPP;
590 }
591
592 return err;
593 }
594
mlx5_sync_reset_unload_flow(struct mlx5_core_dev * dev,bool locked)595 void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked)
596 {
597 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
598 unsigned long timeout;
599 int poll_freq = 20;
600 bool reset_action;
601 u8 rst_state;
602 int err;
603
604 if (locked)
605 mlx5_unload_one_devl_locked(dev, false);
606 else
607 mlx5_unload_one(dev, false);
608
609 if (!test_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags))
610 return;
611
612 mlx5_set_fw_rst_ack(dev);
613 mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");
614
615 reset_action = false;
616 timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
617 do {
618 rst_state = mlx5_get_fw_rst_state(dev);
619 if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
620 rst_state == MLX5_FW_RST_STATE_IDLE) {
621 reset_action = true;
622 break;
623 }
624 if (rst_state == MLX5_FW_RST_STATE_DROP_MODE) {
625 mlx5_core_info(dev, "Sync Reset Drop mode ack\n");
626 mlx5_set_fw_rst_ack(dev);
627 poll_freq = 1000;
628 }
629 msleep(poll_freq);
630 } while (!time_after(jiffies, timeout));
631
632 if (!reset_action) {
633 mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
634 rst_state);
635 fw_reset->ret = -ETIMEDOUT;
636 goto done;
637 }
638
639 mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n",
640 rst_state);
641 if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
642 err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
643 if (err) {
644 mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n",
645 err);
646 fw_reset->ret = err;
647 }
648 }
649
650 done:
651 clear_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
652 }
653
mlx5_sync_reset_now_event(struct work_struct * work)654 static void mlx5_sync_reset_now_event(struct work_struct *work)
655 {
656 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
657 reset_now_work);
658 struct mlx5_core_dev *dev = fw_reset->dev;
659 int err;
660
661 if (mlx5_sync_reset_clear_reset_requested(dev, false))
662 return;
663
664 mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
665
666 err = mlx5_cmd_fast_teardown_hca(dev);
667 if (err) {
668 mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
669 goto done;
670 }
671
672 err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
673 if (err) {
674 mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, no reset done, err %d\n", err);
675 set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags);
676 }
677
678 mlx5_enter_error_state(dev, true);
679 done:
680 fw_reset->ret = err;
681 mlx5_fw_reset_complete_reload(dev);
682 }
683
mlx5_sync_reset_unload_event(struct work_struct * work)684 static void mlx5_sync_reset_unload_event(struct work_struct *work)
685 {
686 struct mlx5_fw_reset *fw_reset;
687 struct mlx5_core_dev *dev;
688 int err;
689
690 fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work);
691 dev = fw_reset->dev;
692
693 if (mlx5_sync_reset_clear_reset_requested(dev, false))
694 return;
695
696 set_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
697 mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n");
698
699 err = mlx5_cmd_fast_teardown_hca(dev);
700 if (err)
701 mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err);
702 else
703 mlx5_enter_error_state(dev, true);
704
705 mlx5_fw_reset_complete_reload(dev);
706 }
707
mlx5_sync_reset_abort_event(struct work_struct * work)708 static void mlx5_sync_reset_abort_event(struct work_struct *work)
709 {
710 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
711 reset_abort_work);
712 struct mlx5_core_dev *dev = fw_reset->dev;
713
714 if (mlx5_sync_reset_clear_reset_requested(dev, true))
715 return;
716 mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
717 }
718
mlx5_sync_reset_events_handle(struct mlx5_fw_reset * fw_reset,struct mlx5_eqe * eqe)719 static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
720 {
721 struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
722 u8 sync_event_rst_type;
723
724 sync_fw_update_eqe = &eqe->data.sync_fw_update;
725 sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
726 switch (sync_event_rst_type) {
727 case MLX5_SYNC_RST_STATE_RESET_REQUEST:
728 queue_work(fw_reset->wq, &fw_reset->reset_request_work);
729 break;
730 case MLX5_SYNC_RST_STATE_RESET_UNLOAD:
731 queue_work(fw_reset->wq, &fw_reset->reset_unload_work);
732 break;
733 case MLX5_SYNC_RST_STATE_RESET_NOW:
734 queue_work(fw_reset->wq, &fw_reset->reset_now_work);
735 break;
736 case MLX5_SYNC_RST_STATE_RESET_ABORT:
737 queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
738 break;
739 }
740 }
741
fw_reset_event_notifier(struct notifier_block * nb,unsigned long action,void * data)742 static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
743 {
744 struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
745 struct mlx5_eqe *eqe = data;
746
747 if (test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
748 return NOTIFY_DONE;
749
750 switch (eqe->sub_type) {
751 case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
752 queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
753 break;
754 case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
755 mlx5_sync_reset_events_handle(fw_reset, eqe);
756 break;
757 default:
758 return NOTIFY_DONE;
759 }
760
761 return NOTIFY_OK;
762 }
763
mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev * dev)764 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
765 {
766 unsigned long pci_sync_update_timeout = mlx5_tout_ms(dev, PCI_SYNC_UPDATE);
767 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
768 unsigned long timeout;
769 int err;
770
771 if (MLX5_CAP_GEN(dev, pci_sync_for_fw_update_with_driver_unload))
772 pci_sync_update_timeout += mlx5_tout_ms(dev, RESET_UNLOAD);
773 timeout = msecs_to_jiffies(pci_sync_update_timeout);
774 if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
775 mlx5_core_warn(dev, "FW sync reset timeout after %lu seconds\n",
776 pci_sync_update_timeout / 1000);
777 err = -ETIMEDOUT;
778 goto out;
779 }
780 err = fw_reset->ret;
781 if (test_and_clear_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags)) {
782 mlx5_unload_one_devl_locked(dev, false);
783 mlx5_load_one_devl_locked(dev, true);
784 }
785 out:
786 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
787 return err;
788 }
789
mlx5_fw_reset_events_start(struct mlx5_core_dev * dev)790 void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
791 {
792 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
793
794 if (!fw_reset)
795 return;
796
797 MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
798 mlx5_eq_notifier_register(dev, &fw_reset->nb);
799 }
800
mlx5_fw_reset_events_stop(struct mlx5_core_dev * dev)801 void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
802 {
803 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
804
805 if (!fw_reset)
806 return;
807
808 mlx5_eq_notifier_unregister(dev, &fw_reset->nb);
809 }
810
mlx5_drain_fw_reset(struct mlx5_core_dev * dev)811 void mlx5_drain_fw_reset(struct mlx5_core_dev *dev)
812 {
813 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
814
815 if (!fw_reset)
816 return;
817
818 set_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags);
819 cancel_work_sync(&fw_reset->fw_live_patch_work);
820 cancel_work_sync(&fw_reset->reset_request_work);
821 cancel_work_sync(&fw_reset->reset_unload_work);
822 cancel_work_sync(&fw_reset->reset_reload_work);
823 cancel_work_sync(&fw_reset->reset_now_work);
824 cancel_work_sync(&fw_reset->reset_abort_work);
825 }
826
827 static const struct devlink_param mlx5_fw_reset_devlink_params[] = {
828 DEVLINK_PARAM_GENERIC(ENABLE_REMOTE_DEV_RESET, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
829 mlx5_fw_reset_enable_remote_dev_reset_get,
830 mlx5_fw_reset_enable_remote_dev_reset_set, NULL),
831 };
832
mlx5_fw_reset_init(struct mlx5_core_dev * dev)833 int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
834 {
835 struct mlx5_fw_reset *fw_reset;
836 int err;
837
838 if (!MLX5_CAP_MCAM_REG(dev, mfrl))
839 return 0;
840
841 fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);
842 if (!fw_reset)
843 return -ENOMEM;
844 fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
845 if (!fw_reset->wq) {
846 kfree(fw_reset);
847 return -ENOMEM;
848 }
849
850 fw_reset->dev = dev;
851 dev->priv.fw_reset = fw_reset;
852
853 err = devl_params_register(priv_to_devlink(dev),
854 mlx5_fw_reset_devlink_params,
855 ARRAY_SIZE(mlx5_fw_reset_devlink_params));
856 if (err) {
857 destroy_workqueue(fw_reset->wq);
858 kfree(fw_reset);
859 return err;
860 }
861
862 INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
863 INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
864 INIT_WORK(&fw_reset->reset_unload_work, mlx5_sync_reset_unload_event);
865 INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
866 INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
867 INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
868
869 init_completion(&fw_reset->done);
870 return 0;
871 }
872
mlx5_fw_reset_cleanup(struct mlx5_core_dev * dev)873 void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
874 {
875 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
876
877 if (!fw_reset)
878 return;
879
880 devl_params_unregister(priv_to_devlink(dev),
881 mlx5_fw_reset_devlink_params,
882 ARRAY_SIZE(mlx5_fw_reset_devlink_params));
883 destroy_workqueue(fw_reset->wq);
884 kfree(dev->priv.fw_reset);
885 }
886