1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/module.h> 35 #include <linux/init.h> 36 #include <linux/errno.h> 37 #include <linux/pci.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/slab.h> 40 #include <linux/interrupt.h> 41 #include <linux/delay.h> 42 #include <linux/mlx5/driver.h> 43 #include <linux/mlx5/cq.h> 44 #include <linux/mlx5/qp.h> 45 #include <linux/debugfs.h> 46 #include <linux/kmod.h> 47 #include <linux/mlx5/mlx5_ifc.h> 48 #include <linux/mlx5/vport.h> 49 #include <linux/version.h> 50 #include <net/devlink.h> 51 #include "mlx5_core.h" 52 #include "lib/eq.h" 53 #include "fs_core.h" 54 #include "lib/mpfs.h" 55 #include "eswitch.h" 56 #include "devlink.h" 57 #include "fw_reset.h" 58 #include "lib/mlx5.h" 59 #include "lib/tout.h" 60 #include "fpga/core.h" 61 #include "en_accel/ipsec.h" 62 #include "lib/clock.h" 63 #include "lib/vxlan.h" 64 #include "lib/geneve.h" 65 #include "lib/devcom.h" 66 #include "lib/pci_vsc.h" 67 #include "diag/fw_tracer.h" 68 #include "ecpf.h" 69 #include "lib/hv_vhca.h" 70 #include "diag/rsc_dump.h" 71 #include "sf/vhca_event.h" 72 #include "sf/dev/dev.h" 73 #include "sf/sf.h" 74 #include "mlx5_irq.h" 75 #include "hwmon.h" 76 #include "lag/lag.h" 77 #include "sh_devlink.h" 78 79 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 80 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver"); 81 MODULE_LICENSE("Dual BSD/GPL"); 82 83 unsigned int mlx5_core_debug_mask; 84 module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644); 85 MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 86 87 static unsigned int prof_sel = MLX5_DEFAULT_PROF; 88 module_param_named(prof_sel, prof_sel, uint, 0444); 89 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 90 91 static u32 sw_owner_id[4]; 92 #define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1) 93 static DEFINE_IDA(sw_vhca_ida); 94 95 enum { 96 MLX5_ATOMIC_REQ_MODE_BE = 0x0, 97 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 98 }; 99 100 #define LOG_MAX_SUPPORTED_QPS 0xff 101 102 static struct mlx5_profile profile[] = { 103 [0] = { 104 .mask = 0, 105 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, 106 }, 107 [1] = { 108 .mask = MLX5_PROF_MASK_QP_SIZE, 109 .log_max_qp = 12, 110 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, 111 112 }, 113 [2] = { 114 .mask = MLX5_PROF_MASK_QP_SIZE, 115 .log_max_qp = LOG_MAX_SUPPORTED_QPS, 116 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, 117 }, 118 [3] = { 119 .mask = MLX5_PROF_MASK_QP_SIZE, 120 .log_max_qp = LOG_MAX_SUPPORTED_QPS, 121 .num_cmd_caches = 0, 122 }, 123 }; 124 125 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, 126 u32 warn_time_mili, const char *init_state) 127 { 128 unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili); 129 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 130 u32 fw_initializing; 131 132 do { 133 fw_initializing = ioread32be(&dev->iseg->initializing); 134 if (!(fw_initializing >> 31)) 135 break; 136 if (time_after(jiffies, end)) { 137 mlx5_core_err(dev, "Firmware over %u MS in %s state, aborting\n", 138 max_wait_mili, init_state); 139 return -ETIMEDOUT; 140 } 141 if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) { 142 mlx5_core_warn(dev, "device is being removed, stop waiting for FW %s\n", 143 init_state); 144 return -ENODEV; 145 } 146 if (warn_time_mili && time_after(jiffies, warn)) { 147 mlx5_core_warn(dev, "Waiting for FW %s, timeout abort in %ds (0x%x)\n", 148 init_state, jiffies_to_msecs(end - warn) / 1000, 149 fw_initializing); 150 warn = jiffies + msecs_to_jiffies(warn_time_mili); 151 } 152 msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT)); 153 } while (true); 154 155 return 0; 156 } 157 158 static void mlx5_set_driver_version(struct mlx5_core_dev *dev) 159 { 160 int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in, 161 driver_version); 162 u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {}; 163 char *string; 164 165 if (!MLX5_CAP_GEN(dev, driver_version)) 166 return; 167 168 string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version); 169 170 snprintf(string, driver_ver_sz, "Linux,%s,%u.%u.%u", 171 KBUILD_MODNAME, LINUX_VERSION_MAJOR, 172 LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL); 173 174 /*Send the command*/ 175 MLX5_SET(set_driver_version_in, in, opcode, 176 MLX5_CMD_OP_SET_DRIVER_VERSION); 177 178 mlx5_cmd_exec_in(dev, set_driver_version, in); 179 } 180 181 static int set_dma_caps(struct pci_dev *pdev) 182 { 183 int err; 184 185 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 186 if (err) { 187 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 188 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 189 if (err) { 190 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 191 return err; 192 } 193 } 194 195 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 196 return err; 197 } 198 199 static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 200 { 201 struct pci_dev *pdev = dev->pdev; 202 int err = 0; 203 204 mutex_lock(&dev->pci_status_mutex); 205 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 206 err = pci_enable_device(pdev); 207 if (!err) 208 dev->pci_status = MLX5_PCI_STATUS_ENABLED; 209 } 210 mutex_unlock(&dev->pci_status_mutex); 211 212 return err; 213 } 214 215 static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 216 { 217 struct pci_dev *pdev = dev->pdev; 218 219 mutex_lock(&dev->pci_status_mutex); 220 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 221 pci_disable_device(pdev); 222 dev->pci_status = MLX5_PCI_STATUS_DISABLED; 223 } 224 mutex_unlock(&dev->pci_status_mutex); 225 } 226 227 static int request_bar(struct pci_dev *pdev) 228 { 229 int err = 0; 230 231 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 232 dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 233 return -ENODEV; 234 } 235 236 err = pci_request_regions(pdev, KBUILD_MODNAME); 237 if (err) 238 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 239 240 return err; 241 } 242 243 static void release_bar(struct pci_dev *pdev) 244 { 245 pci_release_regions(pdev); 246 } 247 248 struct mlx5_reg_host_endianness { 249 u8 he; 250 u8 rsvd[15]; 251 }; 252 253 static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) 254 { 255 switch (size) { 256 case 128: 257 return 0; 258 case 256: 259 return 1; 260 case 512: 261 return 2; 262 case 1024: 263 return 3; 264 case 2048: 265 return 4; 266 case 4096: 267 return 5; 268 default: 269 mlx5_core_warn(dev, "invalid pkey table size %d\n", size); 270 return 0; 271 } 272 } 273 274 void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev) 275 { 276 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); 277 dev->mlx5e_res.uplink_netdev = netdev; 278 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, 279 netdev); 280 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); 281 } 282 283 void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev) 284 { 285 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); 286 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, 287 dev->mlx5e_res.uplink_netdev); 288 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); 289 } 290 EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay); 291 292 void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data) 293 { 294 mlx5_blocking_notifier_call_chain(dev, event, data); 295 } 296 EXPORT_SYMBOL(mlx5_core_mp_event_replay); 297 298 int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 299 enum mlx5_cap_mode cap_mode) 300 { 301 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 302 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 303 void *out, *hca_caps; 304 u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 305 int err; 306 307 if (WARN_ON(!dev->caps.hca[cap_type])) 308 /* this cap_type must be added to mlx5_hca_caps_alloc() */ 309 return -EINVAL; 310 311 memset(in, 0, sizeof(in)); 312 out = kzalloc(out_sz, GFP_KERNEL); 313 if (!out) 314 return -ENOMEM; 315 316 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 317 MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 318 err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out); 319 if (err) { 320 mlx5_core_warn(dev, 321 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 322 cap_type, cap_mode, err); 323 goto query_ex; 324 } 325 326 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 327 328 switch (cap_mode) { 329 case HCA_CAP_OPMOD_GET_MAX: 330 memcpy(dev->caps.hca[cap_type]->max, hca_caps, 331 MLX5_UN_SZ_BYTES(hca_cap_union)); 332 break; 333 case HCA_CAP_OPMOD_GET_CUR: 334 memcpy(dev->caps.hca[cap_type]->cur, hca_caps, 335 MLX5_UN_SZ_BYTES(hca_cap_union)); 336 break; 337 default: 338 mlx5_core_warn(dev, 339 "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 340 cap_type, cap_mode); 341 err = -EINVAL; 342 break; 343 } 344 query_ex: 345 kfree(out); 346 return err; 347 } 348 349 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) 350 { 351 int ret; 352 353 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); 354 if (ret) 355 return ret; 356 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); 357 } 358 359 static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod) 360 { 361 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 362 MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); 363 return mlx5_cmd_exec_in(dev, set_hca_cap, in); 364 } 365 366 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx) 367 { 368 void *set_hca_cap; 369 int req_endianness; 370 int err; 371 372 if (!MLX5_CAP_GEN(dev, atomic)) 373 return 0; 374 375 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); 376 if (err) 377 return err; 378 379 req_endianness = 380 MLX5_CAP_ATOMIC(dev, 381 supported_atomic_req_8B_endianness_mode_1); 382 383 if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 384 return 0; 385 386 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 387 388 /* Set requestor to host endianness */ 389 MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode, 390 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 391 392 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); 393 } 394 395 static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx) 396 { 397 bool do_set = false, mem_page_fault = false; 398 void *set_hca_cap; 399 int err; 400 401 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) || 402 !MLX5_CAP_GEN(dev, pg)) 403 return 0; 404 405 err = mlx5_core_get_caps(dev, MLX5_CAP_ODP); 406 if (err) 407 return err; 408 409 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 410 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur, 411 MLX5_ST_SZ_BYTES(odp_cap)); 412 413 /* For best performance, enable memory scheme ODP only when 414 * it has page prefetch enabled. 415 */ 416 if (MLX5_CAP_ODP_MAX(dev, mem_page_fault) && 417 MLX5_CAP_ODP_MAX(dev, memory_page_fault_scheme_cap.page_prefetch)) { 418 mem_page_fault = true; 419 do_set = true; 420 MLX5_SET(odp_cap, set_hca_cap, mem_page_fault, mem_page_fault); 421 goto set; 422 } 423 424 #define ODP_CAP_SET_MAX(dev, field) \ 425 do { \ 426 u32 _res = MLX5_CAP_ODP_MAX(dev, field); \ 427 if (_res) { \ 428 do_set = true; \ 429 MLX5_SET(odp_cap, set_hca_cap, field, _res); \ 430 } \ 431 } while (0) 432 433 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.ud_odp_caps.srq_receive); 434 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.rc_odp_caps.srq_receive); 435 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.srq_receive); 436 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.send); 437 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.receive); 438 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.write); 439 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.read); 440 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.atomic); 441 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.srq_receive); 442 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.send); 443 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.receive); 444 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.write); 445 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.read); 446 ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.atomic); 447 448 set: 449 if (do_set) 450 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP); 451 452 mlx5_core_dbg(dev, "Using ODP %s scheme\n", 453 mem_page_fault ? "memory" : "transport"); 454 return err; 455 } 456 457 static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) 458 { 459 struct devlink *devlink = priv_to_devlink(dev); 460 union devlink_param_value val; 461 int err; 462 463 err = devl_param_driverinit_value_get(devlink, 464 DEVLINK_PARAM_GENERIC_ID_MAX_MACS, 465 &val); 466 if (!err) 467 return val.vu32; 468 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err); 469 return err; 470 } 471 472 bool mlx5_is_roce_on(struct mlx5_core_dev *dev) 473 { 474 struct devlink *devlink = priv_to_devlink(dev); 475 union devlink_param_value val; 476 int err; 477 478 err = devl_param_driverinit_value_get(devlink, 479 DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, 480 &val); 481 482 if (!err) 483 return val.vbool; 484 485 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err); 486 return MLX5_CAP_GEN(dev, roce); 487 } 488 EXPORT_SYMBOL(mlx5_is_roce_on); 489 490 static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) 491 { 492 bool do_set = false; 493 void *set_hca_cap; 494 int err; 495 496 if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) 497 return 0; 498 499 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); 500 if (err) 501 return err; 502 503 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 504 capability); 505 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, 506 MLX5_ST_SZ_BYTES(cmd_hca_cap_2)); 507 508 if (MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) && 509 dev->priv.sw_vhca_id > 0) { 510 MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1); 511 do_set = true; 512 } 513 514 if (MLX5_CAP_GEN_2_MAX(dev, lag_per_mp_group)) { 515 MLX5_SET(cmd_hca_cap_2, set_hca_cap, lag_per_mp_group, 1); 516 do_set = true; 517 } 518 519 /* some FW versions that support querying MLX5_CAP_GENERAL_2 520 * capabilities but don't support setting them. 521 * Skip unnecessary update to hca_cap_2 when no changes were introduced 522 */ 523 return do_set ? set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2) : 0; 524 } 525 526 static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) 527 { 528 struct mlx5_profile *prof = &dev->profile; 529 void *set_hca_cap; 530 int max_uc_list; 531 int err; 532 533 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 534 if (err) 535 return err; 536 537 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 538 capability); 539 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur, 540 MLX5_ST_SZ_BYTES(cmd_hca_cap)); 541 542 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 543 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 544 128); 545 /* we limit the size of the pkey table to 128 entries for now */ 546 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 547 to_fw_pkey_sz(dev, 128)); 548 549 /* Check log_max_qp from HCA caps to set in current profile */ 550 if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) { 551 prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp)); 552 } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) { 553 mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n", 554 prof->log_max_qp, 555 MLX5_CAP_GEN_MAX(dev, log_max_qp)); 556 prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); 557 } 558 if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 559 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 560 prof->log_max_qp); 561 562 /* disable cmdif checksum */ 563 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 564 565 /* Enable 4K UAR only when HCA supports it and page size is bigger 566 * than 4K. 567 */ 568 if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096) 569 MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1); 570 571 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 572 573 if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte)) 574 MLX5_SET(cmd_hca_cap, 575 set_hca_cap, 576 cache_line_128byte, 577 cache_line_size() >= 128 ? 1 : 0); 578 579 if (MLX5_CAP_GEN_MAX(dev, dct)) 580 MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1); 581 582 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event)) 583 MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_event, 1); 584 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload)) 585 MLX5_SET(cmd_hca_cap, set_hca_cap, 586 pci_sync_for_fw_update_with_driver_unload, 1); 587 if (MLX5_CAP_GEN_MAX(dev, pcie_reset_using_hotreset_method)) 588 MLX5_SET(cmd_hca_cap, set_hca_cap, 589 pcie_reset_using_hotreset_method, 1); 590 591 if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports)) 592 MLX5_SET(cmd_hca_cap, 593 set_hca_cap, 594 num_vhca_ports, 595 MLX5_CAP_GEN_MAX(dev, num_vhca_ports)); 596 597 if (MLX5_CAP_GEN_MAX(dev, release_all_pages)) 598 MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1); 599 600 if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) 601 MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1); 602 603 mlx5_vhca_state_cap_handle(dev, set_hca_cap); 604 605 if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)) 606 MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix, 607 MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)); 608 609 if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce)) 610 MLX5_SET(cmd_hca_cap, set_hca_cap, roce, 611 mlx5_is_roce_on(dev)); 612 613 max_uc_list = max_uc_list_get_devlink_param(dev); 614 if (max_uc_list > 0) 615 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list, 616 ilog2(max_uc_list)); 617 618 /* enable absolute native port num */ 619 if (MLX5_CAP_GEN_MAX(dev, abs_native_port_num)) 620 MLX5_SET(cmd_hca_cap, set_hca_cap, abs_native_port_num, 1); 621 622 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 623 } 624 625 /* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the 626 * boot process. 627 * In case RoCE cap is writable in FW and user/devlink requested to change the 628 * cap, we are yet to query the final state of the above cap. 629 * Hence, the need for this function. 630 * 631 * Returns 632 * True: 633 * 1) RoCE cap is read only in FW and already disabled 634 * OR: 635 * 2) RoCE cap is writable in FW and user/devlink requested it off. 636 * 637 * In any other case, return False. 638 */ 639 static bool is_roce_fw_disabled(struct mlx5_core_dev *dev) 640 { 641 return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) || 642 (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce)); 643 } 644 645 static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx) 646 { 647 void *set_hca_cap; 648 int err; 649 650 if (is_roce_fw_disabled(dev)) 651 return 0; 652 653 err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE); 654 if (err) 655 return err; 656 657 if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) || 658 !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port)) 659 return 0; 660 661 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 662 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur, 663 MLX5_ST_SZ_BYTES(roce_cap)); 664 MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1); 665 666 if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default)) 667 MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1); 668 669 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE); 670 return err; 671 } 672 673 static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev, 674 void *set_ctx) 675 { 676 void *set_hca_cap; 677 int err; 678 679 if (!MLX5_CAP_GEN(dev, port_selection_cap)) 680 return 0; 681 682 err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION); 683 if (err) 684 return err; 685 686 if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) || 687 !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass)) 688 return 0; 689 690 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 691 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur, 692 MLX5_ST_SZ_BYTES(port_selection_cap)); 693 MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1); 694 695 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION); 696 697 return err; 698 } 699 700 static int set_hca_cap(struct mlx5_core_dev *dev) 701 { 702 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 703 void *set_ctx; 704 int err; 705 706 set_ctx = kzalloc(set_sz, GFP_KERNEL); 707 if (!set_ctx) 708 return -ENOMEM; 709 710 err = handle_hca_cap(dev, set_ctx); 711 if (err) { 712 mlx5_core_err(dev, "handle_hca_cap failed\n"); 713 goto out; 714 } 715 716 memset(set_ctx, 0, set_sz); 717 err = handle_hca_cap_atomic(dev, set_ctx); 718 if (err) { 719 mlx5_core_err(dev, "handle_hca_cap_atomic failed\n"); 720 goto out; 721 } 722 723 memset(set_ctx, 0, set_sz); 724 err = handle_hca_cap_odp(dev, set_ctx); 725 if (err) { 726 mlx5_core_err(dev, "handle_hca_cap_odp failed\n"); 727 goto out; 728 } 729 730 memset(set_ctx, 0, set_sz); 731 err = handle_hca_cap_roce(dev, set_ctx); 732 if (err) { 733 mlx5_core_err(dev, "handle_hca_cap_roce failed\n"); 734 goto out; 735 } 736 737 memset(set_ctx, 0, set_sz); 738 err = handle_hca_cap_2(dev, set_ctx); 739 if (err) { 740 mlx5_core_err(dev, "handle_hca_cap_2 failed\n"); 741 goto out; 742 } 743 744 memset(set_ctx, 0, set_sz); 745 err = handle_hca_cap_port_selection(dev, set_ctx); 746 if (err) { 747 mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n"); 748 goto out; 749 } 750 751 out: 752 kfree(set_ctx); 753 return err; 754 } 755 756 static int set_hca_ctrl(struct mlx5_core_dev *dev) 757 { 758 struct mlx5_reg_host_endianness he_in; 759 struct mlx5_reg_host_endianness he_out; 760 int err; 761 762 if (!mlx5_core_is_pf(dev)) 763 return 0; 764 765 memset(&he_in, 0, sizeof(he_in)); 766 he_in.he = MLX5_SET_HOST_ENDIANNESS; 767 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 768 &he_out, sizeof(he_out), 769 MLX5_REG_HOST_ENDIANNESS, 0, 1); 770 return err; 771 } 772 773 static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev) 774 { 775 int ret = 0; 776 777 /* Disable local_lb by default */ 778 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) 779 ret = mlx5_nic_vport_update_local_lb(dev, false); 780 781 return ret; 782 } 783 784 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) 785 { 786 u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {}; 787 788 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 789 MLX5_SET(enable_hca_in, in, function_id, func_id); 790 MLX5_SET(enable_hca_in, in, embedded_cpu_function, 791 dev->caps.embedded_cpu); 792 return mlx5_cmd_exec_in(dev, enable_hca, in); 793 } 794 795 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) 796 { 797 u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {}; 798 799 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 800 MLX5_SET(disable_hca_in, in, function_id, func_id); 801 MLX5_SET(enable_hca_in, in, embedded_cpu_function, 802 dev->caps.embedded_cpu); 803 return mlx5_cmd_exec_in(dev, disable_hca, in); 804 } 805 806 static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 807 { 808 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {}; 809 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {}; 810 u32 sup_issi; 811 int err; 812 813 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 814 err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out); 815 if (err) { 816 u32 syndrome = MLX5_GET(query_issi_out, query_out, syndrome); 817 u8 status = MLX5_GET(query_issi_out, query_out, status); 818 819 if (!status || syndrome == MLX5_DRIVER_SYND) { 820 mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n", 821 err, status, syndrome); 822 return err; 823 } 824 825 mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n"); 826 dev->issi = 0; 827 return 0; 828 } 829 830 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 831 832 if (sup_issi & (1 << 1)) { 833 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {}; 834 835 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 836 MLX5_SET(set_issi_in, set_in, current_issi, 1); 837 err = mlx5_cmd_exec_in(dev, set_issi, set_in); 838 if (err) { 839 mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n", 840 err); 841 return err; 842 } 843 844 dev->issi = 1; 845 846 return 0; 847 } else if (sup_issi & (1 << 0) || !sup_issi) { 848 return 0; 849 } 850 851 return -EOPNOTSUPP; 852 } 853 854 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, 855 const struct pci_device_id *id) 856 { 857 int err = 0; 858 859 mutex_init(&dev->pci_status_mutex); 860 pci_set_drvdata(dev->pdev, dev); 861 862 dev->bar_addr = pci_resource_start(pdev, 0); 863 864 err = mlx5_pci_enable_device(dev); 865 if (err) { 866 mlx5_core_err(dev, "Cannot enable PCI device, aborting\n"); 867 return err; 868 } 869 870 err = request_bar(pdev); 871 if (err) { 872 mlx5_core_err(dev, "error requesting BARs, aborting\n"); 873 goto err_disable; 874 } 875 876 pci_set_master(pdev); 877 878 err = set_dma_caps(pdev); 879 if (err) { 880 mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n"); 881 goto err_clr_master; 882 } 883 884 if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && 885 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && 886 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) 887 mlx5_core_dbg(dev, "Enabling pci atomics failed\n"); 888 889 dev->iseg = ioremap(dev->bar_addr, sizeof(*dev->iseg)); 890 if (!dev->iseg) { 891 err = -ENOMEM; 892 mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n"); 893 goto err_clr_master; 894 } 895 896 mlx5_pci_vsc_init(dev); 897 898 pci_enable_ptm(pdev); 899 900 return 0; 901 902 err_clr_master: 903 release_bar(dev->pdev); 904 err_disable: 905 mlx5_pci_disable_device(dev); 906 return err; 907 } 908 909 static void mlx5_pci_close(struct mlx5_core_dev *dev) 910 { 911 /* health work might still be active, and it needs pci bar in 912 * order to know the NIC state. Therefore, drain the health WQ 913 * before removing the pci bars 914 */ 915 mlx5_drain_health_wq(dev); 916 pci_disable_ptm(dev->pdev); 917 iounmap(dev->iseg); 918 release_bar(dev->pdev); 919 mlx5_pci_disable_device(dev); 920 } 921 922 static int mlx5_init_once(struct mlx5_core_dev *dev) 923 { 924 int err; 925 926 dev->priv.devc = mlx5_devcom_register_device(dev); 927 if (!dev->priv.devc) 928 mlx5_core_warn(dev, "failed to register devcom device\n"); 929 930 err = mlx5_query_board_id(dev); 931 if (err) { 932 mlx5_core_err(dev, "query board id failed\n"); 933 goto err_devcom; 934 } 935 936 err = mlx5_irq_table_init(dev); 937 if (err) { 938 mlx5_core_err(dev, "failed to initialize irq table\n"); 939 goto err_devcom; 940 } 941 942 err = mlx5_eq_table_init(dev); 943 if (err) { 944 mlx5_core_err(dev, "failed to initialize eq\n"); 945 goto err_irq_cleanup; 946 } 947 948 err = mlx5_fw_reset_init(dev); 949 if (err) { 950 mlx5_core_err(dev, "failed to initialize fw reset events\n"); 951 goto err_eq_cleanup; 952 } 953 954 mlx5_cq_debugfs_init(dev); 955 956 mlx5_init_reserved_gids(dev); 957 958 err = mlx5_init_clock(dev); 959 if (err) { 960 mlx5_core_err(dev, "failed to initialize hardware clock\n"); 961 goto err_tables_cleanup; 962 } 963 964 dev->vxlan = mlx5_vxlan_create(dev); 965 dev->geneve = mlx5_geneve_create(dev); 966 967 err = mlx5_init_rl_table(dev); 968 if (err) { 969 mlx5_core_err(dev, "Failed to init rate limiting\n"); 970 goto err_clock_cleanup; 971 } 972 973 err = mlx5_mpfs_init(dev); 974 if (err) { 975 mlx5_core_err(dev, "Failed to init l2 table %d\n", err); 976 goto err_rl_cleanup; 977 } 978 979 err = mlx5_sriov_init(dev); 980 if (err) { 981 mlx5_core_err(dev, "Failed to init sriov %d\n", err); 982 goto err_mpfs_cleanup; 983 } 984 985 err = mlx5_eswitch_init(dev); 986 if (err) { 987 mlx5_core_err(dev, "Failed to init eswitch %d\n", err); 988 goto err_sriov_cleanup; 989 } 990 991 err = mlx5_fpga_init(dev); 992 if (err) { 993 mlx5_core_err(dev, "Failed to init fpga device %d\n", err); 994 goto err_eswitch_cleanup; 995 } 996 997 err = mlx5_vhca_event_init(dev); 998 if (err) { 999 mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err); 1000 goto err_fpga_cleanup; 1001 } 1002 1003 err = mlx5_sf_hw_table_init(dev); 1004 if (err) { 1005 mlx5_core_err(dev, "Failed to init SF HW table %d\n", err); 1006 goto err_sf_hw_table_cleanup; 1007 } 1008 1009 err = mlx5_sf_table_init(dev); 1010 if (err) { 1011 mlx5_core_err(dev, "Failed to init SF table %d\n", err); 1012 goto err_sf_table_cleanup; 1013 } 1014 1015 err = mlx5_fs_core_alloc(dev); 1016 if (err) { 1017 mlx5_core_err(dev, "Failed to alloc flow steering\n"); 1018 goto err_fs; 1019 } 1020 1021 dev->dm = mlx5_dm_create(dev); 1022 dev->st = mlx5_st_create(dev); 1023 dev->tracer = mlx5_fw_tracer_create(dev); 1024 dev->hv_vhca = mlx5_hv_vhca_create(dev); 1025 dev->rsc_dump = mlx5_rsc_dump_create(dev); 1026 1027 return 0; 1028 1029 err_fs: 1030 mlx5_sf_table_cleanup(dev); 1031 err_sf_table_cleanup: 1032 mlx5_sf_hw_table_cleanup(dev); 1033 err_sf_hw_table_cleanup: 1034 mlx5_vhca_event_cleanup(dev); 1035 err_fpga_cleanup: 1036 mlx5_fpga_cleanup(dev); 1037 err_eswitch_cleanup: 1038 mlx5_eswitch_cleanup(dev->priv.eswitch); 1039 err_sriov_cleanup: 1040 mlx5_sriov_cleanup(dev); 1041 err_mpfs_cleanup: 1042 mlx5_mpfs_cleanup(dev); 1043 err_rl_cleanup: 1044 mlx5_cleanup_rl_table(dev); 1045 err_clock_cleanup: 1046 mlx5_geneve_destroy(dev->geneve); 1047 mlx5_vxlan_destroy(dev->vxlan); 1048 mlx5_cleanup_clock(dev); 1049 err_tables_cleanup: 1050 mlx5_cleanup_reserved_gids(dev); 1051 mlx5_cq_debugfs_cleanup(dev); 1052 mlx5_fw_reset_cleanup(dev); 1053 err_eq_cleanup: 1054 mlx5_eq_table_cleanup(dev); 1055 err_irq_cleanup: 1056 mlx5_irq_table_cleanup(dev); 1057 err_devcom: 1058 mlx5_devcom_unregister_device(dev->priv.devc); 1059 1060 return err; 1061 } 1062 1063 static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 1064 { 1065 mlx5_rsc_dump_destroy(dev); 1066 mlx5_hv_vhca_destroy(dev->hv_vhca); 1067 mlx5_fw_tracer_destroy(dev->tracer); 1068 mlx5_st_destroy(dev); 1069 mlx5_dm_cleanup(dev); 1070 mlx5_fs_core_free(dev); 1071 mlx5_sf_table_cleanup(dev); 1072 mlx5_sf_hw_table_cleanup(dev); 1073 mlx5_vhca_event_cleanup(dev); 1074 mlx5_fpga_cleanup(dev); 1075 mlx5_eswitch_cleanup(dev->priv.eswitch); 1076 mlx5_sriov_cleanup(dev); 1077 mlx5_mpfs_cleanup(dev); 1078 mlx5_cleanup_rl_table(dev); 1079 mlx5_geneve_destroy(dev->geneve); 1080 mlx5_vxlan_destroy(dev->vxlan); 1081 mlx5_cleanup_clock(dev); 1082 mlx5_cleanup_reserved_gids(dev); 1083 mlx5_cq_debugfs_cleanup(dev); 1084 mlx5_fw_reset_cleanup(dev); 1085 mlx5_eq_table_cleanup(dev); 1086 mlx5_irq_table_cleanup(dev); 1087 mlx5_devcom_unregister_device(dev->priv.devc); 1088 } 1089 1090 static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout) 1091 { 1092 int err; 1093 1094 mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 1095 fw_rev_min(dev), fw_rev_sub(dev)); 1096 1097 /* Only PFs hold the relevant PCIe information for this query */ 1098 if (mlx5_core_is_pf(dev)) 1099 pcie_print_link_status(dev->pdev); 1100 1101 /* wait for firmware to accept initialization segments configurations 1102 */ 1103 err = wait_fw_init(dev, timeout, 1104 mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL), 1105 "pre-initializing"); 1106 if (err) 1107 return err; 1108 1109 err = mlx5_cmd_enable(dev); 1110 if (err) { 1111 mlx5_core_err(dev, "Failed initializing command interface, aborting\n"); 1112 return err; 1113 } 1114 1115 mlx5_tout_query_iseg(dev); 1116 1117 err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0, "initializing"); 1118 if (err) 1119 goto err_cmd_cleanup; 1120 1121 dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); 1122 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP); 1123 1124 err = mlx5_core_enable_hca(dev, 0); 1125 if (err) { 1126 mlx5_core_err(dev, "enable hca failed\n"); 1127 goto err_cmd_cleanup; 1128 } 1129 1130 mlx5_start_health_poll(dev); 1131 1132 err = mlx5_core_set_issi(dev); 1133 if (err) { 1134 mlx5_core_err(dev, "failed to set issi\n"); 1135 goto stop_health_poll; 1136 } 1137 1138 err = mlx5_satisfy_startup_pages(dev, 1); 1139 if (err) { 1140 mlx5_core_err(dev, "failed to allocate boot pages\n"); 1141 goto stop_health_poll; 1142 } 1143 1144 err = mlx5_tout_query_dtor(dev); 1145 if (err) { 1146 mlx5_core_err(dev, "failed to read dtor\n"); 1147 goto reclaim_boot_pages; 1148 } 1149 1150 return 0; 1151 1152 reclaim_boot_pages: 1153 mlx5_reclaim_startup_pages(dev); 1154 stop_health_poll: 1155 mlx5_stop_health_poll(dev, boot); 1156 mlx5_core_disable_hca(dev, 0); 1157 err_cmd_cleanup: 1158 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); 1159 mlx5_cmd_disable(dev); 1160 1161 return err; 1162 } 1163 1164 static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot) 1165 { 1166 mlx5_reclaim_startup_pages(dev); 1167 mlx5_stop_health_poll(dev, boot); 1168 mlx5_core_disable_hca(dev, 0); 1169 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); 1170 mlx5_cmd_disable(dev); 1171 } 1172 1173 static int mlx5_function_open(struct mlx5_core_dev *dev) 1174 { 1175 int err; 1176 1177 err = set_hca_ctrl(dev); 1178 if (err) { 1179 mlx5_core_err(dev, "set_hca_ctrl failed\n"); 1180 return err; 1181 } 1182 1183 err = set_hca_cap(dev); 1184 if (err) { 1185 mlx5_core_err(dev, "set_hca_cap failed\n"); 1186 return err; 1187 } 1188 1189 err = mlx5_satisfy_startup_pages(dev, 0); 1190 if (err) { 1191 mlx5_core_err(dev, "failed to allocate init pages\n"); 1192 return err; 1193 } 1194 1195 err = mlx5_cmd_init_hca(dev, sw_owner_id); 1196 if (err) { 1197 mlx5_core_err(dev, "init hca failed\n"); 1198 return err; 1199 } 1200 1201 mlx5_set_driver_version(dev); 1202 1203 err = mlx5_query_hca_caps(dev); 1204 if (err) { 1205 mlx5_core_err(dev, "query hca failed\n"); 1206 return err; 1207 } 1208 mlx5_start_health_fw_log_up(dev); 1209 return 0; 1210 } 1211 1212 static int mlx5_function_close(struct mlx5_core_dev *dev) 1213 { 1214 int err; 1215 1216 err = mlx5_cmd_teardown_hca(dev); 1217 if (err) { 1218 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); 1219 return err; 1220 } 1221 1222 return 0; 1223 } 1224 1225 static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout) 1226 { 1227 int err; 1228 1229 err = mlx5_function_enable(dev, boot, timeout); 1230 if (err) 1231 return err; 1232 1233 err = mlx5_function_open(dev); 1234 if (err) 1235 mlx5_function_disable(dev, boot); 1236 return err; 1237 } 1238 1239 static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) 1240 { 1241 int err = mlx5_function_close(dev); 1242 1243 if (!err) 1244 mlx5_function_disable(dev, boot); 1245 else 1246 mlx5_stop_health_poll(dev, boot); 1247 1248 return err; 1249 } 1250 1251 static int mlx5_load(struct mlx5_core_dev *dev) 1252 { 1253 int err; 1254 1255 err = mlx5_alloc_bfreg(dev, &dev->priv.bfreg, false, false); 1256 if (err) { 1257 mlx5_core_err(dev, "Failed allocating bfreg, %d\n", err); 1258 return err; 1259 } 1260 1261 mlx5_events_start(dev); 1262 mlx5_pagealloc_start(dev); 1263 1264 err = mlx5_irq_table_create(dev); 1265 if (err) { 1266 mlx5_core_err(dev, "Failed to alloc IRQs\n"); 1267 goto err_irq_table; 1268 } 1269 1270 err = mlx5_eq_table_create(dev); 1271 if (err) { 1272 mlx5_core_err(dev, "Failed to create EQs\n"); 1273 goto err_eq_table; 1274 } 1275 1276 mlx5_clock_load(dev); 1277 1278 err = mlx5_fw_tracer_init(dev->tracer); 1279 if (err) { 1280 mlx5_core_err(dev, "Failed to init FW tracer %d\n", err); 1281 mlx5_fw_tracer_destroy(dev->tracer); 1282 dev->tracer = NULL; 1283 } 1284 1285 mlx5_fw_reset_events_start(dev); 1286 mlx5_hv_vhca_init(dev->hv_vhca); 1287 1288 err = mlx5_rsc_dump_init(dev); 1289 if (err) { 1290 mlx5_core_err(dev, "Failed to init Resource dump %d\n", err); 1291 mlx5_rsc_dump_destroy(dev); 1292 dev->rsc_dump = NULL; 1293 } 1294 1295 err = mlx5_fpga_device_start(dev); 1296 if (err) { 1297 mlx5_core_err(dev, "fpga device start failed %d\n", err); 1298 goto err_fpga_start; 1299 } 1300 1301 err = mlx5_fs_core_init(dev); 1302 if (err) { 1303 mlx5_core_err(dev, "Failed to init flow steering\n"); 1304 goto err_fs; 1305 } 1306 1307 err = mlx5_core_set_hca_defaults(dev); 1308 if (err) { 1309 mlx5_core_err(dev, "Failed to set hca defaults\n"); 1310 goto err_set_hca; 1311 } 1312 1313 mlx5_vhca_event_start(dev); 1314 1315 err = mlx5_ec_init(dev); 1316 if (err) { 1317 mlx5_core_err(dev, "Failed to init embedded CPU\n"); 1318 goto err_ec; 1319 } 1320 1321 mlx5_lag_add_mdev(dev); 1322 err = mlx5_sriov_attach(dev); 1323 if (err) { 1324 mlx5_core_err(dev, "sriov init failed %d\n", err); 1325 goto err_sriov; 1326 } 1327 1328 mlx5_sf_dev_table_create(dev); 1329 1330 err = mlx5_devlink_traps_register(priv_to_devlink(dev)); 1331 if (err) 1332 goto err_traps_reg; 1333 1334 return 0; 1335 1336 err_traps_reg: 1337 mlx5_sf_dev_table_destroy(dev); 1338 mlx5_sriov_detach(dev); 1339 err_sriov: 1340 mlx5_lag_remove_mdev(dev); 1341 mlx5_ec_cleanup(dev); 1342 err_ec: 1343 mlx5_vhca_event_stop(dev); 1344 err_set_hca: 1345 mlx5_fs_core_cleanup(dev); 1346 err_fs: 1347 mlx5_fpga_device_stop(dev); 1348 err_fpga_start: 1349 mlx5_rsc_dump_cleanup(dev); 1350 mlx5_hv_vhca_cleanup(dev->hv_vhca); 1351 mlx5_fw_reset_events_stop(dev); 1352 mlx5_fw_tracer_cleanup(dev->tracer); 1353 mlx5_clock_unload(dev); 1354 mlx5_eq_table_destroy(dev); 1355 err_eq_table: 1356 mlx5_irq_table_destroy(dev); 1357 err_irq_table: 1358 mlx5_pagealloc_stop(dev); 1359 mlx5_events_stop(dev); 1360 mlx5_free_bfreg(dev, &dev->priv.bfreg); 1361 return err; 1362 } 1363 1364 static void mlx5_unload(struct mlx5_core_dev *dev) 1365 { 1366 mlx5_eswitch_disable(dev->priv.eswitch); 1367 mlx5_devlink_traps_unregister(priv_to_devlink(dev)); 1368 mlx5_vhca_event_stop(dev); 1369 mlx5_sf_dev_table_destroy(dev); 1370 mlx5_sriov_detach(dev); 1371 mlx5_lag_remove_mdev(dev); 1372 mlx5_ec_cleanup(dev); 1373 mlx5_sf_hw_table_destroy(dev); 1374 mlx5_fs_core_cleanup(dev); 1375 mlx5_fpga_device_stop(dev); 1376 mlx5_rsc_dump_cleanup(dev); 1377 mlx5_hv_vhca_cleanup(dev->hv_vhca); 1378 mlx5_fw_reset_events_stop(dev); 1379 mlx5_fw_tracer_cleanup(dev->tracer); 1380 mlx5_clock_unload(dev); 1381 mlx5_eq_table_destroy(dev); 1382 mlx5_irq_table_destroy(dev); 1383 mlx5_pagealloc_stop(dev); 1384 mlx5_events_stop(dev); 1385 mlx5_free_bfreg(dev, &dev->priv.bfreg); 1386 } 1387 1388 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev) 1389 { 1390 bool light_probe = mlx5_dev_is_lightweight(dev); 1391 int err = 0; 1392 1393 mutex_lock(&dev->intf_state_mutex); 1394 dev->state = MLX5_DEVICE_STATE_UP; 1395 1396 err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); 1397 if (err) 1398 goto err_function; 1399 1400 err = mlx5_init_once(dev); 1401 if (err) { 1402 mlx5_core_err(dev, "sw objs init failed\n"); 1403 goto function_teardown; 1404 } 1405 1406 /* In case of light_probe, mlx5_devlink is already registered. 1407 * Hence, don't register devlink again. 1408 */ 1409 if (!light_probe) { 1410 err = mlx5_devlink_params_register(priv_to_devlink(dev)); 1411 if (err) 1412 goto err_devlink_params_reg; 1413 } 1414 1415 err = mlx5_load(dev); 1416 if (err) 1417 goto err_load; 1418 1419 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1420 1421 err = mlx5_register_device(dev); 1422 if (err) 1423 goto err_register; 1424 1425 err = mlx5_crdump_enable(dev); 1426 if (err) 1427 mlx5_core_err(dev, "mlx5_crdump_enable failed with error code %d\n", err); 1428 1429 err = mlx5_hwmon_dev_register(dev); 1430 if (err) 1431 mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n", err); 1432 1433 mutex_unlock(&dev->intf_state_mutex); 1434 return 0; 1435 1436 err_register: 1437 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1438 mlx5_unload(dev); 1439 err_load: 1440 if (!light_probe) 1441 mlx5_devlink_params_unregister(priv_to_devlink(dev)); 1442 err_devlink_params_reg: 1443 mlx5_cleanup_once(dev); 1444 function_teardown: 1445 mlx5_function_teardown(dev, true); 1446 err_function: 1447 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1448 mutex_unlock(&dev->intf_state_mutex); 1449 return err; 1450 } 1451 1452 int mlx5_init_one(struct mlx5_core_dev *dev) 1453 { 1454 struct devlink *devlink = priv_to_devlink(dev); 1455 int err; 1456 1457 devl_lock(devlink); 1458 if (dev->shd) { 1459 err = devl_nested_devlink_set(dev->shd, devlink); 1460 if (err) 1461 goto unlock; 1462 } 1463 devl_register(devlink); 1464 err = mlx5_init_one_devl_locked(dev); 1465 if (err) 1466 devl_unregister(devlink); 1467 unlock: 1468 devl_unlock(devlink); 1469 return err; 1470 } 1471 1472 void mlx5_uninit_one(struct mlx5_core_dev *dev) 1473 { 1474 struct devlink *devlink = priv_to_devlink(dev); 1475 1476 devl_lock(devlink); 1477 mutex_lock(&dev->intf_state_mutex); 1478 1479 mlx5_hwmon_dev_unregister(dev); 1480 mlx5_crdump_disable(dev); 1481 mlx5_unregister_device(dev); 1482 1483 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 1484 mlx5_core_warn(dev, "%s: interface is down, NOP\n", 1485 __func__); 1486 mlx5_devlink_params_unregister(priv_to_devlink(dev)); 1487 mlx5_cleanup_once(dev); 1488 goto out; 1489 } 1490 1491 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1492 mlx5_unload(dev); 1493 mlx5_devlink_params_unregister(priv_to_devlink(dev)); 1494 mlx5_cleanup_once(dev); 1495 mlx5_function_teardown(dev, true); 1496 out: 1497 mutex_unlock(&dev->intf_state_mutex); 1498 devl_unregister(devlink); 1499 devl_unlock(devlink); 1500 } 1501 1502 int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) 1503 { 1504 int err = 0; 1505 u64 timeout; 1506 1507 devl_assert_locked(priv_to_devlink(dev)); 1508 mutex_lock(&dev->intf_state_mutex); 1509 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 1510 mlx5_core_warn(dev, "interface is up, NOP\n"); 1511 goto out; 1512 } 1513 /* remove any previous indication of internal error */ 1514 dev->state = MLX5_DEVICE_STATE_UP; 1515 1516 if (recovery) 1517 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT); 1518 else 1519 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT); 1520 err = mlx5_function_setup(dev, false, timeout); 1521 if (err) 1522 goto err_function; 1523 1524 err = mlx5_load(dev); 1525 if (err) 1526 goto err_load; 1527 1528 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1529 1530 err = mlx5_attach_device(dev); 1531 if (err) 1532 goto err_attach; 1533 1534 mutex_unlock(&dev->intf_state_mutex); 1535 return 0; 1536 1537 err_attach: 1538 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1539 mlx5_unload(dev); 1540 err_load: 1541 mlx5_function_teardown(dev, false); 1542 err_function: 1543 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1544 out: 1545 mutex_unlock(&dev->intf_state_mutex); 1546 return err; 1547 } 1548 1549 int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) 1550 { 1551 struct devlink *devlink = priv_to_devlink(dev); 1552 int ret; 1553 1554 devl_lock(devlink); 1555 ret = mlx5_load_one_devl_locked(dev, recovery); 1556 devl_unlock(devlink); 1557 return ret; 1558 } 1559 1560 void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend) 1561 { 1562 devl_assert_locked(priv_to_devlink(dev)); 1563 mutex_lock(&dev->intf_state_mutex); 1564 1565 mlx5_detach_device(dev, suspend); 1566 1567 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 1568 mlx5_core_warn(dev, "%s: interface is down, NOP\n", 1569 __func__); 1570 goto out; 1571 } 1572 1573 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1574 mlx5_unload(dev); 1575 mlx5_function_teardown(dev, false); 1576 out: 1577 mutex_unlock(&dev->intf_state_mutex); 1578 } 1579 1580 void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend) 1581 { 1582 struct devlink *devlink = priv_to_devlink(dev); 1583 1584 devl_lock(devlink); 1585 mlx5_unload_one_devl_locked(dev, suspend); 1586 devl_unlock(devlink); 1587 } 1588 1589 /* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps. 1590 * A full query of hca_caps will be done when the device will reload. 1591 */ 1592 static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev) 1593 { 1594 int err; 1595 1596 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 1597 if (err) 1598 return err; 1599 1600 if (MLX5_CAP_GEN(dev, eth_net_offloads)) { 1601 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_ETHERNET_OFFLOADS, 1602 HCA_CAP_OPMOD_GET_CUR); 1603 if (err) 1604 return err; 1605 } 1606 1607 if (MLX5_CAP_GEN(dev, nic_flow_table) || 1608 MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) { 1609 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_FLOW_TABLE, 1610 HCA_CAP_OPMOD_GET_CUR); 1611 if (err) 1612 return err; 1613 } 1614 1615 if (MLX5_CAP_GEN_64(dev, general_obj_types) & 1616 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { 1617 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_VDPA_EMULATION, 1618 HCA_CAP_OPMOD_GET_CUR); 1619 if (err) 1620 return err; 1621 } 1622 1623 return 0; 1624 } 1625 1626 int mlx5_init_one_light(struct mlx5_core_dev *dev) 1627 { 1628 struct devlink *devlink = priv_to_devlink(dev); 1629 int err; 1630 1631 devl_lock(devlink); 1632 devl_register(devlink); 1633 dev->state = MLX5_DEVICE_STATE_UP; 1634 err = mlx5_function_enable(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); 1635 if (err) { 1636 mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err); 1637 goto out; 1638 } 1639 1640 err = mlx5_query_hca_caps_light(dev); 1641 if (err) { 1642 mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err); 1643 goto query_hca_caps_err; 1644 } 1645 1646 err = mlx5_devlink_params_register(priv_to_devlink(dev)); 1647 if (err) { 1648 mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n", err); 1649 goto query_hca_caps_err; 1650 } 1651 1652 devl_unlock(devlink); 1653 return 0; 1654 1655 query_hca_caps_err: 1656 mlx5_function_disable(dev, true); 1657 out: 1658 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1659 devl_unregister(devlink); 1660 devl_unlock(devlink); 1661 return err; 1662 } 1663 1664 void mlx5_uninit_one_light(struct mlx5_core_dev *dev) 1665 { 1666 struct devlink *devlink = priv_to_devlink(dev); 1667 1668 devl_lock(devlink); 1669 mlx5_devlink_params_unregister(priv_to_devlink(dev)); 1670 devl_unregister(devlink); 1671 devl_unlock(devlink); 1672 if (dev->state != MLX5_DEVICE_STATE_UP) 1673 return; 1674 mlx5_function_disable(dev, true); 1675 } 1676 1677 /* xxx_light() function are used in order to configure the device without full 1678 * init (light init). e.g.: There isn't a point in reload a device to light state. 1679 * Hence, mlx5_load_one_light() isn't needed. 1680 */ 1681 1682 void mlx5_unload_one_light(struct mlx5_core_dev *dev) 1683 { 1684 if (dev->state != MLX5_DEVICE_STATE_UP) 1685 return; 1686 mlx5_function_disable(dev, false); 1687 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1688 } 1689 1690 static const int types[] = { 1691 MLX5_CAP_GENERAL, 1692 MLX5_CAP_GENERAL_2, 1693 MLX5_CAP_ETHERNET_OFFLOADS, 1694 MLX5_CAP_IPOIB_ENHANCED_OFFLOADS, 1695 MLX5_CAP_ODP, 1696 MLX5_CAP_ATOMIC, 1697 MLX5_CAP_ROCE, 1698 MLX5_CAP_IPOIB_OFFLOADS, 1699 MLX5_CAP_FLOW_TABLE, 1700 MLX5_CAP_ESWITCH_FLOW_TABLE, 1701 MLX5_CAP_ESWITCH, 1702 MLX5_CAP_QOS, 1703 MLX5_CAP_DEBUG, 1704 MLX5_CAP_DEV_MEM, 1705 MLX5_CAP_DEV_EVENT, 1706 MLX5_CAP_TLS, 1707 MLX5_CAP_VDPA_EMULATION, 1708 MLX5_CAP_IPSEC, 1709 MLX5_CAP_PORT_SELECTION, 1710 MLX5_CAP_PSP, 1711 MLX5_CAP_MACSEC, 1712 MLX5_CAP_ADV_VIRTUALIZATION, 1713 MLX5_CAP_CRYPTO, 1714 MLX5_CAP_SHAMPO, 1715 MLX5_CAP_ADV_RDMA, 1716 MLX5_CAP_TLP_EMULATION, 1717 }; 1718 1719 static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) 1720 { 1721 int type; 1722 int i; 1723 1724 for (i = 0; i < ARRAY_SIZE(types); i++) { 1725 type = types[i]; 1726 kfree(dev->caps.hca[type]); 1727 } 1728 } 1729 1730 static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev) 1731 { 1732 struct mlx5_hca_cap *cap; 1733 int type; 1734 int i; 1735 1736 for (i = 0; i < ARRAY_SIZE(types); i++) { 1737 cap = kzalloc_obj(*cap); 1738 if (!cap) 1739 goto err; 1740 type = types[i]; 1741 dev->caps.hca[type] = cap; 1742 } 1743 1744 return 0; 1745 1746 err: 1747 mlx5_hca_caps_free(dev); 1748 return -ENOMEM; 1749 } 1750 1751 static int mlx5_notifiers_init(struct mlx5_core_dev *dev) 1752 { 1753 int err; 1754 1755 err = mlx5_events_init(dev); 1756 if (err) { 1757 mlx5_core_err(dev, "failed to initialize events\n"); 1758 return err; 1759 } 1760 1761 BLOCKING_INIT_NOTIFIER_HEAD(&dev->priv.esw_n_head); 1762 mlx5_vhca_state_notifier_init(dev); 1763 1764 err = mlx5_sf_hw_notifier_init(dev); 1765 if (err) 1766 goto err_sf_hw_notifier; 1767 1768 err = mlx5_sf_notifiers_init(dev); 1769 if (err) 1770 goto err_sf_notifiers; 1771 1772 err = mlx5_sf_dev_notifier_init(dev); 1773 if (err) 1774 goto err_sf_dev_notifier; 1775 1776 return 0; 1777 1778 err_sf_dev_notifier: 1779 mlx5_sf_notifiers_cleanup(dev); 1780 err_sf_notifiers: 1781 mlx5_sf_hw_notifier_cleanup(dev); 1782 err_sf_hw_notifier: 1783 mlx5_events_cleanup(dev); 1784 return err; 1785 } 1786 1787 static void mlx5_notifiers_cleanup(struct mlx5_core_dev *dev) 1788 { 1789 mlx5_sf_dev_notifier_cleanup(dev); 1790 mlx5_sf_notifiers_cleanup(dev); 1791 mlx5_sf_hw_notifier_cleanup(dev); 1792 mlx5_events_cleanup(dev); 1793 } 1794 1795 int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) 1796 { 1797 struct mlx5_priv *priv = &dev->priv; 1798 int err; 1799 1800 memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); 1801 lockdep_register_key(&dev->lock_key); 1802 mutex_init(&dev->intf_state_mutex); 1803 lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); 1804 mutex_init(&dev->mlx5e_res.uplink_netdev_lock); 1805 mutex_init(&dev->wc_state_lock); 1806 1807 mutex_init(&priv->bfregs.reg_head.lock); 1808 mutex_init(&priv->bfregs.wc_head.lock); 1809 INIT_LIST_HEAD(&priv->bfregs.reg_head.list); 1810 INIT_LIST_HEAD(&priv->bfregs.wc_head.list); 1811 1812 mutex_init(&priv->alloc_mutex); 1813 mutex_init(&priv->pgdir_mutex); 1814 INIT_LIST_HEAD(&priv->pgdir_list); 1815 1816 priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev)); 1817 priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device), 1818 mlx5_debugfs_root); 1819 1820 INIT_LIST_HEAD(&priv->traps); 1821 1822 err = mlx5_cmd_init(dev); 1823 if (err) { 1824 mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n"); 1825 goto err_cmd_init; 1826 } 1827 1828 err = mlx5_tout_init(dev); 1829 if (err) { 1830 mlx5_core_err(dev, "Failed initializing timeouts, aborting\n"); 1831 goto err_timeout_init; 1832 } 1833 1834 err = mlx5_health_init(dev); 1835 if (err) 1836 goto err_health_init; 1837 1838 err = mlx5_pagealloc_init(dev); 1839 if (err) 1840 goto err_pagealloc_init; 1841 1842 err = mlx5_adev_init(dev); 1843 if (err) 1844 goto err_adev_init; 1845 1846 err = mlx5_hca_caps_alloc(dev); 1847 if (err) 1848 goto err_hca_caps; 1849 1850 err = mlx5_notifiers_init(dev); 1851 if (err) 1852 goto err_hca_caps; 1853 1854 /* The conjunction of sw_vhca_id with sw_owner_id will be a global 1855 * unique id per function which uses mlx5_core. 1856 * Those values are supplied to FW as part of the init HCA command to 1857 * be used by both driver and FW when it's applicable. 1858 */ 1859 dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1, 1860 MAX_SW_VHCA_ID, 1861 GFP_KERNEL); 1862 if (dev->priv.sw_vhca_id < 0) 1863 mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n", 1864 dev->priv.sw_vhca_id); 1865 1866 return 0; 1867 1868 err_hca_caps: 1869 mlx5_adev_cleanup(dev); 1870 err_adev_init: 1871 mlx5_pagealloc_cleanup(dev); 1872 err_pagealloc_init: 1873 mlx5_health_cleanup(dev); 1874 err_health_init: 1875 mlx5_tout_cleanup(dev); 1876 err_timeout_init: 1877 mlx5_cmd_cleanup(dev); 1878 err_cmd_init: 1879 debugfs_remove(dev->priv.dbg.dbg_root); 1880 mutex_destroy(&priv->pgdir_mutex); 1881 mutex_destroy(&priv->alloc_mutex); 1882 mutex_destroy(&priv->bfregs.wc_head.lock); 1883 mutex_destroy(&priv->bfregs.reg_head.lock); 1884 mutex_destroy(&dev->intf_state_mutex); 1885 lockdep_unregister_key(&dev->lock_key); 1886 return err; 1887 } 1888 1889 void mlx5_mdev_uninit(struct mlx5_core_dev *dev) 1890 { 1891 struct mlx5_priv *priv = &dev->priv; 1892 1893 if (priv->sw_vhca_id > 0) 1894 ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); 1895 1896 mlx5_notifiers_cleanup(dev); 1897 mlx5_hca_caps_free(dev); 1898 mlx5_adev_cleanup(dev); 1899 mlx5_pagealloc_cleanup(dev); 1900 mlx5_health_cleanup(dev); 1901 mlx5_tout_cleanup(dev); 1902 mlx5_cmd_cleanup(dev); 1903 debugfs_remove_recursive(dev->priv.dbg.dbg_root); 1904 mutex_destroy(&priv->pgdir_mutex); 1905 mutex_destroy(&priv->alloc_mutex); 1906 mutex_destroy(&priv->bfregs.wc_head.lock); 1907 mutex_destroy(&priv->bfregs.reg_head.lock); 1908 mutex_destroy(&dev->wc_state_lock); 1909 mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock); 1910 mutex_destroy(&dev->intf_state_mutex); 1911 lockdep_unregister_key(&dev->lock_key); 1912 } 1913 1914 static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) 1915 { 1916 struct mlx5_core_dev *dev; 1917 struct devlink *devlink; 1918 int err; 1919 1920 devlink = mlx5_devlink_alloc(&pdev->dev); 1921 if (!devlink) { 1922 dev_err(&pdev->dev, "devlink alloc failed\n"); 1923 return -ENOMEM; 1924 } 1925 1926 dev = devlink_priv(devlink); 1927 dev->device = &pdev->dev; 1928 dev->pdev = pdev; 1929 1930 dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ? 1931 MLX5_COREDEV_VF : MLX5_COREDEV_PF; 1932 1933 dev->priv.adev_idx = mlx5_adev_idx_alloc(); 1934 if (dev->priv.adev_idx < 0) { 1935 err = dev->priv.adev_idx; 1936 goto adev_init_err; 1937 } 1938 1939 err = mlx5_mdev_init(dev, prof_sel); 1940 if (err) 1941 goto mdev_init_err; 1942 1943 err = mlx5_pci_init(dev, pdev, id); 1944 if (err) { 1945 mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n", 1946 err); 1947 goto pci_init_err; 1948 } 1949 1950 err = mlx5_shd_init(dev); 1951 if (err) { 1952 mlx5_core_err(dev, "mlx5_shd_init failed with error code %d\n", 1953 err); 1954 goto shd_init_err; 1955 } 1956 1957 err = mlx5_init_one(dev); 1958 if (err) { 1959 mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n", 1960 err); 1961 goto err_init_one; 1962 } 1963 1964 mlx5_vhca_debugfs_init(dev); 1965 1966 pci_save_state(pdev); 1967 return 0; 1968 1969 err_init_one: 1970 mlx5_shd_uninit(dev); 1971 shd_init_err: 1972 mlx5_pci_close(dev); 1973 pci_init_err: 1974 mlx5_mdev_uninit(dev); 1975 mdev_init_err: 1976 mlx5_adev_idx_free(dev->priv.adev_idx); 1977 adev_init_err: 1978 mlx5_devlink_free(devlink); 1979 1980 return err; 1981 } 1982 1983 static void remove_one(struct pci_dev *pdev) 1984 { 1985 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1986 struct devlink *devlink = priv_to_devlink(dev); 1987 1988 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state); 1989 mlx5_drain_fw_reset(dev); 1990 mlx5_drain_health_wq(dev); 1991 mlx5_sriov_disable(pdev, false); 1992 mlx5_uninit_one(dev); 1993 mlx5_shd_uninit(dev); 1994 mlx5_pci_close(dev); 1995 mlx5_mdev_uninit(dev); 1996 mlx5_adev_idx_free(dev->priv.adev_idx); 1997 mlx5_devlink_free(devlink); 1998 } 1999 2000 #define mlx5_pci_trace(dev, fmt, ...) ({ \ 2001 struct mlx5_core_dev *__dev = (dev); \ 2002 mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \ 2003 __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \ 2004 __dev->pci_status, ##__VA_ARGS__); \ 2005 }) 2006 2007 static const char *result2str(enum pci_ers_result result) 2008 { 2009 return result == PCI_ERS_RESULT_NEED_RESET ? "need reset" : 2010 result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" : 2011 result == PCI_ERS_RESULT_RECOVERED ? "recovered" : 2012 "unknown"; 2013 } 2014 2015 static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 2016 pci_channel_state_t state) 2017 { 2018 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2019 enum pci_ers_result res; 2020 2021 mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state); 2022 2023 mlx5_enter_error_state(dev, false); 2024 mlx5_error_sw_reset(dev); 2025 mlx5_unload_one(dev, false); 2026 mlx5_drain_health_wq(dev); 2027 mlx5_pci_disable_device(dev); 2028 2029 res = state == pci_channel_io_perm_failure ? 2030 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 2031 2032 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n", 2033 __func__, dev->state, dev->pci_status, res, result2str(res)); 2034 return res; 2035 } 2036 2037 /* wait for the device to show vital signs by waiting 2038 * for the health counter to start counting. 2039 */ 2040 static int wait_vital(struct pci_dev *pdev) 2041 { 2042 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2043 struct mlx5_core_health *health = &dev->priv.health; 2044 const int niter = 100; 2045 u32 last_count = 0; 2046 u32 count; 2047 int i; 2048 2049 for (i = 0; i < niter; i++) { 2050 count = ioread32be(health->health_counter); 2051 if (count && count != 0xffffffff) { 2052 if (last_count && last_count != count) { 2053 mlx5_core_info(dev, 2054 "wait vital counter value 0x%x after %d iterations\n", 2055 count, i); 2056 return 0; 2057 } 2058 last_count = count; 2059 } 2060 msleep(50); 2061 } 2062 2063 return -ETIMEDOUT; 2064 } 2065 2066 static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 2067 { 2068 enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT; 2069 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2070 int err; 2071 2072 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n", 2073 __func__, dev->state, dev->pci_status); 2074 2075 err = mlx5_pci_enable_device(dev); 2076 if (err) { 2077 mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n", 2078 __func__, err); 2079 goto out; 2080 } 2081 2082 pci_set_master(pdev); 2083 pci_restore_state(pdev); 2084 2085 err = wait_vital(pdev); 2086 if (err) { 2087 mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n", 2088 __func__, err); 2089 goto out; 2090 } 2091 2092 res = PCI_ERS_RESULT_RECOVERED; 2093 out: 2094 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n", 2095 __func__, dev->state, dev->pci_status, err, res, result2str(res)); 2096 return res; 2097 } 2098 2099 static void mlx5_pci_resume(struct pci_dev *pdev) 2100 { 2101 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2102 int err; 2103 2104 mlx5_pci_trace(dev, "Enter, loading driver..\n"); 2105 2106 err = mlx5_load_one(dev, false); 2107 2108 if (!err) 2109 devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter, 2110 DEVLINK_HEALTH_REPORTER_STATE_HEALTHY); 2111 2112 mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err, 2113 !err ? "recovered" : "Failed"); 2114 } 2115 2116 static const struct pci_error_handlers mlx5_err_handler = { 2117 .error_detected = mlx5_pci_err_detected, 2118 .slot_reset = mlx5_pci_slot_reset, 2119 .resume = mlx5_pci_resume 2120 }; 2121 2122 static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) 2123 { 2124 bool fast_teardown = false, force_teardown = false; 2125 int ret = 1; 2126 2127 fast_teardown = MLX5_CAP_GEN(dev, fast_teardown); 2128 force_teardown = MLX5_CAP_GEN(dev, force_teardown); 2129 2130 mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown); 2131 mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown); 2132 2133 if (!fast_teardown && !force_teardown) 2134 return -EOPNOTSUPP; 2135 2136 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 2137 mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); 2138 return -EAGAIN; 2139 } 2140 2141 /* Panic tear down fw command will stop the PCI bus communication 2142 * with the HCA, so the health poll is no longer needed. 2143 */ 2144 mlx5_stop_health_poll(dev, false); 2145 2146 ret = mlx5_cmd_fast_teardown_hca(dev); 2147 if (!ret) 2148 goto succeed; 2149 2150 ret = mlx5_cmd_force_teardown_hca(dev); 2151 if (!ret) 2152 goto succeed; 2153 2154 mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret); 2155 mlx5_start_health_poll(dev); 2156 return ret; 2157 2158 succeed: 2159 mlx5_enter_error_state(dev, true); 2160 2161 /* Some platforms requiring freeing the IRQ's in the shutdown 2162 * flow. If they aren't freed they can't be allocated after 2163 * kexec. There is no need to cleanup the mlx5_core software 2164 * contexts. 2165 */ 2166 mlx5_core_eq_free_irqs(dev); 2167 2168 return 0; 2169 } 2170 2171 static void shutdown(struct pci_dev *pdev) 2172 { 2173 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2174 int err; 2175 2176 mlx5_core_info(dev, "Shutdown was called\n"); 2177 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state); 2178 mlx5_drain_fw_reset(dev); 2179 mlx5_drain_health_wq(dev); 2180 err = mlx5_try_fast_unload(dev); 2181 if (err) 2182 mlx5_unload_one(dev, false); 2183 mlx5_pci_disable_device(dev); 2184 } 2185 2186 static int mlx5_suspend(struct pci_dev *pdev, pm_message_t state) 2187 { 2188 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2189 2190 mlx5_unload_one(dev, true); 2191 2192 return 0; 2193 } 2194 2195 static int mlx5_resume(struct pci_dev *pdev) 2196 { 2197 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 2198 2199 return mlx5_load_one(dev, false); 2200 } 2201 2202 static const struct pci_device_id mlx5_core_pci_table[] = { 2203 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) }, 2204 { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ 2205 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) }, 2206 { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ 2207 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) }, 2208 { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ 2209 { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */ 2210 { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */ 2211 { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */ 2212 { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */ 2213 { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */ 2214 { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */ 2215 { PCI_VDEVICE(MELLANOX, 0x101d) }, /* ConnectX-6 Dx */ 2216 { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */ 2217 { PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */ 2218 { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */ 2219 { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */ 2220 { PCI_VDEVICE(MELLANOX, 0x1025) }, /* ConnectX-9 */ 2221 { PCI_VDEVICE(MELLANOX, 0x1027) }, /* ConnectX-10 */ 2222 { PCI_VDEVICE(MELLANOX, 0x2101) }, /* ConnectX-10 NVLink-C2C */ 2223 { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ 2224 { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ 2225 { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ 2226 { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */ 2227 { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */ 2228 { 0, } 2229 }; 2230 2231 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 2232 2233 void mlx5_disable_device(struct mlx5_core_dev *dev) 2234 { 2235 mlx5_error_sw_reset(dev); 2236 mlx5_unload_one_devl_locked(dev, false); 2237 } 2238 2239 int mlx5_recover_device(struct mlx5_core_dev *dev) 2240 { 2241 if (!mlx5_core_is_sf(dev)) { 2242 mlx5_pci_disable_device(dev); 2243 if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED) 2244 return -EIO; 2245 } 2246 2247 return mlx5_load_one_devl_locked(dev, true); 2248 } 2249 2250 static struct pci_driver mlx5_core_driver = { 2251 .name = KBUILD_MODNAME, 2252 .id_table = mlx5_core_pci_table, 2253 .probe = probe_one, 2254 .remove = remove_one, 2255 .suspend = mlx5_suspend, 2256 .resume = mlx5_resume, 2257 .shutdown = shutdown, 2258 .err_handler = &mlx5_err_handler, 2259 .sriov_configure = mlx5_core_sriov_configure, 2260 .sriov_get_vf_total_msix = mlx5_sriov_get_vf_total_msix, 2261 .sriov_set_msix_vec_count = mlx5_core_sriov_set_msix_vec_count, 2262 }; 2263 2264 /** 2265 * mlx5_vf_get_core_dev - Get the mlx5 core device from a given VF PCI device if 2266 * mlx5_core is its driver. 2267 * @pdev: The associated PCI device. 2268 * 2269 * Upon return the interface state lock stay held to let caller uses it safely. 2270 * Caller must ensure to use the returned mlx5 device for a narrow window 2271 * and put it back with mlx5_vf_put_core_dev() immediately once usage was over. 2272 * 2273 * Return: Pointer to the associated mlx5_core_dev or NULL. 2274 */ 2275 struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev) 2276 { 2277 struct mlx5_core_dev *mdev; 2278 2279 mdev = pci_iov_get_pf_drvdata(pdev, &mlx5_core_driver); 2280 if (IS_ERR(mdev)) 2281 return NULL; 2282 2283 mutex_lock(&mdev->intf_state_mutex); 2284 if (!test_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state)) { 2285 mutex_unlock(&mdev->intf_state_mutex); 2286 return NULL; 2287 } 2288 2289 return mdev; 2290 } 2291 EXPORT_SYMBOL(mlx5_vf_get_core_dev); 2292 2293 /** 2294 * mlx5_vf_put_core_dev - Put the mlx5 core device back. 2295 * @mdev: The mlx5 core device. 2296 * 2297 * Upon return the interface state lock is unlocked and caller should not 2298 * access the mdev any more. 2299 */ 2300 void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev) 2301 { 2302 mutex_unlock(&mdev->intf_state_mutex); 2303 } 2304 EXPORT_SYMBOL(mlx5_vf_put_core_dev); 2305 2306 static void mlx5_core_verify_params(void) 2307 { 2308 if (prof_sel >= ARRAY_SIZE(profile)) { 2309 pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n", 2310 prof_sel, 2311 ARRAY_SIZE(profile) - 1, 2312 MLX5_DEFAULT_PROF); 2313 prof_sel = MLX5_DEFAULT_PROF; 2314 } 2315 } 2316 2317 static int __init mlx5_init(void) 2318 { 2319 int err; 2320 2321 WARN_ONCE(strcmp(MLX5_ADEV_NAME, KBUILD_MODNAME), 2322 "mlx5_core name not in sync with kernel module name"); 2323 2324 get_random_bytes(&sw_owner_id, sizeof(sw_owner_id)); 2325 2326 mlx5_core_verify_params(); 2327 mlx5_register_debugfs(); 2328 2329 err = mlx5e_init(); 2330 if (err) 2331 goto err_debug; 2332 2333 err = mlx5_sf_driver_register(); 2334 if (err) 2335 goto err_sf; 2336 2337 err = pci_register_driver(&mlx5_core_driver); 2338 if (err) 2339 goto err_pci; 2340 2341 return 0; 2342 2343 err_pci: 2344 mlx5_sf_driver_unregister(); 2345 err_sf: 2346 mlx5e_cleanup(); 2347 err_debug: 2348 mlx5_unregister_debugfs(); 2349 return err; 2350 } 2351 2352 static void __exit mlx5_cleanup(void) 2353 { 2354 pci_unregister_driver(&mlx5_core_driver); 2355 mlx5_sf_driver_unregister(); 2356 mlx5e_cleanup(); 2357 mlx5_unregister_debugfs(); 2358 } 2359 2360 module_init(mlx5_init); 2361 module_exit(mlx5_cleanup); 2362