1 /* 2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/errno.h> 35 #include <linux/pci.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/slab.h> 38 #include <linux/delay.h> 39 #include <linux/random.h> 40 #include <linux/mlx5/driver.h> 41 #include <linux/mlx5/eq.h> 42 #include <linux/debugfs.h> 43 44 #include "mlx5_core.h" 45 #include "lib/eq.h" 46 #include "lib/tout.h" 47 #define CREATE_TRACE_POINTS 48 #include "diag/cmd_tracepoint.h" 49 50 struct mlx5_ifc_mbox_out_bits { 51 u8 status[0x8]; 52 u8 reserved_at_8[0x18]; 53 54 u8 syndrome[0x20]; 55 56 u8 reserved_at_40[0x40]; 57 }; 58 59 struct mlx5_ifc_mbox_in_bits { 60 u8 opcode[0x10]; 61 u8 uid[0x10]; 62 63 u8 reserved_at_20[0x10]; 64 u8 op_mod[0x10]; 65 66 u8 reserved_at_40[0x40]; 67 }; 68 69 enum { 70 CMD_IF_REV = 5, 71 }; 72 73 enum { 74 CMD_MODE_POLLING, 75 CMD_MODE_EVENTS 76 }; 77 78 enum { 79 MLX5_CMD_DELIVERY_STAT_OK = 0x0, 80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, 81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2, 82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, 83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, 84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, 85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6, 86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, 87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, 88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, 89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, 90 }; 91 92 static u16 in_to_opcode(void *in) 93 { 94 return MLX5_GET(mbox_in, in, opcode); 95 } 96 97 static u16 in_to_uid(void *in) 98 { 99 return MLX5_GET(mbox_in, in, uid); 100 } 101 102 /* Returns true for opcodes that might be triggered very frequently and throttle 103 * the command interface. Limit their command slots usage. 104 */ 105 static bool mlx5_cmd_is_throttle_opcode(u16 op) 106 { 107 switch (op) { 108 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 109 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 110 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 111 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 112 case MLX5_CMD_OP_SYNC_CRYPTO: 113 return true; 114 } 115 return false; 116 } 117 118 static struct mlx5_cmd_work_ent * 119 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in, 120 struct mlx5_cmd_msg *out, void *uout, int uout_size, 121 mlx5_cmd_cbk_t cbk, void *context, int page_queue) 122 { 123 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; 124 struct mlx5_cmd_work_ent *ent; 125 126 ent = kzalloc_obj(*ent, alloc_flags); 127 if (!ent) 128 return ERR_PTR(-ENOMEM); 129 130 ent->idx = -EINVAL; 131 ent->in = in; 132 ent->out = out; 133 ent->uout = uout; 134 ent->uout_size = uout_size; 135 ent->callback = cbk; 136 ent->context = context; 137 ent->cmd = cmd; 138 ent->page_queue = page_queue; 139 ent->op = in_to_opcode(in->first.data); 140 refcount_set(&ent->refcnt, 1); 141 142 return ent; 143 } 144 145 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent) 146 { 147 kfree(ent); 148 } 149 150 static u8 alloc_token(struct mlx5_cmd *cmd) 151 { 152 u8 token; 153 154 spin_lock(&cmd->token_lock); 155 cmd->token++; 156 if (cmd->token == 0) 157 cmd->token++; 158 token = cmd->token; 159 spin_unlock(&cmd->token_lock); 160 161 return token; 162 } 163 164 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent) 165 { 166 unsigned long flags; 167 int ret; 168 169 spin_lock_irqsave(&cmd->alloc_lock, flags); 170 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds); 171 if (ret < cmd->vars.max_reg_cmds) { 172 clear_bit(ret, &cmd->vars.bitmask); 173 ent->idx = ret; 174 cmd->ent_arr[ent->idx] = ent; 175 } 176 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 177 178 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM; 179 } 180 181 static void cmd_free_index(struct mlx5_cmd *cmd, int idx) 182 { 183 lockdep_assert_held(&cmd->alloc_lock); 184 cmd->ent_arr[idx] = NULL; 185 set_bit(idx, &cmd->vars.bitmask); 186 } 187 188 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent) 189 { 190 refcount_inc(&ent->refcnt); 191 } 192 193 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) 194 { 195 struct mlx5_cmd *cmd = ent->cmd; 196 unsigned long flags; 197 198 spin_lock_irqsave(&cmd->alloc_lock, flags); 199 if (!refcount_dec_and_test(&ent->refcnt)) { 200 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 201 return; 202 } 203 204 if (ent->idx >= 0) { 205 cmd_free_index(cmd, ent->idx); 206 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem); 207 } 208 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 209 210 cmd_free_ent(ent); 211 } 212 213 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) 214 { 215 return cmd->cmd_buf + (idx << cmd->vars.log_stride); 216 } 217 218 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg) 219 { 220 int size = msg->len; 221 int blen = size - min_t(int, sizeof(msg->first.data), size); 222 223 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE); 224 } 225 226 static u8 xor8_buf(void *buf, size_t offset, int len) 227 { 228 u8 *ptr = buf; 229 u8 sum = 0; 230 int i; 231 int end = len + offset; 232 233 for (i = offset; i < end; i++) 234 sum ^= ptr[i]; 235 236 return sum; 237 } 238 239 static int verify_block_sig(struct mlx5_cmd_prot_block *block) 240 { 241 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 242 int xor_len = sizeof(*block) - sizeof(block->data) - 1; 243 244 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff) 245 return -EHWPOISON; 246 247 if (xor8_buf(block, 0, sizeof(*block)) != 0xff) 248 return -EHWPOISON; 249 250 return 0; 251 } 252 253 static void calc_block_sig(struct mlx5_cmd_prot_block *block) 254 { 255 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2; 256 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 257 258 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len); 259 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1); 260 } 261 262 static void calc_chain_sig(struct mlx5_cmd_msg *msg) 263 { 264 struct mlx5_cmd_mailbox *next = msg->next; 265 int n = mlx5_calc_cmd_blocks(msg); 266 int i = 0; 267 268 for (i = 0; i < n && next; i++) { 269 calc_block_sig(next->buf); 270 next = next->next; 271 } 272 } 273 274 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum) 275 { 276 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 277 if (csum) { 278 calc_chain_sig(ent->in); 279 calc_chain_sig(ent->out); 280 } 281 } 282 283 static void poll_timeout(struct mlx5_cmd_work_ent *ent) 284 { 285 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); 286 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD); 287 unsigned long poll_end; 288 u8 own; 289 290 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000); 291 292 do { 293 own = READ_ONCE(ent->lay->status_own); 294 if (!(own & CMD_OWNER_HW)) { 295 ent->ret = 0; 296 return; 297 } 298 cond_resched(); 299 if (mlx5_cmd_is_down(dev)) { 300 ent->ret = -ENXIO; 301 return; 302 } 303 } while (time_before(jiffies, poll_end)); 304 305 ent->ret = -ETIMEDOUT; 306 } 307 308 static int verify_signature(struct mlx5_cmd_work_ent *ent) 309 { 310 struct mlx5_cmd_mailbox *next = ent->out->next; 311 int n = mlx5_calc_cmd_blocks(ent->out); 312 int err; 313 u8 sig; 314 int i = 0; 315 316 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 317 if (sig != 0xff) 318 return -EHWPOISON; 319 320 for (i = 0; i < n && next; i++) { 321 err = verify_block_sig(next->buf); 322 if (err) 323 return -EHWPOISON; 324 325 next = next->next; 326 } 327 328 return 0; 329 } 330 331 static void dump_buf(void *buf, int size, int data_only, int offset, int idx) 332 { 333 __be32 *p = buf; 334 int i; 335 336 for (i = 0; i < size; i += 16) { 337 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset, 338 be32_to_cpu(p[0]), be32_to_cpu(p[1]), 339 be32_to_cpu(p[2]), be32_to_cpu(p[3])); 340 p += 4; 341 offset += 16; 342 } 343 if (!data_only) 344 pr_debug("\n"); 345 } 346 347 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 348 u32 *synd, u8 *status) 349 { 350 *synd = 0; 351 *status = 0; 352 353 switch (op) { 354 case MLX5_CMD_OP_TEARDOWN_HCA: 355 case MLX5_CMD_OP_DISABLE_HCA: 356 case MLX5_CMD_OP_MANAGE_PAGES: 357 case MLX5_CMD_OP_DESTROY_MKEY: 358 case MLX5_CMD_OP_DESTROY_EQ: 359 case MLX5_CMD_OP_DESTROY_CQ: 360 case MLX5_CMD_OP_DESTROY_QP: 361 case MLX5_CMD_OP_DESTROY_PSV: 362 case MLX5_CMD_OP_DESTROY_SRQ: 363 case MLX5_CMD_OP_DESTROY_XRC_SRQ: 364 case MLX5_CMD_OP_DESTROY_XRQ: 365 case MLX5_CMD_OP_DESTROY_DCT: 366 case MLX5_CMD_OP_DEALLOC_Q_COUNTER: 367 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT: 368 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT: 369 case MLX5_CMD_OP_DEALLOC_PD: 370 case MLX5_CMD_OP_DEALLOC_UAR: 371 case MLX5_CMD_OP_DETACH_FROM_MCG: 372 case MLX5_CMD_OP_DEALLOC_XRCD: 373 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN: 374 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT: 375 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY: 376 case MLX5_CMD_OP_DESTROY_LAG: 377 case MLX5_CMD_OP_DESTROY_VPORT_LAG: 378 case MLX5_CMD_OP_DESTROY_TIR: 379 case MLX5_CMD_OP_DESTROY_SQ: 380 case MLX5_CMD_OP_DESTROY_RQ: 381 case MLX5_CMD_OP_DESTROY_RMP: 382 case MLX5_CMD_OP_DESTROY_TIS: 383 case MLX5_CMD_OP_DESTROY_RQT: 384 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 385 case MLX5_CMD_OP_DESTROY_FLOW_GROUP: 386 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY: 387 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER: 388 case MLX5_CMD_OP_2ERR_QP: 389 case MLX5_CMD_OP_2RST_QP: 390 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT: 391 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 392 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 393 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT: 394 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT: 395 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT: 396 case MLX5_CMD_OP_FPGA_DESTROY_QP: 397 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 398 case MLX5_CMD_OP_DEALLOC_MEMIC: 399 case MLX5_CMD_OP_PAGE_FAULT_RESUME: 400 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: 401 case MLX5_CMD_OP_DEALLOC_SF: 402 case MLX5_CMD_OP_DESTROY_UCTX: 403 case MLX5_CMD_OP_DESTROY_UMEM: 404 case MLX5_CMD_OP_MODIFY_RQT: 405 return MLX5_CMD_STAT_OK; 406 407 case MLX5_CMD_OP_QUERY_HCA_CAP: 408 case MLX5_CMD_OP_QUERY_ADAPTER: 409 case MLX5_CMD_OP_INIT_HCA: 410 case MLX5_CMD_OP_ENABLE_HCA: 411 case MLX5_CMD_OP_QUERY_PAGES: 412 case MLX5_CMD_OP_SET_HCA_CAP: 413 case MLX5_CMD_OP_QUERY_ISSI: 414 case MLX5_CMD_OP_SET_ISSI: 415 case MLX5_CMD_OP_CREATE_MKEY: 416 case MLX5_CMD_OP_QUERY_MKEY: 417 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS: 418 case MLX5_CMD_OP_CREATE_EQ: 419 case MLX5_CMD_OP_QUERY_EQ: 420 case MLX5_CMD_OP_GEN_EQE: 421 case MLX5_CMD_OP_CREATE_CQ: 422 case MLX5_CMD_OP_QUERY_CQ: 423 case MLX5_CMD_OP_MODIFY_CQ: 424 case MLX5_CMD_OP_CREATE_QP: 425 case MLX5_CMD_OP_RST2INIT_QP: 426 case MLX5_CMD_OP_INIT2RTR_QP: 427 case MLX5_CMD_OP_RTR2RTS_QP: 428 case MLX5_CMD_OP_RTS2RTS_QP: 429 case MLX5_CMD_OP_SQERR2RTS_QP: 430 case MLX5_CMD_OP_QUERY_QP: 431 case MLX5_CMD_OP_SQD_RTS_QP: 432 case MLX5_CMD_OP_INIT2INIT_QP: 433 case MLX5_CMD_OP_CREATE_PSV: 434 case MLX5_CMD_OP_CREATE_SRQ: 435 case MLX5_CMD_OP_QUERY_SRQ: 436 case MLX5_CMD_OP_ARM_RQ: 437 case MLX5_CMD_OP_CREATE_XRC_SRQ: 438 case MLX5_CMD_OP_QUERY_XRC_SRQ: 439 case MLX5_CMD_OP_ARM_XRC_SRQ: 440 case MLX5_CMD_OP_CREATE_XRQ: 441 case MLX5_CMD_OP_QUERY_XRQ: 442 case MLX5_CMD_OP_ARM_XRQ: 443 case MLX5_CMD_OP_CREATE_DCT: 444 case MLX5_CMD_OP_DRAIN_DCT: 445 case MLX5_CMD_OP_QUERY_DCT: 446 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 447 case MLX5_CMD_OP_QUERY_VPORT_STATE: 448 case MLX5_CMD_OP_MODIFY_VPORT_STATE: 449 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 450 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT: 451 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 452 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 453 case MLX5_CMD_OP_SET_ROCE_ADDRESS: 454 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 455 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT: 456 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID: 457 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY: 458 case MLX5_CMD_OP_QUERY_VNIC_ENV: 459 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 460 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 461 case MLX5_CMD_OP_QUERY_Q_COUNTER: 462 case MLX5_CMD_OP_SET_MONITOR_COUNTER: 463 case MLX5_CMD_OP_ARM_MONITOR_COUNTER: 464 case MLX5_CMD_OP_SET_PP_RATE_LIMIT: 465 case MLX5_CMD_OP_QUERY_RATE_LIMIT: 466 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 467 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 468 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 469 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT: 470 case MLX5_CMD_OP_ALLOC_PD: 471 case MLX5_CMD_OP_ALLOC_UAR: 472 case MLX5_CMD_OP_CONFIG_INT_MODERATION: 473 case MLX5_CMD_OP_ACCESS_REG: 474 case MLX5_CMD_OP_ATTACH_TO_MCG: 475 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 476 case MLX5_CMD_OP_MAD_IFC: 477 case MLX5_CMD_OP_QUERY_MAD_DEMUX: 478 case MLX5_CMD_OP_SET_MAD_DEMUX: 479 case MLX5_CMD_OP_NOP: 480 case MLX5_CMD_OP_ALLOC_XRCD: 481 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 482 case MLX5_CMD_OP_QUERY_CONG_STATUS: 483 case MLX5_CMD_OP_MODIFY_CONG_STATUS: 484 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 485 case MLX5_CMD_OP_MODIFY_CONG_PARAMS: 486 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 487 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 488 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 489 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 490 case MLX5_CMD_OP_CREATE_LAG: 491 case MLX5_CMD_OP_MODIFY_LAG: 492 case MLX5_CMD_OP_QUERY_LAG: 493 case MLX5_CMD_OP_CREATE_VPORT_LAG: 494 case MLX5_CMD_OP_CREATE_TIR: 495 case MLX5_CMD_OP_MODIFY_TIR: 496 case MLX5_CMD_OP_QUERY_TIR: 497 case MLX5_CMD_OP_CREATE_SQ: 498 case MLX5_CMD_OP_MODIFY_SQ: 499 case MLX5_CMD_OP_QUERY_SQ: 500 case MLX5_CMD_OP_CREATE_RQ: 501 case MLX5_CMD_OP_MODIFY_RQ: 502 case MLX5_CMD_OP_QUERY_RQ: 503 case MLX5_CMD_OP_CREATE_RMP: 504 case MLX5_CMD_OP_MODIFY_RMP: 505 case MLX5_CMD_OP_QUERY_RMP: 506 case MLX5_CMD_OP_CREATE_TIS: 507 case MLX5_CMD_OP_MODIFY_TIS: 508 case MLX5_CMD_OP_QUERY_TIS: 509 case MLX5_CMD_OP_CREATE_RQT: 510 case MLX5_CMD_OP_QUERY_RQT: 511 512 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 513 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 514 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 515 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 516 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 517 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 518 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 519 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 520 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 521 case MLX5_CMD_OP_FPGA_CREATE_QP: 522 case MLX5_CMD_OP_FPGA_MODIFY_QP: 523 case MLX5_CMD_OP_FPGA_QUERY_QP: 524 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS: 525 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 526 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 527 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 528 case MLX5_CMD_OP_CREATE_UCTX: 529 case MLX5_CMD_OP_CREATE_UMEM: 530 case MLX5_CMD_OP_ALLOC_MEMIC: 531 case MLX5_CMD_OP_MODIFY_XRQ: 532 case MLX5_CMD_OP_RELEASE_XRQ_ERROR: 533 case MLX5_CMD_OP_QUERY_VHCA_STATE: 534 case MLX5_CMD_OP_MODIFY_VHCA_STATE: 535 case MLX5_CMD_OP_ALLOC_SF: 536 case MLX5_CMD_OP_SUSPEND_VHCA: 537 case MLX5_CMD_OP_RESUME_VHCA: 538 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE: 539 case MLX5_CMD_OP_SAVE_VHCA_STATE: 540 case MLX5_CMD_OP_LOAD_VHCA_STATE: 541 case MLX5_CMD_OP_SYNC_CRYPTO: 542 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS: 543 *status = MLX5_DRIVER_STATUS_ABORTED; 544 *synd = MLX5_DRIVER_SYND; 545 return -ENOLINK; 546 default: 547 mlx5_core_err(dev, "Unknown FW command (%d)\n", op); 548 return -EINVAL; 549 } 550 } 551 552 const char *mlx5_command_str(int command) 553 { 554 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd 555 556 switch (command) { 557 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP); 558 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER); 559 MLX5_COMMAND_STR_CASE(INIT_HCA); 560 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA); 561 MLX5_COMMAND_STR_CASE(ENABLE_HCA); 562 MLX5_COMMAND_STR_CASE(DISABLE_HCA); 563 MLX5_COMMAND_STR_CASE(QUERY_PAGES); 564 MLX5_COMMAND_STR_CASE(MANAGE_PAGES); 565 MLX5_COMMAND_STR_CASE(SET_HCA_CAP); 566 MLX5_COMMAND_STR_CASE(QUERY_ISSI); 567 MLX5_COMMAND_STR_CASE(SET_ISSI); 568 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION); 569 MLX5_COMMAND_STR_CASE(CREATE_MKEY); 570 MLX5_COMMAND_STR_CASE(QUERY_MKEY); 571 MLX5_COMMAND_STR_CASE(DESTROY_MKEY); 572 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS); 573 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME); 574 MLX5_COMMAND_STR_CASE(CREATE_EQ); 575 MLX5_COMMAND_STR_CASE(DESTROY_EQ); 576 MLX5_COMMAND_STR_CASE(QUERY_EQ); 577 MLX5_COMMAND_STR_CASE(GEN_EQE); 578 MLX5_COMMAND_STR_CASE(CREATE_CQ); 579 MLX5_COMMAND_STR_CASE(DESTROY_CQ); 580 MLX5_COMMAND_STR_CASE(QUERY_CQ); 581 MLX5_COMMAND_STR_CASE(MODIFY_CQ); 582 MLX5_COMMAND_STR_CASE(CREATE_QP); 583 MLX5_COMMAND_STR_CASE(DESTROY_QP); 584 MLX5_COMMAND_STR_CASE(RST2INIT_QP); 585 MLX5_COMMAND_STR_CASE(INIT2RTR_QP); 586 MLX5_COMMAND_STR_CASE(RTR2RTS_QP); 587 MLX5_COMMAND_STR_CASE(RTS2RTS_QP); 588 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP); 589 MLX5_COMMAND_STR_CASE(2ERR_QP); 590 MLX5_COMMAND_STR_CASE(2RST_QP); 591 MLX5_COMMAND_STR_CASE(QUERY_QP); 592 MLX5_COMMAND_STR_CASE(SQD_RTS_QP); 593 MLX5_COMMAND_STR_CASE(INIT2INIT_QP); 594 MLX5_COMMAND_STR_CASE(CREATE_PSV); 595 MLX5_COMMAND_STR_CASE(DESTROY_PSV); 596 MLX5_COMMAND_STR_CASE(CREATE_SRQ); 597 MLX5_COMMAND_STR_CASE(DESTROY_SRQ); 598 MLX5_COMMAND_STR_CASE(QUERY_SRQ); 599 MLX5_COMMAND_STR_CASE(ARM_RQ); 600 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ); 601 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ); 602 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ); 603 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ); 604 MLX5_COMMAND_STR_CASE(CREATE_DCT); 605 MLX5_COMMAND_STR_CASE(DESTROY_DCT); 606 MLX5_COMMAND_STR_CASE(DRAIN_DCT); 607 MLX5_COMMAND_STR_CASE(QUERY_DCT); 608 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION); 609 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE); 610 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE); 611 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT); 612 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT); 613 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT); 614 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT); 615 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS); 616 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS); 617 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT); 618 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT); 619 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID); 620 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY); 621 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV); 622 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER); 623 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER); 624 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER); 625 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER); 626 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER); 627 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER); 628 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT); 629 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT); 630 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT); 631 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT); 632 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT); 633 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT); 634 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT); 635 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT); 636 MLX5_COMMAND_STR_CASE(ALLOC_PD); 637 MLX5_COMMAND_STR_CASE(DEALLOC_PD); 638 MLX5_COMMAND_STR_CASE(ALLOC_UAR); 639 MLX5_COMMAND_STR_CASE(DEALLOC_UAR); 640 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION); 641 MLX5_COMMAND_STR_CASE(ACCESS_REG); 642 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG); 643 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG); 644 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG); 645 MLX5_COMMAND_STR_CASE(MAD_IFC); 646 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX); 647 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX); 648 MLX5_COMMAND_STR_CASE(NOP); 649 MLX5_COMMAND_STR_CASE(ALLOC_XRCD); 650 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD); 651 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN); 652 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN); 653 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS); 654 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS); 655 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS); 656 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS); 657 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS); 658 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT); 659 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT); 660 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY); 661 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY); 662 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY); 663 MLX5_COMMAND_STR_CASE(SET_WOL_ROL); 664 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL); 665 MLX5_COMMAND_STR_CASE(CREATE_LAG); 666 MLX5_COMMAND_STR_CASE(MODIFY_LAG); 667 MLX5_COMMAND_STR_CASE(QUERY_LAG); 668 MLX5_COMMAND_STR_CASE(DESTROY_LAG); 669 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG); 670 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG); 671 MLX5_COMMAND_STR_CASE(CREATE_TIR); 672 MLX5_COMMAND_STR_CASE(MODIFY_TIR); 673 MLX5_COMMAND_STR_CASE(DESTROY_TIR); 674 MLX5_COMMAND_STR_CASE(QUERY_TIR); 675 MLX5_COMMAND_STR_CASE(CREATE_SQ); 676 MLX5_COMMAND_STR_CASE(MODIFY_SQ); 677 MLX5_COMMAND_STR_CASE(DESTROY_SQ); 678 MLX5_COMMAND_STR_CASE(QUERY_SQ); 679 MLX5_COMMAND_STR_CASE(CREATE_RQ); 680 MLX5_COMMAND_STR_CASE(MODIFY_RQ); 681 MLX5_COMMAND_STR_CASE(DESTROY_RQ); 682 MLX5_COMMAND_STR_CASE(QUERY_RQ); 683 MLX5_COMMAND_STR_CASE(CREATE_RMP); 684 MLX5_COMMAND_STR_CASE(MODIFY_RMP); 685 MLX5_COMMAND_STR_CASE(DESTROY_RMP); 686 MLX5_COMMAND_STR_CASE(QUERY_RMP); 687 MLX5_COMMAND_STR_CASE(CREATE_TIS); 688 MLX5_COMMAND_STR_CASE(MODIFY_TIS); 689 MLX5_COMMAND_STR_CASE(DESTROY_TIS); 690 MLX5_COMMAND_STR_CASE(QUERY_TIS); 691 MLX5_COMMAND_STR_CASE(CREATE_RQT); 692 MLX5_COMMAND_STR_CASE(MODIFY_RQT); 693 MLX5_COMMAND_STR_CASE(DESTROY_RQT); 694 MLX5_COMMAND_STR_CASE(QUERY_RQT); 695 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT); 696 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE); 697 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE); 698 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE); 699 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP); 700 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP); 701 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP); 702 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY); 703 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY); 704 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY); 705 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER); 706 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER); 707 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER); 708 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE); 709 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT); 710 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT); 711 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT); 712 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT); 713 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP); 714 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP); 715 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP); 716 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS); 717 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP); 718 MLX5_COMMAND_STR_CASE(CREATE_XRQ); 719 MLX5_COMMAND_STR_CASE(DESTROY_XRQ); 720 MLX5_COMMAND_STR_CASE(QUERY_XRQ); 721 MLX5_COMMAND_STR_CASE(ARM_XRQ); 722 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT); 723 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT); 724 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT); 725 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT); 726 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT); 727 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC); 728 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC); 729 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS); 730 MLX5_COMMAND_STR_CASE(CREATE_UCTX); 731 MLX5_COMMAND_STR_CASE(DESTROY_UCTX); 732 MLX5_COMMAND_STR_CASE(CREATE_UMEM); 733 MLX5_COMMAND_STR_CASE(DESTROY_UMEM); 734 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR); 735 MLX5_COMMAND_STR_CASE(MODIFY_XRQ); 736 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); 737 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); 738 MLX5_COMMAND_STR_CASE(ALLOC_SF); 739 MLX5_COMMAND_STR_CASE(DEALLOC_SF); 740 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA); 741 MLX5_COMMAND_STR_CASE(RESUME_VHCA); 742 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE); 743 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE); 744 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE); 745 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO); 746 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS); 747 default: return "unknown command opcode"; 748 } 749 } 750 751 static const char *cmd_status_str(u8 status) 752 { 753 switch (status) { 754 case MLX5_CMD_STAT_OK: 755 return "OK"; 756 case MLX5_CMD_STAT_INT_ERR: 757 return "internal error"; 758 case MLX5_CMD_STAT_BAD_OP_ERR: 759 return "bad operation"; 760 case MLX5_CMD_STAT_BAD_PARAM_ERR: 761 return "bad parameter"; 762 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: 763 return "bad system state"; 764 case MLX5_CMD_STAT_BAD_RES_ERR: 765 return "bad resource"; 766 case MLX5_CMD_STAT_RES_BUSY: 767 return "resource busy"; 768 case MLX5_CMD_STAT_NOT_READY: 769 return "FW not ready"; 770 case MLX5_CMD_STAT_LIM_ERR: 771 return "limits exceeded"; 772 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: 773 return "bad resource state"; 774 case MLX5_CMD_STAT_IX_ERR: 775 return "bad index"; 776 case MLX5_CMD_STAT_NO_RES_ERR: 777 return "no resources"; 778 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: 779 return "bad input length"; 780 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: 781 return "bad output length"; 782 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: 783 return "bad QP state"; 784 case MLX5_CMD_STAT_BAD_PKT_ERR: 785 return "bad packet (discarded)"; 786 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: 787 return "bad size too many outstanding CQEs"; 788 default: 789 return "unknown status"; 790 } 791 } 792 793 static int cmd_status_to_err(u8 status) 794 { 795 switch (status) { 796 case MLX5_CMD_STAT_OK: return 0; 797 case MLX5_CMD_STAT_INT_ERR: return -EIO; 798 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL; 799 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; 800 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; 801 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL; 802 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY; 803 case MLX5_CMD_STAT_NOT_READY: return -EAGAIN; 804 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM; 805 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; 806 case MLX5_CMD_STAT_IX_ERR: return -EINVAL; 807 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN; 808 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; 809 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; 810 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; 811 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL; 812 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; 813 default: return -EIO; 814 } 815 } 816 817 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 818 { 819 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 820 u8 status = MLX5_GET(mbox_out, out, status); 821 822 mlx5_core_err_rl(dev, 823 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n", 824 mlx5_command_str(opcode), opcode, op_mod, 825 cmd_status_str(status), status, syndrome, cmd_status_to_err(status)); 826 } 827 EXPORT_SYMBOL(mlx5_cmd_out_err); 828 829 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out) 830 { 831 u16 opcode, op_mod; 832 u8 status; 833 u16 uid; 834 835 opcode = in_to_opcode(in); 836 op_mod = MLX5_GET(mbox_in, in, op_mod); 837 uid = in_to_uid(in); 838 status = MLX5_GET(mbox_out, out, status); 839 840 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY && 841 opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY) 842 mlx5_cmd_out_err(dev, opcode, op_mod, out); 843 } 844 845 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out) 846 { 847 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */ 848 if (err == -ENXIO) { 849 u16 opcode = in_to_opcode(in); 850 u32 syndrome; 851 u8 status; 852 853 /* PCI Error, emulate command return status, for smooth reset */ 854 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status); 855 MLX5_SET(mbox_out, out, status, status); 856 MLX5_SET(mbox_out, out, syndrome, syndrome); 857 if (!err) 858 return 0; 859 } 860 861 /* driver or FW delivery error */ 862 if (err != -EREMOTEIO && err) 863 return err; 864 865 /* check outbox status */ 866 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status)); 867 if (err) 868 cmd_status_print(dev, in, out); 869 870 return err; 871 } 872 EXPORT_SYMBOL(mlx5_cmd_check); 873 874 static void dump_command(struct mlx5_core_dev *dev, 875 struct mlx5_cmd_work_ent *ent, int input) 876 { 877 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out; 878 struct mlx5_cmd_mailbox *next = msg->next; 879 int n = mlx5_calc_cmd_blocks(msg); 880 u16 op = ent->op; 881 int data_only; 882 u32 offset = 0; 883 int dump_len; 884 int i; 885 886 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx); 887 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA)); 888 889 if (data_only) 890 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA, 891 "cmd[%d]: dump command data %s(0x%x) %s\n", 892 ent->idx, mlx5_command_str(op), op, 893 input ? "INPUT" : "OUTPUT"); 894 else 895 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n", 896 ent->idx, mlx5_command_str(op), op, 897 input ? "INPUT" : "OUTPUT"); 898 899 if (data_only) { 900 if (input) { 901 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx); 902 offset += sizeof(ent->lay->in); 903 } else { 904 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx); 905 offset += sizeof(ent->lay->out); 906 } 907 } else { 908 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx); 909 offset += sizeof(*ent->lay); 910 } 911 912 for (i = 0; i < n && next; i++) { 913 if (data_only) { 914 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset); 915 dump_buf(next->buf, dump_len, 1, offset, ent->idx); 916 offset += MLX5_CMD_DATA_BLOCK_SIZE; 917 } else { 918 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx); 919 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset, 920 ent->idx); 921 offset += sizeof(struct mlx5_cmd_prot_block); 922 } 923 next = next->next; 924 } 925 926 if (data_only) 927 pr_debug("\n"); 928 929 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx); 930 } 931 932 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced); 933 934 static void cb_timeout_handler(struct work_struct *work) 935 { 936 struct delayed_work *dwork = to_delayed_work(work); 937 struct mlx5_cmd_work_ent *ent = container_of(dwork, 938 struct mlx5_cmd_work_ent, 939 cb_timeout_work); 940 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, 941 cmd); 942 943 mlx5_cmd_eq_recover(dev); 944 945 /* Maybe got handled by eq recover ? */ 946 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { 947 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, 948 mlx5_command_str(ent->op), ent->op); 949 goto out; /* phew, already handled */ 950 } 951 952 ent->ret = -ETIMEDOUT; 953 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", 954 ent->idx, mlx5_command_str(ent->op), ent->op); 955 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 956 957 out: 958 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ 959 } 960 961 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); 962 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 963 struct mlx5_cmd_msg *msg); 964 965 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) 966 { 967 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) 968 return true; 969 970 return cmd->allowed_opcode == opcode; 971 } 972 973 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev) 974 { 975 return pci_channel_offline(dev->pdev) || 976 dev->cmd.state != MLX5_CMDIF_STATE_UP || 977 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR; 978 } 979 980 static void cmd_work_handler(struct work_struct *work) 981 { 982 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); 983 struct mlx5_cmd *cmd = ent->cmd; 984 bool poll_cmd = ent->polling; 985 struct mlx5_cmd_layout *lay; 986 struct mlx5_core_dev *dev; 987 unsigned long timeout; 988 unsigned long flags; 989 int alloc_ret; 990 int cmd_mode; 991 992 complete(&ent->handling); 993 994 dev = container_of(cmd, struct mlx5_core_dev, cmd); 995 timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 996 997 if (!ent->page_queue) { 998 if (down_timeout(&cmd->vars.sem, timeout)) { 999 mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n", 1000 mlx5_command_str(ent->op), ent->op); 1001 if (ent->callback) { 1002 ent->callback(-EBUSY, ent->context); 1003 mlx5_free_cmd_msg(dev, ent->out); 1004 free_msg(dev, ent->in); 1005 complete(&ent->slotted); 1006 cmd_ent_put(ent); 1007 } else { 1008 ent->ret = -EBUSY; 1009 complete(&ent->done); 1010 complete(&ent->slotted); 1011 } 1012 return; 1013 } 1014 alloc_ret = cmd_alloc_index(cmd, ent); 1015 if (alloc_ret < 0) { 1016 mlx5_core_err_rl(dev, "failed to allocate command entry\n"); 1017 if (ent->callback) { 1018 ent->callback(-EAGAIN, ent->context); 1019 mlx5_free_cmd_msg(dev, ent->out); 1020 free_msg(dev, ent->in); 1021 complete(&ent->slotted); 1022 cmd_ent_put(ent); 1023 } else { 1024 ent->ret = -EAGAIN; 1025 complete(&ent->done); 1026 complete(&ent->slotted); 1027 } 1028 up(&cmd->vars.sem); 1029 return; 1030 } 1031 } else { 1032 down(&cmd->vars.pages_sem); 1033 ent->idx = cmd->vars.max_reg_cmds; 1034 spin_lock_irqsave(&cmd->alloc_lock, flags); 1035 clear_bit(ent->idx, &cmd->vars.bitmask); 1036 cmd->ent_arr[ent->idx] = ent; 1037 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 1038 } 1039 1040 complete(&ent->slotted); 1041 1042 lay = get_inst(cmd, ent->idx); 1043 ent->lay = lay; 1044 memset(lay, 0, sizeof(*lay)); 1045 memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); 1046 if (ent->in->next) 1047 lay->in_ptr = cpu_to_be64(ent->in->next->dma); 1048 lay->inlen = cpu_to_be32(ent->in->len); 1049 if (ent->out->next) 1050 lay->out_ptr = cpu_to_be64(ent->out->next->dma); 1051 lay->outlen = cpu_to_be32(ent->out->len); 1052 lay->type = MLX5_PCI_CMD_XPORT; 1053 lay->token = ent->token; 1054 lay->status_own = CMD_OWNER_HW; 1055 set_signature(ent, !cmd->checksum_disabled); 1056 dump_command(dev, ent, 1); 1057 ent->ts1 = ktime_get_ns(); 1058 cmd_mode = cmd->mode; 1059 1060 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout)) 1061 cmd_ent_get(ent); 1062 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state); 1063 1064 cmd_ent_get(ent); /* for the _real_ FW event on completion */ 1065 /* Skip sending command to fw if internal error */ 1066 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) { 1067 ent->ret = -ENXIO; 1068 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1069 return; 1070 } 1071 1072 /* ring doorbell after the descriptor is valid */ 1073 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); 1074 wmb(); 1075 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); 1076 /* if not in polling don't use ent after this point */ 1077 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { 1078 poll_timeout(ent); 1079 /* make sure we read the descriptor after ownership is SW */ 1080 rmb(); 1081 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret); 1082 } 1083 } 1084 1085 static int deliv_status_to_err(u8 status) 1086 { 1087 switch (status) { 1088 case MLX5_CMD_DELIVERY_STAT_OK: 1089 case MLX5_DRIVER_STATUS_ABORTED: 1090 return 0; 1091 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1092 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1093 return -EBADR; 1094 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1095 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1096 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1097 return -EFAULT; /* Bad address */ 1098 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1099 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1100 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1101 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1102 return -ENOMSG; 1103 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1104 return -EIO; 1105 default: 1106 return -EINVAL; 1107 } 1108 } 1109 1110 static const char *deliv_status_to_str(u8 status) 1111 { 1112 switch (status) { 1113 case MLX5_CMD_DELIVERY_STAT_OK: 1114 return "no errors"; 1115 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1116 return "signature error"; 1117 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1118 return "token error"; 1119 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1120 return "bad block number"; 1121 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1122 return "output pointer not aligned to block size"; 1123 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1124 return "input pointer not aligned to block size"; 1125 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1126 return "firmware internal error"; 1127 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1128 return "command input length error"; 1129 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1130 return "command output length error"; 1131 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1132 return "reserved fields not cleared"; 1133 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1134 return "bad command descriptor type"; 1135 default: 1136 return "unknown status code"; 1137 } 1138 } 1139 1140 enum { 1141 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, 1142 }; 1143 1144 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, 1145 struct mlx5_cmd_work_ent *ent) 1146 { 1147 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); 1148 1149 mlx5_cmd_eq_recover(dev); 1150 1151 /* Re-wait on the ent->done after executing the recovery flow. If the 1152 * recovery flow (or any other recovery flow running simultaneously) 1153 * has recovered an EQE, it should cause the entry to be completed by 1154 * the command interface. 1155 */ 1156 if (wait_for_completion_timeout(&ent->done, timeout)) { 1157 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, 1158 mlx5_command_str(ent->op), ent->op); 1159 return; 1160 } 1161 1162 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, 1163 mlx5_command_str(ent->op), ent->op); 1164 1165 ent->ret = -ETIMEDOUT; 1166 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1167 } 1168 1169 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) 1170 { 1171 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 1172 struct mlx5_cmd *cmd = &dev->cmd; 1173 int err; 1174 1175 if (!wait_for_completion_timeout(&ent->handling, timeout) && 1176 cancel_work_sync(&ent->work)) { 1177 ent->ret = -ECANCELED; 1178 goto out_err; 1179 } 1180 1181 wait_for_completion(&ent->slotted); 1182 1183 if (cmd->mode == CMD_MODE_POLLING || ent->polling) 1184 wait_for_completion(&ent->done); 1185 else if (!wait_for_completion_timeout(&ent->done, timeout)) 1186 wait_func_handle_exec_timeout(dev, ent); 1187 1188 out_err: 1189 err = ent->ret; 1190 1191 if (err == -ETIMEDOUT) { 1192 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", 1193 mlx5_command_str(ent->op), ent->op); 1194 } else if (err == -ECANCELED) { 1195 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n", 1196 mlx5_command_str(ent->op), ent->op); 1197 } else if (err == -EBUSY) { 1198 mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n", 1199 mlx5_command_str(ent->op), ent->op); 1200 } 1201 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", 1202 err, deliv_status_to_str(ent->status), ent->status); 1203 1204 return err; 1205 } 1206 1207 /* Check if all command slots are stalled (timed out and not recovered). 1208 * returns true if all slots timed out on a recent command and have not been 1209 * completed by FW yet. (stalled state) 1210 * false otherwise (at least one slot is not stalled). 1211 * 1212 * In such odd situation "all_stalled", this serves as a protection mechanism 1213 * to avoid blocking the kernel for long periods of time in case FW is not 1214 * responding to commands. 1215 */ 1216 static bool mlx5_cmd_all_stalled(struct mlx5_core_dev *dev) 1217 { 1218 struct mlx5_cmd *cmd = &dev->cmd; 1219 bool all_stalled = true; 1220 unsigned long flags; 1221 int i; 1222 1223 spin_lock_irqsave(&cmd->alloc_lock, flags); 1224 1225 /* at least one command slot is free */ 1226 if (bitmap_weight(&cmd->vars.bitmask, cmd->vars.max_reg_cmds) > 0) { 1227 all_stalled = false; 1228 goto out; 1229 } 1230 1231 for_each_clear_bit(i, &cmd->vars.bitmask, cmd->vars.max_reg_cmds) { 1232 struct mlx5_cmd_work_ent *ent = dev->cmd.ent_arr[i]; 1233 1234 if (!test_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, &ent->state)) { 1235 all_stalled = false; 1236 break; 1237 } 1238 } 1239 out: 1240 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 1241 1242 return all_stalled; 1243 } 1244 1245 /* Notes: 1246 * 1. Callback functions may not sleep 1247 * 2. page queue commands do not support asynchrous completion 1248 * 1249 * return value in case (!callback): 1250 * ret < 0 : Command execution couldn't be submitted by driver 1251 * ret > 0 : Command execution couldn't be performed by firmware 1252 * ret == 0: Command was executed by FW, Caller must check FW outbox status. 1253 * 1254 * return value in case (callback): 1255 * ret < 0 : Command execution couldn't be submitted by driver 1256 * ret == 0: Command will be submitted to FW for execution 1257 * and the callback will be called for further status updates 1258 */ 1259 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 1260 struct mlx5_cmd_msg *out, void *uout, int uout_size, 1261 mlx5_cmd_cbk_t callback, 1262 void *context, int page_queue, 1263 u8 token, bool force_polling) 1264 { 1265 struct mlx5_cmd *cmd = &dev->cmd; 1266 struct mlx5_cmd_work_ent *ent; 1267 struct mlx5_cmd_stats *stats; 1268 u8 status = 0; 1269 int err = 0; 1270 s64 ds; 1271 1272 if (callback && page_queue) 1273 return -EINVAL; 1274 1275 if (!page_queue && mlx5_cmd_all_stalled(dev)) { 1276 mlx5_core_err_rl(dev, 1277 "All CMD slots are stalled, aborting command\n"); 1278 /* there's no reason to wait and block the whole kernel if FW 1279 * isn't currently responding to all slots, fail immediately 1280 */ 1281 return -EAGAIN; 1282 } 1283 1284 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size, 1285 callback, context, page_queue); 1286 if (IS_ERR(ent)) 1287 return PTR_ERR(ent); 1288 1289 /* put for this ent is when consumed, depending on the use case 1290 * 1) (!callback) blocking flow: by caller after wait_func completes 1291 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled 1292 */ 1293 1294 ent->token = token; 1295 ent->polling = force_polling; 1296 1297 init_completion(&ent->handling); 1298 init_completion(&ent->slotted); 1299 if (!callback) 1300 init_completion(&ent->done); 1301 1302 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler); 1303 INIT_WORK(&ent->work, cmd_work_handler); 1304 if (page_queue) { 1305 cmd_work_handler(&ent->work); 1306 } else if (!queue_work(cmd->wq, &ent->work)) { 1307 mlx5_core_warn(dev, "failed to queue work\n"); 1308 err = -EALREADY; 1309 goto out_free; 1310 } 1311 1312 if (callback) 1313 return 0; /* mlx5_cmd_comp_handler() will put(ent) */ 1314 1315 err = wait_func(dev, ent); 1316 if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY) 1317 goto out_free; 1318 1319 ds = ent->ts2 - ent->ts1; 1320 stats = xa_load(&cmd->stats, ent->op); 1321 if (stats) { 1322 spin_lock_irq(&stats->lock); 1323 stats->sum += ds; 1324 ++stats->n; 1325 spin_unlock_irq(&stats->lock); 1326 } 1327 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME, 1328 "fw exec time for %s is %lld nsec\n", 1329 mlx5_command_str(ent->op), ds); 1330 1331 out_free: 1332 status = ent->status; 1333 cmd_ent_put(ent); 1334 return err ? : status; 1335 } 1336 1337 static ssize_t dbg_write(struct file *filp, const char __user *buf, 1338 size_t count, loff_t *pos) 1339 { 1340 struct mlx5_core_dev *dev = filp->private_data; 1341 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1342 char lbuf[3]; 1343 int err; 1344 1345 if (!dbg->in_msg || !dbg->out_msg) 1346 return -ENOMEM; 1347 1348 if (count < sizeof(lbuf) - 1) 1349 return -EINVAL; 1350 1351 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1)) 1352 return -EFAULT; 1353 1354 lbuf[sizeof(lbuf) - 1] = 0; 1355 1356 if (strcmp(lbuf, "go")) 1357 return -EINVAL; 1358 1359 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen); 1360 1361 return err ? err : count; 1362 } 1363 1364 static const struct file_operations fops = { 1365 .owner = THIS_MODULE, 1366 .open = simple_open, 1367 .write = dbg_write, 1368 }; 1369 1370 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size, 1371 u8 token) 1372 { 1373 struct mlx5_cmd_prot_block *block; 1374 struct mlx5_cmd_mailbox *next; 1375 int copy; 1376 1377 if (!to || !from) 1378 return -ENOMEM; 1379 1380 copy = min_t(int, size, sizeof(to->first.data)); 1381 memcpy(to->first.data, from, copy); 1382 size -= copy; 1383 from += copy; 1384 1385 next = to->next; 1386 while (size) { 1387 if (!next) { 1388 /* this is a BUG */ 1389 return -ENOMEM; 1390 } 1391 1392 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1393 block = next->buf; 1394 memcpy(block->data, from, copy); 1395 from += copy; 1396 size -= copy; 1397 block->token = token; 1398 next = next->next; 1399 } 1400 1401 return 0; 1402 } 1403 1404 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size) 1405 { 1406 struct mlx5_cmd_prot_block *block; 1407 struct mlx5_cmd_mailbox *next; 1408 int copy; 1409 1410 if (!to || !from) 1411 return -ENOMEM; 1412 1413 copy = min_t(int, size, sizeof(from->first.data)); 1414 memcpy(to, from->first.data, copy); 1415 size -= copy; 1416 to += copy; 1417 1418 next = from->next; 1419 while (size) { 1420 if (!next) { 1421 /* this is a BUG */ 1422 return -ENOMEM; 1423 } 1424 1425 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1426 block = next->buf; 1427 1428 memcpy(to, block->data, copy); 1429 to += copy; 1430 size -= copy; 1431 next = next->next; 1432 } 1433 1434 return 0; 1435 } 1436 1437 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev, 1438 gfp_t flags) 1439 { 1440 struct mlx5_cmd_mailbox *mailbox; 1441 1442 mailbox = kmalloc_obj(*mailbox, flags); 1443 if (!mailbox) 1444 return ERR_PTR(-ENOMEM); 1445 1446 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags, 1447 &mailbox->dma); 1448 if (!mailbox->buf) { 1449 mlx5_core_dbg(dev, "failed allocation\n"); 1450 kfree(mailbox); 1451 return ERR_PTR(-ENOMEM); 1452 } 1453 mailbox->next = NULL; 1454 1455 return mailbox; 1456 } 1457 1458 static void free_cmd_box(struct mlx5_core_dev *dev, 1459 struct mlx5_cmd_mailbox *mailbox) 1460 { 1461 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); 1462 kfree(mailbox); 1463 } 1464 1465 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, 1466 gfp_t flags, int size, 1467 u8 token) 1468 { 1469 struct mlx5_cmd_mailbox *tmp, *head = NULL; 1470 struct mlx5_cmd_prot_block *block; 1471 struct mlx5_cmd_msg *msg; 1472 int err; 1473 int n; 1474 int i; 1475 1476 msg = kzalloc_obj(*msg, flags); 1477 if (!msg) 1478 return ERR_PTR(-ENOMEM); 1479 1480 msg->len = size; 1481 n = mlx5_calc_cmd_blocks(msg); 1482 1483 for (i = 0; i < n; i++) { 1484 tmp = alloc_cmd_box(dev, flags); 1485 if (IS_ERR(tmp)) { 1486 mlx5_core_warn(dev, "failed allocating block\n"); 1487 err = PTR_ERR(tmp); 1488 goto err_alloc; 1489 } 1490 1491 block = tmp->buf; 1492 tmp->next = head; 1493 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); 1494 block->block_num = cpu_to_be32(n - i - 1); 1495 block->token = token; 1496 head = tmp; 1497 } 1498 msg->next = head; 1499 return msg; 1500 1501 err_alloc: 1502 while (head) { 1503 tmp = head->next; 1504 free_cmd_box(dev, head); 1505 head = tmp; 1506 } 1507 kfree(msg); 1508 1509 return ERR_PTR(err); 1510 } 1511 1512 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 1513 struct mlx5_cmd_msg *msg) 1514 { 1515 struct mlx5_cmd_mailbox *head = msg->next; 1516 struct mlx5_cmd_mailbox *next; 1517 1518 while (head) { 1519 next = head->next; 1520 free_cmd_box(dev, head); 1521 head = next; 1522 } 1523 kfree(msg); 1524 } 1525 1526 static ssize_t data_write(struct file *filp, const char __user *buf, 1527 size_t count, loff_t *pos) 1528 { 1529 struct mlx5_core_dev *dev = filp->private_data; 1530 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1531 void *ptr; 1532 1533 if (*pos != 0) 1534 return -EINVAL; 1535 1536 kfree(dbg->in_msg); 1537 dbg->in_msg = NULL; 1538 dbg->inlen = 0; 1539 ptr = memdup_user(buf, count); 1540 if (IS_ERR(ptr)) 1541 return PTR_ERR(ptr); 1542 dbg->in_msg = ptr; 1543 dbg->inlen = count; 1544 1545 *pos = count; 1546 1547 return count; 1548 } 1549 1550 static ssize_t data_read(struct file *filp, char __user *buf, size_t count, 1551 loff_t *pos) 1552 { 1553 struct mlx5_core_dev *dev = filp->private_data; 1554 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1555 1556 if (!dbg->out_msg) 1557 return -ENOMEM; 1558 1559 return simple_read_from_buffer(buf, count, pos, dbg->out_msg, 1560 dbg->outlen); 1561 } 1562 1563 static const struct file_operations dfops = { 1564 .owner = THIS_MODULE, 1565 .open = simple_open, 1566 .write = data_write, 1567 .read = data_read, 1568 }; 1569 1570 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count, 1571 loff_t *pos) 1572 { 1573 struct mlx5_core_dev *dev = filp->private_data; 1574 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1575 char outlen[8]; 1576 int err; 1577 1578 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen); 1579 if (err < 0) 1580 return err; 1581 1582 return simple_read_from_buffer(buf, count, pos, outlen, err); 1583 } 1584 1585 static ssize_t outlen_write(struct file *filp, const char __user *buf, 1586 size_t count, loff_t *pos) 1587 { 1588 struct mlx5_core_dev *dev = filp->private_data; 1589 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1590 char outlen_str[8] = {0}; 1591 int outlen; 1592 void *ptr; 1593 int err; 1594 1595 if (*pos != 0 || count > 6) 1596 return -EINVAL; 1597 1598 kfree(dbg->out_msg); 1599 dbg->out_msg = NULL; 1600 dbg->outlen = 0; 1601 1602 if (copy_from_user(outlen_str, buf, count)) 1603 return -EFAULT; 1604 1605 err = sscanf(outlen_str, "%d", &outlen); 1606 if (err != 1) 1607 return -EINVAL; 1608 1609 ptr = kzalloc(outlen, GFP_KERNEL); 1610 if (!ptr) 1611 return -ENOMEM; 1612 1613 dbg->out_msg = ptr; 1614 dbg->outlen = outlen; 1615 1616 *pos = count; 1617 1618 return count; 1619 } 1620 1621 static const struct file_operations olfops = { 1622 .owner = THIS_MODULE, 1623 .open = simple_open, 1624 .write = outlen_write, 1625 .read = outlen_read, 1626 }; 1627 1628 static void set_wqname(struct mlx5_core_dev *dev) 1629 { 1630 struct mlx5_cmd *cmd = &dev->cmd; 1631 1632 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s", 1633 dev_name(dev->device)); 1634 } 1635 1636 static void clean_debug_files(struct mlx5_core_dev *dev) 1637 { 1638 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1639 1640 if (!mlx5_debugfs_root) 1641 return; 1642 1643 debugfs_remove_recursive(dbg->dbg_root); 1644 } 1645 1646 static void create_debugfs_files(struct mlx5_core_dev *dev) 1647 { 1648 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1649 1650 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev)); 1651 1652 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops); 1653 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops); 1654 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops); 1655 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); 1656 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops); 1657 } 1658 1659 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) 1660 { 1661 struct mlx5_cmd *cmd = &dev->cmd; 1662 int i; 1663 1664 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1665 down(&cmd->vars.sem); 1666 down(&cmd->vars.pages_sem); 1667 1668 cmd->allowed_opcode = opcode; 1669 1670 up(&cmd->vars.pages_sem); 1671 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1672 up(&cmd->vars.sem); 1673 } 1674 1675 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) 1676 { 1677 struct mlx5_cmd *cmd = &dev->cmd; 1678 int i; 1679 1680 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1681 down(&cmd->vars.sem); 1682 down(&cmd->vars.pages_sem); 1683 1684 cmd->mode = mode; 1685 1686 up(&cmd->vars.pages_sem); 1687 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1688 up(&cmd->vars.sem); 1689 } 1690 1691 static int cmd_comp_notifier(struct notifier_block *nb, 1692 unsigned long type, void *data) 1693 { 1694 struct mlx5_core_dev *dev; 1695 struct mlx5_cmd *cmd; 1696 struct mlx5_eqe *eqe; 1697 1698 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb); 1699 dev = container_of(cmd, struct mlx5_core_dev, cmd); 1700 eqe = data; 1701 1702 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1703 return NOTIFY_DONE; 1704 1705 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false); 1706 1707 return NOTIFY_OK; 1708 } 1709 void mlx5_cmd_use_events(struct mlx5_core_dev *dev) 1710 { 1711 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD); 1712 mlx5_eq_notifier_register(dev, &dev->cmd.nb); 1713 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS); 1714 } 1715 1716 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev) 1717 { 1718 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING); 1719 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb); 1720 } 1721 1722 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) 1723 { 1724 unsigned long flags; 1725 1726 if (msg->parent) { 1727 spin_lock_irqsave(&msg->parent->lock, flags); 1728 list_add_tail(&msg->list, &msg->parent->head); 1729 spin_unlock_irqrestore(&msg->parent->lock, flags); 1730 } else { 1731 mlx5_free_cmd_msg(dev, msg); 1732 } 1733 } 1734 1735 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced) 1736 { 1737 struct mlx5_cmd *cmd = &dev->cmd; 1738 struct mlx5_cmd_work_ent *ent; 1739 mlx5_cmd_cbk_t callback; 1740 void *context; 1741 int err; 1742 int i; 1743 s64 ds; 1744 struct mlx5_cmd_stats *stats; 1745 unsigned long flags; 1746 unsigned long vector; 1747 1748 /* there can be at most 32 command queues */ 1749 vector = vec & 0xffffffff; 1750 for (i = 0; i < (1 << cmd->vars.log_sz); i++) { 1751 if (test_bit(i, &vector)) { 1752 ent = cmd->ent_arr[i]; 1753 1754 if (forced && ent->ret == -ETIMEDOUT) 1755 set_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, 1756 &ent->state); 1757 else if (!forced) /* real FW completion */ 1758 clear_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, 1759 &ent->state); 1760 1761 /* if we already completed the command, ignore it */ 1762 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, 1763 &ent->state)) { 1764 /* only real completion can free the cmd slot */ 1765 if (!forced) { 1766 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n", 1767 ent->idx); 1768 cmd_ent_put(ent); 1769 } 1770 continue; 1771 } 1772 1773 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work)) 1774 cmd_ent_put(ent); /* timeout work was canceled */ 1775 1776 if (!forced || /* Real FW completion */ 1777 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */ 1778 !opcode_allowed(cmd, ent->op)) 1779 cmd_ent_put(ent); 1780 1781 ent->ts2 = ktime_get_ns(); 1782 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); 1783 dump_command(dev, ent, 0); 1784 1785 if (vec & MLX5_TRIGGERED_CMD_COMP) 1786 ent->ret = -ENXIO; 1787 1788 if (!ent->ret) { /* Command completed by FW */ 1789 if (!cmd->checksum_disabled) 1790 ent->ret = verify_signature(ent); 1791 1792 ent->status = ent->lay->status_own >> 1; 1793 1794 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n", 1795 ent->ret, deliv_status_to_str(ent->status), ent->status); 1796 } 1797 1798 if (ent->callback) { 1799 ds = ent->ts2 - ent->ts1; 1800 stats = xa_load(&cmd->stats, ent->op); 1801 if (stats) { 1802 spin_lock_irqsave(&stats->lock, flags); 1803 stats->sum += ds; 1804 ++stats->n; 1805 spin_unlock_irqrestore(&stats->lock, flags); 1806 } 1807 1808 callback = ent->callback; 1809 context = ent->context; 1810 err = ent->ret ? : ent->status; 1811 if (err > 0) /* Failed in FW, command didn't execute */ 1812 err = deliv_status_to_err(err); 1813 1814 if (!err) 1815 err = mlx5_copy_from_msg(ent->uout, 1816 ent->out, 1817 ent->uout_size); 1818 1819 mlx5_free_cmd_msg(dev, ent->out); 1820 free_msg(dev, ent->in); 1821 1822 /* final consumer is done, release ent */ 1823 cmd_ent_put(ent); 1824 callback(err, context); 1825 } else { 1826 /* release wait_func() so mlx5_cmd_invoke() 1827 * can make the final ent_put() 1828 */ 1829 complete(&ent->done); 1830 } 1831 } 1832 } 1833 } 1834 1835 #define MLX5_MAX_MANAGE_PAGES_CMD_ENT 1 1836 #define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \ 1837 MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1) 1838 1839 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) 1840 { 1841 struct mlx5_cmd *cmd = &dev->cmd; 1842 unsigned long bitmask; 1843 unsigned long flags; 1844 u64 vector; 1845 int i; 1846 1847 /* wait for pending handlers to complete */ 1848 mlx5_eq_synchronize_cmd_irq(dev); 1849 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 1850 vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK; 1851 if (!vector) 1852 goto no_trig; 1853 1854 bitmask = vector; 1855 /* we must increment the allocated entries refcount before triggering the completions 1856 * to guarantee pending commands will not get freed in the meanwhile. 1857 * For that reason, it also has to be done inside the alloc_lock. 1858 */ 1859 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1860 cmd_ent_get(cmd->ent_arr[i]); 1861 vector |= MLX5_TRIGGERED_CMD_COMP; 1862 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1863 1864 mlx5_core_dbg(dev, "vector 0x%llx\n", vector); 1865 mlx5_cmd_comp_handler(dev, vector, true); 1866 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1867 cmd_ent_put(cmd->ent_arr[i]); 1868 return; 1869 1870 no_trig: 1871 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1872 } 1873 1874 void mlx5_cmd_flush(struct mlx5_core_dev *dev) 1875 { 1876 struct mlx5_cmd *cmd = &dev->cmd; 1877 int i; 1878 1879 for (i = 0; i < cmd->vars.max_reg_cmds; i++) { 1880 while (down_trylock(&cmd->vars.sem)) { 1881 mlx5_cmd_trigger_completions(dev); 1882 cond_resched(); 1883 } 1884 } 1885 1886 while (down_trylock(&cmd->vars.pages_sem)) { 1887 mlx5_cmd_trigger_completions(dev); 1888 cond_resched(); 1889 } 1890 1891 /* Unlock cmdif */ 1892 up(&cmd->vars.pages_sem); 1893 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1894 up(&cmd->vars.sem); 1895 } 1896 1897 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, 1898 gfp_t gfp) 1899 { 1900 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM); 1901 struct cmd_msg_cache *ch = NULL; 1902 struct mlx5_cmd *cmd = &dev->cmd; 1903 int i; 1904 1905 if (in_size <= 16) 1906 goto cache_miss; 1907 1908 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 1909 ch = &cmd->cache[i]; 1910 if (in_size > ch->max_inbox_size) 1911 continue; 1912 spin_lock_irq(&ch->lock); 1913 if (list_empty(&ch->head)) { 1914 spin_unlock_irq(&ch->lock); 1915 continue; 1916 } 1917 msg = list_entry(ch->head.next, typeof(*msg), list); 1918 /* For cached lists, we must explicitly state what is 1919 * the real size 1920 */ 1921 msg->len = in_size; 1922 list_del(&msg->list); 1923 spin_unlock_irq(&ch->lock); 1924 break; 1925 } 1926 1927 if (!IS_ERR(msg)) 1928 return msg; 1929 1930 cache_miss: 1931 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0); 1932 return msg; 1933 } 1934 1935 static int is_manage_pages(void *in) 1936 { 1937 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES; 1938 } 1939 1940 static bool mlx5_has_privileged_uid(struct mlx5_core_dev *dev) 1941 { 1942 return !xa_empty(&dev->cmd.vars.privileged_uids); 1943 } 1944 1945 static bool mlx5_cmd_is_privileged_uid(struct mlx5_core_dev *dev, 1946 u16 uid) 1947 { 1948 return !!xa_load(&dev->cmd.vars.privileged_uids, uid); 1949 } 1950 1951 /* Notes: 1952 * 1. Callback functions may not sleep 1953 * 2. Page queue commands do not support asynchrous completion 1954 */ 1955 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1956 int out_size, mlx5_cmd_cbk_t callback, void *context, 1957 bool force_polling) 1958 { 1959 struct mlx5_cmd_msg *inb, *outb; 1960 u16 opcode = in_to_opcode(in); 1961 bool throttle_locked = false; 1962 bool unpriv_locked = false; 1963 u16 uid = in_to_uid(in); 1964 int pages_queue; 1965 gfp_t gfp; 1966 u8 token; 1967 int err; 1968 1969 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) 1970 return -ENXIO; 1971 1972 if (!callback) { 1973 /* The semaphore is already held for callback commands. It was 1974 * acquired in mlx5_cmd_exec_cb() 1975 */ 1976 if (uid && mlx5_has_privileged_uid(dev)) { 1977 if (!mlx5_cmd_is_privileged_uid(dev, uid)) { 1978 unpriv_locked = true; 1979 down(&dev->cmd.vars.unprivileged_sem); 1980 } 1981 } else if (mlx5_cmd_is_throttle_opcode(opcode)) { 1982 throttle_locked = true; 1983 down(&dev->cmd.vars.throttle_sem); 1984 } 1985 } 1986 1987 pages_queue = is_manage_pages(in); 1988 gfp = callback ? GFP_ATOMIC : GFP_KERNEL; 1989 1990 inb = alloc_msg(dev, in_size, gfp); 1991 if (IS_ERR(inb)) { 1992 err = PTR_ERR(inb); 1993 goto out_up; 1994 } 1995 1996 token = alloc_token(&dev->cmd); 1997 1998 err = mlx5_copy_to_msg(inb, in, in_size, token); 1999 if (err) { 2000 mlx5_core_warn(dev, "err %d\n", err); 2001 goto out_in; 2002 } 2003 2004 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token); 2005 if (IS_ERR(outb)) { 2006 err = PTR_ERR(outb); 2007 goto out_in; 2008 } 2009 2010 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, 2011 pages_queue, token, force_polling); 2012 if (callback && !err) 2013 return 0; 2014 2015 if (err > 0) /* Failed in FW, command didn't execute */ 2016 err = deliv_status_to_err(err); 2017 2018 if (err) 2019 goto out_out; 2020 2021 /* command completed by FW */ 2022 err = mlx5_copy_from_msg(out, outb, out_size); 2023 out_out: 2024 mlx5_free_cmd_msg(dev, outb); 2025 out_in: 2026 free_msg(dev, inb); 2027 out_up: 2028 if (throttle_locked) 2029 up(&dev->cmd.vars.throttle_sem); 2030 if (unpriv_locked) 2031 up(&dev->cmd.vars.unprivileged_sem); 2032 2033 return err; 2034 } 2035 2036 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 2037 { 2038 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 2039 u8 status = MLX5_GET(mbox_out, out, status); 2040 2041 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod, 2042 cmd_status_str(status), status, syndrome, 2043 cmd_status_to_err(status)); 2044 } 2045 2046 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status, 2047 u32 syndrome, int err) 2048 { 2049 const char *namep = mlx5_command_str(opcode); 2050 struct mlx5_cmd_stats *stats; 2051 unsigned long flags; 2052 2053 if (!err || !(strcmp(namep, "unknown command opcode"))) 2054 return; 2055 2056 stats = xa_load(&dev->cmd.stats, opcode); 2057 if (!stats) 2058 return; 2059 spin_lock_irqsave(&stats->lock, flags); 2060 stats->failed++; 2061 if (err < 0) 2062 stats->last_failed_errno = -err; 2063 if (err == -EREMOTEIO) { 2064 stats->failed_mbox_status++; 2065 stats->last_failed_mbox_status = status; 2066 stats->last_failed_syndrome = syndrome; 2067 } 2068 spin_unlock_irqrestore(&stats->lock, flags); 2069 } 2070 2071 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */ 2072 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out) 2073 { 2074 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 2075 u8 status = MLX5_GET(mbox_out, out, status); 2076 2077 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */ 2078 err = -EIO; 2079 2080 if (!err && status != MLX5_CMD_STAT_OK) { 2081 err = -EREMOTEIO; 2082 mlx5_cmd_err_trace(dev, opcode, op_mod, out); 2083 } 2084 2085 cmd_status_log(dev, opcode, status, syndrome, err); 2086 return err; 2087 } 2088 2089 /** 2090 * mlx5_cmd_do - Executes a fw command, wait for completion. 2091 * Unlike mlx5_cmd_exec, this function will not translate or intercept 2092 * outbox.status and will return -EREMOTEIO when 2093 * outbox.status != MLX5_CMD_STAT_OK 2094 * 2095 * @dev: mlx5 core device 2096 * @in: inbox mlx5_ifc command buffer 2097 * @in_size: inbox buffer size 2098 * @out: outbox mlx5_ifc buffer 2099 * @out_size: outbox size 2100 * 2101 * @return: 2102 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK. 2103 * Caller must check FW outbox status. 2104 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK. 2105 * < 0 : Command execution couldn't be performed by firmware or driver 2106 */ 2107 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size) 2108 { 2109 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false); 2110 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 2111 u16 opcode = in_to_opcode(in); 2112 2113 return cmd_status_err(dev, err, opcode, op_mod, out); 2114 } 2115 EXPORT_SYMBOL(mlx5_cmd_do); 2116 2117 /** 2118 * mlx5_cmd_exec - Executes a fw command, wait for completion 2119 * 2120 * @dev: mlx5 core device 2121 * @in: inbox mlx5_ifc command buffer 2122 * @in_size: inbox buffer size 2123 * @out: outbox mlx5_ifc buffer 2124 * @out_size: outbox size 2125 * 2126 * @return: 0 if no error, FW command execution was successful 2127 * and outbox status is ok. 2128 */ 2129 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 2130 int out_size) 2131 { 2132 int err = mlx5_cmd_do(dev, in, in_size, out, out_size); 2133 2134 return mlx5_cmd_check(dev, err, in, out); 2135 } 2136 EXPORT_SYMBOL(mlx5_cmd_exec); 2137 2138 /** 2139 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion 2140 * Needed for driver force teardown, when command completion EQ 2141 * will not be available to complete the command 2142 * 2143 * @dev: mlx5 core device 2144 * @in: inbox mlx5_ifc command buffer 2145 * @in_size: inbox buffer size 2146 * @out: outbox mlx5_ifc buffer 2147 * @out_size: outbox size 2148 * 2149 * @return: 0 if no error, FW command execution was successful 2150 * and outbox status is ok. 2151 */ 2152 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, 2153 void *out, int out_size) 2154 { 2155 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true); 2156 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 2157 u16 opcode = in_to_opcode(in); 2158 2159 err = cmd_status_err(dev, err, opcode, op_mod, out); 2160 return mlx5_cmd_check(dev, err, in, out); 2161 } 2162 EXPORT_SYMBOL(mlx5_cmd_exec_polling); 2163 2164 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, 2165 struct mlx5_async_ctx *ctx) 2166 { 2167 ctx->dev = dev; 2168 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */ 2169 atomic_set(&ctx->num_inflight, 1); 2170 init_completion(&ctx->inflight_done); 2171 } 2172 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); 2173 2174 /** 2175 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx 2176 * @ctx: The ctx to clean 2177 * 2178 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The 2179 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after 2180 * the call mlx5_cleanup_async_ctx(). 2181 */ 2182 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx) 2183 { 2184 if (!atomic_dec_and_test(&ctx->num_inflight)) 2185 wait_for_completion(&ctx->inflight_done); 2186 } 2187 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); 2188 2189 static void mlx5_cmd_exec_cb_handler(int status, void *_work) 2190 { 2191 struct mlx5_async_work *work = _work; 2192 struct mlx5_async_ctx *ctx; 2193 struct mlx5_core_dev *dev; 2194 bool throttle_locked; 2195 bool unpriv_locked; 2196 2197 ctx = work->ctx; 2198 dev = ctx->dev; 2199 throttle_locked = work->throttle_locked; 2200 unpriv_locked = work->unpriv_locked; 2201 status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out); 2202 work->user_callback(status, work); 2203 /* Can't access "work" from this point on. It could have been freed in 2204 * the callback. 2205 */ 2206 if (throttle_locked) 2207 up(&dev->cmd.vars.throttle_sem); 2208 if (unpriv_locked) 2209 up(&dev->cmd.vars.unprivileged_sem); 2210 if (atomic_dec_and_test(&ctx->num_inflight)) 2211 complete(&ctx->inflight_done); 2212 } 2213 2214 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 2215 void *out, int out_size, mlx5_async_cbk_t callback, 2216 struct mlx5_async_work *work) 2217 { 2218 struct mlx5_core_dev *dev = ctx->dev; 2219 u16 uid; 2220 int ret; 2221 2222 work->ctx = ctx; 2223 work->user_callback = callback; 2224 work->opcode = in_to_opcode(in); 2225 work->op_mod = MLX5_GET(mbox_in, in, op_mod); 2226 work->out = out; 2227 work->throttle_locked = false; 2228 work->unpriv_locked = false; 2229 uid = in_to_uid(in); 2230 2231 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight))) 2232 return -EIO; 2233 2234 if (uid && mlx5_has_privileged_uid(dev)) { 2235 if (!mlx5_cmd_is_privileged_uid(dev, uid)) { 2236 if (down_trylock(&dev->cmd.vars.unprivileged_sem)) { 2237 ret = -EBUSY; 2238 goto dec_num_inflight; 2239 } 2240 work->unpriv_locked = true; 2241 } 2242 } else if (mlx5_cmd_is_throttle_opcode(in_to_opcode(in))) { 2243 if (down_trylock(&dev->cmd.vars.throttle_sem)) { 2244 ret = -EBUSY; 2245 goto dec_num_inflight; 2246 } 2247 work->throttle_locked = true; 2248 } 2249 2250 ret = cmd_exec(dev, in, in_size, out, out_size, 2251 mlx5_cmd_exec_cb_handler, work, false); 2252 if (ret) 2253 goto sem_up; 2254 2255 return 0; 2256 2257 sem_up: 2258 if (work->throttle_locked) 2259 up(&dev->cmd.vars.throttle_sem); 2260 if (work->unpriv_locked) 2261 up(&dev->cmd.vars.unprivileged_sem); 2262 dec_num_inflight: 2263 if (atomic_dec_and_test(&ctx->num_inflight)) 2264 complete(&ctx->inflight_done); 2265 2266 return ret; 2267 } 2268 EXPORT_SYMBOL(mlx5_cmd_exec_cb); 2269 2270 int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev, 2271 struct mlx5_cmd_allow_other_vhca_access_attr *attr) 2272 { 2273 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {}; 2274 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {}; 2275 void *key; 2276 2277 MLX5_SET(allow_other_vhca_access_in, 2278 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS); 2279 MLX5_SET(allow_other_vhca_access_in, 2280 in, object_type_to_be_accessed, attr->obj_type); 2281 MLX5_SET(allow_other_vhca_access_in, 2282 in, object_id_to_be_accessed, attr->obj_id); 2283 2284 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key); 2285 memcpy(key, attr->access_key, sizeof(attr->access_key)); 2286 2287 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2288 } 2289 2290 int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev, 2291 struct mlx5_cmd_alias_obj_create_attr *alias_attr, 2292 u32 *obj_id) 2293 { 2294 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 2295 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {}; 2296 void *param; 2297 void *attr; 2298 void *key; 2299 int ret; 2300 2301 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr); 2302 MLX5_SET(general_obj_in_cmd_hdr, 2303 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 2304 MLX5_SET(general_obj_in_cmd_hdr, 2305 attr, obj_type, alias_attr->obj_type); 2306 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); 2307 MLX5_SET(general_obj_create_param, param, alias_object, 1); 2308 2309 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx); 2310 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id); 2311 MLX5_SET(alias_context, attr, vhca_id_type, alias_attr->vhca_id_type); 2312 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id); 2313 2314 key = MLX5_ADDR_OF(alias_context, attr, access_key); 2315 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key)); 2316 2317 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2318 if (ret) 2319 return ret; 2320 2321 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 2322 2323 return 0; 2324 } 2325 2326 int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id, 2327 u16 obj_type) 2328 { 2329 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 2330 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 2331 2332 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 2333 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type); 2334 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 2335 2336 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2337 } 2338 2339 static void destroy_msg_cache(struct mlx5_core_dev *dev) 2340 { 2341 struct cmd_msg_cache *ch; 2342 struct mlx5_cmd_msg *msg; 2343 struct mlx5_cmd_msg *n; 2344 int i; 2345 2346 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 2347 ch = &dev->cmd.cache[i]; 2348 list_for_each_entry_safe(msg, n, &ch->head, list) { 2349 list_del(&msg->list); 2350 mlx5_free_cmd_msg(dev, msg); 2351 } 2352 } 2353 } 2354 2355 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = { 2356 512, 32, 16, 8, 2 2357 }; 2358 2359 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = { 2360 16 + MLX5_CMD_DATA_BLOCK_SIZE, 2361 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2, 2362 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16, 2363 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256, 2364 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512, 2365 }; 2366 2367 static void create_msg_cache(struct mlx5_core_dev *dev) 2368 { 2369 struct mlx5_cmd *cmd = &dev->cmd; 2370 struct cmd_msg_cache *ch; 2371 struct mlx5_cmd_msg *msg; 2372 int i; 2373 int k; 2374 2375 /* Initialize and fill the caches with initial entries */ 2376 for (k = 0; k < dev->profile.num_cmd_caches; k++) { 2377 ch = &cmd->cache[k]; 2378 spin_lock_init(&ch->lock); 2379 INIT_LIST_HEAD(&ch->head); 2380 ch->num_ent = cmd_cache_num_ent[k]; 2381 ch->max_inbox_size = cmd_cache_ent_size[k]; 2382 for (i = 0; i < ch->num_ent; i++) { 2383 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN, 2384 ch->max_inbox_size, 0); 2385 if (IS_ERR(msg)) 2386 break; 2387 msg->parent = ch; 2388 list_add_tail(&msg->list, &ch->head); 2389 } 2390 } 2391 } 2392 2393 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2394 { 2395 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, 2396 &cmd->alloc_dma, GFP_KERNEL); 2397 if (!cmd->cmd_alloc_buf) 2398 return -ENOMEM; 2399 2400 /* make sure it is aligned to 4K */ 2401 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) { 2402 cmd->cmd_buf = cmd->cmd_alloc_buf; 2403 cmd->dma = cmd->alloc_dma; 2404 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE; 2405 return 0; 2406 } 2407 2408 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, 2409 cmd->alloc_dma); 2410 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), 2411 2 * MLX5_ADAPTER_PAGE_SIZE - 1, 2412 &cmd->alloc_dma, GFP_KERNEL); 2413 if (!cmd->cmd_alloc_buf) 2414 return -ENOMEM; 2415 2416 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE); 2417 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE); 2418 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1; 2419 return 0; 2420 } 2421 2422 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2423 { 2424 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf, 2425 cmd->alloc_dma); 2426 } 2427 2428 static u16 cmdif_rev(struct mlx5_core_dev *dev) 2429 { 2430 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; 2431 } 2432 2433 int mlx5_cmd_init(struct mlx5_core_dev *dev) 2434 { 2435 struct mlx5_cmd *cmd = &dev->cmd; 2436 2437 cmd->checksum_disabled = 1; 2438 2439 spin_lock_init(&cmd->alloc_lock); 2440 spin_lock_init(&cmd->token_lock); 2441 2442 set_wqname(dev); 2443 cmd->wq = create_singlethread_workqueue(cmd->wq_name); 2444 if (!cmd->wq) { 2445 mlx5_core_err(dev, "failed to create command workqueue\n"); 2446 return -ENOMEM; 2447 } 2448 2449 mlx5_cmdif_debugfs_init(dev); 2450 2451 return 0; 2452 } 2453 2454 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) 2455 { 2456 struct mlx5_cmd *cmd = &dev->cmd; 2457 2458 mlx5_cmdif_debugfs_cleanup(dev); 2459 destroy_workqueue(cmd->wq); 2460 } 2461 2462 int mlx5_cmd_enable(struct mlx5_core_dev *dev) 2463 { 2464 int size = sizeof(struct mlx5_cmd_prot_block); 2465 int align = roundup_pow_of_two(size); 2466 struct mlx5_cmd *cmd = &dev->cmd; 2467 u32 cmd_h, cmd_l; 2468 int err; 2469 2470 memset(&cmd->vars, 0, sizeof(cmd->vars)); 2471 cmd->vars.cmdif_rev = cmdif_rev(dev); 2472 if (cmd->vars.cmdif_rev != CMD_IF_REV) { 2473 mlx5_core_err(dev, 2474 "Driver cmdif rev(%d) differs from firmware's(%d)\n", 2475 CMD_IF_REV, cmd->vars.cmdif_rev); 2476 return -EINVAL; 2477 } 2478 2479 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; 2480 cmd->vars.log_sz = cmd_l >> 4 & 0xf; 2481 cmd->vars.log_stride = cmd_l & 0xf; 2482 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) { 2483 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", 2484 1 << cmd->vars.log_sz); 2485 return -EINVAL; 2486 } 2487 2488 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) { 2489 mlx5_core_err(dev, "command queue size overflow\n"); 2490 return -EINVAL; 2491 } 2492 2493 cmd->state = MLX5_CMDIF_STATE_DOWN; 2494 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1; 2495 cmd->vars.bitmask = MLX5_CMD_MASK; 2496 2497 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds); 2498 sema_init(&cmd->vars.pages_sem, 1); 2499 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2)); 2500 sema_init(&cmd->vars.unprivileged_sem, 2501 DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2)); 2502 2503 xa_init(&cmd->vars.privileged_uids); 2504 2505 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); 2506 if (!cmd->pool) { 2507 err = -ENOMEM; 2508 goto err_destroy_xa; 2509 } 2510 2511 err = alloc_cmd_page(dev, cmd); 2512 if (err) 2513 goto err_free_pool; 2514 2515 cmd_h = (u32)((u64)(cmd->dma) >> 32); 2516 cmd_l = (u32)(cmd->dma); 2517 if (cmd_l & 0xfff) { 2518 mlx5_core_err(dev, "invalid command queue address\n"); 2519 err = -ENOMEM; 2520 goto err_cmd_page; 2521 } 2522 2523 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); 2524 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); 2525 2526 /* Make sure firmware sees the complete address before we proceed */ 2527 wmb(); 2528 2529 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); 2530 2531 cmd->mode = CMD_MODE_POLLING; 2532 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; 2533 2534 create_msg_cache(dev); 2535 create_debugfs_files(dev); 2536 2537 return 0; 2538 2539 err_cmd_page: 2540 free_cmd_page(dev, cmd); 2541 err_free_pool: 2542 dma_pool_destroy(cmd->pool); 2543 err_destroy_xa: 2544 xa_destroy(&dev->cmd.vars.privileged_uids); 2545 return err; 2546 } 2547 2548 void mlx5_cmd_disable(struct mlx5_core_dev *dev) 2549 { 2550 struct mlx5_cmd *cmd = &dev->cmd; 2551 2552 flush_workqueue(cmd->wq); 2553 clean_debug_files(dev); 2554 destroy_msg_cache(dev); 2555 free_cmd_page(dev, cmd); 2556 dma_pool_destroy(cmd->pool); 2557 xa_destroy(&dev->cmd.vars.privileged_uids); 2558 } 2559 2560 void mlx5_cmd_set_state(struct mlx5_core_dev *dev, 2561 enum mlx5_cmdif_state cmdif_state) 2562 { 2563 dev->cmd.state = cmdif_state; 2564 } 2565 2566 int mlx5_cmd_add_privileged_uid(struct mlx5_core_dev *dev, u16 uid) 2567 { 2568 return xa_insert(&dev->cmd.vars.privileged_uids, uid, 2569 xa_mk_value(uid), GFP_KERNEL); 2570 } 2571 EXPORT_SYMBOL(mlx5_cmd_add_privileged_uid); 2572 2573 void mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev *dev, u16 uid) 2574 { 2575 void *data = xa_erase(&dev->cmd.vars.privileged_uids, uid); 2576 2577 WARN(!data, "Privileged UID %u does not exist\n", uid); 2578 } 2579 EXPORT_SYMBOL(mlx5_cmd_remove_privileged_uid); 2580