1 /*
2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/errno.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/slab.h>
38 #include <linux/delay.h>
39 #include <linux/random.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/eq.h>
42 #include <linux/debugfs.h>
43
44 #include "mlx5_core.h"
45 #include "lib/eq.h"
46 #include "lib/tout.h"
47 #define CREATE_TRACE_POINTS
48 #include "diag/cmd_tracepoint.h"
49
50 struct mlx5_ifc_mbox_out_bits {
51 u8 status[0x8];
52 u8 reserved_at_8[0x18];
53
54 u8 syndrome[0x20];
55
56 u8 reserved_at_40[0x40];
57 };
58
59 struct mlx5_ifc_mbox_in_bits {
60 u8 opcode[0x10];
61 u8 uid[0x10];
62
63 u8 reserved_at_20[0x10];
64 u8 op_mod[0x10];
65
66 u8 reserved_at_40[0x40];
67 };
68
69 enum {
70 CMD_IF_REV = 5,
71 };
72
73 enum {
74 CMD_MODE_POLLING,
75 CMD_MODE_EVENTS
76 };
77
78 enum {
79 MLX5_CMD_DELIVERY_STAT_OK = 0x0,
80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
90 };
91
in_to_opcode(void * in)92 static u16 in_to_opcode(void *in)
93 {
94 return MLX5_GET(mbox_in, in, opcode);
95 }
96
97 /* Returns true for opcodes that might be triggered very frequently and throttle
98 * the command interface. Limit their command slots usage.
99 */
mlx5_cmd_is_throttle_opcode(u16 op)100 static bool mlx5_cmd_is_throttle_opcode(u16 op)
101 {
102 switch (op) {
103 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
104 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
105 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
106 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
107 case MLX5_CMD_OP_SYNC_CRYPTO:
108 return true;
109 }
110 return false;
111 }
112
113 static struct mlx5_cmd_work_ent *
cmd_alloc_ent(struct mlx5_cmd * cmd,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t cbk,void * context,int page_queue)114 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
115 struct mlx5_cmd_msg *out, void *uout, int uout_size,
116 mlx5_cmd_cbk_t cbk, void *context, int page_queue)
117 {
118 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
119 struct mlx5_cmd_work_ent *ent;
120
121 ent = kzalloc(sizeof(*ent), alloc_flags);
122 if (!ent)
123 return ERR_PTR(-ENOMEM);
124
125 ent->idx = -EINVAL;
126 ent->in = in;
127 ent->out = out;
128 ent->uout = uout;
129 ent->uout_size = uout_size;
130 ent->callback = cbk;
131 ent->context = context;
132 ent->cmd = cmd;
133 ent->page_queue = page_queue;
134 ent->op = in_to_opcode(in->first.data);
135 refcount_set(&ent->refcnt, 1);
136
137 return ent;
138 }
139
cmd_free_ent(struct mlx5_cmd_work_ent * ent)140 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
141 {
142 kfree(ent);
143 }
144
alloc_token(struct mlx5_cmd * cmd)145 static u8 alloc_token(struct mlx5_cmd *cmd)
146 {
147 u8 token;
148
149 spin_lock(&cmd->token_lock);
150 cmd->token++;
151 if (cmd->token == 0)
152 cmd->token++;
153 token = cmd->token;
154 spin_unlock(&cmd->token_lock);
155
156 return token;
157 }
158
cmd_alloc_index(struct mlx5_cmd * cmd,struct mlx5_cmd_work_ent * ent)159 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
160 {
161 unsigned long flags;
162 int ret;
163
164 spin_lock_irqsave(&cmd->alloc_lock, flags);
165 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds);
166 if (ret < cmd->vars.max_reg_cmds) {
167 clear_bit(ret, &cmd->vars.bitmask);
168 ent->idx = ret;
169 cmd->ent_arr[ent->idx] = ent;
170 }
171 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
172
173 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;
174 }
175
cmd_free_index(struct mlx5_cmd * cmd,int idx)176 static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
177 {
178 lockdep_assert_held(&cmd->alloc_lock);
179 set_bit(idx, &cmd->vars.bitmask);
180 }
181
cmd_ent_get(struct mlx5_cmd_work_ent * ent)182 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
183 {
184 refcount_inc(&ent->refcnt);
185 }
186
cmd_ent_put(struct mlx5_cmd_work_ent * ent)187 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
188 {
189 struct mlx5_cmd *cmd = ent->cmd;
190 unsigned long flags;
191
192 spin_lock_irqsave(&cmd->alloc_lock, flags);
193 if (!refcount_dec_and_test(&ent->refcnt))
194 goto out;
195
196 if (ent->idx >= 0) {
197 cmd_free_index(cmd, ent->idx);
198 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
199 }
200
201 cmd_free_ent(ent);
202 out:
203 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
204 }
205
get_inst(struct mlx5_cmd * cmd,int idx)206 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
207 {
208 return cmd->cmd_buf + (idx << cmd->vars.log_stride);
209 }
210
mlx5_calc_cmd_blocks(struct mlx5_cmd_msg * msg)211 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
212 {
213 int size = msg->len;
214 int blen = size - min_t(int, sizeof(msg->first.data), size);
215
216 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
217 }
218
xor8_buf(void * buf,size_t offset,int len)219 static u8 xor8_buf(void *buf, size_t offset, int len)
220 {
221 u8 *ptr = buf;
222 u8 sum = 0;
223 int i;
224 int end = len + offset;
225
226 for (i = offset; i < end; i++)
227 sum ^= ptr[i];
228
229 return sum;
230 }
231
verify_block_sig(struct mlx5_cmd_prot_block * block)232 static int verify_block_sig(struct mlx5_cmd_prot_block *block)
233 {
234 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
235 int xor_len = sizeof(*block) - sizeof(block->data) - 1;
236
237 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
238 return -EHWPOISON;
239
240 if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
241 return -EHWPOISON;
242
243 return 0;
244 }
245
calc_block_sig(struct mlx5_cmd_prot_block * block)246 static void calc_block_sig(struct mlx5_cmd_prot_block *block)
247 {
248 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
249 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
250
251 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
252 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
253 }
254
calc_chain_sig(struct mlx5_cmd_msg * msg)255 static void calc_chain_sig(struct mlx5_cmd_msg *msg)
256 {
257 struct mlx5_cmd_mailbox *next = msg->next;
258 int n = mlx5_calc_cmd_blocks(msg);
259 int i = 0;
260
261 for (i = 0; i < n && next; i++) {
262 calc_block_sig(next->buf);
263 next = next->next;
264 }
265 }
266
set_signature(struct mlx5_cmd_work_ent * ent,int csum)267 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
268 {
269 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay));
270 if (csum) {
271 calc_chain_sig(ent->in);
272 calc_chain_sig(ent->out);
273 }
274 }
275
poll_timeout(struct mlx5_cmd_work_ent * ent)276 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
277 {
278 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
279 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
280 unsigned long poll_end;
281 u8 own;
282
283 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);
284
285 do {
286 own = READ_ONCE(ent->lay->status_own);
287 if (!(own & CMD_OWNER_HW)) {
288 ent->ret = 0;
289 return;
290 }
291 cond_resched();
292 } while (time_before(jiffies, poll_end));
293
294 ent->ret = -ETIMEDOUT;
295 }
296
verify_signature(struct mlx5_cmd_work_ent * ent)297 static int verify_signature(struct mlx5_cmd_work_ent *ent)
298 {
299 struct mlx5_cmd_mailbox *next = ent->out->next;
300 int n = mlx5_calc_cmd_blocks(ent->out);
301 int err;
302 u8 sig;
303 int i = 0;
304
305 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
306 if (sig != 0xff)
307 return -EHWPOISON;
308
309 for (i = 0; i < n && next; i++) {
310 err = verify_block_sig(next->buf);
311 if (err)
312 return -EHWPOISON;
313
314 next = next->next;
315 }
316
317 return 0;
318 }
319
dump_buf(void * buf,int size,int data_only,int offset,int idx)320 static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
321 {
322 __be32 *p = buf;
323 int i;
324
325 for (i = 0; i < size; i += 16) {
326 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
327 be32_to_cpu(p[0]), be32_to_cpu(p[1]),
328 be32_to_cpu(p[2]), be32_to_cpu(p[3]));
329 p += 4;
330 offset += 16;
331 }
332 if (!data_only)
333 pr_debug("\n");
334 }
335
mlx5_internal_err_ret_value(struct mlx5_core_dev * dev,u16 op,u32 * synd,u8 * status)336 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
337 u32 *synd, u8 *status)
338 {
339 *synd = 0;
340 *status = 0;
341
342 switch (op) {
343 case MLX5_CMD_OP_TEARDOWN_HCA:
344 case MLX5_CMD_OP_DISABLE_HCA:
345 case MLX5_CMD_OP_MANAGE_PAGES:
346 case MLX5_CMD_OP_DESTROY_MKEY:
347 case MLX5_CMD_OP_DESTROY_EQ:
348 case MLX5_CMD_OP_DESTROY_CQ:
349 case MLX5_CMD_OP_DESTROY_QP:
350 case MLX5_CMD_OP_DESTROY_PSV:
351 case MLX5_CMD_OP_DESTROY_SRQ:
352 case MLX5_CMD_OP_DESTROY_XRC_SRQ:
353 case MLX5_CMD_OP_DESTROY_XRQ:
354 case MLX5_CMD_OP_DESTROY_DCT:
355 case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
356 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
357 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
358 case MLX5_CMD_OP_DEALLOC_PD:
359 case MLX5_CMD_OP_DEALLOC_UAR:
360 case MLX5_CMD_OP_DETACH_FROM_MCG:
361 case MLX5_CMD_OP_DEALLOC_XRCD:
362 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
363 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
364 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
365 case MLX5_CMD_OP_DESTROY_LAG:
366 case MLX5_CMD_OP_DESTROY_VPORT_LAG:
367 case MLX5_CMD_OP_DESTROY_TIR:
368 case MLX5_CMD_OP_DESTROY_SQ:
369 case MLX5_CMD_OP_DESTROY_RQ:
370 case MLX5_CMD_OP_DESTROY_RMP:
371 case MLX5_CMD_OP_DESTROY_TIS:
372 case MLX5_CMD_OP_DESTROY_RQT:
373 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
374 case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
375 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
376 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
377 case MLX5_CMD_OP_2ERR_QP:
378 case MLX5_CMD_OP_2RST_QP:
379 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
380 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
381 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
382 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
383 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
384 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
385 case MLX5_CMD_OP_FPGA_DESTROY_QP:
386 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
387 case MLX5_CMD_OP_DEALLOC_MEMIC:
388 case MLX5_CMD_OP_PAGE_FAULT_RESUME:
389 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
390 case MLX5_CMD_OP_DEALLOC_SF:
391 case MLX5_CMD_OP_DESTROY_UCTX:
392 case MLX5_CMD_OP_DESTROY_UMEM:
393 case MLX5_CMD_OP_MODIFY_RQT:
394 return MLX5_CMD_STAT_OK;
395
396 case MLX5_CMD_OP_QUERY_HCA_CAP:
397 case MLX5_CMD_OP_QUERY_ADAPTER:
398 case MLX5_CMD_OP_INIT_HCA:
399 case MLX5_CMD_OP_ENABLE_HCA:
400 case MLX5_CMD_OP_QUERY_PAGES:
401 case MLX5_CMD_OP_SET_HCA_CAP:
402 case MLX5_CMD_OP_QUERY_ISSI:
403 case MLX5_CMD_OP_SET_ISSI:
404 case MLX5_CMD_OP_CREATE_MKEY:
405 case MLX5_CMD_OP_QUERY_MKEY:
406 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
407 case MLX5_CMD_OP_CREATE_EQ:
408 case MLX5_CMD_OP_QUERY_EQ:
409 case MLX5_CMD_OP_GEN_EQE:
410 case MLX5_CMD_OP_CREATE_CQ:
411 case MLX5_CMD_OP_QUERY_CQ:
412 case MLX5_CMD_OP_MODIFY_CQ:
413 case MLX5_CMD_OP_CREATE_QP:
414 case MLX5_CMD_OP_RST2INIT_QP:
415 case MLX5_CMD_OP_INIT2RTR_QP:
416 case MLX5_CMD_OP_RTR2RTS_QP:
417 case MLX5_CMD_OP_RTS2RTS_QP:
418 case MLX5_CMD_OP_SQERR2RTS_QP:
419 case MLX5_CMD_OP_QUERY_QP:
420 case MLX5_CMD_OP_SQD_RTS_QP:
421 case MLX5_CMD_OP_INIT2INIT_QP:
422 case MLX5_CMD_OP_CREATE_PSV:
423 case MLX5_CMD_OP_CREATE_SRQ:
424 case MLX5_CMD_OP_QUERY_SRQ:
425 case MLX5_CMD_OP_ARM_RQ:
426 case MLX5_CMD_OP_CREATE_XRC_SRQ:
427 case MLX5_CMD_OP_QUERY_XRC_SRQ:
428 case MLX5_CMD_OP_ARM_XRC_SRQ:
429 case MLX5_CMD_OP_CREATE_XRQ:
430 case MLX5_CMD_OP_QUERY_XRQ:
431 case MLX5_CMD_OP_ARM_XRQ:
432 case MLX5_CMD_OP_CREATE_DCT:
433 case MLX5_CMD_OP_DRAIN_DCT:
434 case MLX5_CMD_OP_QUERY_DCT:
435 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
436 case MLX5_CMD_OP_QUERY_VPORT_STATE:
437 case MLX5_CMD_OP_MODIFY_VPORT_STATE:
438 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
439 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
440 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
441 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
442 case MLX5_CMD_OP_SET_ROCE_ADDRESS:
443 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
444 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
445 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
446 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
447 case MLX5_CMD_OP_QUERY_VNIC_ENV:
448 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
449 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
450 case MLX5_CMD_OP_QUERY_Q_COUNTER:
451 case MLX5_CMD_OP_SET_MONITOR_COUNTER:
452 case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
453 case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
454 case MLX5_CMD_OP_QUERY_RATE_LIMIT:
455 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
456 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
457 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
458 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
459 case MLX5_CMD_OP_ALLOC_PD:
460 case MLX5_CMD_OP_ALLOC_UAR:
461 case MLX5_CMD_OP_CONFIG_INT_MODERATION:
462 case MLX5_CMD_OP_ACCESS_REG:
463 case MLX5_CMD_OP_ATTACH_TO_MCG:
464 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
465 case MLX5_CMD_OP_MAD_IFC:
466 case MLX5_CMD_OP_QUERY_MAD_DEMUX:
467 case MLX5_CMD_OP_SET_MAD_DEMUX:
468 case MLX5_CMD_OP_NOP:
469 case MLX5_CMD_OP_ALLOC_XRCD:
470 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
471 case MLX5_CMD_OP_QUERY_CONG_STATUS:
472 case MLX5_CMD_OP_MODIFY_CONG_STATUS:
473 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
474 case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
475 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
476 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
477 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
478 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
479 case MLX5_CMD_OP_CREATE_LAG:
480 case MLX5_CMD_OP_MODIFY_LAG:
481 case MLX5_CMD_OP_QUERY_LAG:
482 case MLX5_CMD_OP_CREATE_VPORT_LAG:
483 case MLX5_CMD_OP_CREATE_TIR:
484 case MLX5_CMD_OP_MODIFY_TIR:
485 case MLX5_CMD_OP_QUERY_TIR:
486 case MLX5_CMD_OP_CREATE_SQ:
487 case MLX5_CMD_OP_MODIFY_SQ:
488 case MLX5_CMD_OP_QUERY_SQ:
489 case MLX5_CMD_OP_CREATE_RQ:
490 case MLX5_CMD_OP_MODIFY_RQ:
491 case MLX5_CMD_OP_QUERY_RQ:
492 case MLX5_CMD_OP_CREATE_RMP:
493 case MLX5_CMD_OP_MODIFY_RMP:
494 case MLX5_CMD_OP_QUERY_RMP:
495 case MLX5_CMD_OP_CREATE_TIS:
496 case MLX5_CMD_OP_MODIFY_TIS:
497 case MLX5_CMD_OP_QUERY_TIS:
498 case MLX5_CMD_OP_CREATE_RQT:
499 case MLX5_CMD_OP_QUERY_RQT:
500
501 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
502 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
503 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
504 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
505 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
506 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
507 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
508 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
509 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
510 case MLX5_CMD_OP_FPGA_CREATE_QP:
511 case MLX5_CMD_OP_FPGA_MODIFY_QP:
512 case MLX5_CMD_OP_FPGA_QUERY_QP:
513 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
514 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
515 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
516 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
517 case MLX5_CMD_OP_CREATE_UCTX:
518 case MLX5_CMD_OP_CREATE_UMEM:
519 case MLX5_CMD_OP_ALLOC_MEMIC:
520 case MLX5_CMD_OP_MODIFY_XRQ:
521 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
522 case MLX5_CMD_OP_QUERY_VHCA_STATE:
523 case MLX5_CMD_OP_MODIFY_VHCA_STATE:
524 case MLX5_CMD_OP_ALLOC_SF:
525 case MLX5_CMD_OP_SUSPEND_VHCA:
526 case MLX5_CMD_OP_RESUME_VHCA:
527 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
528 case MLX5_CMD_OP_SAVE_VHCA_STATE:
529 case MLX5_CMD_OP_LOAD_VHCA_STATE:
530 case MLX5_CMD_OP_SYNC_CRYPTO:
531 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:
532 *status = MLX5_DRIVER_STATUS_ABORTED;
533 *synd = MLX5_DRIVER_SYND;
534 return -ENOLINK;
535 default:
536 mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
537 return -EINVAL;
538 }
539 }
540
mlx5_command_str(int command)541 const char *mlx5_command_str(int command)
542 {
543 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
544
545 switch (command) {
546 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
547 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
548 MLX5_COMMAND_STR_CASE(INIT_HCA);
549 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
550 MLX5_COMMAND_STR_CASE(ENABLE_HCA);
551 MLX5_COMMAND_STR_CASE(DISABLE_HCA);
552 MLX5_COMMAND_STR_CASE(QUERY_PAGES);
553 MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
554 MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
555 MLX5_COMMAND_STR_CASE(QUERY_ISSI);
556 MLX5_COMMAND_STR_CASE(SET_ISSI);
557 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
558 MLX5_COMMAND_STR_CASE(CREATE_MKEY);
559 MLX5_COMMAND_STR_CASE(QUERY_MKEY);
560 MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
561 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
562 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
563 MLX5_COMMAND_STR_CASE(CREATE_EQ);
564 MLX5_COMMAND_STR_CASE(DESTROY_EQ);
565 MLX5_COMMAND_STR_CASE(QUERY_EQ);
566 MLX5_COMMAND_STR_CASE(GEN_EQE);
567 MLX5_COMMAND_STR_CASE(CREATE_CQ);
568 MLX5_COMMAND_STR_CASE(DESTROY_CQ);
569 MLX5_COMMAND_STR_CASE(QUERY_CQ);
570 MLX5_COMMAND_STR_CASE(MODIFY_CQ);
571 MLX5_COMMAND_STR_CASE(CREATE_QP);
572 MLX5_COMMAND_STR_CASE(DESTROY_QP);
573 MLX5_COMMAND_STR_CASE(RST2INIT_QP);
574 MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
575 MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
576 MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
577 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
578 MLX5_COMMAND_STR_CASE(2ERR_QP);
579 MLX5_COMMAND_STR_CASE(2RST_QP);
580 MLX5_COMMAND_STR_CASE(QUERY_QP);
581 MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
582 MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
583 MLX5_COMMAND_STR_CASE(CREATE_PSV);
584 MLX5_COMMAND_STR_CASE(DESTROY_PSV);
585 MLX5_COMMAND_STR_CASE(CREATE_SRQ);
586 MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
587 MLX5_COMMAND_STR_CASE(QUERY_SRQ);
588 MLX5_COMMAND_STR_CASE(ARM_RQ);
589 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
590 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
591 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
592 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
593 MLX5_COMMAND_STR_CASE(CREATE_DCT);
594 MLX5_COMMAND_STR_CASE(DESTROY_DCT);
595 MLX5_COMMAND_STR_CASE(DRAIN_DCT);
596 MLX5_COMMAND_STR_CASE(QUERY_DCT);
597 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
598 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
599 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
600 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
601 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
602 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
603 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
604 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
605 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
606 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
607 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
608 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
609 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
610 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
611 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
612 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
613 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
614 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
615 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
616 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
617 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
618 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
619 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
620 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
621 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
622 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
623 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
624 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
625 MLX5_COMMAND_STR_CASE(ALLOC_PD);
626 MLX5_COMMAND_STR_CASE(DEALLOC_PD);
627 MLX5_COMMAND_STR_CASE(ALLOC_UAR);
628 MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
629 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
630 MLX5_COMMAND_STR_CASE(ACCESS_REG);
631 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
632 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
633 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
634 MLX5_COMMAND_STR_CASE(MAD_IFC);
635 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
636 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
637 MLX5_COMMAND_STR_CASE(NOP);
638 MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
639 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
640 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
641 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
642 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
643 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
644 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
645 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
646 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
647 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
648 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
649 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
650 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
651 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
652 MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
653 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
654 MLX5_COMMAND_STR_CASE(CREATE_LAG);
655 MLX5_COMMAND_STR_CASE(MODIFY_LAG);
656 MLX5_COMMAND_STR_CASE(QUERY_LAG);
657 MLX5_COMMAND_STR_CASE(DESTROY_LAG);
658 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
659 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
660 MLX5_COMMAND_STR_CASE(CREATE_TIR);
661 MLX5_COMMAND_STR_CASE(MODIFY_TIR);
662 MLX5_COMMAND_STR_CASE(DESTROY_TIR);
663 MLX5_COMMAND_STR_CASE(QUERY_TIR);
664 MLX5_COMMAND_STR_CASE(CREATE_SQ);
665 MLX5_COMMAND_STR_CASE(MODIFY_SQ);
666 MLX5_COMMAND_STR_CASE(DESTROY_SQ);
667 MLX5_COMMAND_STR_CASE(QUERY_SQ);
668 MLX5_COMMAND_STR_CASE(CREATE_RQ);
669 MLX5_COMMAND_STR_CASE(MODIFY_RQ);
670 MLX5_COMMAND_STR_CASE(DESTROY_RQ);
671 MLX5_COMMAND_STR_CASE(QUERY_RQ);
672 MLX5_COMMAND_STR_CASE(CREATE_RMP);
673 MLX5_COMMAND_STR_CASE(MODIFY_RMP);
674 MLX5_COMMAND_STR_CASE(DESTROY_RMP);
675 MLX5_COMMAND_STR_CASE(QUERY_RMP);
676 MLX5_COMMAND_STR_CASE(CREATE_TIS);
677 MLX5_COMMAND_STR_CASE(MODIFY_TIS);
678 MLX5_COMMAND_STR_CASE(DESTROY_TIS);
679 MLX5_COMMAND_STR_CASE(QUERY_TIS);
680 MLX5_COMMAND_STR_CASE(CREATE_RQT);
681 MLX5_COMMAND_STR_CASE(MODIFY_RQT);
682 MLX5_COMMAND_STR_CASE(DESTROY_RQT);
683 MLX5_COMMAND_STR_CASE(QUERY_RQT);
684 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
685 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
686 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
687 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
688 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
689 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
690 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
691 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
692 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
693 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
694 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
695 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
696 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
697 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
698 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
699 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
700 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
701 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
702 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
703 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
704 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
705 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
706 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
707 MLX5_COMMAND_STR_CASE(CREATE_XRQ);
708 MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
709 MLX5_COMMAND_STR_CASE(QUERY_XRQ);
710 MLX5_COMMAND_STR_CASE(ARM_XRQ);
711 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
712 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
713 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
714 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
715 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
716 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
717 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
718 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
719 MLX5_COMMAND_STR_CASE(CREATE_UCTX);
720 MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
721 MLX5_COMMAND_STR_CASE(CREATE_UMEM);
722 MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
723 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
724 MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
725 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
726 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
727 MLX5_COMMAND_STR_CASE(ALLOC_SF);
728 MLX5_COMMAND_STR_CASE(DEALLOC_SF);
729 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
730 MLX5_COMMAND_STR_CASE(RESUME_VHCA);
731 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
732 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
733 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
734 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);
735 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);
736 default: return "unknown command opcode";
737 }
738 }
739
cmd_status_str(u8 status)740 static const char *cmd_status_str(u8 status)
741 {
742 switch (status) {
743 case MLX5_CMD_STAT_OK:
744 return "OK";
745 case MLX5_CMD_STAT_INT_ERR:
746 return "internal error";
747 case MLX5_CMD_STAT_BAD_OP_ERR:
748 return "bad operation";
749 case MLX5_CMD_STAT_BAD_PARAM_ERR:
750 return "bad parameter";
751 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
752 return "bad system state";
753 case MLX5_CMD_STAT_BAD_RES_ERR:
754 return "bad resource";
755 case MLX5_CMD_STAT_RES_BUSY:
756 return "resource busy";
757 case MLX5_CMD_STAT_NOT_READY:
758 return "FW not ready";
759 case MLX5_CMD_STAT_LIM_ERR:
760 return "limits exceeded";
761 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
762 return "bad resource state";
763 case MLX5_CMD_STAT_IX_ERR:
764 return "bad index";
765 case MLX5_CMD_STAT_NO_RES_ERR:
766 return "no resources";
767 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
768 return "bad input length";
769 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
770 return "bad output length";
771 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
772 return "bad QP state";
773 case MLX5_CMD_STAT_BAD_PKT_ERR:
774 return "bad packet (discarded)";
775 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
776 return "bad size too many outstanding CQEs";
777 default:
778 return "unknown status";
779 }
780 }
781
cmd_status_to_err(u8 status)782 static int cmd_status_to_err(u8 status)
783 {
784 switch (status) {
785 case MLX5_CMD_STAT_OK: return 0;
786 case MLX5_CMD_STAT_INT_ERR: return -EIO;
787 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
788 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
789 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
790 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
791 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
792 case MLX5_CMD_STAT_NOT_READY: return -EAGAIN;
793 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;
794 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
795 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
796 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
797 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
798 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
799 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
800 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
801 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
802 default: return -EIO;
803 }
804 }
805
mlx5_cmd_out_err(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)806 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
807 {
808 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
809 u8 status = MLX5_GET(mbox_out, out, status);
810
811 mlx5_core_err_rl(dev,
812 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
813 mlx5_command_str(opcode), opcode, op_mod,
814 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
815 }
816 EXPORT_SYMBOL(mlx5_cmd_out_err);
817
cmd_status_print(struct mlx5_core_dev * dev,void * in,void * out)818 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
819 {
820 u16 opcode, op_mod;
821 u8 status;
822 u16 uid;
823
824 opcode = in_to_opcode(in);
825 op_mod = MLX5_GET(mbox_in, in, op_mod);
826 uid = MLX5_GET(mbox_in, in, uid);
827 status = MLX5_GET(mbox_out, out, status);
828
829 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&
830 opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)
831 mlx5_cmd_out_err(dev, opcode, op_mod, out);
832 }
833
mlx5_cmd_check(struct mlx5_core_dev * dev,int err,void * in,void * out)834 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
835 {
836 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
837 if (err == -ENXIO) {
838 u16 opcode = in_to_opcode(in);
839 u32 syndrome;
840 u8 status;
841
842 /* PCI Error, emulate command return status, for smooth reset */
843 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);
844 MLX5_SET(mbox_out, out, status, status);
845 MLX5_SET(mbox_out, out, syndrome, syndrome);
846 if (!err)
847 return 0;
848 }
849
850 /* driver or FW delivery error */
851 if (err != -EREMOTEIO && err)
852 return err;
853
854 /* check outbox status */
855 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
856 if (err)
857 cmd_status_print(dev, in, out);
858
859 return err;
860 }
861 EXPORT_SYMBOL(mlx5_cmd_check);
862
dump_command(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent,int input)863 static void dump_command(struct mlx5_core_dev *dev,
864 struct mlx5_cmd_work_ent *ent, int input)
865 {
866 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
867 struct mlx5_cmd_mailbox *next = msg->next;
868 int n = mlx5_calc_cmd_blocks(msg);
869 u16 op = ent->op;
870 int data_only;
871 u32 offset = 0;
872 int dump_len;
873 int i;
874
875 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
876 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
877
878 if (data_only)
879 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
880 "cmd[%d]: dump command data %s(0x%x) %s\n",
881 ent->idx, mlx5_command_str(op), op,
882 input ? "INPUT" : "OUTPUT");
883 else
884 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
885 ent->idx, mlx5_command_str(op), op,
886 input ? "INPUT" : "OUTPUT");
887
888 if (data_only) {
889 if (input) {
890 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
891 offset += sizeof(ent->lay->in);
892 } else {
893 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
894 offset += sizeof(ent->lay->out);
895 }
896 } else {
897 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
898 offset += sizeof(*ent->lay);
899 }
900
901 for (i = 0; i < n && next; i++) {
902 if (data_only) {
903 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
904 dump_buf(next->buf, dump_len, 1, offset, ent->idx);
905 offset += MLX5_CMD_DATA_BLOCK_SIZE;
906 } else {
907 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
908 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,
909 ent->idx);
910 offset += sizeof(struct mlx5_cmd_prot_block);
911 }
912 next = next->next;
913 }
914
915 if (data_only)
916 pr_debug("\n");
917
918 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
919 }
920
921 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
922
cb_timeout_handler(struct work_struct * work)923 static void cb_timeout_handler(struct work_struct *work)
924 {
925 struct delayed_work *dwork = container_of(work, struct delayed_work,
926 work);
927 struct mlx5_cmd_work_ent *ent = container_of(dwork,
928 struct mlx5_cmd_work_ent,
929 cb_timeout_work);
930 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
931 cmd);
932
933 mlx5_cmd_eq_recover(dev);
934
935 /* Maybe got handled by eq recover ? */
936 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
937 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
938 mlx5_command_str(ent->op), ent->op);
939 goto out; /* phew, already handled */
940 }
941
942 ent->ret = -ETIMEDOUT;
943 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
944 ent->idx, mlx5_command_str(ent->op), ent->op);
945 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
946
947 out:
948 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
949 }
950
951 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
952 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
953 struct mlx5_cmd_msg *msg);
954
opcode_allowed(struct mlx5_cmd * cmd,u16 opcode)955 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
956 {
957 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
958 return true;
959
960 return cmd->allowed_opcode == opcode;
961 }
962
mlx5_cmd_is_down(struct mlx5_core_dev * dev)963 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
964 {
965 return pci_channel_offline(dev->pdev) ||
966 dev->cmd.state != MLX5_CMDIF_STATE_UP ||
967 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
968 }
969
cmd_work_handler(struct work_struct * work)970 static void cmd_work_handler(struct work_struct *work)
971 {
972 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
973 struct mlx5_cmd *cmd = ent->cmd;
974 bool poll_cmd = ent->polling;
975 struct mlx5_cmd_layout *lay;
976 struct mlx5_core_dev *dev;
977 unsigned long timeout;
978 unsigned long flags;
979 int alloc_ret;
980 int cmd_mode;
981
982 complete(&ent->handling);
983
984 dev = container_of(cmd, struct mlx5_core_dev, cmd);
985 timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
986
987 if (!ent->page_queue) {
988 if (down_timeout(&cmd->vars.sem, timeout)) {
989 mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",
990 mlx5_command_str(ent->op), ent->op);
991 if (ent->callback) {
992 ent->callback(-EBUSY, ent->context);
993 mlx5_free_cmd_msg(dev, ent->out);
994 free_msg(dev, ent->in);
995 cmd_ent_put(ent);
996 } else {
997 ent->ret = -EBUSY;
998 complete(&ent->done);
999 }
1000 complete(&ent->slotted);
1001 return;
1002 }
1003 alloc_ret = cmd_alloc_index(cmd, ent);
1004 if (alloc_ret < 0) {
1005 mlx5_core_err_rl(dev, "failed to allocate command entry\n");
1006 if (ent->callback) {
1007 ent->callback(-EAGAIN, ent->context);
1008 mlx5_free_cmd_msg(dev, ent->out);
1009 free_msg(dev, ent->in);
1010 cmd_ent_put(ent);
1011 } else {
1012 ent->ret = -EAGAIN;
1013 complete(&ent->done);
1014 }
1015 up(&cmd->vars.sem);
1016 return;
1017 }
1018 } else {
1019 down(&cmd->vars.pages_sem);
1020 ent->idx = cmd->vars.max_reg_cmds;
1021 spin_lock_irqsave(&cmd->alloc_lock, flags);
1022 clear_bit(ent->idx, &cmd->vars.bitmask);
1023 cmd->ent_arr[ent->idx] = ent;
1024 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1025 }
1026
1027 complete(&ent->slotted);
1028
1029 lay = get_inst(cmd, ent->idx);
1030 ent->lay = lay;
1031 memset(lay, 0, sizeof(*lay));
1032 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1033 if (ent->in->next)
1034 lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1035 lay->inlen = cpu_to_be32(ent->in->len);
1036 if (ent->out->next)
1037 lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1038 lay->outlen = cpu_to_be32(ent->out->len);
1039 lay->type = MLX5_PCI_CMD_XPORT;
1040 lay->token = ent->token;
1041 lay->status_own = CMD_OWNER_HW;
1042 set_signature(ent, !cmd->checksum_disabled);
1043 dump_command(dev, ent, 1);
1044 ent->ts1 = ktime_get_ns();
1045 cmd_mode = cmd->mode;
1046
1047 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))
1048 cmd_ent_get(ent);
1049 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1050
1051 cmd_ent_get(ent); /* for the _real_ FW event on completion */
1052 /* Skip sending command to fw if internal error */
1053 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1054 ent->ret = -ENXIO;
1055 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1056 return;
1057 }
1058
1059 /* ring doorbell after the descriptor is valid */
1060 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1061 wmb();
1062 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1063 /* if not in polling don't use ent after this point */
1064 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1065 poll_timeout(ent);
1066 /* make sure we read the descriptor after ownership is SW */
1067 rmb();
1068 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT));
1069 }
1070 }
1071
deliv_status_to_err(u8 status)1072 static int deliv_status_to_err(u8 status)
1073 {
1074 switch (status) {
1075 case MLX5_CMD_DELIVERY_STAT_OK:
1076 case MLX5_DRIVER_STATUS_ABORTED:
1077 return 0;
1078 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1079 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1080 return -EBADR;
1081 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1082 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1083 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1084 return -EFAULT; /* Bad address */
1085 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1086 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1087 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1088 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1089 return -ENOMSG;
1090 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1091 return -EIO;
1092 default:
1093 return -EINVAL;
1094 }
1095 }
1096
deliv_status_to_str(u8 status)1097 static const char *deliv_status_to_str(u8 status)
1098 {
1099 switch (status) {
1100 case MLX5_CMD_DELIVERY_STAT_OK:
1101 return "no errors";
1102 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1103 return "signature error";
1104 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1105 return "token error";
1106 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1107 return "bad block number";
1108 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1109 return "output pointer not aligned to block size";
1110 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1111 return "input pointer not aligned to block size";
1112 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1113 return "firmware internal error";
1114 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1115 return "command input length error";
1116 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1117 return "command output length error";
1118 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1119 return "reserved fields not cleared";
1120 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1121 return "bad command descriptor type";
1122 default:
1123 return "unknown status code";
1124 }
1125 }
1126
1127 enum {
1128 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
1129 };
1130
wait_func_handle_exec_timeout(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1131 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1132 struct mlx5_cmd_work_ent *ent)
1133 {
1134 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1135
1136 mlx5_cmd_eq_recover(dev);
1137
1138 /* Re-wait on the ent->done after executing the recovery flow. If the
1139 * recovery flow (or any other recovery flow running simultaneously)
1140 * has recovered an EQE, it should cause the entry to be completed by
1141 * the command interface.
1142 */
1143 if (wait_for_completion_timeout(&ent->done, timeout)) {
1144 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1145 mlx5_command_str(ent->op), ent->op);
1146 return;
1147 }
1148
1149 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1150 mlx5_command_str(ent->op), ent->op);
1151
1152 ent->ret = -ETIMEDOUT;
1153 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1154 }
1155
wait_func(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1156 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1157 {
1158 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1159 struct mlx5_cmd *cmd = &dev->cmd;
1160 int err;
1161
1162 if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1163 cancel_work_sync(&ent->work)) {
1164 ent->ret = -ECANCELED;
1165 goto out_err;
1166 }
1167
1168 wait_for_completion(&ent->slotted);
1169
1170 if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1171 wait_for_completion(&ent->done);
1172 else if (!wait_for_completion_timeout(&ent->done, timeout))
1173 wait_func_handle_exec_timeout(dev, ent);
1174
1175 out_err:
1176 err = ent->ret;
1177
1178 if (err == -ETIMEDOUT) {
1179 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1180 mlx5_command_str(ent->op), ent->op);
1181 } else if (err == -ECANCELED) {
1182 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1183 mlx5_command_str(ent->op), ent->op);
1184 } else if (err == -EBUSY) {
1185 mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",
1186 mlx5_command_str(ent->op), ent->op);
1187 }
1188 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1189 err, deliv_status_to_str(ent->status), ent->status);
1190
1191 return err;
1192 }
1193
1194 /* Notes:
1195 * 1. Callback functions may not sleep
1196 * 2. page queue commands do not support asynchrous completion
1197 *
1198 * return value in case (!callback):
1199 * ret < 0 : Command execution couldn't be submitted by driver
1200 * ret > 0 : Command execution couldn't be performed by firmware
1201 * ret == 0: Command was executed by FW, Caller must check FW outbox status.
1202 *
1203 * return value in case (callback):
1204 * ret < 0 : Command execution couldn't be submitted by driver
1205 * ret == 0: Command will be submitted to FW for execution
1206 * and the callback will be called for further status updates
1207 */
mlx5_cmd_invoke(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t callback,void * context,int page_queue,u8 token,bool force_polling)1208 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1209 struct mlx5_cmd_msg *out, void *uout, int uout_size,
1210 mlx5_cmd_cbk_t callback,
1211 void *context, int page_queue,
1212 u8 token, bool force_polling)
1213 {
1214 struct mlx5_cmd *cmd = &dev->cmd;
1215 struct mlx5_cmd_work_ent *ent;
1216 struct mlx5_cmd_stats *stats;
1217 u8 status = 0;
1218 int err = 0;
1219 s64 ds;
1220
1221 if (callback && page_queue)
1222 return -EINVAL;
1223
1224 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1225 callback, context, page_queue);
1226 if (IS_ERR(ent))
1227 return PTR_ERR(ent);
1228
1229 /* put for this ent is when consumed, depending on the use case
1230 * 1) (!callback) blocking flow: by caller after wait_func completes
1231 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1232 */
1233
1234 ent->token = token;
1235 ent->polling = force_polling;
1236
1237 init_completion(&ent->handling);
1238 init_completion(&ent->slotted);
1239 if (!callback)
1240 init_completion(&ent->done);
1241
1242 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1243 INIT_WORK(&ent->work, cmd_work_handler);
1244 if (page_queue) {
1245 cmd_work_handler(&ent->work);
1246 } else if (!queue_work(cmd->wq, &ent->work)) {
1247 mlx5_core_warn(dev, "failed to queue work\n");
1248 err = -EALREADY;
1249 goto out_free;
1250 }
1251
1252 if (callback)
1253 return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1254
1255 err = wait_func(dev, ent);
1256 if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)
1257 goto out_free;
1258
1259 ds = ent->ts2 - ent->ts1;
1260 stats = xa_load(&cmd->stats, ent->op);
1261 if (stats) {
1262 spin_lock_irq(&stats->lock);
1263 stats->sum += ds;
1264 ++stats->n;
1265 spin_unlock_irq(&stats->lock);
1266 }
1267 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1268 "fw exec time for %s is %lld nsec\n",
1269 mlx5_command_str(ent->op), ds);
1270
1271 out_free:
1272 status = ent->status;
1273 cmd_ent_put(ent);
1274 return err ? : status;
1275 }
1276
dbg_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1277 static ssize_t dbg_write(struct file *filp, const char __user *buf,
1278 size_t count, loff_t *pos)
1279 {
1280 struct mlx5_core_dev *dev = filp->private_data;
1281 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1282 char lbuf[3];
1283 int err;
1284
1285 if (!dbg->in_msg || !dbg->out_msg)
1286 return -ENOMEM;
1287
1288 if (count < sizeof(lbuf) - 1)
1289 return -EINVAL;
1290
1291 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))
1292 return -EFAULT;
1293
1294 lbuf[sizeof(lbuf) - 1] = 0;
1295
1296 if (strcmp(lbuf, "go"))
1297 return -EINVAL;
1298
1299 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
1300
1301 return err ? err : count;
1302 }
1303
1304 static const struct file_operations fops = {
1305 .owner = THIS_MODULE,
1306 .open = simple_open,
1307 .write = dbg_write,
1308 };
1309
mlx5_copy_to_msg(struct mlx5_cmd_msg * to,void * from,int size,u8 token)1310 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1311 u8 token)
1312 {
1313 struct mlx5_cmd_prot_block *block;
1314 struct mlx5_cmd_mailbox *next;
1315 int copy;
1316
1317 if (!to || !from)
1318 return -ENOMEM;
1319
1320 copy = min_t(int, size, sizeof(to->first.data));
1321 memcpy(to->first.data, from, copy);
1322 size -= copy;
1323 from += copy;
1324
1325 next = to->next;
1326 while (size) {
1327 if (!next) {
1328 /* this is a BUG */
1329 return -ENOMEM;
1330 }
1331
1332 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1333 block = next->buf;
1334 memcpy(block->data, from, copy);
1335 from += copy;
1336 size -= copy;
1337 block->token = token;
1338 next = next->next;
1339 }
1340
1341 return 0;
1342 }
1343
mlx5_copy_from_msg(void * to,struct mlx5_cmd_msg * from,int size)1344 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1345 {
1346 struct mlx5_cmd_prot_block *block;
1347 struct mlx5_cmd_mailbox *next;
1348 int copy;
1349
1350 if (!to || !from)
1351 return -ENOMEM;
1352
1353 copy = min_t(int, size, sizeof(from->first.data));
1354 memcpy(to, from->first.data, copy);
1355 size -= copy;
1356 to += copy;
1357
1358 next = from->next;
1359 while (size) {
1360 if (!next) {
1361 /* this is a BUG */
1362 return -ENOMEM;
1363 }
1364
1365 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1366 block = next->buf;
1367
1368 memcpy(to, block->data, copy);
1369 to += copy;
1370 size -= copy;
1371 next = next->next;
1372 }
1373
1374 return 0;
1375 }
1376
alloc_cmd_box(struct mlx5_core_dev * dev,gfp_t flags)1377 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1378 gfp_t flags)
1379 {
1380 struct mlx5_cmd_mailbox *mailbox;
1381
1382 mailbox = kmalloc(sizeof(*mailbox), flags);
1383 if (!mailbox)
1384 return ERR_PTR(-ENOMEM);
1385
1386 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
1387 &mailbox->dma);
1388 if (!mailbox->buf) {
1389 mlx5_core_dbg(dev, "failed allocation\n");
1390 kfree(mailbox);
1391 return ERR_PTR(-ENOMEM);
1392 }
1393 mailbox->next = NULL;
1394
1395 return mailbox;
1396 }
1397
free_cmd_box(struct mlx5_core_dev * dev,struct mlx5_cmd_mailbox * mailbox)1398 static void free_cmd_box(struct mlx5_core_dev *dev,
1399 struct mlx5_cmd_mailbox *mailbox)
1400 {
1401 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
1402 kfree(mailbox);
1403 }
1404
mlx5_alloc_cmd_msg(struct mlx5_core_dev * dev,gfp_t flags,int size,u8 token)1405 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1406 gfp_t flags, int size,
1407 u8 token)
1408 {
1409 struct mlx5_cmd_mailbox *tmp, *head = NULL;
1410 struct mlx5_cmd_prot_block *block;
1411 struct mlx5_cmd_msg *msg;
1412 int err;
1413 int n;
1414 int i;
1415
1416 msg = kzalloc(sizeof(*msg), flags);
1417 if (!msg)
1418 return ERR_PTR(-ENOMEM);
1419
1420 msg->len = size;
1421 n = mlx5_calc_cmd_blocks(msg);
1422
1423 for (i = 0; i < n; i++) {
1424 tmp = alloc_cmd_box(dev, flags);
1425 if (IS_ERR(tmp)) {
1426 mlx5_core_warn(dev, "failed allocating block\n");
1427 err = PTR_ERR(tmp);
1428 goto err_alloc;
1429 }
1430
1431 block = tmp->buf;
1432 tmp->next = head;
1433 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1434 block->block_num = cpu_to_be32(n - i - 1);
1435 block->token = token;
1436 head = tmp;
1437 }
1438 msg->next = head;
1439 return msg;
1440
1441 err_alloc:
1442 while (head) {
1443 tmp = head->next;
1444 free_cmd_box(dev, head);
1445 head = tmp;
1446 }
1447 kfree(msg);
1448
1449 return ERR_PTR(err);
1450 }
1451
mlx5_free_cmd_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1452 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1453 struct mlx5_cmd_msg *msg)
1454 {
1455 struct mlx5_cmd_mailbox *head = msg->next;
1456 struct mlx5_cmd_mailbox *next;
1457
1458 while (head) {
1459 next = head->next;
1460 free_cmd_box(dev, head);
1461 head = next;
1462 }
1463 kfree(msg);
1464 }
1465
data_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1466 static ssize_t data_write(struct file *filp, const char __user *buf,
1467 size_t count, loff_t *pos)
1468 {
1469 struct mlx5_core_dev *dev = filp->private_data;
1470 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1471 void *ptr;
1472
1473 if (*pos != 0)
1474 return -EINVAL;
1475
1476 kfree(dbg->in_msg);
1477 dbg->in_msg = NULL;
1478 dbg->inlen = 0;
1479 ptr = memdup_user(buf, count);
1480 if (IS_ERR(ptr))
1481 return PTR_ERR(ptr);
1482 dbg->in_msg = ptr;
1483 dbg->inlen = count;
1484
1485 *pos = count;
1486
1487 return count;
1488 }
1489
data_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1490 static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1491 loff_t *pos)
1492 {
1493 struct mlx5_core_dev *dev = filp->private_data;
1494 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1495
1496 if (!dbg->out_msg)
1497 return -ENOMEM;
1498
1499 return simple_read_from_buffer(buf, count, pos, dbg->out_msg,
1500 dbg->outlen);
1501 }
1502
1503 static const struct file_operations dfops = {
1504 .owner = THIS_MODULE,
1505 .open = simple_open,
1506 .write = data_write,
1507 .read = data_read,
1508 };
1509
outlen_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1510 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1511 loff_t *pos)
1512 {
1513 struct mlx5_core_dev *dev = filp->private_data;
1514 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1515 char outlen[8];
1516 int err;
1517
1518 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
1519 if (err < 0)
1520 return err;
1521
1522 return simple_read_from_buffer(buf, count, pos, outlen, err);
1523 }
1524
outlen_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1525 static ssize_t outlen_write(struct file *filp, const char __user *buf,
1526 size_t count, loff_t *pos)
1527 {
1528 struct mlx5_core_dev *dev = filp->private_data;
1529 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1530 char outlen_str[8] = {0};
1531 int outlen;
1532 void *ptr;
1533 int err;
1534
1535 if (*pos != 0 || count > 6)
1536 return -EINVAL;
1537
1538 kfree(dbg->out_msg);
1539 dbg->out_msg = NULL;
1540 dbg->outlen = 0;
1541
1542 if (copy_from_user(outlen_str, buf, count))
1543 return -EFAULT;
1544
1545 err = sscanf(outlen_str, "%d", &outlen);
1546 if (err != 1)
1547 return -EINVAL;
1548
1549 ptr = kzalloc(outlen, GFP_KERNEL);
1550 if (!ptr)
1551 return -ENOMEM;
1552
1553 dbg->out_msg = ptr;
1554 dbg->outlen = outlen;
1555
1556 *pos = count;
1557
1558 return count;
1559 }
1560
1561 static const struct file_operations olfops = {
1562 .owner = THIS_MODULE,
1563 .open = simple_open,
1564 .write = outlen_write,
1565 .read = outlen_read,
1566 };
1567
set_wqname(struct mlx5_core_dev * dev)1568 static void set_wqname(struct mlx5_core_dev *dev)
1569 {
1570 struct mlx5_cmd *cmd = &dev->cmd;
1571
1572 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1573 dev_name(dev->device));
1574 }
1575
clean_debug_files(struct mlx5_core_dev * dev)1576 static void clean_debug_files(struct mlx5_core_dev *dev)
1577 {
1578 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1579
1580 if (!mlx5_debugfs_root)
1581 return;
1582
1583 debugfs_remove_recursive(dbg->dbg_root);
1584 }
1585
create_debugfs_files(struct mlx5_core_dev * dev)1586 static void create_debugfs_files(struct mlx5_core_dev *dev)
1587 {
1588 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1589
1590 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));
1591
1592 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1593 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1594 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1595 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);
1596 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1597 }
1598
mlx5_cmd_allowed_opcode(struct mlx5_core_dev * dev,u16 opcode)1599 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1600 {
1601 struct mlx5_cmd *cmd = &dev->cmd;
1602 int i;
1603
1604 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1605 down(&cmd->vars.sem);
1606 down(&cmd->vars.pages_sem);
1607
1608 cmd->allowed_opcode = opcode;
1609
1610 up(&cmd->vars.pages_sem);
1611 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1612 up(&cmd->vars.sem);
1613 }
1614
mlx5_cmd_change_mod(struct mlx5_core_dev * dev,int mode)1615 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1616 {
1617 struct mlx5_cmd *cmd = &dev->cmd;
1618 int i;
1619
1620 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1621 down(&cmd->vars.sem);
1622 down(&cmd->vars.pages_sem);
1623
1624 cmd->mode = mode;
1625
1626 up(&cmd->vars.pages_sem);
1627 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1628 up(&cmd->vars.sem);
1629 }
1630
cmd_comp_notifier(struct notifier_block * nb,unsigned long type,void * data)1631 static int cmd_comp_notifier(struct notifier_block *nb,
1632 unsigned long type, void *data)
1633 {
1634 struct mlx5_core_dev *dev;
1635 struct mlx5_cmd *cmd;
1636 struct mlx5_eqe *eqe;
1637
1638 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1639 dev = container_of(cmd, struct mlx5_core_dev, cmd);
1640 eqe = data;
1641
1642 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1643 return NOTIFY_DONE;
1644
1645 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);
1646
1647 return NOTIFY_OK;
1648 }
mlx5_cmd_use_events(struct mlx5_core_dev * dev)1649 void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1650 {
1651 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1652 mlx5_eq_notifier_register(dev, &dev->cmd.nb);
1653 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);
1654 }
1655
mlx5_cmd_use_polling(struct mlx5_core_dev * dev)1656 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1657 {
1658 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);
1659 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);
1660 }
1661
free_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1662 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1663 {
1664 unsigned long flags;
1665
1666 if (msg->parent) {
1667 spin_lock_irqsave(&msg->parent->lock, flags);
1668 list_add_tail(&msg->list, &msg->parent->head);
1669 spin_unlock_irqrestore(&msg->parent->lock, flags);
1670 } else {
1671 mlx5_free_cmd_msg(dev, msg);
1672 }
1673 }
1674
mlx5_cmd_comp_handler(struct mlx5_core_dev * dev,u64 vec,bool forced)1675 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1676 {
1677 struct mlx5_cmd *cmd = &dev->cmd;
1678 struct mlx5_cmd_work_ent *ent;
1679 mlx5_cmd_cbk_t callback;
1680 void *context;
1681 int err;
1682 int i;
1683 s64 ds;
1684 struct mlx5_cmd_stats *stats;
1685 unsigned long flags;
1686 unsigned long vector;
1687
1688 /* there can be at most 32 command queues */
1689 vector = vec & 0xffffffff;
1690 for (i = 0; i < (1 << cmd->vars.log_sz); i++) {
1691 if (test_bit(i, &vector)) {
1692 ent = cmd->ent_arr[i];
1693
1694 /* if we already completed the command, ignore it */
1695 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,
1696 &ent->state)) {
1697 /* only real completion can free the cmd slot */
1698 if (!forced) {
1699 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1700 ent->idx);
1701 cmd_ent_put(ent);
1702 }
1703 continue;
1704 }
1705
1706 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1707 cmd_ent_put(ent); /* timeout work was canceled */
1708
1709 if (!forced || /* Real FW completion */
1710 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1711 !opcode_allowed(cmd, ent->op))
1712 cmd_ent_put(ent);
1713
1714 ent->ts2 = ktime_get_ns();
1715 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1716 dump_command(dev, ent, 0);
1717
1718 if (vec & MLX5_TRIGGERED_CMD_COMP)
1719 ent->ret = -ENXIO;
1720
1721 if (!ent->ret) { /* Command completed by FW */
1722 if (!cmd->checksum_disabled)
1723 ent->ret = verify_signature(ent);
1724
1725 ent->status = ent->lay->status_own >> 1;
1726
1727 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1728 ent->ret, deliv_status_to_str(ent->status), ent->status);
1729 }
1730
1731 if (ent->callback) {
1732 ds = ent->ts2 - ent->ts1;
1733 stats = xa_load(&cmd->stats, ent->op);
1734 if (stats) {
1735 spin_lock_irqsave(&stats->lock, flags);
1736 stats->sum += ds;
1737 ++stats->n;
1738 spin_unlock_irqrestore(&stats->lock, flags);
1739 }
1740
1741 callback = ent->callback;
1742 context = ent->context;
1743 err = ent->ret ? : ent->status;
1744 if (err > 0) /* Failed in FW, command didn't execute */
1745 err = deliv_status_to_err(err);
1746
1747 if (!err)
1748 err = mlx5_copy_from_msg(ent->uout,
1749 ent->out,
1750 ent->uout_size);
1751
1752 mlx5_free_cmd_msg(dev, ent->out);
1753 free_msg(dev, ent->in);
1754
1755 /* final consumer is done, release ent */
1756 cmd_ent_put(ent);
1757 callback(err, context);
1758 } else {
1759 /* release wait_func() so mlx5_cmd_invoke()
1760 * can make the final ent_put()
1761 */
1762 complete(&ent->done);
1763 }
1764 }
1765 }
1766 }
1767
mlx5_cmd_trigger_completions(struct mlx5_core_dev * dev)1768 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1769 {
1770 struct mlx5_cmd *cmd = &dev->cmd;
1771 unsigned long bitmask;
1772 unsigned long flags;
1773 u64 vector;
1774 int i;
1775
1776 /* wait for pending handlers to complete */
1777 mlx5_eq_synchronize_cmd_irq(dev);
1778 spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1779 vector = ~dev->cmd.vars.bitmask & ((1ul << (1 << dev->cmd.vars.log_sz)) - 1);
1780 if (!vector)
1781 goto no_trig;
1782
1783 bitmask = vector;
1784 /* we must increment the allocated entries refcount before triggering the completions
1785 * to guarantee pending commands will not get freed in the meanwhile.
1786 * For that reason, it also has to be done inside the alloc_lock.
1787 */
1788 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1789 cmd_ent_get(cmd->ent_arr[i]);
1790 vector |= MLX5_TRIGGERED_CMD_COMP;
1791 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1792
1793 mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1794 mlx5_cmd_comp_handler(dev, vector, true);
1795 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1796 cmd_ent_put(cmd->ent_arr[i]);
1797 return;
1798
1799 no_trig:
1800 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1801 }
1802
mlx5_cmd_flush(struct mlx5_core_dev * dev)1803 void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1804 {
1805 struct mlx5_cmd *cmd = &dev->cmd;
1806 int i;
1807
1808 for (i = 0; i < cmd->vars.max_reg_cmds; i++) {
1809 while (down_trylock(&cmd->vars.sem)) {
1810 mlx5_cmd_trigger_completions(dev);
1811 cond_resched();
1812 }
1813 }
1814
1815 while (down_trylock(&cmd->vars.pages_sem)) {
1816 mlx5_cmd_trigger_completions(dev);
1817 cond_resched();
1818 }
1819
1820 /* Unlock cmdif */
1821 up(&cmd->vars.pages_sem);
1822 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1823 up(&cmd->vars.sem);
1824 }
1825
alloc_msg(struct mlx5_core_dev * dev,int in_size,gfp_t gfp)1826 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1827 gfp_t gfp)
1828 {
1829 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1830 struct cmd_msg_cache *ch = NULL;
1831 struct mlx5_cmd *cmd = &dev->cmd;
1832 int i;
1833
1834 if (in_size <= 16)
1835 goto cache_miss;
1836
1837 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
1838 ch = &cmd->cache[i];
1839 if (in_size > ch->max_inbox_size)
1840 continue;
1841 spin_lock_irq(&ch->lock);
1842 if (list_empty(&ch->head)) {
1843 spin_unlock_irq(&ch->lock);
1844 continue;
1845 }
1846 msg = list_entry(ch->head.next, typeof(*msg), list);
1847 /* For cached lists, we must explicitly state what is
1848 * the real size
1849 */
1850 msg->len = in_size;
1851 list_del(&msg->list);
1852 spin_unlock_irq(&ch->lock);
1853 break;
1854 }
1855
1856 if (!IS_ERR(msg))
1857 return msg;
1858
1859 cache_miss:
1860 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
1861 return msg;
1862 }
1863
is_manage_pages(void * in)1864 static int is_manage_pages(void *in)
1865 {
1866 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;
1867 }
1868
1869 /* Notes:
1870 * 1. Callback functions may not sleep
1871 * 2. Page queue commands do not support asynchrous completion
1872 */
cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size,mlx5_cmd_cbk_t callback,void * context,bool force_polling)1873 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1874 int out_size, mlx5_cmd_cbk_t callback, void *context,
1875 bool force_polling)
1876 {
1877 struct mlx5_cmd_msg *inb, *outb;
1878 u16 opcode = in_to_opcode(in);
1879 bool throttle_op;
1880 int pages_queue;
1881 gfp_t gfp;
1882 u8 token;
1883 int err;
1884
1885 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))
1886 return -ENXIO;
1887
1888 throttle_op = mlx5_cmd_is_throttle_opcode(opcode);
1889 if (throttle_op) {
1890 /* atomic context may not sleep */
1891 if (callback)
1892 return -EINVAL;
1893 down(&dev->cmd.vars.throttle_sem);
1894 }
1895
1896 pages_queue = is_manage_pages(in);
1897 gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1898
1899 inb = alloc_msg(dev, in_size, gfp);
1900 if (IS_ERR(inb)) {
1901 err = PTR_ERR(inb);
1902 goto out_up;
1903 }
1904
1905 token = alloc_token(&dev->cmd);
1906
1907 err = mlx5_copy_to_msg(inb, in, in_size, token);
1908 if (err) {
1909 mlx5_core_warn(dev, "err %d\n", err);
1910 goto out_in;
1911 }
1912
1913 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
1914 if (IS_ERR(outb)) {
1915 err = PTR_ERR(outb);
1916 goto out_in;
1917 }
1918
1919 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
1920 pages_queue, token, force_polling);
1921 if (callback)
1922 return err;
1923
1924 if (err > 0) /* Failed in FW, command didn't execute */
1925 err = deliv_status_to_err(err);
1926
1927 if (err)
1928 goto out_out;
1929
1930 /* command completed by FW */
1931 err = mlx5_copy_from_msg(out, outb, out_size);
1932 out_out:
1933 mlx5_free_cmd_msg(dev, outb);
1934 out_in:
1935 free_msg(dev, inb);
1936 out_up:
1937 if (throttle_op)
1938 up(&dev->cmd.vars.throttle_sem);
1939 return err;
1940 }
1941
mlx5_cmd_err_trace(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)1942 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
1943 {
1944 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
1945 u8 status = MLX5_GET(mbox_out, out, status);
1946
1947 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,
1948 cmd_status_str(status), status, syndrome,
1949 cmd_status_to_err(status));
1950 }
1951
cmd_status_log(struct mlx5_core_dev * dev,u16 opcode,u8 status,u32 syndrome,int err)1952 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
1953 u32 syndrome, int err)
1954 {
1955 const char *namep = mlx5_command_str(opcode);
1956 struct mlx5_cmd_stats *stats;
1957 unsigned long flags;
1958
1959 if (!err || !(strcmp(namep, "unknown command opcode")))
1960 return;
1961
1962 stats = xa_load(&dev->cmd.stats, opcode);
1963 if (!stats)
1964 return;
1965 spin_lock_irqsave(&stats->lock, flags);
1966 stats->failed++;
1967 if (err < 0)
1968 stats->last_failed_errno = -err;
1969 if (err == -EREMOTEIO) {
1970 stats->failed_mbox_status++;
1971 stats->last_failed_mbox_status = status;
1972 stats->last_failed_syndrome = syndrome;
1973 }
1974 spin_unlock_irqrestore(&stats->lock, flags);
1975 }
1976
1977 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
cmd_status_err(struct mlx5_core_dev * dev,int err,u16 opcode,u16 op_mod,void * out)1978 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
1979 {
1980 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
1981 u8 status = MLX5_GET(mbox_out, out, status);
1982
1983 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
1984 err = -EIO;
1985
1986 if (!err && status != MLX5_CMD_STAT_OK) {
1987 err = -EREMOTEIO;
1988 mlx5_cmd_err_trace(dev, opcode, op_mod, out);
1989 }
1990
1991 cmd_status_log(dev, opcode, status, syndrome, err);
1992 return err;
1993 }
1994
1995 /**
1996 * mlx5_cmd_do - Executes a fw command, wait for completion.
1997 * Unlike mlx5_cmd_exec, this function will not translate or intercept
1998 * outbox.status and will return -EREMOTEIO when
1999 * outbox.status != MLX5_CMD_STAT_OK
2000 *
2001 * @dev: mlx5 core device
2002 * @in: inbox mlx5_ifc command buffer
2003 * @in_size: inbox buffer size
2004 * @out: outbox mlx5_ifc buffer
2005 * @out_size: outbox size
2006 *
2007 * @return:
2008 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
2009 * Caller must check FW outbox status.
2010 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
2011 * < 0 : Command execution couldn't be performed by firmware or driver
2012 */
mlx5_cmd_do(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2013 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
2014 {
2015 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);
2016 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2017 u16 opcode = in_to_opcode(in);
2018
2019 return cmd_status_err(dev, err, opcode, op_mod, out);
2020 }
2021 EXPORT_SYMBOL(mlx5_cmd_do);
2022
2023 /**
2024 * mlx5_cmd_exec - Executes a fw command, wait for completion
2025 *
2026 * @dev: mlx5 core device
2027 * @in: inbox mlx5_ifc command buffer
2028 * @in_size: inbox buffer size
2029 * @out: outbox mlx5_ifc buffer
2030 * @out_size: outbox size
2031 *
2032 * @return: 0 if no error, FW command execution was successful
2033 * and outbox status is ok.
2034 */
mlx5_cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2035 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
2036 int out_size)
2037 {
2038 int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
2039
2040 return mlx5_cmd_check(dev, err, in, out);
2041 }
2042 EXPORT_SYMBOL(mlx5_cmd_exec);
2043
2044 /**
2045 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion
2046 * Needed for driver force teardown, when command completion EQ
2047 * will not be available to complete the command
2048 *
2049 * @dev: mlx5 core device
2050 * @in: inbox mlx5_ifc command buffer
2051 * @in_size: inbox buffer size
2052 * @out: outbox mlx5_ifc buffer
2053 * @out_size: outbox size
2054 *
2055 * @return: 0 if no error, FW command execution was successful
2056 * and outbox status is ok.
2057 */
mlx5_cmd_exec_polling(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2058 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
2059 void *out, int out_size)
2060 {
2061 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);
2062 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2063 u16 opcode = in_to_opcode(in);
2064
2065 err = cmd_status_err(dev, err, opcode, op_mod, out);
2066 return mlx5_cmd_check(dev, err, in, out);
2067 }
2068 EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2069
mlx5_cmd_init_async_ctx(struct mlx5_core_dev * dev,struct mlx5_async_ctx * ctx)2070 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2071 struct mlx5_async_ctx *ctx)
2072 {
2073 ctx->dev = dev;
2074 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2075 atomic_set(&ctx->num_inflight, 1);
2076 init_completion(&ctx->inflight_done);
2077 }
2078 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2079
2080 /**
2081 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2082 * @ctx: The ctx to clean
2083 *
2084 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2085 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2086 * the call mlx5_cleanup_async_ctx().
2087 */
mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx * ctx)2088 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2089 {
2090 if (!atomic_dec_and_test(&ctx->num_inflight))
2091 wait_for_completion(&ctx->inflight_done);
2092 }
2093 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2094
mlx5_cmd_exec_cb_handler(int status,void * _work)2095 static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2096 {
2097 struct mlx5_async_work *work = _work;
2098 struct mlx5_async_ctx *ctx;
2099
2100 ctx = work->ctx;
2101 status = cmd_status_err(ctx->dev, status, work->opcode, work->op_mod, work->out);
2102 work->user_callback(status, work);
2103 if (atomic_dec_and_test(&ctx->num_inflight))
2104 complete(&ctx->inflight_done);
2105 }
2106
mlx5_cmd_exec_cb(struct mlx5_async_ctx * ctx,void * in,int in_size,void * out,int out_size,mlx5_async_cbk_t callback,struct mlx5_async_work * work)2107 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2108 void *out, int out_size, mlx5_async_cbk_t callback,
2109 struct mlx5_async_work *work)
2110 {
2111 int ret;
2112
2113 work->ctx = ctx;
2114 work->user_callback = callback;
2115 work->opcode = in_to_opcode(in);
2116 work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2117 work->out = out;
2118 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2119 return -EIO;
2120 ret = cmd_exec(ctx->dev, in, in_size, out, out_size,
2121 mlx5_cmd_exec_cb_handler, work, false);
2122 if (ret && atomic_dec_and_test(&ctx->num_inflight))
2123 complete(&ctx->inflight_done);
2124
2125 return ret;
2126 }
2127 EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2128
mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev * dev,struct mlx5_cmd_allow_other_vhca_access_attr * attr)2129 int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,
2130 struct mlx5_cmd_allow_other_vhca_access_attr *attr)
2131 {
2132 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};
2133 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};
2134 void *key;
2135
2136 MLX5_SET(allow_other_vhca_access_in,
2137 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);
2138 MLX5_SET(allow_other_vhca_access_in,
2139 in, object_type_to_be_accessed, attr->obj_type);
2140 MLX5_SET(allow_other_vhca_access_in,
2141 in, object_id_to_be_accessed, attr->obj_id);
2142
2143 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);
2144 memcpy(key, attr->access_key, sizeof(attr->access_key));
2145
2146 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2147 }
2148
mlx5_cmd_alias_obj_create(struct mlx5_core_dev * dev,struct mlx5_cmd_alias_obj_create_attr * alias_attr,u32 * obj_id)2149 int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
2150 struct mlx5_cmd_alias_obj_create_attr *alias_attr,
2151 u32 *obj_id)
2152 {
2153 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2154 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};
2155 void *param;
2156 void *attr;
2157 void *key;
2158 int ret;
2159
2160 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);
2161 MLX5_SET(general_obj_in_cmd_hdr,
2162 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2163 MLX5_SET(general_obj_in_cmd_hdr,
2164 attr, obj_type, alias_attr->obj_type);
2165 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
2166 MLX5_SET(general_obj_create_param, param, alias_object, 1);
2167
2168 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
2169 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
2170 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
2171
2172 key = MLX5_ADDR_OF(alias_context, attr, access_key);
2173 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));
2174
2175 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2176 if (ret)
2177 return ret;
2178
2179 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2180
2181 return 0;
2182 }
2183
mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev * dev,u32 obj_id,u16 obj_type)2184 int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,
2185 u16 obj_type)
2186 {
2187 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2188 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
2189
2190 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
2191 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);
2192 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
2193
2194 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2195 }
2196
destroy_msg_cache(struct mlx5_core_dev * dev)2197 static void destroy_msg_cache(struct mlx5_core_dev *dev)
2198 {
2199 struct cmd_msg_cache *ch;
2200 struct mlx5_cmd_msg *msg;
2201 struct mlx5_cmd_msg *n;
2202 int i;
2203
2204 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
2205 ch = &dev->cmd.cache[i];
2206 list_for_each_entry_safe(msg, n, &ch->head, list) {
2207 list_del(&msg->list);
2208 mlx5_free_cmd_msg(dev, msg);
2209 }
2210 }
2211 }
2212
2213 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2214 512, 32, 16, 8, 2
2215 };
2216
2217 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2218 16 + MLX5_CMD_DATA_BLOCK_SIZE,
2219 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2220 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2221 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2222 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2223 };
2224
create_msg_cache(struct mlx5_core_dev * dev)2225 static void create_msg_cache(struct mlx5_core_dev *dev)
2226 {
2227 struct mlx5_cmd *cmd = &dev->cmd;
2228 struct cmd_msg_cache *ch;
2229 struct mlx5_cmd_msg *msg;
2230 int i;
2231 int k;
2232
2233 /* Initialize and fill the caches with initial entries */
2234 for (k = 0; k < dev->profile.num_cmd_caches; k++) {
2235 ch = &cmd->cache[k];
2236 spin_lock_init(&ch->lock);
2237 INIT_LIST_HEAD(&ch->head);
2238 ch->num_ent = cmd_cache_num_ent[k];
2239 ch->max_inbox_size = cmd_cache_ent_size[k];
2240 for (i = 0; i < ch->num_ent; i++) {
2241 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2242 ch->max_inbox_size, 0);
2243 if (IS_ERR(msg))
2244 break;
2245 msg->parent = ch;
2246 list_add_tail(&msg->list, &ch->head);
2247 }
2248 }
2249 }
2250
alloc_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2251 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2252 {
2253 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,
2254 &cmd->alloc_dma, GFP_KERNEL);
2255 if (!cmd->cmd_alloc_buf)
2256 return -ENOMEM;
2257
2258 /* make sure it is aligned to 4K */
2259 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2260 cmd->cmd_buf = cmd->cmd_alloc_buf;
2261 cmd->dma = cmd->alloc_dma;
2262 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2263 return 0;
2264 }
2265
2266 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,
2267 cmd->alloc_dma);
2268 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),
2269 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2270 &cmd->alloc_dma, GFP_KERNEL);
2271 if (!cmd->cmd_alloc_buf)
2272 return -ENOMEM;
2273
2274 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2275 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2276 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2277 return 0;
2278 }
2279
free_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2280 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2281 {
2282 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,
2283 cmd->alloc_dma);
2284 }
2285
cmdif_rev(struct mlx5_core_dev * dev)2286 static u16 cmdif_rev(struct mlx5_core_dev *dev)
2287 {
2288 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2289 }
2290
mlx5_cmd_init(struct mlx5_core_dev * dev)2291 int mlx5_cmd_init(struct mlx5_core_dev *dev)
2292 {
2293 struct mlx5_cmd *cmd = &dev->cmd;
2294
2295 cmd->checksum_disabled = 1;
2296
2297 spin_lock_init(&cmd->alloc_lock);
2298 spin_lock_init(&cmd->token_lock);
2299
2300 set_wqname(dev);
2301 cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2302 if (!cmd->wq) {
2303 mlx5_core_err(dev, "failed to create command workqueue\n");
2304 return -ENOMEM;
2305 }
2306
2307 mlx5_cmdif_debugfs_init(dev);
2308
2309 return 0;
2310 }
2311
mlx5_cmd_cleanup(struct mlx5_core_dev * dev)2312 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2313 {
2314 struct mlx5_cmd *cmd = &dev->cmd;
2315
2316 mlx5_cmdif_debugfs_cleanup(dev);
2317 destroy_workqueue(cmd->wq);
2318 }
2319
mlx5_cmd_enable(struct mlx5_core_dev * dev)2320 int mlx5_cmd_enable(struct mlx5_core_dev *dev)
2321 {
2322 int size = sizeof(struct mlx5_cmd_prot_block);
2323 int align = roundup_pow_of_two(size);
2324 struct mlx5_cmd *cmd = &dev->cmd;
2325 u32 cmd_h, cmd_l;
2326 int err;
2327
2328 memset(&cmd->vars, 0, sizeof(cmd->vars));
2329 cmd->vars.cmdif_rev = cmdif_rev(dev);
2330 if (cmd->vars.cmdif_rev != CMD_IF_REV) {
2331 mlx5_core_err(dev,
2332 "Driver cmdif rev(%d) differs from firmware's(%d)\n",
2333 CMD_IF_REV, cmd->vars.cmdif_rev);
2334 return -EINVAL;
2335 }
2336
2337 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2338 cmd->vars.log_sz = cmd_l >> 4 & 0xf;
2339 cmd->vars.log_stride = cmd_l & 0xf;
2340 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {
2341 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2342 1 << cmd->vars.log_sz);
2343 return -EINVAL;
2344 }
2345
2346 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2347 mlx5_core_err(dev, "command queue size overflow\n");
2348 return -EINVAL;
2349 }
2350
2351 cmd->state = MLX5_CMDIF_STATE_DOWN;
2352 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;
2353 cmd->vars.bitmask = (1UL << cmd->vars.max_reg_cmds) - 1;
2354
2355 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds);
2356 sema_init(&cmd->vars.pages_sem, 1);
2357 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2358
2359 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);
2360 if (!cmd->pool)
2361 return -ENOMEM;
2362
2363 err = alloc_cmd_page(dev, cmd);
2364 if (err)
2365 goto err_free_pool;
2366
2367 cmd_h = (u32)((u64)(cmd->dma) >> 32);
2368 cmd_l = (u32)(cmd->dma);
2369 if (cmd_l & 0xfff) {
2370 mlx5_core_err(dev, "invalid command queue address\n");
2371 err = -ENOMEM;
2372 goto err_cmd_page;
2373 }
2374
2375 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2376 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2377
2378 /* Make sure firmware sees the complete address before we proceed */
2379 wmb();
2380
2381 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2382
2383 cmd->mode = CMD_MODE_POLLING;
2384 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2385
2386 create_msg_cache(dev);
2387 create_debugfs_files(dev);
2388
2389 return 0;
2390
2391 err_cmd_page:
2392 free_cmd_page(dev, cmd);
2393 err_free_pool:
2394 dma_pool_destroy(cmd->pool);
2395 return err;
2396 }
2397
mlx5_cmd_disable(struct mlx5_core_dev * dev)2398 void mlx5_cmd_disable(struct mlx5_core_dev *dev)
2399 {
2400 struct mlx5_cmd *cmd = &dev->cmd;
2401
2402 flush_workqueue(cmd->wq);
2403 clean_debug_files(dev);
2404 destroy_msg_cache(dev);
2405 free_cmd_page(dev, cmd);
2406 dma_pool_destroy(cmd->pool);
2407 }
2408
mlx5_cmd_set_state(struct mlx5_core_dev * dev,enum mlx5_cmdif_state cmdif_state)2409 void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2410 enum mlx5_cmdif_state cmdif_state)
2411 {
2412 dev->cmd.state = cmdif_state;
2413 }
2414