1 /*
2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/errno.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/slab.h>
38 #include <linux/delay.h>
39 #include <linux/random.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/eq.h>
42 #include <linux/debugfs.h>
43
44 #include "mlx5_core.h"
45 #include "lib/eq.h"
46 #include "lib/tout.h"
47 #define CREATE_TRACE_POINTS
48 #include "diag/cmd_tracepoint.h"
49
50 struct mlx5_ifc_mbox_out_bits {
51 u8 status[0x8];
52 u8 reserved_at_8[0x18];
53
54 u8 syndrome[0x20];
55
56 u8 reserved_at_40[0x40];
57 };
58
59 struct mlx5_ifc_mbox_in_bits {
60 u8 opcode[0x10];
61 u8 uid[0x10];
62
63 u8 reserved_at_20[0x10];
64 u8 op_mod[0x10];
65
66 u8 reserved_at_40[0x40];
67 };
68
69 enum {
70 CMD_IF_REV = 5,
71 };
72
73 enum {
74 CMD_MODE_POLLING,
75 CMD_MODE_EVENTS
76 };
77
78 enum {
79 MLX5_CMD_DELIVERY_STAT_OK = 0x0,
80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
90 };
91
in_to_opcode(void * in)92 static u16 in_to_opcode(void *in)
93 {
94 return MLX5_GET(mbox_in, in, opcode);
95 }
96
in_to_uid(void * in)97 static u16 in_to_uid(void *in)
98 {
99 return MLX5_GET(mbox_in, in, uid);
100 }
101
102 /* Returns true for opcodes that might be triggered very frequently and throttle
103 * the command interface. Limit their command slots usage.
104 */
mlx5_cmd_is_throttle_opcode(u16 op)105 static bool mlx5_cmd_is_throttle_opcode(u16 op)
106 {
107 switch (op) {
108 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
109 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
110 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
111 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
112 case MLX5_CMD_OP_SYNC_CRYPTO:
113 return true;
114 }
115 return false;
116 }
117
118 static struct mlx5_cmd_work_ent *
cmd_alloc_ent(struct mlx5_cmd * cmd,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t cbk,void * context,int page_queue)119 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
120 struct mlx5_cmd_msg *out, void *uout, int uout_size,
121 mlx5_cmd_cbk_t cbk, void *context, int page_queue)
122 {
123 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
124 struct mlx5_cmd_work_ent *ent;
125
126 ent = kzalloc_obj(*ent, alloc_flags);
127 if (!ent)
128 return ERR_PTR(-ENOMEM);
129
130 ent->idx = -EINVAL;
131 ent->in = in;
132 ent->out = out;
133 ent->uout = uout;
134 ent->uout_size = uout_size;
135 ent->callback = cbk;
136 ent->context = context;
137 ent->cmd = cmd;
138 ent->page_queue = page_queue;
139 ent->op = in_to_opcode(in->first.data);
140 refcount_set(&ent->refcnt, 1);
141
142 return ent;
143 }
144
cmd_free_ent(struct mlx5_cmd_work_ent * ent)145 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
146 {
147 kfree(ent);
148 }
149
alloc_token(struct mlx5_cmd * cmd)150 static u8 alloc_token(struct mlx5_cmd *cmd)
151 {
152 u8 token;
153
154 spin_lock(&cmd->token_lock);
155 cmd->token++;
156 if (cmd->token == 0)
157 cmd->token++;
158 token = cmd->token;
159 spin_unlock(&cmd->token_lock);
160
161 return token;
162 }
163
cmd_alloc_index(struct mlx5_cmd * cmd,struct mlx5_cmd_work_ent * ent)164 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
165 {
166 unsigned long flags;
167 int ret;
168
169 spin_lock_irqsave(&cmd->alloc_lock, flags);
170 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds);
171 if (ret < cmd->vars.max_reg_cmds) {
172 clear_bit(ret, &cmd->vars.bitmask);
173 ent->idx = ret;
174 cmd->ent_arr[ent->idx] = ent;
175 }
176 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
177
178 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;
179 }
180
cmd_free_index(struct mlx5_cmd * cmd,int idx)181 static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
182 {
183 lockdep_assert_held(&cmd->alloc_lock);
184 cmd->ent_arr[idx] = NULL;
185 set_bit(idx, &cmd->vars.bitmask);
186 }
187
cmd_ent_get(struct mlx5_cmd_work_ent * ent)188 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
189 {
190 refcount_inc(&ent->refcnt);
191 }
192
cmd_ent_put(struct mlx5_cmd_work_ent * ent)193 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
194 {
195 struct mlx5_cmd *cmd = ent->cmd;
196 unsigned long flags;
197
198 spin_lock_irqsave(&cmd->alloc_lock, flags);
199 if (!refcount_dec_and_test(&ent->refcnt)) {
200 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
201 return;
202 }
203
204 if (ent->idx >= 0) {
205 cmd_free_index(cmd, ent->idx);
206 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
207 }
208 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
209
210 cmd_free_ent(ent);
211 }
212
get_inst(struct mlx5_cmd * cmd,int idx)213 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
214 {
215 return cmd->cmd_buf + (idx << cmd->vars.log_stride);
216 }
217
mlx5_calc_cmd_blocks(struct mlx5_cmd_msg * msg)218 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
219 {
220 int size = msg->len;
221 int blen = size - min_t(int, sizeof(msg->first.data), size);
222
223 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
224 }
225
xor8_buf(void * buf,size_t offset,int len)226 static u8 xor8_buf(void *buf, size_t offset, int len)
227 {
228 u8 *ptr = buf;
229 u8 sum = 0;
230 int i;
231 int end = len + offset;
232
233 for (i = offset; i < end; i++)
234 sum ^= ptr[i];
235
236 return sum;
237 }
238
verify_block_sig(struct mlx5_cmd_prot_block * block)239 static int verify_block_sig(struct mlx5_cmd_prot_block *block)
240 {
241 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
242 int xor_len = sizeof(*block) - sizeof(block->data) - 1;
243
244 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
245 return -EHWPOISON;
246
247 if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
248 return -EHWPOISON;
249
250 return 0;
251 }
252
calc_block_sig(struct mlx5_cmd_prot_block * block)253 static void calc_block_sig(struct mlx5_cmd_prot_block *block)
254 {
255 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
256 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
257
258 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
259 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
260 }
261
calc_chain_sig(struct mlx5_cmd_msg * msg)262 static void calc_chain_sig(struct mlx5_cmd_msg *msg)
263 {
264 struct mlx5_cmd_mailbox *next = msg->next;
265 int n = mlx5_calc_cmd_blocks(msg);
266 int i = 0;
267
268 for (i = 0; i < n && next; i++) {
269 calc_block_sig(next->buf);
270 next = next->next;
271 }
272 }
273
set_signature(struct mlx5_cmd_work_ent * ent,int csum)274 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
275 {
276 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay));
277 if (csum) {
278 calc_chain_sig(ent->in);
279 calc_chain_sig(ent->out);
280 }
281 }
282
poll_timeout(struct mlx5_cmd_work_ent * ent)283 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
284 {
285 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
286 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
287 unsigned long poll_end;
288 u8 own;
289
290 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);
291
292 do {
293 own = READ_ONCE(ent->lay->status_own);
294 if (!(own & CMD_OWNER_HW)) {
295 ent->ret = 0;
296 return;
297 }
298 cond_resched();
299 if (mlx5_cmd_is_down(dev)) {
300 ent->ret = -ENXIO;
301 return;
302 }
303 } while (time_before(jiffies, poll_end));
304
305 ent->ret = -ETIMEDOUT;
306 }
307
verify_signature(struct mlx5_cmd_work_ent * ent)308 static int verify_signature(struct mlx5_cmd_work_ent *ent)
309 {
310 struct mlx5_cmd_mailbox *next = ent->out->next;
311 int n = mlx5_calc_cmd_blocks(ent->out);
312 int err;
313 u8 sig;
314 int i = 0;
315
316 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
317 if (sig != 0xff)
318 return -EHWPOISON;
319
320 for (i = 0; i < n && next; i++) {
321 err = verify_block_sig(next->buf);
322 if (err)
323 return -EHWPOISON;
324
325 next = next->next;
326 }
327
328 return 0;
329 }
330
dump_buf(void * buf,int size,int data_only,int offset,int idx)331 static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
332 {
333 __be32 *p = buf;
334 int i;
335
336 for (i = 0; i < size; i += 16) {
337 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
338 be32_to_cpu(p[0]), be32_to_cpu(p[1]),
339 be32_to_cpu(p[2]), be32_to_cpu(p[3]));
340 p += 4;
341 offset += 16;
342 }
343 if (!data_only)
344 pr_debug("\n");
345 }
346
mlx5_internal_err_ret_value(struct mlx5_core_dev * dev,u16 op,u32 * synd,u8 * status)347 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
348 u32 *synd, u8 *status)
349 {
350 *synd = 0;
351 *status = 0;
352
353 switch (op) {
354 case MLX5_CMD_OP_TEARDOWN_HCA:
355 case MLX5_CMD_OP_DISABLE_HCA:
356 case MLX5_CMD_OP_MANAGE_PAGES:
357 case MLX5_CMD_OP_DESTROY_MKEY:
358 case MLX5_CMD_OP_DESTROY_EQ:
359 case MLX5_CMD_OP_DESTROY_CQ:
360 case MLX5_CMD_OP_DESTROY_QP:
361 case MLX5_CMD_OP_DESTROY_PSV:
362 case MLX5_CMD_OP_DESTROY_SRQ:
363 case MLX5_CMD_OP_DESTROY_XRC_SRQ:
364 case MLX5_CMD_OP_DESTROY_XRQ:
365 case MLX5_CMD_OP_DESTROY_DCT:
366 case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
367 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
368 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
369 case MLX5_CMD_OP_DEALLOC_PD:
370 case MLX5_CMD_OP_DEALLOC_UAR:
371 case MLX5_CMD_OP_DETACH_FROM_MCG:
372 case MLX5_CMD_OP_DEALLOC_XRCD:
373 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
374 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
375 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
376 case MLX5_CMD_OP_DESTROY_LAG:
377 case MLX5_CMD_OP_DESTROY_VPORT_LAG:
378 case MLX5_CMD_OP_DESTROY_TIR:
379 case MLX5_CMD_OP_DESTROY_SQ:
380 case MLX5_CMD_OP_DESTROY_RQ:
381 case MLX5_CMD_OP_DESTROY_RMP:
382 case MLX5_CMD_OP_DESTROY_TIS:
383 case MLX5_CMD_OP_DESTROY_RQT:
384 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
385 case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
386 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
387 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
388 case MLX5_CMD_OP_2ERR_QP:
389 case MLX5_CMD_OP_2RST_QP:
390 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
391 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
392 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
393 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
394 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
395 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
396 case MLX5_CMD_OP_FPGA_DESTROY_QP:
397 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
398 case MLX5_CMD_OP_DEALLOC_MEMIC:
399 case MLX5_CMD_OP_PAGE_FAULT_RESUME:
400 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
401 case MLX5_CMD_OP_DEALLOC_SF:
402 case MLX5_CMD_OP_DESTROY_UCTX:
403 case MLX5_CMD_OP_DESTROY_UMEM:
404 case MLX5_CMD_OP_MODIFY_RQT:
405 return MLX5_CMD_STAT_OK;
406
407 case MLX5_CMD_OP_QUERY_HCA_CAP:
408 case MLX5_CMD_OP_QUERY_ADAPTER:
409 case MLX5_CMD_OP_INIT_HCA:
410 case MLX5_CMD_OP_ENABLE_HCA:
411 case MLX5_CMD_OP_QUERY_PAGES:
412 case MLX5_CMD_OP_SET_HCA_CAP:
413 case MLX5_CMD_OP_QUERY_ISSI:
414 case MLX5_CMD_OP_SET_ISSI:
415 case MLX5_CMD_OP_CREATE_MKEY:
416 case MLX5_CMD_OP_QUERY_MKEY:
417 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
418 case MLX5_CMD_OP_CREATE_EQ:
419 case MLX5_CMD_OP_QUERY_EQ:
420 case MLX5_CMD_OP_GEN_EQE:
421 case MLX5_CMD_OP_CREATE_CQ:
422 case MLX5_CMD_OP_QUERY_CQ:
423 case MLX5_CMD_OP_MODIFY_CQ:
424 case MLX5_CMD_OP_CREATE_QP:
425 case MLX5_CMD_OP_RST2INIT_QP:
426 case MLX5_CMD_OP_INIT2RTR_QP:
427 case MLX5_CMD_OP_RTR2RTS_QP:
428 case MLX5_CMD_OP_RTS2RTS_QP:
429 case MLX5_CMD_OP_SQERR2RTS_QP:
430 case MLX5_CMD_OP_QUERY_QP:
431 case MLX5_CMD_OP_SQD_RTS_QP:
432 case MLX5_CMD_OP_INIT2INIT_QP:
433 case MLX5_CMD_OP_CREATE_PSV:
434 case MLX5_CMD_OP_CREATE_SRQ:
435 case MLX5_CMD_OP_QUERY_SRQ:
436 case MLX5_CMD_OP_ARM_RQ:
437 case MLX5_CMD_OP_CREATE_XRC_SRQ:
438 case MLX5_CMD_OP_QUERY_XRC_SRQ:
439 case MLX5_CMD_OP_ARM_XRC_SRQ:
440 case MLX5_CMD_OP_CREATE_XRQ:
441 case MLX5_CMD_OP_QUERY_XRQ:
442 case MLX5_CMD_OP_ARM_XRQ:
443 case MLX5_CMD_OP_CREATE_DCT:
444 case MLX5_CMD_OP_DRAIN_DCT:
445 case MLX5_CMD_OP_QUERY_DCT:
446 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
447 case MLX5_CMD_OP_QUERY_VPORT_STATE:
448 case MLX5_CMD_OP_MODIFY_VPORT_STATE:
449 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
450 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
451 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
452 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
453 case MLX5_CMD_OP_SET_ROCE_ADDRESS:
454 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
455 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
456 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
457 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
458 case MLX5_CMD_OP_QUERY_VNIC_ENV:
459 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
460 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
461 case MLX5_CMD_OP_QUERY_Q_COUNTER:
462 case MLX5_CMD_OP_SET_MONITOR_COUNTER:
463 case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
464 case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
465 case MLX5_CMD_OP_QUERY_RATE_LIMIT:
466 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
467 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
468 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
469 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
470 case MLX5_CMD_OP_ALLOC_PD:
471 case MLX5_CMD_OP_ALLOC_UAR:
472 case MLX5_CMD_OP_CONFIG_INT_MODERATION:
473 case MLX5_CMD_OP_ACCESS_REG:
474 case MLX5_CMD_OP_ATTACH_TO_MCG:
475 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
476 case MLX5_CMD_OP_MAD_IFC:
477 case MLX5_CMD_OP_QUERY_MAD_DEMUX:
478 case MLX5_CMD_OP_SET_MAD_DEMUX:
479 case MLX5_CMD_OP_NOP:
480 case MLX5_CMD_OP_ALLOC_XRCD:
481 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
482 case MLX5_CMD_OP_QUERY_CONG_STATUS:
483 case MLX5_CMD_OP_MODIFY_CONG_STATUS:
484 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
485 case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
486 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
487 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
488 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
489 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
490 case MLX5_CMD_OP_CREATE_LAG:
491 case MLX5_CMD_OP_MODIFY_LAG:
492 case MLX5_CMD_OP_QUERY_LAG:
493 case MLX5_CMD_OP_CREATE_VPORT_LAG:
494 case MLX5_CMD_OP_CREATE_TIR:
495 case MLX5_CMD_OP_MODIFY_TIR:
496 case MLX5_CMD_OP_QUERY_TIR:
497 case MLX5_CMD_OP_CREATE_SQ:
498 case MLX5_CMD_OP_MODIFY_SQ:
499 case MLX5_CMD_OP_QUERY_SQ:
500 case MLX5_CMD_OP_CREATE_RQ:
501 case MLX5_CMD_OP_MODIFY_RQ:
502 case MLX5_CMD_OP_QUERY_RQ:
503 case MLX5_CMD_OP_CREATE_RMP:
504 case MLX5_CMD_OP_MODIFY_RMP:
505 case MLX5_CMD_OP_QUERY_RMP:
506 case MLX5_CMD_OP_CREATE_TIS:
507 case MLX5_CMD_OP_MODIFY_TIS:
508 case MLX5_CMD_OP_QUERY_TIS:
509 case MLX5_CMD_OP_CREATE_RQT:
510 case MLX5_CMD_OP_QUERY_RQT:
511
512 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
513 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
514 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
515 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
516 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
517 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
518 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
519 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
520 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
521 case MLX5_CMD_OP_FPGA_CREATE_QP:
522 case MLX5_CMD_OP_FPGA_MODIFY_QP:
523 case MLX5_CMD_OP_FPGA_QUERY_QP:
524 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
525 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
526 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
527 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
528 case MLX5_CMD_OP_CREATE_UCTX:
529 case MLX5_CMD_OP_CREATE_UMEM:
530 case MLX5_CMD_OP_ALLOC_MEMIC:
531 case MLX5_CMD_OP_MODIFY_XRQ:
532 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
533 case MLX5_CMD_OP_QUERY_VHCA_STATE:
534 case MLX5_CMD_OP_MODIFY_VHCA_STATE:
535 case MLX5_CMD_OP_ALLOC_SF:
536 case MLX5_CMD_OP_SUSPEND_VHCA:
537 case MLX5_CMD_OP_RESUME_VHCA:
538 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
539 case MLX5_CMD_OP_SAVE_VHCA_STATE:
540 case MLX5_CMD_OP_LOAD_VHCA_STATE:
541 case MLX5_CMD_OP_SYNC_CRYPTO:
542 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:
543 *status = MLX5_DRIVER_STATUS_ABORTED;
544 *synd = MLX5_DRIVER_SYND;
545 return -ENOLINK;
546 default:
547 mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
548 return -EINVAL;
549 }
550 }
551
mlx5_command_str(int command)552 const char *mlx5_command_str(int command)
553 {
554 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
555
556 switch (command) {
557 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
558 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
559 MLX5_COMMAND_STR_CASE(INIT_HCA);
560 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
561 MLX5_COMMAND_STR_CASE(ENABLE_HCA);
562 MLX5_COMMAND_STR_CASE(DISABLE_HCA);
563 MLX5_COMMAND_STR_CASE(QUERY_PAGES);
564 MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
565 MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
566 MLX5_COMMAND_STR_CASE(QUERY_ISSI);
567 MLX5_COMMAND_STR_CASE(SET_ISSI);
568 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
569 MLX5_COMMAND_STR_CASE(CREATE_MKEY);
570 MLX5_COMMAND_STR_CASE(QUERY_MKEY);
571 MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
572 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
573 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
574 MLX5_COMMAND_STR_CASE(CREATE_EQ);
575 MLX5_COMMAND_STR_CASE(DESTROY_EQ);
576 MLX5_COMMAND_STR_CASE(QUERY_EQ);
577 MLX5_COMMAND_STR_CASE(GEN_EQE);
578 MLX5_COMMAND_STR_CASE(CREATE_CQ);
579 MLX5_COMMAND_STR_CASE(DESTROY_CQ);
580 MLX5_COMMAND_STR_CASE(QUERY_CQ);
581 MLX5_COMMAND_STR_CASE(MODIFY_CQ);
582 MLX5_COMMAND_STR_CASE(CREATE_QP);
583 MLX5_COMMAND_STR_CASE(DESTROY_QP);
584 MLX5_COMMAND_STR_CASE(RST2INIT_QP);
585 MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
586 MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
587 MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
588 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
589 MLX5_COMMAND_STR_CASE(2ERR_QP);
590 MLX5_COMMAND_STR_CASE(2RST_QP);
591 MLX5_COMMAND_STR_CASE(QUERY_QP);
592 MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
593 MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
594 MLX5_COMMAND_STR_CASE(CREATE_PSV);
595 MLX5_COMMAND_STR_CASE(DESTROY_PSV);
596 MLX5_COMMAND_STR_CASE(CREATE_SRQ);
597 MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
598 MLX5_COMMAND_STR_CASE(QUERY_SRQ);
599 MLX5_COMMAND_STR_CASE(ARM_RQ);
600 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
601 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
602 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
603 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
604 MLX5_COMMAND_STR_CASE(CREATE_DCT);
605 MLX5_COMMAND_STR_CASE(DESTROY_DCT);
606 MLX5_COMMAND_STR_CASE(DRAIN_DCT);
607 MLX5_COMMAND_STR_CASE(QUERY_DCT);
608 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
609 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
610 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
611 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
612 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
613 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
614 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
615 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
616 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
617 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
618 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
619 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
620 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
621 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
622 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
623 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
624 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
625 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
626 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
627 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
628 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
629 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
630 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
631 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
632 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
633 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
634 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
635 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
636 MLX5_COMMAND_STR_CASE(ALLOC_PD);
637 MLX5_COMMAND_STR_CASE(DEALLOC_PD);
638 MLX5_COMMAND_STR_CASE(ALLOC_UAR);
639 MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
640 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
641 MLX5_COMMAND_STR_CASE(ACCESS_REG);
642 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
643 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
644 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
645 MLX5_COMMAND_STR_CASE(MAD_IFC);
646 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
647 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
648 MLX5_COMMAND_STR_CASE(NOP);
649 MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
650 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
651 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
652 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
653 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
654 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
655 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
656 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
657 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
658 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
659 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
660 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
661 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
662 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
663 MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
664 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
665 MLX5_COMMAND_STR_CASE(CREATE_LAG);
666 MLX5_COMMAND_STR_CASE(MODIFY_LAG);
667 MLX5_COMMAND_STR_CASE(QUERY_LAG);
668 MLX5_COMMAND_STR_CASE(DESTROY_LAG);
669 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
670 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
671 MLX5_COMMAND_STR_CASE(CREATE_TIR);
672 MLX5_COMMAND_STR_CASE(MODIFY_TIR);
673 MLX5_COMMAND_STR_CASE(DESTROY_TIR);
674 MLX5_COMMAND_STR_CASE(QUERY_TIR);
675 MLX5_COMMAND_STR_CASE(CREATE_SQ);
676 MLX5_COMMAND_STR_CASE(MODIFY_SQ);
677 MLX5_COMMAND_STR_CASE(DESTROY_SQ);
678 MLX5_COMMAND_STR_CASE(QUERY_SQ);
679 MLX5_COMMAND_STR_CASE(CREATE_RQ);
680 MLX5_COMMAND_STR_CASE(MODIFY_RQ);
681 MLX5_COMMAND_STR_CASE(DESTROY_RQ);
682 MLX5_COMMAND_STR_CASE(QUERY_RQ);
683 MLX5_COMMAND_STR_CASE(CREATE_RMP);
684 MLX5_COMMAND_STR_CASE(MODIFY_RMP);
685 MLX5_COMMAND_STR_CASE(DESTROY_RMP);
686 MLX5_COMMAND_STR_CASE(QUERY_RMP);
687 MLX5_COMMAND_STR_CASE(CREATE_TIS);
688 MLX5_COMMAND_STR_CASE(MODIFY_TIS);
689 MLX5_COMMAND_STR_CASE(DESTROY_TIS);
690 MLX5_COMMAND_STR_CASE(QUERY_TIS);
691 MLX5_COMMAND_STR_CASE(CREATE_RQT);
692 MLX5_COMMAND_STR_CASE(MODIFY_RQT);
693 MLX5_COMMAND_STR_CASE(DESTROY_RQT);
694 MLX5_COMMAND_STR_CASE(QUERY_RQT);
695 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
696 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
697 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
698 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
699 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
700 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
701 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
702 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
703 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
704 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
705 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
706 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
707 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
708 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
709 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
710 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
711 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
712 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
713 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
714 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
715 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
716 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
717 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
718 MLX5_COMMAND_STR_CASE(CREATE_XRQ);
719 MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
720 MLX5_COMMAND_STR_CASE(QUERY_XRQ);
721 MLX5_COMMAND_STR_CASE(ARM_XRQ);
722 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
723 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
724 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
725 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
726 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
727 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
728 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
729 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
730 MLX5_COMMAND_STR_CASE(CREATE_UCTX);
731 MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
732 MLX5_COMMAND_STR_CASE(CREATE_UMEM);
733 MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
734 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
735 MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
736 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
737 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
738 MLX5_COMMAND_STR_CASE(ALLOC_SF);
739 MLX5_COMMAND_STR_CASE(DEALLOC_SF);
740 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
741 MLX5_COMMAND_STR_CASE(RESUME_VHCA);
742 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
743 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
744 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
745 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);
746 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);
747 default: return "unknown command opcode";
748 }
749 }
750
cmd_status_str(u8 status)751 static const char *cmd_status_str(u8 status)
752 {
753 switch (status) {
754 case MLX5_CMD_STAT_OK:
755 return "OK";
756 case MLX5_CMD_STAT_INT_ERR:
757 return "internal error";
758 case MLX5_CMD_STAT_BAD_OP_ERR:
759 return "bad operation";
760 case MLX5_CMD_STAT_BAD_PARAM_ERR:
761 return "bad parameter";
762 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
763 return "bad system state";
764 case MLX5_CMD_STAT_BAD_RES_ERR:
765 return "bad resource";
766 case MLX5_CMD_STAT_RES_BUSY:
767 return "resource busy";
768 case MLX5_CMD_STAT_NOT_READY:
769 return "FW not ready";
770 case MLX5_CMD_STAT_LIM_ERR:
771 return "limits exceeded";
772 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
773 return "bad resource state";
774 case MLX5_CMD_STAT_IX_ERR:
775 return "bad index";
776 case MLX5_CMD_STAT_NO_RES_ERR:
777 return "no resources";
778 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
779 return "bad input length";
780 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
781 return "bad output length";
782 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
783 return "bad QP state";
784 case MLX5_CMD_STAT_BAD_PKT_ERR:
785 return "bad packet (discarded)";
786 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
787 return "bad size too many outstanding CQEs";
788 default:
789 return "unknown status";
790 }
791 }
792
cmd_status_to_err(u8 status)793 static int cmd_status_to_err(u8 status)
794 {
795 switch (status) {
796 case MLX5_CMD_STAT_OK: return 0;
797 case MLX5_CMD_STAT_INT_ERR: return -EIO;
798 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
799 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
800 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
801 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
802 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
803 case MLX5_CMD_STAT_NOT_READY: return -EAGAIN;
804 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;
805 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
806 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
807 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
808 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
809 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
810 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
811 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
812 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
813 default: return -EIO;
814 }
815 }
816
mlx5_cmd_out_err(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)817 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
818 {
819 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
820 u8 status = MLX5_GET(mbox_out, out, status);
821
822 mlx5_core_err_rl(dev,
823 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
824 mlx5_command_str(opcode), opcode, op_mod,
825 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
826 }
827 EXPORT_SYMBOL(mlx5_cmd_out_err);
828
cmd_status_print(struct mlx5_core_dev * dev,void * in,void * out)829 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
830 {
831 u16 opcode, op_mod;
832 u8 status;
833 u16 uid;
834
835 opcode = in_to_opcode(in);
836 op_mod = MLX5_GET(mbox_in, in, op_mod);
837 uid = in_to_uid(in);
838 status = MLX5_GET(mbox_out, out, status);
839
840 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&
841 opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)
842 mlx5_cmd_out_err(dev, opcode, op_mod, out);
843 }
844
mlx5_cmd_check(struct mlx5_core_dev * dev,int err,void * in,void * out)845 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
846 {
847 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
848 if (err == -ENXIO) {
849 u16 opcode = in_to_opcode(in);
850 u32 syndrome;
851 u8 status;
852
853 /* PCI Error, emulate command return status, for smooth reset */
854 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);
855 MLX5_SET(mbox_out, out, status, status);
856 MLX5_SET(mbox_out, out, syndrome, syndrome);
857 if (!err)
858 return 0;
859 }
860
861 /* driver or FW delivery error */
862 if (err != -EREMOTEIO && err)
863 return err;
864
865 /* check outbox status */
866 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
867 if (err)
868 cmd_status_print(dev, in, out);
869
870 return err;
871 }
872 EXPORT_SYMBOL(mlx5_cmd_check);
873
dump_command(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent,int input)874 static void dump_command(struct mlx5_core_dev *dev,
875 struct mlx5_cmd_work_ent *ent, int input)
876 {
877 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
878 struct mlx5_cmd_mailbox *next = msg->next;
879 int n = mlx5_calc_cmd_blocks(msg);
880 u16 op = ent->op;
881 int data_only;
882 u32 offset = 0;
883 int dump_len;
884 int i;
885
886 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
887 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
888
889 if (data_only)
890 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
891 "cmd[%d]: dump command data %s(0x%x) %s\n",
892 ent->idx, mlx5_command_str(op), op,
893 input ? "INPUT" : "OUTPUT");
894 else
895 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
896 ent->idx, mlx5_command_str(op), op,
897 input ? "INPUT" : "OUTPUT");
898
899 if (data_only) {
900 if (input) {
901 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
902 offset += sizeof(ent->lay->in);
903 } else {
904 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
905 offset += sizeof(ent->lay->out);
906 }
907 } else {
908 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
909 offset += sizeof(*ent->lay);
910 }
911
912 for (i = 0; i < n && next; i++) {
913 if (data_only) {
914 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
915 dump_buf(next->buf, dump_len, 1, offset, ent->idx);
916 offset += MLX5_CMD_DATA_BLOCK_SIZE;
917 } else {
918 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
919 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,
920 ent->idx);
921 offset += sizeof(struct mlx5_cmd_prot_block);
922 }
923 next = next->next;
924 }
925
926 if (data_only)
927 pr_debug("\n");
928
929 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
930 }
931
932 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
933
cb_timeout_handler(struct work_struct * work)934 static void cb_timeout_handler(struct work_struct *work)
935 {
936 struct delayed_work *dwork = to_delayed_work(work);
937 struct mlx5_cmd_work_ent *ent = container_of(dwork,
938 struct mlx5_cmd_work_ent,
939 cb_timeout_work);
940 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
941 cmd);
942
943 mlx5_cmd_eq_recover(dev);
944
945 /* Maybe got handled by eq recover ? */
946 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
947 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
948 mlx5_command_str(ent->op), ent->op);
949 goto out; /* phew, already handled */
950 }
951
952 ent->ret = -ETIMEDOUT;
953 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
954 ent->idx, mlx5_command_str(ent->op), ent->op);
955 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
956
957 out:
958 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
959 }
960
961 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
962 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
963 struct mlx5_cmd_msg *msg);
964
opcode_allowed(struct mlx5_cmd * cmd,u16 opcode)965 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
966 {
967 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
968 return true;
969
970 return cmd->allowed_opcode == opcode;
971 }
972
mlx5_cmd_is_down(struct mlx5_core_dev * dev)973 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
974 {
975 return pci_channel_offline(dev->pdev) ||
976 dev->cmd.state != MLX5_CMDIF_STATE_UP ||
977 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
978 }
979
cmd_work_handler(struct work_struct * work)980 static void cmd_work_handler(struct work_struct *work)
981 {
982 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
983 struct mlx5_cmd *cmd = ent->cmd;
984 bool poll_cmd = ent->polling;
985 struct mlx5_cmd_layout *lay;
986 struct mlx5_core_dev *dev;
987 unsigned long timeout;
988 unsigned long flags;
989 int alloc_ret;
990 int cmd_mode;
991
992 complete(&ent->handling);
993
994 dev = container_of(cmd, struct mlx5_core_dev, cmd);
995 timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
996
997 if (!ent->page_queue) {
998 if (down_timeout(&cmd->vars.sem, timeout)) {
999 mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",
1000 mlx5_command_str(ent->op), ent->op);
1001 if (ent->callback) {
1002 ent->callback(-EBUSY, ent->context);
1003 mlx5_free_cmd_msg(dev, ent->out);
1004 free_msg(dev, ent->in);
1005 complete(&ent->slotted);
1006 cmd_ent_put(ent);
1007 } else {
1008 ent->ret = -EBUSY;
1009 complete(&ent->done);
1010 complete(&ent->slotted);
1011 }
1012 return;
1013 }
1014 alloc_ret = cmd_alloc_index(cmd, ent);
1015 if (alloc_ret < 0) {
1016 mlx5_core_err_rl(dev, "failed to allocate command entry\n");
1017 if (ent->callback) {
1018 ent->callback(-EAGAIN, ent->context);
1019 mlx5_free_cmd_msg(dev, ent->out);
1020 free_msg(dev, ent->in);
1021 complete(&ent->slotted);
1022 cmd_ent_put(ent);
1023 } else {
1024 ent->ret = -EAGAIN;
1025 complete(&ent->done);
1026 complete(&ent->slotted);
1027 }
1028 up(&cmd->vars.sem);
1029 return;
1030 }
1031 } else {
1032 down(&cmd->vars.pages_sem);
1033 ent->idx = cmd->vars.max_reg_cmds;
1034 spin_lock_irqsave(&cmd->alloc_lock, flags);
1035 clear_bit(ent->idx, &cmd->vars.bitmask);
1036 cmd->ent_arr[ent->idx] = ent;
1037 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1038 }
1039
1040 complete(&ent->slotted);
1041
1042 lay = get_inst(cmd, ent->idx);
1043 ent->lay = lay;
1044 memset(lay, 0, sizeof(*lay));
1045 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1046 if (ent->in->next)
1047 lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1048 lay->inlen = cpu_to_be32(ent->in->len);
1049 if (ent->out->next)
1050 lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1051 lay->outlen = cpu_to_be32(ent->out->len);
1052 lay->type = MLX5_PCI_CMD_XPORT;
1053 lay->token = ent->token;
1054 lay->status_own = CMD_OWNER_HW;
1055 set_signature(ent, !cmd->checksum_disabled);
1056 dump_command(dev, ent, 1);
1057 ent->ts1 = ktime_get_ns();
1058 cmd_mode = cmd->mode;
1059
1060 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))
1061 cmd_ent_get(ent);
1062 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1063
1064 cmd_ent_get(ent); /* for the _real_ FW event on completion */
1065 /* Skip sending command to fw if internal error */
1066 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1067 ent->ret = -ENXIO;
1068 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1069 return;
1070 }
1071
1072 /* ring doorbell after the descriptor is valid */
1073 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1074 wmb();
1075 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1076 /* if not in polling don't use ent after this point */
1077 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1078 poll_timeout(ent);
1079 /* make sure we read the descriptor after ownership is SW */
1080 rmb();
1081 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret);
1082 }
1083 }
1084
deliv_status_to_err(u8 status)1085 static int deliv_status_to_err(u8 status)
1086 {
1087 switch (status) {
1088 case MLX5_CMD_DELIVERY_STAT_OK:
1089 case MLX5_DRIVER_STATUS_ABORTED:
1090 return 0;
1091 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1092 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1093 return -EBADR;
1094 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1095 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1096 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1097 return -EFAULT; /* Bad address */
1098 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1099 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1100 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1101 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1102 return -ENOMSG;
1103 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1104 return -EIO;
1105 default:
1106 return -EINVAL;
1107 }
1108 }
1109
deliv_status_to_str(u8 status)1110 static const char *deliv_status_to_str(u8 status)
1111 {
1112 switch (status) {
1113 case MLX5_CMD_DELIVERY_STAT_OK:
1114 return "no errors";
1115 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1116 return "signature error";
1117 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1118 return "token error";
1119 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1120 return "bad block number";
1121 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1122 return "output pointer not aligned to block size";
1123 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1124 return "input pointer not aligned to block size";
1125 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1126 return "firmware internal error";
1127 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1128 return "command input length error";
1129 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1130 return "command output length error";
1131 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1132 return "reserved fields not cleared";
1133 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1134 return "bad command descriptor type";
1135 default:
1136 return "unknown status code";
1137 }
1138 }
1139
1140 enum {
1141 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
1142 };
1143
wait_func_handle_exec_timeout(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1144 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1145 struct mlx5_cmd_work_ent *ent)
1146 {
1147 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1148
1149 mlx5_cmd_eq_recover(dev);
1150
1151 /* Re-wait on the ent->done after executing the recovery flow. If the
1152 * recovery flow (or any other recovery flow running simultaneously)
1153 * has recovered an EQE, it should cause the entry to be completed by
1154 * the command interface.
1155 */
1156 if (wait_for_completion_timeout(&ent->done, timeout)) {
1157 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1158 mlx5_command_str(ent->op), ent->op);
1159 return;
1160 }
1161
1162 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1163 mlx5_command_str(ent->op), ent->op);
1164
1165 ent->ret = -ETIMEDOUT;
1166 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1167 }
1168
wait_func(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1169 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1170 {
1171 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1172 struct mlx5_cmd *cmd = &dev->cmd;
1173 int err;
1174
1175 if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1176 cancel_work_sync(&ent->work)) {
1177 ent->ret = -ECANCELED;
1178 goto out_err;
1179 }
1180
1181 wait_for_completion(&ent->slotted);
1182
1183 if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1184 wait_for_completion(&ent->done);
1185 else if (!wait_for_completion_timeout(&ent->done, timeout))
1186 wait_func_handle_exec_timeout(dev, ent);
1187
1188 out_err:
1189 err = ent->ret;
1190
1191 if (err == -ETIMEDOUT) {
1192 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1193 mlx5_command_str(ent->op), ent->op);
1194 } else if (err == -ECANCELED) {
1195 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1196 mlx5_command_str(ent->op), ent->op);
1197 } else if (err == -EBUSY) {
1198 mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",
1199 mlx5_command_str(ent->op), ent->op);
1200 }
1201 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1202 err, deliv_status_to_str(ent->status), ent->status);
1203
1204 return err;
1205 }
1206
1207 /* Check if all command slots are stalled (timed out and not recovered).
1208 * returns true if all slots timed out on a recent command and have not been
1209 * completed by FW yet. (stalled state)
1210 * false otherwise (at least one slot is not stalled).
1211 *
1212 * In such odd situation "all_stalled", this serves as a protection mechanism
1213 * to avoid blocking the kernel for long periods of time in case FW is not
1214 * responding to commands.
1215 */
mlx5_cmd_all_stalled(struct mlx5_core_dev * dev)1216 static bool mlx5_cmd_all_stalled(struct mlx5_core_dev *dev)
1217 {
1218 struct mlx5_cmd *cmd = &dev->cmd;
1219 bool all_stalled = true;
1220 unsigned long flags;
1221 int i;
1222
1223 spin_lock_irqsave(&cmd->alloc_lock, flags);
1224
1225 /* at least one command slot is free */
1226 if (bitmap_weight(&cmd->vars.bitmask, cmd->vars.max_reg_cmds) > 0) {
1227 all_stalled = false;
1228 goto out;
1229 }
1230
1231 for_each_clear_bit(i, &cmd->vars.bitmask, cmd->vars.max_reg_cmds) {
1232 struct mlx5_cmd_work_ent *ent = dev->cmd.ent_arr[i];
1233
1234 if (!test_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, &ent->state)) {
1235 all_stalled = false;
1236 break;
1237 }
1238 }
1239 out:
1240 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1241
1242 return all_stalled;
1243 }
1244
1245 /* Notes:
1246 * 1. Callback functions may not sleep
1247 * 2. page queue commands do not support asynchrous completion
1248 *
1249 * return value in case (!callback):
1250 * ret < 0 : Command execution couldn't be submitted by driver
1251 * ret > 0 : Command execution couldn't be performed by firmware
1252 * ret == 0: Command was executed by FW, Caller must check FW outbox status.
1253 *
1254 * return value in case (callback):
1255 * ret < 0 : Command execution couldn't be submitted by driver
1256 * ret == 0: Command will be submitted to FW for execution
1257 * and the callback will be called for further status updates
1258 */
mlx5_cmd_invoke(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t callback,void * context,int page_queue,u8 token,bool force_polling)1259 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1260 struct mlx5_cmd_msg *out, void *uout, int uout_size,
1261 mlx5_cmd_cbk_t callback,
1262 void *context, int page_queue,
1263 u8 token, bool force_polling)
1264 {
1265 struct mlx5_cmd *cmd = &dev->cmd;
1266 struct mlx5_cmd_work_ent *ent;
1267 struct mlx5_cmd_stats *stats;
1268 u8 status = 0;
1269 int err = 0;
1270 s64 ds;
1271
1272 if (callback && page_queue)
1273 return -EINVAL;
1274
1275 if (!page_queue && mlx5_cmd_all_stalled(dev)) {
1276 mlx5_core_err_rl(dev,
1277 "All CMD slots are stalled, aborting command\n");
1278 /* there's no reason to wait and block the whole kernel if FW
1279 * isn't currently responding to all slots, fail immediately
1280 */
1281 return -EAGAIN;
1282 }
1283
1284 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1285 callback, context, page_queue);
1286 if (IS_ERR(ent))
1287 return PTR_ERR(ent);
1288
1289 /* put for this ent is when consumed, depending on the use case
1290 * 1) (!callback) blocking flow: by caller after wait_func completes
1291 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1292 */
1293
1294 ent->token = token;
1295 ent->polling = force_polling;
1296
1297 init_completion(&ent->handling);
1298 init_completion(&ent->slotted);
1299 if (!callback)
1300 init_completion(&ent->done);
1301
1302 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1303 INIT_WORK(&ent->work, cmd_work_handler);
1304 if (page_queue) {
1305 cmd_work_handler(&ent->work);
1306 } else if (!queue_work(cmd->wq, &ent->work)) {
1307 mlx5_core_warn(dev, "failed to queue work\n");
1308 err = -EALREADY;
1309 goto out_free;
1310 }
1311
1312 if (callback)
1313 return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1314
1315 err = wait_func(dev, ent);
1316 if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)
1317 goto out_free;
1318
1319 ds = ent->ts2 - ent->ts1;
1320 stats = xa_load(&cmd->stats, ent->op);
1321 if (stats) {
1322 spin_lock_irq(&stats->lock);
1323 stats->sum += ds;
1324 ++stats->n;
1325 spin_unlock_irq(&stats->lock);
1326 }
1327 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1328 "fw exec time for %s is %lld nsec\n",
1329 mlx5_command_str(ent->op), ds);
1330
1331 out_free:
1332 status = ent->status;
1333 cmd_ent_put(ent);
1334 return err ? : status;
1335 }
1336
dbg_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1337 static ssize_t dbg_write(struct file *filp, const char __user *buf,
1338 size_t count, loff_t *pos)
1339 {
1340 struct mlx5_core_dev *dev = filp->private_data;
1341 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1342 char lbuf[3];
1343 int err;
1344
1345 if (!dbg->in_msg || !dbg->out_msg)
1346 return -ENOMEM;
1347
1348 if (count < sizeof(lbuf) - 1)
1349 return -EINVAL;
1350
1351 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))
1352 return -EFAULT;
1353
1354 lbuf[sizeof(lbuf) - 1] = 0;
1355
1356 if (strcmp(lbuf, "go"))
1357 return -EINVAL;
1358
1359 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
1360
1361 return err ? err : count;
1362 }
1363
1364 static const struct file_operations fops = {
1365 .owner = THIS_MODULE,
1366 .open = simple_open,
1367 .write = dbg_write,
1368 };
1369
mlx5_copy_to_msg(struct mlx5_cmd_msg * to,void * from,int size,u8 token)1370 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1371 u8 token)
1372 {
1373 struct mlx5_cmd_prot_block *block;
1374 struct mlx5_cmd_mailbox *next;
1375 int copy;
1376
1377 if (!to || !from)
1378 return -ENOMEM;
1379
1380 copy = min_t(int, size, sizeof(to->first.data));
1381 memcpy(to->first.data, from, copy);
1382 size -= copy;
1383 from += copy;
1384
1385 next = to->next;
1386 while (size) {
1387 if (!next) {
1388 /* this is a BUG */
1389 return -ENOMEM;
1390 }
1391
1392 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1393 block = next->buf;
1394 memcpy(block->data, from, copy);
1395 from += copy;
1396 size -= copy;
1397 block->token = token;
1398 next = next->next;
1399 }
1400
1401 return 0;
1402 }
1403
mlx5_copy_from_msg(void * to,struct mlx5_cmd_msg * from,int size)1404 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1405 {
1406 struct mlx5_cmd_prot_block *block;
1407 struct mlx5_cmd_mailbox *next;
1408 int copy;
1409
1410 if (!to || !from)
1411 return -ENOMEM;
1412
1413 copy = min_t(int, size, sizeof(from->first.data));
1414 memcpy(to, from->first.data, copy);
1415 size -= copy;
1416 to += copy;
1417
1418 next = from->next;
1419 while (size) {
1420 if (!next) {
1421 /* this is a BUG */
1422 return -ENOMEM;
1423 }
1424
1425 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1426 block = next->buf;
1427
1428 memcpy(to, block->data, copy);
1429 to += copy;
1430 size -= copy;
1431 next = next->next;
1432 }
1433
1434 return 0;
1435 }
1436
alloc_cmd_box(struct mlx5_core_dev * dev,gfp_t flags)1437 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1438 gfp_t flags)
1439 {
1440 struct mlx5_cmd_mailbox *mailbox;
1441
1442 mailbox = kmalloc_obj(*mailbox, flags);
1443 if (!mailbox)
1444 return ERR_PTR(-ENOMEM);
1445
1446 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
1447 &mailbox->dma);
1448 if (!mailbox->buf) {
1449 mlx5_core_dbg(dev, "failed allocation\n");
1450 kfree(mailbox);
1451 return ERR_PTR(-ENOMEM);
1452 }
1453 mailbox->next = NULL;
1454
1455 return mailbox;
1456 }
1457
free_cmd_box(struct mlx5_core_dev * dev,struct mlx5_cmd_mailbox * mailbox)1458 static void free_cmd_box(struct mlx5_core_dev *dev,
1459 struct mlx5_cmd_mailbox *mailbox)
1460 {
1461 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
1462 kfree(mailbox);
1463 }
1464
mlx5_alloc_cmd_msg(struct mlx5_core_dev * dev,gfp_t flags,int size,u8 token)1465 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1466 gfp_t flags, int size,
1467 u8 token)
1468 {
1469 struct mlx5_cmd_mailbox *tmp, *head = NULL;
1470 struct mlx5_cmd_prot_block *block;
1471 struct mlx5_cmd_msg *msg;
1472 int err;
1473 int n;
1474 int i;
1475
1476 msg = kzalloc_obj(*msg, flags);
1477 if (!msg)
1478 return ERR_PTR(-ENOMEM);
1479
1480 msg->len = size;
1481 n = mlx5_calc_cmd_blocks(msg);
1482
1483 for (i = 0; i < n; i++) {
1484 tmp = alloc_cmd_box(dev, flags);
1485 if (IS_ERR(tmp)) {
1486 mlx5_core_warn(dev, "failed allocating block\n");
1487 err = PTR_ERR(tmp);
1488 goto err_alloc;
1489 }
1490
1491 block = tmp->buf;
1492 tmp->next = head;
1493 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1494 block->block_num = cpu_to_be32(n - i - 1);
1495 block->token = token;
1496 head = tmp;
1497 }
1498 msg->next = head;
1499 return msg;
1500
1501 err_alloc:
1502 while (head) {
1503 tmp = head->next;
1504 free_cmd_box(dev, head);
1505 head = tmp;
1506 }
1507 kfree(msg);
1508
1509 return ERR_PTR(err);
1510 }
1511
mlx5_free_cmd_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1512 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1513 struct mlx5_cmd_msg *msg)
1514 {
1515 struct mlx5_cmd_mailbox *head = msg->next;
1516 struct mlx5_cmd_mailbox *next;
1517
1518 while (head) {
1519 next = head->next;
1520 free_cmd_box(dev, head);
1521 head = next;
1522 }
1523 kfree(msg);
1524 }
1525
data_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1526 static ssize_t data_write(struct file *filp, const char __user *buf,
1527 size_t count, loff_t *pos)
1528 {
1529 struct mlx5_core_dev *dev = filp->private_data;
1530 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1531 void *ptr;
1532
1533 if (*pos != 0)
1534 return -EINVAL;
1535
1536 kfree(dbg->in_msg);
1537 dbg->in_msg = NULL;
1538 dbg->inlen = 0;
1539 ptr = memdup_user(buf, count);
1540 if (IS_ERR(ptr))
1541 return PTR_ERR(ptr);
1542 dbg->in_msg = ptr;
1543 dbg->inlen = count;
1544
1545 *pos = count;
1546
1547 return count;
1548 }
1549
data_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1550 static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1551 loff_t *pos)
1552 {
1553 struct mlx5_core_dev *dev = filp->private_data;
1554 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1555
1556 if (!dbg->out_msg)
1557 return -ENOMEM;
1558
1559 return simple_read_from_buffer(buf, count, pos, dbg->out_msg,
1560 dbg->outlen);
1561 }
1562
1563 static const struct file_operations dfops = {
1564 .owner = THIS_MODULE,
1565 .open = simple_open,
1566 .write = data_write,
1567 .read = data_read,
1568 };
1569
outlen_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1570 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1571 loff_t *pos)
1572 {
1573 struct mlx5_core_dev *dev = filp->private_data;
1574 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1575 char outlen[8];
1576 int err;
1577
1578 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
1579 if (err < 0)
1580 return err;
1581
1582 return simple_read_from_buffer(buf, count, pos, outlen, err);
1583 }
1584
outlen_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1585 static ssize_t outlen_write(struct file *filp, const char __user *buf,
1586 size_t count, loff_t *pos)
1587 {
1588 struct mlx5_core_dev *dev = filp->private_data;
1589 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1590 char outlen_str[8] = {0};
1591 int outlen;
1592 void *ptr;
1593 int err;
1594
1595 if (*pos != 0 || count > 6)
1596 return -EINVAL;
1597
1598 kfree(dbg->out_msg);
1599 dbg->out_msg = NULL;
1600 dbg->outlen = 0;
1601
1602 if (copy_from_user(outlen_str, buf, count))
1603 return -EFAULT;
1604
1605 err = sscanf(outlen_str, "%d", &outlen);
1606 if (err != 1)
1607 return -EINVAL;
1608
1609 ptr = kzalloc(outlen, GFP_KERNEL);
1610 if (!ptr)
1611 return -ENOMEM;
1612
1613 dbg->out_msg = ptr;
1614 dbg->outlen = outlen;
1615
1616 *pos = count;
1617
1618 return count;
1619 }
1620
1621 static const struct file_operations olfops = {
1622 .owner = THIS_MODULE,
1623 .open = simple_open,
1624 .write = outlen_write,
1625 .read = outlen_read,
1626 };
1627
set_wqname(struct mlx5_core_dev * dev)1628 static void set_wqname(struct mlx5_core_dev *dev)
1629 {
1630 struct mlx5_cmd *cmd = &dev->cmd;
1631
1632 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1633 dev_name(dev->device));
1634 }
1635
clean_debug_files(struct mlx5_core_dev * dev)1636 static void clean_debug_files(struct mlx5_core_dev *dev)
1637 {
1638 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1639
1640 if (!mlx5_debugfs_root)
1641 return;
1642
1643 debugfs_remove_recursive(dbg->dbg_root);
1644 }
1645
create_debugfs_files(struct mlx5_core_dev * dev)1646 static void create_debugfs_files(struct mlx5_core_dev *dev)
1647 {
1648 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1649
1650 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));
1651
1652 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1653 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1654 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1655 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);
1656 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1657 }
1658
mlx5_cmd_allowed_opcode(struct mlx5_core_dev * dev,u16 opcode)1659 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1660 {
1661 struct mlx5_cmd *cmd = &dev->cmd;
1662 int i;
1663
1664 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1665 down(&cmd->vars.sem);
1666 down(&cmd->vars.pages_sem);
1667
1668 cmd->allowed_opcode = opcode;
1669
1670 up(&cmd->vars.pages_sem);
1671 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1672 up(&cmd->vars.sem);
1673 }
1674
mlx5_cmd_change_mod(struct mlx5_core_dev * dev,int mode)1675 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1676 {
1677 struct mlx5_cmd *cmd = &dev->cmd;
1678 int i;
1679
1680 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1681 down(&cmd->vars.sem);
1682 down(&cmd->vars.pages_sem);
1683
1684 cmd->mode = mode;
1685
1686 up(&cmd->vars.pages_sem);
1687 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1688 up(&cmd->vars.sem);
1689 }
1690
cmd_comp_notifier(struct notifier_block * nb,unsigned long type,void * data)1691 static int cmd_comp_notifier(struct notifier_block *nb,
1692 unsigned long type, void *data)
1693 {
1694 struct mlx5_core_dev *dev;
1695 struct mlx5_cmd *cmd;
1696 struct mlx5_eqe *eqe;
1697
1698 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1699 dev = container_of(cmd, struct mlx5_core_dev, cmd);
1700 eqe = data;
1701
1702 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1703 return NOTIFY_DONE;
1704
1705 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);
1706
1707 return NOTIFY_OK;
1708 }
mlx5_cmd_use_events(struct mlx5_core_dev * dev)1709 void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1710 {
1711 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1712 mlx5_eq_notifier_register(dev, &dev->cmd.nb);
1713 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);
1714 }
1715
mlx5_cmd_use_polling(struct mlx5_core_dev * dev)1716 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1717 {
1718 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);
1719 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);
1720 }
1721
free_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1722 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1723 {
1724 unsigned long flags;
1725
1726 if (msg->parent) {
1727 spin_lock_irqsave(&msg->parent->lock, flags);
1728 list_add_tail(&msg->list, &msg->parent->head);
1729 spin_unlock_irqrestore(&msg->parent->lock, flags);
1730 } else {
1731 mlx5_free_cmd_msg(dev, msg);
1732 }
1733 }
1734
mlx5_cmd_comp_handler(struct mlx5_core_dev * dev,u64 vec,bool forced)1735 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1736 {
1737 struct mlx5_cmd *cmd = &dev->cmd;
1738 struct mlx5_cmd_work_ent *ent;
1739 mlx5_cmd_cbk_t callback;
1740 void *context;
1741 int err;
1742 int i;
1743 s64 ds;
1744 struct mlx5_cmd_stats *stats;
1745 unsigned long flags;
1746 unsigned long vector;
1747
1748 /* there can be at most 32 command queues */
1749 vector = vec & 0xffffffff;
1750 for (i = 0; i < (1 << cmd->vars.log_sz); i++) {
1751 if (test_bit(i, &vector)) {
1752 ent = cmd->ent_arr[i];
1753
1754 if (forced && ent->ret == -ETIMEDOUT)
1755 set_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1756 &ent->state);
1757 else if (!forced) /* real FW completion */
1758 clear_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1759 &ent->state);
1760
1761 /* if we already completed the command, ignore it */
1762 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,
1763 &ent->state)) {
1764 /* only real completion can free the cmd slot */
1765 if (!forced) {
1766 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1767 ent->idx);
1768 cmd_ent_put(ent);
1769 }
1770 continue;
1771 }
1772
1773 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1774 cmd_ent_put(ent); /* timeout work was canceled */
1775
1776 if (!forced || /* Real FW completion */
1777 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1778 !opcode_allowed(cmd, ent->op))
1779 cmd_ent_put(ent);
1780
1781 ent->ts2 = ktime_get_ns();
1782 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1783 dump_command(dev, ent, 0);
1784
1785 if (vec & MLX5_TRIGGERED_CMD_COMP)
1786 ent->ret = -ENXIO;
1787
1788 if (!ent->ret) { /* Command completed by FW */
1789 if (!cmd->checksum_disabled)
1790 ent->ret = verify_signature(ent);
1791
1792 ent->status = ent->lay->status_own >> 1;
1793
1794 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1795 ent->ret, deliv_status_to_str(ent->status), ent->status);
1796 }
1797
1798 if (ent->callback) {
1799 ds = ent->ts2 - ent->ts1;
1800 stats = xa_load(&cmd->stats, ent->op);
1801 if (stats) {
1802 spin_lock_irqsave(&stats->lock, flags);
1803 stats->sum += ds;
1804 ++stats->n;
1805 spin_unlock_irqrestore(&stats->lock, flags);
1806 }
1807
1808 callback = ent->callback;
1809 context = ent->context;
1810 err = ent->ret ? : ent->status;
1811 if (err > 0) /* Failed in FW, command didn't execute */
1812 err = deliv_status_to_err(err);
1813
1814 if (!err)
1815 err = mlx5_copy_from_msg(ent->uout,
1816 ent->out,
1817 ent->uout_size);
1818
1819 mlx5_free_cmd_msg(dev, ent->out);
1820 free_msg(dev, ent->in);
1821
1822 /* final consumer is done, release ent */
1823 cmd_ent_put(ent);
1824 callback(err, context);
1825 } else {
1826 /* release wait_func() so mlx5_cmd_invoke()
1827 * can make the final ent_put()
1828 */
1829 complete(&ent->done);
1830 }
1831 }
1832 }
1833 }
1834
1835 #define MLX5_MAX_MANAGE_PAGES_CMD_ENT 1
1836 #define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \
1837 MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1)
1838
mlx5_cmd_trigger_completions(struct mlx5_core_dev * dev)1839 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1840 {
1841 struct mlx5_cmd *cmd = &dev->cmd;
1842 unsigned long bitmask;
1843 unsigned long flags;
1844 u64 vector;
1845 int i;
1846
1847 /* wait for pending handlers to complete */
1848 mlx5_eq_synchronize_cmd_irq(dev);
1849 spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1850 vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK;
1851 if (!vector)
1852 goto no_trig;
1853
1854 bitmask = vector;
1855 /* we must increment the allocated entries refcount before triggering the completions
1856 * to guarantee pending commands will not get freed in the meanwhile.
1857 * For that reason, it also has to be done inside the alloc_lock.
1858 */
1859 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1860 cmd_ent_get(cmd->ent_arr[i]);
1861 vector |= MLX5_TRIGGERED_CMD_COMP;
1862 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1863
1864 mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1865 mlx5_cmd_comp_handler(dev, vector, true);
1866 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1867 cmd_ent_put(cmd->ent_arr[i]);
1868 return;
1869
1870 no_trig:
1871 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1872 }
1873
mlx5_cmd_flush(struct mlx5_core_dev * dev)1874 void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1875 {
1876 struct mlx5_cmd *cmd = &dev->cmd;
1877 int i;
1878
1879 for (i = 0; i < cmd->vars.max_reg_cmds; i++) {
1880 while (down_trylock(&cmd->vars.sem)) {
1881 mlx5_cmd_trigger_completions(dev);
1882 cond_resched();
1883 }
1884 }
1885
1886 while (down_trylock(&cmd->vars.pages_sem)) {
1887 mlx5_cmd_trigger_completions(dev);
1888 cond_resched();
1889 }
1890
1891 /* Unlock cmdif */
1892 up(&cmd->vars.pages_sem);
1893 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1894 up(&cmd->vars.sem);
1895 }
1896
alloc_msg(struct mlx5_core_dev * dev,int in_size,gfp_t gfp)1897 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1898 gfp_t gfp)
1899 {
1900 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1901 struct cmd_msg_cache *ch = NULL;
1902 struct mlx5_cmd *cmd = &dev->cmd;
1903 int i;
1904
1905 if (in_size <= 16)
1906 goto cache_miss;
1907
1908 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
1909 ch = &cmd->cache[i];
1910 if (in_size > ch->max_inbox_size)
1911 continue;
1912 spin_lock_irq(&ch->lock);
1913 if (list_empty(&ch->head)) {
1914 spin_unlock_irq(&ch->lock);
1915 continue;
1916 }
1917 msg = list_entry(ch->head.next, typeof(*msg), list);
1918 /* For cached lists, we must explicitly state what is
1919 * the real size
1920 */
1921 msg->len = in_size;
1922 list_del(&msg->list);
1923 spin_unlock_irq(&ch->lock);
1924 break;
1925 }
1926
1927 if (!IS_ERR(msg))
1928 return msg;
1929
1930 cache_miss:
1931 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
1932 return msg;
1933 }
1934
is_manage_pages(void * in)1935 static int is_manage_pages(void *in)
1936 {
1937 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;
1938 }
1939
mlx5_has_privileged_uid(struct mlx5_core_dev * dev)1940 static bool mlx5_has_privileged_uid(struct mlx5_core_dev *dev)
1941 {
1942 return !xa_empty(&dev->cmd.vars.privileged_uids);
1943 }
1944
mlx5_cmd_is_privileged_uid(struct mlx5_core_dev * dev,u16 uid)1945 static bool mlx5_cmd_is_privileged_uid(struct mlx5_core_dev *dev,
1946 u16 uid)
1947 {
1948 return !!xa_load(&dev->cmd.vars.privileged_uids, uid);
1949 }
1950
1951 /* Notes:
1952 * 1. Callback functions may not sleep
1953 * 2. Page queue commands do not support asynchrous completion
1954 */
cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size,mlx5_cmd_cbk_t callback,void * context,bool force_polling)1955 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1956 int out_size, mlx5_cmd_cbk_t callback, void *context,
1957 bool force_polling)
1958 {
1959 struct mlx5_cmd_msg *inb, *outb;
1960 u16 opcode = in_to_opcode(in);
1961 bool throttle_locked = false;
1962 bool unpriv_locked = false;
1963 u16 uid = in_to_uid(in);
1964 int pages_queue;
1965 gfp_t gfp;
1966 u8 token;
1967 int err;
1968
1969 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))
1970 return -ENXIO;
1971
1972 if (!callback) {
1973 /* The semaphore is already held for callback commands. It was
1974 * acquired in mlx5_cmd_exec_cb()
1975 */
1976 if (uid && mlx5_has_privileged_uid(dev)) {
1977 if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
1978 unpriv_locked = true;
1979 down(&dev->cmd.vars.unprivileged_sem);
1980 }
1981 } else if (mlx5_cmd_is_throttle_opcode(opcode)) {
1982 throttle_locked = true;
1983 down(&dev->cmd.vars.throttle_sem);
1984 }
1985 }
1986
1987 pages_queue = is_manage_pages(in);
1988 gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1989
1990 inb = alloc_msg(dev, in_size, gfp);
1991 if (IS_ERR(inb)) {
1992 err = PTR_ERR(inb);
1993 goto out_up;
1994 }
1995
1996 token = alloc_token(&dev->cmd);
1997
1998 err = mlx5_copy_to_msg(inb, in, in_size, token);
1999 if (err) {
2000 mlx5_core_warn(dev, "err %d\n", err);
2001 goto out_in;
2002 }
2003
2004 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
2005 if (IS_ERR(outb)) {
2006 err = PTR_ERR(outb);
2007 goto out_in;
2008 }
2009
2010 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
2011 pages_queue, token, force_polling);
2012 if (callback && !err)
2013 return 0;
2014
2015 if (err > 0) /* Failed in FW, command didn't execute */
2016 err = deliv_status_to_err(err);
2017
2018 if (err)
2019 goto out_out;
2020
2021 /* command completed by FW */
2022 err = mlx5_copy_from_msg(out, outb, out_size);
2023 out_out:
2024 mlx5_free_cmd_msg(dev, outb);
2025 out_in:
2026 free_msg(dev, inb);
2027 out_up:
2028 if (throttle_locked)
2029 up(&dev->cmd.vars.throttle_sem);
2030 if (unpriv_locked)
2031 up(&dev->cmd.vars.unprivileged_sem);
2032
2033 return err;
2034 }
2035
mlx5_cmd_err_trace(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)2036 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
2037 {
2038 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2039 u8 status = MLX5_GET(mbox_out, out, status);
2040
2041 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,
2042 cmd_status_str(status), status, syndrome,
2043 cmd_status_to_err(status));
2044 }
2045
cmd_status_log(struct mlx5_core_dev * dev,u16 opcode,u8 status,u32 syndrome,int err)2046 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
2047 u32 syndrome, int err)
2048 {
2049 const char *namep = mlx5_command_str(opcode);
2050 struct mlx5_cmd_stats *stats;
2051 unsigned long flags;
2052
2053 if (!err || !(strcmp(namep, "unknown command opcode")))
2054 return;
2055
2056 stats = xa_load(&dev->cmd.stats, opcode);
2057 if (!stats)
2058 return;
2059 spin_lock_irqsave(&stats->lock, flags);
2060 stats->failed++;
2061 if (err < 0)
2062 stats->last_failed_errno = -err;
2063 if (err == -EREMOTEIO) {
2064 stats->failed_mbox_status++;
2065 stats->last_failed_mbox_status = status;
2066 stats->last_failed_syndrome = syndrome;
2067 }
2068 spin_unlock_irqrestore(&stats->lock, flags);
2069 }
2070
2071 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
cmd_status_err(struct mlx5_core_dev * dev,int err,u16 opcode,u16 op_mod,void * out)2072 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
2073 {
2074 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2075 u8 status = MLX5_GET(mbox_out, out, status);
2076
2077 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
2078 err = -EIO;
2079
2080 if (!err && status != MLX5_CMD_STAT_OK) {
2081 err = -EREMOTEIO;
2082 mlx5_cmd_err_trace(dev, opcode, op_mod, out);
2083 }
2084
2085 cmd_status_log(dev, opcode, status, syndrome, err);
2086 return err;
2087 }
2088
2089 /**
2090 * mlx5_cmd_do - Executes a fw command, wait for completion.
2091 * Unlike mlx5_cmd_exec, this function will not translate or intercept
2092 * outbox.status and will return -EREMOTEIO when
2093 * outbox.status != MLX5_CMD_STAT_OK
2094 *
2095 * @dev: mlx5 core device
2096 * @in: inbox mlx5_ifc command buffer
2097 * @in_size: inbox buffer size
2098 * @out: outbox mlx5_ifc buffer
2099 * @out_size: outbox size
2100 *
2101 * @return:
2102 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
2103 * Caller must check FW outbox status.
2104 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
2105 * < 0 : Command execution couldn't be performed by firmware or driver
2106 */
mlx5_cmd_do(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2107 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
2108 {
2109 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);
2110 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2111 u16 opcode = in_to_opcode(in);
2112
2113 return cmd_status_err(dev, err, opcode, op_mod, out);
2114 }
2115 EXPORT_SYMBOL(mlx5_cmd_do);
2116
2117 /**
2118 * mlx5_cmd_exec - Executes a fw command, wait for completion
2119 *
2120 * @dev: mlx5 core device
2121 * @in: inbox mlx5_ifc command buffer
2122 * @in_size: inbox buffer size
2123 * @out: outbox mlx5_ifc buffer
2124 * @out_size: outbox size
2125 *
2126 * @return: 0 if no error, FW command execution was successful
2127 * and outbox status is ok.
2128 */
mlx5_cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2129 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
2130 int out_size)
2131 {
2132 int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
2133
2134 return mlx5_cmd_check(dev, err, in, out);
2135 }
2136 EXPORT_SYMBOL(mlx5_cmd_exec);
2137
2138 /**
2139 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion
2140 * Needed for driver force teardown, when command completion EQ
2141 * will not be available to complete the command
2142 *
2143 * @dev: mlx5 core device
2144 * @in: inbox mlx5_ifc command buffer
2145 * @in_size: inbox buffer size
2146 * @out: outbox mlx5_ifc buffer
2147 * @out_size: outbox size
2148 *
2149 * @return: 0 if no error, FW command execution was successful
2150 * and outbox status is ok.
2151 */
mlx5_cmd_exec_polling(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2152 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
2153 void *out, int out_size)
2154 {
2155 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);
2156 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2157 u16 opcode = in_to_opcode(in);
2158
2159 err = cmd_status_err(dev, err, opcode, op_mod, out);
2160 return mlx5_cmd_check(dev, err, in, out);
2161 }
2162 EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2163
mlx5_cmd_init_async_ctx(struct mlx5_core_dev * dev,struct mlx5_async_ctx * ctx)2164 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2165 struct mlx5_async_ctx *ctx)
2166 {
2167 ctx->dev = dev;
2168 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2169 atomic_set(&ctx->num_inflight, 1);
2170 init_completion(&ctx->inflight_done);
2171 }
2172 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2173
2174 /**
2175 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2176 * @ctx: The ctx to clean
2177 *
2178 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2179 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2180 * the call mlx5_cleanup_async_ctx().
2181 */
mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx * ctx)2182 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2183 {
2184 if (!atomic_dec_and_test(&ctx->num_inflight))
2185 wait_for_completion(&ctx->inflight_done);
2186 }
2187 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2188
mlx5_cmd_exec_cb_handler(int status,void * _work)2189 static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2190 {
2191 struct mlx5_async_work *work = _work;
2192 struct mlx5_async_ctx *ctx;
2193 struct mlx5_core_dev *dev;
2194 bool throttle_locked;
2195 bool unpriv_locked;
2196
2197 ctx = work->ctx;
2198 dev = ctx->dev;
2199 throttle_locked = work->throttle_locked;
2200 unpriv_locked = work->unpriv_locked;
2201 status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out);
2202 work->user_callback(status, work);
2203 /* Can't access "work" from this point on. It could have been freed in
2204 * the callback.
2205 */
2206 if (throttle_locked)
2207 up(&dev->cmd.vars.throttle_sem);
2208 if (unpriv_locked)
2209 up(&dev->cmd.vars.unprivileged_sem);
2210 if (atomic_dec_and_test(&ctx->num_inflight))
2211 complete(&ctx->inflight_done);
2212 }
2213
mlx5_cmd_exec_cb(struct mlx5_async_ctx * ctx,void * in,int in_size,void * out,int out_size,mlx5_async_cbk_t callback,struct mlx5_async_work * work)2214 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2215 void *out, int out_size, mlx5_async_cbk_t callback,
2216 struct mlx5_async_work *work)
2217 {
2218 struct mlx5_core_dev *dev = ctx->dev;
2219 u16 uid;
2220 int ret;
2221
2222 work->ctx = ctx;
2223 work->user_callback = callback;
2224 work->opcode = in_to_opcode(in);
2225 work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2226 work->out = out;
2227 work->throttle_locked = false;
2228 work->unpriv_locked = false;
2229 uid = in_to_uid(in);
2230
2231 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2232 return -EIO;
2233
2234 if (uid && mlx5_has_privileged_uid(dev)) {
2235 if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
2236 if (down_trylock(&dev->cmd.vars.unprivileged_sem)) {
2237 ret = -EBUSY;
2238 goto dec_num_inflight;
2239 }
2240 work->unpriv_locked = true;
2241 }
2242 } else if (mlx5_cmd_is_throttle_opcode(in_to_opcode(in))) {
2243 if (down_trylock(&dev->cmd.vars.throttle_sem)) {
2244 ret = -EBUSY;
2245 goto dec_num_inflight;
2246 }
2247 work->throttle_locked = true;
2248 }
2249
2250 ret = cmd_exec(dev, in, in_size, out, out_size,
2251 mlx5_cmd_exec_cb_handler, work, false);
2252 if (ret)
2253 goto sem_up;
2254
2255 return 0;
2256
2257 sem_up:
2258 if (work->throttle_locked)
2259 up(&dev->cmd.vars.throttle_sem);
2260 if (work->unpriv_locked)
2261 up(&dev->cmd.vars.unprivileged_sem);
2262 dec_num_inflight:
2263 if (atomic_dec_and_test(&ctx->num_inflight))
2264 complete(&ctx->inflight_done);
2265
2266 return ret;
2267 }
2268 EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2269
mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev * dev,struct mlx5_cmd_allow_other_vhca_access_attr * attr)2270 int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,
2271 struct mlx5_cmd_allow_other_vhca_access_attr *attr)
2272 {
2273 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};
2274 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};
2275 void *key;
2276
2277 MLX5_SET(allow_other_vhca_access_in,
2278 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);
2279 MLX5_SET(allow_other_vhca_access_in,
2280 in, object_type_to_be_accessed, attr->obj_type);
2281 MLX5_SET(allow_other_vhca_access_in,
2282 in, object_id_to_be_accessed, attr->obj_id);
2283
2284 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);
2285 memcpy(key, attr->access_key, sizeof(attr->access_key));
2286
2287 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2288 }
2289
mlx5_cmd_alias_obj_create(struct mlx5_core_dev * dev,struct mlx5_cmd_alias_obj_create_attr * alias_attr,u32 * obj_id)2290 int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
2291 struct mlx5_cmd_alias_obj_create_attr *alias_attr,
2292 u32 *obj_id)
2293 {
2294 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2295 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};
2296 void *param;
2297 void *attr;
2298 void *key;
2299 int ret;
2300
2301 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);
2302 MLX5_SET(general_obj_in_cmd_hdr,
2303 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2304 MLX5_SET(general_obj_in_cmd_hdr,
2305 attr, obj_type, alias_attr->obj_type);
2306 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
2307 MLX5_SET(general_obj_create_param, param, alias_object, 1);
2308
2309 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
2310 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
2311 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
2312
2313 key = MLX5_ADDR_OF(alias_context, attr, access_key);
2314 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));
2315
2316 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2317 if (ret)
2318 return ret;
2319
2320 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2321
2322 return 0;
2323 }
2324
mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev * dev,u32 obj_id,u16 obj_type)2325 int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,
2326 u16 obj_type)
2327 {
2328 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2329 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
2330
2331 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
2332 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);
2333 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
2334
2335 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2336 }
2337
destroy_msg_cache(struct mlx5_core_dev * dev)2338 static void destroy_msg_cache(struct mlx5_core_dev *dev)
2339 {
2340 struct cmd_msg_cache *ch;
2341 struct mlx5_cmd_msg *msg;
2342 struct mlx5_cmd_msg *n;
2343 int i;
2344
2345 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
2346 ch = &dev->cmd.cache[i];
2347 list_for_each_entry_safe(msg, n, &ch->head, list) {
2348 list_del(&msg->list);
2349 mlx5_free_cmd_msg(dev, msg);
2350 }
2351 }
2352 }
2353
2354 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2355 512, 32, 16, 8, 2
2356 };
2357
2358 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2359 16 + MLX5_CMD_DATA_BLOCK_SIZE,
2360 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2361 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2362 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2363 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2364 };
2365
create_msg_cache(struct mlx5_core_dev * dev)2366 static void create_msg_cache(struct mlx5_core_dev *dev)
2367 {
2368 struct mlx5_cmd *cmd = &dev->cmd;
2369 struct cmd_msg_cache *ch;
2370 struct mlx5_cmd_msg *msg;
2371 int i;
2372 int k;
2373
2374 /* Initialize and fill the caches with initial entries */
2375 for (k = 0; k < dev->profile.num_cmd_caches; k++) {
2376 ch = &cmd->cache[k];
2377 spin_lock_init(&ch->lock);
2378 INIT_LIST_HEAD(&ch->head);
2379 ch->num_ent = cmd_cache_num_ent[k];
2380 ch->max_inbox_size = cmd_cache_ent_size[k];
2381 for (i = 0; i < ch->num_ent; i++) {
2382 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2383 ch->max_inbox_size, 0);
2384 if (IS_ERR(msg))
2385 break;
2386 msg->parent = ch;
2387 list_add_tail(&msg->list, &ch->head);
2388 }
2389 }
2390 }
2391
alloc_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2392 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2393 {
2394 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,
2395 &cmd->alloc_dma, GFP_KERNEL);
2396 if (!cmd->cmd_alloc_buf)
2397 return -ENOMEM;
2398
2399 /* make sure it is aligned to 4K */
2400 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2401 cmd->cmd_buf = cmd->cmd_alloc_buf;
2402 cmd->dma = cmd->alloc_dma;
2403 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2404 return 0;
2405 }
2406
2407 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,
2408 cmd->alloc_dma);
2409 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),
2410 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2411 &cmd->alloc_dma, GFP_KERNEL);
2412 if (!cmd->cmd_alloc_buf)
2413 return -ENOMEM;
2414
2415 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2416 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2417 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2418 return 0;
2419 }
2420
free_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2421 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2422 {
2423 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,
2424 cmd->alloc_dma);
2425 }
2426
cmdif_rev(struct mlx5_core_dev * dev)2427 static u16 cmdif_rev(struct mlx5_core_dev *dev)
2428 {
2429 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2430 }
2431
mlx5_cmd_init(struct mlx5_core_dev * dev)2432 int mlx5_cmd_init(struct mlx5_core_dev *dev)
2433 {
2434 struct mlx5_cmd *cmd = &dev->cmd;
2435
2436 cmd->checksum_disabled = 1;
2437
2438 spin_lock_init(&cmd->alloc_lock);
2439 spin_lock_init(&cmd->token_lock);
2440
2441 set_wqname(dev);
2442 cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2443 if (!cmd->wq) {
2444 mlx5_core_err(dev, "failed to create command workqueue\n");
2445 return -ENOMEM;
2446 }
2447
2448 mlx5_cmdif_debugfs_init(dev);
2449
2450 return 0;
2451 }
2452
mlx5_cmd_cleanup(struct mlx5_core_dev * dev)2453 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2454 {
2455 struct mlx5_cmd *cmd = &dev->cmd;
2456
2457 mlx5_cmdif_debugfs_cleanup(dev);
2458 destroy_workqueue(cmd->wq);
2459 }
2460
mlx5_cmd_enable(struct mlx5_core_dev * dev)2461 int mlx5_cmd_enable(struct mlx5_core_dev *dev)
2462 {
2463 int size = sizeof(struct mlx5_cmd_prot_block);
2464 int align = roundup_pow_of_two(size);
2465 struct mlx5_cmd *cmd = &dev->cmd;
2466 u32 cmd_h, cmd_l;
2467 int err;
2468
2469 memset(&cmd->vars, 0, sizeof(cmd->vars));
2470 cmd->vars.cmdif_rev = cmdif_rev(dev);
2471 if (cmd->vars.cmdif_rev != CMD_IF_REV) {
2472 mlx5_core_err(dev,
2473 "Driver cmdif rev(%d) differs from firmware's(%d)\n",
2474 CMD_IF_REV, cmd->vars.cmdif_rev);
2475 return -EINVAL;
2476 }
2477
2478 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2479 cmd->vars.log_sz = cmd_l >> 4 & 0xf;
2480 cmd->vars.log_stride = cmd_l & 0xf;
2481 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {
2482 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2483 1 << cmd->vars.log_sz);
2484 return -EINVAL;
2485 }
2486
2487 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2488 mlx5_core_err(dev, "command queue size overflow\n");
2489 return -EINVAL;
2490 }
2491
2492 cmd->state = MLX5_CMDIF_STATE_DOWN;
2493 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;
2494 cmd->vars.bitmask = MLX5_CMD_MASK;
2495
2496 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds);
2497 sema_init(&cmd->vars.pages_sem, 1);
2498 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2499 sema_init(&cmd->vars.unprivileged_sem,
2500 DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2501
2502 xa_init(&cmd->vars.privileged_uids);
2503
2504 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);
2505 if (!cmd->pool) {
2506 err = -ENOMEM;
2507 goto err_destroy_xa;
2508 }
2509
2510 err = alloc_cmd_page(dev, cmd);
2511 if (err)
2512 goto err_free_pool;
2513
2514 cmd_h = (u32)((u64)(cmd->dma) >> 32);
2515 cmd_l = (u32)(cmd->dma);
2516 if (cmd_l & 0xfff) {
2517 mlx5_core_err(dev, "invalid command queue address\n");
2518 err = -ENOMEM;
2519 goto err_cmd_page;
2520 }
2521
2522 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2523 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2524
2525 /* Make sure firmware sees the complete address before we proceed */
2526 wmb();
2527
2528 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2529
2530 cmd->mode = CMD_MODE_POLLING;
2531 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2532
2533 create_msg_cache(dev);
2534 create_debugfs_files(dev);
2535
2536 return 0;
2537
2538 err_cmd_page:
2539 free_cmd_page(dev, cmd);
2540 err_free_pool:
2541 dma_pool_destroy(cmd->pool);
2542 err_destroy_xa:
2543 xa_destroy(&dev->cmd.vars.privileged_uids);
2544 return err;
2545 }
2546
mlx5_cmd_disable(struct mlx5_core_dev * dev)2547 void mlx5_cmd_disable(struct mlx5_core_dev *dev)
2548 {
2549 struct mlx5_cmd *cmd = &dev->cmd;
2550
2551 flush_workqueue(cmd->wq);
2552 clean_debug_files(dev);
2553 destroy_msg_cache(dev);
2554 free_cmd_page(dev, cmd);
2555 dma_pool_destroy(cmd->pool);
2556 xa_destroy(&dev->cmd.vars.privileged_uids);
2557 }
2558
mlx5_cmd_set_state(struct mlx5_core_dev * dev,enum mlx5_cmdif_state cmdif_state)2559 void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2560 enum mlx5_cmdif_state cmdif_state)
2561 {
2562 dev->cmd.state = cmdif_state;
2563 }
2564
mlx5_cmd_add_privileged_uid(struct mlx5_core_dev * dev,u16 uid)2565 int mlx5_cmd_add_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2566 {
2567 return xa_insert(&dev->cmd.vars.privileged_uids, uid,
2568 xa_mk_value(uid), GFP_KERNEL);
2569 }
2570 EXPORT_SYMBOL(mlx5_cmd_add_privileged_uid);
2571
mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev * dev,u16 uid)2572 void mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2573 {
2574 void *data = xa_erase(&dev->cmd.vars.privileged_uids, uid);
2575
2576 WARN(!data, "Privileged UID %u does not exist\n", uid);
2577 }
2578 EXPORT_SYMBOL(mlx5_cmd_remove_privileged_uid);
2579