1 /*
2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/errno.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/slab.h>
38 #include <linux/delay.h>
39 #include <linux/random.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/eq.h>
42 #include <linux/debugfs.h>
43
44 #include "mlx5_core.h"
45 #include "lib/eq.h"
46 #include "lib/tout.h"
47 #define CREATE_TRACE_POINTS
48 #include "diag/cmd_tracepoint.h"
49
50 struct mlx5_ifc_mbox_out_bits {
51 u8 status[0x8];
52 u8 reserved_at_8[0x18];
53
54 u8 syndrome[0x20];
55
56 u8 reserved_at_40[0x40];
57 };
58
59 struct mlx5_ifc_mbox_in_bits {
60 u8 opcode[0x10];
61 u8 uid[0x10];
62
63 u8 reserved_at_20[0x10];
64 u8 op_mod[0x10];
65
66 u8 reserved_at_40[0x40];
67 };
68
69 enum {
70 CMD_IF_REV = 5,
71 };
72
73 enum {
74 CMD_MODE_POLLING,
75 CMD_MODE_EVENTS
76 };
77
78 enum {
79 MLX5_CMD_DELIVERY_STAT_OK = 0x0,
80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
90 };
91
in_to_opcode(void * in)92 static u16 in_to_opcode(void *in)
93 {
94 return MLX5_GET(mbox_in, in, opcode);
95 }
96
in_to_uid(void * in)97 static u16 in_to_uid(void *in)
98 {
99 return MLX5_GET(mbox_in, in, uid);
100 }
101
102 /* Returns true for opcodes that might be triggered very frequently and throttle
103 * the command interface. Limit their command slots usage.
104 */
mlx5_cmd_is_throttle_opcode(u16 op)105 static bool mlx5_cmd_is_throttle_opcode(u16 op)
106 {
107 switch (op) {
108 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
109 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
110 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
111 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
112 case MLX5_CMD_OP_SYNC_CRYPTO:
113 return true;
114 }
115 return false;
116 }
117
118 static struct mlx5_cmd_work_ent *
cmd_alloc_ent(struct mlx5_cmd * cmd,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t cbk,void * context,int page_queue)119 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
120 struct mlx5_cmd_msg *out, void *uout, int uout_size,
121 mlx5_cmd_cbk_t cbk, void *context, int page_queue)
122 {
123 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
124 struct mlx5_cmd_work_ent *ent;
125
126 ent = kzalloc_obj(*ent, alloc_flags);
127 if (!ent)
128 return ERR_PTR(-ENOMEM);
129
130 ent->idx = -EINVAL;
131 ent->in = in;
132 ent->out = out;
133 ent->uout = uout;
134 ent->uout_size = uout_size;
135 ent->callback = cbk;
136 ent->context = context;
137 ent->cmd = cmd;
138 ent->page_queue = page_queue;
139 ent->op = in_to_opcode(in->first.data);
140 refcount_set(&ent->refcnt, 1);
141
142 return ent;
143 }
144
cmd_free_ent(struct mlx5_cmd_work_ent * ent)145 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
146 {
147 kfree(ent);
148 }
149
alloc_token(struct mlx5_cmd * cmd)150 static u8 alloc_token(struct mlx5_cmd *cmd)
151 {
152 u8 token;
153
154 spin_lock(&cmd->token_lock);
155 cmd->token++;
156 if (cmd->token == 0)
157 cmd->token++;
158 token = cmd->token;
159 spin_unlock(&cmd->token_lock);
160
161 return token;
162 }
163
cmd_alloc_index(struct mlx5_cmd * cmd,struct mlx5_cmd_work_ent * ent)164 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
165 {
166 unsigned long flags;
167 int ret;
168
169 spin_lock_irqsave(&cmd->alloc_lock, flags);
170 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds);
171 if (ret < cmd->vars.max_reg_cmds) {
172 clear_bit(ret, &cmd->vars.bitmask);
173 ent->idx = ret;
174 cmd->ent_arr[ent->idx] = ent;
175 }
176 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
177
178 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;
179 }
180
cmd_free_index(struct mlx5_cmd * cmd,int idx)181 static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
182 {
183 lockdep_assert_held(&cmd->alloc_lock);
184 cmd->ent_arr[idx] = NULL;
185 set_bit(idx, &cmd->vars.bitmask);
186 }
187
cmd_ent_get(struct mlx5_cmd_work_ent * ent)188 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
189 {
190 refcount_inc(&ent->refcnt);
191 }
192
cmd_ent_put(struct mlx5_cmd_work_ent * ent)193 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
194 {
195 struct mlx5_cmd *cmd = ent->cmd;
196 unsigned long flags;
197
198 spin_lock_irqsave(&cmd->alloc_lock, flags);
199 if (!refcount_dec_and_test(&ent->refcnt))
200 goto out;
201
202 if (ent->idx >= 0) {
203 cmd_free_index(cmd, ent->idx);
204 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
205 }
206
207 cmd_free_ent(ent);
208 out:
209 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
210 }
211
get_inst(struct mlx5_cmd * cmd,int idx)212 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
213 {
214 return cmd->cmd_buf + (idx << cmd->vars.log_stride);
215 }
216
mlx5_calc_cmd_blocks(struct mlx5_cmd_msg * msg)217 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
218 {
219 int size = msg->len;
220 int blen = size - min_t(int, sizeof(msg->first.data), size);
221
222 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
223 }
224
xor8_buf(void * buf,size_t offset,int len)225 static u8 xor8_buf(void *buf, size_t offset, int len)
226 {
227 u8 *ptr = buf;
228 u8 sum = 0;
229 int i;
230 int end = len + offset;
231
232 for (i = offset; i < end; i++)
233 sum ^= ptr[i];
234
235 return sum;
236 }
237
verify_block_sig(struct mlx5_cmd_prot_block * block)238 static int verify_block_sig(struct mlx5_cmd_prot_block *block)
239 {
240 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
241 int xor_len = sizeof(*block) - sizeof(block->data) - 1;
242
243 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
244 return -EHWPOISON;
245
246 if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
247 return -EHWPOISON;
248
249 return 0;
250 }
251
calc_block_sig(struct mlx5_cmd_prot_block * block)252 static void calc_block_sig(struct mlx5_cmd_prot_block *block)
253 {
254 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
255 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
256
257 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
258 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
259 }
260
calc_chain_sig(struct mlx5_cmd_msg * msg)261 static void calc_chain_sig(struct mlx5_cmd_msg *msg)
262 {
263 struct mlx5_cmd_mailbox *next = msg->next;
264 int n = mlx5_calc_cmd_blocks(msg);
265 int i = 0;
266
267 for (i = 0; i < n && next; i++) {
268 calc_block_sig(next->buf);
269 next = next->next;
270 }
271 }
272
set_signature(struct mlx5_cmd_work_ent * ent,int csum)273 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
274 {
275 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay));
276 if (csum) {
277 calc_chain_sig(ent->in);
278 calc_chain_sig(ent->out);
279 }
280 }
281
poll_timeout(struct mlx5_cmd_work_ent * ent)282 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
283 {
284 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
285 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
286 unsigned long poll_end;
287 u8 own;
288
289 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);
290
291 do {
292 own = READ_ONCE(ent->lay->status_own);
293 if (!(own & CMD_OWNER_HW)) {
294 ent->ret = 0;
295 return;
296 }
297 cond_resched();
298 if (mlx5_cmd_is_down(dev)) {
299 ent->ret = -ENXIO;
300 return;
301 }
302 } while (time_before(jiffies, poll_end));
303
304 ent->ret = -ETIMEDOUT;
305 }
306
verify_signature(struct mlx5_cmd_work_ent * ent)307 static int verify_signature(struct mlx5_cmd_work_ent *ent)
308 {
309 struct mlx5_cmd_mailbox *next = ent->out->next;
310 int n = mlx5_calc_cmd_blocks(ent->out);
311 int err;
312 u8 sig;
313 int i = 0;
314
315 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
316 if (sig != 0xff)
317 return -EHWPOISON;
318
319 for (i = 0; i < n && next; i++) {
320 err = verify_block_sig(next->buf);
321 if (err)
322 return -EHWPOISON;
323
324 next = next->next;
325 }
326
327 return 0;
328 }
329
dump_buf(void * buf,int size,int data_only,int offset,int idx)330 static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
331 {
332 __be32 *p = buf;
333 int i;
334
335 for (i = 0; i < size; i += 16) {
336 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
337 be32_to_cpu(p[0]), be32_to_cpu(p[1]),
338 be32_to_cpu(p[2]), be32_to_cpu(p[3]));
339 p += 4;
340 offset += 16;
341 }
342 if (!data_only)
343 pr_debug("\n");
344 }
345
mlx5_internal_err_ret_value(struct mlx5_core_dev * dev,u16 op,u32 * synd,u8 * status)346 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
347 u32 *synd, u8 *status)
348 {
349 *synd = 0;
350 *status = 0;
351
352 switch (op) {
353 case MLX5_CMD_OP_TEARDOWN_HCA:
354 case MLX5_CMD_OP_DISABLE_HCA:
355 case MLX5_CMD_OP_MANAGE_PAGES:
356 case MLX5_CMD_OP_DESTROY_MKEY:
357 case MLX5_CMD_OP_DESTROY_EQ:
358 case MLX5_CMD_OP_DESTROY_CQ:
359 case MLX5_CMD_OP_DESTROY_QP:
360 case MLX5_CMD_OP_DESTROY_PSV:
361 case MLX5_CMD_OP_DESTROY_SRQ:
362 case MLX5_CMD_OP_DESTROY_XRC_SRQ:
363 case MLX5_CMD_OP_DESTROY_XRQ:
364 case MLX5_CMD_OP_DESTROY_DCT:
365 case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
366 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
367 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
368 case MLX5_CMD_OP_DEALLOC_PD:
369 case MLX5_CMD_OP_DEALLOC_UAR:
370 case MLX5_CMD_OP_DETACH_FROM_MCG:
371 case MLX5_CMD_OP_DEALLOC_XRCD:
372 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
373 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
374 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
375 case MLX5_CMD_OP_DESTROY_LAG:
376 case MLX5_CMD_OP_DESTROY_VPORT_LAG:
377 case MLX5_CMD_OP_DESTROY_TIR:
378 case MLX5_CMD_OP_DESTROY_SQ:
379 case MLX5_CMD_OP_DESTROY_RQ:
380 case MLX5_CMD_OP_DESTROY_RMP:
381 case MLX5_CMD_OP_DESTROY_TIS:
382 case MLX5_CMD_OP_DESTROY_RQT:
383 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
384 case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
385 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
386 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
387 case MLX5_CMD_OP_2ERR_QP:
388 case MLX5_CMD_OP_2RST_QP:
389 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
390 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
391 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
392 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
393 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
394 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
395 case MLX5_CMD_OP_FPGA_DESTROY_QP:
396 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
397 case MLX5_CMD_OP_DEALLOC_MEMIC:
398 case MLX5_CMD_OP_PAGE_FAULT_RESUME:
399 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
400 case MLX5_CMD_OP_DEALLOC_SF:
401 case MLX5_CMD_OP_DESTROY_UCTX:
402 case MLX5_CMD_OP_DESTROY_UMEM:
403 case MLX5_CMD_OP_MODIFY_RQT:
404 return MLX5_CMD_STAT_OK;
405
406 case MLX5_CMD_OP_QUERY_HCA_CAP:
407 case MLX5_CMD_OP_QUERY_ADAPTER:
408 case MLX5_CMD_OP_INIT_HCA:
409 case MLX5_CMD_OP_ENABLE_HCA:
410 case MLX5_CMD_OP_QUERY_PAGES:
411 case MLX5_CMD_OP_SET_HCA_CAP:
412 case MLX5_CMD_OP_QUERY_ISSI:
413 case MLX5_CMD_OP_SET_ISSI:
414 case MLX5_CMD_OP_CREATE_MKEY:
415 case MLX5_CMD_OP_QUERY_MKEY:
416 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
417 case MLX5_CMD_OP_CREATE_EQ:
418 case MLX5_CMD_OP_QUERY_EQ:
419 case MLX5_CMD_OP_GEN_EQE:
420 case MLX5_CMD_OP_CREATE_CQ:
421 case MLX5_CMD_OP_QUERY_CQ:
422 case MLX5_CMD_OP_MODIFY_CQ:
423 case MLX5_CMD_OP_CREATE_QP:
424 case MLX5_CMD_OP_RST2INIT_QP:
425 case MLX5_CMD_OP_INIT2RTR_QP:
426 case MLX5_CMD_OP_RTR2RTS_QP:
427 case MLX5_CMD_OP_RTS2RTS_QP:
428 case MLX5_CMD_OP_SQERR2RTS_QP:
429 case MLX5_CMD_OP_QUERY_QP:
430 case MLX5_CMD_OP_SQD_RTS_QP:
431 case MLX5_CMD_OP_INIT2INIT_QP:
432 case MLX5_CMD_OP_CREATE_PSV:
433 case MLX5_CMD_OP_CREATE_SRQ:
434 case MLX5_CMD_OP_QUERY_SRQ:
435 case MLX5_CMD_OP_ARM_RQ:
436 case MLX5_CMD_OP_CREATE_XRC_SRQ:
437 case MLX5_CMD_OP_QUERY_XRC_SRQ:
438 case MLX5_CMD_OP_ARM_XRC_SRQ:
439 case MLX5_CMD_OP_CREATE_XRQ:
440 case MLX5_CMD_OP_QUERY_XRQ:
441 case MLX5_CMD_OP_ARM_XRQ:
442 case MLX5_CMD_OP_CREATE_DCT:
443 case MLX5_CMD_OP_DRAIN_DCT:
444 case MLX5_CMD_OP_QUERY_DCT:
445 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
446 case MLX5_CMD_OP_QUERY_VPORT_STATE:
447 case MLX5_CMD_OP_MODIFY_VPORT_STATE:
448 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
449 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
450 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
451 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
452 case MLX5_CMD_OP_SET_ROCE_ADDRESS:
453 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
454 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
455 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
456 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
457 case MLX5_CMD_OP_QUERY_VNIC_ENV:
458 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
459 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
460 case MLX5_CMD_OP_QUERY_Q_COUNTER:
461 case MLX5_CMD_OP_SET_MONITOR_COUNTER:
462 case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
463 case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
464 case MLX5_CMD_OP_QUERY_RATE_LIMIT:
465 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
466 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
467 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
468 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
469 case MLX5_CMD_OP_ALLOC_PD:
470 case MLX5_CMD_OP_ALLOC_UAR:
471 case MLX5_CMD_OP_CONFIG_INT_MODERATION:
472 case MLX5_CMD_OP_ACCESS_REG:
473 case MLX5_CMD_OP_ATTACH_TO_MCG:
474 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
475 case MLX5_CMD_OP_MAD_IFC:
476 case MLX5_CMD_OP_QUERY_MAD_DEMUX:
477 case MLX5_CMD_OP_SET_MAD_DEMUX:
478 case MLX5_CMD_OP_NOP:
479 case MLX5_CMD_OP_ALLOC_XRCD:
480 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
481 case MLX5_CMD_OP_QUERY_CONG_STATUS:
482 case MLX5_CMD_OP_MODIFY_CONG_STATUS:
483 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
484 case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
485 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
486 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
487 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
488 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
489 case MLX5_CMD_OP_CREATE_LAG:
490 case MLX5_CMD_OP_MODIFY_LAG:
491 case MLX5_CMD_OP_QUERY_LAG:
492 case MLX5_CMD_OP_CREATE_VPORT_LAG:
493 case MLX5_CMD_OP_CREATE_TIR:
494 case MLX5_CMD_OP_MODIFY_TIR:
495 case MLX5_CMD_OP_QUERY_TIR:
496 case MLX5_CMD_OP_CREATE_SQ:
497 case MLX5_CMD_OP_MODIFY_SQ:
498 case MLX5_CMD_OP_QUERY_SQ:
499 case MLX5_CMD_OP_CREATE_RQ:
500 case MLX5_CMD_OP_MODIFY_RQ:
501 case MLX5_CMD_OP_QUERY_RQ:
502 case MLX5_CMD_OP_CREATE_RMP:
503 case MLX5_CMD_OP_MODIFY_RMP:
504 case MLX5_CMD_OP_QUERY_RMP:
505 case MLX5_CMD_OP_CREATE_TIS:
506 case MLX5_CMD_OP_MODIFY_TIS:
507 case MLX5_CMD_OP_QUERY_TIS:
508 case MLX5_CMD_OP_CREATE_RQT:
509 case MLX5_CMD_OP_QUERY_RQT:
510
511 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
512 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
513 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
514 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
515 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
516 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
517 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
518 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
519 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
520 case MLX5_CMD_OP_FPGA_CREATE_QP:
521 case MLX5_CMD_OP_FPGA_MODIFY_QP:
522 case MLX5_CMD_OP_FPGA_QUERY_QP:
523 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
524 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
525 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
526 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
527 case MLX5_CMD_OP_CREATE_UCTX:
528 case MLX5_CMD_OP_CREATE_UMEM:
529 case MLX5_CMD_OP_ALLOC_MEMIC:
530 case MLX5_CMD_OP_MODIFY_XRQ:
531 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
532 case MLX5_CMD_OP_QUERY_VHCA_STATE:
533 case MLX5_CMD_OP_MODIFY_VHCA_STATE:
534 case MLX5_CMD_OP_ALLOC_SF:
535 case MLX5_CMD_OP_SUSPEND_VHCA:
536 case MLX5_CMD_OP_RESUME_VHCA:
537 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
538 case MLX5_CMD_OP_SAVE_VHCA_STATE:
539 case MLX5_CMD_OP_LOAD_VHCA_STATE:
540 case MLX5_CMD_OP_SYNC_CRYPTO:
541 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:
542 *status = MLX5_DRIVER_STATUS_ABORTED;
543 *synd = MLX5_DRIVER_SYND;
544 return -ENOLINK;
545 default:
546 mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
547 return -EINVAL;
548 }
549 }
550
mlx5_command_str(int command)551 const char *mlx5_command_str(int command)
552 {
553 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
554
555 switch (command) {
556 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
557 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
558 MLX5_COMMAND_STR_CASE(INIT_HCA);
559 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
560 MLX5_COMMAND_STR_CASE(ENABLE_HCA);
561 MLX5_COMMAND_STR_CASE(DISABLE_HCA);
562 MLX5_COMMAND_STR_CASE(QUERY_PAGES);
563 MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
564 MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
565 MLX5_COMMAND_STR_CASE(QUERY_ISSI);
566 MLX5_COMMAND_STR_CASE(SET_ISSI);
567 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
568 MLX5_COMMAND_STR_CASE(CREATE_MKEY);
569 MLX5_COMMAND_STR_CASE(QUERY_MKEY);
570 MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
571 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
572 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
573 MLX5_COMMAND_STR_CASE(CREATE_EQ);
574 MLX5_COMMAND_STR_CASE(DESTROY_EQ);
575 MLX5_COMMAND_STR_CASE(QUERY_EQ);
576 MLX5_COMMAND_STR_CASE(GEN_EQE);
577 MLX5_COMMAND_STR_CASE(CREATE_CQ);
578 MLX5_COMMAND_STR_CASE(DESTROY_CQ);
579 MLX5_COMMAND_STR_CASE(QUERY_CQ);
580 MLX5_COMMAND_STR_CASE(MODIFY_CQ);
581 MLX5_COMMAND_STR_CASE(CREATE_QP);
582 MLX5_COMMAND_STR_CASE(DESTROY_QP);
583 MLX5_COMMAND_STR_CASE(RST2INIT_QP);
584 MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
585 MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
586 MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
587 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
588 MLX5_COMMAND_STR_CASE(2ERR_QP);
589 MLX5_COMMAND_STR_CASE(2RST_QP);
590 MLX5_COMMAND_STR_CASE(QUERY_QP);
591 MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
592 MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
593 MLX5_COMMAND_STR_CASE(CREATE_PSV);
594 MLX5_COMMAND_STR_CASE(DESTROY_PSV);
595 MLX5_COMMAND_STR_CASE(CREATE_SRQ);
596 MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
597 MLX5_COMMAND_STR_CASE(QUERY_SRQ);
598 MLX5_COMMAND_STR_CASE(ARM_RQ);
599 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
600 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
601 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
602 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
603 MLX5_COMMAND_STR_CASE(CREATE_DCT);
604 MLX5_COMMAND_STR_CASE(DESTROY_DCT);
605 MLX5_COMMAND_STR_CASE(DRAIN_DCT);
606 MLX5_COMMAND_STR_CASE(QUERY_DCT);
607 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
608 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
609 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
610 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
611 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
612 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
613 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
614 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
615 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
616 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
617 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
618 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
619 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
620 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
621 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
622 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
623 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
624 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
625 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
626 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
627 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
628 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
629 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
630 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
631 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
632 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
633 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
634 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
635 MLX5_COMMAND_STR_CASE(ALLOC_PD);
636 MLX5_COMMAND_STR_CASE(DEALLOC_PD);
637 MLX5_COMMAND_STR_CASE(ALLOC_UAR);
638 MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
639 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
640 MLX5_COMMAND_STR_CASE(ACCESS_REG);
641 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
642 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
643 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
644 MLX5_COMMAND_STR_CASE(MAD_IFC);
645 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
646 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
647 MLX5_COMMAND_STR_CASE(NOP);
648 MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
649 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
650 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
651 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
652 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
653 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
654 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
655 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
656 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
657 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
658 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
659 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
660 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
661 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
662 MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
663 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
664 MLX5_COMMAND_STR_CASE(CREATE_LAG);
665 MLX5_COMMAND_STR_CASE(MODIFY_LAG);
666 MLX5_COMMAND_STR_CASE(QUERY_LAG);
667 MLX5_COMMAND_STR_CASE(DESTROY_LAG);
668 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
669 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
670 MLX5_COMMAND_STR_CASE(CREATE_TIR);
671 MLX5_COMMAND_STR_CASE(MODIFY_TIR);
672 MLX5_COMMAND_STR_CASE(DESTROY_TIR);
673 MLX5_COMMAND_STR_CASE(QUERY_TIR);
674 MLX5_COMMAND_STR_CASE(CREATE_SQ);
675 MLX5_COMMAND_STR_CASE(MODIFY_SQ);
676 MLX5_COMMAND_STR_CASE(DESTROY_SQ);
677 MLX5_COMMAND_STR_CASE(QUERY_SQ);
678 MLX5_COMMAND_STR_CASE(CREATE_RQ);
679 MLX5_COMMAND_STR_CASE(MODIFY_RQ);
680 MLX5_COMMAND_STR_CASE(DESTROY_RQ);
681 MLX5_COMMAND_STR_CASE(QUERY_RQ);
682 MLX5_COMMAND_STR_CASE(CREATE_RMP);
683 MLX5_COMMAND_STR_CASE(MODIFY_RMP);
684 MLX5_COMMAND_STR_CASE(DESTROY_RMP);
685 MLX5_COMMAND_STR_CASE(QUERY_RMP);
686 MLX5_COMMAND_STR_CASE(CREATE_TIS);
687 MLX5_COMMAND_STR_CASE(MODIFY_TIS);
688 MLX5_COMMAND_STR_CASE(DESTROY_TIS);
689 MLX5_COMMAND_STR_CASE(QUERY_TIS);
690 MLX5_COMMAND_STR_CASE(CREATE_RQT);
691 MLX5_COMMAND_STR_CASE(MODIFY_RQT);
692 MLX5_COMMAND_STR_CASE(DESTROY_RQT);
693 MLX5_COMMAND_STR_CASE(QUERY_RQT);
694 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
695 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
696 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
697 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
698 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
699 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
700 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
701 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
702 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
703 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
704 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
705 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
706 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
707 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
708 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
709 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
710 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
711 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
712 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
713 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
714 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
715 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
716 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
717 MLX5_COMMAND_STR_CASE(CREATE_XRQ);
718 MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
719 MLX5_COMMAND_STR_CASE(QUERY_XRQ);
720 MLX5_COMMAND_STR_CASE(ARM_XRQ);
721 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
722 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
723 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
724 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
725 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
726 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
727 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
728 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
729 MLX5_COMMAND_STR_CASE(CREATE_UCTX);
730 MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
731 MLX5_COMMAND_STR_CASE(CREATE_UMEM);
732 MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
733 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
734 MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
735 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
736 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
737 MLX5_COMMAND_STR_CASE(ALLOC_SF);
738 MLX5_COMMAND_STR_CASE(DEALLOC_SF);
739 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
740 MLX5_COMMAND_STR_CASE(RESUME_VHCA);
741 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
742 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
743 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
744 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);
745 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);
746 default: return "unknown command opcode";
747 }
748 }
749
cmd_status_str(u8 status)750 static const char *cmd_status_str(u8 status)
751 {
752 switch (status) {
753 case MLX5_CMD_STAT_OK:
754 return "OK";
755 case MLX5_CMD_STAT_INT_ERR:
756 return "internal error";
757 case MLX5_CMD_STAT_BAD_OP_ERR:
758 return "bad operation";
759 case MLX5_CMD_STAT_BAD_PARAM_ERR:
760 return "bad parameter";
761 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
762 return "bad system state";
763 case MLX5_CMD_STAT_BAD_RES_ERR:
764 return "bad resource";
765 case MLX5_CMD_STAT_RES_BUSY:
766 return "resource busy";
767 case MLX5_CMD_STAT_NOT_READY:
768 return "FW not ready";
769 case MLX5_CMD_STAT_LIM_ERR:
770 return "limits exceeded";
771 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
772 return "bad resource state";
773 case MLX5_CMD_STAT_IX_ERR:
774 return "bad index";
775 case MLX5_CMD_STAT_NO_RES_ERR:
776 return "no resources";
777 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
778 return "bad input length";
779 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
780 return "bad output length";
781 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
782 return "bad QP state";
783 case MLX5_CMD_STAT_BAD_PKT_ERR:
784 return "bad packet (discarded)";
785 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
786 return "bad size too many outstanding CQEs";
787 default:
788 return "unknown status";
789 }
790 }
791
cmd_status_to_err(u8 status)792 static int cmd_status_to_err(u8 status)
793 {
794 switch (status) {
795 case MLX5_CMD_STAT_OK: return 0;
796 case MLX5_CMD_STAT_INT_ERR: return -EIO;
797 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
798 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
799 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
800 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
801 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
802 case MLX5_CMD_STAT_NOT_READY: return -EAGAIN;
803 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;
804 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
805 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
806 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
807 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
808 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
809 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
810 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
811 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
812 default: return -EIO;
813 }
814 }
815
mlx5_cmd_out_err(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)816 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
817 {
818 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
819 u8 status = MLX5_GET(mbox_out, out, status);
820
821 mlx5_core_err_rl(dev,
822 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
823 mlx5_command_str(opcode), opcode, op_mod,
824 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
825 }
826 EXPORT_SYMBOL(mlx5_cmd_out_err);
827
cmd_status_print(struct mlx5_core_dev * dev,void * in,void * out)828 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
829 {
830 u16 opcode, op_mod;
831 u8 status;
832 u16 uid;
833
834 opcode = in_to_opcode(in);
835 op_mod = MLX5_GET(mbox_in, in, op_mod);
836 uid = in_to_uid(in);
837 status = MLX5_GET(mbox_out, out, status);
838
839 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&
840 opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)
841 mlx5_cmd_out_err(dev, opcode, op_mod, out);
842 }
843
mlx5_cmd_check(struct mlx5_core_dev * dev,int err,void * in,void * out)844 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
845 {
846 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
847 if (err == -ENXIO) {
848 u16 opcode = in_to_opcode(in);
849 u32 syndrome;
850 u8 status;
851
852 /* PCI Error, emulate command return status, for smooth reset */
853 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);
854 MLX5_SET(mbox_out, out, status, status);
855 MLX5_SET(mbox_out, out, syndrome, syndrome);
856 if (!err)
857 return 0;
858 }
859
860 /* driver or FW delivery error */
861 if (err != -EREMOTEIO && err)
862 return err;
863
864 /* check outbox status */
865 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
866 if (err)
867 cmd_status_print(dev, in, out);
868
869 return err;
870 }
871 EXPORT_SYMBOL(mlx5_cmd_check);
872
dump_command(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent,int input)873 static void dump_command(struct mlx5_core_dev *dev,
874 struct mlx5_cmd_work_ent *ent, int input)
875 {
876 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
877 struct mlx5_cmd_mailbox *next = msg->next;
878 int n = mlx5_calc_cmd_blocks(msg);
879 u16 op = ent->op;
880 int data_only;
881 u32 offset = 0;
882 int dump_len;
883 int i;
884
885 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
886 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
887
888 if (data_only)
889 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
890 "cmd[%d]: dump command data %s(0x%x) %s\n",
891 ent->idx, mlx5_command_str(op), op,
892 input ? "INPUT" : "OUTPUT");
893 else
894 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
895 ent->idx, mlx5_command_str(op), op,
896 input ? "INPUT" : "OUTPUT");
897
898 if (data_only) {
899 if (input) {
900 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
901 offset += sizeof(ent->lay->in);
902 } else {
903 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
904 offset += sizeof(ent->lay->out);
905 }
906 } else {
907 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
908 offset += sizeof(*ent->lay);
909 }
910
911 for (i = 0; i < n && next; i++) {
912 if (data_only) {
913 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
914 dump_buf(next->buf, dump_len, 1, offset, ent->idx);
915 offset += MLX5_CMD_DATA_BLOCK_SIZE;
916 } else {
917 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
918 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,
919 ent->idx);
920 offset += sizeof(struct mlx5_cmd_prot_block);
921 }
922 next = next->next;
923 }
924
925 if (data_only)
926 pr_debug("\n");
927
928 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
929 }
930
931 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
932
cb_timeout_handler(struct work_struct * work)933 static void cb_timeout_handler(struct work_struct *work)
934 {
935 struct delayed_work *dwork = to_delayed_work(work);
936 struct mlx5_cmd_work_ent *ent = container_of(dwork,
937 struct mlx5_cmd_work_ent,
938 cb_timeout_work);
939 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
940 cmd);
941
942 mlx5_cmd_eq_recover(dev);
943
944 /* Maybe got handled by eq recover ? */
945 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
946 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
947 mlx5_command_str(ent->op), ent->op);
948 goto out; /* phew, already handled */
949 }
950
951 ent->ret = -ETIMEDOUT;
952 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
953 ent->idx, mlx5_command_str(ent->op), ent->op);
954 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
955
956 out:
957 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
958 }
959
960 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
961 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
962 struct mlx5_cmd_msg *msg);
963
opcode_allowed(struct mlx5_cmd * cmd,u16 opcode)964 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
965 {
966 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
967 return true;
968
969 return cmd->allowed_opcode == opcode;
970 }
971
mlx5_cmd_is_down(struct mlx5_core_dev * dev)972 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
973 {
974 return pci_channel_offline(dev->pdev) ||
975 dev->cmd.state != MLX5_CMDIF_STATE_UP ||
976 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
977 }
978
cmd_work_handler(struct work_struct * work)979 static void cmd_work_handler(struct work_struct *work)
980 {
981 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
982 struct mlx5_cmd *cmd = ent->cmd;
983 bool poll_cmd = ent->polling;
984 struct mlx5_cmd_layout *lay;
985 struct mlx5_core_dev *dev;
986 unsigned long timeout;
987 unsigned long flags;
988 int alloc_ret;
989 int cmd_mode;
990
991 complete(&ent->handling);
992
993 dev = container_of(cmd, struct mlx5_core_dev, cmd);
994 timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
995
996 if (!ent->page_queue) {
997 if (down_timeout(&cmd->vars.sem, timeout)) {
998 mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",
999 mlx5_command_str(ent->op), ent->op);
1000 if (ent->callback) {
1001 ent->callback(-EBUSY, ent->context);
1002 mlx5_free_cmd_msg(dev, ent->out);
1003 free_msg(dev, ent->in);
1004 cmd_ent_put(ent);
1005 } else {
1006 ent->ret = -EBUSY;
1007 complete(&ent->done);
1008 }
1009 complete(&ent->slotted);
1010 return;
1011 }
1012 alloc_ret = cmd_alloc_index(cmd, ent);
1013 if (alloc_ret < 0) {
1014 mlx5_core_err_rl(dev, "failed to allocate command entry\n");
1015 if (ent->callback) {
1016 ent->callback(-EAGAIN, ent->context);
1017 mlx5_free_cmd_msg(dev, ent->out);
1018 free_msg(dev, ent->in);
1019 cmd_ent_put(ent);
1020 } else {
1021 ent->ret = -EAGAIN;
1022 complete(&ent->done);
1023 }
1024 up(&cmd->vars.sem);
1025 complete(&ent->slotted);
1026 return;
1027 }
1028 } else {
1029 down(&cmd->vars.pages_sem);
1030 ent->idx = cmd->vars.max_reg_cmds;
1031 spin_lock_irqsave(&cmd->alloc_lock, flags);
1032 clear_bit(ent->idx, &cmd->vars.bitmask);
1033 cmd->ent_arr[ent->idx] = ent;
1034 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1035 }
1036
1037 complete(&ent->slotted);
1038
1039 lay = get_inst(cmd, ent->idx);
1040 ent->lay = lay;
1041 memset(lay, 0, sizeof(*lay));
1042 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1043 if (ent->in->next)
1044 lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1045 lay->inlen = cpu_to_be32(ent->in->len);
1046 if (ent->out->next)
1047 lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1048 lay->outlen = cpu_to_be32(ent->out->len);
1049 lay->type = MLX5_PCI_CMD_XPORT;
1050 lay->token = ent->token;
1051 lay->status_own = CMD_OWNER_HW;
1052 set_signature(ent, !cmd->checksum_disabled);
1053 dump_command(dev, ent, 1);
1054 ent->ts1 = ktime_get_ns();
1055 cmd_mode = cmd->mode;
1056
1057 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))
1058 cmd_ent_get(ent);
1059 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1060
1061 cmd_ent_get(ent); /* for the _real_ FW event on completion */
1062 /* Skip sending command to fw if internal error */
1063 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1064 ent->ret = -ENXIO;
1065 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1066 return;
1067 }
1068
1069 /* ring doorbell after the descriptor is valid */
1070 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1071 wmb();
1072 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1073 /* if not in polling don't use ent after this point */
1074 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1075 poll_timeout(ent);
1076 /* make sure we read the descriptor after ownership is SW */
1077 rmb();
1078 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret);
1079 }
1080 }
1081
deliv_status_to_err(u8 status)1082 static int deliv_status_to_err(u8 status)
1083 {
1084 switch (status) {
1085 case MLX5_CMD_DELIVERY_STAT_OK:
1086 case MLX5_DRIVER_STATUS_ABORTED:
1087 return 0;
1088 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1089 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1090 return -EBADR;
1091 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1092 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1093 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1094 return -EFAULT; /* Bad address */
1095 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1096 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1097 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1098 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1099 return -ENOMSG;
1100 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1101 return -EIO;
1102 default:
1103 return -EINVAL;
1104 }
1105 }
1106
deliv_status_to_str(u8 status)1107 static const char *deliv_status_to_str(u8 status)
1108 {
1109 switch (status) {
1110 case MLX5_CMD_DELIVERY_STAT_OK:
1111 return "no errors";
1112 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1113 return "signature error";
1114 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1115 return "token error";
1116 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1117 return "bad block number";
1118 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1119 return "output pointer not aligned to block size";
1120 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1121 return "input pointer not aligned to block size";
1122 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1123 return "firmware internal error";
1124 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1125 return "command input length error";
1126 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1127 return "command output length error";
1128 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1129 return "reserved fields not cleared";
1130 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1131 return "bad command descriptor type";
1132 default:
1133 return "unknown status code";
1134 }
1135 }
1136
1137 enum {
1138 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
1139 };
1140
wait_func_handle_exec_timeout(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1141 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1142 struct mlx5_cmd_work_ent *ent)
1143 {
1144 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1145
1146 mlx5_cmd_eq_recover(dev);
1147
1148 /* Re-wait on the ent->done after executing the recovery flow. If the
1149 * recovery flow (or any other recovery flow running simultaneously)
1150 * has recovered an EQE, it should cause the entry to be completed by
1151 * the command interface.
1152 */
1153 if (wait_for_completion_timeout(&ent->done, timeout)) {
1154 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1155 mlx5_command_str(ent->op), ent->op);
1156 return;
1157 }
1158
1159 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1160 mlx5_command_str(ent->op), ent->op);
1161
1162 ent->ret = -ETIMEDOUT;
1163 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1164 }
1165
wait_func(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1166 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1167 {
1168 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1169 struct mlx5_cmd *cmd = &dev->cmd;
1170 int err;
1171
1172 if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1173 cancel_work_sync(&ent->work)) {
1174 ent->ret = -ECANCELED;
1175 goto out_err;
1176 }
1177
1178 wait_for_completion(&ent->slotted);
1179
1180 if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1181 wait_for_completion(&ent->done);
1182 else if (!wait_for_completion_timeout(&ent->done, timeout))
1183 wait_func_handle_exec_timeout(dev, ent);
1184
1185 out_err:
1186 err = ent->ret;
1187
1188 if (err == -ETIMEDOUT) {
1189 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1190 mlx5_command_str(ent->op), ent->op);
1191 } else if (err == -ECANCELED) {
1192 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1193 mlx5_command_str(ent->op), ent->op);
1194 } else if (err == -EBUSY) {
1195 mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",
1196 mlx5_command_str(ent->op), ent->op);
1197 }
1198 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1199 err, deliv_status_to_str(ent->status), ent->status);
1200
1201 return err;
1202 }
1203
1204 /* Check if all command slots are stalled (timed out and not recovered).
1205 * returns true if all slots timed out on a recent command and have not been
1206 * completed by FW yet. (stalled state)
1207 * false otherwise (at least one slot is not stalled).
1208 *
1209 * In such odd situation "all_stalled", this serves as a protection mechanism
1210 * to avoid blocking the kernel for long periods of time in case FW is not
1211 * responding to commands.
1212 */
mlx5_cmd_all_stalled(struct mlx5_core_dev * dev)1213 static bool mlx5_cmd_all_stalled(struct mlx5_core_dev *dev)
1214 {
1215 struct mlx5_cmd *cmd = &dev->cmd;
1216 bool all_stalled = true;
1217 unsigned long flags;
1218 int i;
1219
1220 spin_lock_irqsave(&cmd->alloc_lock, flags);
1221
1222 /* at least one command slot is free */
1223 if (bitmap_weight(&cmd->vars.bitmask, cmd->vars.max_reg_cmds) > 0) {
1224 all_stalled = false;
1225 goto out;
1226 }
1227
1228 for_each_clear_bit(i, &cmd->vars.bitmask, cmd->vars.max_reg_cmds) {
1229 struct mlx5_cmd_work_ent *ent = dev->cmd.ent_arr[i];
1230
1231 if (!test_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, &ent->state)) {
1232 all_stalled = false;
1233 break;
1234 }
1235 }
1236 out:
1237 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1238
1239 return all_stalled;
1240 }
1241
1242 /* Notes:
1243 * 1. Callback functions may not sleep
1244 * 2. page queue commands do not support asynchrous completion
1245 *
1246 * return value in case (!callback):
1247 * ret < 0 : Command execution couldn't be submitted by driver
1248 * ret > 0 : Command execution couldn't be performed by firmware
1249 * ret == 0: Command was executed by FW, Caller must check FW outbox status.
1250 *
1251 * return value in case (callback):
1252 * ret < 0 : Command execution couldn't be submitted by driver
1253 * ret == 0: Command will be submitted to FW for execution
1254 * and the callback will be called for further status updates
1255 */
mlx5_cmd_invoke(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t callback,void * context,int page_queue,u8 token,bool force_polling)1256 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1257 struct mlx5_cmd_msg *out, void *uout, int uout_size,
1258 mlx5_cmd_cbk_t callback,
1259 void *context, int page_queue,
1260 u8 token, bool force_polling)
1261 {
1262 struct mlx5_cmd *cmd = &dev->cmd;
1263 struct mlx5_cmd_work_ent *ent;
1264 struct mlx5_cmd_stats *stats;
1265 u8 status = 0;
1266 int err = 0;
1267 s64 ds;
1268
1269 if (callback && page_queue)
1270 return -EINVAL;
1271
1272 if (!page_queue && mlx5_cmd_all_stalled(dev)) {
1273 mlx5_core_err_rl(dev,
1274 "All CMD slots are stalled, aborting command\n");
1275 /* there's no reason to wait and block the whole kernel if FW
1276 * isn't currently responding to all slots, fail immediately
1277 */
1278 return -EAGAIN;
1279 }
1280
1281 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1282 callback, context, page_queue);
1283 if (IS_ERR(ent))
1284 return PTR_ERR(ent);
1285
1286 /* put for this ent is when consumed, depending on the use case
1287 * 1) (!callback) blocking flow: by caller after wait_func completes
1288 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1289 */
1290
1291 ent->token = token;
1292 ent->polling = force_polling;
1293
1294 init_completion(&ent->handling);
1295 init_completion(&ent->slotted);
1296 if (!callback)
1297 init_completion(&ent->done);
1298
1299 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1300 INIT_WORK(&ent->work, cmd_work_handler);
1301 if (page_queue) {
1302 cmd_work_handler(&ent->work);
1303 } else if (!queue_work(cmd->wq, &ent->work)) {
1304 mlx5_core_warn(dev, "failed to queue work\n");
1305 err = -EALREADY;
1306 goto out_free;
1307 }
1308
1309 if (callback)
1310 return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1311
1312 err = wait_func(dev, ent);
1313 if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)
1314 goto out_free;
1315
1316 ds = ent->ts2 - ent->ts1;
1317 stats = xa_load(&cmd->stats, ent->op);
1318 if (stats) {
1319 spin_lock_irq(&stats->lock);
1320 stats->sum += ds;
1321 ++stats->n;
1322 spin_unlock_irq(&stats->lock);
1323 }
1324 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1325 "fw exec time for %s is %lld nsec\n",
1326 mlx5_command_str(ent->op), ds);
1327
1328 out_free:
1329 status = ent->status;
1330 cmd_ent_put(ent);
1331 return err ? : status;
1332 }
1333
dbg_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1334 static ssize_t dbg_write(struct file *filp, const char __user *buf,
1335 size_t count, loff_t *pos)
1336 {
1337 struct mlx5_core_dev *dev = filp->private_data;
1338 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1339 char lbuf[3];
1340 int err;
1341
1342 if (!dbg->in_msg || !dbg->out_msg)
1343 return -ENOMEM;
1344
1345 if (count < sizeof(lbuf) - 1)
1346 return -EINVAL;
1347
1348 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))
1349 return -EFAULT;
1350
1351 lbuf[sizeof(lbuf) - 1] = 0;
1352
1353 if (strcmp(lbuf, "go"))
1354 return -EINVAL;
1355
1356 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
1357
1358 return err ? err : count;
1359 }
1360
1361 static const struct file_operations fops = {
1362 .owner = THIS_MODULE,
1363 .open = simple_open,
1364 .write = dbg_write,
1365 };
1366
mlx5_copy_to_msg(struct mlx5_cmd_msg * to,void * from,int size,u8 token)1367 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1368 u8 token)
1369 {
1370 struct mlx5_cmd_prot_block *block;
1371 struct mlx5_cmd_mailbox *next;
1372 int copy;
1373
1374 if (!to || !from)
1375 return -ENOMEM;
1376
1377 copy = min_t(int, size, sizeof(to->first.data));
1378 memcpy(to->first.data, from, copy);
1379 size -= copy;
1380 from += copy;
1381
1382 next = to->next;
1383 while (size) {
1384 if (!next) {
1385 /* this is a BUG */
1386 return -ENOMEM;
1387 }
1388
1389 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1390 block = next->buf;
1391 memcpy(block->data, from, copy);
1392 from += copy;
1393 size -= copy;
1394 block->token = token;
1395 next = next->next;
1396 }
1397
1398 return 0;
1399 }
1400
mlx5_copy_from_msg(void * to,struct mlx5_cmd_msg * from,int size)1401 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1402 {
1403 struct mlx5_cmd_prot_block *block;
1404 struct mlx5_cmd_mailbox *next;
1405 int copy;
1406
1407 if (!to || !from)
1408 return -ENOMEM;
1409
1410 copy = min_t(int, size, sizeof(from->first.data));
1411 memcpy(to, from->first.data, copy);
1412 size -= copy;
1413 to += copy;
1414
1415 next = from->next;
1416 while (size) {
1417 if (!next) {
1418 /* this is a BUG */
1419 return -ENOMEM;
1420 }
1421
1422 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1423 block = next->buf;
1424
1425 memcpy(to, block->data, copy);
1426 to += copy;
1427 size -= copy;
1428 next = next->next;
1429 }
1430
1431 return 0;
1432 }
1433
alloc_cmd_box(struct mlx5_core_dev * dev,gfp_t flags)1434 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1435 gfp_t flags)
1436 {
1437 struct mlx5_cmd_mailbox *mailbox;
1438
1439 mailbox = kmalloc_obj(*mailbox, flags);
1440 if (!mailbox)
1441 return ERR_PTR(-ENOMEM);
1442
1443 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
1444 &mailbox->dma);
1445 if (!mailbox->buf) {
1446 mlx5_core_dbg(dev, "failed allocation\n");
1447 kfree(mailbox);
1448 return ERR_PTR(-ENOMEM);
1449 }
1450 mailbox->next = NULL;
1451
1452 return mailbox;
1453 }
1454
free_cmd_box(struct mlx5_core_dev * dev,struct mlx5_cmd_mailbox * mailbox)1455 static void free_cmd_box(struct mlx5_core_dev *dev,
1456 struct mlx5_cmd_mailbox *mailbox)
1457 {
1458 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
1459 kfree(mailbox);
1460 }
1461
mlx5_alloc_cmd_msg(struct mlx5_core_dev * dev,gfp_t flags,int size,u8 token)1462 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1463 gfp_t flags, int size,
1464 u8 token)
1465 {
1466 struct mlx5_cmd_mailbox *tmp, *head = NULL;
1467 struct mlx5_cmd_prot_block *block;
1468 struct mlx5_cmd_msg *msg;
1469 int err;
1470 int n;
1471 int i;
1472
1473 msg = kzalloc_obj(*msg, flags);
1474 if (!msg)
1475 return ERR_PTR(-ENOMEM);
1476
1477 msg->len = size;
1478 n = mlx5_calc_cmd_blocks(msg);
1479
1480 for (i = 0; i < n; i++) {
1481 tmp = alloc_cmd_box(dev, flags);
1482 if (IS_ERR(tmp)) {
1483 mlx5_core_warn(dev, "failed allocating block\n");
1484 err = PTR_ERR(tmp);
1485 goto err_alloc;
1486 }
1487
1488 block = tmp->buf;
1489 tmp->next = head;
1490 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1491 block->block_num = cpu_to_be32(n - i - 1);
1492 block->token = token;
1493 head = tmp;
1494 }
1495 msg->next = head;
1496 return msg;
1497
1498 err_alloc:
1499 while (head) {
1500 tmp = head->next;
1501 free_cmd_box(dev, head);
1502 head = tmp;
1503 }
1504 kfree(msg);
1505
1506 return ERR_PTR(err);
1507 }
1508
mlx5_free_cmd_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1509 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1510 struct mlx5_cmd_msg *msg)
1511 {
1512 struct mlx5_cmd_mailbox *head = msg->next;
1513 struct mlx5_cmd_mailbox *next;
1514
1515 while (head) {
1516 next = head->next;
1517 free_cmd_box(dev, head);
1518 head = next;
1519 }
1520 kfree(msg);
1521 }
1522
data_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1523 static ssize_t data_write(struct file *filp, const char __user *buf,
1524 size_t count, loff_t *pos)
1525 {
1526 struct mlx5_core_dev *dev = filp->private_data;
1527 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1528 void *ptr;
1529
1530 if (*pos != 0)
1531 return -EINVAL;
1532
1533 kfree(dbg->in_msg);
1534 dbg->in_msg = NULL;
1535 dbg->inlen = 0;
1536 ptr = memdup_user(buf, count);
1537 if (IS_ERR(ptr))
1538 return PTR_ERR(ptr);
1539 dbg->in_msg = ptr;
1540 dbg->inlen = count;
1541
1542 *pos = count;
1543
1544 return count;
1545 }
1546
data_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1547 static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1548 loff_t *pos)
1549 {
1550 struct mlx5_core_dev *dev = filp->private_data;
1551 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1552
1553 if (!dbg->out_msg)
1554 return -ENOMEM;
1555
1556 return simple_read_from_buffer(buf, count, pos, dbg->out_msg,
1557 dbg->outlen);
1558 }
1559
1560 static const struct file_operations dfops = {
1561 .owner = THIS_MODULE,
1562 .open = simple_open,
1563 .write = data_write,
1564 .read = data_read,
1565 };
1566
outlen_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1567 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1568 loff_t *pos)
1569 {
1570 struct mlx5_core_dev *dev = filp->private_data;
1571 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1572 char outlen[8];
1573 int err;
1574
1575 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
1576 if (err < 0)
1577 return err;
1578
1579 return simple_read_from_buffer(buf, count, pos, outlen, err);
1580 }
1581
outlen_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1582 static ssize_t outlen_write(struct file *filp, const char __user *buf,
1583 size_t count, loff_t *pos)
1584 {
1585 struct mlx5_core_dev *dev = filp->private_data;
1586 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1587 char outlen_str[8] = {0};
1588 int outlen;
1589 void *ptr;
1590 int err;
1591
1592 if (*pos != 0 || count > 6)
1593 return -EINVAL;
1594
1595 kfree(dbg->out_msg);
1596 dbg->out_msg = NULL;
1597 dbg->outlen = 0;
1598
1599 if (copy_from_user(outlen_str, buf, count))
1600 return -EFAULT;
1601
1602 err = sscanf(outlen_str, "%d", &outlen);
1603 if (err != 1)
1604 return -EINVAL;
1605
1606 ptr = kzalloc(outlen, GFP_KERNEL);
1607 if (!ptr)
1608 return -ENOMEM;
1609
1610 dbg->out_msg = ptr;
1611 dbg->outlen = outlen;
1612
1613 *pos = count;
1614
1615 return count;
1616 }
1617
1618 static const struct file_operations olfops = {
1619 .owner = THIS_MODULE,
1620 .open = simple_open,
1621 .write = outlen_write,
1622 .read = outlen_read,
1623 };
1624
set_wqname(struct mlx5_core_dev * dev)1625 static void set_wqname(struct mlx5_core_dev *dev)
1626 {
1627 struct mlx5_cmd *cmd = &dev->cmd;
1628
1629 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1630 dev_name(dev->device));
1631 }
1632
clean_debug_files(struct mlx5_core_dev * dev)1633 static void clean_debug_files(struct mlx5_core_dev *dev)
1634 {
1635 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1636
1637 if (!mlx5_debugfs_root)
1638 return;
1639
1640 debugfs_remove_recursive(dbg->dbg_root);
1641 }
1642
create_debugfs_files(struct mlx5_core_dev * dev)1643 static void create_debugfs_files(struct mlx5_core_dev *dev)
1644 {
1645 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1646
1647 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));
1648
1649 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1650 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1651 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1652 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);
1653 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1654 }
1655
mlx5_cmd_allowed_opcode(struct mlx5_core_dev * dev,u16 opcode)1656 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1657 {
1658 struct mlx5_cmd *cmd = &dev->cmd;
1659 int i;
1660
1661 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1662 down(&cmd->vars.sem);
1663 down(&cmd->vars.pages_sem);
1664
1665 cmd->allowed_opcode = opcode;
1666
1667 up(&cmd->vars.pages_sem);
1668 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1669 up(&cmd->vars.sem);
1670 }
1671
mlx5_cmd_change_mod(struct mlx5_core_dev * dev,int mode)1672 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1673 {
1674 struct mlx5_cmd *cmd = &dev->cmd;
1675 int i;
1676
1677 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1678 down(&cmd->vars.sem);
1679 down(&cmd->vars.pages_sem);
1680
1681 cmd->mode = mode;
1682
1683 up(&cmd->vars.pages_sem);
1684 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1685 up(&cmd->vars.sem);
1686 }
1687
cmd_comp_notifier(struct notifier_block * nb,unsigned long type,void * data)1688 static int cmd_comp_notifier(struct notifier_block *nb,
1689 unsigned long type, void *data)
1690 {
1691 struct mlx5_core_dev *dev;
1692 struct mlx5_cmd *cmd;
1693 struct mlx5_eqe *eqe;
1694
1695 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1696 dev = container_of(cmd, struct mlx5_core_dev, cmd);
1697 eqe = data;
1698
1699 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1700 return NOTIFY_DONE;
1701
1702 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);
1703
1704 return NOTIFY_OK;
1705 }
mlx5_cmd_use_events(struct mlx5_core_dev * dev)1706 void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1707 {
1708 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1709 mlx5_eq_notifier_register(dev, &dev->cmd.nb);
1710 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);
1711 }
1712
mlx5_cmd_use_polling(struct mlx5_core_dev * dev)1713 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1714 {
1715 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);
1716 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);
1717 }
1718
free_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1719 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1720 {
1721 unsigned long flags;
1722
1723 if (msg->parent) {
1724 spin_lock_irqsave(&msg->parent->lock, flags);
1725 list_add_tail(&msg->list, &msg->parent->head);
1726 spin_unlock_irqrestore(&msg->parent->lock, flags);
1727 } else {
1728 mlx5_free_cmd_msg(dev, msg);
1729 }
1730 }
1731
mlx5_cmd_comp_handler(struct mlx5_core_dev * dev,u64 vec,bool forced)1732 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1733 {
1734 struct mlx5_cmd *cmd = &dev->cmd;
1735 struct mlx5_cmd_work_ent *ent;
1736 mlx5_cmd_cbk_t callback;
1737 void *context;
1738 int err;
1739 int i;
1740 s64 ds;
1741 struct mlx5_cmd_stats *stats;
1742 unsigned long flags;
1743 unsigned long vector;
1744
1745 /* there can be at most 32 command queues */
1746 vector = vec & 0xffffffff;
1747 for (i = 0; i < (1 << cmd->vars.log_sz); i++) {
1748 if (test_bit(i, &vector)) {
1749 ent = cmd->ent_arr[i];
1750
1751 if (forced && ent->ret == -ETIMEDOUT)
1752 set_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1753 &ent->state);
1754 else if (!forced) /* real FW completion */
1755 clear_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1756 &ent->state);
1757
1758 /* if we already completed the command, ignore it */
1759 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,
1760 &ent->state)) {
1761 /* only real completion can free the cmd slot */
1762 if (!forced) {
1763 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1764 ent->idx);
1765 cmd_ent_put(ent);
1766 }
1767 continue;
1768 }
1769
1770 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1771 cmd_ent_put(ent); /* timeout work was canceled */
1772
1773 if (!forced || /* Real FW completion */
1774 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1775 !opcode_allowed(cmd, ent->op))
1776 cmd_ent_put(ent);
1777
1778 ent->ts2 = ktime_get_ns();
1779 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1780 dump_command(dev, ent, 0);
1781
1782 if (vec & MLX5_TRIGGERED_CMD_COMP)
1783 ent->ret = -ENXIO;
1784
1785 if (!ent->ret) { /* Command completed by FW */
1786 if (!cmd->checksum_disabled)
1787 ent->ret = verify_signature(ent);
1788
1789 ent->status = ent->lay->status_own >> 1;
1790
1791 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1792 ent->ret, deliv_status_to_str(ent->status), ent->status);
1793 }
1794
1795 if (ent->callback) {
1796 ds = ent->ts2 - ent->ts1;
1797 stats = xa_load(&cmd->stats, ent->op);
1798 if (stats) {
1799 spin_lock_irqsave(&stats->lock, flags);
1800 stats->sum += ds;
1801 ++stats->n;
1802 spin_unlock_irqrestore(&stats->lock, flags);
1803 }
1804
1805 callback = ent->callback;
1806 context = ent->context;
1807 err = ent->ret ? : ent->status;
1808 if (err > 0) /* Failed in FW, command didn't execute */
1809 err = deliv_status_to_err(err);
1810
1811 if (!err)
1812 err = mlx5_copy_from_msg(ent->uout,
1813 ent->out,
1814 ent->uout_size);
1815
1816 mlx5_free_cmd_msg(dev, ent->out);
1817 free_msg(dev, ent->in);
1818
1819 /* final consumer is done, release ent */
1820 cmd_ent_put(ent);
1821 callback(err, context);
1822 } else {
1823 /* release wait_func() so mlx5_cmd_invoke()
1824 * can make the final ent_put()
1825 */
1826 complete(&ent->done);
1827 }
1828 }
1829 }
1830 }
1831
1832 #define MLX5_MAX_MANAGE_PAGES_CMD_ENT 1
1833 #define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \
1834 MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1)
1835
mlx5_cmd_trigger_completions(struct mlx5_core_dev * dev)1836 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1837 {
1838 struct mlx5_cmd *cmd = &dev->cmd;
1839 unsigned long bitmask;
1840 unsigned long flags;
1841 u64 vector;
1842 int i;
1843
1844 /* wait for pending handlers to complete */
1845 mlx5_eq_synchronize_cmd_irq(dev);
1846 spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1847 vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK;
1848 if (!vector)
1849 goto no_trig;
1850
1851 bitmask = vector;
1852 /* we must increment the allocated entries refcount before triggering the completions
1853 * to guarantee pending commands will not get freed in the meanwhile.
1854 * For that reason, it also has to be done inside the alloc_lock.
1855 */
1856 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1857 cmd_ent_get(cmd->ent_arr[i]);
1858 vector |= MLX5_TRIGGERED_CMD_COMP;
1859 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1860
1861 mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1862 mlx5_cmd_comp_handler(dev, vector, true);
1863 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1864 cmd_ent_put(cmd->ent_arr[i]);
1865 return;
1866
1867 no_trig:
1868 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1869 }
1870
mlx5_cmd_flush(struct mlx5_core_dev * dev)1871 void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1872 {
1873 struct mlx5_cmd *cmd = &dev->cmd;
1874 int i;
1875
1876 for (i = 0; i < cmd->vars.max_reg_cmds; i++) {
1877 while (down_trylock(&cmd->vars.sem)) {
1878 mlx5_cmd_trigger_completions(dev);
1879 cond_resched();
1880 }
1881 }
1882
1883 while (down_trylock(&cmd->vars.pages_sem)) {
1884 mlx5_cmd_trigger_completions(dev);
1885 cond_resched();
1886 }
1887
1888 /* Unlock cmdif */
1889 up(&cmd->vars.pages_sem);
1890 for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1891 up(&cmd->vars.sem);
1892 }
1893
alloc_msg(struct mlx5_core_dev * dev,int in_size,gfp_t gfp)1894 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1895 gfp_t gfp)
1896 {
1897 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1898 struct cmd_msg_cache *ch = NULL;
1899 struct mlx5_cmd *cmd = &dev->cmd;
1900 int i;
1901
1902 if (in_size <= 16)
1903 goto cache_miss;
1904
1905 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
1906 ch = &cmd->cache[i];
1907 if (in_size > ch->max_inbox_size)
1908 continue;
1909 spin_lock_irq(&ch->lock);
1910 if (list_empty(&ch->head)) {
1911 spin_unlock_irq(&ch->lock);
1912 continue;
1913 }
1914 msg = list_entry(ch->head.next, typeof(*msg), list);
1915 /* For cached lists, we must explicitly state what is
1916 * the real size
1917 */
1918 msg->len = in_size;
1919 list_del(&msg->list);
1920 spin_unlock_irq(&ch->lock);
1921 break;
1922 }
1923
1924 if (!IS_ERR(msg))
1925 return msg;
1926
1927 cache_miss:
1928 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
1929 return msg;
1930 }
1931
is_manage_pages(void * in)1932 static int is_manage_pages(void *in)
1933 {
1934 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;
1935 }
1936
mlx5_has_privileged_uid(struct mlx5_core_dev * dev)1937 static bool mlx5_has_privileged_uid(struct mlx5_core_dev *dev)
1938 {
1939 return !xa_empty(&dev->cmd.vars.privileged_uids);
1940 }
1941
mlx5_cmd_is_privileged_uid(struct mlx5_core_dev * dev,u16 uid)1942 static bool mlx5_cmd_is_privileged_uid(struct mlx5_core_dev *dev,
1943 u16 uid)
1944 {
1945 return !!xa_load(&dev->cmd.vars.privileged_uids, uid);
1946 }
1947
1948 /* Notes:
1949 * 1. Callback functions may not sleep
1950 * 2. Page queue commands do not support asynchrous completion
1951 */
cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size,mlx5_cmd_cbk_t callback,void * context,bool force_polling)1952 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1953 int out_size, mlx5_cmd_cbk_t callback, void *context,
1954 bool force_polling)
1955 {
1956 struct mlx5_cmd_msg *inb, *outb;
1957 u16 opcode = in_to_opcode(in);
1958 bool throttle_locked = false;
1959 bool unpriv_locked = false;
1960 u16 uid = in_to_uid(in);
1961 int pages_queue;
1962 gfp_t gfp;
1963 u8 token;
1964 int err;
1965
1966 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))
1967 return -ENXIO;
1968
1969 if (!callback) {
1970 /* The semaphore is already held for callback commands. It was
1971 * acquired in mlx5_cmd_exec_cb()
1972 */
1973 if (uid && mlx5_has_privileged_uid(dev)) {
1974 if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
1975 unpriv_locked = true;
1976 down(&dev->cmd.vars.unprivileged_sem);
1977 }
1978 } else if (mlx5_cmd_is_throttle_opcode(opcode)) {
1979 throttle_locked = true;
1980 down(&dev->cmd.vars.throttle_sem);
1981 }
1982 }
1983
1984 pages_queue = is_manage_pages(in);
1985 gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1986
1987 inb = alloc_msg(dev, in_size, gfp);
1988 if (IS_ERR(inb)) {
1989 err = PTR_ERR(inb);
1990 goto out_up;
1991 }
1992
1993 token = alloc_token(&dev->cmd);
1994
1995 err = mlx5_copy_to_msg(inb, in, in_size, token);
1996 if (err) {
1997 mlx5_core_warn(dev, "err %d\n", err);
1998 goto out_in;
1999 }
2000
2001 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
2002 if (IS_ERR(outb)) {
2003 err = PTR_ERR(outb);
2004 goto out_in;
2005 }
2006
2007 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
2008 pages_queue, token, force_polling);
2009 if (callback && !err)
2010 return 0;
2011
2012 if (err > 0) /* Failed in FW, command didn't execute */
2013 err = deliv_status_to_err(err);
2014
2015 if (err)
2016 goto out_out;
2017
2018 /* command completed by FW */
2019 err = mlx5_copy_from_msg(out, outb, out_size);
2020 out_out:
2021 mlx5_free_cmd_msg(dev, outb);
2022 out_in:
2023 free_msg(dev, inb);
2024 out_up:
2025 if (throttle_locked)
2026 up(&dev->cmd.vars.throttle_sem);
2027 if (unpriv_locked)
2028 up(&dev->cmd.vars.unprivileged_sem);
2029
2030 return err;
2031 }
2032
mlx5_cmd_err_trace(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)2033 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
2034 {
2035 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2036 u8 status = MLX5_GET(mbox_out, out, status);
2037
2038 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,
2039 cmd_status_str(status), status, syndrome,
2040 cmd_status_to_err(status));
2041 }
2042
cmd_status_log(struct mlx5_core_dev * dev,u16 opcode,u8 status,u32 syndrome,int err)2043 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
2044 u32 syndrome, int err)
2045 {
2046 const char *namep = mlx5_command_str(opcode);
2047 struct mlx5_cmd_stats *stats;
2048 unsigned long flags;
2049
2050 if (!err || !(strcmp(namep, "unknown command opcode")))
2051 return;
2052
2053 stats = xa_load(&dev->cmd.stats, opcode);
2054 if (!stats)
2055 return;
2056 spin_lock_irqsave(&stats->lock, flags);
2057 stats->failed++;
2058 if (err < 0)
2059 stats->last_failed_errno = -err;
2060 if (err == -EREMOTEIO) {
2061 stats->failed_mbox_status++;
2062 stats->last_failed_mbox_status = status;
2063 stats->last_failed_syndrome = syndrome;
2064 }
2065 spin_unlock_irqrestore(&stats->lock, flags);
2066 }
2067
2068 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
cmd_status_err(struct mlx5_core_dev * dev,int err,u16 opcode,u16 op_mod,void * out)2069 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
2070 {
2071 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2072 u8 status = MLX5_GET(mbox_out, out, status);
2073
2074 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
2075 err = -EIO;
2076
2077 if (!err && status != MLX5_CMD_STAT_OK) {
2078 err = -EREMOTEIO;
2079 mlx5_cmd_err_trace(dev, opcode, op_mod, out);
2080 }
2081
2082 cmd_status_log(dev, opcode, status, syndrome, err);
2083 return err;
2084 }
2085
2086 /**
2087 * mlx5_cmd_do - Executes a fw command, wait for completion.
2088 * Unlike mlx5_cmd_exec, this function will not translate or intercept
2089 * outbox.status and will return -EREMOTEIO when
2090 * outbox.status != MLX5_CMD_STAT_OK
2091 *
2092 * @dev: mlx5 core device
2093 * @in: inbox mlx5_ifc command buffer
2094 * @in_size: inbox buffer size
2095 * @out: outbox mlx5_ifc buffer
2096 * @out_size: outbox size
2097 *
2098 * @return:
2099 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
2100 * Caller must check FW outbox status.
2101 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
2102 * < 0 : Command execution couldn't be performed by firmware or driver
2103 */
mlx5_cmd_do(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2104 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
2105 {
2106 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);
2107 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2108 u16 opcode = in_to_opcode(in);
2109
2110 return cmd_status_err(dev, err, opcode, op_mod, out);
2111 }
2112 EXPORT_SYMBOL(mlx5_cmd_do);
2113
2114 /**
2115 * mlx5_cmd_exec - Executes a fw command, wait for completion
2116 *
2117 * @dev: mlx5 core device
2118 * @in: inbox mlx5_ifc command buffer
2119 * @in_size: inbox buffer size
2120 * @out: outbox mlx5_ifc buffer
2121 * @out_size: outbox size
2122 *
2123 * @return: 0 if no error, FW command execution was successful
2124 * and outbox status is ok.
2125 */
mlx5_cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2126 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
2127 int out_size)
2128 {
2129 int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
2130
2131 return mlx5_cmd_check(dev, err, in, out);
2132 }
2133 EXPORT_SYMBOL(mlx5_cmd_exec);
2134
2135 /**
2136 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion
2137 * Needed for driver force teardown, when command completion EQ
2138 * will not be available to complete the command
2139 *
2140 * @dev: mlx5 core device
2141 * @in: inbox mlx5_ifc command buffer
2142 * @in_size: inbox buffer size
2143 * @out: outbox mlx5_ifc buffer
2144 * @out_size: outbox size
2145 *
2146 * @return: 0 if no error, FW command execution was successful
2147 * and outbox status is ok.
2148 */
mlx5_cmd_exec_polling(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)2149 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
2150 void *out, int out_size)
2151 {
2152 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);
2153 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2154 u16 opcode = in_to_opcode(in);
2155
2156 err = cmd_status_err(dev, err, opcode, op_mod, out);
2157 return mlx5_cmd_check(dev, err, in, out);
2158 }
2159 EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2160
mlx5_cmd_init_async_ctx(struct mlx5_core_dev * dev,struct mlx5_async_ctx * ctx)2161 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2162 struct mlx5_async_ctx *ctx)
2163 {
2164 ctx->dev = dev;
2165 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2166 atomic_set(&ctx->num_inflight, 1);
2167 init_completion(&ctx->inflight_done);
2168 }
2169 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2170
2171 /**
2172 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2173 * @ctx: The ctx to clean
2174 *
2175 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2176 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2177 * the call mlx5_cleanup_async_ctx().
2178 */
mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx * ctx)2179 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2180 {
2181 if (!atomic_dec_and_test(&ctx->num_inflight))
2182 wait_for_completion(&ctx->inflight_done);
2183 }
2184 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2185
mlx5_cmd_exec_cb_handler(int status,void * _work)2186 static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2187 {
2188 struct mlx5_async_work *work = _work;
2189 struct mlx5_async_ctx *ctx;
2190 struct mlx5_core_dev *dev;
2191 bool throttle_locked;
2192 bool unpriv_locked;
2193
2194 ctx = work->ctx;
2195 dev = ctx->dev;
2196 throttle_locked = work->throttle_locked;
2197 unpriv_locked = work->unpriv_locked;
2198 status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out);
2199 work->user_callback(status, work);
2200 /* Can't access "work" from this point on. It could have been freed in
2201 * the callback.
2202 */
2203 if (throttle_locked)
2204 up(&dev->cmd.vars.throttle_sem);
2205 if (unpriv_locked)
2206 up(&dev->cmd.vars.unprivileged_sem);
2207 if (atomic_dec_and_test(&ctx->num_inflight))
2208 complete(&ctx->inflight_done);
2209 }
2210
mlx5_cmd_exec_cb(struct mlx5_async_ctx * ctx,void * in,int in_size,void * out,int out_size,mlx5_async_cbk_t callback,struct mlx5_async_work * work)2211 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2212 void *out, int out_size, mlx5_async_cbk_t callback,
2213 struct mlx5_async_work *work)
2214 {
2215 struct mlx5_core_dev *dev = ctx->dev;
2216 u16 uid;
2217 int ret;
2218
2219 work->ctx = ctx;
2220 work->user_callback = callback;
2221 work->opcode = in_to_opcode(in);
2222 work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2223 work->out = out;
2224 work->throttle_locked = false;
2225 work->unpriv_locked = false;
2226 uid = in_to_uid(in);
2227
2228 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2229 return -EIO;
2230
2231 if (uid && mlx5_has_privileged_uid(dev)) {
2232 if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
2233 if (down_trylock(&dev->cmd.vars.unprivileged_sem)) {
2234 ret = -EBUSY;
2235 goto dec_num_inflight;
2236 }
2237 work->unpriv_locked = true;
2238 }
2239 } else if (mlx5_cmd_is_throttle_opcode(in_to_opcode(in))) {
2240 if (down_trylock(&dev->cmd.vars.throttle_sem)) {
2241 ret = -EBUSY;
2242 goto dec_num_inflight;
2243 }
2244 work->throttle_locked = true;
2245 }
2246
2247 ret = cmd_exec(dev, in, in_size, out, out_size,
2248 mlx5_cmd_exec_cb_handler, work, false);
2249 if (ret)
2250 goto sem_up;
2251
2252 return 0;
2253
2254 sem_up:
2255 if (work->throttle_locked)
2256 up(&dev->cmd.vars.throttle_sem);
2257 if (work->unpriv_locked)
2258 up(&dev->cmd.vars.unprivileged_sem);
2259 dec_num_inflight:
2260 if (atomic_dec_and_test(&ctx->num_inflight))
2261 complete(&ctx->inflight_done);
2262
2263 return ret;
2264 }
2265 EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2266
mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev * dev,struct mlx5_cmd_allow_other_vhca_access_attr * attr)2267 int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,
2268 struct mlx5_cmd_allow_other_vhca_access_attr *attr)
2269 {
2270 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};
2271 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};
2272 void *key;
2273
2274 MLX5_SET(allow_other_vhca_access_in,
2275 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);
2276 MLX5_SET(allow_other_vhca_access_in,
2277 in, object_type_to_be_accessed, attr->obj_type);
2278 MLX5_SET(allow_other_vhca_access_in,
2279 in, object_id_to_be_accessed, attr->obj_id);
2280
2281 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);
2282 memcpy(key, attr->access_key, sizeof(attr->access_key));
2283
2284 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2285 }
2286
mlx5_cmd_alias_obj_create(struct mlx5_core_dev * dev,struct mlx5_cmd_alias_obj_create_attr * alias_attr,u32 * obj_id)2287 int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
2288 struct mlx5_cmd_alias_obj_create_attr *alias_attr,
2289 u32 *obj_id)
2290 {
2291 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2292 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};
2293 void *param;
2294 void *attr;
2295 void *key;
2296 int ret;
2297
2298 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);
2299 MLX5_SET(general_obj_in_cmd_hdr,
2300 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2301 MLX5_SET(general_obj_in_cmd_hdr,
2302 attr, obj_type, alias_attr->obj_type);
2303 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
2304 MLX5_SET(general_obj_create_param, param, alias_object, 1);
2305
2306 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
2307 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
2308 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
2309
2310 key = MLX5_ADDR_OF(alias_context, attr, access_key);
2311 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));
2312
2313 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2314 if (ret)
2315 return ret;
2316
2317 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2318
2319 return 0;
2320 }
2321
mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev * dev,u32 obj_id,u16 obj_type)2322 int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,
2323 u16 obj_type)
2324 {
2325 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2326 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
2327
2328 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
2329 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);
2330 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
2331
2332 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2333 }
2334
destroy_msg_cache(struct mlx5_core_dev * dev)2335 static void destroy_msg_cache(struct mlx5_core_dev *dev)
2336 {
2337 struct cmd_msg_cache *ch;
2338 struct mlx5_cmd_msg *msg;
2339 struct mlx5_cmd_msg *n;
2340 int i;
2341
2342 for (i = 0; i < dev->profile.num_cmd_caches; i++) {
2343 ch = &dev->cmd.cache[i];
2344 list_for_each_entry_safe(msg, n, &ch->head, list) {
2345 list_del(&msg->list);
2346 mlx5_free_cmd_msg(dev, msg);
2347 }
2348 }
2349 }
2350
2351 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2352 512, 32, 16, 8, 2
2353 };
2354
2355 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2356 16 + MLX5_CMD_DATA_BLOCK_SIZE,
2357 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2358 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2359 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2360 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2361 };
2362
create_msg_cache(struct mlx5_core_dev * dev)2363 static void create_msg_cache(struct mlx5_core_dev *dev)
2364 {
2365 struct mlx5_cmd *cmd = &dev->cmd;
2366 struct cmd_msg_cache *ch;
2367 struct mlx5_cmd_msg *msg;
2368 int i;
2369 int k;
2370
2371 /* Initialize and fill the caches with initial entries */
2372 for (k = 0; k < dev->profile.num_cmd_caches; k++) {
2373 ch = &cmd->cache[k];
2374 spin_lock_init(&ch->lock);
2375 INIT_LIST_HEAD(&ch->head);
2376 ch->num_ent = cmd_cache_num_ent[k];
2377 ch->max_inbox_size = cmd_cache_ent_size[k];
2378 for (i = 0; i < ch->num_ent; i++) {
2379 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2380 ch->max_inbox_size, 0);
2381 if (IS_ERR(msg))
2382 break;
2383 msg->parent = ch;
2384 list_add_tail(&msg->list, &ch->head);
2385 }
2386 }
2387 }
2388
alloc_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2389 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2390 {
2391 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,
2392 &cmd->alloc_dma, GFP_KERNEL);
2393 if (!cmd->cmd_alloc_buf)
2394 return -ENOMEM;
2395
2396 /* make sure it is aligned to 4K */
2397 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2398 cmd->cmd_buf = cmd->cmd_alloc_buf;
2399 cmd->dma = cmd->alloc_dma;
2400 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2401 return 0;
2402 }
2403
2404 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,
2405 cmd->alloc_dma);
2406 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),
2407 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2408 &cmd->alloc_dma, GFP_KERNEL);
2409 if (!cmd->cmd_alloc_buf)
2410 return -ENOMEM;
2411
2412 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2413 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2414 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2415 return 0;
2416 }
2417
free_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2418 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2419 {
2420 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,
2421 cmd->alloc_dma);
2422 }
2423
cmdif_rev(struct mlx5_core_dev * dev)2424 static u16 cmdif_rev(struct mlx5_core_dev *dev)
2425 {
2426 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2427 }
2428
mlx5_cmd_init(struct mlx5_core_dev * dev)2429 int mlx5_cmd_init(struct mlx5_core_dev *dev)
2430 {
2431 struct mlx5_cmd *cmd = &dev->cmd;
2432
2433 cmd->checksum_disabled = 1;
2434
2435 spin_lock_init(&cmd->alloc_lock);
2436 spin_lock_init(&cmd->token_lock);
2437
2438 set_wqname(dev);
2439 cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2440 if (!cmd->wq) {
2441 mlx5_core_err(dev, "failed to create command workqueue\n");
2442 return -ENOMEM;
2443 }
2444
2445 mlx5_cmdif_debugfs_init(dev);
2446
2447 return 0;
2448 }
2449
mlx5_cmd_cleanup(struct mlx5_core_dev * dev)2450 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2451 {
2452 struct mlx5_cmd *cmd = &dev->cmd;
2453
2454 mlx5_cmdif_debugfs_cleanup(dev);
2455 destroy_workqueue(cmd->wq);
2456 }
2457
mlx5_cmd_enable(struct mlx5_core_dev * dev)2458 int mlx5_cmd_enable(struct mlx5_core_dev *dev)
2459 {
2460 int size = sizeof(struct mlx5_cmd_prot_block);
2461 int align = roundup_pow_of_two(size);
2462 struct mlx5_cmd *cmd = &dev->cmd;
2463 u32 cmd_h, cmd_l;
2464 int err;
2465
2466 memset(&cmd->vars, 0, sizeof(cmd->vars));
2467 cmd->vars.cmdif_rev = cmdif_rev(dev);
2468 if (cmd->vars.cmdif_rev != CMD_IF_REV) {
2469 mlx5_core_err(dev,
2470 "Driver cmdif rev(%d) differs from firmware's(%d)\n",
2471 CMD_IF_REV, cmd->vars.cmdif_rev);
2472 return -EINVAL;
2473 }
2474
2475 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2476 cmd->vars.log_sz = cmd_l >> 4 & 0xf;
2477 cmd->vars.log_stride = cmd_l & 0xf;
2478 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {
2479 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2480 1 << cmd->vars.log_sz);
2481 return -EINVAL;
2482 }
2483
2484 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2485 mlx5_core_err(dev, "command queue size overflow\n");
2486 return -EINVAL;
2487 }
2488
2489 cmd->state = MLX5_CMDIF_STATE_DOWN;
2490 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;
2491 cmd->vars.bitmask = MLX5_CMD_MASK;
2492
2493 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds);
2494 sema_init(&cmd->vars.pages_sem, 1);
2495 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2496 sema_init(&cmd->vars.unprivileged_sem,
2497 DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2498
2499 xa_init(&cmd->vars.privileged_uids);
2500
2501 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);
2502 if (!cmd->pool) {
2503 err = -ENOMEM;
2504 goto err_destroy_xa;
2505 }
2506
2507 err = alloc_cmd_page(dev, cmd);
2508 if (err)
2509 goto err_free_pool;
2510
2511 cmd_h = (u32)((u64)(cmd->dma) >> 32);
2512 cmd_l = (u32)(cmd->dma);
2513 if (cmd_l & 0xfff) {
2514 mlx5_core_err(dev, "invalid command queue address\n");
2515 err = -ENOMEM;
2516 goto err_cmd_page;
2517 }
2518
2519 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2520 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2521
2522 /* Make sure firmware sees the complete address before we proceed */
2523 wmb();
2524
2525 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2526
2527 cmd->mode = CMD_MODE_POLLING;
2528 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2529
2530 create_msg_cache(dev);
2531 create_debugfs_files(dev);
2532
2533 return 0;
2534
2535 err_cmd_page:
2536 free_cmd_page(dev, cmd);
2537 err_free_pool:
2538 dma_pool_destroy(cmd->pool);
2539 err_destroy_xa:
2540 xa_destroy(&dev->cmd.vars.privileged_uids);
2541 return err;
2542 }
2543
mlx5_cmd_disable(struct mlx5_core_dev * dev)2544 void mlx5_cmd_disable(struct mlx5_core_dev *dev)
2545 {
2546 struct mlx5_cmd *cmd = &dev->cmd;
2547
2548 flush_workqueue(cmd->wq);
2549 clean_debug_files(dev);
2550 destroy_msg_cache(dev);
2551 free_cmd_page(dev, cmd);
2552 dma_pool_destroy(cmd->pool);
2553 xa_destroy(&dev->cmd.vars.privileged_uids);
2554 }
2555
mlx5_cmd_set_state(struct mlx5_core_dev * dev,enum mlx5_cmdif_state cmdif_state)2556 void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2557 enum mlx5_cmdif_state cmdif_state)
2558 {
2559 dev->cmd.state = cmdif_state;
2560 }
2561
mlx5_cmd_add_privileged_uid(struct mlx5_core_dev * dev,u16 uid)2562 int mlx5_cmd_add_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2563 {
2564 return xa_insert(&dev->cmd.vars.privileged_uids, uid,
2565 xa_mk_value(uid), GFP_KERNEL);
2566 }
2567 EXPORT_SYMBOL(mlx5_cmd_add_privileged_uid);
2568
mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev * dev,u16 uid)2569 void mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2570 {
2571 void *data = xa_erase(&dev->cmd.vars.privileged_uids, uid);
2572
2573 WARN(!data, "Privileged UID %u does not exist\n", uid);
2574 }
2575 EXPORT_SYMBOL(mlx5_cmd_remove_privileged_uid);
2576