1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/mlx5_user_ioctl_cmds.h>
11 #include <rdma/mlx5_user_ioctl_verbs.h>
12 #include <rdma/ib_umem.h>
13 #include <rdma/uverbs_std_types.h>
14 #include <linux/mlx5/driver.h>
15 #include <linux/mlx5/fs.h>
16 #include <rdma/ib_ucaps.h>
17 #include "mlx5_ib.h"
18 #include "devx.h"
19 #include "qp.h"
20 #include <linux/xarray.h>
21
22 #define UVERBS_MODULE_NAME mlx5_ib
23 #include <rdma/uverbs_named_ioctl.h>
24
25 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
26
27 enum devx_obj_flags {
28 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
29 DEVX_OBJ_FLAGS_DCT = 1 << 1,
30 DEVX_OBJ_FLAGS_CQ = 1 << 2,
31 DEVX_OBJ_FLAGS_HW_FREED = 1 << 3,
32 };
33
34 #define MAX_ASYNC_CMDS 8
35
36 struct mlx5_async_cmd {
37 struct ib_uobject *uobject;
38 void *in;
39 int in_size;
40 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
41 int err;
42 struct mlx5_async_work cb_work;
43 struct completion comp;
44 };
45
46 struct devx_async_data {
47 struct mlx5_ib_dev *mdev;
48 struct list_head list;
49 struct devx_async_cmd_event_file *ev_file;
50 struct mlx5_async_work cb_work;
51 u16 cmd_out_len;
52 /* must be last field in this structure */
53 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
54 };
55
56 struct devx_async_event_data {
57 struct list_head list; /* headed in ev_file->event_list */
58 struct mlx5_ib_uapi_devx_async_event_hdr hdr;
59 };
60
61 /* first level XA value data structure */
62 struct devx_event {
63 struct xarray object_ids; /* second XA level, Key = object id */
64 struct list_head unaffiliated_list;
65 };
66
67 /* second level XA value data structure */
68 struct devx_obj_event {
69 struct rcu_head rcu;
70 struct list_head obj_sub_list;
71 };
72
73 struct devx_event_subscription {
74 struct list_head file_list; /* headed in ev_file->
75 * subscribed_events_list
76 */
77 struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
78 * devx_obj_event->obj_sub_list
79 */
80 struct list_head obj_list; /* headed in devx_object */
81 struct list_head event_list; /* headed in ev_file->event_list or in
82 * temp list via subscription
83 */
84
85 u8 is_cleaned:1;
86 u32 xa_key_level1;
87 u32 xa_key_level2;
88 struct rcu_head rcu;
89 u64 cookie;
90 struct devx_async_event_file *ev_file;
91 struct eventfd_ctx *eventfd;
92 };
93
94 struct devx_async_event_file {
95 struct ib_uobject uobj;
96 /* Head of events that are subscribed to this FD */
97 struct list_head subscribed_events_list;
98 spinlock_t lock;
99 wait_queue_head_t poll_wait;
100 struct list_head event_list;
101 struct mlx5_ib_dev *dev;
102 u8 omit_data:1;
103 u8 is_overflow_err:1;
104 u8 is_destroyed:1;
105 };
106
107 struct devx_umem {
108 struct mlx5_core_dev *mdev;
109 struct ib_umem *umem;
110 u32 dinlen;
111 u32 dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];
112 };
113
114 struct devx_umem_reg_cmd {
115 void *in;
116 u32 inlen;
117 u32 out[MLX5_ST_SZ_DW(create_umem_out)];
118 };
119
120 static struct mlx5_ib_ucontext *
devx_ufile2uctx(const struct uverbs_attr_bundle * attrs)121 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
122 {
123 return to_mucontext(ib_uverbs_get_ucontext(attrs));
124 }
125
set_uctx_ucaps(struct mlx5_ib_dev * dev,u64 req_ucaps,u32 * cap)126 static int set_uctx_ucaps(struct mlx5_ib_dev *dev, u64 req_ucaps, u32 *cap)
127 {
128 if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_LOCAL)) {
129 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
130 *cap |= MLX5_UCTX_CAP_RDMA_CTRL;
131 else
132 return -EOPNOTSUPP;
133 }
134
135 if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA)) {
136 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
137 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
138 *cap |= MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA;
139 else
140 return -EOPNOTSUPP;
141 }
142
143 return 0;
144 }
145
mlx5_ib_devx_create(struct mlx5_ib_dev * dev,bool is_user,u64 req_ucaps)146 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)
147 {
148 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};
149 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
150 void *uctx;
151 int err;
152 u16 uid;
153 u32 cap = 0;
154
155 /* 0 means not supported */
156 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
157 return -EINVAL;
158
159 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
160 if (is_user &&
161 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX) &&
162 capable(CAP_NET_RAW))
163 cap |= MLX5_UCTX_CAP_RAW_TX;
164 if (is_user &&
165 (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
166 MLX5_UCTX_CAP_INTERNAL_DEV_RES) &&
167 capable(CAP_SYS_RAWIO))
168 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
169
170 if (req_ucaps) {
171 err = set_uctx_ucaps(dev, req_ucaps, &cap);
172 if (err)
173 return err;
174 }
175
176 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
177 MLX5_SET(uctx, uctx, cap, cap);
178
179 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
180 if (err)
181 return err;
182
183 uid = MLX5_GET(create_uctx_out, out, uid);
184 return uid;
185 }
186
mlx5_ib_devx_destroy(struct mlx5_ib_dev * dev,u16 uid)187 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
188 {
189 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
190 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
191
192 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
193 MLX5_SET(destroy_uctx_in, in, uid, uid);
194
195 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
196 }
197
is_legacy_unaffiliated_event_num(u16 event_num)198 static bool is_legacy_unaffiliated_event_num(u16 event_num)
199 {
200 switch (event_num) {
201 case MLX5_EVENT_TYPE_PORT_CHANGE:
202 return true;
203 default:
204 return false;
205 }
206 }
207
is_legacy_obj_event_num(u16 event_num)208 static bool is_legacy_obj_event_num(u16 event_num)
209 {
210 switch (event_num) {
211 case MLX5_EVENT_TYPE_PATH_MIG:
212 case MLX5_EVENT_TYPE_COMM_EST:
213 case MLX5_EVENT_TYPE_SQ_DRAINED:
214 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
215 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
216 case MLX5_EVENT_TYPE_CQ_ERROR:
217 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
218 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
219 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
220 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
221 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
222 case MLX5_EVENT_TYPE_DCT_DRAINED:
223 case MLX5_EVENT_TYPE_COMP:
224 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
225 case MLX5_EVENT_TYPE_XRQ_ERROR:
226 return true;
227 default:
228 return false;
229 }
230 }
231
get_legacy_obj_type(u16 opcode)232 static u16 get_legacy_obj_type(u16 opcode)
233 {
234 switch (opcode) {
235 case MLX5_CMD_OP_CREATE_RQ:
236 return MLX5_EVENT_QUEUE_TYPE_RQ;
237 case MLX5_CMD_OP_CREATE_QP:
238 return MLX5_EVENT_QUEUE_TYPE_QP;
239 case MLX5_CMD_OP_CREATE_SQ:
240 return MLX5_EVENT_QUEUE_TYPE_SQ;
241 case MLX5_CMD_OP_CREATE_DCT:
242 return MLX5_EVENT_QUEUE_TYPE_DCT;
243 default:
244 return 0;
245 }
246 }
247
get_dec_obj_type(struct devx_obj * obj,u16 event_num)248 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
249 {
250 u16 opcode;
251
252 opcode = (obj->obj_id >> 32) & 0xffff;
253
254 if (is_legacy_obj_event_num(event_num))
255 return get_legacy_obj_type(opcode);
256
257 switch (opcode) {
258 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
259 return (obj->obj_id >> 48);
260 case MLX5_CMD_OP_CREATE_RQ:
261 return MLX5_OBJ_TYPE_RQ;
262 case MLX5_CMD_OP_CREATE_QP:
263 return MLX5_OBJ_TYPE_QP;
264 case MLX5_CMD_OP_CREATE_SQ:
265 return MLX5_OBJ_TYPE_SQ;
266 case MLX5_CMD_OP_CREATE_DCT:
267 return MLX5_OBJ_TYPE_DCT;
268 case MLX5_CMD_OP_CREATE_TIR:
269 return MLX5_OBJ_TYPE_TIR;
270 case MLX5_CMD_OP_CREATE_TIS:
271 return MLX5_OBJ_TYPE_TIS;
272 case MLX5_CMD_OP_CREATE_PSV:
273 return MLX5_OBJ_TYPE_PSV;
274 case MLX5_OBJ_TYPE_MKEY:
275 return MLX5_OBJ_TYPE_MKEY;
276 case MLX5_CMD_OP_CREATE_RMP:
277 return MLX5_OBJ_TYPE_RMP;
278 case MLX5_CMD_OP_CREATE_XRC_SRQ:
279 return MLX5_OBJ_TYPE_XRC_SRQ;
280 case MLX5_CMD_OP_CREATE_XRQ:
281 return MLX5_OBJ_TYPE_XRQ;
282 case MLX5_CMD_OP_CREATE_RQT:
283 return MLX5_OBJ_TYPE_RQT;
284 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
285 return MLX5_OBJ_TYPE_FLOW_COUNTER;
286 case MLX5_CMD_OP_CREATE_CQ:
287 return MLX5_OBJ_TYPE_CQ;
288 default:
289 return 0;
290 }
291 }
292
get_event_obj_type(unsigned long event_type,struct mlx5_eqe * eqe)293 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
294 {
295 switch (event_type) {
296 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
297 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
298 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
299 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
300 case MLX5_EVENT_TYPE_PATH_MIG:
301 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
302 case MLX5_EVENT_TYPE_COMM_EST:
303 case MLX5_EVENT_TYPE_SQ_DRAINED:
304 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
305 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
306 return eqe->data.qp_srq.type;
307 case MLX5_EVENT_TYPE_CQ_ERROR:
308 case MLX5_EVENT_TYPE_XRQ_ERROR:
309 return 0;
310 case MLX5_EVENT_TYPE_DCT_DRAINED:
311 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
312 return MLX5_EVENT_QUEUE_TYPE_DCT;
313 default:
314 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
315 }
316 }
317
get_dec_obj_id(u64 obj_id)318 static u32 get_dec_obj_id(u64 obj_id)
319 {
320 return (obj_id & 0xffffffff);
321 }
322
323 /*
324 * As the obj_id in the firmware is not globally unique the object type
325 * must be considered upon checking for a valid object id.
326 * For that the opcode of the creator command is encoded as part of the obj_id.
327 */
get_enc_obj_id(u32 opcode,u32 obj_id)328 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
329 {
330 return ((u64)opcode << 32) | obj_id;
331 }
332
devx_get_created_obj_id(const void * in,const void * out,u16 opcode)333 static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)
334 {
335 switch (opcode) {
336 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
337 return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
338 case MLX5_CMD_OP_CREATE_UMEM:
339 return MLX5_GET(create_umem_out, out, umem_id);
340 case MLX5_CMD_OP_CREATE_MKEY:
341 return MLX5_GET(create_mkey_out, out, mkey_index);
342 case MLX5_CMD_OP_CREATE_CQ:
343 return MLX5_GET(create_cq_out, out, cqn);
344 case MLX5_CMD_OP_ALLOC_PD:
345 return MLX5_GET(alloc_pd_out, out, pd);
346 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
347 return MLX5_GET(alloc_transport_domain_out, out,
348 transport_domain);
349 case MLX5_CMD_OP_CREATE_RMP:
350 return MLX5_GET(create_rmp_out, out, rmpn);
351 case MLX5_CMD_OP_CREATE_SQ:
352 return MLX5_GET(create_sq_out, out, sqn);
353 case MLX5_CMD_OP_CREATE_RQ:
354 return MLX5_GET(create_rq_out, out, rqn);
355 case MLX5_CMD_OP_CREATE_RQT:
356 return MLX5_GET(create_rqt_out, out, rqtn);
357 case MLX5_CMD_OP_CREATE_TIR:
358 return MLX5_GET(create_tir_out, out, tirn);
359 case MLX5_CMD_OP_CREATE_TIS:
360 return MLX5_GET(create_tis_out, out, tisn);
361 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
362 return MLX5_GET(alloc_q_counter_out, out, counter_set_id);
363 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
364 return MLX5_GET(create_flow_table_out, out, table_id);
365 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
366 return MLX5_GET(create_flow_group_out, out, group_id);
367 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
368 return MLX5_GET(set_fte_in, in, flow_index);
369 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
370 return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
371 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
372 return MLX5_GET(alloc_packet_reformat_context_out, out,
373 packet_reformat_id);
374 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
375 return MLX5_GET(alloc_modify_header_context_out, out,
376 modify_header_id);
377 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
378 return MLX5_GET(create_scheduling_element_out, out,
379 scheduling_element_id);
380 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
381 return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
382 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
383 return MLX5_GET(set_l2_table_entry_in, in, table_index);
384 case MLX5_CMD_OP_CREATE_QP:
385 return MLX5_GET(create_qp_out, out, qpn);
386 case MLX5_CMD_OP_CREATE_SRQ:
387 return MLX5_GET(create_srq_out, out, srqn);
388 case MLX5_CMD_OP_CREATE_XRC_SRQ:
389 return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
390 case MLX5_CMD_OP_CREATE_DCT:
391 return MLX5_GET(create_dct_out, out, dctn);
392 case MLX5_CMD_OP_CREATE_XRQ:
393 return MLX5_GET(create_xrq_out, out, xrqn);
394 case MLX5_CMD_OP_ATTACH_TO_MCG:
395 return MLX5_GET(attach_to_mcg_in, in, qpn);
396 case MLX5_CMD_OP_ALLOC_XRCD:
397 return MLX5_GET(alloc_xrcd_out, out, xrcd);
398 case MLX5_CMD_OP_CREATE_PSV:
399 return MLX5_GET(create_psv_out, out, psv0_index);
400 default:
401 /* The entry must match to one of the devx_is_obj_create_cmd */
402 WARN_ON(true);
403 return 0;
404 }
405 }
406
devx_get_obj_id(const void * in)407 static u64 devx_get_obj_id(const void *in)
408 {
409 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
410 u64 obj_id;
411
412 switch (opcode) {
413 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
414 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
415 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
416 MLX5_GET(general_obj_in_cmd_hdr, in,
417 obj_type) << 16,
418 MLX5_GET(general_obj_in_cmd_hdr, in,
419 obj_id));
420 break;
421 case MLX5_CMD_OP_QUERY_MKEY:
422 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
423 MLX5_GET(query_mkey_in, in,
424 mkey_index));
425 break;
426 case MLX5_CMD_OP_QUERY_CQ:
427 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
428 MLX5_GET(query_cq_in, in, cqn));
429 break;
430 case MLX5_CMD_OP_MODIFY_CQ:
431 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
432 MLX5_GET(modify_cq_in, in, cqn));
433 break;
434 case MLX5_CMD_OP_QUERY_SQ:
435 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
436 MLX5_GET(query_sq_in, in, sqn));
437 break;
438 case MLX5_CMD_OP_MODIFY_SQ:
439 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
440 MLX5_GET(modify_sq_in, in, sqn));
441 break;
442 case MLX5_CMD_OP_QUERY_RQ:
443 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
444 MLX5_GET(query_rq_in, in, rqn));
445 break;
446 case MLX5_CMD_OP_MODIFY_RQ:
447 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
448 MLX5_GET(modify_rq_in, in, rqn));
449 break;
450 case MLX5_CMD_OP_QUERY_RMP:
451 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
452 MLX5_GET(query_rmp_in, in, rmpn));
453 break;
454 case MLX5_CMD_OP_MODIFY_RMP:
455 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
456 MLX5_GET(modify_rmp_in, in, rmpn));
457 break;
458 case MLX5_CMD_OP_QUERY_RQT:
459 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
460 MLX5_GET(query_rqt_in, in, rqtn));
461 break;
462 case MLX5_CMD_OP_MODIFY_RQT:
463 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
464 MLX5_GET(modify_rqt_in, in, rqtn));
465 break;
466 case MLX5_CMD_OP_QUERY_TIR:
467 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
468 MLX5_GET(query_tir_in, in, tirn));
469 break;
470 case MLX5_CMD_OP_MODIFY_TIR:
471 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
472 MLX5_GET(modify_tir_in, in, tirn));
473 break;
474 case MLX5_CMD_OP_QUERY_TIS:
475 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
476 MLX5_GET(query_tis_in, in, tisn));
477 break;
478 case MLX5_CMD_OP_MODIFY_TIS:
479 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
480 MLX5_GET(modify_tis_in, in, tisn));
481 break;
482 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
483 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
484 MLX5_GET(query_flow_table_in, in,
485 table_id));
486 break;
487 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
488 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
489 MLX5_GET(modify_flow_table_in, in,
490 table_id));
491 break;
492 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
493 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
494 MLX5_GET(query_flow_group_in, in,
495 group_id));
496 break;
497 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
498 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
499 MLX5_GET(query_fte_in, in,
500 flow_index));
501 break;
502 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
503 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
504 MLX5_GET(set_fte_in, in, flow_index));
505 break;
506 case MLX5_CMD_OP_QUERY_Q_COUNTER:
507 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
508 MLX5_GET(query_q_counter_in, in,
509 counter_set_id));
510 break;
511 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
512 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
513 MLX5_GET(query_flow_counter_in, in,
514 flow_counter_id));
515 break;
516 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
517 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
518 MLX5_GET(query_modify_header_context_in,
519 in, modify_header_id));
520 break;
521 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
522 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
523 MLX5_GET(query_scheduling_element_in,
524 in, scheduling_element_id));
525 break;
526 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
527 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
528 MLX5_GET(modify_scheduling_element_in,
529 in, scheduling_element_id));
530 break;
531 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
532 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
533 MLX5_GET(add_vxlan_udp_dport_in, in,
534 vxlan_udp_port));
535 break;
536 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
537 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
538 MLX5_GET(query_l2_table_entry_in, in,
539 table_index));
540 break;
541 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
542 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
543 MLX5_GET(set_l2_table_entry_in, in,
544 table_index));
545 break;
546 case MLX5_CMD_OP_QUERY_QP:
547 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
548 MLX5_GET(query_qp_in, in, qpn));
549 break;
550 case MLX5_CMD_OP_RST2INIT_QP:
551 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
552 MLX5_GET(rst2init_qp_in, in, qpn));
553 break;
554 case MLX5_CMD_OP_INIT2INIT_QP:
555 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
556 MLX5_GET(init2init_qp_in, in, qpn));
557 break;
558 case MLX5_CMD_OP_INIT2RTR_QP:
559 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
560 MLX5_GET(init2rtr_qp_in, in, qpn));
561 break;
562 case MLX5_CMD_OP_RTR2RTS_QP:
563 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
564 MLX5_GET(rtr2rts_qp_in, in, qpn));
565 break;
566 case MLX5_CMD_OP_RTS2RTS_QP:
567 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
568 MLX5_GET(rts2rts_qp_in, in, qpn));
569 break;
570 case MLX5_CMD_OP_SQERR2RTS_QP:
571 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
572 MLX5_GET(sqerr2rts_qp_in, in, qpn));
573 break;
574 case MLX5_CMD_OP_2ERR_QP:
575 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
576 MLX5_GET(qp_2err_in, in, qpn));
577 break;
578 case MLX5_CMD_OP_2RST_QP:
579 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
580 MLX5_GET(qp_2rst_in, in, qpn));
581 break;
582 case MLX5_CMD_OP_QUERY_DCT:
583 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
584 MLX5_GET(query_dct_in, in, dctn));
585 break;
586 case MLX5_CMD_OP_QUERY_XRQ:
587 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
588 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
589 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
590 MLX5_GET(query_xrq_in, in, xrqn));
591 break;
592 case MLX5_CMD_OP_QUERY_XRC_SRQ:
593 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
594 MLX5_GET(query_xrc_srq_in, in,
595 xrc_srqn));
596 break;
597 case MLX5_CMD_OP_ARM_XRC_SRQ:
598 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
599 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
600 break;
601 case MLX5_CMD_OP_QUERY_SRQ:
602 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
603 MLX5_GET(query_srq_in, in, srqn));
604 break;
605 case MLX5_CMD_OP_ARM_RQ:
606 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
607 MLX5_GET(arm_rq_in, in, srq_number));
608 break;
609 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
610 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
611 MLX5_GET(drain_dct_in, in, dctn));
612 break;
613 case MLX5_CMD_OP_ARM_XRQ:
614 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
615 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
616 case MLX5_CMD_OP_MODIFY_XRQ:
617 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
618 MLX5_GET(arm_xrq_in, in, xrqn));
619 break;
620 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
621 obj_id = get_enc_obj_id
622 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
623 MLX5_GET(query_packet_reformat_context_in,
624 in, packet_reformat_id));
625 break;
626 default:
627 obj_id = 0;
628 }
629
630 return obj_id;
631 }
632
devx_is_valid_obj_id(struct uverbs_attr_bundle * attrs,struct ib_uobject * uobj,const void * in)633 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
634 struct ib_uobject *uobj, const void *in)
635 {
636 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
637 u64 obj_id = devx_get_obj_id(in);
638
639 if (!obj_id)
640 return false;
641
642 switch (uobj_get_object_id(uobj)) {
643 case UVERBS_OBJECT_CQ:
644 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
645 to_mcq(uobj->object)->mcq.cqn) ==
646 obj_id;
647
648 case UVERBS_OBJECT_SRQ:
649 {
650 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
651 u16 opcode;
652
653 switch (srq->common.res) {
654 case MLX5_RES_XSRQ:
655 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
656 break;
657 case MLX5_RES_XRQ:
658 opcode = MLX5_CMD_OP_CREATE_XRQ;
659 break;
660 default:
661 if (!dev->mdev->issi)
662 opcode = MLX5_CMD_OP_CREATE_SRQ;
663 else
664 opcode = MLX5_CMD_OP_CREATE_RMP;
665 }
666
667 return get_enc_obj_id(opcode,
668 to_msrq(uobj->object)->msrq.srqn) ==
669 obj_id;
670 }
671
672 case UVERBS_OBJECT_QP:
673 {
674 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
675
676 if (qp->type == IB_QPT_RAW_PACKET ||
677 (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
678 struct mlx5_ib_raw_packet_qp *raw_packet_qp =
679 &qp->raw_packet_qp;
680 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
681 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
682
683 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
684 rq->base.mqp.qpn) == obj_id ||
685 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
686 sq->base.mqp.qpn) == obj_id ||
687 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
688 rq->tirn) == obj_id ||
689 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
690 sq->tisn) == obj_id);
691 }
692
693 if (qp->type == MLX5_IB_QPT_DCT)
694 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
695 qp->dct.mdct.mqp.qpn) == obj_id;
696 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
697 qp->ibqp.qp_num) == obj_id;
698 }
699
700 case UVERBS_OBJECT_WQ:
701 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
702 to_mrwq(uobj->object)->core_qp.qpn) ==
703 obj_id;
704
705 case UVERBS_OBJECT_RWQ_IND_TBL:
706 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
707 to_mrwq_ind_table(uobj->object)->rqtn) ==
708 obj_id;
709
710 case MLX5_IB_OBJECT_DEVX_OBJ:
711 {
712 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
713 struct devx_obj *devx_uobj = uobj->object;
714
715 if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &&
716 devx_uobj->flow_counter_bulk_size) {
717 u64 end;
718
719 end = devx_uobj->obj_id +
720 devx_uobj->flow_counter_bulk_size;
721 return devx_uobj->obj_id <= obj_id && end > obj_id;
722 }
723
724 return devx_uobj->obj_id == obj_id;
725 }
726
727 default:
728 return false;
729 }
730 }
731
devx_set_umem_valid(const void * in)732 static void devx_set_umem_valid(const void *in)
733 {
734 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
735
736 switch (opcode) {
737 case MLX5_CMD_OP_CREATE_MKEY:
738 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
739 break;
740 case MLX5_CMD_OP_CREATE_CQ:
741 {
742 void *cqc;
743
744 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
745 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
746 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
747 break;
748 }
749 case MLX5_CMD_OP_CREATE_QP:
750 {
751 void *qpc;
752
753 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
754 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
755 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
756 break;
757 }
758
759 case MLX5_CMD_OP_CREATE_RQ:
760 {
761 void *rqc, *wq;
762
763 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
764 wq = MLX5_ADDR_OF(rqc, rqc, wq);
765 MLX5_SET(wq, wq, dbr_umem_valid, 1);
766 MLX5_SET(wq, wq, wq_umem_valid, 1);
767 break;
768 }
769
770 case MLX5_CMD_OP_CREATE_SQ:
771 {
772 void *sqc, *wq;
773
774 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
775 wq = MLX5_ADDR_OF(sqc, sqc, wq);
776 MLX5_SET(wq, wq, dbr_umem_valid, 1);
777 MLX5_SET(wq, wq, wq_umem_valid, 1);
778 break;
779 }
780
781 case MLX5_CMD_OP_MODIFY_CQ:
782 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
783 break;
784
785 case MLX5_CMD_OP_CREATE_RMP:
786 {
787 void *rmpc, *wq;
788
789 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
790 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
791 MLX5_SET(wq, wq, dbr_umem_valid, 1);
792 MLX5_SET(wq, wq, wq_umem_valid, 1);
793 break;
794 }
795
796 case MLX5_CMD_OP_CREATE_XRQ:
797 {
798 void *xrqc, *wq;
799
800 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
801 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
802 MLX5_SET(wq, wq, dbr_umem_valid, 1);
803 MLX5_SET(wq, wq, wq_umem_valid, 1);
804 break;
805 }
806
807 case MLX5_CMD_OP_CREATE_XRC_SRQ:
808 {
809 void *xrc_srqc;
810
811 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
812 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
813 xrc_srq_context_entry);
814 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
815 break;
816 }
817
818 default:
819 return;
820 }
821 }
822
devx_is_obj_create_cmd(const void * in,u16 * opcode)823 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
824 {
825 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
826
827 switch (*opcode) {
828 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
829 case MLX5_CMD_OP_CREATE_MKEY:
830 case MLX5_CMD_OP_CREATE_CQ:
831 case MLX5_CMD_OP_ALLOC_PD:
832 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
833 case MLX5_CMD_OP_CREATE_RMP:
834 case MLX5_CMD_OP_CREATE_SQ:
835 case MLX5_CMD_OP_CREATE_RQ:
836 case MLX5_CMD_OP_CREATE_RQT:
837 case MLX5_CMD_OP_CREATE_TIR:
838 case MLX5_CMD_OP_CREATE_TIS:
839 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
840 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
841 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
842 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
843 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
844 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
845 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
846 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
847 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
848 case MLX5_CMD_OP_CREATE_QP:
849 case MLX5_CMD_OP_CREATE_SRQ:
850 case MLX5_CMD_OP_CREATE_XRC_SRQ:
851 case MLX5_CMD_OP_CREATE_DCT:
852 case MLX5_CMD_OP_CREATE_XRQ:
853 case MLX5_CMD_OP_ATTACH_TO_MCG:
854 case MLX5_CMD_OP_ALLOC_XRCD:
855 return true;
856 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
857 {
858 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
859 if (op_mod == 0)
860 return true;
861 return false;
862 }
863 case MLX5_CMD_OP_CREATE_PSV:
864 {
865 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
866
867 if (num_psv == 1)
868 return true;
869 return false;
870 }
871 default:
872 return false;
873 }
874 }
875
devx_is_obj_modify_cmd(const void * in)876 static bool devx_is_obj_modify_cmd(const void *in)
877 {
878 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
879
880 switch (opcode) {
881 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
882 case MLX5_CMD_OP_MODIFY_CQ:
883 case MLX5_CMD_OP_MODIFY_RMP:
884 case MLX5_CMD_OP_MODIFY_SQ:
885 case MLX5_CMD_OP_MODIFY_RQ:
886 case MLX5_CMD_OP_MODIFY_RQT:
887 case MLX5_CMD_OP_MODIFY_TIR:
888 case MLX5_CMD_OP_MODIFY_TIS:
889 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
890 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
891 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
892 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
893 case MLX5_CMD_OP_RST2INIT_QP:
894 case MLX5_CMD_OP_INIT2RTR_QP:
895 case MLX5_CMD_OP_INIT2INIT_QP:
896 case MLX5_CMD_OP_RTR2RTS_QP:
897 case MLX5_CMD_OP_RTS2RTS_QP:
898 case MLX5_CMD_OP_SQERR2RTS_QP:
899 case MLX5_CMD_OP_2ERR_QP:
900 case MLX5_CMD_OP_2RST_QP:
901 case MLX5_CMD_OP_ARM_XRC_SRQ:
902 case MLX5_CMD_OP_ARM_RQ:
903 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
904 case MLX5_CMD_OP_ARM_XRQ:
905 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
906 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
907 case MLX5_CMD_OP_MODIFY_XRQ:
908 return true;
909 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
910 {
911 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
912
913 if (op_mod == 1)
914 return true;
915 return false;
916 }
917 default:
918 return false;
919 }
920 }
921
devx_is_obj_query_cmd(const void * in)922 static bool devx_is_obj_query_cmd(const void *in)
923 {
924 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
925
926 switch (opcode) {
927 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
928 case MLX5_CMD_OP_QUERY_MKEY:
929 case MLX5_CMD_OP_QUERY_CQ:
930 case MLX5_CMD_OP_QUERY_RMP:
931 case MLX5_CMD_OP_QUERY_SQ:
932 case MLX5_CMD_OP_QUERY_RQ:
933 case MLX5_CMD_OP_QUERY_RQT:
934 case MLX5_CMD_OP_QUERY_TIR:
935 case MLX5_CMD_OP_QUERY_TIS:
936 case MLX5_CMD_OP_QUERY_Q_COUNTER:
937 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
938 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
939 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
940 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
941 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
942 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
943 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
944 case MLX5_CMD_OP_QUERY_QP:
945 case MLX5_CMD_OP_QUERY_SRQ:
946 case MLX5_CMD_OP_QUERY_XRC_SRQ:
947 case MLX5_CMD_OP_QUERY_DCT:
948 case MLX5_CMD_OP_QUERY_XRQ:
949 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
950 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
951 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
952 return true;
953 default:
954 return false;
955 }
956 }
957
devx_is_whitelist_cmd(void * in)958 static bool devx_is_whitelist_cmd(void *in)
959 {
960 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
961
962 switch (opcode) {
963 case MLX5_CMD_OP_QUERY_HCA_CAP:
964 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
965 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
966 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
967 return true;
968 default:
969 return false;
970 }
971 }
972
devx_get_uid(struct mlx5_ib_ucontext * c,void * cmd_in)973 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
974 {
975 if (devx_is_whitelist_cmd(cmd_in)) {
976 struct mlx5_ib_dev *dev;
977
978 if (c->devx_uid)
979 return c->devx_uid;
980
981 dev = to_mdev(c->ibucontext.device);
982 if (dev->devx_whitelist_uid)
983 return dev->devx_whitelist_uid;
984
985 return -EOPNOTSUPP;
986 }
987
988 if (!c->devx_uid)
989 return -EINVAL;
990
991 return c->devx_uid;
992 }
993
devx_is_general_cmd(void * in,struct mlx5_ib_dev * dev)994 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
995 {
996 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
997
998 /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
999 if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
1000 MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
1001 (opcode >= MLX5_CMD_OP_GENERAL_START &&
1002 opcode < MLX5_CMD_OP_GENERAL_END))
1003 return true;
1004
1005 switch (opcode) {
1006 case MLX5_CMD_OP_QUERY_HCA_CAP:
1007 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
1008 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
1009 case MLX5_CMD_OP_QUERY_VPORT_STATE:
1010 case MLX5_CMD_OP_QUERY_ADAPTER:
1011 case MLX5_CMD_OP_QUERY_ISSI:
1012 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
1013 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
1014 case MLX5_CMD_OP_QUERY_VNIC_ENV:
1015 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
1016 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
1017 case MLX5_CMD_OP_NOP:
1018 case MLX5_CMD_OP_QUERY_CONG_STATUS:
1019 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
1020 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
1021 case MLX5_CMD_OP_QUERY_LAG:
1022 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
1023 return true;
1024 default:
1025 return false;
1026 }
1027 }
1028
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)1029 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
1030 struct uverbs_attr_bundle *attrs)
1031 {
1032 struct mlx5_ib_ucontext *c;
1033 struct mlx5_ib_dev *dev;
1034 int user_vector;
1035 int dev_eqn;
1036 int err;
1037
1038 if (uverbs_copy_from(&user_vector, attrs,
1039 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
1040 return -EFAULT;
1041
1042 c = devx_ufile2uctx(attrs);
1043 if (IS_ERR(c))
1044 return PTR_ERR(c);
1045 dev = to_mdev(c->ibucontext.device);
1046
1047 err = mlx5_comp_eqn_get(dev->mdev, user_vector, &dev_eqn);
1048 if (err < 0)
1049 return err;
1050
1051 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1052 &dev_eqn, sizeof(dev_eqn)))
1053 return -EFAULT;
1054
1055 return 0;
1056 }
1057
1058 /*
1059 *Security note:
1060 * The hardware protection mechanism works like this: Each device object that
1061 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
1062 * the device specification manual) upon its creation. Then upon doorbell,
1063 * hardware fetches the object context for which the doorbell was rang, and
1064 * validates that the UAR through which the DB was rang matches the UAR ID
1065 * of the object.
1066 * If no match the doorbell is silently ignored by the hardware. Of course,
1067 * the user cannot ring a doorbell on a UAR that was not mapped to it.
1068 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1069 * mailboxes (except tagging them with UID), we expose to the user its UAR
1070 * ID, so it can embed it in these objects in the expected specification
1071 * format. So the only thing the user can do is hurt itself by creating a
1072 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1073 * may ring a doorbell on its objects.
1074 * The consequence of that will be that another user can schedule a QP/SQ
1075 * of the buggy user for execution (just insert it to the hardware schedule
1076 * queue or arm its CQ for event generation), no further harm is expected.
1077 */
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)1078 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1079 struct uverbs_attr_bundle *attrs)
1080 {
1081 struct mlx5_ib_ucontext *c;
1082 struct mlx5_ib_dev *dev;
1083 u32 user_idx;
1084 s32 dev_idx;
1085
1086 c = devx_ufile2uctx(attrs);
1087 if (IS_ERR(c))
1088 return PTR_ERR(c);
1089 dev = to_mdev(c->ibucontext.device);
1090
1091 if (uverbs_copy_from(&user_idx, attrs,
1092 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1093 return -EFAULT;
1094
1095 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1096 if (dev_idx < 0)
1097 return dev_idx;
1098
1099 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1100 &dev_idx, sizeof(dev_idx)))
1101 return -EFAULT;
1102
1103 return 0;
1104 }
1105
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)1106 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1107 struct uverbs_attr_bundle *attrs)
1108 {
1109 struct mlx5_ib_ucontext *c;
1110 struct mlx5_ib_dev *dev;
1111 void *cmd_in = uverbs_attr_get_alloced_ptr(
1112 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1113 int cmd_out_len = uverbs_attr_get_len(attrs,
1114 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1115 void *cmd_out;
1116 int err, err2;
1117 int uid;
1118
1119 c = devx_ufile2uctx(attrs);
1120 if (IS_ERR(c))
1121 return PTR_ERR(c);
1122 dev = to_mdev(c->ibucontext.device);
1123
1124 uid = devx_get_uid(c, cmd_in);
1125 if (uid < 0)
1126 return uid;
1127
1128 /* Only white list of some general HCA commands are allowed for this method. */
1129 if (!devx_is_general_cmd(cmd_in, dev))
1130 return -EINVAL;
1131
1132 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1133 if (IS_ERR(cmd_out))
1134 return PTR_ERR(cmd_out);
1135
1136 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1137 err = mlx5_cmd_do(dev->mdev, cmd_in,
1138 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1139 cmd_out, cmd_out_len);
1140 if (err && err != -EREMOTEIO)
1141 return err;
1142
1143 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1144 cmd_out_len);
1145
1146 return err2 ?: err;
1147 }
1148
devx_obj_build_destroy_cmd(void * in,void * out,void * din,u32 * dinlen,u32 * obj_id)1149 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1150 u32 *dinlen,
1151 u32 *obj_id)
1152 {
1153 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
1154 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1155
1156 *obj_id = devx_get_created_obj_id(in, out, opcode);
1157 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1158 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1159
1160 switch (opcode) {
1161 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1162 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1163 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1164 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,
1165 MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));
1166 break;
1167
1168 case MLX5_CMD_OP_CREATE_UMEM:
1169 MLX5_SET(destroy_umem_in, din, opcode,
1170 MLX5_CMD_OP_DESTROY_UMEM);
1171 MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);
1172 break;
1173 case MLX5_CMD_OP_CREATE_MKEY:
1174 MLX5_SET(destroy_mkey_in, din, opcode,
1175 MLX5_CMD_OP_DESTROY_MKEY);
1176 MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);
1177 break;
1178 case MLX5_CMD_OP_CREATE_CQ:
1179 MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1180 MLX5_SET(destroy_cq_in, din, cqn, *obj_id);
1181 break;
1182 case MLX5_CMD_OP_ALLOC_PD:
1183 MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1184 MLX5_SET(dealloc_pd_in, din, pd, *obj_id);
1185 break;
1186 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1187 MLX5_SET(dealloc_transport_domain_in, din, opcode,
1188 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1189 MLX5_SET(dealloc_transport_domain_in, din, transport_domain,
1190 *obj_id);
1191 break;
1192 case MLX5_CMD_OP_CREATE_RMP:
1193 MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1194 MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);
1195 break;
1196 case MLX5_CMD_OP_CREATE_SQ:
1197 MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1198 MLX5_SET(destroy_sq_in, din, sqn, *obj_id);
1199 break;
1200 case MLX5_CMD_OP_CREATE_RQ:
1201 MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1202 MLX5_SET(destroy_rq_in, din, rqn, *obj_id);
1203 break;
1204 case MLX5_CMD_OP_CREATE_RQT:
1205 MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1206 MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);
1207 break;
1208 case MLX5_CMD_OP_CREATE_TIR:
1209 MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1210 MLX5_SET(destroy_tir_in, din, tirn, *obj_id);
1211 break;
1212 case MLX5_CMD_OP_CREATE_TIS:
1213 MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1214 MLX5_SET(destroy_tis_in, din, tisn, *obj_id);
1215 break;
1216 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1217 MLX5_SET(dealloc_q_counter_in, din, opcode,
1218 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1219 MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);
1220 break;
1221 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1222 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1223 MLX5_SET(destroy_flow_table_in, din, other_vport,
1224 MLX5_GET(create_flow_table_in, in, other_vport));
1225 MLX5_SET(destroy_flow_table_in, din, vport_number,
1226 MLX5_GET(create_flow_table_in, in, vport_number));
1227 MLX5_SET(destroy_flow_table_in, din, table_type,
1228 MLX5_GET(create_flow_table_in, in, table_type));
1229 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1230 MLX5_SET(destroy_flow_table_in, din, opcode,
1231 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1232 break;
1233 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1234 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1235 MLX5_SET(destroy_flow_group_in, din, other_vport,
1236 MLX5_GET(create_flow_group_in, in, other_vport));
1237 MLX5_SET(destroy_flow_group_in, din, vport_number,
1238 MLX5_GET(create_flow_group_in, in, vport_number));
1239 MLX5_SET(destroy_flow_group_in, din, table_type,
1240 MLX5_GET(create_flow_group_in, in, table_type));
1241 MLX5_SET(destroy_flow_group_in, din, table_id,
1242 MLX5_GET(create_flow_group_in, in, table_id));
1243 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1244 MLX5_SET(destroy_flow_group_in, din, opcode,
1245 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1246 break;
1247 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1248 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1249 MLX5_SET(delete_fte_in, din, other_vport,
1250 MLX5_GET(set_fte_in, in, other_vport));
1251 MLX5_SET(delete_fte_in, din, vport_number,
1252 MLX5_GET(set_fte_in, in, vport_number));
1253 MLX5_SET(delete_fte_in, din, table_type,
1254 MLX5_GET(set_fte_in, in, table_type));
1255 MLX5_SET(delete_fte_in, din, table_id,
1256 MLX5_GET(set_fte_in, in, table_id));
1257 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1258 MLX5_SET(delete_fte_in, din, opcode,
1259 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1260 break;
1261 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1262 MLX5_SET(dealloc_flow_counter_in, din, opcode,
1263 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1264 MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,
1265 *obj_id);
1266 break;
1267 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1268 MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,
1269 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1270 MLX5_SET(dealloc_packet_reformat_context_in, din,
1271 packet_reformat_id, *obj_id);
1272 break;
1273 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1274 MLX5_SET(dealloc_modify_header_context_in, din, opcode,
1275 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1276 MLX5_SET(dealloc_modify_header_context_in, din,
1277 modify_header_id, *obj_id);
1278 break;
1279 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1280 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1281 MLX5_SET(destroy_scheduling_element_in, din,
1282 scheduling_hierarchy,
1283 MLX5_GET(create_scheduling_element_in, in,
1284 scheduling_hierarchy));
1285 MLX5_SET(destroy_scheduling_element_in, din,
1286 scheduling_element_id, *obj_id);
1287 MLX5_SET(destroy_scheduling_element_in, din, opcode,
1288 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1289 break;
1290 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1291 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1292 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1293 MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,
1294 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1295 break;
1296 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1297 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1298 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1299 MLX5_SET(delete_l2_table_entry_in, din, opcode,
1300 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1301 break;
1302 case MLX5_CMD_OP_CREATE_QP:
1303 MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1304 MLX5_SET(destroy_qp_in, din, qpn, *obj_id);
1305 break;
1306 case MLX5_CMD_OP_CREATE_SRQ:
1307 MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1308 MLX5_SET(destroy_srq_in, din, srqn, *obj_id);
1309 break;
1310 case MLX5_CMD_OP_CREATE_XRC_SRQ:
1311 MLX5_SET(destroy_xrc_srq_in, din, opcode,
1312 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1313 MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);
1314 break;
1315 case MLX5_CMD_OP_CREATE_DCT:
1316 MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1317 MLX5_SET(destroy_dct_in, din, dctn, *obj_id);
1318 break;
1319 case MLX5_CMD_OP_CREATE_XRQ:
1320 MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1321 MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);
1322 break;
1323 case MLX5_CMD_OP_ATTACH_TO_MCG:
1324 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1325 MLX5_SET(detach_from_mcg_in, din, qpn,
1326 MLX5_GET(attach_to_mcg_in, in, qpn));
1327 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1328 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1329 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1330 MLX5_SET(detach_from_mcg_in, din, opcode,
1331 MLX5_CMD_OP_DETACH_FROM_MCG);
1332 MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);
1333 break;
1334 case MLX5_CMD_OP_ALLOC_XRCD:
1335 MLX5_SET(dealloc_xrcd_in, din, opcode,
1336 MLX5_CMD_OP_DEALLOC_XRCD);
1337 MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);
1338 break;
1339 case MLX5_CMD_OP_CREATE_PSV:
1340 MLX5_SET(destroy_psv_in, din, opcode,
1341 MLX5_CMD_OP_DESTROY_PSV);
1342 MLX5_SET(destroy_psv_in, din, psvn, *obj_id);
1343 break;
1344 default:
1345 /* The entry must match to one of the devx_is_obj_create_cmd */
1346 WARN_ON(true);
1347 break;
1348 }
1349 }
1350
devx_handle_mkey_indirect(struct devx_obj * obj,struct mlx5_ib_dev * dev,void * in,void * out)1351 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1352 struct mlx5_ib_dev *dev,
1353 void *in, void *out)
1354 {
1355 struct mlx5_ib_mkey *mkey = &obj->mkey;
1356 void *mkc;
1357 u8 key;
1358
1359 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1360 key = MLX5_GET(mkc, mkc, mkey_7_0);
1361 mkey->key = mlx5_idx_to_mkey(
1362 MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1363 mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1364 mkey->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1365 init_waitqueue_head(&mkey->wait);
1366
1367 return mlx5r_store_odp_mkey(dev, mkey);
1368 }
1369
devx_handle_mkey_create(struct mlx5_ib_dev * dev,struct devx_obj * obj,void * in,int in_len)1370 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1371 struct devx_obj *obj,
1372 void *in, int in_len)
1373 {
1374 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1375 MLX5_FLD_SZ_BYTES(create_mkey_in,
1376 memory_key_mkey_entry);
1377 void *mkc;
1378 u8 access_mode;
1379
1380 if (in_len < min_len)
1381 return -EINVAL;
1382
1383 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1384
1385 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1386 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1387
1388 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1389 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1390 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1391 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1392 return 0;
1393 }
1394
1395 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1396 return 0;
1397 }
1398
devx_cleanup_subscription(struct mlx5_ib_dev * dev,struct devx_event_subscription * sub)1399 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1400 struct devx_event_subscription *sub)
1401 {
1402 struct devx_event *event;
1403 struct devx_obj_event *xa_val_level2;
1404
1405 if (sub->is_cleaned)
1406 return;
1407
1408 sub->is_cleaned = 1;
1409 list_del_rcu(&sub->xa_list);
1410
1411 if (list_empty(&sub->obj_list))
1412 return;
1413
1414 list_del_rcu(&sub->obj_list);
1415 /* check whether key level 1 for this obj_sub_list is empty */
1416 event = xa_load(&dev->devx_event_table.event_xa,
1417 sub->xa_key_level1);
1418 WARN_ON(!event);
1419
1420 xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1421 if (list_empty(&xa_val_level2->obj_sub_list)) {
1422 xa_erase(&event->object_ids,
1423 sub->xa_key_level2);
1424 kfree_rcu(xa_val_level2, rcu);
1425 }
1426 }
1427
devx_obj_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)1428 static int devx_obj_cleanup(struct ib_uobject *uobject,
1429 enum rdma_remove_reason why,
1430 struct uverbs_attr_bundle *attrs)
1431 {
1432 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1433 struct mlx5_devx_event_table *devx_event_table;
1434 struct devx_obj *obj = uobject->object;
1435 struct devx_event_subscription *sub_entry, *tmp;
1436 struct mlx5_ib_dev *dev;
1437 int ret;
1438
1439 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1440 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
1441 xa_erase(&obj->ib_dev->odp_mkeys,
1442 mlx5_base_mkey(obj->mkey.key)))
1443 /*
1444 * The pagefault_single_data_segment() does commands against
1445 * the mmkey, we must wait for that to stop before freeing the
1446 * mkey, as another allocation could get the same mkey #.
1447 */
1448 mlx5r_deref_wait_odp_mkey(&obj->mkey);
1449
1450 if (obj->flags & DEVX_OBJ_FLAGS_HW_FREED)
1451 ret = 0;
1452 else if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1453 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1454 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1455 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1456 else
1457 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1458 obj->dinlen, out, sizeof(out));
1459 if (ret)
1460 return ret;
1461
1462 devx_event_table = &dev->devx_event_table;
1463
1464 mutex_lock(&devx_event_table->event_xa_lock);
1465 list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1466 devx_cleanup_subscription(dev, sub_entry);
1467 mutex_unlock(&devx_event_table->event_xa_lock);
1468
1469 kfree(obj);
1470 return ret;
1471 }
1472
devx_cq_comp(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe)1473 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1474 {
1475 struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1476 struct mlx5_devx_event_table *table;
1477 struct devx_event *event;
1478 struct devx_obj_event *obj_event;
1479 u32 obj_id = mcq->cqn;
1480
1481 table = &obj->ib_dev->devx_event_table;
1482 rcu_read_lock();
1483 event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1484 if (!event)
1485 goto out;
1486
1487 obj_event = xa_load(&event->object_ids, obj_id);
1488 if (!obj_event)
1489 goto out;
1490
1491 dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1492 out:
1493 rcu_read_unlock();
1494 }
1495
is_apu_cq(struct mlx5_ib_dev * dev,const void * in)1496 static bool is_apu_cq(struct mlx5_ib_dev *dev, const void *in)
1497 {
1498 if (!MLX5_CAP_GEN(dev->mdev, apu) ||
1499 !MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), apu_cq))
1500 return false;
1501
1502 return true;
1503 }
1504
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)1505 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1506 struct uverbs_attr_bundle *attrs)
1507 {
1508 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1509 int cmd_out_len = uverbs_attr_get_len(attrs,
1510 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1511 int cmd_in_len = uverbs_attr_get_len(attrs,
1512 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1513 void *cmd_out;
1514 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1515 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1516 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1517 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1518 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1519 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1520 struct devx_obj *obj;
1521 u16 obj_type = 0;
1522 int err, err2 = 0;
1523 int uid;
1524 u32 obj_id;
1525 u16 opcode;
1526
1527 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1528 return -EINVAL;
1529
1530 uid = devx_get_uid(c, cmd_in);
1531 if (uid < 0)
1532 return uid;
1533
1534 if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1535 return -EINVAL;
1536
1537 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1538 if (IS_ERR(cmd_out))
1539 return PTR_ERR(cmd_out);
1540
1541 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1542 if (!obj)
1543 return -ENOMEM;
1544
1545 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1546 if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1547 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1548 if (err)
1549 goto obj_free;
1550 } else {
1551 devx_set_umem_valid(cmd_in);
1552 }
1553
1554 if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1555 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1556 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1557 cmd_in_len, cmd_out, cmd_out_len);
1558 } else if (opcode == MLX5_CMD_OP_CREATE_CQ &&
1559 !is_apu_cq(dev, cmd_in)) {
1560 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1561 obj->core_cq.comp = devx_cq_comp;
1562 err = mlx5_create_cq(dev->mdev, &obj->core_cq,
1563 cmd_in, cmd_in_len, cmd_out,
1564 cmd_out_len);
1565 } else {
1566 err = mlx5_cmd_do(dev->mdev, cmd_in, cmd_in_len,
1567 cmd_out, cmd_out_len);
1568 }
1569
1570 if (err == -EREMOTEIO)
1571 err2 = uverbs_copy_to(attrs,
1572 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1573 cmd_out, cmd_out_len);
1574 if (err)
1575 goto obj_free;
1576
1577 if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1578 u32 bulk = MLX5_GET(alloc_flow_counter_in,
1579 cmd_in,
1580 flow_counter_bulk_log_size);
1581
1582 if (bulk)
1583 bulk = 1 << bulk;
1584 else
1585 bulk = 128UL * MLX5_GET(alloc_flow_counter_in,
1586 cmd_in,
1587 flow_counter_bulk);
1588 obj->flow_counter_bulk_size = bulk;
1589 }
1590
1591 uobj->object = obj;
1592 INIT_LIST_HEAD(&obj->event_sub);
1593 obj->ib_dev = dev;
1594 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1595 &obj_id);
1596 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1597
1598 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1599 if (err)
1600 goto obj_destroy;
1601
1602 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1603 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1604 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1605
1606 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1607 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1608 if (err)
1609 goto obj_destroy;
1610 }
1611 return 0;
1612
1613 obj_destroy:
1614 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1615 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1616 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1617 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1618 else
1619 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1620 sizeof(out));
1621 obj_free:
1622 kfree(obj);
1623 return err2 ?: err;
1624 }
1625
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)1626 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1627 struct uverbs_attr_bundle *attrs)
1628 {
1629 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1630 int cmd_out_len = uverbs_attr_get_len(attrs,
1631 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1632 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1633 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1634 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1635 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1636 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1637 void *cmd_out;
1638 int err, err2;
1639 int uid;
1640
1641 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1642 return -EINVAL;
1643
1644 uid = devx_get_uid(c, cmd_in);
1645 if (uid < 0)
1646 return uid;
1647
1648 if (!devx_is_obj_modify_cmd(cmd_in))
1649 return -EINVAL;
1650
1651 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1652 return -EINVAL;
1653
1654 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1655 if (IS_ERR(cmd_out))
1656 return PTR_ERR(cmd_out);
1657
1658 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1659 devx_set_umem_valid(cmd_in);
1660
1661 err = mlx5_cmd_do(mdev->mdev, cmd_in,
1662 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1663 cmd_out, cmd_out_len);
1664 if (err && err != -EREMOTEIO)
1665 return err;
1666
1667 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1668 cmd_out, cmd_out_len);
1669
1670 return err2 ?: err;
1671 }
1672
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)1673 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1674 struct uverbs_attr_bundle *attrs)
1675 {
1676 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1677 int cmd_out_len = uverbs_attr_get_len(attrs,
1678 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1679 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1680 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1681 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1682 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1683 void *cmd_out;
1684 int err, err2;
1685 int uid;
1686 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1687
1688 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1689 return -EINVAL;
1690
1691 uid = devx_get_uid(c, cmd_in);
1692 if (uid < 0)
1693 return uid;
1694
1695 if (!devx_is_obj_query_cmd(cmd_in))
1696 return -EINVAL;
1697
1698 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1699 return -EINVAL;
1700
1701 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1702 if (IS_ERR(cmd_out))
1703 return PTR_ERR(cmd_out);
1704
1705 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1706 err = mlx5_cmd_do(mdev->mdev, cmd_in,
1707 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1708 cmd_out, cmd_out_len);
1709 if (err && err != -EREMOTEIO)
1710 return err;
1711
1712 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1713 cmd_out, cmd_out_len);
1714
1715 return err2 ?: err;
1716 }
1717
1718 struct devx_async_event_queue {
1719 spinlock_t lock;
1720 wait_queue_head_t poll_wait;
1721 struct list_head event_list;
1722 atomic_t bytes_in_use;
1723 u8 is_destroyed:1;
1724 };
1725
1726 struct devx_async_cmd_event_file {
1727 struct ib_uobject uobj;
1728 struct devx_async_event_queue ev_queue;
1729 struct mlx5_async_ctx async_ctx;
1730 };
1731
devx_init_event_queue(struct devx_async_event_queue * ev_queue)1732 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1733 {
1734 spin_lock_init(&ev_queue->lock);
1735 INIT_LIST_HEAD(&ev_queue->event_list);
1736 init_waitqueue_head(&ev_queue->poll_wait);
1737 atomic_set(&ev_queue->bytes_in_use, 0);
1738 ev_queue->is_destroyed = 0;
1739 }
1740
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)1741 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1742 struct uverbs_attr_bundle *attrs)
1743 {
1744 struct devx_async_cmd_event_file *ev_file;
1745
1746 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1747 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1748 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1749
1750 ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1751 uobj);
1752 devx_init_event_queue(&ev_file->ev_queue);
1753 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1754 return 0;
1755 }
1756
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)1757 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1758 struct uverbs_attr_bundle *attrs)
1759 {
1760 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1761 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1762 struct devx_async_event_file *ev_file;
1763 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1764 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1765 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1766 u32 flags;
1767 int err;
1768
1769 err = uverbs_get_flags32(&flags, attrs,
1770 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1771 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1772
1773 if (err)
1774 return err;
1775
1776 ev_file = container_of(uobj, struct devx_async_event_file,
1777 uobj);
1778 spin_lock_init(&ev_file->lock);
1779 INIT_LIST_HEAD(&ev_file->event_list);
1780 init_waitqueue_head(&ev_file->poll_wait);
1781 if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1782 ev_file->omit_data = 1;
1783 INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1784 ev_file->dev = dev;
1785 get_device(&dev->ib_dev.dev);
1786 return 0;
1787 }
1788
devx_query_callback(int status,struct mlx5_async_work * context)1789 static void devx_query_callback(int status, struct mlx5_async_work *context)
1790 {
1791 struct devx_async_data *async_data =
1792 container_of(context, struct devx_async_data, cb_work);
1793 struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1794 struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1795 unsigned long flags;
1796
1797 /*
1798 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1799 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1800 * routine returns, ensuring that it always remains valid here.
1801 */
1802 spin_lock_irqsave(&ev_queue->lock, flags);
1803 list_add_tail(&async_data->list, &ev_queue->event_list);
1804 spin_unlock_irqrestore(&ev_queue->lock, flags);
1805
1806 wake_up_interruptible(&ev_queue->poll_wait);
1807 }
1808
1809 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1810
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)1811 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1812 struct uverbs_attr_bundle *attrs)
1813 {
1814 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1815 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1816 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1817 attrs,
1818 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1819 u16 cmd_out_len;
1820 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1821 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1822 struct ib_uobject *fd_uobj;
1823 int err;
1824 int uid;
1825 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1826 struct devx_async_cmd_event_file *ev_file;
1827 struct devx_async_data *async_data;
1828
1829 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1830 return -EINVAL;
1831
1832 uid = devx_get_uid(c, cmd_in);
1833 if (uid < 0)
1834 return uid;
1835
1836 if (!devx_is_obj_query_cmd(cmd_in))
1837 return -EINVAL;
1838
1839 err = uverbs_get_const(&cmd_out_len, attrs,
1840 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1841 if (err)
1842 return err;
1843
1844 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1845 return -EINVAL;
1846
1847 fd_uobj = uverbs_attr_get_uobject(attrs,
1848 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1849 if (IS_ERR(fd_uobj))
1850 return PTR_ERR(fd_uobj);
1851
1852 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1853 uobj);
1854
1855 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1856 MAX_ASYNC_BYTES_IN_USE) {
1857 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1858 return -EAGAIN;
1859 }
1860
1861 async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1862 cmd_out_len), GFP_KERNEL);
1863 if (!async_data) {
1864 err = -ENOMEM;
1865 goto sub_bytes;
1866 }
1867
1868 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1869 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1870 if (err)
1871 goto free_async;
1872
1873 async_data->cmd_out_len = cmd_out_len;
1874 async_data->mdev = mdev;
1875 async_data->ev_file = ev_file;
1876
1877 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1878 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1879 uverbs_attr_get_len(attrs,
1880 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1881 async_data->hdr.out_data,
1882 async_data->cmd_out_len,
1883 devx_query_callback, &async_data->cb_work);
1884
1885 if (err)
1886 goto free_async;
1887
1888 return 0;
1889
1890 free_async:
1891 kvfree(async_data);
1892 sub_bytes:
1893 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1894 return err;
1895 }
1896
1897 static void
subscribe_event_xa_dealloc(struct mlx5_devx_event_table * devx_event_table,u32 key_level1,bool is_level2,u32 key_level2)1898 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1899 u32 key_level1,
1900 bool is_level2,
1901 u32 key_level2)
1902 {
1903 struct devx_event *event;
1904 struct devx_obj_event *xa_val_level2;
1905
1906 /* Level 1 is valid for future use, no need to free */
1907 if (!is_level2)
1908 return;
1909
1910 event = xa_load(&devx_event_table->event_xa, key_level1);
1911 WARN_ON(!event);
1912
1913 xa_val_level2 = xa_load(&event->object_ids,
1914 key_level2);
1915 if (list_empty(&xa_val_level2->obj_sub_list)) {
1916 xa_erase(&event->object_ids,
1917 key_level2);
1918 kfree_rcu(xa_val_level2, rcu);
1919 }
1920 }
1921
1922 static int
subscribe_event_xa_alloc(struct mlx5_devx_event_table * devx_event_table,u32 key_level1,bool is_level2,u32 key_level2)1923 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1924 u32 key_level1,
1925 bool is_level2,
1926 u32 key_level2)
1927 {
1928 struct devx_obj_event *obj_event;
1929 struct devx_event *event;
1930 int err;
1931
1932 event = xa_load(&devx_event_table->event_xa, key_level1);
1933 if (!event) {
1934 event = kzalloc(sizeof(*event), GFP_KERNEL);
1935 if (!event)
1936 return -ENOMEM;
1937
1938 INIT_LIST_HEAD(&event->unaffiliated_list);
1939 xa_init(&event->object_ids);
1940
1941 err = xa_insert(&devx_event_table->event_xa,
1942 key_level1,
1943 event,
1944 GFP_KERNEL);
1945 if (err) {
1946 kfree(event);
1947 return err;
1948 }
1949 }
1950
1951 if (!is_level2)
1952 return 0;
1953
1954 obj_event = xa_load(&event->object_ids, key_level2);
1955 if (!obj_event) {
1956 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1957 if (!obj_event)
1958 /* Level1 is valid for future use, no need to free */
1959 return -ENOMEM;
1960
1961 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1962 err = xa_insert(&event->object_ids,
1963 key_level2,
1964 obj_event,
1965 GFP_KERNEL);
1966 if (err) {
1967 kfree(obj_event);
1968 return err;
1969 }
1970 }
1971
1972 return 0;
1973 }
1974
is_valid_events_legacy(int num_events,u16 * event_type_num_list,struct devx_obj * obj)1975 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1976 struct devx_obj *obj)
1977 {
1978 int i;
1979
1980 for (i = 0; i < num_events; i++) {
1981 if (obj) {
1982 if (!is_legacy_obj_event_num(event_type_num_list[i]))
1983 return false;
1984 } else if (!is_legacy_unaffiliated_event_num(
1985 event_type_num_list[i])) {
1986 return false;
1987 }
1988 }
1989
1990 return true;
1991 }
1992
1993 #define MAX_SUPP_EVENT_NUM 255
is_valid_events(struct mlx5_core_dev * dev,int num_events,u16 * event_type_num_list,struct devx_obj * obj)1994 static bool is_valid_events(struct mlx5_core_dev *dev,
1995 int num_events, u16 *event_type_num_list,
1996 struct devx_obj *obj)
1997 {
1998 __be64 *aff_events;
1999 __be64 *unaff_events;
2000 int mask_entry;
2001 int mask_bit;
2002 int i;
2003
2004 if (MLX5_CAP_GEN(dev, event_cap)) {
2005 aff_events = MLX5_CAP_DEV_EVENT(dev,
2006 user_affiliated_events);
2007 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2008 user_unaffiliated_events);
2009 } else {
2010 return is_valid_events_legacy(num_events, event_type_num_list,
2011 obj);
2012 }
2013
2014 for (i = 0; i < num_events; i++) {
2015 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
2016 return false;
2017
2018 mask_entry = event_type_num_list[i] / 64;
2019 mask_bit = event_type_num_list[i] % 64;
2020
2021 if (obj) {
2022 /* CQ completion */
2023 if (event_type_num_list[i] == 0)
2024 continue;
2025
2026 if (!(be64_to_cpu(aff_events[mask_entry]) &
2027 (1ull << mask_bit)))
2028 return false;
2029
2030 continue;
2031 }
2032
2033 if (!(be64_to_cpu(unaff_events[mask_entry]) &
2034 (1ull << mask_bit)))
2035 return false;
2036 }
2037
2038 return true;
2039 }
2040
2041 #define MAX_NUM_EVENTS 16
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)2042 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
2043 struct uverbs_attr_bundle *attrs)
2044 {
2045 struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
2046 attrs,
2047 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
2048 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2049 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2050 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2051 struct ib_uobject *fd_uobj;
2052 struct devx_obj *obj = NULL;
2053 struct devx_async_event_file *ev_file;
2054 struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
2055 u16 *event_type_num_list;
2056 struct devx_event_subscription *event_sub, *tmp_sub;
2057 struct list_head sub_list;
2058 int redirect_fd;
2059 bool use_eventfd = false;
2060 int num_events;
2061 u16 obj_type = 0;
2062 u64 cookie = 0;
2063 u32 obj_id = 0;
2064 int err;
2065 int i;
2066
2067 if (!c->devx_uid)
2068 return -EINVAL;
2069
2070 if (!IS_ERR(devx_uobj)) {
2071 obj = (struct devx_obj *)devx_uobj->object;
2072 if (obj)
2073 obj_id = get_dec_obj_id(obj->obj_id);
2074 }
2075
2076 fd_uobj = uverbs_attr_get_uobject(attrs,
2077 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
2078 if (IS_ERR(fd_uobj))
2079 return PTR_ERR(fd_uobj);
2080
2081 ev_file = container_of(fd_uobj, struct devx_async_event_file,
2082 uobj);
2083
2084 if (uverbs_attr_is_valid(attrs,
2085 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
2086 err = uverbs_copy_from(&redirect_fd, attrs,
2087 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
2088 if (err)
2089 return err;
2090
2091 use_eventfd = true;
2092 }
2093
2094 if (uverbs_attr_is_valid(attrs,
2095 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
2096 if (use_eventfd)
2097 return -EINVAL;
2098
2099 err = uverbs_copy_from(&cookie, attrs,
2100 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
2101 if (err)
2102 return err;
2103 }
2104
2105 num_events = uverbs_attr_ptr_get_array_size(
2106 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2107 sizeof(u16));
2108
2109 if (num_events < 0)
2110 return num_events;
2111
2112 if (num_events > MAX_NUM_EVENTS)
2113 return -EINVAL;
2114
2115 event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2116 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2117
2118 if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2119 return -EINVAL;
2120
2121 INIT_LIST_HEAD(&sub_list);
2122
2123 /* Protect from concurrent subscriptions to same XA entries to allow
2124 * both to succeed
2125 */
2126 mutex_lock(&devx_event_table->event_xa_lock);
2127 for (i = 0; i < num_events; i++) {
2128 u32 key_level1;
2129
2130 if (obj)
2131 obj_type = get_dec_obj_type(obj,
2132 event_type_num_list[i]);
2133 key_level1 = event_type_num_list[i] | obj_type << 16;
2134
2135 err = subscribe_event_xa_alloc(devx_event_table,
2136 key_level1,
2137 obj,
2138 obj_id);
2139 if (err)
2140 goto err;
2141
2142 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2143 if (!event_sub) {
2144 err = -ENOMEM;
2145 goto err;
2146 }
2147
2148 list_add_tail(&event_sub->event_list, &sub_list);
2149 uverbs_uobject_get(&ev_file->uobj);
2150 if (use_eventfd) {
2151 event_sub->eventfd =
2152 eventfd_ctx_fdget(redirect_fd);
2153
2154 if (IS_ERR(event_sub->eventfd)) {
2155 err = PTR_ERR(event_sub->eventfd);
2156 event_sub->eventfd = NULL;
2157 goto err;
2158 }
2159 }
2160
2161 event_sub->cookie = cookie;
2162 event_sub->ev_file = ev_file;
2163 /* May be needed upon cleanup the devx object/subscription */
2164 event_sub->xa_key_level1 = key_level1;
2165 event_sub->xa_key_level2 = obj_id;
2166 INIT_LIST_HEAD(&event_sub->obj_list);
2167 }
2168
2169 /* Once all the allocations and the XA data insertions were done we
2170 * can go ahead and add all the subscriptions to the relevant lists
2171 * without concern of a failure.
2172 */
2173 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2174 struct devx_event *event;
2175 struct devx_obj_event *obj_event;
2176
2177 list_del_init(&event_sub->event_list);
2178
2179 spin_lock_irq(&ev_file->lock);
2180 list_add_tail_rcu(&event_sub->file_list,
2181 &ev_file->subscribed_events_list);
2182 spin_unlock_irq(&ev_file->lock);
2183
2184 event = xa_load(&devx_event_table->event_xa,
2185 event_sub->xa_key_level1);
2186 WARN_ON(!event);
2187
2188 if (!obj) {
2189 list_add_tail_rcu(&event_sub->xa_list,
2190 &event->unaffiliated_list);
2191 continue;
2192 }
2193
2194 obj_event = xa_load(&event->object_ids, obj_id);
2195 WARN_ON(!obj_event);
2196 list_add_tail_rcu(&event_sub->xa_list,
2197 &obj_event->obj_sub_list);
2198 list_add_tail_rcu(&event_sub->obj_list,
2199 &obj->event_sub);
2200 }
2201
2202 mutex_unlock(&devx_event_table->event_xa_lock);
2203 return 0;
2204
2205 err:
2206 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2207 list_del(&event_sub->event_list);
2208
2209 subscribe_event_xa_dealloc(devx_event_table,
2210 event_sub->xa_key_level1,
2211 obj,
2212 obj_id);
2213
2214 if (event_sub->eventfd)
2215 eventfd_ctx_put(event_sub->eventfd);
2216 uverbs_uobject_put(&event_sub->ev_file->uobj);
2217 kfree(event_sub);
2218 }
2219
2220 mutex_unlock(&devx_event_table->event_xa_lock);
2221 return err;
2222 }
2223
devx_umem_get(struct mlx5_ib_dev * dev,struct ib_ucontext * ucontext,struct uverbs_attr_bundle * attrs,struct devx_umem * obj,u32 access_flags)2224 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2225 struct uverbs_attr_bundle *attrs,
2226 struct devx_umem *obj, u32 access_flags)
2227 {
2228 u64 addr;
2229 size_t size;
2230 int err;
2231
2232 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2233 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2234 return -EFAULT;
2235
2236 err = ib_check_mr_access(&dev->ib_dev, access_flags);
2237 if (err)
2238 return err;
2239
2240 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) {
2241 struct ib_umem_dmabuf *umem_dmabuf;
2242 int dmabuf_fd;
2243
2244 err = uverbs_get_raw_fd(&dmabuf_fd, attrs,
2245 MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD);
2246 if (err)
2247 return -EFAULT;
2248
2249 umem_dmabuf = ib_umem_dmabuf_get_pinned(
2250 &dev->ib_dev, addr, size, dmabuf_fd, access_flags);
2251 if (IS_ERR(umem_dmabuf))
2252 return PTR_ERR(umem_dmabuf);
2253 obj->umem = &umem_dmabuf->umem;
2254 } else {
2255 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags);
2256 if (IS_ERR(obj->umem))
2257 return PTR_ERR(obj->umem);
2258 }
2259 return 0;
2260 }
2261
devx_umem_find_best_pgsize(struct ib_umem * umem,unsigned long pgsz_bitmap)2262 static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,
2263 unsigned long pgsz_bitmap)
2264 {
2265 unsigned long page_size;
2266
2267 /* Don't bother checking larger page sizes as offset must be zero and
2268 * total DEVX umem length must be equal to total umem length.
2269 */
2270 pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
2271 PAGE_SHIFT),
2272 MLX5_ADAPTER_PAGE_SHIFT);
2273 if (!pgsz_bitmap)
2274 return 0;
2275
2276 page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);
2277 if (!page_size)
2278 return 0;
2279
2280 /* If the page_size is less than the CPU page size then we can use the
2281 * offset and create a umem which is a subset of the page list.
2282 * For larger page sizes we can't be sure the DMA list reflects the
2283 * VA so we must ensure that the umem extent is exactly equal to the
2284 * page list. Reduce the page size until one of these cases is true.
2285 */
2286 while ((ib_umem_dma_offset(umem, page_size) != 0 ||
2287 (umem->length % page_size) != 0) &&
2288 page_size > PAGE_SIZE)
2289 page_size /= 2;
2290
2291 return page_size;
2292 }
2293
devx_umem_reg_cmd_alloc(struct mlx5_ib_dev * dev,struct uverbs_attr_bundle * attrs,struct devx_umem * obj,struct devx_umem_reg_cmd * cmd,int access)2294 static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2295 struct uverbs_attr_bundle *attrs,
2296 struct devx_umem *obj,
2297 struct devx_umem_reg_cmd *cmd,
2298 int access)
2299 {
2300 unsigned long pgsz_bitmap;
2301 unsigned int page_size;
2302 __be64 *mtt;
2303 void *umem;
2304 int ret;
2305
2306 /*
2307 * If the user does not pass in pgsz_bitmap then the user promises not
2308 * to use umem_offset!=0 in any commands that allocate on top of the
2309 * umem.
2310 *
2311 * If the user wants to use a umem_offset then it must pass in
2312 * pgsz_bitmap which guides the maximum page size and thus maximum
2313 * object alignment inside the umem. See the PRM.
2314 *
2315 * Users are not allowed to use IOVA here, mkeys are not supported on
2316 * umem.
2317 */
2318 ret = uverbs_get_const_default(&pgsz_bitmap, attrs,
2319 MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2320 GENMASK_ULL(63,
2321 min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));
2322 if (ret)
2323 return ret;
2324
2325 page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap);
2326 if (!page_size)
2327 return -EINVAL;
2328
2329 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2330 (MLX5_ST_SZ_BYTES(mtt) *
2331 ib_umem_num_dma_blocks(obj->umem, page_size));
2332 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2333 if (IS_ERR(cmd->in))
2334 return PTR_ERR(cmd->in);
2335
2336 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2337 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2338
2339 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2340 MLX5_SET64(umem, umem, num_of_mtt,
2341 ib_umem_num_dma_blocks(obj->umem, page_size));
2342 MLX5_SET(umem, umem, log_page_size,
2343 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2344 MLX5_SET(umem, umem, page_offset,
2345 ib_umem_dma_offset(obj->umem, page_size));
2346
2347 if (mlx5_umem_needs_ats(dev, obj->umem, access))
2348 MLX5_SET(umem, umem, ats, 1);
2349
2350 mlx5_ib_populate_pas(obj->umem, page_size, mtt,
2351 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2352 MLX5_IB_MTT_READ);
2353 return 0;
2354 }
2355
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)2356 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2357 struct uverbs_attr_bundle *attrs)
2358 {
2359 struct devx_umem_reg_cmd cmd;
2360 struct devx_umem *obj;
2361 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2362 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2363 u32 obj_id;
2364 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2365 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2366 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2367 int access_flags;
2368 int err;
2369
2370 if (!c->devx_uid)
2371 return -EINVAL;
2372
2373 err = uverbs_get_flags32(&access_flags, attrs,
2374 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2375 IB_ACCESS_LOCAL_WRITE |
2376 IB_ACCESS_REMOTE_WRITE |
2377 IB_ACCESS_REMOTE_READ |
2378 IB_ACCESS_RELAXED_ORDERING);
2379 if (err)
2380 return err;
2381
2382 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2383 if (!obj)
2384 return -ENOMEM;
2385
2386 err = devx_umem_get(dev, &c->ibucontext, attrs, obj, access_flags);
2387 if (err)
2388 goto err_obj_free;
2389
2390 err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd, access_flags);
2391 if (err)
2392 goto err_umem_release;
2393
2394 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2395 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2396 sizeof(cmd.out));
2397 if (err)
2398 goto err_umem_release;
2399
2400 obj->mdev = dev->mdev;
2401 uobj->object = obj;
2402 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2403 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2404
2405 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2406 sizeof(obj_id));
2407 return err;
2408
2409 err_umem_release:
2410 ib_umem_release(obj->umem);
2411 err_obj_free:
2412 kfree(obj);
2413 return err;
2414 }
2415
devx_umem_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2416 static int devx_umem_cleanup(struct ib_uobject *uobject,
2417 enum rdma_remove_reason why,
2418 struct uverbs_attr_bundle *attrs)
2419 {
2420 struct devx_umem *obj = uobject->object;
2421 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2422 int err;
2423
2424 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2425 if (err)
2426 return err;
2427
2428 ib_umem_release(obj->umem);
2429 kfree(obj);
2430 return 0;
2431 }
2432
is_unaffiliated_event(struct mlx5_core_dev * dev,unsigned long event_type)2433 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2434 unsigned long event_type)
2435 {
2436 __be64 *unaff_events;
2437 int mask_entry;
2438 int mask_bit;
2439
2440 if (!MLX5_CAP_GEN(dev, event_cap))
2441 return is_legacy_unaffiliated_event_num(event_type);
2442
2443 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2444 user_unaffiliated_events);
2445 WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2446
2447 mask_entry = event_type / 64;
2448 mask_bit = event_type % 64;
2449
2450 if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2451 return false;
2452
2453 return true;
2454 }
2455
devx_get_obj_id_from_event(unsigned long event_type,void * data)2456 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2457 {
2458 struct mlx5_eqe *eqe = data;
2459 u32 obj_id = 0;
2460
2461 switch (event_type) {
2462 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2463 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2464 case MLX5_EVENT_TYPE_PATH_MIG:
2465 case MLX5_EVENT_TYPE_COMM_EST:
2466 case MLX5_EVENT_TYPE_SQ_DRAINED:
2467 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2468 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2469 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2470 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2471 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2472 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2473 break;
2474 case MLX5_EVENT_TYPE_XRQ_ERROR:
2475 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2476 break;
2477 case MLX5_EVENT_TYPE_DCT_DRAINED:
2478 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2479 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2480 break;
2481 case MLX5_EVENT_TYPE_CQ_ERROR:
2482 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2483 break;
2484 default:
2485 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2486 break;
2487 }
2488
2489 return obj_id;
2490 }
2491
deliver_event(struct devx_event_subscription * event_sub,const void * data)2492 static int deliver_event(struct devx_event_subscription *event_sub,
2493 const void *data)
2494 {
2495 struct devx_async_event_file *ev_file;
2496 struct devx_async_event_data *event_data;
2497 unsigned long flags;
2498
2499 ev_file = event_sub->ev_file;
2500
2501 if (ev_file->omit_data) {
2502 spin_lock_irqsave(&ev_file->lock, flags);
2503 if (!list_empty(&event_sub->event_list) ||
2504 ev_file->is_destroyed) {
2505 spin_unlock_irqrestore(&ev_file->lock, flags);
2506 return 0;
2507 }
2508
2509 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2510 spin_unlock_irqrestore(&ev_file->lock, flags);
2511 wake_up_interruptible(&ev_file->poll_wait);
2512 return 0;
2513 }
2514
2515 event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2516 GFP_ATOMIC);
2517 if (!event_data) {
2518 spin_lock_irqsave(&ev_file->lock, flags);
2519 ev_file->is_overflow_err = 1;
2520 spin_unlock_irqrestore(&ev_file->lock, flags);
2521 return -ENOMEM;
2522 }
2523
2524 event_data->hdr.cookie = event_sub->cookie;
2525 memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2526
2527 spin_lock_irqsave(&ev_file->lock, flags);
2528 if (!ev_file->is_destroyed)
2529 list_add_tail(&event_data->list, &ev_file->event_list);
2530 else
2531 kfree(event_data);
2532 spin_unlock_irqrestore(&ev_file->lock, flags);
2533 wake_up_interruptible(&ev_file->poll_wait);
2534
2535 return 0;
2536 }
2537
dispatch_event_fd(struct list_head * fd_list,const void * data)2538 static void dispatch_event_fd(struct list_head *fd_list,
2539 const void *data)
2540 {
2541 struct devx_event_subscription *item;
2542
2543 list_for_each_entry_rcu(item, fd_list, xa_list) {
2544 if (item->eventfd)
2545 eventfd_signal(item->eventfd);
2546 else
2547 deliver_event(item, data);
2548 }
2549 }
2550
devx_event_notifier(struct notifier_block * nb,unsigned long event_type,void * data)2551 static int devx_event_notifier(struct notifier_block *nb,
2552 unsigned long event_type, void *data)
2553 {
2554 struct mlx5_devx_event_table *table;
2555 struct mlx5_ib_dev *dev;
2556 struct devx_event *event;
2557 struct devx_obj_event *obj_event;
2558 u16 obj_type = 0;
2559 bool is_unaffiliated;
2560 u32 obj_id;
2561
2562 /* Explicit filtering to kernel events which may occur frequently */
2563 if (event_type == MLX5_EVENT_TYPE_CMD ||
2564 event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2565 return NOTIFY_OK;
2566
2567 table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2568 dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2569 is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2570
2571 if (!is_unaffiliated)
2572 obj_type = get_event_obj_type(event_type, data);
2573
2574 rcu_read_lock();
2575 event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2576 if (!event) {
2577 rcu_read_unlock();
2578 return NOTIFY_DONE;
2579 }
2580
2581 if (is_unaffiliated) {
2582 dispatch_event_fd(&event->unaffiliated_list, data);
2583 rcu_read_unlock();
2584 return NOTIFY_OK;
2585 }
2586
2587 obj_id = devx_get_obj_id_from_event(event_type, data);
2588 obj_event = xa_load(&event->object_ids, obj_id);
2589 if (!obj_event) {
2590 rcu_read_unlock();
2591 return NOTIFY_DONE;
2592 }
2593
2594 dispatch_event_fd(&obj_event->obj_sub_list, data);
2595
2596 rcu_read_unlock();
2597 return NOTIFY_OK;
2598 }
2599
mlx5_ib_devx_init(struct mlx5_ib_dev * dev)2600 int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
2601 {
2602 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2603 int uid;
2604
2605 uid = mlx5_ib_devx_create(dev, false, 0);
2606 if (uid > 0) {
2607 dev->devx_whitelist_uid = uid;
2608 xa_init(&table->event_xa);
2609 mutex_init(&table->event_xa_lock);
2610 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2611 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2612 }
2613
2614 return 0;
2615 }
2616
mlx5_ib_devx_cleanup(struct mlx5_ib_dev * dev)2617 void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
2618 {
2619 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2620 struct devx_event_subscription *sub, *tmp;
2621 struct devx_event *event;
2622 void *entry;
2623 unsigned long id;
2624
2625 if (dev->devx_whitelist_uid) {
2626 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2627 mutex_lock(&dev->devx_event_table.event_xa_lock);
2628 xa_for_each(&table->event_xa, id, entry) {
2629 event = entry;
2630 list_for_each_entry_safe(
2631 sub, tmp, &event->unaffiliated_list, xa_list)
2632 devx_cleanup_subscription(dev, sub);
2633 kfree(entry);
2634 }
2635 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2636 xa_destroy(&table->event_xa);
2637
2638 mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
2639 }
2640 }
2641
devx_async_destroy_cb(int status,struct mlx5_async_work * context)2642 static void devx_async_destroy_cb(int status, struct mlx5_async_work *context)
2643 {
2644 struct mlx5_async_cmd *devx_out = container_of(context,
2645 struct mlx5_async_cmd, cb_work);
2646 struct devx_obj *obj = devx_out->uobject->object;
2647
2648 if (!status)
2649 obj->flags |= DEVX_OBJ_FLAGS_HW_FREED;
2650
2651 complete(&devx_out->comp);
2652 }
2653
devx_async_destroy(struct mlx5_ib_dev * dev,struct mlx5_async_cmd * cmd)2654 static void devx_async_destroy(struct mlx5_ib_dev *dev,
2655 struct mlx5_async_cmd *cmd)
2656 {
2657 init_completion(&cmd->comp);
2658 cmd->err = mlx5_cmd_exec_cb(&dev->async_ctx, cmd->in, cmd->in_size,
2659 &cmd->out, sizeof(cmd->out),
2660 devx_async_destroy_cb, &cmd->cb_work);
2661 }
2662
devx_wait_async_destroy(struct mlx5_async_cmd * cmd)2663 static void devx_wait_async_destroy(struct mlx5_async_cmd *cmd)
2664 {
2665 if (!cmd->err)
2666 wait_for_completion(&cmd->comp);
2667 atomic_set(&cmd->uobject->usecnt, 0);
2668 }
2669
mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file * ufile)2670 void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)
2671 {
2672 struct mlx5_async_cmd *async_cmd;
2673 struct ib_ucontext *ucontext = ufile->ucontext;
2674 struct ib_device *device = ucontext->device;
2675 struct mlx5_ib_dev *dev = to_mdev(device);
2676 struct ib_uobject *uobject;
2677 struct devx_obj *obj;
2678 int head = 0;
2679 int tail = 0;
2680
2681 async_cmd = kcalloc(MAX_ASYNC_CMDS, sizeof(*async_cmd), GFP_KERNEL);
2682 if (!async_cmd)
2683 return;
2684
2685 list_for_each_entry(uobject, &ufile->uobjects, list) {
2686 WARN_ON(uverbs_try_lock_object(uobject, UVERBS_LOOKUP_WRITE));
2687
2688 /*
2689 * Currently we only support QP destruction, if other objects
2690 * are to be destroyed need to add type synchronization to the
2691 * cleanup algorithm and handle pre/post FW cleanup for the
2692 * new types if needed.
2693 */
2694 if (uobj_get_object_id(uobject) != MLX5_IB_OBJECT_DEVX_OBJ ||
2695 (get_dec_obj_type(uobject->object, MLX5_EVENT_TYPE_MAX) !=
2696 MLX5_OBJ_TYPE_QP)) {
2697 atomic_set(&uobject->usecnt, 0);
2698 continue;
2699 }
2700
2701 obj = uobject->object;
2702
2703 async_cmd[tail % MAX_ASYNC_CMDS].in = obj->dinbox;
2704 async_cmd[tail % MAX_ASYNC_CMDS].in_size = obj->dinlen;
2705 async_cmd[tail % MAX_ASYNC_CMDS].uobject = uobject;
2706
2707 devx_async_destroy(dev, &async_cmd[tail % MAX_ASYNC_CMDS]);
2708 tail++;
2709
2710 if (tail - head == MAX_ASYNC_CMDS) {
2711 devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2712 head++;
2713 }
2714 }
2715
2716 while (head != tail) {
2717 devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2718 head++;
2719 }
2720
2721 kfree(async_cmd);
2722 }
2723
devx_async_cmd_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)2724 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2725 size_t count, loff_t *pos)
2726 {
2727 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2728 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2729 struct devx_async_data *event;
2730 int ret = 0;
2731 size_t eventsz;
2732
2733 spin_lock_irq(&ev_queue->lock);
2734
2735 while (list_empty(&ev_queue->event_list)) {
2736 spin_unlock_irq(&ev_queue->lock);
2737
2738 if (filp->f_flags & O_NONBLOCK)
2739 return -EAGAIN;
2740
2741 if (wait_event_interruptible(
2742 ev_queue->poll_wait,
2743 (!list_empty(&ev_queue->event_list) ||
2744 ev_queue->is_destroyed))) {
2745 return -ERESTARTSYS;
2746 }
2747
2748 spin_lock_irq(&ev_queue->lock);
2749 if (ev_queue->is_destroyed) {
2750 spin_unlock_irq(&ev_queue->lock);
2751 return -EIO;
2752 }
2753 }
2754
2755 event = list_entry(ev_queue->event_list.next,
2756 struct devx_async_data, list);
2757 eventsz = event->cmd_out_len +
2758 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2759
2760 if (eventsz > count) {
2761 spin_unlock_irq(&ev_queue->lock);
2762 return -ENOSPC;
2763 }
2764
2765 list_del(ev_queue->event_list.next);
2766 spin_unlock_irq(&ev_queue->lock);
2767
2768 if (copy_to_user(buf, &event->hdr, eventsz))
2769 ret = -EFAULT;
2770 else
2771 ret = eventsz;
2772
2773 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2774 kvfree(event);
2775 return ret;
2776 }
2777
devx_async_cmd_event_poll(struct file * filp,struct poll_table_struct * wait)2778 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2779 struct poll_table_struct *wait)
2780 {
2781 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2782 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2783 __poll_t pollflags = 0;
2784
2785 poll_wait(filp, &ev_queue->poll_wait, wait);
2786
2787 spin_lock_irq(&ev_queue->lock);
2788 if (ev_queue->is_destroyed)
2789 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2790 else if (!list_empty(&ev_queue->event_list))
2791 pollflags = EPOLLIN | EPOLLRDNORM;
2792 spin_unlock_irq(&ev_queue->lock);
2793
2794 return pollflags;
2795 }
2796
2797 static const struct file_operations devx_async_cmd_event_fops = {
2798 .owner = THIS_MODULE,
2799 .read = devx_async_cmd_event_read,
2800 .poll = devx_async_cmd_event_poll,
2801 .release = uverbs_uobject_fd_release,
2802 };
2803
devx_async_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)2804 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2805 size_t count, loff_t *pos)
2806 {
2807 struct devx_async_event_file *ev_file = filp->private_data;
2808 struct devx_event_subscription *event_sub;
2809 struct devx_async_event_data *event;
2810 int ret = 0;
2811 size_t eventsz;
2812 bool omit_data;
2813 void *event_data;
2814
2815 omit_data = ev_file->omit_data;
2816
2817 spin_lock_irq(&ev_file->lock);
2818
2819 if (ev_file->is_overflow_err) {
2820 ev_file->is_overflow_err = 0;
2821 spin_unlock_irq(&ev_file->lock);
2822 return -EOVERFLOW;
2823 }
2824
2825
2826 while (list_empty(&ev_file->event_list)) {
2827 spin_unlock_irq(&ev_file->lock);
2828
2829 if (filp->f_flags & O_NONBLOCK)
2830 return -EAGAIN;
2831
2832 if (wait_event_interruptible(ev_file->poll_wait,
2833 (!list_empty(&ev_file->event_list) ||
2834 ev_file->is_destroyed))) {
2835 return -ERESTARTSYS;
2836 }
2837
2838 spin_lock_irq(&ev_file->lock);
2839 if (ev_file->is_destroyed) {
2840 spin_unlock_irq(&ev_file->lock);
2841 return -EIO;
2842 }
2843 }
2844
2845 if (omit_data) {
2846 event_sub = list_first_entry(&ev_file->event_list,
2847 struct devx_event_subscription,
2848 event_list);
2849 eventsz = sizeof(event_sub->cookie);
2850 event_data = &event_sub->cookie;
2851 } else {
2852 event = list_first_entry(&ev_file->event_list,
2853 struct devx_async_event_data, list);
2854 eventsz = sizeof(struct mlx5_eqe) +
2855 sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2856 event_data = &event->hdr;
2857 }
2858
2859 if (eventsz > count) {
2860 spin_unlock_irq(&ev_file->lock);
2861 return -EINVAL;
2862 }
2863
2864 if (omit_data)
2865 list_del_init(&event_sub->event_list);
2866 else
2867 list_del(&event->list);
2868
2869 spin_unlock_irq(&ev_file->lock);
2870
2871 if (copy_to_user(buf, event_data, eventsz))
2872 /* This points to an application issue, not a kernel concern */
2873 ret = -EFAULT;
2874 else
2875 ret = eventsz;
2876
2877 if (!omit_data)
2878 kfree(event);
2879 return ret;
2880 }
2881
devx_async_event_poll(struct file * filp,struct poll_table_struct * wait)2882 static __poll_t devx_async_event_poll(struct file *filp,
2883 struct poll_table_struct *wait)
2884 {
2885 struct devx_async_event_file *ev_file = filp->private_data;
2886 __poll_t pollflags = 0;
2887
2888 poll_wait(filp, &ev_file->poll_wait, wait);
2889
2890 spin_lock_irq(&ev_file->lock);
2891 if (ev_file->is_destroyed)
2892 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2893 else if (!list_empty(&ev_file->event_list))
2894 pollflags = EPOLLIN | EPOLLRDNORM;
2895 spin_unlock_irq(&ev_file->lock);
2896
2897 return pollflags;
2898 }
2899
devx_free_subscription(struct rcu_head * rcu)2900 static void devx_free_subscription(struct rcu_head *rcu)
2901 {
2902 struct devx_event_subscription *event_sub =
2903 container_of(rcu, struct devx_event_subscription, rcu);
2904
2905 if (event_sub->eventfd)
2906 eventfd_ctx_put(event_sub->eventfd);
2907 uverbs_uobject_put(&event_sub->ev_file->uobj);
2908 kfree(event_sub);
2909 }
2910
2911 static const struct file_operations devx_async_event_fops = {
2912 .owner = THIS_MODULE,
2913 .read = devx_async_event_read,
2914 .poll = devx_async_event_poll,
2915 .release = uverbs_uobject_fd_release,
2916 };
2917
devx_async_cmd_event_destroy_uobj(struct ib_uobject * uobj,enum rdma_remove_reason why)2918 static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2919 enum rdma_remove_reason why)
2920 {
2921 struct devx_async_cmd_event_file *comp_ev_file =
2922 container_of(uobj, struct devx_async_cmd_event_file,
2923 uobj);
2924 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2925 struct devx_async_data *entry, *tmp;
2926
2927 spin_lock_irq(&ev_queue->lock);
2928 ev_queue->is_destroyed = 1;
2929 spin_unlock_irq(&ev_queue->lock);
2930 wake_up_interruptible(&ev_queue->poll_wait);
2931
2932 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2933
2934 spin_lock_irq(&comp_ev_file->ev_queue.lock);
2935 list_for_each_entry_safe(entry, tmp,
2936 &comp_ev_file->ev_queue.event_list, list) {
2937 list_del(&entry->list);
2938 kvfree(entry);
2939 }
2940 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2941 };
2942
devx_async_event_destroy_uobj(struct ib_uobject * uobj,enum rdma_remove_reason why)2943 static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2944 enum rdma_remove_reason why)
2945 {
2946 struct devx_async_event_file *ev_file =
2947 container_of(uobj, struct devx_async_event_file,
2948 uobj);
2949 struct devx_event_subscription *event_sub, *event_sub_tmp;
2950 struct mlx5_ib_dev *dev = ev_file->dev;
2951
2952 spin_lock_irq(&ev_file->lock);
2953 ev_file->is_destroyed = 1;
2954
2955 /* free the pending events allocation */
2956 if (ev_file->omit_data) {
2957 struct devx_event_subscription *event_sub, *tmp;
2958
2959 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2960 event_list)
2961 list_del_init(&event_sub->event_list);
2962
2963 } else {
2964 struct devx_async_event_data *entry, *tmp;
2965
2966 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2967 list) {
2968 list_del(&entry->list);
2969 kfree(entry);
2970 }
2971 }
2972
2973 spin_unlock_irq(&ev_file->lock);
2974 wake_up_interruptible(&ev_file->poll_wait);
2975
2976 mutex_lock(&dev->devx_event_table.event_xa_lock);
2977 /* delete the subscriptions which are related to this FD */
2978 list_for_each_entry_safe(event_sub, event_sub_tmp,
2979 &ev_file->subscribed_events_list, file_list) {
2980 devx_cleanup_subscription(dev, event_sub);
2981 list_del_rcu(&event_sub->file_list);
2982 /* subscription may not be used by the read API any more */
2983 call_rcu(&event_sub->rcu, devx_free_subscription);
2984 }
2985 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2986
2987 put_device(&dev->ib_dev.dev);
2988 };
2989
2990 DECLARE_UVERBS_NAMED_METHOD(
2991 MLX5_IB_METHOD_DEVX_UMEM_REG,
2992 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2993 MLX5_IB_OBJECT_DEVX_UMEM,
2994 UVERBS_ACCESS_NEW,
2995 UA_MANDATORY),
2996 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2997 UVERBS_ATTR_TYPE(u64),
2998 UA_MANDATORY),
2999 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
3000 UVERBS_ATTR_TYPE(u64),
3001 UA_MANDATORY),
3002 UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD,
3003 UA_OPTIONAL),
3004 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
3005 enum ib_access_flags),
3006 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
3007 u64),
3008 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
3009 UVERBS_ATTR_TYPE(u32),
3010 UA_MANDATORY));
3011
3012 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3013 MLX5_IB_METHOD_DEVX_UMEM_DEREG,
3014 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
3015 MLX5_IB_OBJECT_DEVX_UMEM,
3016 UVERBS_ACCESS_DESTROY,
3017 UA_MANDATORY));
3018
3019 DECLARE_UVERBS_NAMED_METHOD(
3020 MLX5_IB_METHOD_DEVX_QUERY_EQN,
3021 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
3022 UVERBS_ATTR_TYPE(u32),
3023 UA_MANDATORY),
3024 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
3025 UVERBS_ATTR_TYPE(u32),
3026 UA_MANDATORY));
3027
3028 DECLARE_UVERBS_NAMED_METHOD(
3029 MLX5_IB_METHOD_DEVX_QUERY_UAR,
3030 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
3031 UVERBS_ATTR_TYPE(u32),
3032 UA_MANDATORY),
3033 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
3034 UVERBS_ATTR_TYPE(u32),
3035 UA_MANDATORY));
3036
3037 DECLARE_UVERBS_NAMED_METHOD(
3038 MLX5_IB_METHOD_DEVX_OTHER,
3039 UVERBS_ATTR_PTR_IN(
3040 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
3041 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3042 UA_MANDATORY,
3043 UA_ALLOC_AND_COPY),
3044 UVERBS_ATTR_PTR_OUT(
3045 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
3046 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3047 UA_MANDATORY));
3048
3049 DECLARE_UVERBS_NAMED_METHOD(
3050 MLX5_IB_METHOD_DEVX_OBJ_CREATE,
3051 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
3052 MLX5_IB_OBJECT_DEVX_OBJ,
3053 UVERBS_ACCESS_NEW,
3054 UA_MANDATORY),
3055 UVERBS_ATTR_PTR_IN(
3056 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
3057 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3058 UA_MANDATORY,
3059 UA_ALLOC_AND_COPY),
3060 UVERBS_ATTR_PTR_OUT(
3061 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
3062 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3063 UA_MANDATORY));
3064
3065 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3066 MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
3067 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
3068 MLX5_IB_OBJECT_DEVX_OBJ,
3069 UVERBS_ACCESS_DESTROY,
3070 UA_MANDATORY));
3071
3072 DECLARE_UVERBS_NAMED_METHOD(
3073 MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
3074 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
3075 UVERBS_IDR_ANY_OBJECT,
3076 UVERBS_ACCESS_READ,
3077 UA_MANDATORY),
3078 UVERBS_ATTR_PTR_IN(
3079 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
3080 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3081 UA_MANDATORY,
3082 UA_ALLOC_AND_COPY),
3083 UVERBS_ATTR_PTR_OUT(
3084 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
3085 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3086 UA_MANDATORY));
3087
3088 DECLARE_UVERBS_NAMED_METHOD(
3089 MLX5_IB_METHOD_DEVX_OBJ_QUERY,
3090 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3091 UVERBS_IDR_ANY_OBJECT,
3092 UVERBS_ACCESS_READ,
3093 UA_MANDATORY),
3094 UVERBS_ATTR_PTR_IN(
3095 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3096 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3097 UA_MANDATORY,
3098 UA_ALLOC_AND_COPY),
3099 UVERBS_ATTR_PTR_OUT(
3100 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
3101 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3102 UA_MANDATORY));
3103
3104 DECLARE_UVERBS_NAMED_METHOD(
3105 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
3106 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3107 UVERBS_IDR_ANY_OBJECT,
3108 UVERBS_ACCESS_READ,
3109 UA_MANDATORY),
3110 UVERBS_ATTR_PTR_IN(
3111 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3112 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3113 UA_MANDATORY,
3114 UA_ALLOC_AND_COPY),
3115 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
3116 u16, UA_MANDATORY),
3117 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
3118 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3119 UVERBS_ACCESS_READ,
3120 UA_MANDATORY),
3121 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
3122 UVERBS_ATTR_TYPE(u64),
3123 UA_MANDATORY));
3124
3125 DECLARE_UVERBS_NAMED_METHOD(
3126 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
3127 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
3128 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3129 UVERBS_ACCESS_READ,
3130 UA_MANDATORY),
3131 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
3132 MLX5_IB_OBJECT_DEVX_OBJ,
3133 UVERBS_ACCESS_READ,
3134 UA_OPTIONAL),
3135 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
3136 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
3137 UA_MANDATORY,
3138 UA_ALLOC_AND_COPY),
3139 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
3140 UVERBS_ATTR_TYPE(u64),
3141 UA_OPTIONAL),
3142 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
3143 UVERBS_ATTR_TYPE(u32),
3144 UA_OPTIONAL));
3145
3146 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
3147 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
3148 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
3149 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
3150 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
3151
3152 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
3153 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
3154 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
3155 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
3156 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
3157 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
3158 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
3159
3160 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
3161 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
3162 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
3163 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
3164
3165
3166 DECLARE_UVERBS_NAMED_METHOD(
3167 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
3168 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
3169 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3170 UVERBS_ACCESS_NEW,
3171 UA_MANDATORY));
3172
3173 DECLARE_UVERBS_NAMED_OBJECT(
3174 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3175 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
3176 devx_async_cmd_event_destroy_uobj,
3177 &devx_async_cmd_event_fops, "[devx_async_cmd]",
3178 O_RDONLY),
3179 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
3180
3181 DECLARE_UVERBS_NAMED_METHOD(
3182 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
3183 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
3184 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3185 UVERBS_ACCESS_NEW,
3186 UA_MANDATORY),
3187 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
3188 enum mlx5_ib_uapi_devx_create_event_channel_flags,
3189 UA_MANDATORY));
3190
3191 DECLARE_UVERBS_NAMED_OBJECT(
3192 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3193 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
3194 devx_async_event_destroy_uobj,
3195 &devx_async_event_fops, "[devx_async_event]",
3196 O_RDONLY),
3197 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
3198
devx_is_supported(struct ib_device * device)3199 static bool devx_is_supported(struct ib_device *device)
3200 {
3201 struct mlx5_ib_dev *dev = to_mdev(device);
3202
3203 return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
3204 }
3205
3206 const struct uapi_definition mlx5_ib_devx_defs[] = {
3207 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3208 MLX5_IB_OBJECT_DEVX,
3209 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3210 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3211 MLX5_IB_OBJECT_DEVX_OBJ,
3212 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3213 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3214 MLX5_IB_OBJECT_DEVX_UMEM,
3215 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3216 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3217 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3218 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3219 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3220 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3221 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3222 {},
3223 };
3224