1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/mlx5_user_ioctl_cmds.h>
11 #include <rdma/mlx5_user_ioctl_verbs.h>
12 #include <rdma/ib_umem.h>
13 #include <rdma/uverbs_std_types.h>
14 #include <linux/mlx5/driver.h>
15 #include <linux/mlx5/fs.h>
16 #include <rdma/ib_ucaps.h>
17 #include "mlx5_ib.h"
18 #include "devx.h"
19 #include "qp.h"
20 #include <linux/xarray.h>
21
22 #define UVERBS_MODULE_NAME mlx5_ib
23 #include <rdma/uverbs_named_ioctl.h>
24
25 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
26
27 enum devx_obj_flags {
28 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
29 DEVX_OBJ_FLAGS_DCT = 1 << 1,
30 DEVX_OBJ_FLAGS_CQ = 1 << 2,
31 DEVX_OBJ_FLAGS_HW_FREED = 1 << 3,
32 };
33
34 #define MAX_ASYNC_CMDS 8
35
36 struct mlx5_async_cmd {
37 struct ib_uobject *uobject;
38 void *in;
39 int in_size;
40 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
41 int err;
42 struct mlx5_async_work cb_work;
43 struct completion comp;
44 };
45
46 struct devx_async_data {
47 struct mlx5_ib_dev *mdev;
48 struct list_head list;
49 struct devx_async_cmd_event_file *ev_file;
50 struct mlx5_async_work cb_work;
51 u16 cmd_out_len;
52 /* must be last field in this structure */
53 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
54 };
55
56 struct devx_async_event_data {
57 struct list_head list; /* headed in ev_file->event_list */
58 struct mlx5_ib_uapi_devx_async_event_hdr hdr;
59 };
60
61 /* first level XA value data structure */
62 struct devx_event {
63 struct xarray object_ids; /* second XA level, Key = object id */
64 struct list_head unaffiliated_list;
65 };
66
67 /* second level XA value data structure */
68 struct devx_obj_event {
69 struct rcu_head rcu;
70 struct list_head obj_sub_list;
71 };
72
73 struct devx_event_subscription {
74 struct list_head file_list; /* headed in ev_file->
75 * subscribed_events_list
76 */
77 struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
78 * devx_obj_event->obj_sub_list
79 */
80 struct list_head obj_list; /* headed in devx_object */
81 struct list_head event_list; /* headed in ev_file->event_list or in
82 * temp list via subscription
83 */
84
85 u8 is_cleaned:1;
86 u32 xa_key_level1;
87 u32 xa_key_level2;
88 struct rcu_head rcu;
89 u64 cookie;
90 struct devx_async_event_file *ev_file;
91 struct eventfd_ctx *eventfd;
92 };
93
94 struct devx_async_event_file {
95 struct ib_uobject uobj;
96 /* Head of events that are subscribed to this FD */
97 struct list_head subscribed_events_list;
98 spinlock_t lock;
99 wait_queue_head_t poll_wait;
100 struct list_head event_list;
101 struct mlx5_ib_dev *dev;
102 u8 omit_data:1;
103 u8 is_overflow_err:1;
104 u8 is_destroyed:1;
105 };
106
107 struct devx_umem {
108 struct mlx5_core_dev *mdev;
109 struct ib_umem *umem;
110 u32 dinlen;
111 u32 dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];
112 };
113
114 struct devx_umem_reg_cmd {
115 void *in;
116 u32 inlen;
117 u32 out[MLX5_ST_SZ_DW(create_umem_out)];
118 };
119
120 static struct mlx5_ib_ucontext *
devx_ufile2uctx(const struct uverbs_attr_bundle * attrs)121 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
122 {
123 return to_mucontext(ib_uverbs_get_ucontext(attrs));
124 }
125
set_uctx_ucaps(struct mlx5_ib_dev * dev,u64 req_ucaps,u32 * cap)126 static int set_uctx_ucaps(struct mlx5_ib_dev *dev, u64 req_ucaps, u32 *cap)
127 {
128 if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_LOCAL)) {
129 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
130 *cap |= MLX5_UCTX_CAP_RDMA_CTRL;
131 else
132 return -EOPNOTSUPP;
133 }
134
135 if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA)) {
136 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
137 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
138 *cap |= MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA;
139 else
140 return -EOPNOTSUPP;
141 }
142
143 return 0;
144 }
145
mlx5_ib_devx_create(struct mlx5_ib_dev * dev,bool is_user,u64 req_ucaps)146 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)
147 {
148 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};
149 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
150 void *uctx;
151 int err;
152 u16 uid;
153 u32 cap = 0;
154
155 /* 0 means not supported */
156 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
157 return -EINVAL;
158
159 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
160 if (is_user &&
161 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX) &&
162 rdma_dev_has_raw_cap(&dev->ib_dev))
163 cap |= MLX5_UCTX_CAP_RAW_TX;
164 if (is_user &&
165 (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
166 MLX5_UCTX_CAP_INTERNAL_DEV_RES) &&
167 capable(CAP_SYS_RAWIO))
168 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
169
170 if (req_ucaps) {
171 err = set_uctx_ucaps(dev, req_ucaps, &cap);
172 if (err)
173 return err;
174 }
175
176 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
177 MLX5_SET(uctx, uctx, cap, cap);
178
179 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
180 if (err)
181 return err;
182
183 uid = MLX5_GET(create_uctx_out, out, uid);
184 return uid;
185 }
186
mlx5_ib_devx_destroy(struct mlx5_ib_dev * dev,u16 uid)187 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
188 {
189 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
190 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
191
192 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
193 MLX5_SET(destroy_uctx_in, in, uid, uid);
194
195 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
196 }
197
is_legacy_unaffiliated_event_num(u16 event_num)198 static bool is_legacy_unaffiliated_event_num(u16 event_num)
199 {
200 switch (event_num) {
201 case MLX5_EVENT_TYPE_PORT_CHANGE:
202 return true;
203 default:
204 return false;
205 }
206 }
207
is_legacy_obj_event_num(u16 event_num)208 static bool is_legacy_obj_event_num(u16 event_num)
209 {
210 switch (event_num) {
211 case MLX5_EVENT_TYPE_PATH_MIG:
212 case MLX5_EVENT_TYPE_COMM_EST:
213 case MLX5_EVENT_TYPE_SQ_DRAINED:
214 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
215 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
216 case MLX5_EVENT_TYPE_CQ_ERROR:
217 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
218 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
219 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
220 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
221 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
222 case MLX5_EVENT_TYPE_DCT_DRAINED:
223 case MLX5_EVENT_TYPE_COMP:
224 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
225 case MLX5_EVENT_TYPE_XRQ_ERROR:
226 return true;
227 default:
228 return false;
229 }
230 }
231
get_legacy_obj_type(u16 opcode)232 static u16 get_legacy_obj_type(u16 opcode)
233 {
234 switch (opcode) {
235 case MLX5_CMD_OP_CREATE_RQ:
236 return MLX5_EVENT_QUEUE_TYPE_RQ;
237 case MLX5_CMD_OP_CREATE_QP:
238 return MLX5_EVENT_QUEUE_TYPE_QP;
239 case MLX5_CMD_OP_CREATE_SQ:
240 return MLX5_EVENT_QUEUE_TYPE_SQ;
241 case MLX5_CMD_OP_CREATE_DCT:
242 return MLX5_EVENT_QUEUE_TYPE_DCT;
243 default:
244 return 0;
245 }
246 }
247
get_dec_obj_type(struct devx_obj * obj,u16 event_num)248 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
249 {
250 u16 opcode;
251
252 opcode = (obj->obj_id >> 32) & 0xffff;
253
254 if (is_legacy_obj_event_num(event_num))
255 return get_legacy_obj_type(opcode);
256
257 switch (opcode) {
258 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
259 return (obj->obj_id >> 48);
260 case MLX5_CMD_OP_CREATE_RQ:
261 return MLX5_OBJ_TYPE_RQ;
262 case MLX5_CMD_OP_CREATE_QP:
263 return MLX5_OBJ_TYPE_QP;
264 case MLX5_CMD_OP_CREATE_SQ:
265 return MLX5_OBJ_TYPE_SQ;
266 case MLX5_CMD_OP_CREATE_DCT:
267 return MLX5_OBJ_TYPE_DCT;
268 case MLX5_CMD_OP_CREATE_TIR:
269 return MLX5_OBJ_TYPE_TIR;
270 case MLX5_CMD_OP_CREATE_TIS:
271 return MLX5_OBJ_TYPE_TIS;
272 case MLX5_CMD_OP_CREATE_PSV:
273 return MLX5_OBJ_TYPE_PSV;
274 case MLX5_OBJ_TYPE_MKEY:
275 return MLX5_OBJ_TYPE_MKEY;
276 case MLX5_CMD_OP_CREATE_RMP:
277 return MLX5_OBJ_TYPE_RMP;
278 case MLX5_CMD_OP_CREATE_XRC_SRQ:
279 return MLX5_OBJ_TYPE_XRC_SRQ;
280 case MLX5_CMD_OP_CREATE_XRQ:
281 return MLX5_OBJ_TYPE_XRQ;
282 case MLX5_CMD_OP_CREATE_RQT:
283 return MLX5_OBJ_TYPE_RQT;
284 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
285 return MLX5_OBJ_TYPE_FLOW_COUNTER;
286 case MLX5_CMD_OP_CREATE_CQ:
287 return MLX5_OBJ_TYPE_CQ;
288 default:
289 return 0;
290 }
291 }
292
get_event_obj_type(unsigned long event_type,struct mlx5_eqe * eqe)293 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
294 {
295 switch (event_type) {
296 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
297 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
298 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
299 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
300 case MLX5_EVENT_TYPE_PATH_MIG:
301 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
302 case MLX5_EVENT_TYPE_COMM_EST:
303 case MLX5_EVENT_TYPE_SQ_DRAINED:
304 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
305 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
306 return eqe->data.qp_srq.type;
307 case MLX5_EVENT_TYPE_CQ_ERROR:
308 case MLX5_EVENT_TYPE_XRQ_ERROR:
309 return 0;
310 case MLX5_EVENT_TYPE_DCT_DRAINED:
311 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
312 return MLX5_EVENT_QUEUE_TYPE_DCT;
313 default:
314 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
315 }
316 }
317
get_dec_obj_id(u64 obj_id)318 static u32 get_dec_obj_id(u64 obj_id)
319 {
320 return (obj_id & 0xffffffff);
321 }
322
323 /*
324 * As the obj_id in the firmware is not globally unique the object type
325 * must be considered upon checking for a valid object id.
326 * For that the opcode of the creator command is encoded as part of the obj_id.
327 */
get_enc_obj_id(u32 opcode,u32 obj_id)328 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
329 {
330 return ((u64)opcode << 32) | obj_id;
331 }
332
devx_get_created_obj_id(const void * in,const void * out,u16 opcode)333 static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)
334 {
335 switch (opcode) {
336 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
337 return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
338 case MLX5_CMD_OP_CREATE_UMEM:
339 return MLX5_GET(create_umem_out, out, umem_id);
340 case MLX5_CMD_OP_CREATE_MKEY:
341 return MLX5_GET(create_mkey_out, out, mkey_index);
342 case MLX5_CMD_OP_CREATE_CQ:
343 return MLX5_GET(create_cq_out, out, cqn);
344 case MLX5_CMD_OP_ALLOC_PD:
345 return MLX5_GET(alloc_pd_out, out, pd);
346 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
347 return MLX5_GET(alloc_transport_domain_out, out,
348 transport_domain);
349 case MLX5_CMD_OP_CREATE_RMP:
350 return MLX5_GET(create_rmp_out, out, rmpn);
351 case MLX5_CMD_OP_CREATE_SQ:
352 return MLX5_GET(create_sq_out, out, sqn);
353 case MLX5_CMD_OP_CREATE_RQ:
354 return MLX5_GET(create_rq_out, out, rqn);
355 case MLX5_CMD_OP_CREATE_RQT:
356 return MLX5_GET(create_rqt_out, out, rqtn);
357 case MLX5_CMD_OP_CREATE_TIR:
358 return MLX5_GET(create_tir_out, out, tirn);
359 case MLX5_CMD_OP_CREATE_TIS:
360 return MLX5_GET(create_tis_out, out, tisn);
361 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
362 return MLX5_GET(alloc_q_counter_out, out, counter_set_id);
363 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
364 return MLX5_GET(create_flow_table_out, out, table_id);
365 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
366 return MLX5_GET(create_flow_group_out, out, group_id);
367 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
368 return MLX5_GET(set_fte_in, in, flow_index);
369 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
370 return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
371 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
372 return MLX5_GET(alloc_packet_reformat_context_out, out,
373 packet_reformat_id);
374 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
375 return MLX5_GET(alloc_modify_header_context_out, out,
376 modify_header_id);
377 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
378 return MLX5_GET(create_scheduling_element_out, out,
379 scheduling_element_id);
380 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
381 return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
382 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
383 return MLX5_GET(set_l2_table_entry_in, in, table_index);
384 case MLX5_CMD_OP_CREATE_QP:
385 return MLX5_GET(create_qp_out, out, qpn);
386 case MLX5_CMD_OP_CREATE_SRQ:
387 return MLX5_GET(create_srq_out, out, srqn);
388 case MLX5_CMD_OP_CREATE_XRC_SRQ:
389 return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
390 case MLX5_CMD_OP_CREATE_DCT:
391 return MLX5_GET(create_dct_out, out, dctn);
392 case MLX5_CMD_OP_CREATE_XRQ:
393 return MLX5_GET(create_xrq_out, out, xrqn);
394 case MLX5_CMD_OP_ATTACH_TO_MCG:
395 return MLX5_GET(attach_to_mcg_in, in, qpn);
396 case MLX5_CMD_OP_ALLOC_XRCD:
397 return MLX5_GET(alloc_xrcd_out, out, xrcd);
398 case MLX5_CMD_OP_CREATE_PSV:
399 return MLX5_GET(create_psv_out, out, psv0_index);
400 default:
401 /* The entry must match to one of the devx_is_obj_create_cmd */
402 WARN_ON(true);
403 return 0;
404 }
405 }
406
devx_get_obj_id(const void * in)407 static u64 devx_get_obj_id(const void *in)
408 {
409 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
410 u64 obj_id;
411
412 switch (opcode) {
413 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
414 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
415 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
416 MLX5_GET(general_obj_in_cmd_hdr, in,
417 obj_type) << 16,
418 MLX5_GET(general_obj_in_cmd_hdr, in,
419 obj_id));
420 break;
421 case MLX5_CMD_OP_QUERY_MKEY:
422 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
423 MLX5_GET(query_mkey_in, in,
424 mkey_index));
425 break;
426 case MLX5_CMD_OP_QUERY_CQ:
427 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
428 MLX5_GET(query_cq_in, in, cqn));
429 break;
430 case MLX5_CMD_OP_MODIFY_CQ:
431 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
432 MLX5_GET(modify_cq_in, in, cqn));
433 break;
434 case MLX5_CMD_OP_QUERY_SQ:
435 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
436 MLX5_GET(query_sq_in, in, sqn));
437 break;
438 case MLX5_CMD_OP_MODIFY_SQ:
439 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
440 MLX5_GET(modify_sq_in, in, sqn));
441 break;
442 case MLX5_CMD_OP_QUERY_RQ:
443 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
444 MLX5_GET(query_rq_in, in, rqn));
445 break;
446 case MLX5_CMD_OP_MODIFY_RQ:
447 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
448 MLX5_GET(modify_rq_in, in, rqn));
449 break;
450 case MLX5_CMD_OP_QUERY_RMP:
451 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
452 MLX5_GET(query_rmp_in, in, rmpn));
453 break;
454 case MLX5_CMD_OP_MODIFY_RMP:
455 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
456 MLX5_GET(modify_rmp_in, in, rmpn));
457 break;
458 case MLX5_CMD_OP_QUERY_RQT:
459 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
460 MLX5_GET(query_rqt_in, in, rqtn));
461 break;
462 case MLX5_CMD_OP_MODIFY_RQT:
463 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
464 MLX5_GET(modify_rqt_in, in, rqtn));
465 break;
466 case MLX5_CMD_OP_QUERY_TIR:
467 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
468 MLX5_GET(query_tir_in, in, tirn));
469 break;
470 case MLX5_CMD_OP_MODIFY_TIR:
471 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
472 MLX5_GET(modify_tir_in, in, tirn));
473 break;
474 case MLX5_CMD_OP_QUERY_TIS:
475 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
476 MLX5_GET(query_tis_in, in, tisn));
477 break;
478 case MLX5_CMD_OP_MODIFY_TIS:
479 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
480 MLX5_GET(modify_tis_in, in, tisn));
481 break;
482 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
483 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
484 MLX5_GET(query_flow_table_in, in,
485 table_id));
486 break;
487 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
488 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
489 MLX5_GET(modify_flow_table_in, in,
490 table_id));
491 break;
492 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
493 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
494 MLX5_GET(query_flow_group_in, in,
495 group_id));
496 break;
497 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
498 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
499 MLX5_GET(query_fte_in, in,
500 flow_index));
501 break;
502 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
503 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
504 MLX5_GET(set_fte_in, in, flow_index));
505 break;
506 case MLX5_CMD_OP_QUERY_Q_COUNTER:
507 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
508 MLX5_GET(query_q_counter_in, in,
509 counter_set_id));
510 break;
511 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
512 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
513 MLX5_GET(query_flow_counter_in, in,
514 flow_counter_id));
515 break;
516 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
517 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
518 MLX5_GET(query_modify_header_context_in,
519 in, modify_header_id));
520 break;
521 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
522 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
523 MLX5_GET(query_scheduling_element_in,
524 in, scheduling_element_id));
525 break;
526 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
527 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
528 MLX5_GET(modify_scheduling_element_in,
529 in, scheduling_element_id));
530 break;
531 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
532 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
533 MLX5_GET(add_vxlan_udp_dport_in, in,
534 vxlan_udp_port));
535 break;
536 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
537 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
538 MLX5_GET(query_l2_table_entry_in, in,
539 table_index));
540 break;
541 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
542 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
543 MLX5_GET(set_l2_table_entry_in, in,
544 table_index));
545 break;
546 case MLX5_CMD_OP_QUERY_QP:
547 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
548 MLX5_GET(query_qp_in, in, qpn));
549 break;
550 case MLX5_CMD_OP_RST2INIT_QP:
551 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
552 MLX5_GET(rst2init_qp_in, in, qpn));
553 break;
554 case MLX5_CMD_OP_INIT2INIT_QP:
555 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
556 MLX5_GET(init2init_qp_in, in, qpn));
557 break;
558 case MLX5_CMD_OP_INIT2RTR_QP:
559 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
560 MLX5_GET(init2rtr_qp_in, in, qpn));
561 break;
562 case MLX5_CMD_OP_RTR2RTS_QP:
563 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
564 MLX5_GET(rtr2rts_qp_in, in, qpn));
565 break;
566 case MLX5_CMD_OP_RTS2RTS_QP:
567 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
568 MLX5_GET(rts2rts_qp_in, in, qpn));
569 break;
570 case MLX5_CMD_OP_SQERR2RTS_QP:
571 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
572 MLX5_GET(sqerr2rts_qp_in, in, qpn));
573 break;
574 case MLX5_CMD_OP_2ERR_QP:
575 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
576 MLX5_GET(qp_2err_in, in, qpn));
577 break;
578 case MLX5_CMD_OP_2RST_QP:
579 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
580 MLX5_GET(qp_2rst_in, in, qpn));
581 break;
582 case MLX5_CMD_OP_QUERY_DCT:
583 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
584 MLX5_GET(query_dct_in, in, dctn));
585 break;
586 case MLX5_CMD_OP_QUERY_XRQ:
587 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
588 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
589 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
590 MLX5_GET(query_xrq_in, in, xrqn));
591 break;
592 case MLX5_CMD_OP_QUERY_XRC_SRQ:
593 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
594 MLX5_GET(query_xrc_srq_in, in,
595 xrc_srqn));
596 break;
597 case MLX5_CMD_OP_ARM_XRC_SRQ:
598 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
599 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
600 break;
601 case MLX5_CMD_OP_QUERY_SRQ:
602 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
603 MLX5_GET(query_srq_in, in, srqn));
604 break;
605 case MLX5_CMD_OP_ARM_RQ:
606 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
607 MLX5_GET(arm_rq_in, in, srq_number));
608 break;
609 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
610 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
611 MLX5_GET(drain_dct_in, in, dctn));
612 break;
613 case MLX5_CMD_OP_ARM_XRQ:
614 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
615 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
616 case MLX5_CMD_OP_MODIFY_XRQ:
617 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
618 MLX5_GET(arm_xrq_in, in, xrqn));
619 break;
620 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
621 obj_id = get_enc_obj_id
622 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
623 MLX5_GET(query_packet_reformat_context_in,
624 in, packet_reformat_id));
625 break;
626 default:
627 obj_id = 0;
628 }
629
630 return obj_id;
631 }
632
devx_is_valid_obj_id(struct uverbs_attr_bundle * attrs,struct ib_uobject * uobj,const void * in)633 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
634 struct ib_uobject *uobj, const void *in)
635 {
636 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
637 u64 obj_id = devx_get_obj_id(in);
638
639 if (!obj_id)
640 return false;
641
642 switch (uobj_get_object_id(uobj)) {
643 case UVERBS_OBJECT_CQ:
644 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
645 to_mcq(uobj->object)->mcq.cqn) ==
646 obj_id;
647
648 case UVERBS_OBJECT_SRQ:
649 {
650 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
651 u16 opcode;
652
653 switch (srq->common.res) {
654 case MLX5_RES_XSRQ:
655 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
656 break;
657 case MLX5_RES_XRQ:
658 opcode = MLX5_CMD_OP_CREATE_XRQ;
659 break;
660 default:
661 if (!dev->mdev->issi)
662 opcode = MLX5_CMD_OP_CREATE_SRQ;
663 else
664 opcode = MLX5_CMD_OP_CREATE_RMP;
665 }
666
667 return get_enc_obj_id(opcode,
668 to_msrq(uobj->object)->msrq.srqn) ==
669 obj_id;
670 }
671
672 case UVERBS_OBJECT_QP:
673 {
674 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
675
676 if (qp->type == IB_QPT_RAW_PACKET ||
677 (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
678 struct mlx5_ib_raw_packet_qp *raw_packet_qp =
679 &qp->raw_packet_qp;
680 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
681 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
682
683 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
684 rq->base.mqp.qpn) == obj_id ||
685 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
686 sq->base.mqp.qpn) == obj_id ||
687 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
688 rq->tirn) == obj_id ||
689 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
690 sq->tisn) == obj_id);
691 }
692
693 if (qp->type == MLX5_IB_QPT_DCT)
694 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
695 qp->dct.mdct.mqp.qpn) == obj_id;
696 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
697 qp->ibqp.qp_num) == obj_id;
698 }
699
700 case UVERBS_OBJECT_WQ:
701 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
702 to_mrwq(uobj->object)->core_qp.qpn) ==
703 obj_id;
704
705 case UVERBS_OBJECT_RWQ_IND_TBL:
706 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
707 to_mrwq_ind_table(uobj->object)->rqtn) ==
708 obj_id;
709
710 case MLX5_IB_OBJECT_DEVX_OBJ:
711 {
712 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
713 struct devx_obj *devx_uobj = uobj->object;
714
715 if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &&
716 devx_uobj->flow_counter_bulk_size) {
717 u64 end;
718
719 end = devx_uobj->obj_id +
720 devx_uobj->flow_counter_bulk_size;
721 return devx_uobj->obj_id <= obj_id && end > obj_id;
722 }
723
724 return devx_uobj->obj_id == obj_id;
725 }
726
727 default:
728 return false;
729 }
730 }
731
devx_set_umem_valid(const void * in)732 static void devx_set_umem_valid(const void *in)
733 {
734 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
735
736 switch (opcode) {
737 case MLX5_CMD_OP_CREATE_MKEY:
738 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
739 break;
740 case MLX5_CMD_OP_CREATE_CQ:
741 {
742 void *cqc;
743
744 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
745 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
746 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
747 break;
748 }
749 case MLX5_CMD_OP_CREATE_QP:
750 {
751 void *qpc;
752
753 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
754 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
755 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
756 break;
757 }
758
759 case MLX5_CMD_OP_CREATE_RQ:
760 {
761 void *rqc, *wq;
762
763 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
764 wq = MLX5_ADDR_OF(rqc, rqc, wq);
765 MLX5_SET(wq, wq, dbr_umem_valid, 1);
766 MLX5_SET(wq, wq, wq_umem_valid, 1);
767 break;
768 }
769
770 case MLX5_CMD_OP_CREATE_SQ:
771 {
772 void *sqc, *wq;
773
774 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
775 wq = MLX5_ADDR_OF(sqc, sqc, wq);
776 MLX5_SET(wq, wq, dbr_umem_valid, 1);
777 MLX5_SET(wq, wq, wq_umem_valid, 1);
778 break;
779 }
780
781 case MLX5_CMD_OP_MODIFY_CQ:
782 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
783 break;
784
785 case MLX5_CMD_OP_CREATE_RMP:
786 {
787 void *rmpc, *wq;
788
789 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
790 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
791 MLX5_SET(wq, wq, dbr_umem_valid, 1);
792 MLX5_SET(wq, wq, wq_umem_valid, 1);
793 break;
794 }
795
796 case MLX5_CMD_OP_CREATE_XRQ:
797 {
798 void *xrqc, *wq;
799
800 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
801 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
802 MLX5_SET(wq, wq, dbr_umem_valid, 1);
803 MLX5_SET(wq, wq, wq_umem_valid, 1);
804 break;
805 }
806
807 case MLX5_CMD_OP_CREATE_XRC_SRQ:
808 {
809 void *xrc_srqc;
810
811 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
812 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
813 xrc_srq_context_entry);
814 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
815 break;
816 }
817
818 default:
819 return;
820 }
821 }
822
devx_is_obj_create_cmd(const void * in,u16 * opcode)823 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
824 {
825 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
826
827 switch (*opcode) {
828 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
829 case MLX5_CMD_OP_CREATE_MKEY:
830 case MLX5_CMD_OP_CREATE_CQ:
831 case MLX5_CMD_OP_ALLOC_PD:
832 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
833 case MLX5_CMD_OP_CREATE_RMP:
834 case MLX5_CMD_OP_CREATE_SQ:
835 case MLX5_CMD_OP_CREATE_RQ:
836 case MLX5_CMD_OP_CREATE_RQT:
837 case MLX5_CMD_OP_CREATE_TIR:
838 case MLX5_CMD_OP_CREATE_TIS:
839 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
840 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
841 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
842 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
843 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
844 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
845 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
846 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
847 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
848 case MLX5_CMD_OP_CREATE_QP:
849 case MLX5_CMD_OP_CREATE_SRQ:
850 case MLX5_CMD_OP_CREATE_XRC_SRQ:
851 case MLX5_CMD_OP_CREATE_DCT:
852 case MLX5_CMD_OP_CREATE_XRQ:
853 case MLX5_CMD_OP_ATTACH_TO_MCG:
854 case MLX5_CMD_OP_ALLOC_XRCD:
855 return true;
856 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
857 {
858 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
859 if (op_mod == 0)
860 return true;
861 return false;
862 }
863 case MLX5_CMD_OP_CREATE_PSV:
864 {
865 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
866
867 if (num_psv == 1)
868 return true;
869 return false;
870 }
871 default:
872 return false;
873 }
874 }
875
devx_is_obj_modify_cmd(const void * in)876 static bool devx_is_obj_modify_cmd(const void *in)
877 {
878 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
879
880 switch (opcode) {
881 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
882 case MLX5_CMD_OP_MODIFY_CQ:
883 case MLX5_CMD_OP_MODIFY_RMP:
884 case MLX5_CMD_OP_MODIFY_SQ:
885 case MLX5_CMD_OP_MODIFY_RQ:
886 case MLX5_CMD_OP_MODIFY_RQT:
887 case MLX5_CMD_OP_MODIFY_TIR:
888 case MLX5_CMD_OP_MODIFY_TIS:
889 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
890 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
891 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
892 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
893 case MLX5_CMD_OP_RST2INIT_QP:
894 case MLX5_CMD_OP_INIT2RTR_QP:
895 case MLX5_CMD_OP_INIT2INIT_QP:
896 case MLX5_CMD_OP_RTR2RTS_QP:
897 case MLX5_CMD_OP_RTS2RTS_QP:
898 case MLX5_CMD_OP_SQERR2RTS_QP:
899 case MLX5_CMD_OP_2ERR_QP:
900 case MLX5_CMD_OP_2RST_QP:
901 case MLX5_CMD_OP_ARM_XRC_SRQ:
902 case MLX5_CMD_OP_ARM_RQ:
903 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
904 case MLX5_CMD_OP_ARM_XRQ:
905 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
906 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
907 case MLX5_CMD_OP_MODIFY_XRQ:
908 return true;
909 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
910 {
911 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
912
913 if (op_mod == 1)
914 return true;
915 return false;
916 }
917 default:
918 return false;
919 }
920 }
921
devx_is_obj_query_cmd(const void * in)922 static bool devx_is_obj_query_cmd(const void *in)
923 {
924 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
925
926 switch (opcode) {
927 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
928 case MLX5_CMD_OP_QUERY_MKEY:
929 case MLX5_CMD_OP_QUERY_CQ:
930 case MLX5_CMD_OP_QUERY_RMP:
931 case MLX5_CMD_OP_QUERY_SQ:
932 case MLX5_CMD_OP_QUERY_RQ:
933 case MLX5_CMD_OP_QUERY_RQT:
934 case MLX5_CMD_OP_QUERY_TIR:
935 case MLX5_CMD_OP_QUERY_TIS:
936 case MLX5_CMD_OP_QUERY_Q_COUNTER:
937 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
938 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
939 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
940 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
941 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
942 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
943 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
944 case MLX5_CMD_OP_QUERY_QP:
945 case MLX5_CMD_OP_QUERY_SRQ:
946 case MLX5_CMD_OP_QUERY_XRC_SRQ:
947 case MLX5_CMD_OP_QUERY_DCT:
948 case MLX5_CMD_OP_QUERY_XRQ:
949 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
950 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
951 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
952 return true;
953 default:
954 return false;
955 }
956 }
957
devx_is_whitelist_cmd(void * in)958 static bool devx_is_whitelist_cmd(void *in)
959 {
960 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
961
962 switch (opcode) {
963 case MLX5_CMD_OP_QUERY_HCA_CAP:
964 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
965 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
966 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
967 return true;
968 default:
969 return false;
970 }
971 }
972
devx_get_uid(struct mlx5_ib_ucontext * c,void * cmd_in)973 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
974 {
975 if (devx_is_whitelist_cmd(cmd_in)) {
976 struct mlx5_ib_dev *dev;
977
978 if (c->devx_uid)
979 return c->devx_uid;
980
981 dev = to_mdev(c->ibucontext.device);
982 if (dev->devx_whitelist_uid)
983 return dev->devx_whitelist_uid;
984
985 return -EOPNOTSUPP;
986 }
987
988 if (!c->devx_uid)
989 return -EINVAL;
990
991 return c->devx_uid;
992 }
993
devx_is_general_cmd(void * in,struct mlx5_ib_dev * dev)994 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
995 {
996 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
997
998 /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
999 if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
1000 MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
1001 (opcode >= MLX5_CMD_OP_GENERAL_START &&
1002 opcode < MLX5_CMD_OP_GENERAL_END))
1003 return true;
1004
1005 switch (opcode) {
1006 case MLX5_CMD_OP_QUERY_HCA_CAP:
1007 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
1008 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
1009 case MLX5_CMD_OP_QUERY_VPORT_STATE:
1010 case MLX5_CMD_OP_QUERY_ADAPTER:
1011 case MLX5_CMD_OP_QUERY_ISSI:
1012 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
1013 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
1014 case MLX5_CMD_OP_QUERY_VNIC_ENV:
1015 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
1016 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
1017 case MLX5_CMD_OP_NOP:
1018 case MLX5_CMD_OP_QUERY_CONG_STATUS:
1019 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
1020 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
1021 case MLX5_CMD_OP_QUERY_LAG:
1022 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
1023 return true;
1024 default:
1025 return false;
1026 }
1027 }
1028
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)1029 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
1030 struct uverbs_attr_bundle *attrs)
1031 {
1032 struct mlx5_ib_ucontext *c;
1033 struct mlx5_ib_dev *dev;
1034 int user_vector;
1035 int dev_eqn;
1036 int err;
1037
1038 if (uverbs_copy_from(&user_vector, attrs,
1039 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
1040 return -EFAULT;
1041
1042 c = devx_ufile2uctx(attrs);
1043 if (IS_ERR(c))
1044 return PTR_ERR(c);
1045 dev = to_mdev(c->ibucontext.device);
1046
1047 err = mlx5_comp_eqn_get(dev->mdev, user_vector, &dev_eqn);
1048 if (err < 0)
1049 return err;
1050
1051 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1052 &dev_eqn, sizeof(dev_eqn)))
1053 return -EFAULT;
1054
1055 return 0;
1056 }
1057
1058 /*
1059 *Security note:
1060 * The hardware protection mechanism works like this: Each device object that
1061 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
1062 * the device specification manual) upon its creation. Then upon doorbell,
1063 * hardware fetches the object context for which the doorbell was rang, and
1064 * validates that the UAR through which the DB was rang matches the UAR ID
1065 * of the object.
1066 * If no match the doorbell is silently ignored by the hardware. Of course,
1067 * the user cannot ring a doorbell on a UAR that was not mapped to it.
1068 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1069 * mailboxes (except tagging them with UID), we expose to the user its UAR
1070 * ID, so it can embed it in these objects in the expected specification
1071 * format. So the only thing the user can do is hurt itself by creating a
1072 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1073 * may ring a doorbell on its objects.
1074 * The consequence of that will be that another user can schedule a QP/SQ
1075 * of the buggy user for execution (just insert it to the hardware schedule
1076 * queue or arm its CQ for event generation), no further harm is expected.
1077 */
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)1078 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1079 struct uverbs_attr_bundle *attrs)
1080 {
1081 struct mlx5_ib_ucontext *c;
1082 struct mlx5_ib_dev *dev;
1083 u32 user_idx;
1084 s32 dev_idx;
1085
1086 c = devx_ufile2uctx(attrs);
1087 if (IS_ERR(c))
1088 return PTR_ERR(c);
1089 dev = to_mdev(c->ibucontext.device);
1090
1091 if (uverbs_copy_from(&user_idx, attrs,
1092 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1093 return -EFAULT;
1094
1095 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1096 if (dev_idx < 0)
1097 return dev_idx;
1098
1099 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1100 &dev_idx, sizeof(dev_idx)))
1101 return -EFAULT;
1102
1103 return 0;
1104 }
1105
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)1106 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1107 struct uverbs_attr_bundle *attrs)
1108 {
1109 struct mlx5_ib_ucontext *c;
1110 struct mlx5_ib_dev *dev;
1111 void *cmd_in = uverbs_attr_get_alloced_ptr(
1112 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1113 int cmd_out_len = uverbs_attr_get_len(attrs,
1114 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1115 void *cmd_out;
1116 int err, err2;
1117 int uid;
1118
1119 c = devx_ufile2uctx(attrs);
1120 if (IS_ERR(c))
1121 return PTR_ERR(c);
1122 dev = to_mdev(c->ibucontext.device);
1123
1124 uid = devx_get_uid(c, cmd_in);
1125 if (uid < 0)
1126 return uid;
1127
1128 /* Only white list of some general HCA commands are allowed for this method. */
1129 if (!devx_is_general_cmd(cmd_in, dev))
1130 return -EINVAL;
1131
1132 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1133 if (IS_ERR(cmd_out))
1134 return PTR_ERR(cmd_out);
1135
1136 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1137 err = mlx5_cmd_do(dev->mdev, cmd_in,
1138 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1139 cmd_out, cmd_out_len);
1140 if (err && err != -EREMOTEIO)
1141 return err;
1142
1143 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1144 cmd_out_len);
1145
1146 return err2 ?: err;
1147 }
1148
devx_obj_build_destroy_cmd(void * in,void * out,void * din,u32 * dinlen,u32 * obj_id)1149 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1150 u32 *dinlen,
1151 u32 *obj_id)
1152 {
1153 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
1154 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1155
1156 *obj_id = devx_get_created_obj_id(in, out, opcode);
1157 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1158 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1159
1160 switch (opcode) {
1161 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1162 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1163 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1164 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,
1165 MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));
1166 break;
1167
1168 case MLX5_CMD_OP_CREATE_UMEM:
1169 MLX5_SET(destroy_umem_in, din, opcode,
1170 MLX5_CMD_OP_DESTROY_UMEM);
1171 MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);
1172 break;
1173 case MLX5_CMD_OP_CREATE_MKEY:
1174 MLX5_SET(destroy_mkey_in, din, opcode,
1175 MLX5_CMD_OP_DESTROY_MKEY);
1176 MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);
1177 break;
1178 case MLX5_CMD_OP_CREATE_CQ:
1179 MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1180 MLX5_SET(destroy_cq_in, din, cqn, *obj_id);
1181 break;
1182 case MLX5_CMD_OP_ALLOC_PD:
1183 MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1184 MLX5_SET(dealloc_pd_in, din, pd, *obj_id);
1185 break;
1186 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1187 MLX5_SET(dealloc_transport_domain_in, din, opcode,
1188 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1189 MLX5_SET(dealloc_transport_domain_in, din, transport_domain,
1190 *obj_id);
1191 break;
1192 case MLX5_CMD_OP_CREATE_RMP:
1193 MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1194 MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);
1195 break;
1196 case MLX5_CMD_OP_CREATE_SQ:
1197 MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1198 MLX5_SET(destroy_sq_in, din, sqn, *obj_id);
1199 break;
1200 case MLX5_CMD_OP_CREATE_RQ:
1201 MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1202 MLX5_SET(destroy_rq_in, din, rqn, *obj_id);
1203 break;
1204 case MLX5_CMD_OP_CREATE_RQT:
1205 MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1206 MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);
1207 break;
1208 case MLX5_CMD_OP_CREATE_TIR:
1209 MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1210 MLX5_SET(destroy_tir_in, din, tirn, *obj_id);
1211 break;
1212 case MLX5_CMD_OP_CREATE_TIS:
1213 MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1214 MLX5_SET(destroy_tis_in, din, tisn, *obj_id);
1215 break;
1216 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1217 MLX5_SET(dealloc_q_counter_in, din, opcode,
1218 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1219 MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);
1220 break;
1221 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1222 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1223 MLX5_SET(destroy_flow_table_in, din, other_vport,
1224 MLX5_GET(create_flow_table_in, in, other_vport));
1225 MLX5_SET(destroy_flow_table_in, din, vport_number,
1226 MLX5_GET(create_flow_table_in, in, vport_number));
1227 MLX5_SET(destroy_flow_table_in, din, table_type,
1228 MLX5_GET(create_flow_table_in, in, table_type));
1229 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1230 MLX5_SET(destroy_flow_table_in, din, opcode,
1231 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1232 break;
1233 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1234 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1235 MLX5_SET(destroy_flow_group_in, din, other_vport,
1236 MLX5_GET(create_flow_group_in, in, other_vport));
1237 MLX5_SET(destroy_flow_group_in, din, vport_number,
1238 MLX5_GET(create_flow_group_in, in, vport_number));
1239 MLX5_SET(destroy_flow_group_in, din, table_type,
1240 MLX5_GET(create_flow_group_in, in, table_type));
1241 MLX5_SET(destroy_flow_group_in, din, table_id,
1242 MLX5_GET(create_flow_group_in, in, table_id));
1243 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1244 MLX5_SET(destroy_flow_group_in, din, opcode,
1245 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1246 break;
1247 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1248 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1249 MLX5_SET(delete_fte_in, din, other_vport,
1250 MLX5_GET(set_fte_in, in, other_vport));
1251 MLX5_SET(delete_fte_in, din, vport_number,
1252 MLX5_GET(set_fte_in, in, vport_number));
1253 MLX5_SET(delete_fte_in, din, table_type,
1254 MLX5_GET(set_fte_in, in, table_type));
1255 MLX5_SET(delete_fte_in, din, table_id,
1256 MLX5_GET(set_fte_in, in, table_id));
1257 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1258 MLX5_SET(delete_fte_in, din, opcode,
1259 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1260 break;
1261 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1262 MLX5_SET(dealloc_flow_counter_in, din, opcode,
1263 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1264 MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,
1265 *obj_id);
1266 break;
1267 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1268 MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,
1269 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1270 MLX5_SET(dealloc_packet_reformat_context_in, din,
1271 packet_reformat_id, *obj_id);
1272 break;
1273 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1274 MLX5_SET(dealloc_modify_header_context_in, din, opcode,
1275 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1276 MLX5_SET(dealloc_modify_header_context_in, din,
1277 modify_header_id, *obj_id);
1278 break;
1279 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1280 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1281 MLX5_SET(destroy_scheduling_element_in, din,
1282 scheduling_hierarchy,
1283 MLX5_GET(create_scheduling_element_in, in,
1284 scheduling_hierarchy));
1285 MLX5_SET(destroy_scheduling_element_in, din,
1286 scheduling_element_id, *obj_id);
1287 MLX5_SET(destroy_scheduling_element_in, din, opcode,
1288 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1289 break;
1290 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1291 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1292 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1293 MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,
1294 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1295 break;
1296 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1297 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1298 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1299 MLX5_SET(delete_l2_table_entry_in, din, opcode,
1300 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1301 break;
1302 case MLX5_CMD_OP_CREATE_QP:
1303 MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1304 MLX5_SET(destroy_qp_in, din, qpn, *obj_id);
1305 break;
1306 case MLX5_CMD_OP_CREATE_SRQ:
1307 MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1308 MLX5_SET(destroy_srq_in, din, srqn, *obj_id);
1309 break;
1310 case MLX5_CMD_OP_CREATE_XRC_SRQ:
1311 MLX5_SET(destroy_xrc_srq_in, din, opcode,
1312 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1313 MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);
1314 break;
1315 case MLX5_CMD_OP_CREATE_DCT:
1316 MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1317 MLX5_SET(destroy_dct_in, din, dctn, *obj_id);
1318 break;
1319 case MLX5_CMD_OP_CREATE_XRQ:
1320 MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1321 MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);
1322 break;
1323 case MLX5_CMD_OP_ATTACH_TO_MCG:
1324 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1325 MLX5_SET(detach_from_mcg_in, din, qpn,
1326 MLX5_GET(attach_to_mcg_in, in, qpn));
1327 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1328 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1329 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1330 MLX5_SET(detach_from_mcg_in, din, opcode,
1331 MLX5_CMD_OP_DETACH_FROM_MCG);
1332 MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);
1333 break;
1334 case MLX5_CMD_OP_ALLOC_XRCD:
1335 MLX5_SET(dealloc_xrcd_in, din, opcode,
1336 MLX5_CMD_OP_DEALLOC_XRCD);
1337 MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);
1338 break;
1339 case MLX5_CMD_OP_CREATE_PSV:
1340 MLX5_SET(destroy_psv_in, din, opcode,
1341 MLX5_CMD_OP_DESTROY_PSV);
1342 MLX5_SET(destroy_psv_in, din, psvn, *obj_id);
1343 break;
1344 default:
1345 /* The entry must match to one of the devx_is_obj_create_cmd */
1346 WARN_ON(true);
1347 break;
1348 }
1349 }
1350
devx_handle_mkey_indirect(struct devx_obj * obj,struct mlx5_ib_dev * dev,void * in,void * out)1351 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1352 struct mlx5_ib_dev *dev,
1353 void *in, void *out)
1354 {
1355 struct mlx5_ib_mkey *mkey = &obj->mkey;
1356 void *mkc;
1357 u8 key;
1358
1359 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1360 key = MLX5_GET(mkc, mkc, mkey_7_0);
1361 mkey->key = mlx5_idx_to_mkey(
1362 MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1363 mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1364 mkey->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1365 init_waitqueue_head(&mkey->wait);
1366
1367 return mlx5r_store_odp_mkey(dev, mkey);
1368 }
1369
devx_handle_mkey_create(struct mlx5_ib_dev * dev,struct devx_obj * obj,void * in,int in_len)1370 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1371 struct devx_obj *obj,
1372 void *in, int in_len)
1373 {
1374 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1375 MLX5_FLD_SZ_BYTES(create_mkey_in,
1376 memory_key_mkey_entry);
1377 void *mkc;
1378 u8 access_mode;
1379
1380 if (in_len < min_len)
1381 return -EINVAL;
1382
1383 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1384
1385 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1386 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1387
1388 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1389 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1390 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1391 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1392 return 0;
1393 }
1394
1395 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1396 /* TPH is not allowed to bypass the regular kernel's verbs flow */
1397 MLX5_SET(mkc, mkc, pcie_tph_en, 0);
1398 MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
1399 MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX);
1400 return 0;
1401 }
1402
devx_cleanup_subscription(struct mlx5_ib_dev * dev,struct devx_event_subscription * sub)1403 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1404 struct devx_event_subscription *sub)
1405 {
1406 struct devx_event *event;
1407 struct devx_obj_event *xa_val_level2;
1408
1409 if (sub->is_cleaned)
1410 return;
1411
1412 sub->is_cleaned = 1;
1413 list_del_rcu(&sub->xa_list);
1414
1415 if (list_empty(&sub->obj_list))
1416 return;
1417
1418 list_del_rcu(&sub->obj_list);
1419 /* check whether key level 1 for this obj_sub_list is empty */
1420 event = xa_load(&dev->devx_event_table.event_xa,
1421 sub->xa_key_level1);
1422 WARN_ON(!event);
1423
1424 xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1425 if (list_empty(&xa_val_level2->obj_sub_list)) {
1426 xa_erase(&event->object_ids,
1427 sub->xa_key_level2);
1428 kfree_rcu(xa_val_level2, rcu);
1429 }
1430 }
1431
devx_obj_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)1432 static int devx_obj_cleanup(struct ib_uobject *uobject,
1433 enum rdma_remove_reason why,
1434 struct uverbs_attr_bundle *attrs)
1435 {
1436 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1437 struct mlx5_devx_event_table *devx_event_table;
1438 struct devx_obj *obj = uobject->object;
1439 struct devx_event_subscription *sub_entry, *tmp;
1440 struct mlx5_ib_dev *dev;
1441 int ret;
1442
1443 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1444 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
1445 xa_erase(&obj->ib_dev->odp_mkeys,
1446 mlx5_base_mkey(obj->mkey.key)))
1447 /*
1448 * The pagefault_single_data_segment() does commands against
1449 * the mmkey, we must wait for that to stop before freeing the
1450 * mkey, as another allocation could get the same mkey #.
1451 */
1452 mlx5r_deref_wait_odp_mkey(&obj->mkey);
1453
1454 if (obj->flags & DEVX_OBJ_FLAGS_HW_FREED)
1455 ret = 0;
1456 else if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1457 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1458 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1459 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1460 else
1461 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1462 obj->dinlen, out, sizeof(out));
1463 if (ret)
1464 return ret;
1465
1466 devx_event_table = &dev->devx_event_table;
1467
1468 mutex_lock(&devx_event_table->event_xa_lock);
1469 list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1470 devx_cleanup_subscription(dev, sub_entry);
1471 mutex_unlock(&devx_event_table->event_xa_lock);
1472
1473 kfree(obj);
1474 return ret;
1475 }
1476
devx_cq_comp(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe)1477 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1478 {
1479 struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1480 struct mlx5_devx_event_table *table;
1481 struct devx_event *event;
1482 struct devx_obj_event *obj_event;
1483 u32 obj_id = mcq->cqn;
1484
1485 table = &obj->ib_dev->devx_event_table;
1486 rcu_read_lock();
1487 event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1488 if (!event)
1489 goto out;
1490
1491 obj_event = xa_load(&event->object_ids, obj_id);
1492 if (!obj_event)
1493 goto out;
1494
1495 dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1496 out:
1497 rcu_read_unlock();
1498 }
1499
is_apu_cq(struct mlx5_ib_dev * dev,const void * in)1500 static bool is_apu_cq(struct mlx5_ib_dev *dev, const void *in)
1501 {
1502 if (!MLX5_CAP_GEN(dev->mdev, apu) ||
1503 !MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), apu_cq))
1504 return false;
1505
1506 return true;
1507 }
1508
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)1509 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1510 struct uverbs_attr_bundle *attrs)
1511 {
1512 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1513 int cmd_out_len = uverbs_attr_get_len(attrs,
1514 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1515 int cmd_in_len = uverbs_attr_get_len(attrs,
1516 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1517 void *cmd_out;
1518 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1519 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1520 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1521 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1522 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1523 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1524 struct devx_obj *obj;
1525 u16 obj_type = 0;
1526 int err, err2 = 0;
1527 int uid;
1528 u32 obj_id;
1529 u16 opcode;
1530
1531 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1532 return -EINVAL;
1533
1534 uid = devx_get_uid(c, cmd_in);
1535 if (uid < 0)
1536 return uid;
1537
1538 if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1539 return -EINVAL;
1540
1541 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1542 if (IS_ERR(cmd_out))
1543 return PTR_ERR(cmd_out);
1544
1545 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1546 if (!obj)
1547 return -ENOMEM;
1548
1549 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1550 if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1551 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1552 if (err)
1553 goto obj_free;
1554 } else {
1555 devx_set_umem_valid(cmd_in);
1556 }
1557
1558 if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1559 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1560 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1561 cmd_in_len, cmd_out, cmd_out_len);
1562 } else if (opcode == MLX5_CMD_OP_CREATE_CQ &&
1563 !is_apu_cq(dev, cmd_in)) {
1564 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1565 obj->core_cq.comp = devx_cq_comp;
1566 err = mlx5_create_cq(dev->mdev, &obj->core_cq,
1567 cmd_in, cmd_in_len, cmd_out,
1568 cmd_out_len);
1569 } else {
1570 err = mlx5_cmd_do(dev->mdev, cmd_in, cmd_in_len,
1571 cmd_out, cmd_out_len);
1572 }
1573
1574 if (err == -EREMOTEIO)
1575 err2 = uverbs_copy_to(attrs,
1576 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1577 cmd_out, cmd_out_len);
1578 if (err)
1579 goto obj_free;
1580
1581 if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1582 u32 bulk = MLX5_GET(alloc_flow_counter_in,
1583 cmd_in,
1584 flow_counter_bulk_log_size);
1585
1586 if (bulk)
1587 bulk = 1 << bulk;
1588 else
1589 bulk = 128UL * MLX5_GET(alloc_flow_counter_in,
1590 cmd_in,
1591 flow_counter_bulk);
1592 obj->flow_counter_bulk_size = bulk;
1593 }
1594
1595 uobj->object = obj;
1596 INIT_LIST_HEAD(&obj->event_sub);
1597 obj->ib_dev = dev;
1598 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1599 &obj_id);
1600 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1601
1602 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1603 if (err)
1604 goto obj_destroy;
1605
1606 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1607 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1608 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1609
1610 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1611 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1612 if (err)
1613 goto obj_destroy;
1614 }
1615 return 0;
1616
1617 obj_destroy:
1618 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1619 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1620 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1621 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1622 else
1623 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1624 sizeof(out));
1625 obj_free:
1626 kfree(obj);
1627 return err2 ?: err;
1628 }
1629
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)1630 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1631 struct uverbs_attr_bundle *attrs)
1632 {
1633 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1634 int cmd_out_len = uverbs_attr_get_len(attrs,
1635 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1636 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1637 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1638 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1639 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1640 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1641 void *cmd_out;
1642 int err, err2;
1643 int uid;
1644
1645 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1646 return -EINVAL;
1647
1648 uid = devx_get_uid(c, cmd_in);
1649 if (uid < 0)
1650 return uid;
1651
1652 if (!devx_is_obj_modify_cmd(cmd_in))
1653 return -EINVAL;
1654
1655 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1656 return -EINVAL;
1657
1658 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1659 if (IS_ERR(cmd_out))
1660 return PTR_ERR(cmd_out);
1661
1662 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1663 devx_set_umem_valid(cmd_in);
1664
1665 err = mlx5_cmd_do(mdev->mdev, cmd_in,
1666 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1667 cmd_out, cmd_out_len);
1668 if (err && err != -EREMOTEIO)
1669 return err;
1670
1671 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1672 cmd_out, cmd_out_len);
1673
1674 return err2 ?: err;
1675 }
1676
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)1677 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1678 struct uverbs_attr_bundle *attrs)
1679 {
1680 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1681 int cmd_out_len = uverbs_attr_get_len(attrs,
1682 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1683 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1684 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1685 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1686 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1687 void *cmd_out;
1688 int err, err2;
1689 int uid;
1690 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1691
1692 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1693 return -EINVAL;
1694
1695 uid = devx_get_uid(c, cmd_in);
1696 if (uid < 0)
1697 return uid;
1698
1699 if (!devx_is_obj_query_cmd(cmd_in))
1700 return -EINVAL;
1701
1702 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1703 return -EINVAL;
1704
1705 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1706 if (IS_ERR(cmd_out))
1707 return PTR_ERR(cmd_out);
1708
1709 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1710 err = mlx5_cmd_do(mdev->mdev, cmd_in,
1711 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1712 cmd_out, cmd_out_len);
1713 if (err && err != -EREMOTEIO)
1714 return err;
1715
1716 err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1717 cmd_out, cmd_out_len);
1718
1719 return err2 ?: err;
1720 }
1721
1722 struct devx_async_event_queue {
1723 spinlock_t lock;
1724 wait_queue_head_t poll_wait;
1725 struct list_head event_list;
1726 atomic_t bytes_in_use;
1727 u8 is_destroyed:1;
1728 };
1729
1730 struct devx_async_cmd_event_file {
1731 struct ib_uobject uobj;
1732 struct devx_async_event_queue ev_queue;
1733 struct mlx5_async_ctx async_ctx;
1734 };
1735
devx_init_event_queue(struct devx_async_event_queue * ev_queue)1736 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1737 {
1738 spin_lock_init(&ev_queue->lock);
1739 INIT_LIST_HEAD(&ev_queue->event_list);
1740 init_waitqueue_head(&ev_queue->poll_wait);
1741 atomic_set(&ev_queue->bytes_in_use, 0);
1742 ev_queue->is_destroyed = 0;
1743 }
1744
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)1745 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1746 struct uverbs_attr_bundle *attrs)
1747 {
1748 struct devx_async_cmd_event_file *ev_file;
1749
1750 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1751 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1752 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1753
1754 ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1755 uobj);
1756 devx_init_event_queue(&ev_file->ev_queue);
1757 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1758 return 0;
1759 }
1760
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)1761 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1762 struct uverbs_attr_bundle *attrs)
1763 {
1764 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1765 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1766 struct devx_async_event_file *ev_file;
1767 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1768 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1769 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1770 u32 flags;
1771 int err;
1772
1773 err = uverbs_get_flags32(&flags, attrs,
1774 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1775 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1776
1777 if (err)
1778 return err;
1779
1780 ev_file = container_of(uobj, struct devx_async_event_file,
1781 uobj);
1782 spin_lock_init(&ev_file->lock);
1783 INIT_LIST_HEAD(&ev_file->event_list);
1784 init_waitqueue_head(&ev_file->poll_wait);
1785 if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1786 ev_file->omit_data = 1;
1787 INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1788 ev_file->dev = dev;
1789 get_device(&dev->ib_dev.dev);
1790 return 0;
1791 }
1792
devx_query_callback(int status,struct mlx5_async_work * context)1793 static void devx_query_callback(int status, struct mlx5_async_work *context)
1794 {
1795 struct devx_async_data *async_data =
1796 container_of(context, struct devx_async_data, cb_work);
1797 struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1798 struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1799 unsigned long flags;
1800
1801 /*
1802 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1803 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1804 * routine returns, ensuring that it always remains valid here.
1805 */
1806 spin_lock_irqsave(&ev_queue->lock, flags);
1807 list_add_tail(&async_data->list, &ev_queue->event_list);
1808 spin_unlock_irqrestore(&ev_queue->lock, flags);
1809
1810 wake_up_interruptible(&ev_queue->poll_wait);
1811 }
1812
1813 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1814
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)1815 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1816 struct uverbs_attr_bundle *attrs)
1817 {
1818 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1819 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1820 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1821 attrs,
1822 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1823 u16 cmd_out_len;
1824 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1825 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1826 struct ib_uobject *fd_uobj;
1827 int err;
1828 int uid;
1829 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1830 struct devx_async_cmd_event_file *ev_file;
1831 struct devx_async_data *async_data;
1832
1833 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1834 return -EINVAL;
1835
1836 uid = devx_get_uid(c, cmd_in);
1837 if (uid < 0)
1838 return uid;
1839
1840 if (!devx_is_obj_query_cmd(cmd_in))
1841 return -EINVAL;
1842
1843 err = uverbs_get_const(&cmd_out_len, attrs,
1844 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1845 if (err)
1846 return err;
1847
1848 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1849 return -EINVAL;
1850
1851 fd_uobj = uverbs_attr_get_uobject(attrs,
1852 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1853 if (IS_ERR(fd_uobj))
1854 return PTR_ERR(fd_uobj);
1855
1856 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1857 uobj);
1858
1859 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1860 MAX_ASYNC_BYTES_IN_USE) {
1861 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1862 return -EAGAIN;
1863 }
1864
1865 async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1866 cmd_out_len), GFP_KERNEL);
1867 if (!async_data) {
1868 err = -ENOMEM;
1869 goto sub_bytes;
1870 }
1871
1872 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1873 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1874 if (err)
1875 goto free_async;
1876
1877 async_data->cmd_out_len = cmd_out_len;
1878 async_data->mdev = mdev;
1879 async_data->ev_file = ev_file;
1880
1881 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1882 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1883 uverbs_attr_get_len(attrs,
1884 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1885 async_data->hdr.out_data,
1886 async_data->cmd_out_len,
1887 devx_query_callback, &async_data->cb_work);
1888
1889 if (err)
1890 goto free_async;
1891
1892 return 0;
1893
1894 free_async:
1895 kvfree(async_data);
1896 sub_bytes:
1897 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1898 return err;
1899 }
1900
1901 static void
subscribe_event_xa_dealloc(struct mlx5_devx_event_table * devx_event_table,u32 key_level1,bool is_level2,u32 key_level2)1902 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1903 u32 key_level1,
1904 bool is_level2,
1905 u32 key_level2)
1906 {
1907 struct devx_event *event;
1908 struct devx_obj_event *xa_val_level2;
1909
1910 /* Level 1 is valid for future use, no need to free */
1911 if (!is_level2)
1912 return;
1913
1914 event = xa_load(&devx_event_table->event_xa, key_level1);
1915 WARN_ON(!event);
1916
1917 xa_val_level2 = xa_load(&event->object_ids,
1918 key_level2);
1919 if (list_empty(&xa_val_level2->obj_sub_list)) {
1920 xa_erase(&event->object_ids,
1921 key_level2);
1922 kfree_rcu(xa_val_level2, rcu);
1923 }
1924 }
1925
1926 static int
subscribe_event_xa_alloc(struct mlx5_devx_event_table * devx_event_table,u32 key_level1,bool is_level2,u32 key_level2)1927 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1928 u32 key_level1,
1929 bool is_level2,
1930 u32 key_level2)
1931 {
1932 struct devx_obj_event *obj_event;
1933 struct devx_event *event;
1934 int err;
1935
1936 event = xa_load(&devx_event_table->event_xa, key_level1);
1937 if (!event) {
1938 event = kzalloc(sizeof(*event), GFP_KERNEL);
1939 if (!event)
1940 return -ENOMEM;
1941
1942 INIT_LIST_HEAD(&event->unaffiliated_list);
1943 xa_init(&event->object_ids);
1944
1945 err = xa_insert(&devx_event_table->event_xa,
1946 key_level1,
1947 event,
1948 GFP_KERNEL);
1949 if (err) {
1950 kfree(event);
1951 return err;
1952 }
1953 }
1954
1955 if (!is_level2)
1956 return 0;
1957
1958 obj_event = xa_load(&event->object_ids, key_level2);
1959 if (!obj_event) {
1960 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1961 if (!obj_event)
1962 /* Level1 is valid for future use, no need to free */
1963 return -ENOMEM;
1964
1965 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1966 err = xa_insert(&event->object_ids,
1967 key_level2,
1968 obj_event,
1969 GFP_KERNEL);
1970 if (err) {
1971 kfree(obj_event);
1972 return err;
1973 }
1974 }
1975
1976 return 0;
1977 }
1978
is_valid_events_legacy(int num_events,u16 * event_type_num_list,struct devx_obj * obj)1979 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1980 struct devx_obj *obj)
1981 {
1982 int i;
1983
1984 for (i = 0; i < num_events; i++) {
1985 if (obj) {
1986 if (!is_legacy_obj_event_num(event_type_num_list[i]))
1987 return false;
1988 } else if (!is_legacy_unaffiliated_event_num(
1989 event_type_num_list[i])) {
1990 return false;
1991 }
1992 }
1993
1994 return true;
1995 }
1996
1997 #define MAX_SUPP_EVENT_NUM 255
is_valid_events(struct mlx5_core_dev * dev,int num_events,u16 * event_type_num_list,struct devx_obj * obj)1998 static bool is_valid_events(struct mlx5_core_dev *dev,
1999 int num_events, u16 *event_type_num_list,
2000 struct devx_obj *obj)
2001 {
2002 __be64 *aff_events;
2003 __be64 *unaff_events;
2004 int mask_entry;
2005 int mask_bit;
2006 int i;
2007
2008 if (MLX5_CAP_GEN(dev, event_cap)) {
2009 aff_events = MLX5_CAP_DEV_EVENT(dev,
2010 user_affiliated_events);
2011 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2012 user_unaffiliated_events);
2013 } else {
2014 return is_valid_events_legacy(num_events, event_type_num_list,
2015 obj);
2016 }
2017
2018 for (i = 0; i < num_events; i++) {
2019 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
2020 return false;
2021
2022 mask_entry = event_type_num_list[i] / 64;
2023 mask_bit = event_type_num_list[i] % 64;
2024
2025 if (obj) {
2026 /* CQ completion */
2027 if (event_type_num_list[i] == 0)
2028 continue;
2029
2030 if (!(be64_to_cpu(aff_events[mask_entry]) &
2031 (1ull << mask_bit)))
2032 return false;
2033
2034 continue;
2035 }
2036
2037 if (!(be64_to_cpu(unaff_events[mask_entry]) &
2038 (1ull << mask_bit)))
2039 return false;
2040 }
2041
2042 return true;
2043 }
2044
2045 #define MAX_NUM_EVENTS 16
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)2046 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
2047 struct uverbs_attr_bundle *attrs)
2048 {
2049 struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
2050 attrs,
2051 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
2052 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2053 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2054 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2055 struct ib_uobject *fd_uobj;
2056 struct devx_obj *obj = NULL;
2057 struct devx_async_event_file *ev_file;
2058 struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
2059 u16 *event_type_num_list;
2060 struct devx_event_subscription *event_sub, *tmp_sub;
2061 struct list_head sub_list;
2062 int redirect_fd;
2063 bool use_eventfd = false;
2064 int num_events;
2065 u16 obj_type = 0;
2066 u64 cookie = 0;
2067 u32 obj_id = 0;
2068 int err;
2069 int i;
2070
2071 if (!c->devx_uid)
2072 return -EINVAL;
2073
2074 if (!IS_ERR(devx_uobj)) {
2075 obj = (struct devx_obj *)devx_uobj->object;
2076 if (obj)
2077 obj_id = get_dec_obj_id(obj->obj_id);
2078 }
2079
2080 fd_uobj = uverbs_attr_get_uobject(attrs,
2081 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
2082 if (IS_ERR(fd_uobj))
2083 return PTR_ERR(fd_uobj);
2084
2085 ev_file = container_of(fd_uobj, struct devx_async_event_file,
2086 uobj);
2087
2088 if (uverbs_attr_is_valid(attrs,
2089 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
2090 err = uverbs_copy_from(&redirect_fd, attrs,
2091 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
2092 if (err)
2093 return err;
2094
2095 use_eventfd = true;
2096 }
2097
2098 if (uverbs_attr_is_valid(attrs,
2099 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
2100 if (use_eventfd)
2101 return -EINVAL;
2102
2103 err = uverbs_copy_from(&cookie, attrs,
2104 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
2105 if (err)
2106 return err;
2107 }
2108
2109 num_events = uverbs_attr_ptr_get_array_size(
2110 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2111 sizeof(u16));
2112
2113 if (num_events < 0)
2114 return num_events;
2115
2116 if (num_events > MAX_NUM_EVENTS)
2117 return -EINVAL;
2118
2119 event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2120 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2121
2122 if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2123 return -EINVAL;
2124
2125 INIT_LIST_HEAD(&sub_list);
2126
2127 /* Protect from concurrent subscriptions to same XA entries to allow
2128 * both to succeed
2129 */
2130 mutex_lock(&devx_event_table->event_xa_lock);
2131 for (i = 0; i < num_events; i++) {
2132 u32 key_level1;
2133
2134 if (obj)
2135 obj_type = get_dec_obj_type(obj,
2136 event_type_num_list[i]);
2137 key_level1 = event_type_num_list[i] | obj_type << 16;
2138
2139 err = subscribe_event_xa_alloc(devx_event_table,
2140 key_level1,
2141 obj,
2142 obj_id);
2143 if (err)
2144 goto err;
2145
2146 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2147 if (!event_sub) {
2148 err = -ENOMEM;
2149 goto err;
2150 }
2151
2152 list_add_tail(&event_sub->event_list, &sub_list);
2153 uverbs_uobject_get(&ev_file->uobj);
2154 if (use_eventfd) {
2155 event_sub->eventfd =
2156 eventfd_ctx_fdget(redirect_fd);
2157
2158 if (IS_ERR(event_sub->eventfd)) {
2159 err = PTR_ERR(event_sub->eventfd);
2160 event_sub->eventfd = NULL;
2161 goto err;
2162 }
2163 }
2164
2165 event_sub->cookie = cookie;
2166 event_sub->ev_file = ev_file;
2167 /* May be needed upon cleanup the devx object/subscription */
2168 event_sub->xa_key_level1 = key_level1;
2169 event_sub->xa_key_level2 = obj_id;
2170 INIT_LIST_HEAD(&event_sub->obj_list);
2171 }
2172
2173 /* Once all the allocations and the XA data insertions were done we
2174 * can go ahead and add all the subscriptions to the relevant lists
2175 * without concern of a failure.
2176 */
2177 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2178 struct devx_event *event;
2179 struct devx_obj_event *obj_event;
2180
2181 list_del_init(&event_sub->event_list);
2182
2183 spin_lock_irq(&ev_file->lock);
2184 list_add_tail_rcu(&event_sub->file_list,
2185 &ev_file->subscribed_events_list);
2186 spin_unlock_irq(&ev_file->lock);
2187
2188 event = xa_load(&devx_event_table->event_xa,
2189 event_sub->xa_key_level1);
2190 WARN_ON(!event);
2191
2192 if (!obj) {
2193 list_add_tail_rcu(&event_sub->xa_list,
2194 &event->unaffiliated_list);
2195 continue;
2196 }
2197
2198 obj_event = xa_load(&event->object_ids, obj_id);
2199 WARN_ON(!obj_event);
2200 list_add_tail_rcu(&event_sub->xa_list,
2201 &obj_event->obj_sub_list);
2202 list_add_tail_rcu(&event_sub->obj_list,
2203 &obj->event_sub);
2204 }
2205
2206 mutex_unlock(&devx_event_table->event_xa_lock);
2207 return 0;
2208
2209 err:
2210 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2211 list_del(&event_sub->event_list);
2212
2213 subscribe_event_xa_dealloc(devx_event_table,
2214 event_sub->xa_key_level1,
2215 obj,
2216 obj_id);
2217
2218 if (event_sub->eventfd)
2219 eventfd_ctx_put(event_sub->eventfd);
2220 uverbs_uobject_put(&event_sub->ev_file->uobj);
2221 kfree(event_sub);
2222 }
2223
2224 mutex_unlock(&devx_event_table->event_xa_lock);
2225 return err;
2226 }
2227
devx_umem_get(struct mlx5_ib_dev * dev,struct ib_ucontext * ucontext,struct uverbs_attr_bundle * attrs,struct devx_umem * obj,u32 access_flags)2228 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2229 struct uverbs_attr_bundle *attrs,
2230 struct devx_umem *obj, u32 access_flags)
2231 {
2232 u64 addr;
2233 size_t size;
2234 int err;
2235
2236 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2237 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2238 return -EFAULT;
2239
2240 err = ib_check_mr_access(&dev->ib_dev, access_flags);
2241 if (err)
2242 return err;
2243
2244 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) {
2245 struct ib_umem_dmabuf *umem_dmabuf;
2246 int dmabuf_fd;
2247
2248 err = uverbs_get_raw_fd(&dmabuf_fd, attrs,
2249 MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD);
2250 if (err)
2251 return -EFAULT;
2252
2253 umem_dmabuf = ib_umem_dmabuf_get_pinned(
2254 &dev->ib_dev, addr, size, dmabuf_fd, access_flags);
2255 if (IS_ERR(umem_dmabuf))
2256 return PTR_ERR(umem_dmabuf);
2257 obj->umem = &umem_dmabuf->umem;
2258 } else {
2259 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags);
2260 if (IS_ERR(obj->umem))
2261 return PTR_ERR(obj->umem);
2262 }
2263 return 0;
2264 }
2265
devx_umem_find_best_pgsize(struct ib_umem * umem,unsigned long pgsz_bitmap)2266 static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,
2267 unsigned long pgsz_bitmap)
2268 {
2269 unsigned long page_size;
2270
2271 /* Don't bother checking larger page sizes as offset must be zero and
2272 * total DEVX umem length must be equal to total umem length.
2273 */
2274 pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
2275 PAGE_SHIFT),
2276 MLX5_ADAPTER_PAGE_SHIFT);
2277 if (!pgsz_bitmap)
2278 return 0;
2279
2280 page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);
2281 if (!page_size)
2282 return 0;
2283
2284 /* If the page_size is less than the CPU page size then we can use the
2285 * offset and create a umem which is a subset of the page list.
2286 * For larger page sizes we can't be sure the DMA list reflects the
2287 * VA so we must ensure that the umem extent is exactly equal to the
2288 * page list. Reduce the page size until one of these cases is true.
2289 */
2290 while ((ib_umem_dma_offset(umem, page_size) != 0 ||
2291 (umem->length % page_size) != 0) &&
2292 page_size > PAGE_SIZE)
2293 page_size /= 2;
2294
2295 return page_size;
2296 }
2297
devx_umem_reg_cmd_alloc(struct mlx5_ib_dev * dev,struct uverbs_attr_bundle * attrs,struct devx_umem * obj,struct devx_umem_reg_cmd * cmd,int access)2298 static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2299 struct uverbs_attr_bundle *attrs,
2300 struct devx_umem *obj,
2301 struct devx_umem_reg_cmd *cmd,
2302 int access)
2303 {
2304 unsigned long pgsz_bitmap;
2305 unsigned int page_size;
2306 __be64 *mtt;
2307 void *umem;
2308 int ret;
2309
2310 /*
2311 * If the user does not pass in pgsz_bitmap then the user promises not
2312 * to use umem_offset!=0 in any commands that allocate on top of the
2313 * umem.
2314 *
2315 * If the user wants to use a umem_offset then it must pass in
2316 * pgsz_bitmap which guides the maximum page size and thus maximum
2317 * object alignment inside the umem. See the PRM.
2318 *
2319 * Users are not allowed to use IOVA here, mkeys are not supported on
2320 * umem.
2321 */
2322 ret = uverbs_get_const_default(&pgsz_bitmap, attrs,
2323 MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2324 GENMASK_ULL(63,
2325 min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));
2326 if (ret)
2327 return ret;
2328
2329 page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap);
2330 if (!page_size)
2331 return -EINVAL;
2332
2333 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2334 (MLX5_ST_SZ_BYTES(mtt) *
2335 ib_umem_num_dma_blocks(obj->umem, page_size));
2336 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2337 if (IS_ERR(cmd->in))
2338 return PTR_ERR(cmd->in);
2339
2340 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2341 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2342
2343 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2344 MLX5_SET64(umem, umem, num_of_mtt,
2345 ib_umem_num_dma_blocks(obj->umem, page_size));
2346 MLX5_SET(umem, umem, log_page_size,
2347 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2348 MLX5_SET(umem, umem, page_offset,
2349 ib_umem_dma_offset(obj->umem, page_size));
2350
2351 if (mlx5_umem_needs_ats(dev, obj->umem, access))
2352 MLX5_SET(umem, umem, ats, 1);
2353
2354 mlx5_ib_populate_pas(obj->umem, page_size, mtt,
2355 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2356 MLX5_IB_MTT_READ);
2357 return 0;
2358 }
2359
UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)2360 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2361 struct uverbs_attr_bundle *attrs)
2362 {
2363 struct devx_umem_reg_cmd cmd;
2364 struct devx_umem *obj;
2365 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2366 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2367 u32 obj_id;
2368 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2369 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2370 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2371 int access_flags;
2372 int err;
2373
2374 if (!c->devx_uid)
2375 return -EINVAL;
2376
2377 err = uverbs_get_flags32(&access_flags, attrs,
2378 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2379 IB_ACCESS_LOCAL_WRITE |
2380 IB_ACCESS_REMOTE_WRITE |
2381 IB_ACCESS_REMOTE_READ |
2382 IB_ACCESS_RELAXED_ORDERING);
2383 if (err)
2384 return err;
2385
2386 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2387 if (!obj)
2388 return -ENOMEM;
2389
2390 err = devx_umem_get(dev, &c->ibucontext, attrs, obj, access_flags);
2391 if (err)
2392 goto err_obj_free;
2393
2394 err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd, access_flags);
2395 if (err)
2396 goto err_umem_release;
2397
2398 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2399 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2400 sizeof(cmd.out));
2401 if (err)
2402 goto err_umem_release;
2403
2404 obj->mdev = dev->mdev;
2405 uobj->object = obj;
2406 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2407 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2408
2409 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2410 sizeof(obj_id));
2411 return err;
2412
2413 err_umem_release:
2414 ib_umem_release(obj->umem);
2415 err_obj_free:
2416 kfree(obj);
2417 return err;
2418 }
2419
devx_umem_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2420 static int devx_umem_cleanup(struct ib_uobject *uobject,
2421 enum rdma_remove_reason why,
2422 struct uverbs_attr_bundle *attrs)
2423 {
2424 struct devx_umem *obj = uobject->object;
2425 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2426 int err;
2427
2428 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2429 if (err)
2430 return err;
2431
2432 ib_umem_release(obj->umem);
2433 kfree(obj);
2434 return 0;
2435 }
2436
is_unaffiliated_event(struct mlx5_core_dev * dev,unsigned long event_type)2437 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2438 unsigned long event_type)
2439 {
2440 __be64 *unaff_events;
2441 int mask_entry;
2442 int mask_bit;
2443
2444 if (!MLX5_CAP_GEN(dev, event_cap))
2445 return is_legacy_unaffiliated_event_num(event_type);
2446
2447 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2448 user_unaffiliated_events);
2449 WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2450
2451 mask_entry = event_type / 64;
2452 mask_bit = event_type % 64;
2453
2454 if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2455 return false;
2456
2457 return true;
2458 }
2459
devx_get_obj_id_from_event(unsigned long event_type,void * data)2460 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2461 {
2462 struct mlx5_eqe *eqe = data;
2463 u32 obj_id = 0;
2464
2465 switch (event_type) {
2466 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2467 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2468 case MLX5_EVENT_TYPE_PATH_MIG:
2469 case MLX5_EVENT_TYPE_COMM_EST:
2470 case MLX5_EVENT_TYPE_SQ_DRAINED:
2471 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2472 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2473 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2474 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2475 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2476 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2477 break;
2478 case MLX5_EVENT_TYPE_XRQ_ERROR:
2479 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2480 break;
2481 case MLX5_EVENT_TYPE_DCT_DRAINED:
2482 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2483 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2484 break;
2485 case MLX5_EVENT_TYPE_CQ_ERROR:
2486 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2487 break;
2488 default:
2489 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2490 break;
2491 }
2492
2493 return obj_id;
2494 }
2495
deliver_event(struct devx_event_subscription * event_sub,const void * data)2496 static int deliver_event(struct devx_event_subscription *event_sub,
2497 const void *data)
2498 {
2499 struct devx_async_event_file *ev_file;
2500 struct devx_async_event_data *event_data;
2501 unsigned long flags;
2502
2503 ev_file = event_sub->ev_file;
2504
2505 if (ev_file->omit_data) {
2506 spin_lock_irqsave(&ev_file->lock, flags);
2507 if (!list_empty(&event_sub->event_list) ||
2508 ev_file->is_destroyed) {
2509 spin_unlock_irqrestore(&ev_file->lock, flags);
2510 return 0;
2511 }
2512
2513 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2514 spin_unlock_irqrestore(&ev_file->lock, flags);
2515 wake_up_interruptible(&ev_file->poll_wait);
2516 return 0;
2517 }
2518
2519 event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2520 GFP_ATOMIC);
2521 if (!event_data) {
2522 spin_lock_irqsave(&ev_file->lock, flags);
2523 ev_file->is_overflow_err = 1;
2524 spin_unlock_irqrestore(&ev_file->lock, flags);
2525 return -ENOMEM;
2526 }
2527
2528 event_data->hdr.cookie = event_sub->cookie;
2529 memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2530
2531 spin_lock_irqsave(&ev_file->lock, flags);
2532 if (!ev_file->is_destroyed)
2533 list_add_tail(&event_data->list, &ev_file->event_list);
2534 else
2535 kfree(event_data);
2536 spin_unlock_irqrestore(&ev_file->lock, flags);
2537 wake_up_interruptible(&ev_file->poll_wait);
2538
2539 return 0;
2540 }
2541
dispatch_event_fd(struct list_head * fd_list,const void * data)2542 static void dispatch_event_fd(struct list_head *fd_list,
2543 const void *data)
2544 {
2545 struct devx_event_subscription *item;
2546
2547 list_for_each_entry_rcu(item, fd_list, xa_list) {
2548 if (item->eventfd)
2549 eventfd_signal(item->eventfd);
2550 else
2551 deliver_event(item, data);
2552 }
2553 }
2554
devx_event_notifier(struct notifier_block * nb,unsigned long event_type,void * data)2555 static int devx_event_notifier(struct notifier_block *nb,
2556 unsigned long event_type, void *data)
2557 {
2558 struct mlx5_devx_event_table *table;
2559 struct mlx5_ib_dev *dev;
2560 struct devx_event *event;
2561 struct devx_obj_event *obj_event;
2562 u16 obj_type = 0;
2563 bool is_unaffiliated;
2564 u32 obj_id;
2565
2566 /* Explicit filtering to kernel events which may occur frequently */
2567 if (event_type == MLX5_EVENT_TYPE_CMD ||
2568 event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2569 return NOTIFY_OK;
2570
2571 table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2572 dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2573 is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2574
2575 if (!is_unaffiliated)
2576 obj_type = get_event_obj_type(event_type, data);
2577
2578 rcu_read_lock();
2579 event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2580 if (!event) {
2581 rcu_read_unlock();
2582 return NOTIFY_DONE;
2583 }
2584
2585 if (is_unaffiliated) {
2586 dispatch_event_fd(&event->unaffiliated_list, data);
2587 rcu_read_unlock();
2588 return NOTIFY_OK;
2589 }
2590
2591 obj_id = devx_get_obj_id_from_event(event_type, data);
2592 obj_event = xa_load(&event->object_ids, obj_id);
2593 if (!obj_event) {
2594 rcu_read_unlock();
2595 return NOTIFY_DONE;
2596 }
2597
2598 dispatch_event_fd(&obj_event->obj_sub_list, data);
2599
2600 rcu_read_unlock();
2601 return NOTIFY_OK;
2602 }
2603
mlx5_ib_devx_init(struct mlx5_ib_dev * dev)2604 int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
2605 {
2606 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2607 int uid;
2608
2609 uid = mlx5_ib_devx_create(dev, false, 0);
2610 if (uid > 0) {
2611 dev->devx_whitelist_uid = uid;
2612 xa_init(&table->event_xa);
2613 mutex_init(&table->event_xa_lock);
2614 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2615 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2616 }
2617
2618 return 0;
2619 }
2620
mlx5_ib_devx_cleanup(struct mlx5_ib_dev * dev)2621 void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
2622 {
2623 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2624 struct devx_event_subscription *sub, *tmp;
2625 struct devx_event *event;
2626 void *entry;
2627 unsigned long id;
2628
2629 if (dev->devx_whitelist_uid) {
2630 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2631 mutex_lock(&dev->devx_event_table.event_xa_lock);
2632 xa_for_each(&table->event_xa, id, entry) {
2633 event = entry;
2634 list_for_each_entry_safe(
2635 sub, tmp, &event->unaffiliated_list, xa_list)
2636 devx_cleanup_subscription(dev, sub);
2637 kfree(entry);
2638 }
2639 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2640 xa_destroy(&table->event_xa);
2641
2642 mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
2643 }
2644 }
2645
devx_async_destroy_cb(int status,struct mlx5_async_work * context)2646 static void devx_async_destroy_cb(int status, struct mlx5_async_work *context)
2647 {
2648 struct mlx5_async_cmd *devx_out = container_of(context,
2649 struct mlx5_async_cmd, cb_work);
2650 struct devx_obj *obj = devx_out->uobject->object;
2651
2652 if (!status)
2653 obj->flags |= DEVX_OBJ_FLAGS_HW_FREED;
2654
2655 complete(&devx_out->comp);
2656 }
2657
devx_async_destroy(struct mlx5_ib_dev * dev,struct mlx5_async_cmd * cmd)2658 static void devx_async_destroy(struct mlx5_ib_dev *dev,
2659 struct mlx5_async_cmd *cmd)
2660 {
2661 init_completion(&cmd->comp);
2662 cmd->err = mlx5_cmd_exec_cb(&dev->async_ctx, cmd->in, cmd->in_size,
2663 &cmd->out, sizeof(cmd->out),
2664 devx_async_destroy_cb, &cmd->cb_work);
2665 }
2666
devx_wait_async_destroy(struct mlx5_async_cmd * cmd)2667 static void devx_wait_async_destroy(struct mlx5_async_cmd *cmd)
2668 {
2669 if (!cmd->err)
2670 wait_for_completion(&cmd->comp);
2671 atomic_set(&cmd->uobject->usecnt, 0);
2672 }
2673
mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file * ufile)2674 void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)
2675 {
2676 struct mlx5_async_cmd *async_cmd;
2677 struct ib_ucontext *ucontext = ufile->ucontext;
2678 struct ib_device *device = ucontext->device;
2679 struct mlx5_ib_dev *dev = to_mdev(device);
2680 struct ib_uobject *uobject;
2681 struct devx_obj *obj;
2682 int head = 0;
2683 int tail = 0;
2684
2685 async_cmd = kcalloc(MAX_ASYNC_CMDS, sizeof(*async_cmd), GFP_KERNEL);
2686 if (!async_cmd)
2687 return;
2688
2689 list_for_each_entry(uobject, &ufile->uobjects, list) {
2690 WARN_ON(uverbs_try_lock_object(uobject, UVERBS_LOOKUP_WRITE));
2691
2692 /*
2693 * Currently we only support QP destruction, if other objects
2694 * are to be destroyed need to add type synchronization to the
2695 * cleanup algorithm and handle pre/post FW cleanup for the
2696 * new types if needed.
2697 */
2698 if (uobj_get_object_id(uobject) != MLX5_IB_OBJECT_DEVX_OBJ ||
2699 (get_dec_obj_type(uobject->object, MLX5_EVENT_TYPE_MAX) !=
2700 MLX5_OBJ_TYPE_QP)) {
2701 atomic_set(&uobject->usecnt, 0);
2702 continue;
2703 }
2704
2705 obj = uobject->object;
2706
2707 async_cmd[tail % MAX_ASYNC_CMDS].in = obj->dinbox;
2708 async_cmd[tail % MAX_ASYNC_CMDS].in_size = obj->dinlen;
2709 async_cmd[tail % MAX_ASYNC_CMDS].uobject = uobject;
2710
2711 devx_async_destroy(dev, &async_cmd[tail % MAX_ASYNC_CMDS]);
2712 tail++;
2713
2714 if (tail - head == MAX_ASYNC_CMDS) {
2715 devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2716 head++;
2717 }
2718 }
2719
2720 while (head != tail) {
2721 devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2722 head++;
2723 }
2724
2725 kfree(async_cmd);
2726 }
2727
devx_async_cmd_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)2728 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2729 size_t count, loff_t *pos)
2730 {
2731 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2732 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2733 struct devx_async_data *event;
2734 int ret = 0;
2735 size_t eventsz;
2736
2737 spin_lock_irq(&ev_queue->lock);
2738
2739 while (list_empty(&ev_queue->event_list)) {
2740 spin_unlock_irq(&ev_queue->lock);
2741
2742 if (filp->f_flags & O_NONBLOCK)
2743 return -EAGAIN;
2744
2745 if (wait_event_interruptible(
2746 ev_queue->poll_wait,
2747 (!list_empty(&ev_queue->event_list) ||
2748 ev_queue->is_destroyed))) {
2749 return -ERESTARTSYS;
2750 }
2751
2752 spin_lock_irq(&ev_queue->lock);
2753 if (ev_queue->is_destroyed) {
2754 spin_unlock_irq(&ev_queue->lock);
2755 return -EIO;
2756 }
2757 }
2758
2759 event = list_entry(ev_queue->event_list.next,
2760 struct devx_async_data, list);
2761 eventsz = event->cmd_out_len +
2762 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2763
2764 if (eventsz > count) {
2765 spin_unlock_irq(&ev_queue->lock);
2766 return -ENOSPC;
2767 }
2768
2769 list_del(ev_queue->event_list.next);
2770 spin_unlock_irq(&ev_queue->lock);
2771
2772 if (copy_to_user(buf, &event->hdr, eventsz))
2773 ret = -EFAULT;
2774 else
2775 ret = eventsz;
2776
2777 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2778 kvfree(event);
2779 return ret;
2780 }
2781
devx_async_cmd_event_poll(struct file * filp,struct poll_table_struct * wait)2782 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2783 struct poll_table_struct *wait)
2784 {
2785 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2786 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2787 __poll_t pollflags = 0;
2788
2789 poll_wait(filp, &ev_queue->poll_wait, wait);
2790
2791 spin_lock_irq(&ev_queue->lock);
2792 if (ev_queue->is_destroyed)
2793 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2794 else if (!list_empty(&ev_queue->event_list))
2795 pollflags = EPOLLIN | EPOLLRDNORM;
2796 spin_unlock_irq(&ev_queue->lock);
2797
2798 return pollflags;
2799 }
2800
2801 static const struct file_operations devx_async_cmd_event_fops = {
2802 .owner = THIS_MODULE,
2803 .read = devx_async_cmd_event_read,
2804 .poll = devx_async_cmd_event_poll,
2805 .release = uverbs_uobject_fd_release,
2806 };
2807
devx_async_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)2808 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2809 size_t count, loff_t *pos)
2810 {
2811 struct devx_async_event_file *ev_file = filp->private_data;
2812 struct devx_event_subscription *event_sub;
2813 struct devx_async_event_data *event;
2814 int ret = 0;
2815 size_t eventsz;
2816 bool omit_data;
2817 void *event_data;
2818
2819 omit_data = ev_file->omit_data;
2820
2821 spin_lock_irq(&ev_file->lock);
2822
2823 if (ev_file->is_overflow_err) {
2824 ev_file->is_overflow_err = 0;
2825 spin_unlock_irq(&ev_file->lock);
2826 return -EOVERFLOW;
2827 }
2828
2829
2830 while (list_empty(&ev_file->event_list)) {
2831 spin_unlock_irq(&ev_file->lock);
2832
2833 if (filp->f_flags & O_NONBLOCK)
2834 return -EAGAIN;
2835
2836 if (wait_event_interruptible(ev_file->poll_wait,
2837 (!list_empty(&ev_file->event_list) ||
2838 ev_file->is_destroyed))) {
2839 return -ERESTARTSYS;
2840 }
2841
2842 spin_lock_irq(&ev_file->lock);
2843 if (ev_file->is_destroyed) {
2844 spin_unlock_irq(&ev_file->lock);
2845 return -EIO;
2846 }
2847 }
2848
2849 if (omit_data) {
2850 event_sub = list_first_entry(&ev_file->event_list,
2851 struct devx_event_subscription,
2852 event_list);
2853 eventsz = sizeof(event_sub->cookie);
2854 event_data = &event_sub->cookie;
2855 } else {
2856 event = list_first_entry(&ev_file->event_list,
2857 struct devx_async_event_data, list);
2858 eventsz = sizeof(struct mlx5_eqe) +
2859 sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2860 event_data = &event->hdr;
2861 }
2862
2863 if (eventsz > count) {
2864 spin_unlock_irq(&ev_file->lock);
2865 return -EINVAL;
2866 }
2867
2868 if (omit_data)
2869 list_del_init(&event_sub->event_list);
2870 else
2871 list_del(&event->list);
2872
2873 spin_unlock_irq(&ev_file->lock);
2874
2875 if (copy_to_user(buf, event_data, eventsz))
2876 /* This points to an application issue, not a kernel concern */
2877 ret = -EFAULT;
2878 else
2879 ret = eventsz;
2880
2881 if (!omit_data)
2882 kfree(event);
2883 return ret;
2884 }
2885
devx_async_event_poll(struct file * filp,struct poll_table_struct * wait)2886 static __poll_t devx_async_event_poll(struct file *filp,
2887 struct poll_table_struct *wait)
2888 {
2889 struct devx_async_event_file *ev_file = filp->private_data;
2890 __poll_t pollflags = 0;
2891
2892 poll_wait(filp, &ev_file->poll_wait, wait);
2893
2894 spin_lock_irq(&ev_file->lock);
2895 if (ev_file->is_destroyed)
2896 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2897 else if (!list_empty(&ev_file->event_list))
2898 pollflags = EPOLLIN | EPOLLRDNORM;
2899 spin_unlock_irq(&ev_file->lock);
2900
2901 return pollflags;
2902 }
2903
devx_free_subscription(struct rcu_head * rcu)2904 static void devx_free_subscription(struct rcu_head *rcu)
2905 {
2906 struct devx_event_subscription *event_sub =
2907 container_of(rcu, struct devx_event_subscription, rcu);
2908
2909 if (event_sub->eventfd)
2910 eventfd_ctx_put(event_sub->eventfd);
2911 uverbs_uobject_put(&event_sub->ev_file->uobj);
2912 kfree(event_sub);
2913 }
2914
2915 static const struct file_operations devx_async_event_fops = {
2916 .owner = THIS_MODULE,
2917 .read = devx_async_event_read,
2918 .poll = devx_async_event_poll,
2919 .release = uverbs_uobject_fd_release,
2920 };
2921
devx_async_cmd_event_destroy_uobj(struct ib_uobject * uobj,enum rdma_remove_reason why)2922 static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2923 enum rdma_remove_reason why)
2924 {
2925 struct devx_async_cmd_event_file *comp_ev_file =
2926 container_of(uobj, struct devx_async_cmd_event_file,
2927 uobj);
2928 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2929 struct devx_async_data *entry, *tmp;
2930
2931 spin_lock_irq(&ev_queue->lock);
2932 ev_queue->is_destroyed = 1;
2933 spin_unlock_irq(&ev_queue->lock);
2934 wake_up_interruptible(&ev_queue->poll_wait);
2935
2936 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2937
2938 spin_lock_irq(&comp_ev_file->ev_queue.lock);
2939 list_for_each_entry_safe(entry, tmp,
2940 &comp_ev_file->ev_queue.event_list, list) {
2941 list_del(&entry->list);
2942 kvfree(entry);
2943 }
2944 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2945 };
2946
devx_async_event_destroy_uobj(struct ib_uobject * uobj,enum rdma_remove_reason why)2947 static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2948 enum rdma_remove_reason why)
2949 {
2950 struct devx_async_event_file *ev_file =
2951 container_of(uobj, struct devx_async_event_file,
2952 uobj);
2953 struct devx_event_subscription *event_sub, *event_sub_tmp;
2954 struct mlx5_ib_dev *dev = ev_file->dev;
2955
2956 spin_lock_irq(&ev_file->lock);
2957 ev_file->is_destroyed = 1;
2958
2959 /* free the pending events allocation */
2960 if (ev_file->omit_data) {
2961 struct devx_event_subscription *event_sub, *tmp;
2962
2963 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2964 event_list)
2965 list_del_init(&event_sub->event_list);
2966
2967 } else {
2968 struct devx_async_event_data *entry, *tmp;
2969
2970 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2971 list) {
2972 list_del(&entry->list);
2973 kfree(entry);
2974 }
2975 }
2976
2977 spin_unlock_irq(&ev_file->lock);
2978 wake_up_interruptible(&ev_file->poll_wait);
2979
2980 mutex_lock(&dev->devx_event_table.event_xa_lock);
2981 /* delete the subscriptions which are related to this FD */
2982 list_for_each_entry_safe(event_sub, event_sub_tmp,
2983 &ev_file->subscribed_events_list, file_list) {
2984 devx_cleanup_subscription(dev, event_sub);
2985 list_del_rcu(&event_sub->file_list);
2986 /* subscription may not be used by the read API any more */
2987 call_rcu(&event_sub->rcu, devx_free_subscription);
2988 }
2989 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2990
2991 put_device(&dev->ib_dev.dev);
2992 };
2993
2994 DECLARE_UVERBS_NAMED_METHOD(
2995 MLX5_IB_METHOD_DEVX_UMEM_REG,
2996 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2997 MLX5_IB_OBJECT_DEVX_UMEM,
2998 UVERBS_ACCESS_NEW,
2999 UA_MANDATORY),
3000 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
3001 UVERBS_ATTR_TYPE(u64),
3002 UA_MANDATORY),
3003 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
3004 UVERBS_ATTR_TYPE(u64),
3005 UA_MANDATORY),
3006 UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD,
3007 UA_OPTIONAL),
3008 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
3009 enum ib_access_flags),
3010 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
3011 u64),
3012 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
3013 UVERBS_ATTR_TYPE(u32),
3014 UA_MANDATORY));
3015
3016 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3017 MLX5_IB_METHOD_DEVX_UMEM_DEREG,
3018 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
3019 MLX5_IB_OBJECT_DEVX_UMEM,
3020 UVERBS_ACCESS_DESTROY,
3021 UA_MANDATORY));
3022
3023 DECLARE_UVERBS_NAMED_METHOD(
3024 MLX5_IB_METHOD_DEVX_QUERY_EQN,
3025 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
3026 UVERBS_ATTR_TYPE(u32),
3027 UA_MANDATORY),
3028 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
3029 UVERBS_ATTR_TYPE(u32),
3030 UA_MANDATORY));
3031
3032 DECLARE_UVERBS_NAMED_METHOD(
3033 MLX5_IB_METHOD_DEVX_QUERY_UAR,
3034 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
3035 UVERBS_ATTR_TYPE(u32),
3036 UA_MANDATORY),
3037 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
3038 UVERBS_ATTR_TYPE(u32),
3039 UA_MANDATORY));
3040
3041 DECLARE_UVERBS_NAMED_METHOD(
3042 MLX5_IB_METHOD_DEVX_OTHER,
3043 UVERBS_ATTR_PTR_IN(
3044 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
3045 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3046 UA_MANDATORY,
3047 UA_ALLOC_AND_COPY),
3048 UVERBS_ATTR_PTR_OUT(
3049 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
3050 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3051 UA_MANDATORY));
3052
3053 DECLARE_UVERBS_NAMED_METHOD(
3054 MLX5_IB_METHOD_DEVX_OBJ_CREATE,
3055 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
3056 MLX5_IB_OBJECT_DEVX_OBJ,
3057 UVERBS_ACCESS_NEW,
3058 UA_MANDATORY),
3059 UVERBS_ATTR_PTR_IN(
3060 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
3061 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3062 UA_MANDATORY,
3063 UA_ALLOC_AND_COPY),
3064 UVERBS_ATTR_PTR_OUT(
3065 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
3066 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3067 UA_MANDATORY));
3068
3069 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3070 MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
3071 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
3072 MLX5_IB_OBJECT_DEVX_OBJ,
3073 UVERBS_ACCESS_DESTROY,
3074 UA_MANDATORY));
3075
3076 DECLARE_UVERBS_NAMED_METHOD(
3077 MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
3078 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
3079 UVERBS_IDR_ANY_OBJECT,
3080 UVERBS_ACCESS_READ,
3081 UA_MANDATORY),
3082 UVERBS_ATTR_PTR_IN(
3083 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
3084 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3085 UA_MANDATORY,
3086 UA_ALLOC_AND_COPY),
3087 UVERBS_ATTR_PTR_OUT(
3088 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
3089 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3090 UA_MANDATORY));
3091
3092 DECLARE_UVERBS_NAMED_METHOD(
3093 MLX5_IB_METHOD_DEVX_OBJ_QUERY,
3094 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3095 UVERBS_IDR_ANY_OBJECT,
3096 UVERBS_ACCESS_READ,
3097 UA_MANDATORY),
3098 UVERBS_ATTR_PTR_IN(
3099 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3100 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3101 UA_MANDATORY,
3102 UA_ALLOC_AND_COPY),
3103 UVERBS_ATTR_PTR_OUT(
3104 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
3105 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3106 UA_MANDATORY));
3107
3108 DECLARE_UVERBS_NAMED_METHOD(
3109 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
3110 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3111 UVERBS_IDR_ANY_OBJECT,
3112 UVERBS_ACCESS_READ,
3113 UA_MANDATORY),
3114 UVERBS_ATTR_PTR_IN(
3115 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3116 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3117 UA_MANDATORY,
3118 UA_ALLOC_AND_COPY),
3119 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
3120 u16, UA_MANDATORY),
3121 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
3122 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3123 UVERBS_ACCESS_READ,
3124 UA_MANDATORY),
3125 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
3126 UVERBS_ATTR_TYPE(u64),
3127 UA_MANDATORY));
3128
3129 DECLARE_UVERBS_NAMED_METHOD(
3130 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
3131 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
3132 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3133 UVERBS_ACCESS_READ,
3134 UA_MANDATORY),
3135 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
3136 MLX5_IB_OBJECT_DEVX_OBJ,
3137 UVERBS_ACCESS_READ,
3138 UA_OPTIONAL),
3139 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
3140 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
3141 UA_MANDATORY,
3142 UA_ALLOC_AND_COPY),
3143 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
3144 UVERBS_ATTR_TYPE(u64),
3145 UA_OPTIONAL),
3146 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
3147 UVERBS_ATTR_TYPE(u32),
3148 UA_OPTIONAL));
3149
3150 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
3151 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
3152 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
3153 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
3154 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
3155
3156 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
3157 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
3158 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
3159 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
3160 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
3161 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
3162 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
3163
3164 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
3165 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
3166 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
3167 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
3168
3169
3170 DECLARE_UVERBS_NAMED_METHOD(
3171 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
3172 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
3173 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3174 UVERBS_ACCESS_NEW,
3175 UA_MANDATORY));
3176
3177 DECLARE_UVERBS_NAMED_OBJECT(
3178 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3179 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
3180 devx_async_cmd_event_destroy_uobj,
3181 &devx_async_cmd_event_fops, "[devx_async_cmd]",
3182 O_RDONLY),
3183 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
3184
3185 DECLARE_UVERBS_NAMED_METHOD(
3186 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
3187 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
3188 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3189 UVERBS_ACCESS_NEW,
3190 UA_MANDATORY),
3191 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
3192 enum mlx5_ib_uapi_devx_create_event_channel_flags,
3193 UA_MANDATORY));
3194
3195 DECLARE_UVERBS_NAMED_OBJECT(
3196 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3197 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
3198 devx_async_event_destroy_uobj,
3199 &devx_async_event_fops, "[devx_async_event]",
3200 O_RDONLY),
3201 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
3202
devx_is_supported(struct ib_device * device)3203 static bool devx_is_supported(struct ib_device *device)
3204 {
3205 struct mlx5_ib_dev *dev = to_mdev(device);
3206
3207 return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
3208 }
3209
3210 const struct uapi_definition mlx5_ib_devx_defs[] = {
3211 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3212 MLX5_IB_OBJECT_DEVX,
3213 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3214 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3215 MLX5_IB_OBJECT_DEVX_OBJ,
3216 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3217 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3218 MLX5_IB_OBJECT_DEVX_UMEM,
3219 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3220 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3221 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3222 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3223 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3224 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3225 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3226 {},
3227 };
3228