xref: /linux/drivers/infiniband/hw/mlx5/main.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4  * Copyright (c) 2020, Intel Corporation. All rights reserved.
5  */
6 
7 #include <linux/debugfs.h>
8 #include <linux/highmem.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/errno.h>
12 #include <linux/pci.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/slab.h>
15 #include <linux/bitmap.h>
16 #include <linux/sched.h>
17 #include <linux/sched/mm.h>
18 #include <linux/sched/task.h>
19 #include <linux/delay.h>
20 #include <rdma/ib_user_verbs.h>
21 #include <rdma/ib_addr.h>
22 #include <rdma/ib_cache.h>
23 #include <linux/mlx5/port.h>
24 #include <linux/mlx5/vport.h>
25 #include <linux/mlx5/fs.h>
26 #include <linux/mlx5/eswitch.h>
27 #include <linux/mlx5/driver.h>
28 #include <linux/list.h>
29 #include <rdma/ib_smi.h>
30 #include <rdma/ib_umem_odp.h>
31 #include <rdma/lag.h>
32 #include <linux/in.h>
33 #include <linux/etherdevice.h>
34 #include "mlx5_ib.h"
35 #include "ib_rep.h"
36 #include "cmd.h"
37 #include "devx.h"
38 #include "dm.h"
39 #include "fs.h"
40 #include "srq.h"
41 #include "qp.h"
42 #include "wr.h"
43 #include "restrack.h"
44 #include "counters.h"
45 #include "umr.h"
46 #include <rdma/uverbs_std_types.h>
47 #include <rdma/uverbs_ioctl.h>
48 #include <rdma/mlx5_user_ioctl_verbs.h>
49 #include <rdma/mlx5_user_ioctl_cmds.h>
50 #include "macsec.h"
51 #include "data_direct.h"
52 
53 #define UVERBS_MODULE_NAME mlx5_ib
54 #include <rdma/uverbs_named_ioctl.h>
55 
56 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
57 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
58 MODULE_LICENSE("Dual BSD/GPL");
59 
60 struct mlx5_ib_event_work {
61 	struct work_struct	work;
62 	union {
63 		struct mlx5_ib_dev	      *dev;
64 		struct mlx5_ib_multiport_info *mpi;
65 	};
66 	bool			is_slave;
67 	unsigned int		event;
68 	void			*param;
69 };
70 
71 enum {
72 	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
73 };
74 
75 static struct workqueue_struct *mlx5_ib_event_wq;
76 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
77 static LIST_HEAD(mlx5_ib_dev_list);
78 /*
79  * This mutex should be held when accessing either of the above lists
80  */
81 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
82 
mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info * mpi)83 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
84 {
85 	struct mlx5_ib_dev *dev;
86 
87 	mutex_lock(&mlx5_ib_multiport_mutex);
88 	dev = mpi->ibdev;
89 	mutex_unlock(&mlx5_ib_multiport_mutex);
90 	return dev;
91 }
92 
93 static enum rdma_link_layer
mlx5_port_type_cap_to_rdma_ll(int port_type_cap)94 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
95 {
96 	switch (port_type_cap) {
97 	case MLX5_CAP_PORT_TYPE_IB:
98 		return IB_LINK_LAYER_INFINIBAND;
99 	case MLX5_CAP_PORT_TYPE_ETH:
100 		return IB_LINK_LAYER_ETHERNET;
101 	default:
102 		return IB_LINK_LAYER_UNSPECIFIED;
103 	}
104 }
105 
106 static enum rdma_link_layer
mlx5_ib_port_link_layer(struct ib_device * device,u32 port_num)107 mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
108 {
109 	struct mlx5_ib_dev *dev = to_mdev(device);
110 	int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
111 
112 	return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
113 }
114 
get_port_state(struct ib_device * ibdev,u32 port_num,enum ib_port_state * state)115 static int get_port_state(struct ib_device *ibdev,
116 			  u32 port_num,
117 			  enum ib_port_state *state)
118 {
119 	struct ib_port_attr attr;
120 	int ret;
121 
122 	memset(&attr, 0, sizeof(attr));
123 	ret = ibdev->ops.query_port(ibdev, port_num, &attr);
124 	if (!ret)
125 		*state = attr.state;
126 	return ret;
127 }
128 
mlx5_get_rep_roce(struct mlx5_ib_dev * dev,struct net_device * ndev,struct net_device * upper,u32 * port_num)129 static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
130 					   struct net_device *ndev,
131 					   struct net_device *upper,
132 					   u32 *port_num)
133 {
134 	struct net_device *rep_ndev;
135 	struct mlx5_ib_port *port;
136 	int i;
137 
138 	for (i = 0; i < dev->num_ports; i++) {
139 		port  = &dev->port[i];
140 		if (!port->rep)
141 			continue;
142 
143 		if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
144 			*port_num = i + 1;
145 			return &port->roce;
146 		}
147 
148 		if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
149 			continue;
150 		rep_ndev = ib_device_get_netdev(&dev->ib_dev, i + 1);
151 		if (rep_ndev && rep_ndev == ndev) {
152 			dev_put(rep_ndev);
153 			*port_num = i + 1;
154 			return &port->roce;
155 		}
156 
157 		dev_put(rep_ndev);
158 	}
159 
160 	return NULL;
161 }
162 
mlx5_netdev_send_event(struct mlx5_ib_dev * dev,struct net_device * ndev,struct net_device * upper,struct net_device * ib_ndev)163 static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev,
164 				   struct net_device *ndev,
165 				   struct net_device *upper,
166 				   struct net_device *ib_ndev)
167 {
168 	if (!dev->ib_active)
169 		return false;
170 
171 	/* Event is about our upper device */
172 	if (upper == ndev)
173 		return true;
174 
175 	/* RDMA device is not in lag and not in switchdev */
176 	if (!dev->is_rep && !upper && ndev == ib_ndev)
177 		return true;
178 
179 	/* RDMA devie is in switchdev */
180 	if (dev->is_rep && ndev == ib_ndev)
181 		return true;
182 
183 	return false;
184 }
185 
mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev * ibdev)186 static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev)
187 {
188 	struct mlx5_ib_port *port;
189 	int i;
190 
191 	for (i = 0; i < ibdev->num_ports; i++) {
192 		port = &ibdev->port[i];
193 		if (port->rep && port->rep->vport == MLX5_VPORT_UPLINK) {
194 			return ib_device_get_netdev(&ibdev->ib_dev, i + 1);
195 		}
196 	}
197 
198 	return NULL;
199 }
200 
mlx5_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)201 static int mlx5_netdev_event(struct notifier_block *this,
202 			     unsigned long event, void *ptr)
203 {
204 	struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
205 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
206 	u32 port_num = roce->native_port_num;
207 	struct net_device *ib_ndev = NULL;
208 	struct mlx5_core_dev *mdev;
209 	struct mlx5_ib_dev *ibdev;
210 
211 	ibdev = roce->dev;
212 	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
213 	if (!mdev)
214 		return NOTIFY_DONE;
215 
216 	switch (event) {
217 	case NETDEV_REGISTER:
218 		/* Should already be registered during the load */
219 		if (ibdev->is_rep)
220 			break;
221 
222 		ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
223 		/* Exit if already registered */
224 		if (ib_ndev)
225 			goto put_ndev;
226 
227 		if (ndev->dev.parent == mdev->device)
228 			ib_device_set_netdev(&ibdev->ib_dev, ndev, port_num);
229 		break;
230 
231 	case NETDEV_UNREGISTER:
232 		/* In case of reps, ib device goes away before the netdevs */
233 		if (ibdev->is_rep)
234 			break;
235 		ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
236 		if (ib_ndev == ndev)
237 			ib_device_set_netdev(&ibdev->ib_dev, NULL, port_num);
238 		goto put_ndev;
239 
240 	case NETDEV_CHANGE:
241 	case NETDEV_UP:
242 	case NETDEV_DOWN: {
243 		struct net_device *upper = NULL;
244 
245 		if (!netif_is_lag_master(ndev) && !netif_is_lag_port(ndev) &&
246 		    !mlx5_core_mp_enabled(mdev))
247 			return NOTIFY_DONE;
248 
249 		if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
250 			struct net_device *lag_ndev;
251 
252 			if(mlx5_lag_is_roce(mdev))
253 				lag_ndev = ib_device_get_netdev(&ibdev->ib_dev, 1);
254 			else /* sriov lag */
255 				lag_ndev = mlx5_ib_get_rep_uplink_netdev(ibdev);
256 
257 			if (lag_ndev) {
258 				upper = netdev_master_upper_dev_get(lag_ndev);
259 				dev_put(lag_ndev);
260 			} else {
261 				goto done;
262 			}
263 		}
264 
265 		if (ibdev->is_rep)
266 			roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
267 		if (!roce)
268 			return NOTIFY_DONE;
269 
270 		ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
271 
272 		if (mlx5_netdev_send_event(ibdev, ndev, upper, ib_ndev)) {
273 			struct ib_event ibev = { };
274 			enum ib_port_state port_state;
275 
276 			if (get_port_state(&ibdev->ib_dev, port_num,
277 					   &port_state))
278 				goto put_ndev;
279 
280 			if (roce->last_port_state == port_state)
281 				goto put_ndev;
282 
283 			roce->last_port_state = port_state;
284 			ibev.device = &ibdev->ib_dev;
285 			if (port_state == IB_PORT_DOWN)
286 				ibev.event = IB_EVENT_PORT_ERR;
287 			else if (port_state == IB_PORT_ACTIVE)
288 				ibev.event = IB_EVENT_PORT_ACTIVE;
289 			else
290 				goto put_ndev;
291 
292 			ibev.element.port_num = port_num;
293 			ib_dispatch_event(&ibev);
294 		}
295 		break;
296 	}
297 
298 	default:
299 		break;
300 	}
301 put_ndev:
302 	dev_put(ib_ndev);
303 done:
304 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
305 	return NOTIFY_DONE;
306 }
307 
mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev * ibdev,u32 ib_port_num,u32 * native_port_num)308 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
309 						   u32 ib_port_num,
310 						   u32 *native_port_num)
311 {
312 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
313 							  ib_port_num);
314 	struct mlx5_core_dev *mdev = NULL;
315 	struct mlx5_ib_multiport_info *mpi;
316 	struct mlx5_ib_port *port;
317 
318 	if (ibdev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
319 		if (native_port_num)
320 			*native_port_num = smi_to_native_portnum(ibdev,
321 								 ib_port_num);
322 		return ibdev->mdev;
323 
324 	}
325 
326 	if (!mlx5_core_mp_enabled(ibdev->mdev) ||
327 	    ll != IB_LINK_LAYER_ETHERNET) {
328 		if (native_port_num)
329 			*native_port_num = ib_port_num;
330 		return ibdev->mdev;
331 	}
332 
333 	if (native_port_num)
334 		*native_port_num = 1;
335 
336 	port = &ibdev->port[ib_port_num - 1];
337 	spin_lock(&port->mp.mpi_lock);
338 	mpi = ibdev->port[ib_port_num - 1].mp.mpi;
339 	if (mpi && !mpi->unaffiliate) {
340 		mdev = mpi->mdev;
341 		/* If it's the master no need to refcount, it'll exist
342 		 * as long as the ib_dev exists.
343 		 */
344 		if (!mpi->is_master)
345 			mpi->mdev_refcnt++;
346 	}
347 	spin_unlock(&port->mp.mpi_lock);
348 
349 	return mdev;
350 }
351 
mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev * ibdev,u32 port_num)352 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
353 {
354 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
355 							  port_num);
356 	struct mlx5_ib_multiport_info *mpi;
357 	struct mlx5_ib_port *port;
358 
359 	if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
360 		return;
361 
362 	port = &ibdev->port[port_num - 1];
363 
364 	spin_lock(&port->mp.mpi_lock);
365 	mpi = ibdev->port[port_num - 1].mp.mpi;
366 	if (mpi->is_master)
367 		goto out;
368 
369 	mpi->mdev_refcnt--;
370 	if (mpi->unaffiliate)
371 		complete(&mpi->unref_comp);
372 out:
373 	spin_unlock(&port->mp.mpi_lock);
374 }
375 
translate_eth_legacy_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width)376 static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
377 					   u16 *active_speed, u8 *active_width)
378 {
379 	switch (eth_proto_oper) {
380 	case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
381 	case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
382 	case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
383 	case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
384 		*active_width = IB_WIDTH_1X;
385 		*active_speed = IB_SPEED_SDR;
386 		break;
387 	case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
388 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
389 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
390 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
391 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
392 	case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
393 	case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
394 		*active_width = IB_WIDTH_1X;
395 		*active_speed = IB_SPEED_QDR;
396 		break;
397 	case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
398 	case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
399 	case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
400 		*active_width = IB_WIDTH_1X;
401 		*active_speed = IB_SPEED_EDR;
402 		break;
403 	case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
404 	case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
405 	case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
406 	case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
407 		*active_width = IB_WIDTH_4X;
408 		*active_speed = IB_SPEED_QDR;
409 		break;
410 	case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
411 	case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
412 	case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
413 		*active_width = IB_WIDTH_1X;
414 		*active_speed = IB_SPEED_HDR;
415 		break;
416 	case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
417 		*active_width = IB_WIDTH_4X;
418 		*active_speed = IB_SPEED_FDR;
419 		break;
420 	case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
421 	case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
422 	case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
423 	case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
424 		*active_width = IB_WIDTH_4X;
425 		*active_speed = IB_SPEED_EDR;
426 		break;
427 	default:
428 		return -EINVAL;
429 	}
430 
431 	return 0;
432 }
433 
translate_eth_ext_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width)434 static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
435 					u8 *active_width)
436 {
437 	switch (eth_proto_oper) {
438 	case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
439 	case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
440 		*active_width = IB_WIDTH_1X;
441 		*active_speed = IB_SPEED_SDR;
442 		break;
443 	case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
444 		*active_width = IB_WIDTH_1X;
445 		*active_speed = IB_SPEED_DDR;
446 		break;
447 	case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
448 		*active_width = IB_WIDTH_1X;
449 		*active_speed = IB_SPEED_QDR;
450 		break;
451 	case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
452 		*active_width = IB_WIDTH_4X;
453 		*active_speed = IB_SPEED_QDR;
454 		break;
455 	case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
456 		*active_width = IB_WIDTH_1X;
457 		*active_speed = IB_SPEED_EDR;
458 		break;
459 	case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
460 		*active_width = IB_WIDTH_2X;
461 		*active_speed = IB_SPEED_EDR;
462 		break;
463 	case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
464 		*active_width = IB_WIDTH_1X;
465 		*active_speed = IB_SPEED_HDR;
466 		break;
467 	case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
468 		*active_width = IB_WIDTH_4X;
469 		*active_speed = IB_SPEED_EDR;
470 		break;
471 	case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
472 		*active_width = IB_WIDTH_2X;
473 		*active_speed = IB_SPEED_HDR;
474 		break;
475 	case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
476 		*active_width = IB_WIDTH_1X;
477 		*active_speed = IB_SPEED_NDR;
478 		break;
479 	case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
480 		*active_width = IB_WIDTH_4X;
481 		*active_speed = IB_SPEED_HDR;
482 		break;
483 	case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
484 		*active_width = IB_WIDTH_2X;
485 		*active_speed = IB_SPEED_NDR;
486 		break;
487 	case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8):
488 		*active_width = IB_WIDTH_8X;
489 		*active_speed = IB_SPEED_HDR;
490 		break;
491 	case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
492 		*active_width = IB_WIDTH_4X;
493 		*active_speed = IB_SPEED_NDR;
494 		break;
495 	case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8):
496 		*active_width = IB_WIDTH_8X;
497 		*active_speed = IB_SPEED_NDR;
498 		break;
499 	default:
500 		return -EINVAL;
501 	}
502 
503 	return 0;
504 }
505 
translate_eth_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width,bool ext)506 static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
507 				    u8 *active_width, bool ext)
508 {
509 	return ext ?
510 		translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
511 					     active_width) :
512 		translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
513 						active_width);
514 }
515 
mlx5_query_port_roce(struct ib_device * device,u32 port_num,struct ib_port_attr * props)516 static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
517 				struct ib_port_attr *props)
518 {
519 	struct mlx5_ib_dev *dev = to_mdev(device);
520 	u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
521 	struct mlx5_core_dev *mdev;
522 	struct net_device *ndev, *upper;
523 	enum ib_mtu ndev_ib_mtu;
524 	bool put_mdev = true;
525 	u32 eth_prot_oper;
526 	u32 mdev_port_num;
527 	bool ext;
528 	int err;
529 
530 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
531 	if (!mdev) {
532 		/* This means the port isn't affiliated yet. Get the
533 		 * info for the master port instead.
534 		 */
535 		put_mdev = false;
536 		mdev = dev->mdev;
537 		mdev_port_num = 1;
538 		port_num = 1;
539 	}
540 
541 	/* Possible bad flows are checked before filling out props so in case
542 	 * of an error it will still be zeroed out.
543 	 * Use native port in case of reps
544 	 */
545 	if (dev->is_rep)
546 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
547 					   1, 0);
548 	else
549 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
550 					   mdev_port_num, 0);
551 	if (err)
552 		goto out;
553 	ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
554 	eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
555 
556 	props->active_width     = IB_WIDTH_4X;
557 	props->active_speed     = IB_SPEED_QDR;
558 
559 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
560 				 &props->active_width, ext);
561 
562 	if (!dev->is_rep && dev->mdev->roce.roce_en) {
563 		u16 qkey_viol_cntr;
564 
565 		props->port_cap_flags |= IB_PORT_CM_SUP;
566 		props->ip_gids = true;
567 		props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
568 						   roce_address_table_size);
569 		mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
570 		props->qkey_viol_cntr = qkey_viol_cntr;
571 	}
572 	props->max_mtu          = IB_MTU_4096;
573 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
574 	props->pkey_tbl_len     = 1;
575 	props->state            = IB_PORT_DOWN;
576 	props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
577 
578 	/* If this is a stub query for an unaffiliated port stop here */
579 	if (!put_mdev)
580 		goto out;
581 
582 	ndev = ib_device_get_netdev(device, port_num);
583 	if (!ndev)
584 		goto out;
585 
586 	if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
587 		rcu_read_lock();
588 		upper = netdev_master_upper_dev_get_rcu(ndev);
589 		if (upper) {
590 			dev_put(ndev);
591 			ndev = upper;
592 			dev_hold(ndev);
593 		}
594 		rcu_read_unlock();
595 	}
596 
597 	if (netif_running(ndev) && netif_carrier_ok(ndev)) {
598 		props->state      = IB_PORT_ACTIVE;
599 		props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
600 	}
601 
602 	ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
603 
604 	dev_put(ndev);
605 
606 	props->active_mtu	= min(props->max_mtu, ndev_ib_mtu);
607 out:
608 	if (put_mdev)
609 		mlx5_ib_put_native_port_mdev(dev, port_num);
610 	return err;
611 }
612 
set_roce_addr(struct mlx5_ib_dev * dev,u32 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr)613 int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
614 		  unsigned int index, const union ib_gid *gid,
615 		  const struct ib_gid_attr *attr)
616 {
617 	enum ib_gid_type gid_type;
618 	u16 vlan_id = 0xffff;
619 	u8 roce_version = 0;
620 	u8 roce_l3_type = 0;
621 	u8 mac[ETH_ALEN];
622 	int ret;
623 
624 	gid_type = attr->gid_type;
625 	if (gid) {
626 		ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
627 		if (ret)
628 			return ret;
629 	}
630 
631 	switch (gid_type) {
632 	case IB_GID_TYPE_ROCE:
633 		roce_version = MLX5_ROCE_VERSION_1;
634 		break;
635 	case IB_GID_TYPE_ROCE_UDP_ENCAP:
636 		roce_version = MLX5_ROCE_VERSION_2;
637 		if (gid && ipv6_addr_v4mapped((void *)gid))
638 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
639 		else
640 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
641 		break;
642 
643 	default:
644 		mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
645 	}
646 
647 	return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
648 				      roce_l3_type, gid->raw, mac,
649 				      vlan_id < VLAN_CFI_MASK, vlan_id,
650 				      port_num);
651 }
652 
mlx5_ib_add_gid(const struct ib_gid_attr * attr,__always_unused void ** context)653 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
654 			   __always_unused void **context)
655 {
656 	int ret;
657 
658 	ret = mlx5r_add_gid_macsec_operations(attr);
659 	if (ret)
660 		return ret;
661 
662 	return set_roce_addr(to_mdev(attr->device), attr->port_num,
663 			     attr->index, &attr->gid, attr);
664 }
665 
mlx5_ib_del_gid(const struct ib_gid_attr * attr,__always_unused void ** context)666 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
667 			   __always_unused void **context)
668 {
669 	int ret;
670 
671 	ret = set_roce_addr(to_mdev(attr->device), attr->port_num,
672 			    attr->index, NULL, attr);
673 	if (ret)
674 		return ret;
675 
676 	mlx5r_del_gid_macsec_operations(attr);
677 	return 0;
678 }
679 
mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev * dev,const struct ib_gid_attr * attr)680 __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
681 				   const struct ib_gid_attr *attr)
682 {
683 	if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
684 		return 0;
685 
686 	return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
687 }
688 
mlx5_use_mad_ifc(struct mlx5_ib_dev * dev)689 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
690 {
691 	if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
692 		return !MLX5_CAP_GEN(dev->mdev, ib_virt);
693 	return 0;
694 }
695 
696 enum {
697 	MLX5_VPORT_ACCESS_METHOD_MAD,
698 	MLX5_VPORT_ACCESS_METHOD_HCA,
699 	MLX5_VPORT_ACCESS_METHOD_NIC,
700 };
701 
mlx5_get_vport_access_method(struct ib_device * ibdev)702 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
703 {
704 	if (mlx5_use_mad_ifc(to_mdev(ibdev)))
705 		return MLX5_VPORT_ACCESS_METHOD_MAD;
706 
707 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
708 	    IB_LINK_LAYER_ETHERNET)
709 		return MLX5_VPORT_ACCESS_METHOD_NIC;
710 
711 	return MLX5_VPORT_ACCESS_METHOD_HCA;
712 }
713 
get_atomic_caps(struct mlx5_ib_dev * dev,u8 atomic_size_qp,struct ib_device_attr * props)714 static void get_atomic_caps(struct mlx5_ib_dev *dev,
715 			    u8 atomic_size_qp,
716 			    struct ib_device_attr *props)
717 {
718 	u8 tmp;
719 	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
720 	u8 atomic_req_8B_endianness_mode =
721 		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
722 
723 	/* Check if HW supports 8 bytes standard atomic operations and capable
724 	 * of host endianness respond
725 	 */
726 	tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
727 	if (((atomic_operations & tmp) == tmp) &&
728 	    (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
729 	    (atomic_req_8B_endianness_mode)) {
730 		props->atomic_cap = IB_ATOMIC_HCA;
731 	} else {
732 		props->atomic_cap = IB_ATOMIC_NONE;
733 	}
734 }
735 
get_atomic_caps_qp(struct mlx5_ib_dev * dev,struct ib_device_attr * props)736 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
737 			       struct ib_device_attr *props)
738 {
739 	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
740 
741 	get_atomic_caps(dev, atomic_size_qp, props);
742 }
743 
mlx5_query_system_image_guid(struct ib_device * ibdev,__be64 * sys_image_guid)744 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
745 					__be64 *sys_image_guid)
746 {
747 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
748 	struct mlx5_core_dev *mdev = dev->mdev;
749 	u64 tmp;
750 	int err;
751 
752 	switch (mlx5_get_vport_access_method(ibdev)) {
753 	case MLX5_VPORT_ACCESS_METHOD_MAD:
754 		return mlx5_query_mad_ifc_system_image_guid(ibdev,
755 							    sys_image_guid);
756 
757 	case MLX5_VPORT_ACCESS_METHOD_HCA:
758 		err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
759 		break;
760 
761 	case MLX5_VPORT_ACCESS_METHOD_NIC:
762 		err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
763 		break;
764 
765 	default:
766 		return -EINVAL;
767 	}
768 
769 	if (!err)
770 		*sys_image_guid = cpu_to_be64(tmp);
771 
772 	return err;
773 
774 }
775 
mlx5_query_max_pkeys(struct ib_device * ibdev,u16 * max_pkeys)776 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
777 				u16 *max_pkeys)
778 {
779 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
780 	struct mlx5_core_dev *mdev = dev->mdev;
781 
782 	switch (mlx5_get_vport_access_method(ibdev)) {
783 	case MLX5_VPORT_ACCESS_METHOD_MAD:
784 		return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
785 
786 	case MLX5_VPORT_ACCESS_METHOD_HCA:
787 	case MLX5_VPORT_ACCESS_METHOD_NIC:
788 		*max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
789 						pkey_table_size));
790 		return 0;
791 
792 	default:
793 		return -EINVAL;
794 	}
795 }
796 
mlx5_query_vendor_id(struct ib_device * ibdev,u32 * vendor_id)797 static int mlx5_query_vendor_id(struct ib_device *ibdev,
798 				u32 *vendor_id)
799 {
800 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
801 
802 	switch (mlx5_get_vport_access_method(ibdev)) {
803 	case MLX5_VPORT_ACCESS_METHOD_MAD:
804 		return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
805 
806 	case MLX5_VPORT_ACCESS_METHOD_HCA:
807 	case MLX5_VPORT_ACCESS_METHOD_NIC:
808 		return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
809 
810 	default:
811 		return -EINVAL;
812 	}
813 }
814 
mlx5_query_node_guid(struct mlx5_ib_dev * dev,__be64 * node_guid)815 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
816 				__be64 *node_guid)
817 {
818 	u64 tmp;
819 	int err;
820 
821 	switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
822 	case MLX5_VPORT_ACCESS_METHOD_MAD:
823 		return mlx5_query_mad_ifc_node_guid(dev, node_guid);
824 
825 	case MLX5_VPORT_ACCESS_METHOD_HCA:
826 		err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
827 		break;
828 
829 	case MLX5_VPORT_ACCESS_METHOD_NIC:
830 		err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
831 		break;
832 
833 	default:
834 		return -EINVAL;
835 	}
836 
837 	if (!err)
838 		*node_guid = cpu_to_be64(tmp);
839 
840 	return err;
841 }
842 
843 struct mlx5_reg_node_desc {
844 	u8	desc[IB_DEVICE_NODE_DESC_MAX];
845 };
846 
mlx5_query_node_desc(struct mlx5_ib_dev * dev,char * node_desc)847 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
848 {
849 	struct mlx5_reg_node_desc in;
850 
851 	if (mlx5_use_mad_ifc(dev))
852 		return mlx5_query_mad_ifc_node_desc(dev, node_desc);
853 
854 	memset(&in, 0, sizeof(in));
855 
856 	return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
857 				    sizeof(struct mlx5_reg_node_desc),
858 				    MLX5_REG_NODE_DESC, 0, 0);
859 }
860 
fill_esw_mgr_reg_c0(struct mlx5_core_dev * mdev,struct mlx5_ib_query_device_resp * resp)861 static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev,
862 				struct mlx5_ib_query_device_resp *resp)
863 {
864 	struct mlx5_eswitch *esw = mdev->priv.eswitch;
865 	u16 vport = mlx5_eswitch_manager_vport(mdev);
866 
867 	resp->reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match(esw,
868 								      vport);
869 	resp->reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask();
870 }
871 
mlx5_ib_query_device(struct ib_device * ibdev,struct ib_device_attr * props,struct ib_udata * uhw)872 static int mlx5_ib_query_device(struct ib_device *ibdev,
873 				struct ib_device_attr *props,
874 				struct ib_udata *uhw)
875 {
876 	size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
877 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
878 	struct mlx5_core_dev *mdev = dev->mdev;
879 	int err = -ENOMEM;
880 	int max_sq_desc;
881 	int max_rq_sg;
882 	int max_sq_sg;
883 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
884 	bool raw_support = !mlx5_core_mp_enabled(mdev);
885 	struct mlx5_ib_query_device_resp resp = {};
886 	size_t resp_len;
887 	u64 max_tso;
888 
889 	resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
890 	if (uhw_outlen && uhw_outlen < resp_len)
891 		return -EINVAL;
892 
893 	resp.response_length = resp_len;
894 
895 	if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
896 		return -EINVAL;
897 
898 	memset(props, 0, sizeof(*props));
899 	err = mlx5_query_system_image_guid(ibdev,
900 					   &props->sys_image_guid);
901 	if (err)
902 		return err;
903 
904 	props->max_pkeys = dev->pkey_table_len;
905 
906 	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
907 	if (err)
908 		return err;
909 
910 	props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
911 		(fw_rev_min(dev->mdev) << 16) |
912 		fw_rev_sub(dev->mdev);
913 	props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
914 		IB_DEVICE_PORT_ACTIVE_EVENT		|
915 		IB_DEVICE_SYS_IMAGE_GUID		|
916 		IB_DEVICE_RC_RNR_NAK_GEN;
917 
918 	if (MLX5_CAP_GEN(mdev, pkv))
919 		props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
920 	if (MLX5_CAP_GEN(mdev, qkv))
921 		props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
922 	if (MLX5_CAP_GEN(mdev, apm))
923 		props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
924 	if (MLX5_CAP_GEN(mdev, xrc))
925 		props->device_cap_flags |= IB_DEVICE_XRC;
926 	if (MLX5_CAP_GEN(mdev, imaicl)) {
927 		props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
928 					   IB_DEVICE_MEM_WINDOW_TYPE_2B;
929 		props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
930 		/* We support 'Gappy' memory registration too */
931 		props->kernel_cap_flags |= IBK_SG_GAPS_REG;
932 	}
933 	/* IB_WR_REG_MR always requires changing the entity size with UMR */
934 	if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
935 		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
936 	if (MLX5_CAP_GEN(mdev, sho)) {
937 		props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
938 		/* At this stage no support for signature handover */
939 		props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
940 				      IB_PROT_T10DIF_TYPE_2 |
941 				      IB_PROT_T10DIF_TYPE_3;
942 		props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
943 				       IB_GUARD_T10DIF_CSUM;
944 	}
945 	if (MLX5_CAP_GEN(mdev, block_lb_mc))
946 		props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
947 
948 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
949 		if (MLX5_CAP_ETH(mdev, csum_cap)) {
950 			/* Legacy bit to support old userspace libraries */
951 			props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
952 			props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
953 		}
954 
955 		if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
956 			props->raw_packet_caps |=
957 				IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
958 
959 		if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
960 			max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
961 			if (max_tso) {
962 				resp.tso_caps.max_tso = 1 << max_tso;
963 				resp.tso_caps.supported_qpts |=
964 					1 << IB_QPT_RAW_PACKET;
965 				resp.response_length += sizeof(resp.tso_caps);
966 			}
967 		}
968 
969 		if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
970 			resp.rss_caps.rx_hash_function =
971 						MLX5_RX_HASH_FUNC_TOEPLITZ;
972 			resp.rss_caps.rx_hash_fields_mask =
973 						MLX5_RX_HASH_SRC_IPV4 |
974 						MLX5_RX_HASH_DST_IPV4 |
975 						MLX5_RX_HASH_SRC_IPV6 |
976 						MLX5_RX_HASH_DST_IPV6 |
977 						MLX5_RX_HASH_SRC_PORT_TCP |
978 						MLX5_RX_HASH_DST_PORT_TCP |
979 						MLX5_RX_HASH_SRC_PORT_UDP |
980 						MLX5_RX_HASH_DST_PORT_UDP |
981 						MLX5_RX_HASH_INNER;
982 			resp.response_length += sizeof(resp.rss_caps);
983 		}
984 	} else {
985 		if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
986 			resp.response_length += sizeof(resp.tso_caps);
987 		if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
988 			resp.response_length += sizeof(resp.rss_caps);
989 	}
990 
991 	if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
992 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
993 		props->kernel_cap_flags |= IBK_UD_TSO;
994 	}
995 
996 	if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
997 	    MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
998 	    raw_support)
999 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
1000 
1001 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
1002 	    MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
1003 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1004 
1005 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1006 	    MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
1007 	    raw_support) {
1008 		/* Legacy bit to support old userspace libraries */
1009 		props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
1010 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
1011 	}
1012 
1013 	if (MLX5_CAP_DEV_MEM(mdev, memic)) {
1014 		props->max_dm_size =
1015 			MLX5_CAP_DEV_MEM(mdev, max_memic_size);
1016 	}
1017 
1018 	if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
1019 		props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
1020 
1021 	if (MLX5_CAP_GEN(mdev, end_pad))
1022 		props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
1023 
1024 	props->vendor_part_id	   = mdev->pdev->device;
1025 	props->hw_ver		   = mdev->pdev->revision;
1026 
1027 	props->max_mr_size	   = ~0ull;
1028 	props->page_size_cap	   = ~(min_page_size - 1);
1029 	props->max_qp		   = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
1030 	props->max_qp_wr	   = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1031 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
1032 		     sizeof(struct mlx5_wqe_data_seg);
1033 	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
1034 	max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
1035 		     sizeof(struct mlx5_wqe_raddr_seg)) /
1036 		sizeof(struct mlx5_wqe_data_seg);
1037 	props->max_send_sge = max_sq_sg;
1038 	props->max_recv_sge = max_rq_sg;
1039 	props->max_sge_rd	   = MLX5_MAX_SGE_RD;
1040 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
1041 	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
1042 	props->max_mr		   = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
1043 	props->max_pd		   = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
1044 	props->max_qp_rd_atom	   = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
1045 	props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
1046 	props->max_srq		   = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
1047 	props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
1048 	props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
1049 	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
1050 	props->max_srq_sge	   = max_rq_sg - 1;
1051 	props->max_fast_reg_page_list_len =
1052 		1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
1053 	props->max_pi_fast_reg_page_list_len =
1054 		props->max_fast_reg_page_list_len / 2;
1055 	props->max_sgl_rd =
1056 		MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
1057 	get_atomic_caps_qp(dev, props);
1058 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
1059 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
1060 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
1061 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
1062 					   props->max_mcast_grp;
1063 	props->max_ah = INT_MAX;
1064 	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
1065 	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
1066 
1067 	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1068 		if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
1069 			props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
1070 		props->odp_caps = dev->odp_caps;
1071 		if (!uhw) {
1072 			/* ODP for kernel QPs is not implemented for receive
1073 			 * WQEs and SRQ WQEs
1074 			 */
1075 			props->odp_caps.per_transport_caps.rc_odp_caps &=
1076 				~(IB_ODP_SUPPORT_READ |
1077 				  IB_ODP_SUPPORT_SRQ_RECV);
1078 			props->odp_caps.per_transport_caps.uc_odp_caps &=
1079 				~(IB_ODP_SUPPORT_READ |
1080 				  IB_ODP_SUPPORT_SRQ_RECV);
1081 			props->odp_caps.per_transport_caps.ud_odp_caps &=
1082 				~(IB_ODP_SUPPORT_READ |
1083 				  IB_ODP_SUPPORT_SRQ_RECV);
1084 			props->odp_caps.per_transport_caps.xrc_odp_caps &=
1085 				~(IB_ODP_SUPPORT_READ |
1086 				  IB_ODP_SUPPORT_SRQ_RECV);
1087 		}
1088 	}
1089 
1090 	if (mlx5_core_is_vf(mdev))
1091 		props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
1092 
1093 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
1094 	    IB_LINK_LAYER_ETHERNET && raw_support) {
1095 		props->rss_caps.max_rwq_indirection_tables =
1096 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1097 		props->rss_caps.max_rwq_indirection_table_size =
1098 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1099 		props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1100 		props->max_wq_type_rq =
1101 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1102 	}
1103 
1104 	if (MLX5_CAP_GEN(mdev, tag_matching)) {
1105 		props->tm_caps.max_num_tags =
1106 			(1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1107 		props->tm_caps.max_ops =
1108 			1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1109 		props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1110 	}
1111 
1112 	if (MLX5_CAP_GEN(mdev, tag_matching) &&
1113 	    MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1114 		props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1115 		props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1116 	}
1117 
1118 	if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1119 		props->cq_caps.max_cq_moderation_count =
1120 						MLX5_MAX_CQ_COUNT;
1121 		props->cq_caps.max_cq_moderation_period =
1122 						MLX5_MAX_CQ_PERIOD;
1123 	}
1124 
1125 	if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
1126 		resp.response_length += sizeof(resp.cqe_comp_caps);
1127 
1128 		if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1129 			resp.cqe_comp_caps.max_num =
1130 				MLX5_CAP_GEN(dev->mdev,
1131 					     cqe_compression_max_num);
1132 
1133 			resp.cqe_comp_caps.supported_format =
1134 				MLX5_IB_CQE_RES_FORMAT_HASH |
1135 				MLX5_IB_CQE_RES_FORMAT_CSUM;
1136 
1137 			if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1138 				resp.cqe_comp_caps.supported_format |=
1139 					MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1140 		}
1141 	}
1142 
1143 	if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
1144 	    raw_support) {
1145 		if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1146 		    MLX5_CAP_GEN(mdev, qos)) {
1147 			resp.packet_pacing_caps.qp_rate_limit_max =
1148 				MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1149 			resp.packet_pacing_caps.qp_rate_limit_min =
1150 				MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1151 			resp.packet_pacing_caps.supported_qpts |=
1152 				1 << IB_QPT_RAW_PACKET;
1153 			if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1154 			    MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1155 				resp.packet_pacing_caps.cap_flags |=
1156 					MLX5_IB_PP_SUPPORT_BURST;
1157 		}
1158 		resp.response_length += sizeof(resp.packet_pacing_caps);
1159 	}
1160 
1161 	if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1162 	    uhw_outlen) {
1163 		if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1164 			resp.mlx5_ib_support_multi_pkt_send_wqes =
1165 				MLX5_IB_ALLOW_MPW;
1166 
1167 		if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1168 			resp.mlx5_ib_support_multi_pkt_send_wqes |=
1169 				MLX5_IB_SUPPORT_EMPW;
1170 
1171 		resp.response_length +=
1172 			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1173 	}
1174 
1175 	if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
1176 		resp.response_length += sizeof(resp.flags);
1177 
1178 		if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1179 			resp.flags |=
1180 				MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1181 
1182 		if (MLX5_CAP_GEN(mdev, cqe_128_always))
1183 			resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1184 		if (MLX5_CAP_GEN(mdev, qp_packet_based))
1185 			resp.flags |=
1186 				MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1187 
1188 		resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1189 
1190 		if (MLX5_CAP_GEN_2(mdev, dp_ordering_force) &&
1191 		    (MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc) ||
1192 		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc) ||
1193 		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc) ||
1194 		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud) ||
1195 		    MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc)))
1196 			resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_OOO_DP;
1197 	}
1198 
1199 	if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
1200 		resp.response_length += sizeof(resp.sw_parsing_caps);
1201 		if (MLX5_CAP_ETH(mdev, swp)) {
1202 			resp.sw_parsing_caps.sw_parsing_offloads |=
1203 				MLX5_IB_SW_PARSING;
1204 
1205 			if (MLX5_CAP_ETH(mdev, swp_csum))
1206 				resp.sw_parsing_caps.sw_parsing_offloads |=
1207 					MLX5_IB_SW_PARSING_CSUM;
1208 
1209 			if (MLX5_CAP_ETH(mdev, swp_lso))
1210 				resp.sw_parsing_caps.sw_parsing_offloads |=
1211 					MLX5_IB_SW_PARSING_LSO;
1212 
1213 			if (resp.sw_parsing_caps.sw_parsing_offloads)
1214 				resp.sw_parsing_caps.supported_qpts =
1215 					BIT(IB_QPT_RAW_PACKET);
1216 		}
1217 	}
1218 
1219 	if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
1220 	    raw_support) {
1221 		resp.response_length += sizeof(resp.striding_rq_caps);
1222 		if (MLX5_CAP_GEN(mdev, striding_rq)) {
1223 			resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1224 				MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1225 			resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1226 				MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1227 			if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1228 				resp.striding_rq_caps
1229 					.min_single_wqe_log_num_of_strides =
1230 					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1231 			else
1232 				resp.striding_rq_caps
1233 					.min_single_wqe_log_num_of_strides =
1234 					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1235 			resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1236 				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1237 			resp.striding_rq_caps.supported_qpts =
1238 				BIT(IB_QPT_RAW_PACKET);
1239 		}
1240 	}
1241 
1242 	if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
1243 		resp.response_length += sizeof(resp.tunnel_offloads_caps);
1244 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1245 			resp.tunnel_offloads_caps |=
1246 				MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1247 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1248 			resp.tunnel_offloads_caps |=
1249 				MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1250 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1251 			resp.tunnel_offloads_caps |=
1252 				MLX5_IB_TUNNELED_OFFLOADS_GRE;
1253 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
1254 			resp.tunnel_offloads_caps |=
1255 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1256 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
1257 			resp.tunnel_offloads_caps |=
1258 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1259 	}
1260 
1261 	if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1262 		resp.response_length += sizeof(resp.dci_streams_caps);
1263 
1264 		resp.dci_streams_caps.max_log_num_concurent =
1265 			MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1266 
1267 		resp.dci_streams_caps.max_log_num_errored =
1268 			MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1269 	}
1270 
1271 	if (offsetofend(typeof(resp), reserved) <= uhw_outlen)
1272 		resp.response_length += sizeof(resp.reserved);
1273 
1274 	if (offsetofend(typeof(resp), reg_c0) <= uhw_outlen) {
1275 		struct mlx5_eswitch *esw = mdev->priv.eswitch;
1276 
1277 		resp.response_length += sizeof(resp.reg_c0);
1278 
1279 		if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS &&
1280 		    mlx5_eswitch_vport_match_metadata_enabled(esw))
1281 			fill_esw_mgr_reg_c0(mdev, &resp);
1282 	}
1283 
1284 	if (uhw_outlen) {
1285 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1286 
1287 		if (err)
1288 			return err;
1289 	}
1290 
1291 	return 0;
1292 }
1293 
translate_active_width(struct ib_device * ibdev,u16 active_width,u8 * ib_width)1294 static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1295 				   u8 *ib_width)
1296 {
1297 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1298 
1299 	if (active_width & MLX5_PTYS_WIDTH_1X)
1300 		*ib_width = IB_WIDTH_1X;
1301 	else if (active_width & MLX5_PTYS_WIDTH_2X)
1302 		*ib_width = IB_WIDTH_2X;
1303 	else if (active_width & MLX5_PTYS_WIDTH_4X)
1304 		*ib_width = IB_WIDTH_4X;
1305 	else if (active_width & MLX5_PTYS_WIDTH_8X)
1306 		*ib_width = IB_WIDTH_8X;
1307 	else if (active_width & MLX5_PTYS_WIDTH_12X)
1308 		*ib_width = IB_WIDTH_12X;
1309 	else {
1310 		mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1311 			    active_width);
1312 		*ib_width = IB_WIDTH_4X;
1313 	}
1314 
1315 	return;
1316 }
1317 
mlx5_mtu_to_ib_mtu(int mtu)1318 static int mlx5_mtu_to_ib_mtu(int mtu)
1319 {
1320 	switch (mtu) {
1321 	case 256: return 1;
1322 	case 512: return 2;
1323 	case 1024: return 3;
1324 	case 2048: return 4;
1325 	case 4096: return 5;
1326 	default:
1327 		pr_warn("invalid mtu\n");
1328 		return -1;
1329 	}
1330 }
1331 
1332 enum ib_max_vl_num {
1333 	__IB_MAX_VL_0		= 1,
1334 	__IB_MAX_VL_0_1		= 2,
1335 	__IB_MAX_VL_0_3		= 3,
1336 	__IB_MAX_VL_0_7		= 4,
1337 	__IB_MAX_VL_0_14	= 5,
1338 };
1339 
1340 enum mlx5_vl_hw_cap {
1341 	MLX5_VL_HW_0	= 1,
1342 	MLX5_VL_HW_0_1	= 2,
1343 	MLX5_VL_HW_0_2	= 3,
1344 	MLX5_VL_HW_0_3	= 4,
1345 	MLX5_VL_HW_0_4	= 5,
1346 	MLX5_VL_HW_0_5	= 6,
1347 	MLX5_VL_HW_0_6	= 7,
1348 	MLX5_VL_HW_0_7	= 8,
1349 	MLX5_VL_HW_0_14	= 15
1350 };
1351 
translate_max_vl_num(struct ib_device * ibdev,u8 vl_hw_cap,u8 * max_vl_num)1352 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1353 				u8 *max_vl_num)
1354 {
1355 	switch (vl_hw_cap) {
1356 	case MLX5_VL_HW_0:
1357 		*max_vl_num = __IB_MAX_VL_0;
1358 		break;
1359 	case MLX5_VL_HW_0_1:
1360 		*max_vl_num = __IB_MAX_VL_0_1;
1361 		break;
1362 	case MLX5_VL_HW_0_3:
1363 		*max_vl_num = __IB_MAX_VL_0_3;
1364 		break;
1365 	case MLX5_VL_HW_0_7:
1366 		*max_vl_num = __IB_MAX_VL_0_7;
1367 		break;
1368 	case MLX5_VL_HW_0_14:
1369 		*max_vl_num = __IB_MAX_VL_0_14;
1370 		break;
1371 
1372 	default:
1373 		return -EINVAL;
1374 	}
1375 
1376 	return 0;
1377 }
1378 
mlx5_query_hca_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1379 static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1380 			       struct ib_port_attr *props)
1381 {
1382 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1383 	struct mlx5_core_dev *mdev = dev->mdev;
1384 	struct mlx5_hca_vport_context *rep;
1385 	u8 vl_hw_cap, plane_index = 0;
1386 	u16 max_mtu;
1387 	u16 oper_mtu;
1388 	int err;
1389 	u16 ib_link_width_oper;
1390 
1391 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1392 	if (!rep) {
1393 		err = -ENOMEM;
1394 		goto out;
1395 	}
1396 
1397 	/* props being zeroed by the caller, avoid zeroing it here */
1398 
1399 	if (ibdev->type == RDMA_DEVICE_TYPE_SMI) {
1400 		plane_index = port;
1401 		port = smi_to_native_portnum(dev, port);
1402 	}
1403 
1404 	err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1405 	if (err)
1406 		goto out;
1407 
1408 	props->lid		= rep->lid;
1409 	props->lmc		= rep->lmc;
1410 	props->sm_lid		= rep->sm_lid;
1411 	props->sm_sl		= rep->sm_sl;
1412 	props->state		= rep->vport_state;
1413 	props->phys_state	= rep->port_physical_state;
1414 
1415 	props->port_cap_flags = rep->cap_mask1;
1416 	if (dev->num_plane) {
1417 		props->port_cap_flags |= IB_PORT_SM_DISABLED;
1418 		props->port_cap_flags &= ~IB_PORT_SM;
1419 	} else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
1420 		props->port_cap_flags &= ~IB_PORT_CM_SUP;
1421 
1422 	props->gid_tbl_len	= mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1423 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1424 	props->pkey_tbl_len	= mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1425 	props->bad_pkey_cntr	= rep->pkey_violation_counter;
1426 	props->qkey_viol_cntr	= rep->qkey_violation_counter;
1427 	props->subnet_timeout	= rep->subnet_timeout;
1428 	props->init_type_reply	= rep->init_type_reply;
1429 
1430 	if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1431 		props->port_cap_flags2 = rep->cap_mask2;
1432 
1433 	err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
1434 				      &props->active_speed, port, plane_index);
1435 	if (err)
1436 		goto out;
1437 
1438 	translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1439 
1440 	mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1441 
1442 	props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1443 
1444 	mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1445 
1446 	props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1447 
1448 	err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1449 	if (err)
1450 		goto out;
1451 
1452 	err = translate_max_vl_num(ibdev, vl_hw_cap,
1453 				   &props->max_vl_num);
1454 out:
1455 	kfree(rep);
1456 	return err;
1457 }
1458 
mlx5_ib_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1459 int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1460 		       struct ib_port_attr *props)
1461 {
1462 	unsigned int count;
1463 	int ret;
1464 
1465 	switch (mlx5_get_vport_access_method(ibdev)) {
1466 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1467 		ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1468 		break;
1469 
1470 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1471 		ret = mlx5_query_hca_port(ibdev, port, props);
1472 		break;
1473 
1474 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1475 		ret = mlx5_query_port_roce(ibdev, port, props);
1476 		break;
1477 
1478 	default:
1479 		ret = -EINVAL;
1480 	}
1481 
1482 	if (!ret && props) {
1483 		struct mlx5_ib_dev *dev = to_mdev(ibdev);
1484 		struct mlx5_core_dev *mdev;
1485 		bool put_mdev = true;
1486 
1487 		mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1488 		if (!mdev) {
1489 			/* If the port isn't affiliated yet query the master.
1490 			 * The master and slave will have the same values.
1491 			 */
1492 			mdev = dev->mdev;
1493 			port = 1;
1494 			put_mdev = false;
1495 		}
1496 		count = mlx5_core_reserved_gids_count(mdev);
1497 		if (put_mdev)
1498 			mlx5_ib_put_native_port_mdev(dev, port);
1499 		props->gid_tbl_len -= count;
1500 	}
1501 	return ret;
1502 }
1503 
mlx5_ib_rep_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1504 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
1505 				  struct ib_port_attr *props)
1506 {
1507 	return mlx5_query_port_roce(ibdev, port, props);
1508 }
1509 
mlx5_ib_rep_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1510 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1511 				  u16 *pkey)
1512 {
1513 	/* Default special Pkey for representor device port as per the
1514 	 * IB specification 1.3 section 10.9.1.2.
1515 	 */
1516 	*pkey = 0xffff;
1517 	return 0;
1518 }
1519 
mlx5_ib_query_gid(struct ib_device * ibdev,u32 port,int index,union ib_gid * gid)1520 static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1521 			     union ib_gid *gid)
1522 {
1523 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1524 	struct mlx5_core_dev *mdev = dev->mdev;
1525 
1526 	switch (mlx5_get_vport_access_method(ibdev)) {
1527 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1528 		return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1529 
1530 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1531 		return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1532 
1533 	default:
1534 		return -EINVAL;
1535 	}
1536 
1537 }
1538 
mlx5_query_hca_nic_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1539 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
1540 				   u16 index, u16 *pkey)
1541 {
1542 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1543 	struct mlx5_core_dev *mdev;
1544 	bool put_mdev = true;
1545 	u32 mdev_port_num;
1546 	int err;
1547 
1548 	mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1549 	if (!mdev) {
1550 		/* The port isn't affiliated yet, get the PKey from the master
1551 		 * port. For RoCE the PKey tables will be the same.
1552 		 */
1553 		put_mdev = false;
1554 		mdev = dev->mdev;
1555 		mdev_port_num = 1;
1556 	}
1557 
1558 	err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1559 					index, pkey);
1560 	if (put_mdev)
1561 		mlx5_ib_put_native_port_mdev(dev, port);
1562 
1563 	return err;
1564 }
1565 
mlx5_ib_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1566 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1567 			      u16 *pkey)
1568 {
1569 	switch (mlx5_get_vport_access_method(ibdev)) {
1570 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1571 		return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1572 
1573 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1574 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1575 		return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1576 	default:
1577 		return -EINVAL;
1578 	}
1579 }
1580 
mlx5_ib_modify_device(struct ib_device * ibdev,int mask,struct ib_device_modify * props)1581 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1582 				 struct ib_device_modify *props)
1583 {
1584 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1585 	struct mlx5_reg_node_desc in;
1586 	struct mlx5_reg_node_desc out;
1587 	int err;
1588 
1589 	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1590 		return -EOPNOTSUPP;
1591 
1592 	if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1593 		return 0;
1594 
1595 	/*
1596 	 * If possible, pass node desc to FW, so it can generate
1597 	 * a 144 trap.  If cmd fails, just ignore.
1598 	 */
1599 	memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1600 	err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1601 				   sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1602 	if (err)
1603 		return err;
1604 
1605 	memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1606 
1607 	return err;
1608 }
1609 
set_port_caps_atomic(struct mlx5_ib_dev * dev,u32 port_num,u32 mask,u32 value)1610 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
1611 				u32 value)
1612 {
1613 	struct mlx5_hca_vport_context ctx = {};
1614 	struct mlx5_core_dev *mdev;
1615 	u32 mdev_port_num;
1616 	int err;
1617 
1618 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1619 	if (!mdev)
1620 		return -ENODEV;
1621 
1622 	err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1623 	if (err)
1624 		goto out;
1625 
1626 	if (~ctx.cap_mask1_perm & mask) {
1627 		mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1628 			     mask, ctx.cap_mask1_perm);
1629 		err = -EINVAL;
1630 		goto out;
1631 	}
1632 
1633 	ctx.cap_mask1 = value;
1634 	ctx.cap_mask1_perm = mask;
1635 	err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1636 						 0, &ctx);
1637 
1638 out:
1639 	mlx5_ib_put_native_port_mdev(dev, port_num);
1640 
1641 	return err;
1642 }
1643 
mlx5_ib_modify_port(struct ib_device * ibdev,u32 port,int mask,struct ib_port_modify * props)1644 static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
1645 			       struct ib_port_modify *props)
1646 {
1647 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1648 	struct ib_port_attr attr;
1649 	u32 tmp;
1650 	int err;
1651 	u32 change_mask;
1652 	u32 value;
1653 	bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1654 		      IB_LINK_LAYER_INFINIBAND);
1655 
1656 	/* CM layer calls ib_modify_port() regardless of the link layer. For
1657 	 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1658 	 */
1659 	if (!is_ib)
1660 		return 0;
1661 
1662 	if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1663 		change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1664 		value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1665 		return set_port_caps_atomic(dev, port, change_mask, value);
1666 	}
1667 
1668 	mutex_lock(&dev->cap_mask_mutex);
1669 
1670 	err = ib_query_port(ibdev, port, &attr);
1671 	if (err)
1672 		goto out;
1673 
1674 	tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1675 		~props->clr_port_cap_mask;
1676 
1677 	err = mlx5_set_port_caps(dev->mdev, port, tmp);
1678 
1679 out:
1680 	mutex_unlock(&dev->cap_mask_mutex);
1681 	return err;
1682 }
1683 
print_lib_caps(struct mlx5_ib_dev * dev,u64 caps)1684 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1685 {
1686 	mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1687 		    caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1688 }
1689 
calc_dynamic_bfregs(int uars_per_sys_page)1690 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1691 {
1692 	/* Large page with non 4k uar support might limit the dynamic size */
1693 	if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1694 		return MLX5_MIN_DYN_BFREGS;
1695 
1696 	return MLX5_MAX_DYN_BFREGS;
1697 }
1698 
calc_total_bfregs(struct mlx5_ib_dev * dev,bool lib_uar_4k,struct mlx5_ib_alloc_ucontext_req_v2 * req,struct mlx5_bfreg_info * bfregi)1699 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1700 			     struct mlx5_ib_alloc_ucontext_req_v2 *req,
1701 			     struct mlx5_bfreg_info *bfregi)
1702 {
1703 	int uars_per_sys_page;
1704 	int bfregs_per_sys_page;
1705 	int ref_bfregs = req->total_num_bfregs;
1706 
1707 	if (req->total_num_bfregs == 0)
1708 		return -EINVAL;
1709 
1710 	BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1711 	BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1712 
1713 	if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1714 		return -ENOMEM;
1715 
1716 	uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1717 	bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1718 	/* This holds the required static allocation asked by the user */
1719 	req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1720 	if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1721 		return -EINVAL;
1722 
1723 	bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1724 	bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1725 	bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1726 	bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1727 
1728 	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1729 		    MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1730 		    lib_uar_4k ? "yes" : "no", ref_bfregs,
1731 		    req->total_num_bfregs, bfregi->total_num_bfregs,
1732 		    bfregi->num_sys_pages);
1733 
1734 	return 0;
1735 }
1736 
allocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1737 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1738 {
1739 	struct mlx5_bfreg_info *bfregi;
1740 	int err;
1741 	int i;
1742 
1743 	bfregi = &context->bfregi;
1744 	for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1745 		err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1746 					 context->devx_uid);
1747 		if (err)
1748 			goto error;
1749 
1750 		mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1751 	}
1752 
1753 	for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1754 		bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1755 
1756 	return 0;
1757 
1758 error:
1759 	for (--i; i >= 0; i--)
1760 		if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1761 					 context->devx_uid))
1762 			mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1763 
1764 	return err;
1765 }
1766 
deallocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1767 static void deallocate_uars(struct mlx5_ib_dev *dev,
1768 			    struct mlx5_ib_ucontext *context)
1769 {
1770 	struct mlx5_bfreg_info *bfregi;
1771 	int i;
1772 
1773 	bfregi = &context->bfregi;
1774 	for (i = 0; i < bfregi->num_sys_pages; i++)
1775 		if (i < bfregi->num_static_sys_pages ||
1776 		    bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1777 			mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1778 					     context->devx_uid);
1779 }
1780 
mlx5_ib_enable_lb(struct mlx5_ib_dev * dev,bool td,bool qp)1781 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1782 {
1783 	int err = 0;
1784 
1785 	mutex_lock(&dev->lb.mutex);
1786 	if (td)
1787 		dev->lb.user_td++;
1788 	if (qp)
1789 		dev->lb.qps++;
1790 
1791 	if (dev->lb.user_td == 2 ||
1792 	    dev->lb.qps == 1) {
1793 		if (!dev->lb.enabled) {
1794 			err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1795 			dev->lb.enabled = true;
1796 		}
1797 	}
1798 
1799 	mutex_unlock(&dev->lb.mutex);
1800 
1801 	return err;
1802 }
1803 
mlx5_ib_disable_lb(struct mlx5_ib_dev * dev,bool td,bool qp)1804 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1805 {
1806 	mutex_lock(&dev->lb.mutex);
1807 	if (td)
1808 		dev->lb.user_td--;
1809 	if (qp)
1810 		dev->lb.qps--;
1811 
1812 	if (dev->lb.user_td == 1 &&
1813 	    dev->lb.qps == 0) {
1814 		if (dev->lb.enabled) {
1815 			mlx5_nic_vport_update_local_lb(dev->mdev, false);
1816 			dev->lb.enabled = false;
1817 		}
1818 	}
1819 
1820 	mutex_unlock(&dev->lb.mutex);
1821 }
1822 
mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev * dev,u32 * tdn,u16 uid)1823 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1824 					  u16 uid)
1825 {
1826 	int err;
1827 
1828 	if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1829 		return 0;
1830 
1831 	err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1832 	if (err)
1833 		return err;
1834 
1835 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1836 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1837 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1838 		return err;
1839 
1840 	return mlx5_ib_enable_lb(dev, true, false);
1841 }
1842 
mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev * dev,u32 tdn,u16 uid)1843 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1844 					     u16 uid)
1845 {
1846 	if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1847 		return;
1848 
1849 	mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1850 
1851 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1852 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1853 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1854 		return;
1855 
1856 	mlx5_ib_disable_lb(dev, true, false);
1857 }
1858 
set_ucontext_resp(struct ib_ucontext * uctx,struct mlx5_ib_alloc_ucontext_resp * resp)1859 static int set_ucontext_resp(struct ib_ucontext *uctx,
1860 			     struct mlx5_ib_alloc_ucontext_resp *resp)
1861 {
1862 	struct ib_device *ibdev = uctx->device;
1863 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1864 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1865 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
1866 
1867 	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1868 		resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
1869 		resp->comp_mask |=
1870 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1871 	}
1872 
1873 	resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1874 	if (mlx5_wc_support_get(dev->mdev))
1875 		resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1876 						      log_bf_reg_size);
1877 	resp->cache_line_size = cache_line_size();
1878 	resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1879 	resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1880 	resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1881 	resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1882 	resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1883 	resp->cqe_version = context->cqe_version;
1884 	resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1885 				MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1886 	resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1887 					MLX5_CAP_GEN(dev->mdev,
1888 						     num_of_uars_per_page) : 1;
1889 	resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1890 			bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1891 	resp->num_ports = dev->num_ports;
1892 	resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1893 				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1894 
1895 	if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1896 		mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1897 		resp->eth_min_inline++;
1898 	}
1899 
1900 	if (dev->mdev->clock_info)
1901 		resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1902 
1903 	/*
1904 	 * We don't want to expose information from the PCI bar that is located
1905 	 * after 4096 bytes, so if the arch only supports larger pages, let's
1906 	 * pretend we don't support reading the HCA's core clock. This is also
1907 	 * forced by mmap function.
1908 	 */
1909 	if (PAGE_SIZE <= 4096) {
1910 		resp->comp_mask |=
1911 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1912 		resp->hca_core_clock_offset =
1913 			offsetof(struct mlx5_init_seg,
1914 				 internal_timer_h) % PAGE_SIZE;
1915 	}
1916 
1917 	if (MLX5_CAP_GEN(dev->mdev, ece_support))
1918 		resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1919 
1920 	if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1921 	    rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1922 	    rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1923 		resp->comp_mask |=
1924 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1925 
1926 	resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
1927 
1928 	if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1929 		resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1930 
1931 	resp->comp_mask |=
1932 		MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;
1933 
1934 	return 0;
1935 }
1936 
mlx5_ib_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)1937 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1938 				  struct ib_udata *udata)
1939 {
1940 	struct ib_device *ibdev = uctx->device;
1941 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1942 	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1943 	struct mlx5_ib_alloc_ucontext_resp resp = {};
1944 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1945 	struct mlx5_bfreg_info *bfregi;
1946 	int ver;
1947 	int err;
1948 	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1949 				     max_cqe_version);
1950 	bool lib_uar_4k;
1951 	bool lib_uar_dyn;
1952 
1953 	if (!dev->ib_active)
1954 		return -EAGAIN;
1955 
1956 	if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1957 		ver = 0;
1958 	else if (udata->inlen >= min_req_v2)
1959 		ver = 2;
1960 	else
1961 		return -EINVAL;
1962 
1963 	err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1964 	if (err)
1965 		return err;
1966 
1967 	if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
1968 		return -EOPNOTSUPP;
1969 
1970 	if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1971 		return -EOPNOTSUPP;
1972 
1973 	req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1974 				    MLX5_NON_FP_BFREGS_PER_UAR);
1975 	if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1976 		return -EINVAL;
1977 
1978 	if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
1979 		err = mlx5_ib_devx_create(dev, true);
1980 		if (err < 0)
1981 			goto out_ctx;
1982 		context->devx_uid = err;
1983 	}
1984 
1985 	lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1986 	lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
1987 	bfregi = &context->bfregi;
1988 
1989 	if (lib_uar_dyn) {
1990 		bfregi->lib_uar_dyn = lib_uar_dyn;
1991 		goto uar_done;
1992 	}
1993 
1994 	/* updates req->total_num_bfregs */
1995 	err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1996 	if (err)
1997 		goto out_devx;
1998 
1999 	mutex_init(&bfregi->lock);
2000 	bfregi->lib_uar_4k = lib_uar_4k;
2001 	bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
2002 				GFP_KERNEL);
2003 	if (!bfregi->count) {
2004 		err = -ENOMEM;
2005 		goto out_devx;
2006 	}
2007 
2008 	bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
2009 				    sizeof(*bfregi->sys_pages),
2010 				    GFP_KERNEL);
2011 	if (!bfregi->sys_pages) {
2012 		err = -ENOMEM;
2013 		goto out_count;
2014 	}
2015 
2016 	err = allocate_uars(dev, context);
2017 	if (err)
2018 		goto out_sys_pages;
2019 
2020 uar_done:
2021 	err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
2022 					     context->devx_uid);
2023 	if (err)
2024 		goto out_uars;
2025 
2026 	INIT_LIST_HEAD(&context->db_page_list);
2027 	mutex_init(&context->db_page_mutex);
2028 
2029 	context->cqe_version = min_t(__u8,
2030 				 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
2031 				 req.max_cqe_version);
2032 
2033 	err = set_ucontext_resp(uctx, &resp);
2034 	if (err)
2035 		goto out_mdev;
2036 
2037 	resp.response_length = min(udata->outlen, sizeof(resp));
2038 	err = ib_copy_to_udata(udata, &resp, resp.response_length);
2039 	if (err)
2040 		goto out_mdev;
2041 
2042 	bfregi->ver = ver;
2043 	bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
2044 	context->lib_caps = req.lib_caps;
2045 	print_lib_caps(dev, context->lib_caps);
2046 
2047 	if (mlx5_ib_lag_should_assign_affinity(dev)) {
2048 		u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
2049 
2050 		atomic_set(&context->tx_port_affinity,
2051 			   atomic_add_return(
2052 				   1, &dev->port[port].roce.tx_port_affinity));
2053 	}
2054 
2055 	return 0;
2056 
2057 out_mdev:
2058 	mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2059 
2060 out_uars:
2061 	deallocate_uars(dev, context);
2062 
2063 out_sys_pages:
2064 	kfree(bfregi->sys_pages);
2065 
2066 out_count:
2067 	kfree(bfregi->count);
2068 
2069 out_devx:
2070 	if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
2071 		mlx5_ib_devx_destroy(dev, context->devx_uid);
2072 
2073 out_ctx:
2074 	return err;
2075 }
2076 
mlx5_ib_query_ucontext(struct ib_ucontext * ibcontext,struct uverbs_attr_bundle * attrs)2077 static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
2078 				  struct uverbs_attr_bundle *attrs)
2079 {
2080 	struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
2081 	int ret;
2082 
2083 	ret = set_ucontext_resp(ibcontext, &uctx_resp);
2084 	if (ret)
2085 		return ret;
2086 
2087 	uctx_resp.response_length =
2088 		min_t(size_t,
2089 		      uverbs_attr_get_len(attrs,
2090 				MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
2091 		      sizeof(uctx_resp));
2092 
2093 	ret = uverbs_copy_to_struct_or_zero(attrs,
2094 					MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
2095 					&uctx_resp,
2096 					sizeof(uctx_resp));
2097 	return ret;
2098 }
2099 
mlx5_ib_dealloc_ucontext(struct ib_ucontext * ibcontext)2100 static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
2101 {
2102 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2103 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2104 	struct mlx5_bfreg_info *bfregi;
2105 
2106 	bfregi = &context->bfregi;
2107 	mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2108 
2109 	deallocate_uars(dev, context);
2110 	kfree(bfregi->sys_pages);
2111 	kfree(bfregi->count);
2112 
2113 	if (context->devx_uid)
2114 		mlx5_ib_devx_destroy(dev, context->devx_uid);
2115 }
2116 
uar_index2pfn(struct mlx5_ib_dev * dev,int uar_idx)2117 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2118 				 int uar_idx)
2119 {
2120 	int fw_uars_per_page;
2121 
2122 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2123 
2124 	return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2125 }
2126 
uar_index2paddress(struct mlx5_ib_dev * dev,int uar_idx)2127 static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2128 				 int uar_idx)
2129 {
2130 	unsigned int fw_uars_per_page;
2131 
2132 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2133 				MLX5_UARS_IN_PAGE : 1;
2134 
2135 	return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2136 }
2137 
get_command(unsigned long offset)2138 static int get_command(unsigned long offset)
2139 {
2140 	return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2141 }
2142 
get_arg(unsigned long offset)2143 static int get_arg(unsigned long offset)
2144 {
2145 	return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2146 }
2147 
get_index(unsigned long offset)2148 static int get_index(unsigned long offset)
2149 {
2150 	return get_arg(offset);
2151 }
2152 
2153 /* Index resides in an extra byte to enable larger values than 255 */
get_extended_index(unsigned long offset)2154 static int get_extended_index(unsigned long offset)
2155 {
2156 	return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2157 }
2158 
2159 
mlx5_ib_disassociate_ucontext(struct ib_ucontext * ibcontext)2160 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2161 {
2162 }
2163 
mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)2164 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2165 {
2166 	switch (cmd) {
2167 	case MLX5_IB_MMAP_WC_PAGE:
2168 		return "WC";
2169 	case MLX5_IB_MMAP_REGULAR_PAGE:
2170 		return "best effort WC";
2171 	case MLX5_IB_MMAP_NC_PAGE:
2172 		return "NC";
2173 	case MLX5_IB_MMAP_DEVICE_MEM:
2174 		return "Device Memory";
2175 	default:
2176 		return "Unknown";
2177 	}
2178 }
2179 
mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)2180 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2181 					struct vm_area_struct *vma,
2182 					struct mlx5_ib_ucontext *context)
2183 {
2184 	if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2185 	    !(vma->vm_flags & VM_SHARED))
2186 		return -EINVAL;
2187 
2188 	if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2189 		return -EOPNOTSUPP;
2190 
2191 	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2192 		return -EPERM;
2193 	vm_flags_clear(vma, VM_MAYWRITE);
2194 
2195 	if (!dev->mdev->clock_info)
2196 		return -EOPNOTSUPP;
2197 
2198 	return vm_insert_page(vma, vma->vm_start,
2199 			      virt_to_page(dev->mdev->clock_info));
2200 }
2201 
mlx5_ib_mmap_free(struct rdma_user_mmap_entry * entry)2202 static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2203 {
2204 	struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2205 	struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2206 	struct mlx5_var_table *var_table = &dev->var_table;
2207 	struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
2208 
2209 	switch (mentry->mmap_flag) {
2210 	case MLX5_IB_MMAP_TYPE_MEMIC:
2211 	case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2212 		mlx5_ib_dm_mmap_free(dev, mentry);
2213 		break;
2214 	case MLX5_IB_MMAP_TYPE_VAR:
2215 		mutex_lock(&var_table->bitmap_lock);
2216 		clear_bit(mentry->page_idx, var_table->bitmap);
2217 		mutex_unlock(&var_table->bitmap_lock);
2218 		kfree(mentry);
2219 		break;
2220 	case MLX5_IB_MMAP_TYPE_UAR_WC:
2221 	case MLX5_IB_MMAP_TYPE_UAR_NC:
2222 		mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2223 				     context->devx_uid);
2224 		kfree(mentry);
2225 		break;
2226 	default:
2227 		WARN_ON(true);
2228 	}
2229 }
2230 
uar_mmap(struct mlx5_ib_dev * dev,enum mlx5_ib_mmap_cmd cmd,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)2231 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2232 		    struct vm_area_struct *vma,
2233 		    struct mlx5_ib_ucontext *context)
2234 {
2235 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
2236 	int err;
2237 	unsigned long idx;
2238 	phys_addr_t pfn;
2239 	pgprot_t prot;
2240 	u32 bfreg_dyn_idx = 0;
2241 	u32 uar_index;
2242 	int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2243 	int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2244 				bfregi->num_static_sys_pages;
2245 
2246 	if (bfregi->lib_uar_dyn)
2247 		return -EINVAL;
2248 
2249 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2250 		return -EINVAL;
2251 
2252 	if (dyn_uar)
2253 		idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2254 	else
2255 		idx = get_index(vma->vm_pgoff);
2256 
2257 	if (idx >= max_valid_idx) {
2258 		mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2259 			     idx, max_valid_idx);
2260 		return -EINVAL;
2261 	}
2262 
2263 	switch (cmd) {
2264 	case MLX5_IB_MMAP_WC_PAGE:
2265 	case MLX5_IB_MMAP_ALLOC_WC:
2266 	case MLX5_IB_MMAP_REGULAR_PAGE:
2267 		/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2268 		prot = pgprot_writecombine(vma->vm_page_prot);
2269 		break;
2270 	case MLX5_IB_MMAP_NC_PAGE:
2271 		prot = pgprot_noncached(vma->vm_page_prot);
2272 		break;
2273 	default:
2274 		return -EINVAL;
2275 	}
2276 
2277 	if (dyn_uar) {
2278 		int uars_per_page;
2279 
2280 		uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2281 		bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2282 		if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2283 			mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2284 				     bfreg_dyn_idx, bfregi->total_num_bfregs);
2285 			return -EINVAL;
2286 		}
2287 
2288 		mutex_lock(&bfregi->lock);
2289 		/* Fail if uar already allocated, first bfreg index of each
2290 		 * page holds its count.
2291 		 */
2292 		if (bfregi->count[bfreg_dyn_idx]) {
2293 			mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2294 			mutex_unlock(&bfregi->lock);
2295 			return -EINVAL;
2296 		}
2297 
2298 		bfregi->count[bfreg_dyn_idx]++;
2299 		mutex_unlock(&bfregi->lock);
2300 
2301 		err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2302 					 context->devx_uid);
2303 		if (err) {
2304 			mlx5_ib_warn(dev, "UAR alloc failed\n");
2305 			goto free_bfreg;
2306 		}
2307 	} else {
2308 		uar_index = bfregi->sys_pages[idx];
2309 	}
2310 
2311 	pfn = uar_index2pfn(dev, uar_index);
2312 	mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2313 
2314 	err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2315 				prot, NULL);
2316 	if (err) {
2317 		mlx5_ib_err(dev,
2318 			    "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2319 			    err, mmap_cmd2str(cmd));
2320 		goto err;
2321 	}
2322 
2323 	if (dyn_uar)
2324 		bfregi->sys_pages[idx] = uar_index;
2325 	return 0;
2326 
2327 err:
2328 	if (!dyn_uar)
2329 		return err;
2330 
2331 	mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
2332 
2333 free_bfreg:
2334 	mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2335 
2336 	return err;
2337 }
2338 
mlx5_vma_to_pgoff(struct vm_area_struct * vma)2339 static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2340 {
2341 	unsigned long idx;
2342 	u8 command;
2343 
2344 	command = get_command(vma->vm_pgoff);
2345 	idx = get_extended_index(vma->vm_pgoff);
2346 
2347 	return (command << 16 | idx);
2348 }
2349 
mlx5_ib_mmap_offset(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct ib_ucontext * ucontext)2350 static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2351 			       struct vm_area_struct *vma,
2352 			       struct ib_ucontext *ucontext)
2353 {
2354 	struct mlx5_user_mmap_entry *mentry;
2355 	struct rdma_user_mmap_entry *entry;
2356 	unsigned long pgoff;
2357 	pgprot_t prot;
2358 	phys_addr_t pfn;
2359 	int ret;
2360 
2361 	pgoff = mlx5_vma_to_pgoff(vma);
2362 	entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2363 	if (!entry)
2364 		return -EINVAL;
2365 
2366 	mentry = to_mmmap(entry);
2367 	pfn = (mentry->address >> PAGE_SHIFT);
2368 	if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2369 	    mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
2370 		prot = pgprot_noncached(vma->vm_page_prot);
2371 	else
2372 		prot = pgprot_writecombine(vma->vm_page_prot);
2373 	ret = rdma_user_mmap_io(ucontext, vma, pfn,
2374 				entry->npages * PAGE_SIZE,
2375 				prot,
2376 				entry);
2377 	rdma_user_mmap_entry_put(&mentry->rdma_entry);
2378 	return ret;
2379 }
2380 
mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry * entry)2381 static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2382 {
2383 	u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2384 	u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
2385 
2386 	return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2387 		(index & 0xFF)) << PAGE_SHIFT;
2388 }
2389 
mlx5_ib_mmap(struct ib_ucontext * ibcontext,struct vm_area_struct * vma)2390 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2391 {
2392 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2393 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2394 	unsigned long command;
2395 	phys_addr_t pfn;
2396 
2397 	command = get_command(vma->vm_pgoff);
2398 	switch (command) {
2399 	case MLX5_IB_MMAP_WC_PAGE:
2400 	case MLX5_IB_MMAP_ALLOC_WC:
2401 		if (!mlx5_wc_support_get(dev->mdev))
2402 			return -EPERM;
2403 		fallthrough;
2404 	case MLX5_IB_MMAP_NC_PAGE:
2405 	case MLX5_IB_MMAP_REGULAR_PAGE:
2406 		return uar_mmap(dev, command, vma, context);
2407 
2408 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2409 		return -ENOSYS;
2410 
2411 	case MLX5_IB_MMAP_CORE_CLOCK:
2412 		if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2413 			return -EINVAL;
2414 
2415 		if (vma->vm_flags & VM_WRITE)
2416 			return -EPERM;
2417 		vm_flags_clear(vma, VM_MAYWRITE);
2418 
2419 		/* Don't expose to user-space information it shouldn't have */
2420 		if (PAGE_SIZE > 4096)
2421 			return -EOPNOTSUPP;
2422 
2423 		pfn = (dev->mdev->iseg_base +
2424 		       offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2425 			PAGE_SHIFT;
2426 		return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2427 					 PAGE_SIZE,
2428 					 pgprot_noncached(vma->vm_page_prot),
2429 					 NULL);
2430 	case MLX5_IB_MMAP_CLOCK_INFO:
2431 		return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2432 
2433 	default:
2434 		return mlx5_ib_mmap_offset(dev, vma, ibcontext);
2435 	}
2436 
2437 	return 0;
2438 }
2439 
mlx5_ib_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)2440 static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2441 {
2442 	struct mlx5_ib_pd *pd = to_mpd(ibpd);
2443 	struct ib_device *ibdev = ibpd->device;
2444 	struct mlx5_ib_alloc_pd_resp resp;
2445 	int err;
2446 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2447 	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
2448 	u16 uid = 0;
2449 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2450 		udata, struct mlx5_ib_ucontext, ibucontext);
2451 
2452 	uid = context ? context->devx_uid : 0;
2453 	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2454 	MLX5_SET(alloc_pd_in, in, uid, uid);
2455 	err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
2456 	if (err)
2457 		return err;
2458 
2459 	pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2460 	pd->uid = uid;
2461 	if (udata) {
2462 		resp.pdn = pd->pdn;
2463 		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2464 			mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2465 			return -EFAULT;
2466 		}
2467 	}
2468 
2469 	return 0;
2470 }
2471 
mlx5_ib_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)2472 static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2473 {
2474 	struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2475 	struct mlx5_ib_pd *mpd = to_mpd(pd);
2476 
2477 	return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2478 }
2479 
mlx5_ib_mcg_attach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)2480 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2481 {
2482 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2483 	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2484 	int err;
2485 	u16 uid;
2486 
2487 	uid = ibqp->pd ?
2488 		to_mpd(ibqp->pd)->uid : 0;
2489 
2490 	if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2491 		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2492 		return -EOPNOTSUPP;
2493 	}
2494 
2495 	err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2496 	if (err)
2497 		mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2498 			     ibqp->qp_num, gid->raw);
2499 
2500 	return err;
2501 }
2502 
mlx5_ib_mcg_detach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)2503 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2504 {
2505 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2506 	int err;
2507 	u16 uid;
2508 
2509 	uid = ibqp->pd ?
2510 		to_mpd(ibqp->pd)->uid : 0;
2511 	err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2512 	if (err)
2513 		mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2514 			     ibqp->qp_num, gid->raw);
2515 
2516 	return err;
2517 }
2518 
init_node_data(struct mlx5_ib_dev * dev)2519 static int init_node_data(struct mlx5_ib_dev *dev)
2520 {
2521 	int err;
2522 
2523 	err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2524 	if (err)
2525 		return err;
2526 
2527 	dev->mdev->rev_id = dev->mdev->pdev->revision;
2528 
2529 	return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2530 }
2531 
fw_pages_show(struct device * device,struct device_attribute * attr,char * buf)2532 static ssize_t fw_pages_show(struct device *device,
2533 			     struct device_attribute *attr, char *buf)
2534 {
2535 	struct mlx5_ib_dev *dev =
2536 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2537 
2538 	return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
2539 }
2540 static DEVICE_ATTR_RO(fw_pages);
2541 
reg_pages_show(struct device * device,struct device_attribute * attr,char * buf)2542 static ssize_t reg_pages_show(struct device *device,
2543 			      struct device_attribute *attr, char *buf)
2544 {
2545 	struct mlx5_ib_dev *dev =
2546 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2547 
2548 	return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2549 }
2550 static DEVICE_ATTR_RO(reg_pages);
2551 
hca_type_show(struct device * device,struct device_attribute * attr,char * buf)2552 static ssize_t hca_type_show(struct device *device,
2553 			     struct device_attribute *attr, char *buf)
2554 {
2555 	struct mlx5_ib_dev *dev =
2556 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2557 
2558 	return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
2559 }
2560 static DEVICE_ATTR_RO(hca_type);
2561 
hw_rev_show(struct device * device,struct device_attribute * attr,char * buf)2562 static ssize_t hw_rev_show(struct device *device,
2563 			   struct device_attribute *attr, char *buf)
2564 {
2565 	struct mlx5_ib_dev *dev =
2566 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2567 
2568 	return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
2569 }
2570 static DEVICE_ATTR_RO(hw_rev);
2571 
board_id_show(struct device * device,struct device_attribute * attr,char * buf)2572 static ssize_t board_id_show(struct device *device,
2573 			     struct device_attribute *attr, char *buf)
2574 {
2575 	struct mlx5_ib_dev *dev =
2576 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2577 
2578 	return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2579 			  dev->mdev->board_id);
2580 }
2581 static DEVICE_ATTR_RO(board_id);
2582 
2583 static struct attribute *mlx5_class_attributes[] = {
2584 	&dev_attr_hw_rev.attr,
2585 	&dev_attr_hca_type.attr,
2586 	&dev_attr_board_id.attr,
2587 	&dev_attr_fw_pages.attr,
2588 	&dev_attr_reg_pages.attr,
2589 	NULL,
2590 };
2591 
2592 static const struct attribute_group mlx5_attr_group = {
2593 	.attrs = mlx5_class_attributes,
2594 };
2595 
pkey_change_handler(struct work_struct * work)2596 static void pkey_change_handler(struct work_struct *work)
2597 {
2598 	struct mlx5_ib_port_resources *ports =
2599 		container_of(work, struct mlx5_ib_port_resources,
2600 			     pkey_change_work);
2601 
2602 	if (!ports->gsi)
2603 		/*
2604 		 * We got this event before device was fully configured
2605 		 * and MAD registration code wasn't called/finished yet.
2606 		 */
2607 		return;
2608 
2609 	mlx5_ib_gsi_pkey_change(ports->gsi);
2610 }
2611 
mlx5_ib_handle_internal_error(struct mlx5_ib_dev * ibdev)2612 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2613 {
2614 	struct mlx5_ib_qp *mqp;
2615 	struct mlx5_ib_cq *send_mcq, *recv_mcq;
2616 	struct mlx5_core_cq *mcq;
2617 	struct list_head cq_armed_list;
2618 	unsigned long flags_qp;
2619 	unsigned long flags_cq;
2620 	unsigned long flags;
2621 
2622 	INIT_LIST_HEAD(&cq_armed_list);
2623 
2624 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2625 	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2626 	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2627 		spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2628 		if (mqp->sq.tail != mqp->sq.head) {
2629 			send_mcq = to_mcq(mqp->ibqp.send_cq);
2630 			spin_lock_irqsave(&send_mcq->lock, flags_cq);
2631 			if (send_mcq->mcq.comp &&
2632 			    mqp->ibqp.send_cq->comp_handler) {
2633 				if (!send_mcq->mcq.reset_notify_added) {
2634 					send_mcq->mcq.reset_notify_added = 1;
2635 					list_add_tail(&send_mcq->mcq.reset_notify,
2636 						      &cq_armed_list);
2637 				}
2638 			}
2639 			spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2640 		}
2641 		spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2642 		spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2643 		/* no handling is needed for SRQ */
2644 		if (!mqp->ibqp.srq) {
2645 			if (mqp->rq.tail != mqp->rq.head) {
2646 				recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2647 				spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2648 				if (recv_mcq->mcq.comp &&
2649 				    mqp->ibqp.recv_cq->comp_handler) {
2650 					if (!recv_mcq->mcq.reset_notify_added) {
2651 						recv_mcq->mcq.reset_notify_added = 1;
2652 						list_add_tail(&recv_mcq->mcq.reset_notify,
2653 							      &cq_armed_list);
2654 					}
2655 				}
2656 				spin_unlock_irqrestore(&recv_mcq->lock,
2657 						       flags_cq);
2658 			}
2659 		}
2660 		spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2661 	}
2662 	/*At that point all inflight post send were put to be executed as of we
2663 	 * lock/unlock above locks Now need to arm all involved CQs.
2664 	 */
2665 	list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2666 		mcq->comp(mcq, NULL);
2667 	}
2668 	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2669 }
2670 
delay_drop_handler(struct work_struct * work)2671 static void delay_drop_handler(struct work_struct *work)
2672 {
2673 	int err;
2674 	struct mlx5_ib_delay_drop *delay_drop =
2675 		container_of(work, struct mlx5_ib_delay_drop,
2676 			     delay_drop_work);
2677 
2678 	atomic_inc(&delay_drop->events_cnt);
2679 
2680 	mutex_lock(&delay_drop->lock);
2681 	err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
2682 	if (err) {
2683 		mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2684 			     delay_drop->timeout);
2685 		delay_drop->activate = false;
2686 	}
2687 	mutex_unlock(&delay_drop->lock);
2688 }
2689 
handle_general_event(struct mlx5_ib_dev * ibdev,struct mlx5_eqe * eqe,struct ib_event * ibev)2690 static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2691 				 struct ib_event *ibev)
2692 {
2693 	u32 port = (eqe->data.port.port >> 4) & 0xf;
2694 
2695 	switch (eqe->sub_type) {
2696 	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
2697 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2698 					    IB_LINK_LAYER_ETHERNET)
2699 			schedule_work(&ibdev->delay_drop.delay_drop_work);
2700 		break;
2701 	default: /* do nothing */
2702 		return;
2703 	}
2704 }
2705 
handle_port_change(struct mlx5_ib_dev * ibdev,struct mlx5_eqe * eqe,struct ib_event * ibev)2706 static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2707 			      struct ib_event *ibev)
2708 {
2709 	u32 port = (eqe->data.port.port >> 4) & 0xf;
2710 
2711 	ibev->element.port_num = port;
2712 
2713 	switch (eqe->sub_type) {
2714 	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2715 	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2716 	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2717 		/* In RoCE, port up/down events are handled in
2718 		 * mlx5_netdev_event().
2719 		 */
2720 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2721 					    IB_LINK_LAYER_ETHERNET)
2722 			return -EINVAL;
2723 
2724 		ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2725 				IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2726 		break;
2727 
2728 	case MLX5_PORT_CHANGE_SUBTYPE_LID:
2729 		ibev->event = IB_EVENT_LID_CHANGE;
2730 		break;
2731 
2732 	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2733 		ibev->event = IB_EVENT_PKEY_CHANGE;
2734 		schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2735 		break;
2736 
2737 	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2738 		ibev->event = IB_EVENT_GID_CHANGE;
2739 		break;
2740 
2741 	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2742 		ibev->event = IB_EVENT_CLIENT_REREGISTER;
2743 		break;
2744 	default:
2745 		return -EINVAL;
2746 	}
2747 
2748 	return 0;
2749 }
2750 
mlx5_ib_handle_event(struct work_struct * _work)2751 static void mlx5_ib_handle_event(struct work_struct *_work)
2752 {
2753 	struct mlx5_ib_event_work *work =
2754 		container_of(_work, struct mlx5_ib_event_work, work);
2755 	struct mlx5_ib_dev *ibdev;
2756 	struct ib_event ibev;
2757 	bool fatal = false;
2758 
2759 	if (work->is_slave) {
2760 		ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
2761 		if (!ibdev)
2762 			goto out;
2763 	} else {
2764 		ibdev = work->dev;
2765 	}
2766 
2767 	switch (work->event) {
2768 	case MLX5_DEV_EVENT_SYS_ERROR:
2769 		ibev.event = IB_EVENT_DEVICE_FATAL;
2770 		mlx5_ib_handle_internal_error(ibdev);
2771 		ibev.element.port_num  = (u8)(unsigned long)work->param;
2772 		fatal = true;
2773 		break;
2774 	case MLX5_EVENT_TYPE_PORT_CHANGE:
2775 		if (handle_port_change(ibdev, work->param, &ibev))
2776 			goto out;
2777 		break;
2778 	case MLX5_EVENT_TYPE_GENERAL_EVENT:
2779 		handle_general_event(ibdev, work->param, &ibev);
2780 		fallthrough;
2781 	default:
2782 		goto out;
2783 	}
2784 
2785 	ibev.device = &ibdev->ib_dev;
2786 
2787 	if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2788 		mlx5_ib_warn(ibdev, "warning: event on port %d\n",  ibev.element.port_num);
2789 		goto out;
2790 	}
2791 
2792 	if (ibdev->ib_active)
2793 		ib_dispatch_event(&ibev);
2794 
2795 	if (fatal)
2796 		ibdev->ib_active = false;
2797 out:
2798 	kfree(work);
2799 }
2800 
mlx5_ib_event(struct notifier_block * nb,unsigned long event,void * param)2801 static int mlx5_ib_event(struct notifier_block *nb,
2802 			 unsigned long event, void *param)
2803 {
2804 	struct mlx5_ib_event_work *work;
2805 
2806 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
2807 	if (!work)
2808 		return NOTIFY_DONE;
2809 
2810 	INIT_WORK(&work->work, mlx5_ib_handle_event);
2811 	work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2812 	work->is_slave = false;
2813 	work->param = param;
2814 	work->event = event;
2815 
2816 	queue_work(mlx5_ib_event_wq, &work->work);
2817 
2818 	return NOTIFY_OK;
2819 }
2820 
mlx5_ib_event_slave_port(struct notifier_block * nb,unsigned long event,void * param)2821 static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2822 				    unsigned long event, void *param)
2823 {
2824 	struct mlx5_ib_event_work *work;
2825 
2826 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
2827 	if (!work)
2828 		return NOTIFY_DONE;
2829 
2830 	INIT_WORK(&work->work, mlx5_ib_handle_event);
2831 	work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2832 	work->is_slave = true;
2833 	work->param = param;
2834 	work->event = event;
2835 	queue_work(mlx5_ib_event_wq, &work->work);
2836 
2837 	return NOTIFY_OK;
2838 }
2839 
mlx5_ib_get_plane_num(struct mlx5_core_dev * mdev,u8 * num_plane)2840 static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane)
2841 {
2842 	struct mlx5_hca_vport_context vport_ctx;
2843 	int err;
2844 
2845 	*num_plane = 0;
2846 	if (!MLX5_CAP_GEN(mdev, ib_virt) || !MLX5_CAP_GEN_2(mdev, multiplane))
2847 		return 0;
2848 
2849 	err = mlx5_query_hca_vport_context(mdev, 0, 1, 0, &vport_ctx);
2850 	if (err)
2851 		return err;
2852 
2853 	*num_plane = vport_ctx.num_plane;
2854 	return 0;
2855 }
2856 
set_has_smi_cap(struct mlx5_ib_dev * dev)2857 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2858 {
2859 	struct mlx5_hca_vport_context vport_ctx;
2860 	int err;
2861 	int port;
2862 
2863 	if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2864 		return 0;
2865 
2866 	for (port = 1; port <= dev->num_ports; port++) {
2867 		if (dev->num_plane) {
2868 			dev->port_caps[port - 1].has_smi = false;
2869 			continue;
2870 		} else if (!MLX5_CAP_GEN(dev->mdev, ib_virt) ||
2871 			dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
2872 			dev->port_caps[port - 1].has_smi = true;
2873 			continue;
2874 		}
2875 
2876 		err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2877 						   &vport_ctx);
2878 		if (err) {
2879 			mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2880 				    port, err);
2881 			return err;
2882 		}
2883 		dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
2884 	}
2885 
2886 	return 0;
2887 }
2888 
get_ext_port_caps(struct mlx5_ib_dev * dev)2889 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2890 {
2891 	unsigned int port;
2892 
2893 	rdma_for_each_port (&dev->ib_dev, port)
2894 		mlx5_query_ext_port_caps(dev, port);
2895 }
2896 
mlx5_get_umr_fence(u8 umr_fence_cap)2897 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2898 {
2899 	switch (umr_fence_cap) {
2900 	case MLX5_CAP_UMR_FENCE_NONE:
2901 		return MLX5_FENCE_MODE_NONE;
2902 	case MLX5_CAP_UMR_FENCE_SMALL:
2903 		return MLX5_FENCE_MODE_INITIATOR_SMALL;
2904 	default:
2905 		return MLX5_FENCE_MODE_STRONG_ORDERING;
2906 	}
2907 }
2908 
mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev * dev)2909 int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
2910 {
2911 	struct mlx5_ib_resources *devr = &dev->devr;
2912 	struct ib_cq_init_attr cq_attr = {.cqe = 1};
2913 	struct ib_device *ibdev;
2914 	struct ib_pd *pd;
2915 	struct ib_cq *cq;
2916 	int ret = 0;
2917 
2918 
2919 	/*
2920 	 * devr->c0 is set once, never changed until device unload.
2921 	 * Avoid taking the mutex if initialization is already done.
2922 	 */
2923 	if (devr->c0)
2924 		return 0;
2925 
2926 	mutex_lock(&devr->cq_lock);
2927 	if (devr->c0)
2928 		goto unlock;
2929 
2930 	ibdev = &dev->ib_dev;
2931 	pd = ib_alloc_pd(ibdev, 0);
2932 	if (IS_ERR(pd)) {
2933 		ret = PTR_ERR(pd);
2934 		mlx5_ib_err(dev, "Couldn't allocate PD for res init, err=%d\n", ret);
2935 		goto unlock;
2936 	}
2937 
2938 	cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
2939 	if (IS_ERR(cq)) {
2940 		ret = PTR_ERR(cq);
2941 		mlx5_ib_err(dev, "Couldn't create CQ for res init, err=%d\n", ret);
2942 		ib_dealloc_pd(pd);
2943 		goto unlock;
2944 	}
2945 
2946 	devr->p0 = pd;
2947 	devr->c0 = cq;
2948 
2949 unlock:
2950 	mutex_unlock(&devr->cq_lock);
2951 	return ret;
2952 }
2953 
mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev * dev)2954 int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
2955 {
2956 	struct mlx5_ib_resources *devr = &dev->devr;
2957 	struct ib_srq_init_attr attr;
2958 	struct ib_srq *s0, *s1;
2959 	int ret = 0;
2960 
2961 	/*
2962 	 * devr->s1 is set once, never changed until device unload.
2963 	 * Avoid taking the mutex if initialization is already done.
2964 	 */
2965 	if (devr->s1)
2966 		return 0;
2967 
2968 	mutex_lock(&devr->srq_lock);
2969 	if (devr->s1)
2970 		goto unlock;
2971 
2972 	ret = mlx5_ib_dev_res_cq_init(dev);
2973 	if (ret)
2974 		goto unlock;
2975 
2976 	memset(&attr, 0, sizeof(attr));
2977 	attr.attr.max_sge = 1;
2978 	attr.attr.max_wr = 1;
2979 	attr.srq_type = IB_SRQT_XRC;
2980 	attr.ext.cq = devr->c0;
2981 
2982 	s0 = ib_create_srq(devr->p0, &attr);
2983 	if (IS_ERR(s0)) {
2984 		ret = PTR_ERR(s0);
2985 		mlx5_ib_err(dev, "Couldn't create SRQ 0 for res init, err=%d\n", ret);
2986 		goto unlock;
2987 	}
2988 
2989 	memset(&attr, 0, sizeof(attr));
2990 	attr.attr.max_sge = 1;
2991 	attr.attr.max_wr = 1;
2992 	attr.srq_type = IB_SRQT_BASIC;
2993 
2994 	s1 = ib_create_srq(devr->p0, &attr);
2995 	if (IS_ERR(s1)) {
2996 		ret = PTR_ERR(s1);
2997 		mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%d\n", ret);
2998 		ib_destroy_srq(s0);
2999 	}
3000 
3001 	devr->s0 = s0;
3002 	devr->s1 = s1;
3003 
3004 unlock:
3005 	mutex_unlock(&devr->srq_lock);
3006 	return ret;
3007 }
3008 
mlx5_ib_dev_res_init(struct mlx5_ib_dev * dev)3009 static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
3010 {
3011 	struct mlx5_ib_resources *devr = &dev->devr;
3012 	int ret;
3013 
3014 	if (!MLX5_CAP_GEN(dev->mdev, xrc))
3015 		return -EOPNOTSUPP;
3016 
3017 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
3018 	if (ret)
3019 		return ret;
3020 
3021 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
3022 	if (ret) {
3023 		mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3024 		return ret;
3025 	}
3026 
3027 	mutex_init(&devr->cq_lock);
3028 	mutex_init(&devr->srq_lock);
3029 
3030 	return 0;
3031 }
3032 
mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev * dev)3033 static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
3034 {
3035 	struct mlx5_ib_resources *devr = &dev->devr;
3036 
3037 	/* After s0/s1 init, they are not unset during the device lifetime. */
3038 	if (devr->s1) {
3039 		ib_destroy_srq(devr->s1);
3040 		ib_destroy_srq(devr->s0);
3041 	}
3042 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
3043 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3044 	/* After p0/c0 init, they are not unset during the device lifetime. */
3045 	if (devr->c0) {
3046 		ib_destroy_cq(devr->c0);
3047 		ib_dealloc_pd(devr->p0);
3048 	}
3049 	mutex_destroy(&devr->cq_lock);
3050 	mutex_destroy(&devr->srq_lock);
3051 }
3052 
3053 static int
mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev * dev)3054 mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
3055 {
3056 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3057 	struct mlx5_core_dev *mdev = dev->mdev;
3058 	void *mkc;
3059 	u32 mkey;
3060 	u32 pdn;
3061 	u32 *in;
3062 	int err;
3063 
3064 	err = mlx5_core_alloc_pd(mdev, &pdn);
3065 	if (err)
3066 		return err;
3067 
3068 	in = kvzalloc(inlen, GFP_KERNEL);
3069 	if (!in) {
3070 		err = -ENOMEM;
3071 		goto err;
3072 	}
3073 
3074 	MLX5_SET(create_mkey_in, in, data_direct, 1);
3075 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3076 	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
3077 	MLX5_SET(mkc, mkc, lw, 1);
3078 	MLX5_SET(mkc, mkc, lr, 1);
3079 	MLX5_SET(mkc, mkc, rw, 1);
3080 	MLX5_SET(mkc, mkc, rr, 1);
3081 	MLX5_SET(mkc, mkc, a, 1);
3082 	MLX5_SET(mkc, mkc, pd, pdn);
3083 	MLX5_SET(mkc, mkc, length64, 1);
3084 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
3085 	err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
3086 	kvfree(in);
3087 	if (err)
3088 		goto err;
3089 
3090 	dev->ddr.mkey = mkey;
3091 	dev->ddr.pdn = pdn;
3092 	return 0;
3093 
3094 err:
3095 	mlx5_core_dealloc_pd(mdev, pdn);
3096 	return err;
3097 }
3098 
3099 static void
mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev * dev)3100 mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
3101 {
3102 	mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
3103 	mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
3104 }
3105 
get_core_cap_flags(struct ib_device * ibdev,struct mlx5_hca_vport_context * rep)3106 static u32 get_core_cap_flags(struct ib_device *ibdev,
3107 			      struct mlx5_hca_vport_context *rep)
3108 {
3109 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
3110 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3111 	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3112 	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3113 	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
3114 	u32 ret = 0;
3115 
3116 	if (rep->grh_required)
3117 		ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
3118 
3119 	if (dev->num_plane)
3120 		return ret | RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_IB_MAD |
3121 			RDMA_CORE_CAP_IB_CM | RDMA_CORE_CAP_IB_SA |
3122 			RDMA_CORE_CAP_AF_IB;
3123 	else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3124 		return ret | RDMA_CORE_CAP_IB_MAD | RDMA_CORE_CAP_IB_SMI;
3125 
3126 	if (ll == IB_LINK_LAYER_INFINIBAND)
3127 		return ret | RDMA_CORE_PORT_IBA_IB;
3128 
3129 	if (raw_support)
3130 		ret |= RDMA_CORE_PORT_RAW_PACKET;
3131 
3132 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3133 		return ret;
3134 
3135 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3136 		return ret;
3137 
3138 	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3139 		ret |= RDMA_CORE_PORT_IBA_ROCE;
3140 
3141 	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3142 		ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3143 
3144 	return ret;
3145 }
3146 
mlx5_port_immutable(struct ib_device * ibdev,u32 port_num,struct ib_port_immutable * immutable)3147 static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
3148 			       struct ib_port_immutable *immutable)
3149 {
3150 	struct ib_port_attr attr;
3151 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
3152 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
3153 	struct mlx5_hca_vport_context rep = {0};
3154 	int err;
3155 
3156 	err = ib_query_port(ibdev, port_num, &attr);
3157 	if (err)
3158 		return err;
3159 
3160 	if (ll == IB_LINK_LAYER_INFINIBAND) {
3161 		if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3162 			port_num = smi_to_native_portnum(dev, port_num);
3163 
3164 		err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
3165 						   &rep);
3166 		if (err)
3167 			return err;
3168 	}
3169 
3170 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
3171 	immutable->gid_tbl_len = attr.gid_tbl_len;
3172 	immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
3173 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3174 
3175 	return 0;
3176 }
3177 
mlx5_port_rep_immutable(struct ib_device * ibdev,u32 port_num,struct ib_port_immutable * immutable)3178 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
3179 				   struct ib_port_immutable *immutable)
3180 {
3181 	struct ib_port_attr attr;
3182 	int err;
3183 
3184 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3185 
3186 	err = ib_query_port(ibdev, port_num, &attr);
3187 	if (err)
3188 		return err;
3189 
3190 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
3191 	immutable->gid_tbl_len = attr.gid_tbl_len;
3192 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3193 
3194 	return 0;
3195 }
3196 
get_dev_fw_str(struct ib_device * ibdev,char * str)3197 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
3198 {
3199 	struct mlx5_ib_dev *dev =
3200 		container_of(ibdev, struct mlx5_ib_dev, ib_dev);
3201 	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
3202 		 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
3203 		 fw_rev_sub(dev->mdev));
3204 }
3205 
lag_event(struct notifier_block * nb,unsigned long event,void * data)3206 static int lag_event(struct notifier_block *nb, unsigned long event, void *data)
3207 {
3208 	struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev,
3209 					       lag_events);
3210 	struct mlx5_core_dev *mdev = dev->mdev;
3211 	struct ib_device *ibdev = &dev->ib_dev;
3212 	struct net_device *old_ndev = NULL;
3213 	struct mlx5_ib_port *port;
3214 	struct net_device *ndev;
3215 	u32 portnum = 0;
3216 	int ret = 0;
3217 	int i;
3218 
3219 	switch (event) {
3220 	case MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE:
3221 		ndev = data;
3222 		if (ndev) {
3223 			if (!mlx5_lag_is_roce(mdev)) {
3224 				// sriov lag
3225 				for (i = 0; i < dev->num_ports; i++) {
3226 					port = &dev->port[i];
3227 					if (port->rep && port->rep->vport ==
3228 					    MLX5_VPORT_UPLINK) {
3229 						portnum = i;
3230 						break;
3231 					}
3232 				}
3233 			}
3234 			old_ndev = ib_device_get_netdev(ibdev, portnum + 1);
3235 			ret = ib_device_set_netdev(ibdev, ndev, portnum + 1);
3236 			if (ret)
3237 				goto out;
3238 
3239 			if (old_ndev)
3240 				roce_del_all_netdev_gids(ibdev, portnum + 1,
3241 							 old_ndev);
3242 			rdma_roce_rescan_port(ibdev, portnum + 1);
3243 		}
3244 		break;
3245 	default:
3246 		return NOTIFY_DONE;
3247 	}
3248 
3249 out:
3250 	dev_put(old_ndev);
3251 	return notifier_from_errno(ret);
3252 }
3253 
mlx5e_lag_event_register(struct mlx5_ib_dev * dev)3254 static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev)
3255 {
3256 	dev->lag_events.notifier_call = lag_event;
3257 	blocking_notifier_chain_register(&dev->mdev->priv.lag_nh,
3258 					 &dev->lag_events);
3259 }
3260 
mlx5e_lag_event_unregister(struct mlx5_ib_dev * dev)3261 static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)
3262 {
3263 	blocking_notifier_chain_unregister(&dev->mdev->priv.lag_nh,
3264 					   &dev->lag_events);
3265 }
3266 
mlx5_eth_lag_init(struct mlx5_ib_dev * dev)3267 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
3268 {
3269 	struct mlx5_core_dev *mdev = dev->mdev;
3270 	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
3271 								 MLX5_FLOW_NAMESPACE_LAG);
3272 	struct mlx5_flow_table *ft;
3273 	int err;
3274 
3275 	if (!ns || !mlx5_lag_is_active(mdev))
3276 		return 0;
3277 
3278 	err = mlx5_cmd_create_vport_lag(mdev);
3279 	if (err)
3280 		return err;
3281 
3282 	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
3283 	if (IS_ERR(ft)) {
3284 		err = PTR_ERR(ft);
3285 		goto err_destroy_vport_lag;
3286 	}
3287 
3288 	mlx5e_lag_event_register(dev);
3289 	dev->flow_db->lag_demux_ft = ft;
3290 	dev->lag_ports = mlx5_lag_get_num_ports(mdev);
3291 	dev->lag_active = true;
3292 	return 0;
3293 
3294 err_destroy_vport_lag:
3295 	mlx5_cmd_destroy_vport_lag(mdev);
3296 	return err;
3297 }
3298 
mlx5_eth_lag_cleanup(struct mlx5_ib_dev * dev)3299 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3300 {
3301 	struct mlx5_core_dev *mdev = dev->mdev;
3302 
3303 	if (dev->lag_active) {
3304 		dev->lag_active = false;
3305 
3306 		mlx5e_lag_event_unregister(dev);
3307 		mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3308 		dev->flow_db->lag_demux_ft = NULL;
3309 
3310 		mlx5_cmd_destroy_vport_lag(mdev);
3311 	}
3312 }
3313 
mlx5_netdev_notifier_register(struct mlx5_roce * roce,struct net_device * netdev)3314 static void mlx5_netdev_notifier_register(struct mlx5_roce *roce,
3315 					  struct net_device *netdev)
3316 {
3317 	int err;
3318 
3319 	if (roce->tracking_netdev)
3320 		return;
3321 	roce->tracking_netdev = netdev;
3322 	roce->nb.notifier_call = mlx5_netdev_event;
3323 	err = register_netdevice_notifier_dev_net(netdev, &roce->nb, &roce->nn);
3324 	WARN_ON(err);
3325 }
3326 
mlx5_netdev_notifier_unregister(struct mlx5_roce * roce)3327 static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce)
3328 {
3329 	if (!roce->tracking_netdev)
3330 		return;
3331 	unregister_netdevice_notifier_dev_net(roce->tracking_netdev, &roce->nb,
3332 					      &roce->nn);
3333 	roce->tracking_netdev = NULL;
3334 }
3335 
mlx5e_mdev_notifier_event(struct notifier_block * nb,unsigned long event,void * data)3336 static int mlx5e_mdev_notifier_event(struct notifier_block *nb,
3337 				     unsigned long event, void *data)
3338 {
3339 	struct mlx5_roce *roce = container_of(nb, struct mlx5_roce, mdev_nb);
3340 	struct net_device *netdev = data;
3341 
3342 	switch (event) {
3343 	case MLX5_DRIVER_EVENT_UPLINK_NETDEV:
3344 		if (netdev)
3345 			mlx5_netdev_notifier_register(roce, netdev);
3346 		else
3347 			mlx5_netdev_notifier_unregister(roce);
3348 		break;
3349 	default:
3350 		return NOTIFY_DONE;
3351 	}
3352 
3353 	return NOTIFY_OK;
3354 }
3355 
mlx5_mdev_netdev_track(struct mlx5_ib_dev * dev,u32 port_num)3356 static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num)
3357 {
3358 	struct mlx5_roce *roce = &dev->port[port_num].roce;
3359 
3360 	roce->mdev_nb.notifier_call = mlx5e_mdev_notifier_event;
3361 	mlx5_blocking_notifier_register(dev->mdev, &roce->mdev_nb);
3362 	mlx5_core_uplink_netdev_event_replay(dev->mdev);
3363 }
3364 
mlx5_mdev_netdev_untrack(struct mlx5_ib_dev * dev,u32 port_num)3365 static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num)
3366 {
3367 	struct mlx5_roce *roce = &dev->port[port_num].roce;
3368 
3369 	mlx5_blocking_notifier_unregister(dev->mdev, &roce->mdev_nb);
3370 	mlx5_netdev_notifier_unregister(roce);
3371 }
3372 
mlx5_enable_eth(struct mlx5_ib_dev * dev)3373 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3374 {
3375 	int err;
3376 
3377 	if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3378 		err = mlx5_nic_vport_enable_roce(dev->mdev);
3379 		if (err)
3380 			return err;
3381 	}
3382 
3383 	err = mlx5_eth_lag_init(dev);
3384 	if (err)
3385 		goto err_disable_roce;
3386 
3387 	return 0;
3388 
3389 err_disable_roce:
3390 	if (!dev->is_rep && dev->profile != &raw_eth_profile)
3391 		mlx5_nic_vport_disable_roce(dev->mdev);
3392 
3393 	return err;
3394 }
3395 
mlx5_disable_eth(struct mlx5_ib_dev * dev)3396 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3397 {
3398 	mlx5_eth_lag_cleanup(dev);
3399 	if (!dev->is_rep && dev->profile != &raw_eth_profile)
3400 		mlx5_nic_vport_disable_roce(dev->mdev);
3401 }
3402 
mlx5_ib_rn_get_params(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,struct rdma_netdev_alloc_params * params)3403 static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
3404 				 enum rdma_netdev_t type,
3405 				 struct rdma_netdev_alloc_params *params)
3406 {
3407 	if (type != RDMA_NETDEV_IPOIB)
3408 		return -EOPNOTSUPP;
3409 
3410 	return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
3411 }
3412 
delay_drop_timeout_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)3413 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3414 				       size_t count, loff_t *pos)
3415 {
3416 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3417 	char lbuf[20];
3418 	int len;
3419 
3420 	len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3421 	return simple_read_from_buffer(buf, count, pos, lbuf, len);
3422 }
3423 
delay_drop_timeout_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)3424 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3425 					size_t count, loff_t *pos)
3426 {
3427 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3428 	u32 timeout;
3429 	u32 var;
3430 
3431 	if (kstrtouint_from_user(buf, count, 0, &var))
3432 		return -EFAULT;
3433 
3434 	timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3435 			1000);
3436 	if (timeout != var)
3437 		mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3438 			    timeout);
3439 
3440 	delay_drop->timeout = timeout;
3441 
3442 	return count;
3443 }
3444 
3445 static const struct file_operations fops_delay_drop_timeout = {
3446 	.owner	= THIS_MODULE,
3447 	.open	= simple_open,
3448 	.write	= delay_drop_timeout_write,
3449 	.read	= delay_drop_timeout_read,
3450 };
3451 
mlx5_ib_unbind_slave_port(struct mlx5_ib_dev * ibdev,struct mlx5_ib_multiport_info * mpi)3452 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3453 				      struct mlx5_ib_multiport_info *mpi)
3454 {
3455 	u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3456 	struct mlx5_ib_port *port = &ibdev->port[port_num];
3457 	int comps;
3458 	int err;
3459 	int i;
3460 
3461 	lockdep_assert_held(&mlx5_ib_multiport_mutex);
3462 
3463 	mlx5_core_mp_event_replay(ibdev->mdev,
3464 				  MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3465 				  NULL);
3466 	mlx5_core_mp_event_replay(mpi->mdev,
3467 				  MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3468 				  NULL);
3469 
3470 	mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3471 
3472 	spin_lock(&port->mp.mpi_lock);
3473 	if (!mpi->ibdev) {
3474 		spin_unlock(&port->mp.mpi_lock);
3475 		return;
3476 	}
3477 
3478 	mpi->ibdev = NULL;
3479 
3480 	spin_unlock(&port->mp.mpi_lock);
3481 	if (mpi->mdev_events.notifier_call)
3482 		mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3483 	mpi->mdev_events.notifier_call = NULL;
3484 	mlx5_mdev_netdev_untrack(ibdev, port_num);
3485 	spin_lock(&port->mp.mpi_lock);
3486 
3487 	comps = mpi->mdev_refcnt;
3488 	if (comps) {
3489 		mpi->unaffiliate = true;
3490 		init_completion(&mpi->unref_comp);
3491 		spin_unlock(&port->mp.mpi_lock);
3492 
3493 		for (i = 0; i < comps; i++)
3494 			wait_for_completion(&mpi->unref_comp);
3495 
3496 		spin_lock(&port->mp.mpi_lock);
3497 		mpi->unaffiliate = false;
3498 	}
3499 
3500 	port->mp.mpi = NULL;
3501 
3502 	spin_unlock(&port->mp.mpi_lock);
3503 
3504 	err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3505 
3506 	mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
3507 	/* Log an error, still needed to cleanup the pointers and add
3508 	 * it back to the list.
3509 	 */
3510 	if (err)
3511 		mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3512 			    port_num + 1);
3513 
3514 	ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
3515 }
3516 
mlx5_ib_bind_slave_port(struct mlx5_ib_dev * ibdev,struct mlx5_ib_multiport_info * mpi)3517 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3518 				    struct mlx5_ib_multiport_info *mpi)
3519 {
3520 	u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3521 	u64 key;
3522 	int err;
3523 
3524 	lockdep_assert_held(&mlx5_ib_multiport_mutex);
3525 
3526 	spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3527 	if (ibdev->port[port_num].mp.mpi) {
3528 		mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
3529 			    port_num + 1);
3530 		spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3531 		return false;
3532 	}
3533 
3534 	ibdev->port[port_num].mp.mpi = mpi;
3535 	mpi->ibdev = ibdev;
3536 	mpi->mdev_events.notifier_call = NULL;
3537 	spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3538 
3539 	err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3540 	if (err)
3541 		goto unbind;
3542 
3543 	mlx5_mdev_netdev_track(ibdev, port_num);
3544 
3545 	mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3546 	mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3547 
3548 	mlx5_ib_init_cong_debugfs(ibdev, port_num);
3549 
3550 	key = mpi->mdev->priv.adev_idx;
3551 	mlx5_core_mp_event_replay(mpi->mdev,
3552 				  MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3553 				  &key);
3554 	mlx5_core_mp_event_replay(ibdev->mdev,
3555 				  MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3556 				  &key);
3557 
3558 	return true;
3559 
3560 unbind:
3561 	mlx5_ib_unbind_slave_port(ibdev, mpi);
3562 	return false;
3563 }
3564 
mlx5_ib_data_direct_init(struct mlx5_ib_dev * dev)3565 static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
3566 {
3567 	char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
3568 	int ret;
3569 
3570 	if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3571 	    !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3572 		return 0;
3573 
3574 	ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
3575 	if (ret)
3576 		return ret;
3577 
3578 	ret = mlx5_ib_create_data_direct_resources(dev);
3579 	if (ret)
3580 		return ret;
3581 
3582 	INIT_LIST_HEAD(&dev->data_direct_mr_list);
3583 	ret = mlx5_data_direct_ib_reg(dev, vuid);
3584 	if (ret)
3585 		mlx5_ib_free_data_direct_resources(dev);
3586 
3587 	return ret;
3588 }
3589 
mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev * dev)3590 static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
3591 {
3592 	if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3593 	    !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3594 		return;
3595 
3596 	mlx5_data_direct_ib_unreg(dev);
3597 	mlx5_ib_free_data_direct_resources(dev);
3598 }
3599 
mlx5_ib_init_multiport_master(struct mlx5_ib_dev * dev)3600 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3601 {
3602 	u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3603 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3604 							  port_num + 1);
3605 	struct mlx5_ib_multiport_info *mpi;
3606 	int err;
3607 	u32 i;
3608 
3609 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3610 		return 0;
3611 
3612 	err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3613 						     &dev->sys_image_guid);
3614 	if (err)
3615 		return err;
3616 
3617 	err = mlx5_nic_vport_enable_roce(dev->mdev);
3618 	if (err)
3619 		return err;
3620 
3621 	mutex_lock(&mlx5_ib_multiport_mutex);
3622 	for (i = 0; i < dev->num_ports; i++) {
3623 		bool bound = false;
3624 
3625 		/* build a stub multiport info struct for the native port. */
3626 		if (i == port_num) {
3627 			mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3628 			if (!mpi) {
3629 				mutex_unlock(&mlx5_ib_multiport_mutex);
3630 				mlx5_nic_vport_disable_roce(dev->mdev);
3631 				return -ENOMEM;
3632 			}
3633 
3634 			mpi->is_master = true;
3635 			mpi->mdev = dev->mdev;
3636 			mpi->sys_image_guid = dev->sys_image_guid;
3637 			dev->port[i].mp.mpi = mpi;
3638 			mpi->ibdev = dev;
3639 			mpi = NULL;
3640 			continue;
3641 		}
3642 
3643 		list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3644 				    list) {
3645 			if (dev->sys_image_guid == mpi->sys_image_guid &&
3646 			    (mlx5_core_native_port_num(mpi->mdev) - 1) == i &&
3647 			    mlx5_core_same_coredev_type(dev->mdev, mpi->mdev)) {
3648 				bound = mlx5_ib_bind_slave_port(dev, mpi);
3649 			}
3650 
3651 			if (bound) {
3652 				dev_dbg(mpi->mdev->device,
3653 					"removing port from unaffiliated list.\n");
3654 				mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3655 				list_del(&mpi->list);
3656 				break;
3657 			}
3658 		}
3659 		if (!bound)
3660 			mlx5_ib_dbg(dev, "no free port found for port %d\n",
3661 				    i + 1);
3662 	}
3663 
3664 	list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3665 	mutex_unlock(&mlx5_ib_multiport_mutex);
3666 	return err;
3667 }
3668 
mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev * dev)3669 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3670 {
3671 	u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3672 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3673 							  port_num + 1);
3674 	u32 i;
3675 
3676 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3677 		return;
3678 
3679 	mutex_lock(&mlx5_ib_multiport_mutex);
3680 	for (i = 0; i < dev->num_ports; i++) {
3681 		if (dev->port[i].mp.mpi) {
3682 			/* Destroy the native port stub */
3683 			if (i == port_num) {
3684 				kfree(dev->port[i].mp.mpi);
3685 				dev->port[i].mp.mpi = NULL;
3686 			} else {
3687 				mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3688 					    i + 1);
3689 				list_add_tail(&dev->port[i].mp.mpi->list,
3690 					      &mlx5_ib_unaffiliated_port_list);
3691 				mlx5_ib_unbind_slave_port(dev,
3692 							  dev->port[i].mp.mpi);
3693 			}
3694 		}
3695 	}
3696 
3697 	mlx5_ib_dbg(dev, "removing from devlist\n");
3698 	list_del(&dev->ib_dev_list);
3699 	mutex_unlock(&mlx5_ib_multiport_mutex);
3700 
3701 	mlx5_nic_vport_disable_roce(dev->mdev);
3702 }
3703 
mmap_obj_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)3704 static int mmap_obj_cleanup(struct ib_uobject *uobject,
3705 			    enum rdma_remove_reason why,
3706 			    struct uverbs_attr_bundle *attrs)
3707 {
3708 	struct mlx5_user_mmap_entry *obj = uobject->object;
3709 
3710 	rdma_user_mmap_entry_remove(&obj->rdma_entry);
3711 	return 0;
3712 }
3713 
mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext * c,struct mlx5_user_mmap_entry * entry,size_t length)3714 static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3715 					    struct mlx5_user_mmap_entry *entry,
3716 					    size_t length)
3717 {
3718 	return rdma_user_mmap_entry_insert_range(
3719 		&c->ibucontext, &entry->rdma_entry, length,
3720 		(MLX5_IB_MMAP_OFFSET_START << 16),
3721 		((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3722 }
3723 
3724 static struct mlx5_user_mmap_entry *
alloc_var_entry(struct mlx5_ib_ucontext * c)3725 alloc_var_entry(struct mlx5_ib_ucontext *c)
3726 {
3727 	struct mlx5_user_mmap_entry *entry;
3728 	struct mlx5_var_table *var_table;
3729 	u32 page_idx;
3730 	int err;
3731 
3732 	var_table = &to_mdev(c->ibucontext.device)->var_table;
3733 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3734 	if (!entry)
3735 		return ERR_PTR(-ENOMEM);
3736 
3737 	mutex_lock(&var_table->bitmap_lock);
3738 	page_idx = find_first_zero_bit(var_table->bitmap,
3739 				       var_table->num_var_hw_entries);
3740 	if (page_idx >= var_table->num_var_hw_entries) {
3741 		err = -ENOSPC;
3742 		mutex_unlock(&var_table->bitmap_lock);
3743 		goto end;
3744 	}
3745 
3746 	set_bit(page_idx, var_table->bitmap);
3747 	mutex_unlock(&var_table->bitmap_lock);
3748 
3749 	entry->address = var_table->hw_start_addr +
3750 				(page_idx * var_table->stride_size);
3751 	entry->page_idx = page_idx;
3752 	entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3753 
3754 	err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3755 					       var_table->stride_size);
3756 	if (err)
3757 		goto err_insert;
3758 
3759 	return entry;
3760 
3761 err_insert:
3762 	mutex_lock(&var_table->bitmap_lock);
3763 	clear_bit(page_idx, var_table->bitmap);
3764 	mutex_unlock(&var_table->bitmap_lock);
3765 end:
3766 	kfree(entry);
3767 	return ERR_PTR(err);
3768 }
3769 
UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)3770 static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3771 	struct uverbs_attr_bundle *attrs)
3772 {
3773 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3774 		attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3775 	struct mlx5_ib_ucontext *c;
3776 	struct mlx5_user_mmap_entry *entry;
3777 	u64 mmap_offset;
3778 	u32 length;
3779 	int err;
3780 
3781 	c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3782 	if (IS_ERR(c))
3783 		return PTR_ERR(c);
3784 
3785 	entry = alloc_var_entry(c);
3786 	if (IS_ERR(entry))
3787 		return PTR_ERR(entry);
3788 
3789 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
3790 	length = entry->rdma_entry.npages * PAGE_SIZE;
3791 	uobj->object = entry;
3792 	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3793 
3794 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3795 			     &mmap_offset, sizeof(mmap_offset));
3796 	if (err)
3797 		return err;
3798 
3799 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3800 			     &entry->page_idx, sizeof(entry->page_idx));
3801 	if (err)
3802 		return err;
3803 
3804 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3805 			     &length, sizeof(length));
3806 	return err;
3807 }
3808 
3809 DECLARE_UVERBS_NAMED_METHOD(
3810 	MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3811 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3812 			MLX5_IB_OBJECT_VAR,
3813 			UVERBS_ACCESS_NEW,
3814 			UA_MANDATORY),
3815 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3816 			   UVERBS_ATTR_TYPE(u32),
3817 			   UA_MANDATORY),
3818 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3819 			   UVERBS_ATTR_TYPE(u32),
3820 			   UA_MANDATORY),
3821 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3822 			    UVERBS_ATTR_TYPE(u64),
3823 			    UA_MANDATORY));
3824 
3825 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3826 	MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3827 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3828 			MLX5_IB_OBJECT_VAR,
3829 			UVERBS_ACCESS_DESTROY,
3830 			UA_MANDATORY));
3831 
3832 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
3833 			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3834 			    &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3835 			    &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3836 
var_is_supported(struct ib_device * device)3837 static bool var_is_supported(struct ib_device *device)
3838 {
3839 	struct mlx5_ib_dev *dev = to_mdev(device);
3840 
3841 	return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3842 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3843 }
3844 
3845 static struct mlx5_user_mmap_entry *
alloc_uar_entry(struct mlx5_ib_ucontext * c,enum mlx5_ib_uapi_uar_alloc_type alloc_type)3846 alloc_uar_entry(struct mlx5_ib_ucontext *c,
3847 		enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3848 {
3849 	struct mlx5_user_mmap_entry *entry;
3850 	struct mlx5_ib_dev *dev;
3851 	u32 uar_index;
3852 	int err;
3853 
3854 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3855 	if (!entry)
3856 		return ERR_PTR(-ENOMEM);
3857 
3858 	dev = to_mdev(c->ibucontext.device);
3859 	err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
3860 	if (err)
3861 		goto end;
3862 
3863 	entry->page_idx = uar_index;
3864 	entry->address = uar_index2paddress(dev, uar_index);
3865 	if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3866 		entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3867 	else
3868 		entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3869 
3870 	err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3871 	if (err)
3872 		goto err_insert;
3873 
3874 	return entry;
3875 
3876 err_insert:
3877 	mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
3878 end:
3879 	kfree(entry);
3880 	return ERR_PTR(err);
3881 }
3882 
UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)3883 static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3884 	struct uverbs_attr_bundle *attrs)
3885 {
3886 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3887 		attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3888 	enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3889 	struct mlx5_ib_ucontext *c;
3890 	struct mlx5_user_mmap_entry *entry;
3891 	u64 mmap_offset;
3892 	u32 length;
3893 	int err;
3894 
3895 	c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3896 	if (IS_ERR(c))
3897 		return PTR_ERR(c);
3898 
3899 	err = uverbs_get_const(&alloc_type, attrs,
3900 			       MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3901 	if (err)
3902 		return err;
3903 
3904 	if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3905 	    alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3906 		return -EOPNOTSUPP;
3907 
3908 	if (!mlx5_wc_support_get(to_mdev(c->ibucontext.device)->mdev) &&
3909 	    alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3910 		return -EOPNOTSUPP;
3911 
3912 	entry = alloc_uar_entry(c, alloc_type);
3913 	if (IS_ERR(entry))
3914 		return PTR_ERR(entry);
3915 
3916 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
3917 	length = entry->rdma_entry.npages * PAGE_SIZE;
3918 	uobj->object = entry;
3919 	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3920 
3921 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3922 			     &mmap_offset, sizeof(mmap_offset));
3923 	if (err)
3924 		return err;
3925 
3926 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3927 			     &entry->page_idx, sizeof(entry->page_idx));
3928 	if (err)
3929 		return err;
3930 
3931 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3932 			     &length, sizeof(length));
3933 	return err;
3934 }
3935 
3936 DECLARE_UVERBS_NAMED_METHOD(
3937 	MLX5_IB_METHOD_UAR_OBJ_ALLOC,
3938 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
3939 			MLX5_IB_OBJECT_UAR,
3940 			UVERBS_ACCESS_NEW,
3941 			UA_MANDATORY),
3942 	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
3943 			     enum mlx5_ib_uapi_uar_alloc_type,
3944 			     UA_MANDATORY),
3945 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3946 			   UVERBS_ATTR_TYPE(u32),
3947 			   UA_MANDATORY),
3948 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3949 			   UVERBS_ATTR_TYPE(u32),
3950 			   UA_MANDATORY),
3951 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3952 			    UVERBS_ATTR_TYPE(u64),
3953 			    UA_MANDATORY));
3954 
3955 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3956 	MLX5_IB_METHOD_UAR_OBJ_DESTROY,
3957 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
3958 			MLX5_IB_OBJECT_UAR,
3959 			UVERBS_ACCESS_DESTROY,
3960 			UA_MANDATORY));
3961 
3962 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
3963 			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3964 			    &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
3965 			    &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
3966 
3967 ADD_UVERBS_ATTRIBUTES_SIMPLE(
3968 	mlx5_ib_query_context,
3969 	UVERBS_OBJECT_DEVICE,
3970 	UVERBS_METHOD_QUERY_CONTEXT,
3971 	UVERBS_ATTR_PTR_OUT(
3972 		MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
3973 		UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
3974 				   dump_fill_mkey),
3975 		UA_MANDATORY));
3976 
3977 ADD_UVERBS_ATTRIBUTES_SIMPLE(
3978 	mlx5_ib_reg_dmabuf_mr,
3979 	UVERBS_OBJECT_MR,
3980 	UVERBS_METHOD_REG_DMABUF_MR,
3981 	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,
3982 			     enum mlx5_ib_uapi_reg_dmabuf_flags,
3983 			     UA_OPTIONAL));
3984 
3985 static const struct uapi_definition mlx5_ib_defs[] = {
3986 	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
3987 	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
3988 	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
3989 	UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
3990 	UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
3991 	UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),
3992 
3993 	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
3994 	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr),
3995 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
3996 				UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
3997 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
3998 	{}
3999 };
4000 
mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev * dev)4001 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
4002 {
4003 	mlx5_ib_data_direct_cleanup(dev);
4004 	mlx5_ib_cleanup_multiport_master(dev);
4005 	WARN_ON(!xa_empty(&dev->odp_mkeys));
4006 	mutex_destroy(&dev->cap_mask_mutex);
4007 	WARN_ON(!xa_empty(&dev->sig_mrs));
4008 	WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
4009 	mlx5r_macsec_dealloc_gids(dev);
4010 }
4011 
mlx5_ib_stage_init_init(struct mlx5_ib_dev * dev)4012 static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
4013 {
4014 	struct mlx5_core_dev *mdev = dev->mdev;
4015 	int err, i;
4016 
4017 	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
4018 	dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
4019 	dev->ib_dev.dev.parent = mdev->device;
4020 	dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
4021 
4022 	for (i = 0; i < dev->num_ports; i++) {
4023 		spin_lock_init(&dev->port[i].mp.mpi_lock);
4024 		dev->port[i].roce.dev = dev;
4025 		dev->port[i].roce.native_port_num = i + 1;
4026 		dev->port[i].roce.last_port_state = IB_PORT_DOWN;
4027 	}
4028 
4029 	err = mlx5r_cmd_query_special_mkeys(dev);
4030 	if (err)
4031 		return err;
4032 
4033 	err = mlx5r_macsec_init_gids_and_devlist(dev);
4034 	if (err)
4035 		return err;
4036 
4037 	err = mlx5_ib_init_multiport_master(dev);
4038 	if (err)
4039 		goto err;
4040 
4041 	err = set_has_smi_cap(dev);
4042 	if (err)
4043 		goto err_mp;
4044 
4045 	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
4046 	if (err)
4047 		goto err_mp;
4048 
4049 	if (mlx5_use_mad_ifc(dev))
4050 		get_ext_port_caps(dev);
4051 
4052 	dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_max(mdev);
4053 
4054 	mutex_init(&dev->cap_mask_mutex);
4055 	mutex_init(&dev->data_direct_lock);
4056 	INIT_LIST_HEAD(&dev->qp_list);
4057 	spin_lock_init(&dev->reset_flow_resource_lock);
4058 	xa_init(&dev->odp_mkeys);
4059 	xa_init(&dev->sig_mrs);
4060 	atomic_set(&dev->mkey_var, 0);
4061 
4062 	spin_lock_init(&dev->dm.lock);
4063 	dev->dm.dev = mdev;
4064 	err = mlx5_ib_data_direct_init(dev);
4065 	if (err)
4066 		goto err_mp;
4067 
4068 	return 0;
4069 err_mp:
4070 	mlx5_ib_cleanup_multiport_master(dev);
4071 err:
4072 	mlx5r_macsec_dealloc_gids(dev);
4073 	return err;
4074 }
4075 
4076 static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4077 					     enum rdma_nl_dev_type type,
4078 					     const char *name);
4079 static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev);
4080 
4081 static const struct ib_device_ops mlx5_ib_dev_ops = {
4082 	.owner = THIS_MODULE,
4083 	.driver_id = RDMA_DRIVER_MLX5,
4084 	.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION,
4085 
4086 	.add_gid = mlx5_ib_add_gid,
4087 	.add_sub_dev = mlx5_ib_add_sub_dev,
4088 	.alloc_mr = mlx5_ib_alloc_mr,
4089 	.alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
4090 	.alloc_pd = mlx5_ib_alloc_pd,
4091 	.alloc_ucontext = mlx5_ib_alloc_ucontext,
4092 	.attach_mcast = mlx5_ib_mcg_attach,
4093 	.check_mr_status = mlx5_ib_check_mr_status,
4094 	.create_ah = mlx5_ib_create_ah,
4095 	.create_cq = mlx5_ib_create_cq,
4096 	.create_qp = mlx5_ib_create_qp,
4097 	.create_srq = mlx5_ib_create_srq,
4098 	.create_user_ah = mlx5_ib_create_ah,
4099 	.dealloc_pd = mlx5_ib_dealloc_pd,
4100 	.dealloc_ucontext = mlx5_ib_dealloc_ucontext,
4101 	.del_gid = mlx5_ib_del_gid,
4102 	.del_sub_dev = mlx5_ib_del_sub_dev,
4103 	.dereg_mr = mlx5_ib_dereg_mr,
4104 	.destroy_ah = mlx5_ib_destroy_ah,
4105 	.destroy_cq = mlx5_ib_destroy_cq,
4106 	.destroy_qp = mlx5_ib_destroy_qp,
4107 	.destroy_srq = mlx5_ib_destroy_srq,
4108 	.detach_mcast = mlx5_ib_mcg_detach,
4109 	.disassociate_ucontext = mlx5_ib_disassociate_ucontext,
4110 	.drain_rq = mlx5_ib_drain_rq,
4111 	.drain_sq = mlx5_ib_drain_sq,
4112 	.device_group = &mlx5_attr_group,
4113 	.get_dev_fw_str = get_dev_fw_str,
4114 	.get_dma_mr = mlx5_ib_get_dma_mr,
4115 	.get_link_layer = mlx5_ib_port_link_layer,
4116 	.map_mr_sg = mlx5_ib_map_mr_sg,
4117 	.map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
4118 	.mmap = mlx5_ib_mmap,
4119 	.mmap_free = mlx5_ib_mmap_free,
4120 	.modify_cq = mlx5_ib_modify_cq,
4121 	.modify_device = mlx5_ib_modify_device,
4122 	.modify_port = mlx5_ib_modify_port,
4123 	.modify_qp = mlx5_ib_modify_qp,
4124 	.modify_srq = mlx5_ib_modify_srq,
4125 	.poll_cq = mlx5_ib_poll_cq,
4126 	.post_recv = mlx5_ib_post_recv_nodrain,
4127 	.post_send = mlx5_ib_post_send_nodrain,
4128 	.post_srq_recv = mlx5_ib_post_srq_recv,
4129 	.process_mad = mlx5_ib_process_mad,
4130 	.query_ah = mlx5_ib_query_ah,
4131 	.query_device = mlx5_ib_query_device,
4132 	.query_gid = mlx5_ib_query_gid,
4133 	.query_pkey = mlx5_ib_query_pkey,
4134 	.query_qp = mlx5_ib_query_qp,
4135 	.query_srq = mlx5_ib_query_srq,
4136 	.query_ucontext = mlx5_ib_query_ucontext,
4137 	.reg_user_mr = mlx5_ib_reg_user_mr,
4138 	.reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
4139 	.req_notify_cq = mlx5_ib_arm_cq,
4140 	.rereg_user_mr = mlx5_ib_rereg_user_mr,
4141 	.resize_cq = mlx5_ib_resize_cq,
4142 	.ufile_hw_cleanup = mlx5_ib_ufile_hw_cleanup,
4143 
4144 	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
4145 	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
4146 	INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
4147 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
4148 	INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
4149 	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
4150 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
4151 };
4152 
4153 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
4154 	.rdma_netdev_get_params = mlx5_ib_rn_get_params,
4155 };
4156 
4157 static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
4158 	.get_vf_config = mlx5_ib_get_vf_config,
4159 	.get_vf_guid = mlx5_ib_get_vf_guid,
4160 	.get_vf_stats = mlx5_ib_get_vf_stats,
4161 	.set_vf_guid = mlx5_ib_set_vf_guid,
4162 	.set_vf_link_state = mlx5_ib_set_vf_link_state,
4163 };
4164 
4165 static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
4166 	.alloc_mw = mlx5_ib_alloc_mw,
4167 	.dealloc_mw = mlx5_ib_dealloc_mw,
4168 
4169 	INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
4170 };
4171 
4172 static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
4173 	.alloc_xrcd = mlx5_ib_alloc_xrcd,
4174 	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,
4175 
4176 	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
4177 };
4178 
mlx5_ib_init_var_table(struct mlx5_ib_dev * dev)4179 static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4180 {
4181 	struct mlx5_core_dev *mdev = dev->mdev;
4182 	struct mlx5_var_table *var_table = &dev->var_table;
4183 	u8 log_doorbell_bar_size;
4184 	u8 log_doorbell_stride;
4185 	u64 bar_size;
4186 
4187 	log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4188 					log_doorbell_bar_size);
4189 	log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4190 					log_doorbell_stride);
4191 	var_table->hw_start_addr = dev->mdev->bar_addr +
4192 				MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
4193 					doorbell_bar_offset);
4194 	bar_size = (1ULL << log_doorbell_bar_size) * 4096;
4195 	var_table->stride_size = 1ULL << log_doorbell_stride;
4196 	var_table->num_var_hw_entries = div_u64(bar_size,
4197 						var_table->stride_size);
4198 	mutex_init(&var_table->bitmap_lock);
4199 	var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
4200 					  GFP_KERNEL);
4201 	return (var_table->bitmap) ? 0 : -ENOMEM;
4202 }
4203 
mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev * dev)4204 static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
4205 {
4206 	bitmap_free(dev->var_table.bitmap);
4207 }
4208 
mlx5_ib_stage_caps_init(struct mlx5_ib_dev * dev)4209 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
4210 {
4211 	struct mlx5_core_dev *mdev = dev->mdev;
4212 	int err;
4213 
4214 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
4215 	    IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
4216 		ib_set_device_ops(&dev->ib_dev,
4217 				  &mlx5_ib_dev_ipoib_enhanced_ops);
4218 
4219 	if (mlx5_core_is_pf(mdev))
4220 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
4221 
4222 	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4223 
4224 	if (MLX5_CAP_GEN(mdev, imaicl))
4225 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
4226 
4227 	if (MLX5_CAP_GEN(mdev, xrc))
4228 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
4229 
4230 	if (MLX5_CAP_DEV_MEM(mdev, memic) ||
4231 	    MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4232 	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
4233 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
4234 
4235 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
4236 
4237 	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
4238 		dev->ib_dev.driver_def = mlx5_ib_defs;
4239 
4240 	err = init_node_data(dev);
4241 	if (err)
4242 		return err;
4243 
4244 	if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4245 	    (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
4246 	     MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
4247 		mutex_init(&dev->lb.mutex);
4248 
4249 	if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4250 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
4251 		err = mlx5_ib_init_var_table(dev);
4252 		if (err)
4253 			return err;
4254 	}
4255 
4256 	dev->ib_dev.use_cq_dim = true;
4257 
4258 	return 0;
4259 }
4260 
4261 static const struct ib_device_ops mlx5_ib_dev_port_ops = {
4262 	.get_port_immutable = mlx5_port_immutable,
4263 	.query_port = mlx5_ib_query_port,
4264 };
4265 
mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev * dev)4266 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
4267 {
4268 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
4269 	return 0;
4270 }
4271 
4272 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
4273 	.get_port_immutable = mlx5_port_rep_immutable,
4274 	.query_port = mlx5_ib_rep_query_port,
4275 	.query_pkey = mlx5_ib_rep_query_pkey,
4276 };
4277 
mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev * dev)4278 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
4279 {
4280 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
4281 	return 0;
4282 }
4283 
4284 static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
4285 	.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
4286 	.create_wq = mlx5_ib_create_wq,
4287 	.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
4288 	.destroy_wq = mlx5_ib_destroy_wq,
4289 	.modify_wq = mlx5_ib_modify_wq,
4290 
4291 	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
4292 			   ib_rwq_ind_tbl),
4293 };
4294 
mlx5_ib_roce_init(struct mlx5_ib_dev * dev)4295 static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
4296 {
4297 	struct mlx5_core_dev *mdev = dev->mdev;
4298 	enum rdma_link_layer ll;
4299 	int port_type_cap;
4300 	u32 port_num = 0;
4301 	int err;
4302 
4303 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4304 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4305 
4306 	if (ll == IB_LINK_LAYER_ETHERNET) {
4307 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
4308 
4309 		port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4310 
4311 		/* Register only for native ports */
4312 		mlx5_mdev_netdev_track(dev, port_num);
4313 
4314 		err = mlx5_enable_eth(dev);
4315 		if (err)
4316 			goto cleanup;
4317 	}
4318 
4319 	return 0;
4320 cleanup:
4321 	mlx5_mdev_netdev_untrack(dev, port_num);
4322 	return err;
4323 }
4324 
mlx5_ib_roce_cleanup(struct mlx5_ib_dev * dev)4325 static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
4326 {
4327 	struct mlx5_core_dev *mdev = dev->mdev;
4328 	enum rdma_link_layer ll;
4329 	int port_type_cap;
4330 	u32 port_num;
4331 
4332 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4333 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4334 
4335 	if (ll == IB_LINK_LAYER_ETHERNET) {
4336 		mlx5_disable_eth(dev);
4337 
4338 		port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4339 		mlx5_mdev_netdev_untrack(dev, port_num);
4340 	}
4341 }
4342 
mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev * dev)4343 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
4344 {
4345 	mlx5_ib_init_cong_debugfs(dev,
4346 				  mlx5_core_native_port_num(dev->mdev) - 1);
4347 	return 0;
4348 }
4349 
mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev * dev)4350 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
4351 {
4352 	mlx5_ib_cleanup_cong_debugfs(dev,
4353 				     mlx5_core_native_port_num(dev->mdev) - 1);
4354 }
4355 
mlx5_ib_stage_uar_init(struct mlx5_ib_dev * dev)4356 static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
4357 {
4358 	dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
4359 	return PTR_ERR_OR_ZERO(dev->mdev->priv.uar);
4360 }
4361 
mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev * dev)4362 static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
4363 {
4364 	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
4365 }
4366 
mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev * dev)4367 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
4368 {
4369 	int err;
4370 
4371 	err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4372 	if (err)
4373 		return err;
4374 
4375 	err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4376 	if (err)
4377 		mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4378 
4379 	return err;
4380 }
4381 
mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev * dev)4382 static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
4383 {
4384 	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4385 	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4386 }
4387 
mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev * dev)4388 static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
4389 {
4390 	const char *name;
4391 
4392 	if (dev->sub_dev_name) {
4393 		name = dev->sub_dev_name;
4394 		ib_mark_name_assigned_by_user(&dev->ib_dev);
4395 	} else if (!mlx5_lag_is_active(dev->mdev))
4396 		name = "mlx5_%d";
4397 	else
4398 		name = "mlx5_bond_%d";
4399 	return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
4400 }
4401 
mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev * dev)4402 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
4403 {
4404 	mlx5_mkey_cache_cleanup(dev);
4405 	mlx5r_umr_resource_cleanup(dev);
4406 	mlx5r_umr_cleanup(dev);
4407 }
4408 
mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev * dev)4409 static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4410 {
4411 	ib_unregister_device(&dev->ib_dev);
4412 }
4413 
mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev * dev)4414 static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
4415 {
4416 	int ret;
4417 
4418 	ret = mlx5r_umr_init(dev);
4419 	if (ret)
4420 		return ret;
4421 
4422 	ret = mlx5_mkey_cache_init(dev);
4423 	if (ret)
4424 		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4425 	return ret;
4426 }
4427 
mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev * dev)4428 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4429 {
4430 	struct dentry *root;
4431 
4432 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4433 		return 0;
4434 
4435 	mutex_init(&dev->delay_drop.lock);
4436 	dev->delay_drop.dev = dev;
4437 	dev->delay_drop.activate = false;
4438 	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4439 	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4440 	atomic_set(&dev->delay_drop.rqs_cnt, 0);
4441 	atomic_set(&dev->delay_drop.events_cnt, 0);
4442 
4443 	if (!mlx5_debugfs_root)
4444 		return 0;
4445 
4446 	root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
4447 	dev->delay_drop.dir_debugfs = root;
4448 
4449 	debugfs_create_atomic_t("num_timeout_events", 0400, root,
4450 				&dev->delay_drop.events_cnt);
4451 	debugfs_create_atomic_t("num_rqs", 0400, root,
4452 				&dev->delay_drop.rqs_cnt);
4453 	debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4454 			    &fops_delay_drop_timeout);
4455 	return 0;
4456 }
4457 
mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev * dev)4458 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4459 {
4460 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4461 		return;
4462 
4463 	cancel_work_sync(&dev->delay_drop.delay_drop_work);
4464 	if (!dev->delay_drop.dir_debugfs)
4465 		return;
4466 
4467 	debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4468 	dev->delay_drop.dir_debugfs = NULL;
4469 }
4470 
mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev * dev)4471 static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4472 {
4473 	struct mlx5_ib_resources *devr = &dev->devr;
4474 	int port;
4475 
4476 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4477 		INIT_WORK(&devr->ports[port].pkey_change_work,
4478 			  pkey_change_handler);
4479 
4480 	dev->mdev_events.notifier_call = mlx5_ib_event;
4481 	mlx5_notifier_register(dev->mdev, &dev->mdev_events);
4482 
4483 	mlx5r_macsec_event_register(dev);
4484 
4485 	return 0;
4486 }
4487 
mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev * dev)4488 static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4489 {
4490 	struct mlx5_ib_resources *devr = &dev->devr;
4491 	int port;
4492 
4493 	mlx5r_macsec_event_unregister(dev);
4494 	mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
4495 
4496 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4497 		cancel_work_sync(&devr->ports[port].pkey_change_work);
4498 }
4499 
mlx5_ib_data_direct_bind(struct mlx5_ib_dev * ibdev,struct mlx5_data_direct_dev * dev)4500 void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
4501 			      struct mlx5_data_direct_dev *dev)
4502 {
4503 	mutex_lock(&ibdev->data_direct_lock);
4504 	ibdev->data_direct_dev = dev;
4505 	mutex_unlock(&ibdev->data_direct_lock);
4506 }
4507 
mlx5_ib_data_direct_unbind(struct mlx5_ib_dev * ibdev)4508 void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
4509 {
4510 	mutex_lock(&ibdev->data_direct_lock);
4511 	mlx5_ib_revoke_data_direct_mrs(ibdev);
4512 	ibdev->data_direct_dev = NULL;
4513 	mutex_unlock(&ibdev->data_direct_lock);
4514 }
4515 
__mlx5_ib_remove(struct mlx5_ib_dev * dev,const struct mlx5_ib_profile * profile,int stage)4516 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4517 		      const struct mlx5_ib_profile *profile,
4518 		      int stage)
4519 {
4520 	dev->ib_active = false;
4521 
4522 	/* Number of stages to cleanup */
4523 	while (stage) {
4524 		stage--;
4525 		if (profile->stage[stage].cleanup)
4526 			profile->stage[stage].cleanup(dev);
4527 	}
4528 
4529 	kfree(dev->port);
4530 	ib_dealloc_device(&dev->ib_dev);
4531 }
4532 
__mlx5_ib_add(struct mlx5_ib_dev * dev,const struct mlx5_ib_profile * profile)4533 int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4534 		  const struct mlx5_ib_profile *profile)
4535 {
4536 	int err;
4537 	int i;
4538 
4539 	dev->profile = profile;
4540 
4541 	for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4542 		if (profile->stage[i].init) {
4543 			err = profile->stage[i].init(dev);
4544 			if (err)
4545 				goto err_out;
4546 		}
4547 	}
4548 
4549 	dev->ib_active = true;
4550 	return 0;
4551 
4552 err_out:
4553 	/* Clean up stages which were initialized */
4554 	while (i) {
4555 		i--;
4556 		if (profile->stage[i].cleanup)
4557 			profile->stage[i].cleanup(dev);
4558 	}
4559 	return -ENOMEM;
4560 }
4561 
4562 static const struct mlx5_ib_profile pf_profile = {
4563 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
4564 		     mlx5_ib_stage_init_init,
4565 		     mlx5_ib_stage_init_cleanup),
4566 	STAGE_CREATE(MLX5_IB_STAGE_FS,
4567 		     mlx5_ib_fs_init,
4568 		     mlx5_ib_fs_cleanup),
4569 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4570 		     mlx5_ib_stage_caps_init,
4571 		     mlx5_ib_stage_caps_cleanup),
4572 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4573 		     mlx5_ib_stage_non_default_cb,
4574 		     NULL),
4575 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4576 		     mlx5_ib_roce_init,
4577 		     mlx5_ib_roce_cleanup),
4578 	STAGE_CREATE(MLX5_IB_STAGE_QP,
4579 		     mlx5_init_qp_table,
4580 		     mlx5_cleanup_qp_table),
4581 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4582 		     mlx5_init_srq_table,
4583 		     mlx5_cleanup_srq_table),
4584 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4585 		     mlx5_ib_dev_res_init,
4586 		     mlx5_ib_dev_res_cleanup),
4587 	STAGE_CREATE(MLX5_IB_STAGE_ODP,
4588 		     mlx5_ib_odp_init_one,
4589 		     mlx5_ib_odp_cleanup_one),
4590 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4591 		     mlx5_ib_counters_init,
4592 		     mlx5_ib_counters_cleanup),
4593 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4594 		     mlx5_ib_stage_cong_debugfs_init,
4595 		     mlx5_ib_stage_cong_debugfs_cleanup),
4596 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
4597 		     mlx5_ib_stage_uar_init,
4598 		     mlx5_ib_stage_uar_cleanup),
4599 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4600 		     mlx5_ib_stage_bfrag_init,
4601 		     mlx5_ib_stage_bfrag_cleanup),
4602 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4603 		     NULL,
4604 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4605 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4606 		     mlx5_ib_devx_init,
4607 		     mlx5_ib_devx_cleanup),
4608 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4609 		     mlx5_ib_stage_ib_reg_init,
4610 		     mlx5_ib_stage_ib_reg_cleanup),
4611 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4612 		     mlx5_ib_stage_dev_notifier_init,
4613 		     mlx5_ib_stage_dev_notifier_cleanup),
4614 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4615 		     mlx5_ib_stage_post_ib_reg_umr_init,
4616 		     NULL),
4617 	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4618 		     mlx5_ib_stage_delay_drop_init,
4619 		     mlx5_ib_stage_delay_drop_cleanup),
4620 	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4621 		     mlx5_ib_restrack_init,
4622 		     NULL),
4623 };
4624 
4625 const struct mlx5_ib_profile raw_eth_profile = {
4626 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
4627 		     mlx5_ib_stage_init_init,
4628 		     mlx5_ib_stage_init_cleanup),
4629 	STAGE_CREATE(MLX5_IB_STAGE_FS,
4630 		     mlx5_ib_fs_init,
4631 		     mlx5_ib_fs_cleanup),
4632 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4633 		     mlx5_ib_stage_caps_init,
4634 		     mlx5_ib_stage_caps_cleanup),
4635 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4636 		     mlx5_ib_stage_raw_eth_non_default_cb,
4637 		     NULL),
4638 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4639 		     mlx5_ib_roce_init,
4640 		     mlx5_ib_roce_cleanup),
4641 	STAGE_CREATE(MLX5_IB_STAGE_QP,
4642 		     mlx5_init_qp_table,
4643 		     mlx5_cleanup_qp_table),
4644 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4645 		     mlx5_init_srq_table,
4646 		     mlx5_cleanup_srq_table),
4647 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4648 		     mlx5_ib_dev_res_init,
4649 		     mlx5_ib_dev_res_cleanup),
4650 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4651 		     mlx5_ib_counters_init,
4652 		     mlx5_ib_counters_cleanup),
4653 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4654 		     mlx5_ib_stage_cong_debugfs_init,
4655 		     mlx5_ib_stage_cong_debugfs_cleanup),
4656 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
4657 		     mlx5_ib_stage_uar_init,
4658 		     mlx5_ib_stage_uar_cleanup),
4659 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4660 		     mlx5_ib_stage_bfrag_init,
4661 		     mlx5_ib_stage_bfrag_cleanup),
4662 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4663 		     NULL,
4664 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4665 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4666 		     mlx5_ib_devx_init,
4667 		     mlx5_ib_devx_cleanup),
4668 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4669 		     mlx5_ib_stage_ib_reg_init,
4670 		     mlx5_ib_stage_ib_reg_cleanup),
4671 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4672 		     mlx5_ib_stage_dev_notifier_init,
4673 		     mlx5_ib_stage_dev_notifier_cleanup),
4674 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4675 		     mlx5_ib_stage_post_ib_reg_umr_init,
4676 		     NULL),
4677 	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4678 		     mlx5_ib_stage_delay_drop_init,
4679 		     mlx5_ib_stage_delay_drop_cleanup),
4680 	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4681 		     mlx5_ib_restrack_init,
4682 		     NULL),
4683 };
4684 
4685 static const struct mlx5_ib_profile plane_profile = {
4686 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
4687 		     mlx5_ib_stage_init_init,
4688 		     mlx5_ib_stage_init_cleanup),
4689 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4690 		     mlx5_ib_stage_caps_init,
4691 		     mlx5_ib_stage_caps_cleanup),
4692 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4693 		     mlx5_ib_stage_non_default_cb,
4694 		     NULL),
4695 	STAGE_CREATE(MLX5_IB_STAGE_QP,
4696 		     mlx5_init_qp_table,
4697 		     mlx5_cleanup_qp_table),
4698 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4699 		     mlx5_init_srq_table,
4700 		     mlx5_cleanup_srq_table),
4701 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4702 		     mlx5_ib_dev_res_init,
4703 		     mlx5_ib_dev_res_cleanup),
4704 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4705 		     mlx5_ib_stage_bfrag_init,
4706 		     mlx5_ib_stage_bfrag_cleanup),
4707 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4708 		     mlx5_ib_stage_ib_reg_init,
4709 		     mlx5_ib_stage_ib_reg_cleanup),
4710 };
4711 
mlx5_ib_add_sub_dev(struct ib_device * parent,enum rdma_nl_dev_type type,const char * name)4712 static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4713 					     enum rdma_nl_dev_type type,
4714 					     const char *name)
4715 {
4716 	struct mlx5_ib_dev *mparent = to_mdev(parent), *mplane;
4717 	enum rdma_link_layer ll;
4718 	int ret;
4719 
4720 	if (mparent->smi_dev)
4721 		return ERR_PTR(-EEXIST);
4722 
4723 	ll = mlx5_port_type_cap_to_rdma_ll(MLX5_CAP_GEN(mparent->mdev,
4724 							port_type));
4725 	if (type != RDMA_DEVICE_TYPE_SMI || !mparent->num_plane ||
4726 	    ll != IB_LINK_LAYER_INFINIBAND ||
4727 	    !MLX5_CAP_GEN_2(mparent->mdev, multiplane_qp_ud))
4728 		return ERR_PTR(-EOPNOTSUPP);
4729 
4730 	mplane = ib_alloc_device(mlx5_ib_dev, ib_dev);
4731 	if (!mplane)
4732 		return ERR_PTR(-ENOMEM);
4733 
4734 	mplane->port = kcalloc(mparent->num_plane * mparent->num_ports,
4735 			       sizeof(*mplane->port), GFP_KERNEL);
4736 	if (!mplane->port) {
4737 		ret = -ENOMEM;
4738 		goto fail_kcalloc;
4739 	}
4740 
4741 	mplane->ib_dev.type = type;
4742 	mplane->mdev = mparent->mdev;
4743 	mplane->num_ports = mparent->num_plane;
4744 	mplane->sub_dev_name = name;
4745 	mplane->ib_dev.phys_port_cnt = mplane->num_ports;
4746 
4747 	ret = __mlx5_ib_add(mplane, &plane_profile);
4748 	if (ret)
4749 		goto fail_ib_add;
4750 
4751 	mparent->smi_dev = mplane;
4752 	return &mplane->ib_dev;
4753 
4754 fail_ib_add:
4755 	kfree(mplane->port);
4756 fail_kcalloc:
4757 	ib_dealloc_device(&mplane->ib_dev);
4758 	return ERR_PTR(ret);
4759 }
4760 
mlx5_ib_del_sub_dev(struct ib_device * sub_dev)4761 static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev)
4762 {
4763 	struct mlx5_ib_dev *mdev = to_mdev(sub_dev);
4764 
4765 	to_mdev(sub_dev->parent)->smi_dev = NULL;
4766 	__mlx5_ib_remove(mdev, mdev->profile, MLX5_IB_STAGE_MAX);
4767 }
4768 
mlx5r_mp_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)4769 static int mlx5r_mp_probe(struct auxiliary_device *adev,
4770 			  const struct auxiliary_device_id *id)
4771 {
4772 	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4773 	struct mlx5_core_dev *mdev = idev->mdev;
4774 	struct mlx5_ib_multiport_info *mpi;
4775 	struct mlx5_ib_dev *dev;
4776 	bool bound = false;
4777 	int err;
4778 
4779 	mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4780 	if (!mpi)
4781 		return -ENOMEM;
4782 
4783 	mpi->mdev = mdev;
4784 	err = mlx5_query_nic_vport_system_image_guid(mdev,
4785 						     &mpi->sys_image_guid);
4786 	if (err) {
4787 		kfree(mpi);
4788 		return err;
4789 	}
4790 
4791 	mutex_lock(&mlx5_ib_multiport_mutex);
4792 	list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
4793 		if (dev->sys_image_guid == mpi->sys_image_guid &&
4794 		    mlx5_core_same_coredev_type(dev->mdev, mpi->mdev))
4795 			bound = mlx5_ib_bind_slave_port(dev, mpi);
4796 
4797 		if (bound) {
4798 			rdma_roce_rescan_device(&dev->ib_dev);
4799 			mpi->ibdev->ib_active = true;
4800 			break;
4801 		}
4802 	}
4803 
4804 	if (!bound) {
4805 		list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4806 		dev_dbg(mdev->device,
4807 			"no suitable IB device found to bind to, added to unaffiliated list.\n");
4808 	}
4809 	mutex_unlock(&mlx5_ib_multiport_mutex);
4810 
4811 	auxiliary_set_drvdata(adev, mpi);
4812 	return 0;
4813 }
4814 
mlx5r_mp_remove(struct auxiliary_device * adev)4815 static void mlx5r_mp_remove(struct auxiliary_device *adev)
4816 {
4817 	struct mlx5_ib_multiport_info *mpi;
4818 
4819 	mpi = auxiliary_get_drvdata(adev);
4820 	mutex_lock(&mlx5_ib_multiport_mutex);
4821 	if (mpi->ibdev)
4822 		mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
4823 	else
4824 		list_del(&mpi->list);
4825 	mutex_unlock(&mlx5_ib_multiport_mutex);
4826 	kfree(mpi);
4827 }
4828 
mlx5r_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)4829 static int mlx5r_probe(struct auxiliary_device *adev,
4830 		       const struct auxiliary_device_id *id)
4831 {
4832 	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4833 	struct mlx5_core_dev *mdev = idev->mdev;
4834 	const struct mlx5_ib_profile *profile;
4835 	int port_type_cap, num_ports, ret;
4836 	enum rdma_link_layer ll;
4837 	struct mlx5_ib_dev *dev;
4838 
4839 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4840 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4841 
4842 	num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4843 			MLX5_CAP_GEN(mdev, num_vhca_ports));
4844 	dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
4845 	if (!dev)
4846 		return -ENOMEM;
4847 
4848 	if (ll == IB_LINK_LAYER_INFINIBAND) {
4849 		ret = mlx5_ib_get_plane_num(mdev, &dev->num_plane);
4850 		if (ret)
4851 			goto fail;
4852 	}
4853 
4854 	dev->port = kcalloc(num_ports, sizeof(*dev->port),
4855 			     GFP_KERNEL);
4856 	if (!dev->port) {
4857 		ret = -ENOMEM;
4858 		goto fail;
4859 	}
4860 
4861 	dev->mdev = mdev;
4862 	dev->num_ports = num_ports;
4863 	dev->ib_dev.phys_port_cnt = num_ports;
4864 
4865 	if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
4866 		profile = &raw_eth_profile;
4867 	else
4868 		profile = &pf_profile;
4869 
4870 	ret = __mlx5_ib_add(dev, profile);
4871 	if (ret)
4872 		goto fail_ib_add;
4873 
4874 	auxiliary_set_drvdata(adev, dev);
4875 	return 0;
4876 
4877 fail_ib_add:
4878 	kfree(dev->port);
4879 fail:
4880 	ib_dealloc_device(&dev->ib_dev);
4881 	return ret;
4882 }
4883 
mlx5r_remove(struct auxiliary_device * adev)4884 static void mlx5r_remove(struct auxiliary_device *adev)
4885 {
4886 	struct mlx5_ib_dev *dev;
4887 
4888 	dev = auxiliary_get_drvdata(adev);
4889 	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4890 }
4891 
4892 static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4893 	{ .name = MLX5_ADEV_NAME ".multiport", },
4894 	{},
4895 };
4896 
4897 static const struct auxiliary_device_id mlx5r_id_table[] = {
4898 	{ .name = MLX5_ADEV_NAME ".rdma", },
4899 	{},
4900 };
4901 
4902 MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
4903 MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
4904 
4905 static struct auxiliary_driver mlx5r_mp_driver = {
4906 	.name = "multiport",
4907 	.probe = mlx5r_mp_probe,
4908 	.remove = mlx5r_mp_remove,
4909 	.id_table = mlx5r_mp_id_table,
4910 };
4911 
4912 static struct auxiliary_driver mlx5r_driver = {
4913 	.name = "rdma",
4914 	.probe = mlx5r_probe,
4915 	.remove = mlx5r_remove,
4916 	.id_table = mlx5r_id_table,
4917 };
4918 
mlx5_ib_init(void)4919 static int __init mlx5_ib_init(void)
4920 {
4921 	int ret;
4922 
4923 	xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
4924 	if (!xlt_emergency_page)
4925 		return -ENOMEM;
4926 
4927 	mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
4928 	if (!mlx5_ib_event_wq) {
4929 		free_page((unsigned long)xlt_emergency_page);
4930 		return -ENOMEM;
4931 	}
4932 
4933 	ret = mlx5_ib_qp_event_init();
4934 	if (ret)
4935 		goto qp_event_err;
4936 
4937 	mlx5_ib_odp_init();
4938 	ret = mlx5r_rep_init();
4939 	if (ret)
4940 		goto rep_err;
4941 	ret = mlx5_data_direct_driver_register();
4942 	if (ret)
4943 		goto dd_err;
4944 	ret = auxiliary_driver_register(&mlx5r_mp_driver);
4945 	if (ret)
4946 		goto mp_err;
4947 	ret = auxiliary_driver_register(&mlx5r_driver);
4948 	if (ret)
4949 		goto drv_err;
4950 
4951 	return 0;
4952 
4953 drv_err:
4954 	auxiliary_driver_unregister(&mlx5r_mp_driver);
4955 mp_err:
4956 	mlx5_data_direct_driver_unregister();
4957 dd_err:
4958 	mlx5r_rep_cleanup();
4959 rep_err:
4960 	mlx5_ib_qp_event_cleanup();
4961 qp_event_err:
4962 	destroy_workqueue(mlx5_ib_event_wq);
4963 	free_page((unsigned long)xlt_emergency_page);
4964 	return ret;
4965 }
4966 
mlx5_ib_cleanup(void)4967 static void __exit mlx5_ib_cleanup(void)
4968 {
4969 	mlx5_data_direct_driver_unregister();
4970 	auxiliary_driver_unregister(&mlx5r_driver);
4971 	auxiliary_driver_unregister(&mlx5r_mp_driver);
4972 	mlx5r_rep_cleanup();
4973 
4974 	mlx5_ib_qp_event_cleanup();
4975 	destroy_workqueue(mlx5_ib_event_wq);
4976 	free_page((unsigned long)xlt_emergency_page);
4977 }
4978 
4979 module_init(mlx5_ib_init);
4980 module_exit(mlx5_ib_cleanup);
4981