1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include "setup.h"
5 #include "en/params.h"
6 #include "en/txrx.h"
7 #include "en/health.h"
8 #include <net/xdp_sock_drv.h>
9
mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk)10 static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
11 struct mlx5e_params *params,
12 struct mlx5e_xsk_param *xsk)
13 {
14 if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
15 mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
16 return -EINVAL;
17 }
18
19 return 0;
20 }
21
22 /* The limitation of 2048 can be altered, but shouldn't go beyond the minimal
23 * stride size of striding RQ.
24 */
25 #define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
26
mlx5e_validate_xsk_param(struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5_core_dev * mdev)27 bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
28 struct mlx5e_xsk_param *xsk,
29 struct mlx5_core_dev *mdev)
30 {
31 /* AF_XDP doesn't support frames larger than PAGE_SIZE,
32 * and xsk->chunk_size is limited to 65535 bytes.
33 */
34 if ((size_t)xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) {
35 mlx5_core_err(mdev, "XSK chunk size %u out of bounds [%u, %lu]\n", xsk->chunk_size,
36 MLX5E_MIN_XSK_CHUNK_SIZE, PAGE_SIZE);
37 return false;
38 }
39
40 /* frag_sz is different for regular and XSK RQs, so ensure that linear
41 * SKB mode is possible.
42 */
43 switch (params->rq_wq_type) {
44 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
45 return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
46 default: /* MLX5_WQ_TYPE_CYCLIC */
47 return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
48 }
49 }
50
mlx5e_build_xsk_cparam(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct mlx5e_channel_param * cparam)51 static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
52 struct mlx5e_params *params,
53 struct mlx5e_xsk_param *xsk,
54 struct mlx5e_channel_param *cparam)
55 {
56 mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
57 mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq);
58 }
59
mlx5e_init_xsk_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct xsk_buff_pool * pool,struct mlx5e_xsk_param * xsk,struct mlx5e_rq * rq)60 static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
61 struct mlx5e_params *params,
62 struct xsk_buff_pool *pool,
63 struct mlx5e_xsk_param *xsk,
64 struct mlx5e_rq *rq)
65 {
66 struct mlx5_core_dev *mdev = c->mdev;
67 int rq_xdp_ix;
68 int err;
69
70 rq->wq_type = params->rq_wq_type;
71 rq->pdev = c->pdev;
72 rq->netdev = c->netdev;
73 rq->priv = c->priv;
74 rq->tstamp = c->tstamp;
75 rq->clock = &mdev->clock;
76 rq->icosq = &c->icosq;
77 rq->ix = c->ix;
78 rq->channel = c;
79 rq->mdev = mdev;
80 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
81 rq->xdpsq = &c->rq_xdpsq;
82 rq->xsk_pool = pool;
83 rq->stats = &c->priv->channel_stats[c->ix]->xskrq;
84 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
85 rq_xdp_ix = c->ix;
86 err = mlx5e_rq_set_handlers(rq, params, xsk);
87 if (err)
88 return err;
89
90 return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
91 }
92
mlx5e_open_xsk_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_rq_param * rq_params,struct xsk_buff_pool * pool,struct mlx5e_xsk_param * xsk)93 static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
94 struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
95 struct mlx5e_xsk_param *xsk)
96 {
97 u16 q_counter = c->priv->q_counter[c->sd_ix];
98 struct mlx5e_rq *xskrq = &c->xskrq;
99 int err;
100
101 err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
102 if (err)
103 return err;
104
105 err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
106 if (err)
107 return err;
108
109 __set_bit(MLX5E_RQ_STATE_XSK, &xskrq->state);
110 return 0;
111 }
112
mlx5e_open_xsk(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_xsk_param * xsk,struct xsk_buff_pool * pool,struct mlx5e_channel * c)113 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
114 struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
115 struct mlx5e_channel *c)
116 {
117 struct mlx5e_channel_param *cparam;
118 struct mlx5e_create_cq_param ccp;
119 int err;
120
121 mlx5e_build_create_cq_param(&ccp, c);
122
123 if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
124 return -EINVAL;
125
126 cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
127 if (!cparam)
128 return -ENOMEM;
129
130 mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
131
132 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
133 &c->xskrq.cq);
134 if (unlikely(err))
135 goto err_free_cparam;
136
137 err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
138 if (unlikely(err))
139 goto err_close_rx_cq;
140
141 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
142 &c->xsksq.cq);
143 if (unlikely(err))
144 goto err_close_rq;
145
146 /* Create a separate SQ, so that when the buff pool is disabled, we could
147 * close this SQ safely and stop receiving CQEs. In other case, e.g., if
148 * the XDPSQ was used instead, we might run into trouble when the buff pool
149 * is disabled and then re-enabled, but the SQ continues receiving CQEs
150 * from the old buff pool.
151 */
152 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, pool, &c->xsksq, true);
153 if (unlikely(err))
154 goto err_close_tx_cq;
155
156 kvfree(cparam);
157
158 set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
159
160 return 0;
161
162 err_close_tx_cq:
163 mlx5e_close_cq(&c->xsksq.cq);
164
165 err_close_rq:
166 mlx5e_close_rq(&c->xskrq);
167
168 err_close_rx_cq:
169 mlx5e_close_cq(&c->xskrq.cq);
170
171 err_free_cparam:
172 kvfree(cparam);
173
174 return err;
175 }
176
mlx5e_close_xsk(struct mlx5e_channel * c)177 void mlx5e_close_xsk(struct mlx5e_channel *c)
178 {
179 clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
180 synchronize_net(); /* Sync with NAPI. */
181
182 mlx5e_close_rq(&c->xskrq);
183 mlx5e_close_cq(&c->xskrq.cq);
184 mlx5e_close_xdpsq(&c->xsksq);
185 mlx5e_close_cq(&c->xsksq.cq);
186
187 memset(&c->xskrq, 0, sizeof(c->xskrq));
188 memset(&c->xsksq, 0, sizeof(c->xsksq));
189 }
190
mlx5e_activate_xsk(struct mlx5e_channel * c)191 void mlx5e_activate_xsk(struct mlx5e_channel *c)
192 {
193 /* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid
194 * activating XSKRQ in the middle of recovery.
195 */
196 mlx5e_reporter_icosq_suspend_recovery(c);
197 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
198 mlx5e_reporter_icosq_resume_recovery(c);
199
200 /* TX queue is created active. */
201 }
202
mlx5e_deactivate_xsk(struct mlx5e_channel * c)203 void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
204 {
205 /* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the
206 * middle of recovery. Suspend the recovery to avoid it.
207 */
208 mlx5e_reporter_icosq_suspend_recovery(c);
209 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
210 mlx5e_reporter_icosq_resume_recovery(c);
211 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
212
213 /* TX queue is disabled on close. */
214 }
215