1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
3
4 #include <linux/iova.h>
5 #include <linux/mlx5/driver.h>
6 #include <linux/moduleparam.h>
7 #include "mlx5_vdpa.h"
8
9 int mlx5_vdpa_max_iotlb_entries = 2048;
10 module_param_named(max_iotlb_entries, mlx5_vdpa_max_iotlb_entries, int, 0444);
11 MODULE_PARM_DESC(max_iotlb_entries,
12 "Maximum number of iotlb entries. (default: 2048)");
13
alloc_pd(struct mlx5_vdpa_dev * dev,u32 * pdn,u16 uid)14 static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid)
15 {
16 struct mlx5_core_dev *mdev = dev->mdev;
17
18 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
19 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
20 int err;
21
22 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
23 MLX5_SET(alloc_pd_in, in, uid, uid);
24
25 err = mlx5_cmd_exec_inout(mdev, alloc_pd, in, out);
26 if (!err)
27 *pdn = MLX5_GET(alloc_pd_out, out, pd);
28
29 return err;
30 }
31
dealloc_pd(struct mlx5_vdpa_dev * dev,u32 pdn,u16 uid)32 static int dealloc_pd(struct mlx5_vdpa_dev *dev, u32 pdn, u16 uid)
33 {
34 u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};
35 struct mlx5_core_dev *mdev = dev->mdev;
36
37 MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
38 MLX5_SET(dealloc_pd_in, in, pd, pdn);
39 MLX5_SET(dealloc_pd_in, in, uid, uid);
40 return mlx5_cmd_exec_in(mdev, dealloc_pd, in);
41 }
42
get_null_mkey(struct mlx5_vdpa_dev * dev,u32 * null_mkey)43 static int get_null_mkey(struct mlx5_vdpa_dev *dev, u32 *null_mkey)
44 {
45 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
46 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
47 struct mlx5_core_dev *mdev = dev->mdev;
48 int err;
49
50 MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
51 err = mlx5_cmd_exec_inout(mdev, query_special_contexts, in, out);
52 if (!err)
53 *null_mkey = MLX5_GET(query_special_contexts_out, out, null_mkey);
54 return err;
55 }
56
create_uctx(struct mlx5_vdpa_dev * mvdev,u16 * uid)57 static int create_uctx(struct mlx5_vdpa_dev *mvdev, u16 *uid)
58 {
59 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
60 int inlen;
61 void *in;
62 int err;
63
64 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0))
65 return 0;
66
67 /* 0 means not supported */
68 if (!MLX5_CAP_GEN(mvdev->mdev, log_max_uctx))
69 return -EOPNOTSUPP;
70
71 inlen = MLX5_ST_SZ_BYTES(create_uctx_in);
72 in = kzalloc(inlen, GFP_KERNEL);
73 if (!in)
74 return -ENOMEM;
75
76 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
77 MLX5_SET(create_uctx_in, in, uctx.cap, MLX5_UCTX_CAP_RAW_TX);
78
79 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
80 kfree(in);
81 if (!err)
82 *uid = MLX5_GET(create_uctx_out, out, uid);
83
84 return err;
85 }
86
destroy_uctx(struct mlx5_vdpa_dev * mvdev,u32 uid)87 static void destroy_uctx(struct mlx5_vdpa_dev *mvdev, u32 uid)
88 {
89 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
90 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
91
92 if (!uid)
93 return;
94
95 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
96 MLX5_SET(destroy_uctx_in, in, uid, uid);
97
98 mlx5_cmd_exec(mvdev->mdev, in, sizeof(in), out, sizeof(out));
99 }
100
mlx5_vdpa_create_tis(struct mlx5_vdpa_dev * mvdev,void * in,u32 * tisn)101 int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn)
102 {
103 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
104 int err;
105
106 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
107 MLX5_SET(create_tis_in, in, uid, mvdev->res.uid);
108 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tis, in, out);
109 if (!err)
110 *tisn = MLX5_GET(create_tis_out, out, tisn);
111
112 return err;
113 }
114
mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev * mvdev,u32 tisn)115 void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev *mvdev, u32 tisn)
116 {
117 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
118
119 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
120 MLX5_SET(destroy_tis_in, in, uid, mvdev->res.uid);
121 MLX5_SET(destroy_tis_in, in, tisn, tisn);
122 mlx5_cmd_exec_in(mvdev->mdev, destroy_tis, in);
123 }
124
mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev * mvdev,void * in,int inlen,u32 * rqtn)125 int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 *rqtn)
126 {
127 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
128 int err;
129
130 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
131 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
132 if (!err)
133 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
134
135 return err;
136 }
137
mlx5_vdpa_modify_rqt(struct mlx5_vdpa_dev * mvdev,void * in,int inlen,u32 rqtn)138 int mlx5_vdpa_modify_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 rqtn)
139 {
140 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
141
142 MLX5_SET(modify_rqt_in, in, uid, mvdev->res.uid);
143 MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
144 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
145 return mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
146 }
147
mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev * mvdev,u32 rqtn)148 void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev *mvdev, u32 rqtn)
149 {
150 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
151
152 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
153 MLX5_SET(destroy_rqt_in, in, uid, mvdev->res.uid);
154 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
155 mlx5_cmd_exec_in(mvdev->mdev, destroy_rqt, in);
156 }
157
mlx5_vdpa_create_tir(struct mlx5_vdpa_dev * mvdev,void * in,u32 * tirn)158 int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tirn)
159 {
160 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
161 int err;
162
163 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
164 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tir, in, out);
165 if (!err)
166 *tirn = MLX5_GET(create_tir_out, out, tirn);
167
168 return err;
169 }
170
mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev * mvdev,u32 tirn)171 void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev *mvdev, u32 tirn)
172 {
173 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
174
175 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
176 MLX5_SET(destroy_tir_in, in, uid, mvdev->res.uid);
177 MLX5_SET(destroy_tir_in, in, tirn, tirn);
178 mlx5_cmd_exec_in(mvdev->mdev, destroy_tir, in);
179 }
180
mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev * mvdev,u32 * tdn)181 int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn)
182 {
183 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
184 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
185 int err;
186
187 MLX5_SET(alloc_transport_domain_in, in, opcode, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
188 MLX5_SET(alloc_transport_domain_in, in, uid, mvdev->res.uid);
189
190 err = mlx5_cmd_exec_inout(mvdev->mdev, alloc_transport_domain, in, out);
191 if (!err)
192 *tdn = MLX5_GET(alloc_transport_domain_out, out, transport_domain);
193
194 return err;
195 }
196
mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev * mvdev,u32 tdn)197 void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn)
198 {
199 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
200
201 MLX5_SET(dealloc_transport_domain_in, in, opcode, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
202 MLX5_SET(dealloc_transport_domain_in, in, uid, mvdev->res.uid);
203 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
204 mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in);
205 }
206
mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev * mvdev,u32 * mkey,u32 * in,int inlen)207 int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, u32 *mkey, u32 *in,
208 int inlen)
209 {
210 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {};
211 u32 mkey_index;
212 int err;
213
214 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
215 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
216
217 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, lout, sizeof(lout));
218 if (err)
219 return err;
220
221 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
222 *mkey = mlx5_idx_to_mkey(mkey_index);
223 return 0;
224 }
225
mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev * mvdev,u32 mkey)226 int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey)
227 {
228 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
229
230 MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
231 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
232 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey));
233 return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in);
234 }
235
init_ctrl_vq(struct mlx5_vdpa_dev * mvdev)236 static int init_ctrl_vq(struct mlx5_vdpa_dev *mvdev)
237 {
238 if (mlx5_vdpa_max_iotlb_entries < 2)
239 return -EINVAL;
240
241 mvdev->cvq.iotlb = vhost_iotlb_alloc(mlx5_vdpa_max_iotlb_entries, 0);
242 if (!mvdev->cvq.iotlb)
243 return -ENOMEM;
244
245 spin_lock_init(&mvdev->cvq.iommu_lock);
246 vringh_set_iotlb(&mvdev->cvq.vring, mvdev->cvq.iotlb, &mvdev->cvq.iommu_lock);
247
248 return 0;
249 }
250
cleanup_ctrl_vq(struct mlx5_vdpa_dev * mvdev)251 static void cleanup_ctrl_vq(struct mlx5_vdpa_dev *mvdev)
252 {
253 vhost_iotlb_free(mvdev->cvq.iotlb);
254 }
255
mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev * mvdev)256 int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev)
257 {
258 u64 offset = MLX5_CAP64_DEV_VDPA_EMULATION(mvdev->mdev, doorbell_bar_offset);
259 struct mlx5_vdpa_resources *res = &mvdev->res;
260 struct mlx5_core_dev *mdev = mvdev->mdev;
261 u64 kick_addr;
262 int err;
263
264 if (res->valid) {
265 mlx5_vdpa_warn(mvdev, "resources already allocated\n");
266 return -EINVAL;
267 }
268 res->uar = mlx5_get_uars_page(mdev);
269 if (IS_ERR(res->uar)) {
270 err = PTR_ERR(res->uar);
271 goto err_uars;
272 }
273
274 err = create_uctx(mvdev, &res->uid);
275 if (err)
276 goto err_uctx;
277
278 err = alloc_pd(mvdev, &res->pdn, res->uid);
279 if (err)
280 goto err_pd;
281
282 err = get_null_mkey(mvdev, &res->null_mkey);
283 if (err)
284 goto err_key;
285
286 kick_addr = mdev->bar_addr + offset;
287 res->phys_kick_addr = kick_addr;
288
289 res->kick_addr = ioremap(kick_addr, PAGE_SIZE);
290 if (!res->kick_addr) {
291 err = -ENOMEM;
292 goto err_key;
293 }
294
295 err = init_ctrl_vq(mvdev);
296 if (err)
297 goto err_ctrl;
298
299 res->valid = true;
300
301 return 0;
302
303 err_ctrl:
304 iounmap(res->kick_addr);
305 err_key:
306 dealloc_pd(mvdev, res->pdn, res->uid);
307 err_pd:
308 destroy_uctx(mvdev, res->uid);
309 err_uctx:
310 mlx5_put_uars_page(mdev, res->uar);
311 err_uars:
312 return err;
313 }
314
mlx5_vdpa_free_resources(struct mlx5_vdpa_dev * mvdev)315 void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev)
316 {
317 struct mlx5_vdpa_resources *res = &mvdev->res;
318
319 if (!res->valid)
320 return;
321
322 cleanup_ctrl_vq(mvdev);
323 iounmap(res->kick_addr);
324 res->kick_addr = NULL;
325 dealloc_pd(mvdev, res->pdn, res->uid);
326 destroy_uctx(mvdev, res->uid);
327 mlx5_put_uars_page(mvdev->mdev, res->uar);
328 res->valid = false;
329 }
330
virtqueue_cmd_callback(int status,struct mlx5_async_work * context)331 static void virtqueue_cmd_callback(int status, struct mlx5_async_work *context)
332 {
333 struct mlx5_vdpa_async_cmd *cmd =
334 container_of(context, struct mlx5_vdpa_async_cmd, cb_work);
335
336 cmd->err = mlx5_cmd_check(context->ctx->dev, status, cmd->in, cmd->out);
337 complete(&cmd->cmd_done);
338 }
339
issue_async_cmd(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_async_cmd * cmds,int issued,int * completed)340 static int issue_async_cmd(struct mlx5_vdpa_dev *mvdev,
341 struct mlx5_vdpa_async_cmd *cmds,
342 int issued,
343 int *completed)
344
345 {
346 struct mlx5_vdpa_async_cmd *cmd = &cmds[issued];
347 int err;
348
349 retry:
350 err = mlx5_cmd_exec_cb(&mvdev->async_ctx,
351 cmd->in, cmd->inlen,
352 cmd->out, cmd->outlen,
353 virtqueue_cmd_callback,
354 &cmd->cb_work);
355 if (err == -EBUSY) {
356 if (*completed < issued) {
357 /* Throttled by own commands: wait for oldest completion. */
358 wait_for_completion(&cmds[*completed].cmd_done);
359 (*completed)++;
360
361 goto retry;
362 } else {
363 /* Throttled by external commands: switch to sync api. */
364 err = mlx5_cmd_exec(mvdev->mdev,
365 cmd->in, cmd->inlen,
366 cmd->out, cmd->outlen);
367 if (!err)
368 (*completed)++;
369 }
370 }
371
372 return err;
373 }
374
mlx5_vdpa_exec_async_cmds(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_async_cmd * cmds,int num_cmds)375 int mlx5_vdpa_exec_async_cmds(struct mlx5_vdpa_dev *mvdev,
376 struct mlx5_vdpa_async_cmd *cmds,
377 int num_cmds)
378 {
379 int completed = 0;
380 int issued = 0;
381 int err = 0;
382
383 for (int i = 0; i < num_cmds; i++)
384 init_completion(&cmds[i].cmd_done);
385
386 while (issued < num_cmds) {
387
388 err = issue_async_cmd(mvdev, cmds, issued, &completed);
389 if (err) {
390 mlx5_vdpa_err(mvdev, "error issuing command %d of %d: %d\n",
391 issued, num_cmds, err);
392 break;
393 }
394
395 issued++;
396 }
397
398 while (completed < issued)
399 wait_for_completion(&cmds[completed++].cmd_done);
400
401 return err;
402 }
403