1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 3 4 #include <linux/iova.h> 5 #include <linux/mlx5/driver.h> 6 #include <linux/moduleparam.h> 7 #include "mlx5_vdpa.h" 8 9 int mlx5_vdpa_max_iotlb_entries = 2048; 10 module_param_named(max_iotlb_entries, mlx5_vdpa_max_iotlb_entries, int, 0444); 11 MODULE_PARM_DESC(max_iotlb_entries, 12 "Maximum number of iotlb entries. (default: 2048)"); 13 14 static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid) 15 { 16 struct mlx5_core_dev *mdev = dev->mdev; 17 18 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; 19 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; 20 int err; 21 22 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD); 23 MLX5_SET(alloc_pd_in, in, uid, uid); 24 25 err = mlx5_cmd_exec_inout(mdev, alloc_pd, in, out); 26 if (!err) 27 *pdn = MLX5_GET(alloc_pd_out, out, pd); 28 29 return err; 30 } 31 32 static int dealloc_pd(struct mlx5_vdpa_dev *dev, u32 pdn, u16 uid) 33 { 34 u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; 35 struct mlx5_core_dev *mdev = dev->mdev; 36 37 MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); 38 MLX5_SET(dealloc_pd_in, in, pd, pdn); 39 MLX5_SET(dealloc_pd_in, in, uid, uid); 40 return mlx5_cmd_exec_in(mdev, dealloc_pd, in); 41 } 42 43 static int get_null_mkey(struct mlx5_vdpa_dev *dev, u32 *null_mkey) 44 { 45 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; 46 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; 47 struct mlx5_core_dev *mdev = dev->mdev; 48 int err; 49 50 MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 51 err = mlx5_cmd_exec_inout(mdev, query_special_contexts, in, out); 52 if (!err) 53 *null_mkey = MLX5_GET(query_special_contexts_out, out, null_mkey); 54 return err; 55 } 56 57 static int create_uctx(struct mlx5_vdpa_dev *mvdev, u16 *uid) 58 { 59 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {}; 60 int inlen; 61 void *in; 62 int err; 63 64 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) 65 return 0; 66 67 /* 0 means not supported */ 68 if (!MLX5_CAP_GEN(mvdev->mdev, log_max_uctx)) 69 return -EOPNOTSUPP; 70 71 inlen = MLX5_ST_SZ_BYTES(create_uctx_in); 72 in = kzalloc(inlen, GFP_KERNEL); 73 if (!in) 74 return -ENOMEM; 75 76 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX); 77 MLX5_SET(create_uctx_in, in, uctx.cap, MLX5_UCTX_CAP_RAW_TX); 78 79 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out)); 80 kfree(in); 81 if (!err) 82 *uid = MLX5_GET(create_uctx_out, out, uid); 83 84 return err; 85 } 86 87 static void destroy_uctx(struct mlx5_vdpa_dev *mvdev, u32 uid) 88 { 89 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {}; 90 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {}; 91 92 if (!uid) 93 return; 94 95 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX); 96 MLX5_SET(destroy_uctx_in, in, uid, uid); 97 98 mlx5_cmd_exec(mvdev->mdev, in, sizeof(in), out, sizeof(out)); 99 } 100 101 int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn) 102 { 103 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {}; 104 int err; 105 106 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 107 MLX5_SET(create_tis_in, in, uid, mvdev->res.uid); 108 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tis, in, out); 109 if (!err) 110 *tisn = MLX5_GET(create_tis_out, out, tisn); 111 112 return err; 113 } 114 115 void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev *mvdev, u32 tisn) 116 { 117 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; 118 119 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 120 MLX5_SET(destroy_tis_in, in, uid, mvdev->res.uid); 121 MLX5_SET(destroy_tis_in, in, tisn, tisn); 122 mlx5_cmd_exec_in(mvdev->mdev, destroy_tis, in); 123 } 124 125 int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 *rqtn) 126 { 127 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {}; 128 int err; 129 130 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); 131 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out)); 132 if (!err) 133 *rqtn = MLX5_GET(create_rqt_out, out, rqtn); 134 135 return err; 136 } 137 138 int mlx5_vdpa_modify_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 rqtn) 139 { 140 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {}; 141 142 MLX5_SET(modify_rqt_in, in, uid, mvdev->res.uid); 143 MLX5_SET(modify_rqt_in, in, rqtn, rqtn); 144 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT); 145 return mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out)); 146 } 147 148 void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev *mvdev, u32 rqtn) 149 { 150 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {}; 151 152 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); 153 MLX5_SET(destroy_rqt_in, in, uid, mvdev->res.uid); 154 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); 155 mlx5_cmd_exec_in(mvdev->mdev, destroy_rqt, in); 156 } 157 158 int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tirn) 159 { 160 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; 161 int err; 162 163 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); 164 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tir, in, out); 165 if (!err) 166 *tirn = MLX5_GET(create_tir_out, out, tirn); 167 168 return err; 169 } 170 171 void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev *mvdev, u32 tirn) 172 { 173 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; 174 175 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR); 176 MLX5_SET(destroy_tir_in, in, uid, mvdev->res.uid); 177 MLX5_SET(destroy_tir_in, in, tirn, tirn); 178 mlx5_cmd_exec_in(mvdev->mdev, destroy_tir, in); 179 } 180 181 int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn) 182 { 183 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {}; 184 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {}; 185 int err; 186 187 MLX5_SET(alloc_transport_domain_in, in, opcode, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN); 188 MLX5_SET(alloc_transport_domain_in, in, uid, mvdev->res.uid); 189 190 err = mlx5_cmd_exec_inout(mvdev->mdev, alloc_transport_domain, in, out); 191 if (!err) 192 *tdn = MLX5_GET(alloc_transport_domain_out, out, transport_domain); 193 194 return err; 195 } 196 197 void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn) 198 { 199 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {}; 200 201 MLX5_SET(dealloc_transport_domain_in, in, opcode, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 202 MLX5_SET(dealloc_transport_domain_in, in, uid, mvdev->res.uid); 203 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn); 204 mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in); 205 } 206 207 int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, u32 *mkey, u32 *in, 208 int inlen) 209 { 210 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {}; 211 u32 mkey_index; 212 int err; 213 214 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 215 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid); 216 217 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, lout, sizeof(lout)); 218 if (err) 219 return err; 220 221 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); 222 *mkey = mlx5_idx_to_mkey(mkey_index); 223 return 0; 224 } 225 226 int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey) 227 { 228 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; 229 230 MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid); 231 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); 232 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey)); 233 return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in); 234 } 235 236 static int init_ctrl_vq(struct mlx5_vdpa_dev *mvdev) 237 { 238 if (mlx5_vdpa_max_iotlb_entries < 2) 239 return -EINVAL; 240 241 mvdev->cvq.iotlb = vhost_iotlb_alloc(mlx5_vdpa_max_iotlb_entries, 0); 242 if (!mvdev->cvq.iotlb) 243 return -ENOMEM; 244 245 spin_lock_init(&mvdev->cvq.iommu_lock); 246 vringh_set_iotlb(&mvdev->cvq.vring, mvdev->cvq.iotlb, &mvdev->cvq.iommu_lock); 247 248 return 0; 249 } 250 251 static void cleanup_ctrl_vq(struct mlx5_vdpa_dev *mvdev) 252 { 253 vhost_iotlb_free(mvdev->cvq.iotlb); 254 } 255 256 int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev) 257 { 258 u64 offset = MLX5_CAP64_DEV_VDPA_EMULATION(mvdev->mdev, doorbell_bar_offset); 259 struct mlx5_vdpa_resources *res = &mvdev->res; 260 struct mlx5_core_dev *mdev = mvdev->mdev; 261 u64 kick_addr; 262 int err; 263 264 if (res->valid) { 265 mlx5_vdpa_warn(mvdev, "resources already allocated\n"); 266 return -EINVAL; 267 } 268 res->uar = mlx5_get_uars_page(mdev); 269 if (IS_ERR(res->uar)) { 270 err = PTR_ERR(res->uar); 271 goto err_uars; 272 } 273 274 err = create_uctx(mvdev, &res->uid); 275 if (err) 276 goto err_uctx; 277 278 err = alloc_pd(mvdev, &res->pdn, res->uid); 279 if (err) 280 goto err_pd; 281 282 err = get_null_mkey(mvdev, &res->null_mkey); 283 if (err) 284 goto err_key; 285 286 kick_addr = mdev->bar_addr + offset; 287 res->phys_kick_addr = kick_addr; 288 289 res->kick_addr = ioremap(kick_addr, PAGE_SIZE); 290 if (!res->kick_addr) { 291 err = -ENOMEM; 292 goto err_key; 293 } 294 295 err = init_ctrl_vq(mvdev); 296 if (err) 297 goto err_ctrl; 298 299 res->valid = true; 300 301 return 0; 302 303 err_ctrl: 304 iounmap(res->kick_addr); 305 err_key: 306 dealloc_pd(mvdev, res->pdn, res->uid); 307 err_pd: 308 destroy_uctx(mvdev, res->uid); 309 err_uctx: 310 mlx5_put_uars_page(mdev, res->uar); 311 err_uars: 312 return err; 313 } 314 315 void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev) 316 { 317 struct mlx5_vdpa_resources *res = &mvdev->res; 318 319 if (!res->valid) 320 return; 321 322 cleanup_ctrl_vq(mvdev); 323 iounmap(res->kick_addr); 324 res->kick_addr = NULL; 325 dealloc_pd(mvdev, res->pdn, res->uid); 326 destroy_uctx(mvdev, res->uid); 327 mlx5_put_uars_page(mvdev->mdev, res->uar); 328 res->valid = false; 329 } 330 331 static void virtqueue_cmd_callback(int status, struct mlx5_async_work *context) 332 { 333 struct mlx5_vdpa_async_cmd *cmd = 334 container_of(context, struct mlx5_vdpa_async_cmd, cb_work); 335 336 cmd->err = mlx5_cmd_check(context->ctx->dev, status, cmd->in, cmd->out); 337 complete(&cmd->cmd_done); 338 } 339 340 static int issue_async_cmd(struct mlx5_vdpa_dev *mvdev, 341 struct mlx5_vdpa_async_cmd *cmds, 342 int issued, 343 int *completed) 344 345 { 346 struct mlx5_vdpa_async_cmd *cmd = &cmds[issued]; 347 int err; 348 349 retry: 350 err = mlx5_cmd_exec_cb(&mvdev->async_ctx, 351 cmd->in, cmd->inlen, 352 cmd->out, cmd->outlen, 353 virtqueue_cmd_callback, 354 &cmd->cb_work); 355 if (err == -EBUSY) { 356 if (*completed < issued) { 357 /* Throttled by own commands: wait for oldest completion. */ 358 wait_for_completion(&cmds[*completed].cmd_done); 359 (*completed)++; 360 361 goto retry; 362 } else { 363 /* Throttled by external commands: switch to sync api. */ 364 err = mlx5_cmd_exec(mvdev->mdev, 365 cmd->in, cmd->inlen, 366 cmd->out, cmd->outlen); 367 if (!err) 368 (*completed)++; 369 } 370 } 371 372 return err; 373 } 374 375 int mlx5_vdpa_exec_async_cmds(struct mlx5_vdpa_dev *mvdev, 376 struct mlx5_vdpa_async_cmd *cmds, 377 int num_cmds) 378 { 379 int completed = 0; 380 int issued = 0; 381 int err = 0; 382 383 for (int i = 0; i < num_cmds; i++) 384 init_completion(&cmds[i].cmd_done); 385 386 while (issued < num_cmds) { 387 388 err = issue_async_cmd(mvdev, cmds, issued, &completed); 389 if (err) { 390 mlx5_vdpa_err(mvdev, "error issuing command %d of %d: %d\n", 391 issued, num_cmds, err); 392 break; 393 } 394 395 issued++; 396 } 397 398 while (completed < issued) 399 wait_for_completion(&cmds[completed++].cmd_done); 400 401 return err; 402 } 403