1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 #define VALID_MR_FLAGS \ 9 (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ) 10 11 #define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE) 12 13 static enum gdma_mr_access_flags 14 mana_ib_verbs_to_gdma_access_flags(int access_flags) 15 { 16 enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ; 17 18 if (access_flags & IB_ACCESS_LOCAL_WRITE) 19 flags |= GDMA_ACCESS_FLAG_LOCAL_WRITE; 20 21 if (access_flags & IB_ACCESS_REMOTE_WRITE) 22 flags |= GDMA_ACCESS_FLAG_REMOTE_WRITE; 23 24 if (access_flags & IB_ACCESS_REMOTE_READ) 25 flags |= GDMA_ACCESS_FLAG_REMOTE_READ; 26 27 return flags; 28 } 29 30 static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr, 31 struct gdma_create_mr_params *mr_params) 32 { 33 struct gdma_create_mr_response resp = {}; 34 struct gdma_create_mr_request req = {}; 35 struct gdma_context *gc = mdev_to_gc(dev); 36 int err; 37 38 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), 39 sizeof(resp)); 40 req.pd_handle = mr_params->pd_handle; 41 req.mr_type = mr_params->mr_type; 42 43 switch (mr_params->mr_type) { 44 case GDMA_MR_TYPE_GPA: 45 break; 46 case GDMA_MR_TYPE_GVA: 47 req.gva.dma_region_handle = mr_params->gva.dma_region_handle; 48 req.gva.virtual_address = mr_params->gva.virtual_address; 49 req.gva.access_flags = mr_params->gva.access_flags; 50 break; 51 52 default: 53 ibdev_dbg(&dev->ib_dev, 54 "invalid param (GDMA_MR_TYPE) passed, type %d\n", 55 req.mr_type); 56 return -EINVAL; 57 } 58 59 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 60 61 if (err || resp.hdr.status) { 62 ibdev_dbg(&dev->ib_dev, "Failed to create mr %d, %u", err, 63 resp.hdr.status); 64 if (!err) 65 err = -EPROTO; 66 67 return err; 68 } 69 70 mr->ibmr.lkey = resp.lkey; 71 mr->ibmr.rkey = resp.rkey; 72 mr->mr_handle = resp.mr_handle; 73 74 return 0; 75 } 76 77 static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle) 78 { 79 struct gdma_destroy_mr_response resp = {}; 80 struct gdma_destroy_mr_request req = {}; 81 struct gdma_context *gc = mdev_to_gc(dev); 82 int err; 83 84 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req), 85 sizeof(resp)); 86 87 req.mr_handle = mr_handle; 88 89 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 90 if (err || resp.hdr.status) { 91 dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err, 92 resp.hdr.status); 93 if (!err) 94 err = -EPROTO; 95 return err; 96 } 97 98 return 0; 99 } 100 101 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 102 u64 iova, int access_flags, 103 struct ib_udata *udata) 104 { 105 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 106 struct gdma_create_mr_params mr_params = {}; 107 struct ib_device *ibdev = ibpd->device; 108 struct mana_ib_dev *dev; 109 struct mana_ib_mr *mr; 110 u64 dma_region_handle; 111 int err; 112 113 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 114 115 ibdev_dbg(ibdev, 116 "start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x", 117 start, iova, length, access_flags); 118 119 access_flags &= ~IB_ACCESS_OPTIONAL; 120 if (access_flags & ~VALID_MR_FLAGS) 121 return ERR_PTR(-EINVAL); 122 123 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 124 if (!mr) 125 return ERR_PTR(-ENOMEM); 126 127 mr->umem = ib_umem_get(ibdev, start, length, access_flags); 128 if (IS_ERR(mr->umem)) { 129 err = PTR_ERR(mr->umem); 130 ibdev_dbg(ibdev, 131 "Failed to get umem for register user-mr, %d\n", err); 132 goto err_free; 133 } 134 135 err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova); 136 if (err) { 137 ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n", 138 err); 139 goto err_umem; 140 } 141 142 ibdev_dbg(ibdev, 143 "created dma region for user-mr 0x%llx\n", 144 dma_region_handle); 145 146 mr_params.pd_handle = pd->pd_handle; 147 mr_params.mr_type = GDMA_MR_TYPE_GVA; 148 mr_params.gva.dma_region_handle = dma_region_handle; 149 mr_params.gva.virtual_address = iova; 150 mr_params.gva.access_flags = 151 mana_ib_verbs_to_gdma_access_flags(access_flags); 152 153 err = mana_ib_gd_create_mr(dev, mr, &mr_params); 154 if (err) 155 goto err_dma_region; 156 157 /* 158 * There is no need to keep track of dma_region_handle after MR is 159 * successfully created. The dma_region_handle is tracked in the PF 160 * as part of the lifecycle of this MR. 161 */ 162 163 return &mr->ibmr; 164 165 err_dma_region: 166 mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle); 167 168 err_umem: 169 ib_umem_release(mr->umem); 170 171 err_free: 172 kfree(mr); 173 return ERR_PTR(err); 174 } 175 176 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, 177 u64 iova, int fd, int access_flags, 178 struct uverbs_attr_bundle *attrs) 179 { 180 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 181 struct gdma_create_mr_params mr_params = {}; 182 struct ib_device *ibdev = ibpd->device; 183 struct ib_umem_dmabuf *umem_dmabuf; 184 struct mana_ib_dev *dev; 185 struct mana_ib_mr *mr; 186 u64 dma_region_handle; 187 int err; 188 189 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 190 191 access_flags &= ~IB_ACCESS_OPTIONAL; 192 if (access_flags & ~VALID_MR_FLAGS) 193 return ERR_PTR(-EOPNOTSUPP); 194 195 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 196 if (!mr) 197 return ERR_PTR(-ENOMEM); 198 199 umem_dmabuf = ib_umem_dmabuf_get_pinned(ibdev, start, length, fd, access_flags); 200 if (IS_ERR(umem_dmabuf)) { 201 err = PTR_ERR(umem_dmabuf); 202 ibdev_dbg(ibdev, "Failed to get dmabuf umem, %d\n", err); 203 goto err_free; 204 } 205 206 mr->umem = &umem_dmabuf->umem; 207 208 err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova); 209 if (err) { 210 ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n", 211 err); 212 goto err_umem; 213 } 214 215 mr_params.pd_handle = pd->pd_handle; 216 mr_params.mr_type = GDMA_MR_TYPE_GVA; 217 mr_params.gva.dma_region_handle = dma_region_handle; 218 mr_params.gva.virtual_address = iova; 219 mr_params.gva.access_flags = 220 mana_ib_verbs_to_gdma_access_flags(access_flags); 221 222 err = mana_ib_gd_create_mr(dev, mr, &mr_params); 223 if (err) 224 goto err_dma_region; 225 226 /* 227 * There is no need to keep track of dma_region_handle after MR is 228 * successfully created. The dma_region_handle is tracked in the PF 229 * as part of the lifecycle of this MR. 230 */ 231 232 return &mr->ibmr; 233 234 err_dma_region: 235 mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle); 236 237 err_umem: 238 ib_umem_release(mr->umem); 239 240 err_free: 241 kfree(mr); 242 return ERR_PTR(err); 243 } 244 245 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) 246 { 247 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 248 struct gdma_create_mr_params mr_params = {}; 249 struct ib_device *ibdev = ibpd->device; 250 struct mana_ib_dev *dev; 251 struct mana_ib_mr *mr; 252 int err; 253 254 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 255 256 if (access_flags & ~VALID_DMA_MR_FLAGS) 257 return ERR_PTR(-EINVAL); 258 259 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 260 if (!mr) 261 return ERR_PTR(-ENOMEM); 262 263 mr_params.pd_handle = pd->pd_handle; 264 mr_params.mr_type = GDMA_MR_TYPE_GPA; 265 266 err = mana_ib_gd_create_mr(dev, mr, &mr_params); 267 if (err) 268 goto err_free; 269 270 return &mr->ibmr; 271 272 err_free: 273 kfree(mr); 274 return ERR_PTR(err); 275 } 276 277 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 278 { 279 struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr); 280 struct ib_device *ibdev = ibmr->device; 281 struct mana_ib_dev *dev; 282 int err; 283 284 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 285 286 err = mana_ib_gd_destroy_mr(dev, mr->mr_handle); 287 if (err) 288 return err; 289 290 if (mr->umem) 291 ib_umem_release(mr->umem); 292 293 kfree(mr); 294 295 return 0; 296 } 297