1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause 2 /* 3 * Copyright (c) 2025, Broadcom. All rights reserved. The term 4 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 5 * 6 * Description: uapi interpreter 7 */ 8 9 #include <rdma/ib_addr.h> 10 #include <rdma/uverbs_types.h> 11 #include <rdma/uverbs_std_types.h> 12 #include <rdma/ib_user_ioctl_cmds.h> 13 #define UVERBS_MODULE_NAME bnxt_re 14 #include <rdma/uverbs_named_ioctl.h> 15 #include <rdma/bnxt_re-abi.h> 16 17 #include "roce_hsi.h" 18 #include "qplib_res.h" 19 #include "qplib_sp.h" 20 #include "qplib_fp.h" 21 #include "qplib_rcfw.h" 22 #include "bnxt_re.h" 23 #include "ib_verbs.h" 24 25 static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id) 26 { 27 struct bnxt_re_cq *cq = NULL, *tmp_cq; 28 29 hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) { 30 if (tmp_cq->qplib_cq.id == cq_id) { 31 cq = tmp_cq; 32 break; 33 } 34 } 35 return cq; 36 } 37 38 static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id) 39 { 40 struct bnxt_re_srq *srq = NULL, *tmp_srq; 41 42 hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) { 43 if (tmp_srq->qplib_srq.id == srq_id) { 44 srq = tmp_srq; 45 break; 46 } 47 } 48 return srq; 49 } 50 51 static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs) 52 { 53 struct bnxt_re_ucontext *uctx; 54 struct ib_ucontext *ib_uctx; 55 56 ib_uctx = ib_uverbs_get_ucontext(attrs); 57 if (IS_ERR(ib_uctx)) 58 return PTR_ERR(ib_uctx); 59 60 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 61 if (IS_ERR(uctx)) 62 return PTR_ERR(uctx); 63 64 bnxt_re_pacing_alert(uctx->rdev); 65 return 0; 66 } 67 68 static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs) 69 { 70 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); 71 enum bnxt_re_alloc_page_type alloc_type; 72 struct bnxt_re_user_mmap_entry *entry; 73 enum bnxt_re_mmap_flag mmap_flag; 74 struct bnxt_qplib_chip_ctx *cctx; 75 struct bnxt_re_ucontext *uctx; 76 struct ib_ucontext *ib_uctx; 77 struct bnxt_re_dev *rdev; 78 u64 mmap_offset; 79 u32 length; 80 u32 dpi; 81 u64 addr; 82 int err; 83 84 ib_uctx = ib_uverbs_get_ucontext(attrs); 85 if (IS_ERR(ib_uctx)) 86 return PTR_ERR(ib_uctx); 87 88 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 89 if (IS_ERR(uctx)) 90 return PTR_ERR(uctx); 91 92 err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE); 93 if (err) 94 return err; 95 96 rdev = uctx->rdev; 97 cctx = rdev->chip_ctx; 98 99 switch (alloc_type) { 100 case BNXT_RE_ALLOC_WC_PAGE: 101 if (cctx->modes.db_push) { 102 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, 103 uctx, BNXT_QPLIB_DPI_TYPE_WC)) 104 return -ENOMEM; 105 length = PAGE_SIZE; 106 dpi = uctx->wcdpi.dpi; 107 addr = (u64)uctx->wcdpi.umdbr; 108 mmap_flag = BNXT_RE_MMAP_WC_DB; 109 } else { 110 return -EINVAL; 111 } 112 113 break; 114 case BNXT_RE_ALLOC_DBR_BAR_PAGE: 115 length = PAGE_SIZE; 116 addr = (u64)rdev->pacing.dbr_bar_addr; 117 mmap_flag = BNXT_RE_MMAP_DBR_BAR; 118 break; 119 120 case BNXT_RE_ALLOC_DBR_PAGE: 121 length = PAGE_SIZE; 122 addr = (u64)rdev->pacing.dbr_page; 123 mmap_flag = BNXT_RE_MMAP_DBR_PAGE; 124 break; 125 126 default: 127 return -EOPNOTSUPP; 128 } 129 130 entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); 131 if (!entry) 132 return -ENOMEM; 133 134 uobj->object = entry; 135 uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); 136 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET, 137 &mmap_offset, sizeof(mmap_offset)); 138 if (err) 139 return err; 140 141 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH, 142 &length, sizeof(length)); 143 if (err) 144 return err; 145 146 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI, 147 &dpi, sizeof(dpi)); 148 if (err) 149 return err; 150 151 return 0; 152 } 153 154 static int alloc_page_obj_cleanup(struct ib_uobject *uobject, 155 enum rdma_remove_reason why, 156 struct uverbs_attr_bundle *attrs) 157 { 158 struct bnxt_re_user_mmap_entry *entry = uobject->object; 159 struct bnxt_re_ucontext *uctx = entry->uctx; 160 161 switch (entry->mmap_flag) { 162 case BNXT_RE_MMAP_WC_DB: 163 if (uctx && uctx->wcdpi.dbr) { 164 struct bnxt_re_dev *rdev = uctx->rdev; 165 166 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); 167 uctx->wcdpi.dbr = NULL; 168 } 169 break; 170 case BNXT_RE_MMAP_DBR_BAR: 171 case BNXT_RE_MMAP_DBR_PAGE: 172 break; 173 default: 174 goto exit; 175 } 176 rdma_user_mmap_entry_remove(&entry->rdma_entry); 177 exit: 178 return 0; 179 } 180 181 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE, 182 UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE, 183 BNXT_RE_OBJECT_ALLOC_PAGE, 184 UVERBS_ACCESS_NEW, 185 UA_MANDATORY), 186 UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE, 187 enum bnxt_re_alloc_page_type, 188 UA_MANDATORY), 189 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET, 190 UVERBS_ATTR_TYPE(u64), 191 UA_MANDATORY), 192 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH, 193 UVERBS_ATTR_TYPE(u32), 194 UA_MANDATORY), 195 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI, 196 UVERBS_ATTR_TYPE(u32), 197 UA_MANDATORY)); 198 199 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE, 200 UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE, 201 BNXT_RE_OBJECT_ALLOC_PAGE, 202 UVERBS_ACCESS_DESTROY, 203 UA_MANDATORY)); 204 205 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE, 206 UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup), 207 &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE), 208 &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE)); 209 210 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV); 211 212 DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV, 213 &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV)); 214 215 /* Toggle MEM */ 216 static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs) 217 { 218 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE); 219 enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE; 220 enum bnxt_re_get_toggle_mem_type res_type; 221 struct bnxt_re_user_mmap_entry *entry; 222 struct bnxt_re_ucontext *uctx; 223 struct ib_ucontext *ib_uctx; 224 struct bnxt_re_dev *rdev; 225 struct bnxt_re_srq *srq; 226 u32 length = PAGE_SIZE; 227 struct bnxt_re_cq *cq; 228 u64 mem_offset; 229 u32 offset = 0; 230 u64 addr = 0; 231 u32 res_id; 232 int err; 233 234 ib_uctx = ib_uverbs_get_ucontext(attrs); 235 if (IS_ERR(ib_uctx)) 236 return PTR_ERR(ib_uctx); 237 238 err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE); 239 if (err) 240 return err; 241 242 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 243 rdev = uctx->rdev; 244 err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID); 245 if (err) 246 return err; 247 248 switch (res_type) { 249 case BNXT_RE_CQ_TOGGLE_MEM: 250 cq = bnxt_re_search_for_cq(rdev, res_id); 251 if (!cq) 252 return -EINVAL; 253 254 addr = (u64)cq->uctx_cq_page; 255 break; 256 case BNXT_RE_SRQ_TOGGLE_MEM: 257 srq = bnxt_re_search_for_srq(rdev, res_id); 258 if (!srq) 259 return -EINVAL; 260 261 addr = (u64)srq->uctx_srq_page; 262 break; 263 264 default: 265 return -EOPNOTSUPP; 266 } 267 268 entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset); 269 if (!entry) 270 return -ENOMEM; 271 272 uobj->object = entry; 273 uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE); 274 err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE, 275 &mem_offset, sizeof(mem_offset)); 276 if (err) 277 return err; 278 279 err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH, 280 &length, sizeof(length)); 281 if (err) 282 return err; 283 284 err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET, 285 &offset, sizeof(offset)); 286 if (err) 287 return err; 288 289 return 0; 290 } 291 292 static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject, 293 enum rdma_remove_reason why, 294 struct uverbs_attr_bundle *attrs) 295 { 296 struct bnxt_re_user_mmap_entry *entry = uobject->object; 297 298 rdma_user_mmap_entry_remove(&entry->rdma_entry); 299 return 0; 300 } 301 302 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM, 303 UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE, 304 BNXT_RE_OBJECT_GET_TOGGLE_MEM, 305 UVERBS_ACCESS_NEW, 306 UA_MANDATORY), 307 UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE, 308 enum bnxt_re_get_toggle_mem_type, 309 UA_MANDATORY), 310 UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID, 311 UVERBS_ATTR_TYPE(u32), 312 UA_MANDATORY), 313 UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE, 314 UVERBS_ATTR_TYPE(u64), 315 UA_MANDATORY), 316 UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET, 317 UVERBS_ATTR_TYPE(u32), 318 UA_MANDATORY), 319 UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH, 320 UVERBS_ATTR_TYPE(u32), 321 UA_MANDATORY)); 322 323 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM, 324 UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE, 325 BNXT_RE_OBJECT_GET_TOGGLE_MEM, 326 UVERBS_ACCESS_DESTROY, 327 UA_MANDATORY)); 328 329 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM, 330 UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup), 331 &UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM), 332 &UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM)); 333 334 static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *attrs) 335 { 336 struct bnxt_re_db_region dbr = {}; 337 struct bnxt_re_ucontext *uctx; 338 struct bnxt_re_dbr_obj *obj; 339 struct ib_ucontext *ib_uctx; 340 struct bnxt_qplib_dpi *dpi; 341 struct bnxt_re_dev *rdev; 342 struct ib_uobject *uobj; 343 u64 mmap_offset; 344 int ret; 345 346 ib_uctx = ib_uverbs_get_ucontext(attrs); 347 if (IS_ERR(ib_uctx)) 348 return PTR_ERR(ib_uctx); 349 350 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 351 rdev = uctx->rdev; 352 uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_DBR_HANDLE); 353 354 obj = kzalloc_obj(*obj); 355 if (!obj) 356 return -ENOMEM; 357 358 dpi = &obj->dpi; 359 ret = bnxt_qplib_alloc_uc_dpi(&rdev->qplib_res, dpi); 360 if (ret) 361 goto free_mem; 362 363 obj->entry = bnxt_re_mmap_entry_insert(uctx, dpi->umdbr, 364 BNXT_RE_MMAP_UC_DB, 365 &mmap_offset); 366 if (!obj->entry) { 367 ret = -ENOMEM; 368 goto free_dpi; 369 } 370 371 obj->rdev = rdev; 372 uobj->object = obj; 373 uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); 374 375 dbr.umdbr = dpi->umdbr; 376 dbr.dpi = dpi->dpi; 377 ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_ALLOC_DBR_ATTR, 378 &dbr, sizeof(dbr)); 379 if (ret) 380 return ret; 381 382 ret = uverbs_copy_to(attrs, BNXT_RE_ALLOC_DBR_OFFSET, 383 &mmap_offset, sizeof(mmap_offset)); 384 if (ret) 385 return ret; 386 return 0; 387 free_dpi: 388 bnxt_qplib_free_uc_dpi(&rdev->qplib_res, dpi); 389 free_mem: 390 kfree(obj); 391 return ret; 392 } 393 394 static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, 395 enum rdma_remove_reason why, 396 struct uverbs_attr_bundle *attrs) 397 { 398 struct bnxt_re_dbr_obj *obj = uobject->object; 399 struct bnxt_re_dev *rdev = obj->rdev; 400 401 rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); 402 bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); 403 return 0; 404 } 405 406 static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_DEFAULT_DBR)(struct uverbs_attr_bundle *attrs) 407 { 408 struct bnxt_re_db_region dpi = {}; 409 struct bnxt_re_ucontext *uctx; 410 struct ib_ucontext *ib_uctx; 411 int ret; 412 413 ib_uctx = ib_uverbs_get_ucontext(attrs); 414 if (IS_ERR(ib_uctx)) 415 return PTR_ERR(ib_uctx); 416 417 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 418 dpi.umdbr = uctx->dpi.umdbr; 419 dpi.dpi = uctx->dpi.dpi; 420 421 ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_DEFAULT_DBR_ATTR, 422 &dpi, sizeof(dpi)); 423 if (ret) 424 return ret; 425 426 return 0; 427 } 428 429 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_DBR_ALLOC, 430 UVERBS_ATTR_IDR(BNXT_RE_ALLOC_DBR_HANDLE, 431 BNXT_RE_OBJECT_DBR, 432 UVERBS_ACCESS_NEW, 433 UA_MANDATORY), 434 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_ATTR, 435 UVERBS_ATTR_STRUCT(struct bnxt_re_db_region, 436 umdbr), 437 UA_MANDATORY), 438 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_OFFSET, 439 UVERBS_ATTR_TYPE(u64), 440 UA_MANDATORY)); 441 442 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DBR_FREE, 443 UVERBS_ATTR_IDR(BNXT_RE_FREE_DBR_HANDLE, 444 BNXT_RE_OBJECT_DBR, 445 UVERBS_ACCESS_DESTROY, 446 UA_MANDATORY)); 447 448 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_DBR, 449 UVERBS_TYPE_ALLOC_IDR(bnxt_re_dbr_cleanup), 450 &UVERBS_METHOD(BNXT_RE_METHOD_DBR_ALLOC), 451 &UVERBS_METHOD(BNXT_RE_METHOD_DBR_FREE)); 452 453 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, 454 UVERBS_ATTR_PTR_OUT(BNXT_RE_DEFAULT_DBR_ATTR, 455 UVERBS_ATTR_STRUCT(struct bnxt_re_db_region, 456 umdbr), 457 UA_MANDATORY)); 458 459 DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, 460 &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); 461 462 const struct uapi_definition bnxt_re_uapi_defs[] = { 463 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), 464 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), 465 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), 466 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), 467 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), 468 {} 469 }; 470