1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved. 4 */ 5 6 #include <rdma/rdma_cm.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/restrack.h> 9 #include <rdma/rdma_counter.h> 10 #include <linux/mutex.h> 11 #include <linux/sched/task.h> 12 #include <linux/pid_namespace.h> 13 14 #include "cma_priv.h" 15 #include "restrack.h" 16 17 /** 18 * rdma_restrack_init() - initialize and allocate resource tracking 19 * @dev: IB device 20 * 21 * Return: 0 on success 22 */ 23 int rdma_restrack_init(struct ib_device *dev) 24 { 25 struct rdma_restrack_root *rt; 26 int i; 27 28 dev->res = kzalloc_objs(*rt, RDMA_RESTRACK_MAX); 29 if (!dev->res) 30 return -ENOMEM; 31 32 rt = dev->res; 33 34 for (i = 0; i < RDMA_RESTRACK_MAX; i++) 35 xa_init_flags(&rt[i].xa, XA_FLAGS_ALLOC); 36 37 return 0; 38 } 39 40 /** 41 * rdma_restrack_clean() - clean resource tracking 42 * @dev: IB device 43 */ 44 void rdma_restrack_clean(struct ib_device *dev) 45 { 46 struct rdma_restrack_root *rt = dev->res; 47 int i; 48 49 for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) { 50 struct xarray *xa = &dev->res[i].xa; 51 52 WARN_ON(!xa_empty(xa)); 53 xa_destroy(xa); 54 } 55 kfree(rt); 56 } 57 58 /** 59 * rdma_restrack_count() - the current usage of specific object 60 * @dev: IB device 61 * @type: actual type of object to operate 62 * @show_details: count driver specific objects 63 */ 64 u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, 65 bool show_details) 66 { 67 struct rdma_restrack_root *rt = &dev->res[type]; 68 struct rdma_restrack_entry *e; 69 XA_STATE(xas, &rt->xa, 0); 70 u32 cnt = 0; 71 72 xa_lock(&rt->xa); 73 xas_for_each(&xas, e, U32_MAX) { 74 if (xa_is_zero(e)) 75 continue; 76 if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) 77 continue; 78 cnt++; 79 } 80 xa_unlock(&rt->xa); 81 return cnt; 82 } 83 EXPORT_SYMBOL(rdma_restrack_count); 84 85 static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) 86 { 87 switch (res->type) { 88 case RDMA_RESTRACK_PD: 89 return container_of(res, struct ib_pd, res)->device; 90 case RDMA_RESTRACK_CQ: 91 return container_of(res, struct ib_cq, res)->device; 92 case RDMA_RESTRACK_QP: 93 return container_of(res, struct ib_qp, res)->device; 94 case RDMA_RESTRACK_CM_ID: 95 return container_of(res, struct rdma_id_private, 96 res)->id.device; 97 case RDMA_RESTRACK_MR: 98 return container_of(res, struct ib_mr, res)->device; 99 case RDMA_RESTRACK_CTX: 100 return container_of(res, struct ib_ucontext, res)->device; 101 case RDMA_RESTRACK_COUNTER: 102 return container_of(res, struct rdma_counter, res)->device; 103 case RDMA_RESTRACK_SRQ: 104 return container_of(res, struct ib_srq, res)->device; 105 case RDMA_RESTRACK_DMAH: 106 return container_of(res, struct ib_dmah, res)->device; 107 case RDMA_RESTRACK_COMP_CNTR: 108 return container_of(res, struct ib_comp_cntr, res)->device; 109 default: 110 WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type); 111 return NULL; 112 } 113 } 114 115 /** 116 * rdma_restrack_attach_task() - attach the task onto this resource, 117 * valid for user space restrack entries. 118 * @res: resource entry 119 * @task: the task to attach 120 */ 121 static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, 122 struct task_struct *task) 123 { 124 if (WARN_ON_ONCE(!task)) 125 return; 126 127 if (res->task) 128 put_task_struct(res->task); 129 get_task_struct(task); 130 res->task = task; 131 res->user = true; 132 } 133 134 static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) 135 { 136 struct ib_device *dev = res_to_dev(res); 137 138 if (WARN_ON(!dev)) 139 return NULL; 140 141 return &dev->res[res->type]; 142 } 143 144 static void restrack_drain_res(struct rdma_restrack_root *rt, 145 struct rdma_restrack_entry *res) 146 { 147 if (rt) { 148 struct rdma_restrack_entry *old; 149 150 old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, 151 GFP_KERNEL); 152 WARN_ON(old != res); 153 } 154 155 rdma_restrack_put(res); 156 wait_for_completion(&res->comp); 157 } 158 159 static void restrack_restore_res(struct rdma_restrack_root *rt, 160 struct rdma_restrack_entry *res) 161 { 162 reinit_completion(&res->comp); 163 kref_init(&res->kref); 164 165 if (rt) { 166 struct rdma_restrack_entry *old; 167 168 old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, 169 GFP_KERNEL); 170 WARN_ON(old); 171 } 172 } 173 174 /** 175 * rdma_restrack_set_name() - set the task for this resource 176 * @res: resource entry 177 * @caller: kernel name, the current task will be used if the caller is NULL. 178 */ 179 void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller) 180 { 181 if (caller) { 182 res->kern_name = caller; 183 return; 184 } 185 186 rdma_restrack_attach_task(res, current); 187 } 188 EXPORT_SYMBOL(rdma_restrack_set_name); 189 190 /** 191 * rdma_restrack_parent_name() - set the restrack name properties based 192 * on parent restrack 193 * @dst: destination resource entry 194 * @parent: parent resource entry 195 */ 196 void rdma_restrack_parent_name(struct rdma_restrack_entry *dst, 197 const struct rdma_restrack_entry *parent) 198 { 199 if (rdma_is_kernel_res(parent)) 200 dst->kern_name = parent->kern_name; 201 else 202 rdma_restrack_attach_task(dst, parent->task); 203 } 204 EXPORT_SYMBOL(rdma_restrack_parent_name); 205 206 /** 207 * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface 208 * to release memory in fully automatic way. 209 * @res: Entry to initialize 210 * @type: REstrack type 211 */ 212 void rdma_restrack_new(struct rdma_restrack_entry *res, 213 enum rdma_restrack_type type) 214 { 215 kref_init(&res->kref); 216 init_completion(&res->comp); 217 res->type = type; 218 } 219 EXPORT_SYMBOL(rdma_restrack_new); 220 221 /** 222 * rdma_restrack_add() - add object to the resource tracking database. 223 * If this resource reuses an ID of a resource that was already destroyed 224 * after calling rdma_restrack_begin() but didn't yet call 225 * rdma_restrack_commit_del() it can result in an untracked QP. 226 * @res: resource entry 227 */ 228 void rdma_restrack_add(struct rdma_restrack_entry *res) 229 { 230 struct rdma_restrack_root *rt; 231 int ret = 0; 232 233 if (res->no_track) 234 goto out; 235 236 rt = res_to_rt(res); 237 if (!rt) 238 return; 239 240 if (res->type == RDMA_RESTRACK_QP) { 241 /* Special case to ensure that LQPN points to right QP */ 242 struct ib_qp *qp = container_of(res, struct ib_qp, res); 243 244 WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8, 245 "QP number 0x%0X and port 0x%0X", qp->qp_num, 246 qp->port); 247 res->id = qp->qp_num; 248 if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI) 249 res->id |= qp->port << 24; 250 ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); 251 if (ret) 252 res->id = 0; 253 254 if (qp->qp_type >= IB_QPT_DRIVER) 255 xa_set_mark(&rt->xa, res->id, RESTRACK_DD); 256 } else if (res->type == RDMA_RESTRACK_COUNTER) { 257 /* Special case to ensure that cntn points to right counter */ 258 struct rdma_counter *counter; 259 260 counter = container_of(res, struct rdma_counter, res); 261 ret = xa_insert(&rt->xa, counter->id, res, GFP_KERNEL); 262 res->id = ret ? 0 : counter->id; 263 } else { 264 ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b, 265 &rt->next_id, GFP_KERNEL); 266 ret = (ret < 0) ? ret : 0; 267 } 268 269 out: 270 if (!ret) 271 res->valid = true; 272 } 273 EXPORT_SYMBOL(rdma_restrack_add); 274 275 /** 276 * rdma_restrack_abort_del() - re-add object to the resource tracking database 277 * it can only be used after rdma_restrack_begin_del(). 278 * @res: resource entry 279 */ 280 void rdma_restrack_abort_del(struct rdma_restrack_entry *res) 281 { 282 struct rdma_restrack_root *rt = NULL; 283 284 if (!res->valid) 285 return; 286 287 if (!res->no_track) { 288 rt = res_to_rt(res); 289 if (!rt) 290 return; 291 } 292 293 restrack_restore_res(rt, res); 294 } 295 EXPORT_SYMBOL(rdma_restrack_abort_del); 296 297 int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) 298 { 299 return kref_get_unless_zero(&res->kref); 300 } 301 EXPORT_SYMBOL(rdma_restrack_get); 302 303 /** 304 * rdma_restrack_get_byid() - translate from ID to restrack object 305 * @dev: IB device 306 * @type: resource track type 307 * @id: ID to take a look 308 * 309 * Return: Pointer to restrack entry or -ENOENT in case of error. 310 */ 311 struct rdma_restrack_entry * 312 rdma_restrack_get_byid(struct ib_device *dev, 313 enum rdma_restrack_type type, u32 id) 314 { 315 struct rdma_restrack_root *rt = &dev->res[type]; 316 struct rdma_restrack_entry *res; 317 318 xa_lock(&rt->xa); 319 res = xa_load(&rt->xa, id); 320 if (!res || !rdma_restrack_get(res)) 321 res = ERR_PTR(-ENOENT); 322 xa_unlock(&rt->xa); 323 324 return res; 325 } 326 EXPORT_SYMBOL(rdma_restrack_get_byid); 327 328 static void restrack_release(struct kref *kref) 329 { 330 struct rdma_restrack_entry *res; 331 332 res = container_of(kref, struct rdma_restrack_entry, kref); 333 if (res->task && !res->valid) { 334 put_task_struct(res->task); 335 res->task = NULL; 336 } 337 complete(&res->comp); 338 } 339 340 int rdma_restrack_put(struct rdma_restrack_entry *res) 341 { 342 return kref_put(&res->kref, restrack_release); 343 } 344 EXPORT_SYMBOL(rdma_restrack_put); 345 346 /** 347 * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry 348 * @res: resource entry 349 * 350 * After this returns any concurrent netlink dump threads will see the current 351 * value of the object. This is useful if the object has to be changed and there 352 * is not locking to protect the nl side. Eg for mr->pd. This effectively 353 * destroys the object from a kref/xarray perspective and then immediately 354 * restores it. The kref is acting like a lock to barrier concurrent nl threads. 355 * Callers must ensure rdma_restrack_del() is not concurrently called. 356 */ 357 void rdma_restrack_sync(struct rdma_restrack_entry *res) 358 { 359 struct rdma_restrack_root *rt; 360 361 if (!res->valid || res->no_track) 362 return; 363 364 rt = res_to_rt(res); 365 if (!rt) 366 return; 367 368 if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) 369 return; 370 371 restrack_drain_res(rt, res); 372 restrack_restore_res(rt, res); 373 } 374 EXPORT_SYMBOL(rdma_restrack_sync); 375 376 /** 377 * rdma_restrack_del() - delete object from the resource tracking database 378 * @res: resource entry 379 */ 380 void rdma_restrack_del(struct rdma_restrack_entry *res) 381 { 382 struct rdma_restrack_entry *old; 383 struct rdma_restrack_root *rt; 384 385 if (!res->valid) { 386 if (res->task) { 387 put_task_struct(res->task); 388 res->task = NULL; 389 } 390 return; 391 } 392 393 if (res->no_track) 394 goto out; 395 396 rt = res_to_rt(res); 397 if (!rt) 398 return; 399 400 old = xa_erase(&rt->xa, res->id); 401 WARN_ON(old != res); 402 403 out: 404 res->valid = false; 405 rdma_restrack_put(res); 406 wait_for_completion(&res->comp); 407 if (res->task) { 408 put_task_struct(res->task); 409 res->task = NULL; 410 } 411 } 412 EXPORT_SYMBOL(rdma_restrack_del); 413 414 /** 415 * rdma_restrack_begin_del() - invalidate the object from the resource tracking 416 * database but preserve its index in the array. 417 * Since this preserves the index in the array until rdma_restrack_commit_del() 418 * is called, if rdma_restrack_add() is called in between with an old QP ID it 419 * can result in an untracked QP. 420 * @res: resource entry 421 */ 422 void rdma_restrack_begin_del(struct rdma_restrack_entry *res) 423 { 424 struct rdma_restrack_root *rt = NULL; 425 426 if (!res->valid) 427 return; 428 429 if (!res->no_track) { 430 rt = res_to_rt(res); 431 if (!rt) 432 return; 433 } 434 435 restrack_drain_res(rt, res); 436 } 437 EXPORT_SYMBOL(rdma_restrack_begin_del); 438 439 /** 440 * rdma_restrack_commit_del() - delete object from the resource tracking 441 * database and free the task. 442 * @res: resource entry 443 */ 444 void rdma_restrack_commit_del(struct rdma_restrack_entry *res) 445 { 446 struct rdma_restrack_root *rt; 447 448 if (!res->valid || res->no_track) 449 goto out; 450 451 rt = res_to_rt(res); 452 if (!rt) 453 return; 454 455 xa_erase(&rt->xa, res->id); 456 457 out: 458 res->valid = false; 459 if (res->task) { 460 put_task_struct(res->task); 461 res->task = NULL; 462 } 463 } 464 EXPORT_SYMBOL(rdma_restrack_commit_del); 465