1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved. 4 */ 5 6 #include <rdma/rdma_cm.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/restrack.h> 9 #include <rdma/rdma_counter.h> 10 #include <linux/mutex.h> 11 #include <linux/sched/task.h> 12 #include <linux/pid_namespace.h> 13 14 #include "cma_priv.h" 15 #include "restrack.h" 16 17 /** 18 * rdma_restrack_init() - initialize and allocate resource tracking 19 * @dev: IB device 20 * 21 * Return: 0 on success 22 */ 23 int rdma_restrack_init(struct ib_device *dev) 24 { 25 struct rdma_restrack_root *rt; 26 int i; 27 28 dev->res = kzalloc_objs(*rt, RDMA_RESTRACK_MAX); 29 if (!dev->res) 30 return -ENOMEM; 31 32 rt = dev->res; 33 34 for (i = 0; i < RDMA_RESTRACK_MAX; i++) 35 xa_init_flags(&rt[i].xa, XA_FLAGS_ALLOC); 36 37 return 0; 38 } 39 40 /** 41 * rdma_restrack_clean() - clean resource tracking 42 * @dev: IB device 43 */ 44 void rdma_restrack_clean(struct ib_device *dev) 45 { 46 struct rdma_restrack_root *rt = dev->res; 47 int i; 48 49 for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) { 50 struct xarray *xa = &dev->res[i].xa; 51 52 WARN_ON(!xa_empty(xa)); 53 xa_destroy(xa); 54 } 55 kfree(rt); 56 } 57 58 /** 59 * rdma_restrack_count() - the current usage of specific object 60 * @dev: IB device 61 * @type: actual type of object to operate 62 * @show_details: count driver specific objects 63 */ 64 int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, 65 bool show_details) 66 { 67 struct rdma_restrack_root *rt = &dev->res[type]; 68 struct rdma_restrack_entry *e; 69 XA_STATE(xas, &rt->xa, 0); 70 u32 cnt = 0; 71 72 xa_lock(&rt->xa); 73 xas_for_each(&xas, e, U32_MAX) { 74 if (xa_is_zero(e)) 75 continue; 76 if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) 77 continue; 78 cnt++; 79 } 80 xa_unlock(&rt->xa); 81 return cnt; 82 } 83 EXPORT_SYMBOL(rdma_restrack_count); 84 85 static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) 86 { 87 switch (res->type) { 88 case RDMA_RESTRACK_PD: 89 return container_of(res, struct ib_pd, res)->device; 90 case RDMA_RESTRACK_CQ: 91 return container_of(res, struct ib_cq, res)->device; 92 case RDMA_RESTRACK_QP: 93 return container_of(res, struct ib_qp, res)->device; 94 case RDMA_RESTRACK_CM_ID: 95 return container_of(res, struct rdma_id_private, 96 res)->id.device; 97 case RDMA_RESTRACK_MR: 98 return container_of(res, struct ib_mr, res)->device; 99 case RDMA_RESTRACK_CTX: 100 return container_of(res, struct ib_ucontext, res)->device; 101 case RDMA_RESTRACK_COUNTER: 102 return container_of(res, struct rdma_counter, res)->device; 103 case RDMA_RESTRACK_SRQ: 104 return container_of(res, struct ib_srq, res)->device; 105 case RDMA_RESTRACK_DMAH: 106 return container_of(res, struct ib_dmah, res)->device; 107 default: 108 WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type); 109 return NULL; 110 } 111 } 112 113 /** 114 * rdma_restrack_attach_task() - attach the task onto this resource, 115 * valid for user space restrack entries. 116 * @res: resource entry 117 * @task: the task to attach 118 */ 119 static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, 120 struct task_struct *task) 121 { 122 if (WARN_ON_ONCE(!task)) 123 return; 124 125 if (res->task) 126 put_task_struct(res->task); 127 get_task_struct(task); 128 res->task = task; 129 res->user = true; 130 } 131 132 /** 133 * rdma_restrack_set_name() - set the task for this resource 134 * @res: resource entry 135 * @caller: kernel name, the current task will be used if the caller is NULL. 136 */ 137 void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller) 138 { 139 if (caller) { 140 res->kern_name = caller; 141 return; 142 } 143 144 rdma_restrack_attach_task(res, current); 145 } 146 EXPORT_SYMBOL(rdma_restrack_set_name); 147 148 /** 149 * rdma_restrack_parent_name() - set the restrack name properties based 150 * on parent restrack 151 * @dst: destination resource entry 152 * @parent: parent resource entry 153 */ 154 void rdma_restrack_parent_name(struct rdma_restrack_entry *dst, 155 const struct rdma_restrack_entry *parent) 156 { 157 if (rdma_is_kernel_res(parent)) 158 dst->kern_name = parent->kern_name; 159 else 160 rdma_restrack_attach_task(dst, parent->task); 161 } 162 EXPORT_SYMBOL(rdma_restrack_parent_name); 163 164 /** 165 * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface 166 * to release memory in fully automatic way. 167 * @res: Entry to initialize 168 * @type: REstrack type 169 */ 170 void rdma_restrack_new(struct rdma_restrack_entry *res, 171 enum rdma_restrack_type type) 172 { 173 kref_init(&res->kref); 174 init_completion(&res->comp); 175 res->type = type; 176 } 177 EXPORT_SYMBOL(rdma_restrack_new); 178 179 /** 180 * rdma_restrack_add() - add object to the resource tracking database 181 * @res: resource entry 182 */ 183 void rdma_restrack_add(struct rdma_restrack_entry *res) 184 { 185 struct ib_device *dev = res_to_dev(res); 186 struct rdma_restrack_root *rt; 187 int ret = 0; 188 189 if (!dev) 190 return; 191 192 if (res->no_track) 193 goto out; 194 195 rt = &dev->res[res->type]; 196 197 if (res->type == RDMA_RESTRACK_QP) { 198 /* Special case to ensure that LQPN points to right QP */ 199 struct ib_qp *qp = container_of(res, struct ib_qp, res); 200 201 WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8, 202 "QP number 0x%0X and port 0x%0X", qp->qp_num, 203 qp->port); 204 res->id = qp->qp_num; 205 if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI) 206 res->id |= qp->port << 24; 207 ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); 208 if (ret) 209 res->id = 0; 210 211 if (qp->qp_type >= IB_QPT_DRIVER) 212 xa_set_mark(&rt->xa, res->id, RESTRACK_DD); 213 } else if (res->type == RDMA_RESTRACK_COUNTER) { 214 /* Special case to ensure that cntn points to right counter */ 215 struct rdma_counter *counter; 216 217 counter = container_of(res, struct rdma_counter, res); 218 ret = xa_insert(&rt->xa, counter->id, res, GFP_KERNEL); 219 res->id = ret ? 0 : counter->id; 220 } else { 221 ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b, 222 &rt->next_id, GFP_KERNEL); 223 ret = (ret < 0) ? ret : 0; 224 } 225 226 out: 227 if (!ret) 228 res->valid = true; 229 } 230 EXPORT_SYMBOL(rdma_restrack_add); 231 232 int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) 233 { 234 return kref_get_unless_zero(&res->kref); 235 } 236 EXPORT_SYMBOL(rdma_restrack_get); 237 238 /** 239 * rdma_restrack_get_byid() - translate from ID to restrack object 240 * @dev: IB device 241 * @type: resource track type 242 * @id: ID to take a look 243 * 244 * Return: Pointer to restrack entry or -ENOENT in case of error. 245 */ 246 struct rdma_restrack_entry * 247 rdma_restrack_get_byid(struct ib_device *dev, 248 enum rdma_restrack_type type, u32 id) 249 { 250 struct rdma_restrack_root *rt = &dev->res[type]; 251 struct rdma_restrack_entry *res; 252 253 xa_lock(&rt->xa); 254 res = xa_load(&rt->xa, id); 255 if (!res || !rdma_restrack_get(res)) 256 res = ERR_PTR(-ENOENT); 257 xa_unlock(&rt->xa); 258 259 return res; 260 } 261 EXPORT_SYMBOL(rdma_restrack_get_byid); 262 263 static void restrack_release(struct kref *kref) 264 { 265 struct rdma_restrack_entry *res; 266 267 res = container_of(kref, struct rdma_restrack_entry, kref); 268 if (res->task) { 269 put_task_struct(res->task); 270 res->task = NULL; 271 } 272 complete(&res->comp); 273 } 274 275 int rdma_restrack_put(struct rdma_restrack_entry *res) 276 { 277 return kref_put(&res->kref, restrack_release); 278 } 279 EXPORT_SYMBOL(rdma_restrack_put); 280 281 /** 282 * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry 283 * @res: resource entry 284 * 285 * After this returns any concurrent netlink dump threads will see the current 286 * value of the object. This is useful if the object has to be changed and there 287 * is not locking to protect the nl side. Eg for mr->pd. This effectively 288 * destroys the object from a kref/xarray perspective and then immediately 289 * restores it. The kref is acting like a lock to barrier concurrent nl threads. 290 * Callers must ensure rdma_restrack_del() is not concurrently called. 291 */ 292 void rdma_restrack_sync(struct rdma_restrack_entry *res) 293 { 294 struct rdma_restrack_entry *old; 295 struct rdma_restrack_root *rt; 296 struct task_struct *task; 297 struct ib_device *dev; 298 299 if (!res->valid || res->no_track) 300 return; 301 302 dev = res_to_dev(res); 303 if (WARN_ON(!dev)) 304 return; 305 306 rt = &dev->res[res->type]; 307 if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) 308 return; 309 310 old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); 311 if (WARN_ON(old != res)) 312 return; 313 314 task = res->task; 315 if (task) 316 get_task_struct(task); 317 rdma_restrack_put(res); 318 wait_for_completion(&res->comp); 319 reinit_completion(&res->comp); 320 if (task) 321 res->task = task; 322 kref_init(&res->kref); 323 324 xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); 325 } 326 EXPORT_SYMBOL(rdma_restrack_sync); 327 328 /** 329 * rdma_restrack_del() - delete object from the resource tracking database 330 * @res: resource entry 331 */ 332 void rdma_restrack_del(struct rdma_restrack_entry *res) 333 { 334 struct rdma_restrack_entry *old; 335 struct rdma_restrack_root *rt; 336 struct ib_device *dev; 337 338 if (!res->valid) { 339 if (res->task) { 340 put_task_struct(res->task); 341 res->task = NULL; 342 } 343 return; 344 } 345 346 if (res->no_track) 347 goto out; 348 349 dev = res_to_dev(res); 350 if (WARN_ON(!dev)) 351 return; 352 353 rt = &dev->res[res->type]; 354 355 old = xa_erase(&rt->xa, res->id); 356 WARN_ON(old != res); 357 358 out: 359 res->valid = false; 360 rdma_restrack_put(res); 361 wait_for_completion(&res->comp); 362 } 363 EXPORT_SYMBOL(rdma_restrack_del); 364