1 /* 2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include "rxe.h" 35 #include "rxe_loc.h" 36 37 /* info about object pools 38 * note that mr and mw share a single index space 39 * so that one can map an lkey to the correct type of object 40 */ 41 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { 42 [RXE_TYPE_UC] = { 43 .name = "rxe-uc", 44 .size = sizeof(struct rxe_ucontext), 45 }, 46 [RXE_TYPE_PD] = { 47 .name = "rxe-pd", 48 .size = sizeof(struct rxe_pd), 49 }, 50 [RXE_TYPE_AH] = { 51 .name = "rxe-ah", 52 .size = sizeof(struct rxe_ah), 53 .flags = RXE_POOL_ATOMIC, 54 }, 55 [RXE_TYPE_SRQ] = { 56 .name = "rxe-srq", 57 .size = sizeof(struct rxe_srq), 58 .flags = RXE_POOL_INDEX, 59 .min_index = RXE_MIN_SRQ_INDEX, 60 .max_index = RXE_MAX_SRQ_INDEX, 61 }, 62 [RXE_TYPE_QP] = { 63 .name = "rxe-qp", 64 .size = sizeof(struct rxe_qp), 65 .cleanup = rxe_qp_cleanup, 66 .flags = RXE_POOL_INDEX, 67 .min_index = RXE_MIN_QP_INDEX, 68 .max_index = RXE_MAX_QP_INDEX, 69 }, 70 [RXE_TYPE_CQ] = { 71 .name = "rxe-cq", 72 .size = sizeof(struct rxe_cq), 73 .cleanup = rxe_cq_cleanup, 74 }, 75 [RXE_TYPE_MR] = { 76 .name = "rxe-mr", 77 .size = sizeof(struct rxe_mem), 78 .cleanup = rxe_mem_cleanup, 79 .flags = RXE_POOL_INDEX, 80 .max_index = RXE_MAX_MR_INDEX, 81 .min_index = RXE_MIN_MR_INDEX, 82 }, 83 [RXE_TYPE_MW] = { 84 .name = "rxe-mw", 85 .size = sizeof(struct rxe_mem), 86 .flags = RXE_POOL_INDEX, 87 .max_index = RXE_MAX_MW_INDEX, 88 .min_index = RXE_MIN_MW_INDEX, 89 }, 90 [RXE_TYPE_MC_GRP] = { 91 .name = "rxe-mc_grp", 92 .size = sizeof(struct rxe_mc_grp), 93 .cleanup = rxe_mc_cleanup, 94 .flags = RXE_POOL_KEY, 95 .key_offset = offsetof(struct rxe_mc_grp, mgid), 96 .key_size = sizeof(union ib_gid), 97 }, 98 [RXE_TYPE_MC_ELEM] = { 99 .name = "rxe-mc_elem", 100 .size = sizeof(struct rxe_mc_elem), 101 .flags = RXE_POOL_ATOMIC, 102 }, 103 }; 104 105 static inline char *pool_name(struct rxe_pool *pool) 106 { 107 return rxe_type_info[pool->type].name; 108 } 109 110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool) 111 { 112 return rxe_type_info[pool->type].cache; 113 } 114 115 static inline enum rxe_elem_type rxe_type(void *arg) 116 { 117 struct rxe_pool_entry *elem = arg; 118 119 return elem->pool->type; 120 } 121 122 int rxe_cache_init(void) 123 { 124 int err; 125 int i; 126 size_t size; 127 struct rxe_type_info *type; 128 129 for (i = 0; i < RXE_NUM_TYPES; i++) { 130 type = &rxe_type_info[i]; 131 size = ALIGN(type->size, RXE_POOL_ALIGN); 132 type->cache = kmem_cache_create(type->name, size, 133 RXE_POOL_ALIGN, 134 RXE_POOL_CACHE_FLAGS, NULL); 135 if (!type->cache) { 136 pr_err("Unable to init kmem cache for %s\n", 137 type->name); 138 err = -ENOMEM; 139 goto err1; 140 } 141 } 142 143 return 0; 144 145 err1: 146 while (--i >= 0) { 147 kmem_cache_destroy(type->cache); 148 type->cache = NULL; 149 } 150 151 return err; 152 } 153 154 void rxe_cache_exit(void) 155 { 156 int i; 157 struct rxe_type_info *type; 158 159 for (i = 0; i < RXE_NUM_TYPES; i++) { 160 type = &rxe_type_info[i]; 161 kmem_cache_destroy(type->cache); 162 type->cache = NULL; 163 } 164 } 165 166 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) 167 { 168 int err = 0; 169 size_t size; 170 171 if ((max - min + 1) < pool->max_elem) { 172 pr_warn("not enough indices for max_elem\n"); 173 err = -EINVAL; 174 goto out; 175 } 176 177 pool->max_index = max; 178 pool->min_index = min; 179 180 size = BITS_TO_LONGS(max - min + 1) * sizeof(long); 181 pool->table = kmalloc(size, GFP_KERNEL); 182 if (!pool->table) { 183 err = -ENOMEM; 184 goto out; 185 } 186 187 pool->table_size = size; 188 bitmap_zero(pool->table, max - min + 1); 189 190 out: 191 return err; 192 } 193 194 int rxe_pool_init( 195 struct rxe_dev *rxe, 196 struct rxe_pool *pool, 197 enum rxe_elem_type type, 198 unsigned max_elem) 199 { 200 int err = 0; 201 size_t size = rxe_type_info[type].size; 202 203 memset(pool, 0, sizeof(*pool)); 204 205 pool->rxe = rxe; 206 pool->type = type; 207 pool->max_elem = max_elem; 208 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN); 209 pool->flags = rxe_type_info[type].flags; 210 pool->tree = RB_ROOT; 211 pool->cleanup = rxe_type_info[type].cleanup; 212 213 atomic_set(&pool->num_elem, 0); 214 215 kref_init(&pool->ref_cnt); 216 217 spin_lock_init(&pool->pool_lock); 218 219 if (rxe_type_info[type].flags & RXE_POOL_INDEX) { 220 err = rxe_pool_init_index(pool, 221 rxe_type_info[type].max_index, 222 rxe_type_info[type].min_index); 223 if (err) 224 goto out; 225 } 226 227 if (rxe_type_info[type].flags & RXE_POOL_KEY) { 228 pool->key_offset = rxe_type_info[type].key_offset; 229 pool->key_size = rxe_type_info[type].key_size; 230 } 231 232 pool->state = rxe_pool_valid; 233 234 out: 235 return err; 236 } 237 238 static void rxe_pool_release(struct kref *kref) 239 { 240 struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt); 241 242 pool->state = rxe_pool_invalid; 243 kfree(pool->table); 244 } 245 246 static void rxe_pool_put(struct rxe_pool *pool) 247 { 248 kref_put(&pool->ref_cnt, rxe_pool_release); 249 } 250 251 int rxe_pool_cleanup(struct rxe_pool *pool) 252 { 253 unsigned long flags; 254 255 spin_lock_irqsave(&pool->pool_lock, flags); 256 pool->state = rxe_pool_invalid; 257 if (atomic_read(&pool->num_elem) > 0) 258 pr_warn("%s pool destroyed with unfree'd elem\n", 259 pool_name(pool)); 260 spin_unlock_irqrestore(&pool->pool_lock, flags); 261 262 rxe_pool_put(pool); 263 264 return 0; 265 } 266 267 static u32 alloc_index(struct rxe_pool *pool) 268 { 269 u32 index; 270 u32 range = pool->max_index - pool->min_index + 1; 271 272 index = find_next_zero_bit(pool->table, range, pool->last); 273 if (index >= range) 274 index = find_first_zero_bit(pool->table, range); 275 276 set_bit(index, pool->table); 277 pool->last = index; 278 return index + pool->min_index; 279 } 280 281 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) 282 { 283 struct rb_node **link = &pool->tree.rb_node; 284 struct rb_node *parent = NULL; 285 struct rxe_pool_entry *elem; 286 287 while (*link) { 288 parent = *link; 289 elem = rb_entry(parent, struct rxe_pool_entry, node); 290 291 if (elem->index == new->index) { 292 pr_warn("element already exists!\n"); 293 goto out; 294 } 295 296 if (elem->index > new->index) 297 link = &(*link)->rb_left; 298 else 299 link = &(*link)->rb_right; 300 } 301 302 rb_link_node(&new->node, parent, link); 303 rb_insert_color(&new->node, &pool->tree); 304 out: 305 return; 306 } 307 308 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) 309 { 310 struct rb_node **link = &pool->tree.rb_node; 311 struct rb_node *parent = NULL; 312 struct rxe_pool_entry *elem; 313 int cmp; 314 315 while (*link) { 316 parent = *link; 317 elem = rb_entry(parent, struct rxe_pool_entry, node); 318 319 cmp = memcmp((u8 *)elem + pool->key_offset, 320 (u8 *)new + pool->key_offset, pool->key_size); 321 322 if (cmp == 0) { 323 pr_warn("key already exists!\n"); 324 goto out; 325 } 326 327 if (cmp > 0) 328 link = &(*link)->rb_left; 329 else 330 link = &(*link)->rb_right; 331 } 332 333 rb_link_node(&new->node, parent, link); 334 rb_insert_color(&new->node, &pool->tree); 335 out: 336 return; 337 } 338 339 void rxe_add_key(void *arg, void *key) 340 { 341 struct rxe_pool_entry *elem = arg; 342 struct rxe_pool *pool = elem->pool; 343 unsigned long flags; 344 345 spin_lock_irqsave(&pool->pool_lock, flags); 346 memcpy((u8 *)elem + pool->key_offset, key, pool->key_size); 347 insert_key(pool, elem); 348 spin_unlock_irqrestore(&pool->pool_lock, flags); 349 } 350 351 void rxe_drop_key(void *arg) 352 { 353 struct rxe_pool_entry *elem = arg; 354 struct rxe_pool *pool = elem->pool; 355 unsigned long flags; 356 357 spin_lock_irqsave(&pool->pool_lock, flags); 358 rb_erase(&elem->node, &pool->tree); 359 spin_unlock_irqrestore(&pool->pool_lock, flags); 360 } 361 362 void rxe_add_index(void *arg) 363 { 364 struct rxe_pool_entry *elem = arg; 365 struct rxe_pool *pool = elem->pool; 366 unsigned long flags; 367 368 spin_lock_irqsave(&pool->pool_lock, flags); 369 elem->index = alloc_index(pool); 370 insert_index(pool, elem); 371 spin_unlock_irqrestore(&pool->pool_lock, flags); 372 } 373 374 void rxe_drop_index(void *arg) 375 { 376 struct rxe_pool_entry *elem = arg; 377 struct rxe_pool *pool = elem->pool; 378 unsigned long flags; 379 380 spin_lock_irqsave(&pool->pool_lock, flags); 381 clear_bit(elem->index - pool->min_index, pool->table); 382 rb_erase(&elem->node, &pool->tree); 383 spin_unlock_irqrestore(&pool->pool_lock, flags); 384 } 385 386 void *rxe_alloc(struct rxe_pool *pool) 387 { 388 struct rxe_pool_entry *elem; 389 unsigned long flags; 390 391 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC)); 392 393 spin_lock_irqsave(&pool->pool_lock, flags); 394 if (pool->state != rxe_pool_valid) { 395 spin_unlock_irqrestore(&pool->pool_lock, flags); 396 return NULL; 397 } 398 kref_get(&pool->ref_cnt); 399 spin_unlock_irqrestore(&pool->pool_lock, flags); 400 401 kref_get(&pool->rxe->ref_cnt); 402 403 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) { 404 atomic_dec(&pool->num_elem); 405 rxe_dev_put(pool->rxe); 406 rxe_pool_put(pool); 407 return NULL; 408 } 409 410 elem = kmem_cache_zalloc(pool_cache(pool), 411 (pool->flags & RXE_POOL_ATOMIC) ? 412 GFP_ATOMIC : GFP_KERNEL); 413 414 elem->pool = pool; 415 kref_init(&elem->ref_cnt); 416 417 return elem; 418 } 419 420 void rxe_elem_release(struct kref *kref) 421 { 422 struct rxe_pool_entry *elem = 423 container_of(kref, struct rxe_pool_entry, ref_cnt); 424 struct rxe_pool *pool = elem->pool; 425 426 if (pool->cleanup) 427 pool->cleanup(elem); 428 429 kmem_cache_free(pool_cache(pool), elem); 430 atomic_dec(&pool->num_elem); 431 rxe_dev_put(pool->rxe); 432 rxe_pool_put(pool); 433 } 434 435 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) 436 { 437 struct rb_node *node = NULL; 438 struct rxe_pool_entry *elem = NULL; 439 unsigned long flags; 440 441 spin_lock_irqsave(&pool->pool_lock, flags); 442 443 if (pool->state != rxe_pool_valid) 444 goto out; 445 446 node = pool->tree.rb_node; 447 448 while (node) { 449 elem = rb_entry(node, struct rxe_pool_entry, node); 450 451 if (elem->index > index) 452 node = node->rb_left; 453 else if (elem->index < index) 454 node = node->rb_right; 455 else 456 break; 457 } 458 459 if (node) 460 kref_get(&elem->ref_cnt); 461 462 out: 463 spin_unlock_irqrestore(&pool->pool_lock, flags); 464 return node ? (void *)elem : NULL; 465 } 466 467 void *rxe_pool_get_key(struct rxe_pool *pool, void *key) 468 { 469 struct rb_node *node = NULL; 470 struct rxe_pool_entry *elem = NULL; 471 int cmp; 472 unsigned long flags; 473 474 spin_lock_irqsave(&pool->pool_lock, flags); 475 476 if (pool->state != rxe_pool_valid) 477 goto out; 478 479 node = pool->tree.rb_node; 480 481 while (node) { 482 elem = rb_entry(node, struct rxe_pool_entry, node); 483 484 cmp = memcmp((u8 *)elem + pool->key_offset, 485 key, pool->key_size); 486 487 if (cmp > 0) 488 node = node->rb_left; 489 else if (cmp < 0) 490 node = node->rb_right; 491 else 492 break; 493 } 494 495 if (node) 496 kref_get(&elem->ref_cnt); 497 498 out: 499 spin_unlock_irqrestore(&pool->pool_lock, flags); 500 return node ? ((void *)elem) : NULL; 501 } 502