1 /* 2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <config.h> 34 35 #include <signal.h> 36 #include <sys/ipc.h> 37 #include <sys/shm.h> 38 #include <stdio.h> 39 #include <stdlib.h> 40 #include <errno.h> 41 42 #include "mlx5.h" 43 #include "bitmap.h" 44 45 static int mlx5_bitmap_init(struct mlx5_bitmap *bitmap, uint32_t num, 46 uint32_t mask) 47 { 48 bitmap->last = 0; 49 bitmap->top = 0; 50 bitmap->max = num; 51 bitmap->avail = num; 52 bitmap->mask = mask; 53 bitmap->avail = bitmap->max; 54 bitmap->table = calloc(BITS_TO_LONGS(bitmap->max), sizeof(uint32_t)); 55 if (!bitmap->table) 56 return -ENOMEM; 57 58 return 0; 59 } 60 61 static void bitmap_free_range(struct mlx5_bitmap *bitmap, uint32_t obj, 62 int cnt) 63 { 64 int i; 65 66 obj &= bitmap->max - 1; 67 68 for (i = 0; i < cnt; i++) 69 mlx5_clear_bit(obj + i, bitmap->table); 70 bitmap->last = min(bitmap->last, obj); 71 bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask; 72 bitmap->avail += cnt; 73 } 74 75 static int bitmap_empty(struct mlx5_bitmap *bitmap) 76 { 77 return (bitmap->avail == bitmap->max) ? 1 : 0; 78 } 79 80 static int bitmap_avail(struct mlx5_bitmap *bitmap) 81 { 82 return bitmap->avail; 83 } 84 85 static void mlx5_bitmap_cleanup(struct mlx5_bitmap *bitmap) 86 { 87 if (bitmap->table) 88 free(bitmap->table); 89 } 90 91 static void free_huge_mem(struct mlx5_hugetlb_mem *hmem) 92 { 93 mlx5_bitmap_cleanup(&hmem->bitmap); 94 if (shmdt(hmem->shmaddr) == -1) 95 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "%s\n", strerror(errno)); 96 shmctl(hmem->shmid, IPC_RMID, NULL); 97 free(hmem); 98 } 99 100 static int mlx5_bitmap_alloc(struct mlx5_bitmap *bitmap) 101 { 102 uint32_t obj; 103 int ret; 104 105 obj = mlx5_find_first_zero_bit(bitmap->table, bitmap->max); 106 if (obj < bitmap->max) { 107 mlx5_set_bit(obj, bitmap->table); 108 bitmap->last = (obj + 1); 109 if (bitmap->last == bitmap->max) 110 bitmap->last = 0; 111 obj |= bitmap->top; 112 ret = obj; 113 } else 114 ret = -1; 115 116 if (ret != -1) 117 --bitmap->avail; 118 119 return ret; 120 } 121 122 static uint32_t find_aligned_range(unsigned long *bitmap, 123 uint32_t start, uint32_t nbits, 124 int len, int alignment) 125 { 126 uint32_t end, i; 127 128 again: 129 start = align(start, alignment); 130 131 while ((start < nbits) && mlx5_test_bit(start, bitmap)) 132 start += alignment; 133 134 if (start >= nbits) 135 return -1; 136 137 end = start + len; 138 if (end > nbits) 139 return -1; 140 141 for (i = start + 1; i < end; i++) { 142 if (mlx5_test_bit(i, bitmap)) { 143 start = i + 1; 144 goto again; 145 } 146 } 147 148 return start; 149 } 150 151 static int bitmap_alloc_range(struct mlx5_bitmap *bitmap, int cnt, 152 int align) 153 { 154 uint32_t obj; 155 int ret, i; 156 157 if (cnt == 1 && align == 1) 158 return mlx5_bitmap_alloc(bitmap); 159 160 if (cnt > bitmap->max) 161 return -1; 162 163 obj = find_aligned_range(bitmap->table, bitmap->last, 164 bitmap->max, cnt, align); 165 if (obj >= bitmap->max) { 166 bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask; 167 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 168 cnt, align); 169 } 170 171 if (obj < bitmap->max) { 172 for (i = 0; i < cnt; i++) 173 mlx5_set_bit(obj + i, bitmap->table); 174 if (obj == bitmap->last) { 175 bitmap->last = (obj + cnt); 176 if (bitmap->last >= bitmap->max) 177 bitmap->last = 0; 178 } 179 obj |= bitmap->top; 180 ret = obj; 181 } else 182 ret = -1; 183 184 if (ret != -1) 185 bitmap->avail -= cnt; 186 187 return obj; 188 } 189 190 static struct mlx5_hugetlb_mem *alloc_huge_mem(size_t size) 191 { 192 struct mlx5_hugetlb_mem *hmem; 193 size_t shm_len; 194 195 hmem = malloc(sizeof(*hmem)); 196 if (!hmem) 197 return NULL; 198 199 shm_len = align(size, MLX5_SHM_LENGTH); 200 hmem->shmid = shmget(IPC_PRIVATE, shm_len, SHM_HUGETLB | SHM_R | SHM_W); 201 if (hmem->shmid == -1) { 202 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "%s\n", strerror(errno)); 203 goto out_free; 204 } 205 206 hmem->shmaddr = shmat(hmem->shmid, MLX5_SHM_ADDR, MLX5_SHMAT_FLAGS); 207 if (hmem->shmaddr == (void *)-1) { 208 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "%s\n", strerror(errno)); 209 goto out_rmid; 210 } 211 212 if (mlx5_bitmap_init(&hmem->bitmap, shm_len / MLX5_Q_CHUNK_SIZE, 213 shm_len / MLX5_Q_CHUNK_SIZE - 1)) { 214 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "%s\n", strerror(errno)); 215 goto out_shmdt; 216 } 217 218 /* 219 * Marked to be destroyed when process detaches from shmget segment 220 */ 221 shmctl(hmem->shmid, IPC_RMID, NULL); 222 223 return hmem; 224 225 out_shmdt: 226 if (shmdt(hmem->shmaddr) == -1) 227 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "%s\n", strerror(errno)); 228 229 out_rmid: 230 shmctl(hmem->shmid, IPC_RMID, NULL); 231 232 out_free: 233 free(hmem); 234 return NULL; 235 } 236 237 static int alloc_huge_buf(struct mlx5_context *mctx, struct mlx5_buf *buf, 238 size_t size, int page_size) 239 { 240 int found = 0; 241 int nchunk; 242 struct mlx5_hugetlb_mem *hmem; 243 int ret; 244 245 buf->length = align(size, MLX5_Q_CHUNK_SIZE); 246 nchunk = buf->length / MLX5_Q_CHUNK_SIZE; 247 248 mlx5_spin_lock(&mctx->hugetlb_lock); 249 TAILQ_FOREACH(hmem, &mctx->hugetlb_list, entry) { 250 if (bitmap_avail(&hmem->bitmap)) { 251 buf->base = bitmap_alloc_range(&hmem->bitmap, nchunk, 1); 252 if (buf->base != -1) { 253 buf->hmem = hmem; 254 found = 1; 255 break; 256 } 257 } 258 } 259 mlx5_spin_unlock(&mctx->hugetlb_lock); 260 261 if (!found) { 262 hmem = alloc_huge_mem(buf->length); 263 if (!hmem) 264 return -1; 265 266 buf->base = bitmap_alloc_range(&hmem->bitmap, nchunk, 1); 267 if (buf->base == -1) { 268 free_huge_mem(hmem); 269 /* TBD: remove after proven stability */ 270 fprintf(stderr, "BUG: huge allocation\n"); 271 return -1; 272 } 273 274 buf->hmem = hmem; 275 276 mlx5_spin_lock(&mctx->hugetlb_lock); 277 if (bitmap_avail(&hmem->bitmap)) 278 TAILQ_INSERT_HEAD(&mctx->hugetlb_list, hmem, entry); 279 else 280 TAILQ_INSERT_TAIL(&mctx->hugetlb_list, hmem, entry); 281 mlx5_spin_unlock(&mctx->hugetlb_lock); 282 } 283 284 buf->buf = hmem->shmaddr + buf->base * MLX5_Q_CHUNK_SIZE; 285 286 ret = ibv_dontfork_range(buf->buf, buf->length); 287 if (ret) { 288 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "\n"); 289 goto out_fork; 290 } 291 buf->type = MLX5_ALLOC_TYPE_HUGE; 292 293 return 0; 294 295 out_fork: 296 mlx5_spin_lock(&mctx->hugetlb_lock); 297 bitmap_free_range(&hmem->bitmap, buf->base, nchunk); 298 if (bitmap_empty(&hmem->bitmap)) { 299 TAILQ_REMOVE(&mctx->hugetlb_list, hmem, entry); 300 mlx5_spin_unlock(&mctx->hugetlb_lock); 301 free_huge_mem(hmem); 302 } else 303 mlx5_spin_unlock(&mctx->hugetlb_lock); 304 305 return -1; 306 } 307 308 static void free_huge_buf(struct mlx5_context *ctx, struct mlx5_buf *buf) 309 { 310 int nchunk; 311 312 nchunk = buf->length / MLX5_Q_CHUNK_SIZE; 313 mlx5_spin_lock(&ctx->hugetlb_lock); 314 bitmap_free_range(&buf->hmem->bitmap, buf->base, nchunk); 315 if (bitmap_empty(&buf->hmem->bitmap)) { 316 TAILQ_REMOVE(&ctx->hugetlb_list, buf->hmem, entry); 317 mlx5_spin_unlock(&ctx->hugetlb_lock); 318 free_huge_mem(buf->hmem); 319 } else 320 mlx5_spin_unlock(&ctx->hugetlb_lock); 321 } 322 323 void mlx5_free_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf) 324 { 325 ibv_dofork_range(buf->buf, buf->length); 326 ctx->extern_alloc.free(buf->buf, ctx->extern_alloc.data); 327 } 328 329 int mlx5_alloc_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf, 330 size_t size) 331 { 332 void *addr; 333 334 addr = ctx->extern_alloc.alloc(size, ctx->extern_alloc.data); 335 if (addr || size == 0) { 336 if (ibv_dontfork_range(addr, size)) { 337 mlx5_dbg(stderr, MLX5_DBG_CONTIG, 338 "External mode dontfork_range failed\n"); 339 ctx->extern_alloc.free(addr, 340 ctx->extern_alloc.data); 341 return -1; 342 } 343 buf->buf = addr; 344 buf->length = size; 345 buf->type = MLX5_ALLOC_TYPE_EXTERNAL; 346 return 0; 347 } 348 349 mlx5_dbg(stderr, MLX5_DBG_CONTIG, "External alloc failed\n"); 350 return -1; 351 } 352 353 int mlx5_alloc_prefered_buf(struct mlx5_context *mctx, 354 struct mlx5_buf *buf, 355 size_t size, int page_size, 356 enum mlx5_alloc_type type, 357 const char *component) 358 { 359 int ret; 360 361 /* 362 * Fallback mechanism priority: 363 * huge pages 364 * contig pages 365 * default 366 */ 367 if (type == MLX5_ALLOC_TYPE_HUGE || 368 type == MLX5_ALLOC_TYPE_PREFER_HUGE || 369 type == MLX5_ALLOC_TYPE_ALL) { 370 ret = alloc_huge_buf(mctx, buf, size, page_size); 371 if (!ret) 372 return 0; 373 374 if (type == MLX5_ALLOC_TYPE_HUGE) 375 return -1; 376 377 mlx5_dbg(stderr, MLX5_DBG_CONTIG, 378 "Huge mode allocation failed, fallback to %s mode\n", 379 MLX5_ALLOC_TYPE_ALL ? "contig" : "default"); 380 } 381 382 if (type == MLX5_ALLOC_TYPE_CONTIG || 383 type == MLX5_ALLOC_TYPE_PREFER_CONTIG || 384 type == MLX5_ALLOC_TYPE_ALL) { 385 ret = mlx5_alloc_buf_contig(mctx, buf, size, page_size, component); 386 if (!ret) 387 return 0; 388 389 if (type == MLX5_ALLOC_TYPE_CONTIG) 390 return -1; 391 mlx5_dbg(stderr, MLX5_DBG_CONTIG, 392 "Contig allocation failed, fallback to default mode\n"); 393 } 394 395 if (type == MLX5_ALLOC_TYPE_EXTERNAL) 396 return mlx5_alloc_buf_extern(mctx, buf, size); 397 398 return mlx5_alloc_buf(buf, size, page_size); 399 400 } 401 402 int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf) 403 { 404 int err = 0; 405 406 switch (buf->type) { 407 case MLX5_ALLOC_TYPE_ANON: 408 mlx5_free_buf(buf); 409 break; 410 411 case MLX5_ALLOC_TYPE_HUGE: 412 free_huge_buf(ctx, buf); 413 break; 414 415 case MLX5_ALLOC_TYPE_CONTIG: 416 mlx5_free_buf_contig(ctx, buf); 417 break; 418 419 case MLX5_ALLOC_TYPE_EXTERNAL: 420 mlx5_free_buf_extern(ctx, buf); 421 break; 422 423 default: 424 fprintf(stderr, "Bad allocation type\n"); 425 } 426 427 return err; 428 } 429 430 /* This function computes log2(v) rounded up. 431 We don't want to have a dependency to libm which exposes ceil & log2 APIs. 432 Code was written based on public domain code: 433 URL: http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog. 434 */ 435 static uint32_t mlx5_get_block_order(uint32_t v) 436 { 437 static const uint32_t bits_arr[] = {0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}; 438 static const uint32_t shift_arr[] = {1, 2, 4, 8, 16}; 439 int i; 440 uint32_t input_val = v; 441 442 register uint32_t r = 0;/* result of log2(v) will go here */ 443 for (i = 4; i >= 0; i--) { 444 if (v & bits_arr[i]) { 445 v >>= shift_arr[i]; 446 r |= shift_arr[i]; 447 } 448 } 449 /* Rounding up if required */ 450 r += !!(input_val & ((1 << r) - 1)); 451 452 return r; 453 } 454 455 bool mlx5_is_extern_alloc(struct mlx5_context *context) 456 { 457 return context->extern_alloc.alloc && context->extern_alloc.free; 458 } 459 460 void mlx5_get_alloc_type(struct mlx5_context *context, 461 const char *component, 462 enum mlx5_alloc_type *alloc_type, 463 enum mlx5_alloc_type default_type) 464 465 { 466 char *env_value; 467 char name[128]; 468 469 if (mlx5_is_extern_alloc(context)) { 470 *alloc_type = MLX5_ALLOC_TYPE_EXTERNAL; 471 return; 472 } 473 474 snprintf(name, sizeof(name), "%s_ALLOC_TYPE", component); 475 476 *alloc_type = default_type; 477 478 env_value = getenv(name); 479 if (env_value) { 480 if (!strcasecmp(env_value, "ANON")) 481 *alloc_type = MLX5_ALLOC_TYPE_ANON; 482 else if (!strcasecmp(env_value, "HUGE")) 483 *alloc_type = MLX5_ALLOC_TYPE_HUGE; 484 else if (!strcasecmp(env_value, "CONTIG")) 485 *alloc_type = MLX5_ALLOC_TYPE_CONTIG; 486 else if (!strcasecmp(env_value, "PREFER_CONTIG")) 487 *alloc_type = MLX5_ALLOC_TYPE_PREFER_CONTIG; 488 else if (!strcasecmp(env_value, "PREFER_HUGE")) 489 *alloc_type = MLX5_ALLOC_TYPE_PREFER_HUGE; 490 else if (!strcasecmp(env_value, "ALL")) 491 *alloc_type = MLX5_ALLOC_TYPE_ALL; 492 } 493 } 494 495 static void mlx5_alloc_get_env_info(int *max_block_log, 496 int *min_block_log, 497 const char *component) 498 499 { 500 char *env; 501 int value; 502 char name[128]; 503 504 /* First set defaults */ 505 *max_block_log = MLX5_MAX_LOG2_CONTIG_BLOCK_SIZE; 506 *min_block_log = MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE; 507 508 snprintf(name, sizeof(name), "%s_MAX_LOG2_CONTIG_BSIZE", component); 509 env = getenv(name); 510 if (env) { 511 value = atoi(env); 512 if (value <= MLX5_MAX_LOG2_CONTIG_BLOCK_SIZE && 513 value >= MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE) 514 *max_block_log = value; 515 else 516 fprintf(stderr, "Invalid value %d for %s\n", 517 value, name); 518 } 519 sprintf(name, "%s_MIN_LOG2_CONTIG_BSIZE", component); 520 env = getenv(name); 521 if (env) { 522 value = atoi(env); 523 if (value >= MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE && 524 value <= *max_block_log) 525 *min_block_log = value; 526 else 527 fprintf(stderr, "Invalid value %d for %s\n", 528 value, name); 529 } 530 } 531 532 int mlx5_alloc_buf_contig(struct mlx5_context *mctx, 533 struct mlx5_buf *buf, size_t size, 534 int page_size, 535 const char *component) 536 { 537 void *addr = MAP_FAILED; 538 int block_size_exp; 539 int max_block_log; 540 int min_block_log; 541 struct ibv_context *context = &mctx->ibv_ctx; 542 off_t offset; 543 544 mlx5_alloc_get_env_info(&max_block_log, 545 &min_block_log, 546 component); 547 548 block_size_exp = mlx5_get_block_order(size); 549 550 if (block_size_exp > max_block_log) 551 block_size_exp = max_block_log; 552 553 do { 554 offset = 0; 555 set_command(MLX5_MMAP_GET_CONTIGUOUS_PAGES_CMD, &offset); 556 set_order(block_size_exp, &offset); 557 addr = mmap(NULL , size, PROT_WRITE | PROT_READ, MAP_SHARED, 558 context->cmd_fd, page_size * offset); 559 if (addr != MAP_FAILED) 560 break; 561 562 /* 563 * The kernel returns EINVAL if not supported 564 */ 565 if (errno == EINVAL) 566 return -1; 567 568 block_size_exp -= 1; 569 } while (block_size_exp >= min_block_log); 570 mlx5_dbg(mctx->dbg_fp, MLX5_DBG_CONTIG, "block order %d, addr %p\n", 571 block_size_exp, addr); 572 573 if (addr == MAP_FAILED) 574 return -1; 575 576 if (ibv_dontfork_range(addr, size)) { 577 munmap(addr, size); 578 return -1; 579 } 580 581 buf->buf = addr; 582 buf->length = size; 583 buf->type = MLX5_ALLOC_TYPE_CONTIG; 584 585 return 0; 586 } 587 588 void mlx5_free_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf) 589 { 590 ibv_dofork_range(buf->buf, buf->length); 591 munmap(buf->buf, buf->length); 592 } 593 594 int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size) 595 { 596 int ret; 597 int al_size; 598 599 al_size = align(size, page_size); 600 ret = posix_memalign(&buf->buf, page_size, al_size); 601 if (ret) 602 return ret; 603 604 ret = ibv_dontfork_range(buf->buf, al_size); 605 if (ret) 606 free(buf->buf); 607 608 if (!ret) { 609 buf->length = al_size; 610 buf->type = MLX5_ALLOC_TYPE_ANON; 611 } 612 613 return ret; 614 } 615 616 void mlx5_free_buf(struct mlx5_buf *buf) 617 { 618 ibv_dofork_range(buf->buf, buf->length); 619 free(buf->buf); 620 } 621