1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/errno.h> 34 #include <linux/slab.h> 35 #include <linux/mm.h> 36 #include <linux/export.h> 37 #include <linux/bitmap.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/vmalloc.h> 40 #include <linux/nodemask.h> 41 #include <linux/debugfs.h> 42 #include <linux/seq_file.h> 43 #include <linux/mlx5/driver.h> 44 45 #include "mlx5_core.h" 46 47 #define MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT MLX5_ADAPTER_PAGE_SHIFT 48 #define MLX5_FRAG_BUF_POOLS_NUM \ 49 (PAGE_SHIFT - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + 1) 50 51 struct mlx5_db_pgdir { 52 struct list_head list; 53 unsigned long *bitmap; 54 __be32 *db_page; 55 dma_addr_t db_dma; 56 }; 57 58 struct mlx5_dma_pool { 59 /* Protects page_list and per-page allocation bitmaps. */ 60 struct mutex lock; 61 struct list_head page_list; 62 struct mlx5_core_dev *dev; 63 int node; 64 u8 block_shift; 65 }; 66 67 struct mlx5_dma_pool_page { 68 struct mlx5_dma_pool *pool; 69 struct list_head pool_link; 70 unsigned long *bitmap; 71 void *buf; 72 dma_addr_t dma; 73 }; 74 75 struct mlx5_frag_buf_node_pools { 76 struct mlx5_dma_pool *pools[MLX5_FRAG_BUF_POOLS_NUM]; 77 }; 78 79 struct mlx5_dma_pool_stats { 80 int node; 81 size_t block_size; 82 size_t used_blocks; 83 size_t allocated_blocks; 84 }; 85 86 /* Handling for queue buffers -- we allocate a bunch of memory and 87 * register it in a memory region at HCA virtual address 0. 88 */ 89 90 static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, 91 size_t size, dma_addr_t *dma_handle, 92 int node) 93 { 94 struct device *device = mlx5_core_dma_dev(dev); 95 struct mlx5_priv *priv = &dev->priv; 96 int original_node; 97 void *cpu_handle; 98 99 mutex_lock(&priv->alloc_mutex); 100 original_node = dev_to_node(device); 101 set_dev_node(device, node); 102 cpu_handle = dma_alloc_coherent(device, size, dma_handle, 103 GFP_KERNEL); 104 set_dev_node(device, original_node); 105 mutex_unlock(&priv->alloc_mutex); 106 return cpu_handle; 107 } 108 109 static void mlx5_dma_pool_destroy(struct mlx5_dma_pool *pool) 110 { 111 mutex_destroy(&pool->lock); 112 kfree(pool); 113 } 114 115 static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev, 116 int node, u8 block_shift) 117 { 118 struct mlx5_dma_pool *pool; 119 120 pool = kzalloc_obj(*pool); 121 if (!pool) 122 return NULL; 123 124 INIT_LIST_HEAD(&pool->page_list); 125 mutex_init(&pool->lock); 126 pool->dev = dev; 127 pool->node = node; 128 pool->block_shift = block_shift; 129 return pool; 130 } 131 132 static struct mlx5_dma_pool_page * 133 mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool) 134 { 135 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift); 136 struct mlx5_dma_pool_page *page; 137 138 page = kzalloc_obj(*page); 139 if (!page) 140 goto err_out; 141 142 page->pool = pool; 143 page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL); 144 if (!page->bitmap) 145 goto err_free_page; 146 147 bitmap_fill(page->bitmap, blocks_per_page); 148 page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE, 149 &page->dma, pool->node); 150 if (!page->buf) 151 goto err_free_bitmap; 152 153 return page; 154 155 err_free_bitmap: 156 bitmap_free(page->bitmap); 157 err_free_page: 158 kfree(page); 159 err_out: 160 return NULL; 161 } 162 163 static void mlx5_dma_pool_page_free(struct mlx5_core_dev *dev, 164 struct mlx5_dma_pool_page *page) 165 { 166 dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, page->buf, 167 page->dma); 168 bitmap_free(page->bitmap); 169 kfree(page); 170 } 171 172 static int mlx5_dma_pool_alloc_from_page(struct mlx5_dma_pool *pool, 173 struct mlx5_dma_pool_page *page, 174 unsigned long *idx_out) 175 { 176 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift); 177 178 *idx_out = find_first_bit(page->bitmap, blocks_per_page); 179 if (*idx_out >= blocks_per_page) 180 return -ENOMEM; 181 182 __clear_bit(*idx_out, page->bitmap); 183 184 if (bitmap_empty(page->bitmap, blocks_per_page)) 185 list_move_tail(&page->pool_link, &pool->page_list); 186 187 return 0; 188 } 189 190 static struct mlx5_dma_pool_page * 191 mlx5_dma_pool_alloc(struct mlx5_dma_pool *pool, unsigned long *idx_out) 192 { 193 struct mlx5_dma_pool_page *page; 194 195 mutex_lock(&pool->lock); 196 197 page = list_first_entry_or_null(&pool->page_list, 198 struct mlx5_dma_pool_page, pool_link); 199 if (page && !mlx5_dma_pool_alloc_from_page(pool, page, idx_out)) 200 goto unlock; /* successfully allocated from existing page */ 201 202 page = mlx5_dma_pool_page_alloc(pool); 203 if (!page) 204 goto unlock; 205 206 list_add(&page->pool_link, &pool->page_list); 207 mlx5_dma_pool_alloc_from_page(pool, page, idx_out); 208 209 unlock: 210 mutex_unlock(&pool->lock); 211 return page; 212 } 213 214 static void mlx5_dma_pool_free(struct mlx5_dma_pool *pool, 215 struct mlx5_dma_pool_page *page, 216 unsigned long idx) 217 { 218 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift); 219 bool was_full; 220 221 mutex_lock(&pool->lock); 222 was_full = bitmap_empty(page->bitmap, blocks_per_page); 223 __set_bit(idx, page->bitmap); 224 225 if (bitmap_full(page->bitmap, blocks_per_page)) { 226 list_del(&page->pool_link); 227 mlx5_dma_pool_page_free(pool->dev, page); 228 } else { 229 memset((u8 *)page->buf + (idx << pool->block_shift), 0, 230 BIT(pool->block_shift)); 231 if (was_full) 232 list_move(&page->pool_link, &pool->page_list); 233 } 234 mutex_unlock(&pool->lock); 235 } 236 237 static void mlx5_dma_pool_debugfs_get_stats(struct mlx5_dma_pool *pool, 238 struct mlx5_dma_pool_stats *stats) 239 { 240 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift); 241 struct mlx5_dma_pool_page *page; 242 size_t free_blocks = 0; 243 size_t pages = 0; 244 245 mutex_lock(&pool->lock); 246 list_for_each_entry(page, &pool->page_list, pool_link) { 247 pages++; 248 free_blocks += bitmap_weight(page->bitmap, blocks_per_page); 249 } 250 mutex_unlock(&pool->lock); 251 252 stats->node = pool->node; 253 stats->block_size = BIT(pool->block_shift); 254 stats->allocated_blocks = pages * blocks_per_page; 255 stats->used_blocks = stats->allocated_blocks - free_blocks; 256 } 257 258 static void mlx5_dma_pool_debugfs_stats_print(struct seq_file *file, 259 struct mlx5_dma_pool *pool) 260 { 261 struct mlx5_dma_pool_stats stats = {}; 262 263 mlx5_dma_pool_debugfs_get_stats(pool, &stats); 264 seq_printf(file, "%4d %5zu %7zu %7zu\n", 265 stats.node, stats.block_size, stats.used_blocks, 266 stats.allocated_blocks); 267 } 268 269 static void mlx5_dma_pools_debugfs_print_header(struct seq_file *file) 270 { 271 seq_puts(file, "node block_size used_blocks allocated_blocks\n"); 272 } 273 274 static void 275 mlx5_frag_buf_node_pools_destroy(struct mlx5_frag_buf_node_pools *node_pools) 276 { 277 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) 278 if (node_pools->pools[i]) 279 mlx5_dma_pool_destroy(node_pools->pools[i]); 280 kfree(node_pools); 281 } 282 283 static struct mlx5_frag_buf_node_pools * 284 mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node) 285 { 286 struct mlx5_frag_buf_node_pools *node_pools; 287 288 node_pools = kzalloc_obj(*node_pools); 289 if (!node_pools) 290 return NULL; 291 292 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) { 293 u8 block_shift = MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + i; 294 295 node_pools->pools[i] = mlx5_dma_pool_create(dev, node, 296 block_shift); 297 if (!node_pools->pools[i]) { 298 mlx5_frag_buf_node_pools_destroy(node_pools); 299 return NULL; 300 } 301 } 302 303 return node_pools; 304 } 305 306 static int 307 mlx5_frag_buf_dma_pools_debugfs_show(struct seq_file *file, void *priv) 308 { 309 struct mlx5_core_dev *dev = file->private; 310 int node; 311 312 mlx5_dma_pools_debugfs_print_header(file); 313 314 if (!dev->priv.frag_buf_node_pools) 315 return 0; 316 317 for_each_node_state(node, N_POSSIBLE) { 318 struct mlx5_frag_buf_node_pools *node_pools; 319 320 node_pools = dev->priv.frag_buf_node_pools[node]; 321 if (!node_pools) 322 continue; 323 324 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) { 325 struct mlx5_dma_pool *pool = node_pools->pools[i]; 326 327 if (!pool) 328 continue; 329 330 mlx5_dma_pool_debugfs_stats_print(file, pool); 331 } 332 } 333 334 return 0; 335 } 336 DEFINE_SHOW_ATTRIBUTE(mlx5_frag_buf_dma_pools_debugfs); 337 338 void mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev *dev) 339 { 340 struct mlx5_priv *priv = &dev->priv; 341 int node; 342 343 debugfs_remove(priv->dbg.frag_buf_dma_pools_debugfs); 344 priv->dbg.frag_buf_dma_pools_debugfs = NULL; 345 346 for_each_node_state(node, N_POSSIBLE) { 347 struct mlx5_frag_buf_node_pools *node_pools; 348 349 node_pools = priv->frag_buf_node_pools[node]; 350 if (!node_pools) 351 continue; 352 mlx5_frag_buf_node_pools_destroy(node_pools); 353 } 354 355 kfree(priv->frag_buf_node_pools); 356 priv->frag_buf_node_pools = NULL; 357 } 358 359 int mlx5_frag_buf_pools_init(struct mlx5_core_dev *dev) 360 { 361 struct mlx5_priv *priv = &dev->priv; 362 int node; 363 364 priv->frag_buf_node_pools = kzalloc_objs(*priv->frag_buf_node_pools, 365 nr_node_ids); 366 if (!priv->frag_buf_node_pools) 367 return -ENOMEM; 368 369 for_each_node_state(node, N_POSSIBLE) { 370 struct mlx5_frag_buf_node_pools *node_pools; 371 372 node_pools = mlx5_frag_buf_node_pools_create(dev, node); 373 if (!node_pools) { 374 mlx5_frag_buf_pools_cleanup(dev); 375 return -ENOMEM; 376 } 377 priv->frag_buf_node_pools[node] = node_pools; 378 } 379 380 priv->dbg.frag_buf_dma_pools_debugfs = 381 debugfs_create_file("frag_buf_dma_pools", 0444, 382 priv->dbg.dbg_root, dev, 383 &mlx5_frag_buf_dma_pools_debugfs_fops); 384 385 return 0; 386 } 387 388 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, 389 struct mlx5_frag_buf *buf, int node) 390 { 391 struct mlx5_dma_pool *pool; 392 int pool_idx; 393 394 node = node == NUMA_NO_NODE ? numa_mem_id() : node; 395 396 buf->size = size; 397 buf->npages = DIV_ROUND_UP(size, PAGE_SIZE); 398 buf->page_shift = clamp_t(int, order_base_2(size), 399 MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT, 400 PAGE_SHIFT); 401 buf->frags = kcalloc_node(buf->npages, sizeof(*buf->frags), 402 GFP_KERNEL, node); 403 if (!buf->frags) 404 return -ENOMEM; 405 406 pool_idx = buf->page_shift - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT; 407 pool = dev->priv.frag_buf_node_pools[node]->pools[pool_idx]; 408 for (int i = 0; i < buf->npages; i++) { 409 struct mlx5_buf_list *frag = &buf->frags[i]; 410 struct mlx5_dma_pool_page *page; 411 unsigned long idx; 412 413 page = mlx5_dma_pool_alloc(pool, &idx); 414 if (!page) { 415 mlx5_frag_buf_free(dev, buf); 416 return -ENOMEM; 417 } 418 frag->buf = (u8 *)page->buf + (idx << pool->block_shift); 419 frag->map = page->dma + (idx << pool->block_shift); 420 frag->frag_page = page; 421 } 422 423 return 0; 424 } 425 EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node); 426 427 void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) 428 { 429 for (int i = 0; i < buf->npages; i++) { 430 struct mlx5_buf_list *frag = &buf->frags[i]; 431 struct mlx5_dma_pool_page *page; 432 struct mlx5_dma_pool *pool; 433 unsigned long idx; 434 435 if (!frag->buf) 436 continue; 437 438 page = frag->frag_page; 439 pool = page->pool; 440 idx = (frag->map - page->dma) >> pool->block_shift; 441 mlx5_dma_pool_free(pool, page, idx); 442 } 443 kfree(buf->frags); 444 } 445 EXPORT_SYMBOL_GPL(mlx5_frag_buf_free); 446 447 static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, 448 int node) 449 { 450 u32 db_per_page = PAGE_SIZE / cache_line_size(); 451 struct mlx5_db_pgdir *pgdir; 452 453 pgdir = kzalloc_node(sizeof(*pgdir), GFP_KERNEL, node); 454 if (!pgdir) 455 return NULL; 456 457 pgdir->bitmap = bitmap_zalloc_node(db_per_page, GFP_KERNEL, node); 458 if (!pgdir->bitmap) { 459 kfree(pgdir); 460 return NULL; 461 } 462 463 bitmap_fill(pgdir->bitmap, db_per_page); 464 465 pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, 466 &pgdir->db_dma, node); 467 if (!pgdir->db_page) { 468 bitmap_free(pgdir->bitmap); 469 kfree(pgdir); 470 return NULL; 471 } 472 473 return pgdir; 474 } 475 476 static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir, 477 struct mlx5_db *db) 478 { 479 u32 db_per_page = PAGE_SIZE / cache_line_size(); 480 int offset; 481 int i; 482 483 i = find_first_bit(pgdir->bitmap, db_per_page); 484 if (i >= db_per_page) 485 return -ENOMEM; 486 487 __clear_bit(i, pgdir->bitmap); 488 489 db->u.pgdir = pgdir; 490 db->index = i; 491 offset = db->index * cache_line_size(); 492 db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); 493 db->dma = pgdir->db_dma + offset; 494 495 db->db[0] = 0; 496 db->db[1] = 0; 497 498 return 0; 499 } 500 501 int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node) 502 { 503 struct mlx5_db_pgdir *pgdir; 504 int ret = 0; 505 506 mutex_lock(&dev->priv.pgdir_mutex); 507 508 list_for_each_entry(pgdir, &dev->priv.pgdir_list, list) 509 if (!mlx5_alloc_db_from_pgdir(pgdir, db)) 510 goto out; 511 512 pgdir = mlx5_alloc_db_pgdir(dev, node); 513 if (!pgdir) { 514 ret = -ENOMEM; 515 goto out; 516 } 517 518 list_add(&pgdir->list, &dev->priv.pgdir_list); 519 520 /* This should never fail -- we just allocated an empty page: */ 521 WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db)); 522 523 out: 524 mutex_unlock(&dev->priv.pgdir_mutex); 525 526 return ret; 527 } 528 EXPORT_SYMBOL_GPL(mlx5_db_alloc_node); 529 530 void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) 531 { 532 u32 db_per_page = PAGE_SIZE / cache_line_size(); 533 534 mutex_lock(&dev->priv.pgdir_mutex); 535 536 __set_bit(db->index, db->u.pgdir->bitmap); 537 538 if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { 539 dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, 540 db->u.pgdir->db_page, db->u.pgdir->db_dma); 541 list_del(&db->u.pgdir->list); 542 bitmap_free(db->u.pgdir->bitmap); 543 kfree(db->u.pgdir); 544 } 545 546 mutex_unlock(&dev->priv.pgdir_mutex); 547 } 548 EXPORT_SYMBOL_GPL(mlx5_db_free); 549 550 void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm) 551 { 552 int i; 553 554 WARN_ON(perm & 0xfc); 555 for (i = 0; i < buf->npages; i++) 556 pas[i] = cpu_to_be64(buf->frags[i].map | perm); 557 } 558 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm); 559 560 void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) 561 { 562 mlx5_fill_page_frag_array_perm(buf, pas, 0); 563 } 564 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); 565