1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/errno.h>
34 #include <linux/slab.h>
35 #include <linux/mm.h>
36 #include <linux/export.h>
37 #include <linux/bitmap.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/vmalloc.h>
40 #include <linux/nodemask.h>
41 #include <linux/debugfs.h>
42 #include <linux/seq_file.h>
43 #include <linux/mlx5/driver.h>
44
45 #include "mlx5_core.h"
46
47 #define MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT MLX5_ADAPTER_PAGE_SHIFT
48 #define MLX5_FRAG_BUF_POOLS_NUM \
49 (PAGE_SHIFT - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + 1)
50
51 struct mlx5_dma_pool {
52 /* Protects page_list and per-page allocation bitmaps. */
53 struct mutex lock;
54 struct list_head page_list;
55 struct mlx5_core_dev *dev;
56 int node;
57 u8 block_shift;
58 };
59
60 struct mlx5_dma_pool_page {
61 struct mlx5_dma_pool *pool;
62 struct list_head pool_link;
63 unsigned long *bitmap;
64 void *buf;
65 dma_addr_t dma;
66 };
67
68 struct mlx5_frag_buf_node_pools {
69 struct mlx5_dma_pool *pools[MLX5_FRAG_BUF_POOLS_NUM];
70 };
71
72 struct mlx5_dma_pool_stats {
73 int node;
74 size_t block_size;
75 size_t used_blocks;
76 size_t allocated_blocks;
77 };
78
79 /* Handling for queue buffers -- we allocate a bunch of memory and
80 * register it in a memory region at HCA virtual address 0.
81 */
82
mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev * dev,size_t size,dma_addr_t * dma_handle,int node)83 static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
84 size_t size, dma_addr_t *dma_handle,
85 int node)
86 {
87 struct device *device = mlx5_core_dma_dev(dev);
88 struct mlx5_priv *priv = &dev->priv;
89 int original_node;
90 void *cpu_handle;
91
92 mutex_lock(&priv->alloc_mutex);
93 original_node = dev_to_node(device);
94 set_dev_node(device, node);
95 cpu_handle = dma_alloc_coherent(device, size, dma_handle,
96 GFP_KERNEL);
97 set_dev_node(device, original_node);
98 mutex_unlock(&priv->alloc_mutex);
99 return cpu_handle;
100 }
101
mlx5_dma_pool_destroy(struct mlx5_dma_pool * pool)102 static void mlx5_dma_pool_destroy(struct mlx5_dma_pool *pool)
103 {
104 mutex_destroy(&pool->lock);
105 kfree(pool);
106 }
107
mlx5_dma_pool_create(struct mlx5_core_dev * dev,int node,u8 block_shift)108 static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev,
109 int node, u8 block_shift)
110 {
111 struct mlx5_dma_pool *pool;
112
113 pool = kzalloc_obj(*pool);
114 if (!pool)
115 return NULL;
116
117 INIT_LIST_HEAD(&pool->page_list);
118 mutex_init(&pool->lock);
119 pool->dev = dev;
120 pool->node = node;
121 pool->block_shift = block_shift;
122 return pool;
123 }
124
125 static struct mlx5_dma_pool_page *
mlx5_dma_pool_page_alloc(struct mlx5_dma_pool * pool)126 mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool)
127 {
128 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
129 struct mlx5_dma_pool_page *page;
130
131 page = kzalloc_obj(*page);
132 if (!page)
133 goto err_out;
134
135 page->pool = pool;
136 page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL);
137 if (!page->bitmap)
138 goto err_free_page;
139
140 bitmap_fill(page->bitmap, blocks_per_page);
141 page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE,
142 &page->dma, pool->node);
143 if (!page->buf)
144 goto err_free_bitmap;
145
146 return page;
147
148 err_free_bitmap:
149 bitmap_free(page->bitmap);
150 err_free_page:
151 kfree(page);
152 err_out:
153 return NULL;
154 }
155
mlx5_dma_pool_page_free(struct mlx5_core_dev * dev,struct mlx5_dma_pool_page * page)156 static void mlx5_dma_pool_page_free(struct mlx5_core_dev *dev,
157 struct mlx5_dma_pool_page *page)
158 {
159 dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, page->buf,
160 page->dma);
161 bitmap_free(page->bitmap);
162 kfree(page);
163 }
164
mlx5_dma_pool_alloc_from_page(struct mlx5_dma_pool * pool,struct mlx5_dma_pool_page * page,unsigned long * idx_out)165 static int mlx5_dma_pool_alloc_from_page(struct mlx5_dma_pool *pool,
166 struct mlx5_dma_pool_page *page,
167 unsigned long *idx_out)
168 {
169 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
170
171 *idx_out = find_first_bit(page->bitmap, blocks_per_page);
172 if (*idx_out >= blocks_per_page)
173 return -ENOMEM;
174
175 __clear_bit(*idx_out, page->bitmap);
176
177 if (bitmap_empty(page->bitmap, blocks_per_page))
178 list_move_tail(&page->pool_link, &pool->page_list);
179
180 return 0;
181 }
182
183 static struct mlx5_dma_pool_page *
mlx5_dma_pool_alloc(struct mlx5_dma_pool * pool,unsigned long * idx_out)184 mlx5_dma_pool_alloc(struct mlx5_dma_pool *pool, unsigned long *idx_out)
185 {
186 struct mlx5_dma_pool_page *page;
187
188 mutex_lock(&pool->lock);
189
190 page = list_first_entry_or_null(&pool->page_list,
191 struct mlx5_dma_pool_page, pool_link);
192 if (page && !mlx5_dma_pool_alloc_from_page(pool, page, idx_out))
193 goto unlock; /* successfully allocated from existing page */
194
195 page = mlx5_dma_pool_page_alloc(pool);
196 if (!page)
197 goto unlock;
198
199 list_add(&page->pool_link, &pool->page_list);
200 mlx5_dma_pool_alloc_from_page(pool, page, idx_out);
201
202 unlock:
203 mutex_unlock(&pool->lock);
204 return page;
205 }
206
mlx5_dma_pool_free(struct mlx5_dma_pool * pool,struct mlx5_dma_pool_page * page,unsigned long idx)207 static void mlx5_dma_pool_free(struct mlx5_dma_pool *pool,
208 struct mlx5_dma_pool_page *page,
209 unsigned long idx)
210 {
211 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
212 bool was_full;
213
214 mutex_lock(&pool->lock);
215 was_full = bitmap_empty(page->bitmap, blocks_per_page);
216 __set_bit(idx, page->bitmap);
217
218 if (bitmap_full(page->bitmap, blocks_per_page)) {
219 list_del(&page->pool_link);
220 mlx5_dma_pool_page_free(pool->dev, page);
221 } else {
222 memset((u8 *)page->buf + (idx << pool->block_shift), 0,
223 BIT(pool->block_shift));
224 if (was_full)
225 list_move(&page->pool_link, &pool->page_list);
226 }
227 mutex_unlock(&pool->lock);
228 }
229
mlx5_dma_pool_debugfs_get_stats(struct mlx5_dma_pool * pool,struct mlx5_dma_pool_stats * stats)230 static void mlx5_dma_pool_debugfs_get_stats(struct mlx5_dma_pool *pool,
231 struct mlx5_dma_pool_stats *stats)
232 {
233 int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
234 struct mlx5_dma_pool_page *page;
235 size_t free_blocks = 0;
236 size_t pages = 0;
237
238 mutex_lock(&pool->lock);
239 list_for_each_entry(page, &pool->page_list, pool_link) {
240 pages++;
241 free_blocks += bitmap_weight(page->bitmap, blocks_per_page);
242 }
243 mutex_unlock(&pool->lock);
244
245 stats->node = pool->node;
246 stats->block_size = BIT(pool->block_shift);
247 stats->allocated_blocks = pages * blocks_per_page;
248 stats->used_blocks = stats->allocated_blocks - free_blocks;
249 }
250
mlx5_dma_pool_debugfs_stats_print(struct seq_file * file,struct mlx5_dma_pool * pool)251 static void mlx5_dma_pool_debugfs_stats_print(struct seq_file *file,
252 struct mlx5_dma_pool *pool)
253 {
254 struct mlx5_dma_pool_stats stats = {};
255
256 mlx5_dma_pool_debugfs_get_stats(pool, &stats);
257 seq_printf(file, "%4d %5zu %7zu %7zu\n",
258 stats.node, stats.block_size, stats.used_blocks,
259 stats.allocated_blocks);
260 }
261
mlx5_dma_pools_debugfs_print_header(struct seq_file * file)262 static void mlx5_dma_pools_debugfs_print_header(struct seq_file *file)
263 {
264 seq_puts(file, "node block_size used_blocks allocated_blocks\n");
265 }
266
267 static void
mlx5_frag_buf_node_pools_destroy(struct mlx5_frag_buf_node_pools * node_pools)268 mlx5_frag_buf_node_pools_destroy(struct mlx5_frag_buf_node_pools *node_pools)
269 {
270 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++)
271 if (node_pools->pools[i])
272 mlx5_dma_pool_destroy(node_pools->pools[i]);
273 kfree(node_pools);
274 }
275
276 static struct mlx5_frag_buf_node_pools *
mlx5_frag_buf_node_pools_create(struct mlx5_core_dev * dev,int node)277 mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node)
278 {
279 struct mlx5_frag_buf_node_pools *node_pools;
280
281 node_pools = kzalloc_obj(*node_pools);
282 if (!node_pools)
283 return NULL;
284
285 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) {
286 u8 block_shift = MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + i;
287
288 node_pools->pools[i] = mlx5_dma_pool_create(dev, node,
289 block_shift);
290 if (!node_pools->pools[i]) {
291 mlx5_frag_buf_node_pools_destroy(node_pools);
292 return NULL;
293 }
294 }
295
296 return node_pools;
297 }
298
299 static int
mlx5_frag_buf_dma_pools_debugfs_show(struct seq_file * file,void * priv)300 mlx5_frag_buf_dma_pools_debugfs_show(struct seq_file *file, void *priv)
301 {
302 struct mlx5_core_dev *dev = file->private;
303 int node;
304
305 mlx5_dma_pools_debugfs_print_header(file);
306
307 if (!dev->priv.frag_buf_node_pools)
308 return 0;
309
310 for_each_node_state(node, N_POSSIBLE) {
311 struct mlx5_frag_buf_node_pools *node_pools;
312
313 node_pools = dev->priv.frag_buf_node_pools[node];
314 if (!node_pools)
315 continue;
316
317 for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) {
318 struct mlx5_dma_pool *pool = node_pools->pools[i];
319
320 if (!pool)
321 continue;
322
323 mlx5_dma_pool_debugfs_stats_print(file, pool);
324 }
325 }
326
327 return 0;
328 }
329 DEFINE_SHOW_ATTRIBUTE(mlx5_frag_buf_dma_pools_debugfs);
330
mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev * dev)331 void mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev *dev)
332 {
333 struct mlx5_priv *priv = &dev->priv;
334 int node;
335
336 debugfs_remove(priv->dbg.frag_buf_dma_pools_debugfs);
337 priv->dbg.frag_buf_dma_pools_debugfs = NULL;
338
339 for_each_node_state(node, N_POSSIBLE) {
340 struct mlx5_frag_buf_node_pools *node_pools;
341
342 node_pools = priv->frag_buf_node_pools[node];
343 if (!node_pools)
344 continue;
345 mlx5_frag_buf_node_pools_destroy(node_pools);
346 }
347
348 kfree(priv->frag_buf_node_pools);
349 priv->frag_buf_node_pools = NULL;
350 }
351
mlx5_frag_buf_pools_init(struct mlx5_core_dev * dev)352 int mlx5_frag_buf_pools_init(struct mlx5_core_dev *dev)
353 {
354 struct mlx5_priv *priv = &dev->priv;
355 int node;
356
357 priv->frag_buf_node_pools = kzalloc_objs(*priv->frag_buf_node_pools,
358 nr_node_ids);
359 if (!priv->frag_buf_node_pools)
360 return -ENOMEM;
361
362 for_each_node_state(node, N_POSSIBLE) {
363 struct mlx5_frag_buf_node_pools *node_pools;
364
365 node_pools = mlx5_frag_buf_node_pools_create(dev, node);
366 if (!node_pools) {
367 mlx5_frag_buf_pools_cleanup(dev);
368 return -ENOMEM;
369 }
370 priv->frag_buf_node_pools[node] = node_pools;
371 }
372
373 priv->dbg.frag_buf_dma_pools_debugfs =
374 debugfs_create_file("frag_buf_dma_pools", 0444,
375 priv->dbg.dbg_root, dev,
376 &mlx5_frag_buf_dma_pools_debugfs_fops);
377
378 return 0;
379 }
380
mlx5_frag_buf_alloc_node(struct mlx5_core_dev * dev,int size,struct mlx5_frag_buf * buf,int node)381 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size,
382 struct mlx5_frag_buf *buf, int node)
383 {
384 struct mlx5_dma_pool *pool;
385 int pool_idx;
386
387 node = node == NUMA_NO_NODE ? numa_mem_id() : node;
388
389 buf->size = size;
390 buf->npages = DIV_ROUND_UP(size, PAGE_SIZE);
391 buf->page_shift = clamp_t(int, order_base_2(size),
392 MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT,
393 PAGE_SHIFT);
394 buf->frags = kcalloc_node(buf->npages, sizeof(*buf->frags),
395 GFP_KERNEL, node);
396 if (!buf->frags)
397 return -ENOMEM;
398
399 pool_idx = buf->page_shift - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT;
400 pool = dev->priv.frag_buf_node_pools[node]->pools[pool_idx];
401 for (int i = 0; i < buf->npages; i++) {
402 struct mlx5_buf_list *frag = &buf->frags[i];
403 struct mlx5_dma_pool_page *page;
404 unsigned long idx;
405
406 page = mlx5_dma_pool_alloc(pool, &idx);
407 if (!page) {
408 mlx5_frag_buf_free(dev, buf);
409 return -ENOMEM;
410 }
411 frag->buf = (u8 *)page->buf + (idx << pool->block_shift);
412 frag->map = page->dma + (idx << pool->block_shift);
413 frag->frag_page = page;
414 }
415
416 return 0;
417 }
418 EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node);
419
mlx5_frag_buf_free(struct mlx5_core_dev * dev,struct mlx5_frag_buf * buf)420 void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)
421 {
422 for (int i = 0; i < buf->npages; i++) {
423 struct mlx5_buf_list *frag = &buf->frags[i];
424 struct mlx5_dma_pool_page *page;
425 struct mlx5_dma_pool *pool;
426 unsigned long idx;
427
428 if (!frag->buf)
429 continue;
430
431 page = frag->frag_page;
432 pool = page->pool;
433 idx = (frag->map - page->dma) >> pool->block_shift;
434 mlx5_dma_pool_free(pool, page, idx);
435 }
436 kfree(buf->frags);
437 }
438 EXPORT_SYMBOL_GPL(mlx5_frag_buf_free);
439
mlx5_db_dma_pools_debugfs_show(struct seq_file * file,void * priv)440 static int mlx5_db_dma_pools_debugfs_show(struct seq_file *file, void *priv)
441 {
442 struct mlx5_core_dev *dev = file->private;
443 int node;
444
445 mlx5_dma_pools_debugfs_print_header(file);
446
447 for_each_node_state(node, N_POSSIBLE) {
448 struct mlx5_dma_pool *pool = dev->priv.db_node_pools[node];
449
450 if (!pool)
451 continue;
452
453 mlx5_dma_pool_debugfs_stats_print(file, pool);
454 }
455
456 return 0;
457 }
458 DEFINE_SHOW_ATTRIBUTE(mlx5_db_dma_pools_debugfs);
459
mlx5_db_pools_cleanup(struct mlx5_core_dev * dev)460 void mlx5_db_pools_cleanup(struct mlx5_core_dev *dev)
461 {
462 struct mlx5_priv *priv = &dev->priv;
463 int node;
464
465 debugfs_remove(priv->dbg.db_dma_pools_debugfs);
466 priv->dbg.db_dma_pools_debugfs = NULL;
467
468 for_each_node_state(node, N_POSSIBLE)
469 if (priv->db_node_pools[node])
470 mlx5_dma_pool_destroy(priv->db_node_pools[node]);
471
472 kfree(priv->db_node_pools);
473 priv->db_node_pools = NULL;
474 }
475
mlx5_db_pools_init(struct mlx5_core_dev * dev)476 int mlx5_db_pools_init(struct mlx5_core_dev *dev)
477 {
478 struct mlx5_priv *priv = &dev->priv;
479 int node;
480
481 priv->db_node_pools = kzalloc_objs(*priv->db_node_pools, nr_node_ids);
482 if (!priv->db_node_pools)
483 return -ENOMEM;
484
485 for_each_node_state(node, N_POSSIBLE) {
486 struct mlx5_dma_pool *pool;
487
488 pool = mlx5_dma_pool_create(dev, node,
489 order_base_2(cache_line_size()));
490 if (!pool) {
491 mlx5_db_pools_cleanup(dev);
492 return -ENOMEM;
493 }
494 priv->db_node_pools[node] = pool;
495 }
496
497 priv->dbg.db_dma_pools_debugfs =
498 debugfs_create_file("db_dma_pools", 0444, priv->dbg.dbg_root,
499 dev, &mlx5_db_dma_pools_debugfs_fops);
500
501 return 0;
502 }
503
mlx5_db_alloc_node(struct mlx5_core_dev * dev,struct mlx5_db * db,int node)504 int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node)
505 {
506 struct mlx5_dma_pool_page *page;
507 struct mlx5_dma_pool *pool;
508 unsigned long idx;
509 int offset;
510
511 node = node == NUMA_NO_NODE ? numa_mem_id() : node;
512
513 pool = dev->priv.db_node_pools[node];
514 page = mlx5_dma_pool_alloc(pool, &idx);
515 if (!page)
516 return -ENOMEM;
517
518 offset = idx << pool->block_shift;
519 db->u.pool_page = page;
520 db->index = idx;
521 db->db = (__be32 *)((u8 *)page->buf + offset);
522 db->dma = page->dma + offset;
523
524 return 0;
525 }
526 EXPORT_SYMBOL_GPL(mlx5_db_alloc_node);
527
mlx5_db_free(struct mlx5_core_dev * dev,struct mlx5_db * db)528 void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db)
529 {
530 struct mlx5_dma_pool_page *page = db->u.pool_page;
531 struct mlx5_dma_pool *pool = page->pool;
532
533 mlx5_dma_pool_free(pool, page, db->index);
534 }
535 EXPORT_SYMBOL_GPL(mlx5_db_free);
536
mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf * buf,__be64 * pas,u8 perm)537 void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm)
538 {
539 int i;
540
541 WARN_ON(perm & 0xfc);
542 for (i = 0; i < buf->npages; i++)
543 pas[i] = cpu_to_be64(buf->frags[i].map | perm);
544 }
545 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm);
546
mlx5_fill_page_frag_array(struct mlx5_frag_buf * buf,__be64 * pas)547 void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
548 {
549 mlx5_fill_page_frag_array_perm(buf, pas, 0);
550 }
551 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array);
552