blk-mq.c (a347c7ad8edf4c5685154f3fdc3c12fc1db800ba) blk-mq.c (590b5b7d8671e011d1a8e1ab20c60addb249d015)
1/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
7#include <linux/kernel.h>
8#include <linux/module.h>

--- 1889 unchanged lines hidden (view full) ---

1898 if (node == NUMA_NO_NODE)
1899 node = set->numa_node;
1900
1901 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
1902 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1903 if (!tags)
1904 return NULL;
1905
1/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
7#include <linux/kernel.h>
8#include <linux/module.h>

--- 1889 unchanged lines hidden (view full) ---

1898 if (node == NUMA_NO_NODE)
1899 node = set->numa_node;
1900
1901 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
1902 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1903 if (!tags)
1904 return NULL;
1905
1906 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1906 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
1907 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1908 node);
1909 if (!tags->rqs) {
1910 blk_mq_free_tags(tags);
1911 return NULL;
1912 }
1913
1907 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1908 node);
1909 if (!tags->rqs) {
1910 blk_mq_free_tags(tags);
1911 return NULL;
1912 }
1913
1914 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1915 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1916 node);
1914 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
1915 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1916 node);
1917 if (!tags->static_rqs) {
1918 kfree(tags->rqs);
1919 blk_mq_free_tags(tags);
1920 return NULL;
1921 }
1922
1923 return tags;
1924}

--- 419 unchanged lines hidden (view full) ---

2344}
2345
2346static void blk_mq_del_queue_tag_set(struct request_queue *q)
2347{
2348 struct blk_mq_tag_set *set = q->tag_set;
2349
2350 mutex_lock(&set->tag_list_lock);
2351 list_del_rcu(&q->tag_set_list);
1917 if (!tags->static_rqs) {
1918 kfree(tags->rqs);
1919 blk_mq_free_tags(tags);
1920 return NULL;
1921 }
1922
1923 return tags;
1924}

--- 419 unchanged lines hidden (view full) ---

2344}
2345
2346static void blk_mq_del_queue_tag_set(struct request_queue *q)
2347{
2348 struct blk_mq_tag_set *set = q->tag_set;
2349
2350 mutex_lock(&set->tag_list_lock);
2351 list_del_rcu(&q->tag_set_list);
2352 INIT_LIST_HEAD(&q->tag_set_list);
2352 if (list_is_singular(&set->tag_list)) {
2353 /* just transitioned to unshared */
2354 set->flags &= ~BLK_MQ_F_TAG_SHARED;
2355 /* update existing queue */
2356 blk_mq_update_tag_set_depth(set, false);
2357 }
2358 mutex_unlock(&set->tag_list_lock);
2353 if (list_is_singular(&set->tag_list)) {
2354 /* just transitioned to unshared */
2355 set->flags &= ~BLK_MQ_F_TAG_SHARED;
2356 /* update existing queue */
2357 blk_mq_update_tag_set_depth(set, false);
2358 }
2359 mutex_unlock(&set->tag_list_lock);
2360
2359 synchronize_rcu();
2361 synchronize_rcu();
2360 INIT_LIST_HEAD(&q->tag_set_list);
2361}
2362
2363static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2364 struct request_queue *q)
2365{
2366 q->tag_set = set;
2367
2368 mutex_lock(&set->tag_list_lock);

--- 147 unchanged lines hidden (view full) ---

2516
2517 q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2518 if (!q->queue_ctx)
2519 goto err_exit;
2520
2521 /* init q->mq_kobj and sw queues' kobjects */
2522 blk_mq_sysfs_init(q);
2523
2362}
2363
2364static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2365 struct request_queue *q)
2366{
2367 q->tag_set = set;
2368
2369 mutex_lock(&set->tag_list_lock);

--- 147 unchanged lines hidden (view full) ---

2517
2518 q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2519 if (!q->queue_ctx)
2520 goto err_exit;
2521
2522 /* init q->mq_kobj and sw queues' kobjects */
2523 blk_mq_sysfs_init(q);
2524
2524 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2525 q->queue_hw_ctx = kcalloc_node(nr_cpu_ids, sizeof(*(q->queue_hw_ctx)),
2525 GFP_KERNEL, set->numa_node);
2526 if (!q->queue_hw_ctx)
2527 goto err_percpu;
2528
2529 q->mq_map = set->mq_map;
2530
2531 blk_mq_realloc_hw_ctxs(set, q);
2532 if (!q->nr_hw_queues)

--- 202 unchanged lines hidden (view full) ---

2735 set->queue_depth = min(64U, set->queue_depth);
2736 }
2737 /*
2738 * There is no use for more h/w queues than cpus.
2739 */
2740 if (set->nr_hw_queues > nr_cpu_ids)
2741 set->nr_hw_queues = nr_cpu_ids;
2742
2526 GFP_KERNEL, set->numa_node);
2527 if (!q->queue_hw_ctx)
2528 goto err_percpu;
2529
2530 q->mq_map = set->mq_map;
2531
2532 blk_mq_realloc_hw_ctxs(set, q);
2533 if (!q->nr_hw_queues)

--- 202 unchanged lines hidden (view full) ---

2736 set->queue_depth = min(64U, set->queue_depth);
2737 }
2738 /*
2739 * There is no use for more h/w queues than cpus.
2740 */
2741 if (set->nr_hw_queues > nr_cpu_ids)
2742 set->nr_hw_queues = nr_cpu_ids;
2743
2743 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2744 set->tags = kcalloc_node(nr_cpu_ids, sizeof(struct blk_mq_tags *),
2744 GFP_KERNEL, set->numa_node);
2745 if (!set->tags)
2746 return -ENOMEM;
2747
2748 ret = -ENOMEM;
2745 GFP_KERNEL, set->numa_node);
2746 if (!set->tags)
2747 return -ENOMEM;
2748
2749 ret = -ENOMEM;
2749 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
2750 GFP_KERNEL, set->numa_node);
2750 set->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*set->mq_map),
2751 GFP_KERNEL, set->numa_node);
2751 if (!set->mq_map)
2752 goto out_free_tags;
2753
2754 ret = blk_mq_update_queue_map(set);
2755 if (ret)
2756 goto out_free_mq_map;
2757
2758 ret = blk_mq_alloc_rq_maps(set);

--- 309 unchanged lines hidden --- 		2752 if (!set->mq_map)
2753 goto out_free_tags;
2754
2755 ret = blk_mq_update_queue_map(set);
2756 if (ret)
2757 goto out_free_mq_map;
2758
2759 ret = blk_mq_alloc_rq_maps(set);

--- 309 unchanged lines hidden ---