1 #include <linux/kernel.h> 2 #include <linux/module.h> 3 #include <linux/percpu_ida.h> 4 5 #include <linux/blk-mq.h> 6 #include "blk.h" 7 #include "blk-mq.h" 8 #include "blk-mq-tag.h" 9 10 /* 11 * Per tagged queue (tag address space) map 12 */ 13 struct blk_mq_tags { 14 unsigned int nr_tags; 15 unsigned int nr_reserved_tags; 16 unsigned int nr_batch_move; 17 unsigned int nr_max_cache; 18 19 struct percpu_ida free_tags; 20 struct percpu_ida reserved_tags; 21 }; 22 23 void blk_mq_wait_for_tags(struct blk_mq_tags *tags) 24 { 25 int tag = blk_mq_get_tag(tags, __GFP_WAIT, false); 26 blk_mq_put_tag(tags, tag); 27 } 28 29 bool blk_mq_has_free_tags(struct blk_mq_tags *tags) 30 { 31 return !tags || 32 percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids) != 0; 33 } 34 35 static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp) 36 { 37 int tag; 38 39 tag = percpu_ida_alloc(&tags->free_tags, gfp); 40 if (tag < 0) 41 return BLK_MQ_TAG_FAIL; 42 return tag + tags->nr_reserved_tags; 43 } 44 45 static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags, 46 gfp_t gfp) 47 { 48 int tag; 49 50 if (unlikely(!tags->nr_reserved_tags)) { 51 WARN_ON_ONCE(1); 52 return BLK_MQ_TAG_FAIL; 53 } 54 55 tag = percpu_ida_alloc(&tags->reserved_tags, gfp); 56 if (tag < 0) 57 return BLK_MQ_TAG_FAIL; 58 return tag; 59 } 60 61 unsigned int blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp, bool reserved) 62 { 63 if (!reserved) 64 return __blk_mq_get_tag(tags, gfp); 65 66 return __blk_mq_get_reserved_tag(tags, gfp); 67 } 68 69 static void __blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag) 70 { 71 BUG_ON(tag >= tags->nr_tags); 72 73 percpu_ida_free(&tags->free_tags, tag - tags->nr_reserved_tags); 74 } 75 76 static void __blk_mq_put_reserved_tag(struct blk_mq_tags *tags, 77 unsigned int tag) 78 { 79 BUG_ON(tag >= tags->nr_reserved_tags); 80 81 percpu_ida_free(&tags->reserved_tags, tag); 82 } 83 84 void blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag) 85 { 86 if (tag >= tags->nr_reserved_tags) 87 __blk_mq_put_tag(tags, tag); 88 else 89 __blk_mq_put_reserved_tag(tags, tag); 90 } 91 92 static int __blk_mq_tag_iter(unsigned id, void *data) 93 { 94 unsigned long *tag_map = data; 95 __set_bit(id, tag_map); 96 return 0; 97 } 98 99 void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, 100 void (*fn)(void *, unsigned long *), void *data) 101 { 102 unsigned long *tag_map; 103 size_t map_size; 104 105 map_size = ALIGN(tags->nr_tags, BITS_PER_LONG) / BITS_PER_LONG; 106 tag_map = kzalloc(map_size * sizeof(unsigned long), GFP_ATOMIC); 107 if (!tag_map) 108 return; 109 110 percpu_ida_for_each_free(&tags->free_tags, __blk_mq_tag_iter, tag_map); 111 if (tags->nr_reserved_tags) 112 percpu_ida_for_each_free(&tags->reserved_tags, __blk_mq_tag_iter, 113 tag_map); 114 115 fn(data, tag_map); 116 kfree(tag_map); 117 } 118 119 struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, 120 unsigned int reserved_tags, int node) 121 { 122 unsigned int nr_tags, nr_cache; 123 struct blk_mq_tags *tags; 124 int ret; 125 126 if (total_tags > BLK_MQ_TAG_MAX) { 127 pr_err("blk-mq: tag depth too large\n"); 128 return NULL; 129 } 130 131 tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); 132 if (!tags) 133 return NULL; 134 135 nr_tags = total_tags - reserved_tags; 136 nr_cache = nr_tags / num_possible_cpus(); 137 138 if (nr_cache < BLK_MQ_TAG_CACHE_MIN) 139 nr_cache = BLK_MQ_TAG_CACHE_MIN; 140 else if (nr_cache > BLK_MQ_TAG_CACHE_MAX) 141 nr_cache = BLK_MQ_TAG_CACHE_MAX; 142 143 tags->nr_tags = total_tags; 144 tags->nr_reserved_tags = reserved_tags; 145 tags->nr_max_cache = nr_cache; 146 tags->nr_batch_move = max(1u, nr_cache / 2); 147 148 ret = __percpu_ida_init(&tags->free_tags, tags->nr_tags - 149 tags->nr_reserved_tags, 150 tags->nr_max_cache, 151 tags->nr_batch_move); 152 if (ret) 153 goto err_free_tags; 154 155 if (reserved_tags) { 156 /* 157 * With max_cahe and batch set to 1, the allocator fallbacks to 158 * no cached. It's fine reserved tags allocation is slow. 159 */ 160 ret = __percpu_ida_init(&tags->reserved_tags, reserved_tags, 161 1, 1); 162 if (ret) 163 goto err_reserved_tags; 164 } 165 166 return tags; 167 168 err_reserved_tags: 169 percpu_ida_destroy(&tags->free_tags); 170 err_free_tags: 171 kfree(tags); 172 return NULL; 173 } 174 175 void blk_mq_free_tags(struct blk_mq_tags *tags) 176 { 177 percpu_ida_destroy(&tags->free_tags); 178 percpu_ida_destroy(&tags->reserved_tags); 179 kfree(tags); 180 } 181 182 ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page) 183 { 184 char *orig_page = page; 185 int cpu; 186 187 if (!tags) 188 return 0; 189 190 page += sprintf(page, "nr_tags=%u, reserved_tags=%u, batch_move=%u," 191 " max_cache=%u\n", tags->nr_tags, tags->nr_reserved_tags, 192 tags->nr_batch_move, tags->nr_max_cache); 193 194 page += sprintf(page, "nr_free=%u, nr_reserved=%u\n", 195 percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids), 196 percpu_ida_free_tags(&tags->reserved_tags, nr_cpu_ids)); 197 198 for_each_possible_cpu(cpu) { 199 page += sprintf(page, " cpu%02u: nr_free=%u\n", cpu, 200 percpu_ida_free_tags(&tags->free_tags, cpu)); 201 } 202 203 return page - orig_page; 204 } 205