1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2017 Facebook Inc. 5 * Copyright (C) 2017 Dennis Zhou <dennis@kernel.org> 6 * 7 * Prints statistics about the percpu allocator and backing chunks. 8 */ 9 #include <linux/debugfs.h> 10 #include <linux/list.h> 11 #include <linux/percpu.h> 12 #include <linux/seq_file.h> 13 #include <linux/sort.h> 14 #include <linux/vmalloc.h> 15 16 #include "percpu-internal.h" 17 18 #define P(X, Y) \ 19 seq_printf(m, " %-20s: %12lld\n", X, (long long int)Y) 20 21 struct percpu_stats pcpu_stats; 22 struct pcpu_alloc_info pcpu_stats_ai; 23 24 static int cmpint(const void *a, const void *b) 25 { 26 return *(int *)a - *(int *)b; 27 } 28 29 /* 30 * Iterates over all chunks to find the max nr_alloc entries. 31 */ 32 static int find_max_nr_alloc(void) 33 { 34 struct pcpu_chunk *chunk; 35 int slot, max_nr_alloc; 36 37 max_nr_alloc = 0; 38 for (slot = 0; slot < pcpu_nr_slots; slot++) 39 list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) 40 max_nr_alloc = max(max_nr_alloc, chunk->nr_alloc); 41 42 return max_nr_alloc; 43 } 44 45 /* 46 * Prints out chunk state. Fragmentation is considered between 47 * the beginning of the chunk to the last allocation. 48 * 49 * All statistics are in bytes unless stated otherwise. 50 */ 51 static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk, 52 int *buffer) 53 { 54 struct pcpu_block_md *chunk_md = &chunk->chunk_md; 55 int i, last_alloc, as_len, start, end; 56 int *alloc_sizes, *p; 57 /* statistics */ 58 int sum_frag = 0, max_frag = 0; 59 int cur_min_alloc = 0, cur_med_alloc = 0, cur_max_alloc = 0; 60 61 alloc_sizes = buffer; 62 63 /* 64 * find_last_bit returns the start value if nothing found. 65 * Therefore, we must determine if it is a failure of find_last_bit 66 * and set the appropriate value. 67 */ 68 last_alloc = find_last_bit(chunk->alloc_map, 69 pcpu_chunk_map_bits(chunk) - 70 chunk->end_offset / PCPU_MIN_ALLOC_SIZE - 1); 71 last_alloc = test_bit(last_alloc, chunk->alloc_map) ? 72 last_alloc + 1 : 0; 73 74 as_len = 0; 75 start = chunk->start_offset / PCPU_MIN_ALLOC_SIZE; 76 77 /* 78 * If a bit is set in the allocation map, the bound_map identifies 79 * where the allocation ends. If the allocation is not set, the 80 * bound_map does not identify free areas as it is only kept accurate 81 * on allocation, not free. 82 * 83 * Positive values are allocations and negative values are free 84 * fragments. 85 */ 86 while (start < last_alloc) { 87 if (test_bit(start, chunk->alloc_map)) { 88 end = find_next_bit(chunk->bound_map, last_alloc, 89 start + 1); 90 alloc_sizes[as_len] = 1; 91 } else { 92 end = find_next_bit(chunk->alloc_map, last_alloc, 93 start + 1); 94 alloc_sizes[as_len] = -1; 95 } 96 97 alloc_sizes[as_len++] *= (end - start) * PCPU_MIN_ALLOC_SIZE; 98 99 start = end; 100 } 101 102 /* 103 * The negative values are free fragments and thus sorting gives the 104 * free fragments at the beginning in largest first order. 105 */ 106 if (as_len > 0) { 107 sort(alloc_sizes, as_len, sizeof(int), cmpint, NULL); 108 109 /* iterate through the unallocated fragments */ 110 for (i = 0, p = alloc_sizes; *p < 0 && i < as_len; i++, p++) { 111 sum_frag -= *p; 112 max_frag = max(max_frag, -1 * (*p)); 113 } 114 115 cur_min_alloc = alloc_sizes[i]; 116 cur_med_alloc = alloc_sizes[(i + as_len - 1) / 2]; 117 cur_max_alloc = alloc_sizes[as_len - 1]; 118 } 119 120 P("nr_alloc", chunk->nr_alloc); 121 P("max_alloc_size", chunk->max_alloc_size); 122 P("empty_pop_pages", chunk->nr_empty_pop_pages); 123 P("first_bit", chunk_md->first_free); 124 P("free_bytes", chunk->free_bytes); 125 P("contig_bytes", chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE); 126 P("sum_frag", sum_frag); 127 P("max_frag", max_frag); 128 P("cur_min_alloc", cur_min_alloc); 129 P("cur_med_alloc", cur_med_alloc); 130 P("cur_max_alloc", cur_max_alloc); 131 seq_putc(m, '\n'); 132 } 133 134 static int percpu_stats_show(struct seq_file *m, void *v) 135 { 136 struct pcpu_chunk *chunk; 137 int slot, max_nr_alloc; 138 int *buffer; 139 140 alloc_buffer: 141 spin_lock_irq(&pcpu_lock); 142 max_nr_alloc = find_max_nr_alloc(); 143 spin_unlock_irq(&pcpu_lock); 144 145 /* there can be at most this many free and allocated fragments */ 146 buffer = vmalloc_array(2 * max_nr_alloc + 1, sizeof(int)); 147 if (!buffer) 148 return -ENOMEM; 149 150 spin_lock_irq(&pcpu_lock); 151 152 /* if the buffer allocated earlier is too small */ 153 if (max_nr_alloc < find_max_nr_alloc()) { 154 spin_unlock_irq(&pcpu_lock); 155 vfree(buffer); 156 goto alloc_buffer; 157 } 158 159 #define PL(X) \ 160 seq_printf(m, " %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X) 161 162 seq_printf(m, 163 "Percpu Memory Statistics\n" 164 "Allocation Info:\n" 165 "----------------------------------------\n"); 166 PL(unit_size); 167 PL(static_size); 168 PL(reserved_size); 169 PL(dyn_size); 170 PL(atom_size); 171 PL(alloc_size); 172 seq_putc(m, '\n'); 173 174 #undef PL 175 176 #define PU(X) \ 177 seq_printf(m, " %-20s: %12llu\n", #X, (unsigned long long)pcpu_stats.X) 178 179 seq_printf(m, 180 "Global Stats:\n" 181 "----------------------------------------\n"); 182 PU(nr_alloc); 183 PU(nr_dealloc); 184 PU(nr_cur_alloc); 185 PU(nr_max_alloc); 186 PU(nr_chunks); 187 PU(nr_max_chunks); 188 PU(min_alloc_size); 189 PU(max_alloc_size); 190 P("empty_pop_pages", pcpu_nr_empty_pop_pages); 191 seq_putc(m, '\n'); 192 193 #undef PU 194 195 seq_printf(m, 196 "Per Chunk Stats:\n" 197 "----------------------------------------\n"); 198 199 if (pcpu_reserved_chunk) { 200 seq_puts(m, "Chunk: <- Reserved Chunk\n"); 201 chunk_map_stats(m, pcpu_reserved_chunk, buffer); 202 } 203 204 for (slot = 0; slot < pcpu_nr_slots; slot++) { 205 list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) { 206 if (chunk == pcpu_first_chunk) 207 seq_puts(m, "Chunk: <- First Chunk\n"); 208 else if (slot == pcpu_to_depopulate_slot) 209 seq_puts(m, "Chunk (to_depopulate)\n"); 210 else if (slot == pcpu_sidelined_slot) 211 seq_puts(m, "Chunk (sidelined):\n"); 212 else 213 seq_puts(m, "Chunk:\n"); 214 chunk_map_stats(m, chunk, buffer); 215 } 216 } 217 218 spin_unlock_irq(&pcpu_lock); 219 220 vfree(buffer); 221 222 return 0; 223 } 224 DEFINE_SHOW_ATTRIBUTE(percpu_stats); 225 226 static int __init init_percpu_stats_debugfs(void) 227 { 228 debugfs_create_file("percpu_stats", 0444, NULL, NULL, 229 &percpu_stats_fops); 230 231 return 0; 232 } 233 234 late_initcall(init_percpu_stats_debugfs); 235