1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) IBM Corporation, 2014, 2017 4 * Anton Blanchard, Rashmica Gupta. 5 */ 6 7 #define pr_fmt(fmt) "memtrace: " fmt 8 9 #include <linux/bitops.h> 10 #include <linux/string.h> 11 #include <linux/memblock.h> 12 #include <linux/init.h> 13 #include <linux/moduleparam.h> 14 #include <linux/fs.h> 15 #include <linux/debugfs.h> 16 #include <linux/slab.h> 17 #include <linux/memory.h> 18 #include <linux/memory_hotplug.h> 19 #include <linux/numa.h> 20 #include <asm/machdep.h> 21 #include <asm/cacheflush.h> 22 23 /* This enables us to keep track of the memory removed from each node. */ 24 struct memtrace_entry { 25 void *mem; 26 u64 start; 27 u64 size; 28 u32 nid; 29 struct dentry *dir; 30 char name[16]; 31 }; 32 33 static DEFINE_MUTEX(memtrace_mutex); 34 static u64 memtrace_size; 35 36 static struct memtrace_entry *memtrace_array; 37 static unsigned int memtrace_array_nr; 38 39 40 static ssize_t memtrace_read(struct file *filp, char __user *ubuf, 41 size_t count, loff_t *ppos) 42 { 43 struct memtrace_entry *ent = filp->private_data; 44 45 return simple_read_from_buffer(ubuf, count, ppos, ent->mem, ent->size); 46 } 47 48 static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma) 49 { 50 struct memtrace_entry *ent = filp->private_data; 51 52 if (ent->size < vma->vm_end - vma->vm_start) 53 return -EINVAL; 54 55 if (vma->vm_pgoff << PAGE_SHIFT >= ent->size) 56 return -EINVAL; 57 58 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 59 return remap_pfn_range(vma, vma->vm_start, PHYS_PFN(ent->start) + vma->vm_pgoff, 60 vma->vm_end - vma->vm_start, vma->vm_page_prot); 61 } 62 63 static const struct file_operations memtrace_fops = { 64 .llseek = default_llseek, 65 .read = memtrace_read, 66 .open = simple_open, 67 .mmap = memtrace_mmap, 68 }; 69 70 #define FLUSH_CHUNK_SIZE SZ_1G 71 /** 72 * flush_dcache_range_chunked(): Write any modified data cache blocks out to 73 * memory and invalidate them, in chunks of up to FLUSH_CHUNK_SIZE 74 * Does not invalidate the corresponding instruction cache blocks. 75 * 76 * @start: the start address 77 * @stop: the stop address (exclusive) 78 * @chunk: the max size of the chunks 79 */ 80 static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, 81 unsigned long chunk) 82 { 83 unsigned long i; 84 85 for (i = start; i < stop; i += chunk) { 86 flush_dcache_range(i, min(stop, i + chunk)); 87 cond_resched(); 88 } 89 } 90 91 static u64 memtrace_alloc_node(u32 nid, u64 size) 92 { 93 const unsigned long nr_pages = PHYS_PFN(size); 94 unsigned long pfn, start_pfn; 95 struct page *page; 96 97 /* 98 * Trace memory needs to be aligned to the size, which is guaranteed 99 * by alloc_contig_pages(). 100 */ 101 page = alloc_contig_pages(nr_pages, GFP_KERNEL | __GFP_THISNODE | 102 __GFP_NOWARN | __GFP_ZERO, nid, NULL); 103 if (!page) 104 return 0; 105 start_pfn = page_to_pfn(page); 106 107 /* 108 * Before we go ahead and use this range as cache inhibited range 109 * flush the cache. 110 */ 111 flush_dcache_range_chunked((unsigned long)pfn_to_kaddr(start_pfn), 112 (unsigned long)pfn_to_kaddr(start_pfn + nr_pages), 113 FLUSH_CHUNK_SIZE); 114 115 /* 116 * Set pages PageOffline(), to indicate that nobody (e.g., hibernation, 117 * dumping, ...) should be touching these pages. 118 */ 119 for (pfn = start_pfn; pfn < start_pfn + nr_pages; pfn++) 120 __SetPageOffline(pfn_to_page(pfn)); 121 122 arch_remove_linear_mapping(PFN_PHYS(start_pfn), size); 123 124 return PFN_PHYS(start_pfn); 125 } 126 127 static int memtrace_init_regions_runtime(u64 size) 128 { 129 u32 nid; 130 u64 m; 131 132 memtrace_array = kcalloc(num_online_nodes(), 133 sizeof(struct memtrace_entry), GFP_KERNEL); 134 if (!memtrace_array) { 135 pr_err("Failed to allocate memtrace_array\n"); 136 return -EINVAL; 137 } 138 139 for_each_online_node(nid) { 140 m = memtrace_alloc_node(nid, size); 141 142 /* 143 * A node might not have any local memory, so warn but 144 * continue on. 145 */ 146 if (!m) { 147 pr_err("Failed to allocate trace memory on node %d\n", nid); 148 continue; 149 } 150 151 pr_info("Allocated trace memory on node %d at 0x%016llx\n", nid, m); 152 153 memtrace_array[memtrace_array_nr].start = m; 154 memtrace_array[memtrace_array_nr].size = size; 155 memtrace_array[memtrace_array_nr].nid = nid; 156 memtrace_array_nr++; 157 } 158 159 return 0; 160 } 161 162 static struct dentry *memtrace_debugfs_dir; 163 164 static int memtrace_init_debugfs(void) 165 { 166 int ret = 0; 167 int i; 168 169 for (i = 0; i < memtrace_array_nr; i++) { 170 struct dentry *dir; 171 struct memtrace_entry *ent = &memtrace_array[i]; 172 173 ent->mem = ioremap(ent->start, ent->size); 174 /* Warn but continue on */ 175 if (!ent->mem) { 176 pr_err("Failed to map trace memory at 0x%llx\n", 177 ent->start); 178 ret = -1; 179 continue; 180 } 181 182 snprintf(ent->name, 16, "%08x", ent->nid); 183 dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir); 184 185 ent->dir = dir; 186 debugfs_create_file_unsafe("trace", 0600, dir, ent, &memtrace_fops); 187 debugfs_create_x64("start", 0400, dir, &ent->start); 188 debugfs_create_x64("size", 0400, dir, &ent->size); 189 } 190 191 return ret; 192 } 193 194 static int memtrace_free(int nid, u64 start, u64 size) 195 { 196 struct mhp_params params = { .pgprot = PAGE_KERNEL }; 197 const unsigned long nr_pages = PHYS_PFN(size); 198 const unsigned long start_pfn = PHYS_PFN(start); 199 unsigned long pfn; 200 int ret; 201 202 ret = arch_create_linear_mapping(nid, start, size, ¶ms); 203 if (ret) 204 return ret; 205 206 for (pfn = start_pfn; pfn < start_pfn + nr_pages; pfn++) 207 __ClearPageOffline(pfn_to_page(pfn)); 208 209 free_contig_range(start_pfn, nr_pages); 210 return 0; 211 } 212 213 /* 214 * Iterate through the chunks of memory we allocated and attempt to expose 215 * them back to the kernel. 216 */ 217 static int memtrace_free_regions(void) 218 { 219 int i, ret = 0; 220 struct memtrace_entry *ent; 221 222 for (i = memtrace_array_nr - 1; i >= 0; i--) { 223 ent = &memtrace_array[i]; 224 225 /* We have freed this chunk previously */ 226 if (ent->nid == NUMA_NO_NODE) 227 continue; 228 229 /* Remove from io mappings */ 230 if (ent->mem) { 231 iounmap(ent->mem); 232 ent->mem = 0; 233 } 234 235 if (memtrace_free(ent->nid, ent->start, ent->size)) { 236 pr_err("Failed to free trace memory on node %d\n", 237 ent->nid); 238 ret += 1; 239 continue; 240 } 241 242 /* 243 * Memory was freed successfully so clean up references to it 244 * so on reentry we can tell that this chunk was freed. 245 */ 246 debugfs_remove_recursive(ent->dir); 247 pr_info("Freed trace memory back on node %d\n", ent->nid); 248 ent->size = ent->start = ent->nid = NUMA_NO_NODE; 249 } 250 if (ret) 251 return ret; 252 253 /* If all chunks of memory were freed successfully, reset globals */ 254 kfree(memtrace_array); 255 memtrace_array = NULL; 256 memtrace_size = 0; 257 memtrace_array_nr = 0; 258 return 0; 259 } 260 261 static int memtrace_enable_set(void *data, u64 val) 262 { 263 int rc = -EAGAIN; 264 u64 bytes; 265 266 /* 267 * Don't attempt to do anything if size isn't aligned to a memory 268 * block or equal to zero. 269 */ 270 bytes = memory_block_size_bytes(); 271 if (val & (bytes - 1)) { 272 pr_err("Value must be aligned with 0x%llx\n", bytes); 273 return -EINVAL; 274 } 275 276 mutex_lock(&memtrace_mutex); 277 278 /* Free all previously allocated memory. */ 279 if (memtrace_size && memtrace_free_regions()) 280 goto out_unlock; 281 282 if (!val) { 283 rc = 0; 284 goto out_unlock; 285 } 286 287 /* Allocate memory. */ 288 if (memtrace_init_regions_runtime(val)) 289 goto out_unlock; 290 291 if (memtrace_init_debugfs()) 292 goto out_unlock; 293 294 memtrace_size = val; 295 rc = 0; 296 out_unlock: 297 mutex_unlock(&memtrace_mutex); 298 return rc; 299 } 300 301 static int memtrace_enable_get(void *data, u64 *val) 302 { 303 *val = memtrace_size; 304 return 0; 305 } 306 307 DEFINE_SIMPLE_ATTRIBUTE(memtrace_init_fops, memtrace_enable_get, 308 memtrace_enable_set, "0x%016llx\n"); 309 310 static int memtrace_init(void) 311 { 312 memtrace_debugfs_dir = debugfs_create_dir("memtrace", 313 arch_debugfs_dir); 314 315 debugfs_create_file("enable", 0600, memtrace_debugfs_dir, 316 NULL, &memtrace_init_fops); 317 318 return 0; 319 } 320 machine_device_initcall(powernv, memtrace_init); 321