1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Author: Xiang Gao <gaoxiang@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 #include <linux/init.h> 9 #include <linux/kernel.h> 10 #include <linux/mm.h> 11 #include <linux/mmzone.h> 12 #include <linux/export.h> 13 #include <linux/nodemask.h> 14 #include <linux/swap.h> 15 #include <linux/memblock.h> 16 #include <linux/pfn.h> 17 #include <linux/acpi.h> 18 #include <linux/efi.h> 19 #include <linux/irq.h> 20 #include <linux/pci.h> 21 #include <asm/bootinfo.h> 22 #include <asm/loongson.h> 23 #include <asm/numa.h> 24 #include <asm/page.h> 25 #include <asm/pgalloc.h> 26 #include <asm/sections.h> 27 #include <asm/time.h> 28 29 int numa_off; 30 unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES]; 31 EXPORT_SYMBOL(node_distances); 32 33 static struct numa_meminfo numa_meminfo; 34 cpumask_t cpus_on_node[MAX_NUMNODES]; 35 cpumask_t phys_cpus_on_node[MAX_NUMNODES]; 36 EXPORT_SYMBOL(cpus_on_node); 37 38 /* 39 * apicid, cpu, node mappings 40 */ 41 s16 __cpuid_to_node[CONFIG_NR_CPUS] = { 42 [0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE 43 }; 44 EXPORT_SYMBOL(__cpuid_to_node); 45 46 nodemask_t numa_nodes_parsed __initdata; 47 48 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 49 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 50 EXPORT_SYMBOL(__per_cpu_offset); 51 52 static int __init pcpu_cpu_to_node(int cpu) 53 { 54 return early_cpu_to_node(cpu); 55 } 56 57 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 58 { 59 if (early_cpu_to_node(from) == early_cpu_to_node(to)) 60 return LOCAL_DISTANCE; 61 else 62 return REMOTE_DISTANCE; 63 } 64 65 void __init pcpu_populate_pte(unsigned long addr) 66 { 67 populate_kernel_pte(addr); 68 } 69 70 void __init setup_per_cpu_areas(void) 71 { 72 unsigned long delta; 73 unsigned int cpu; 74 int rc = -EINVAL; 75 76 if (pcpu_chosen_fc == PCPU_FC_AUTO) { 77 if (nr_node_ids >= 8) 78 pcpu_chosen_fc = PCPU_FC_PAGE; 79 else 80 pcpu_chosen_fc = PCPU_FC_EMBED; 81 } 82 83 /* 84 * Always reserve area for module percpu variables. That's 85 * what the legacy allocator did. 86 */ 87 if (pcpu_chosen_fc != PCPU_FC_PAGE) { 88 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 89 PERCPU_DYNAMIC_RESERVE, PMD_SIZE, 90 pcpu_cpu_distance, pcpu_cpu_to_node); 91 if (rc < 0) 92 pr_warn("%s allocator failed (%d), falling back to page size\n", 93 pcpu_fc_names[pcpu_chosen_fc], rc); 94 } 95 if (rc < 0) 96 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node); 97 if (rc < 0) 98 panic("cannot initialize percpu area (err=%d)", rc); 99 100 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 101 for_each_possible_cpu(cpu) 102 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; 103 } 104 #endif 105 106 /* 107 * Get nodeid by logical cpu number. 108 * __cpuid_to_node maps phyical cpu id to node, so we 109 * should use cpu_logical_map(cpu) to index it. 110 * 111 * This routine is only used in early phase during 112 * booting, after setup_per_cpu_areas calling and numa_node 113 * initialization, cpu_to_node will be used instead. 114 */ 115 int early_cpu_to_node(int cpu) 116 { 117 int physid = cpu_logical_map(cpu); 118 119 if (physid < 0) 120 return NUMA_NO_NODE; 121 122 return __cpuid_to_node[physid]; 123 } 124 125 void __init early_numa_add_cpu(int cpuid, s16 node) 126 { 127 int cpu = __cpu_number_map[cpuid]; 128 129 if (cpu < 0) 130 return; 131 132 cpumask_set_cpu(cpu, &cpus_on_node[node]); 133 cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]); 134 } 135 136 void numa_add_cpu(unsigned int cpu) 137 { 138 int nid = cpu_to_node(cpu); 139 cpumask_set_cpu(cpu, &cpus_on_node[nid]); 140 } 141 142 void numa_remove_cpu(unsigned int cpu) 143 { 144 int nid = cpu_to_node(cpu); 145 cpumask_clear_cpu(cpu, &cpus_on_node[nid]); 146 } 147 148 static int __init numa_add_memblk_to(int nid, u64 start, u64 end, 149 struct numa_meminfo *mi) 150 { 151 /* ignore zero length blks */ 152 if (start == end) 153 return 0; 154 155 /* whine about and ignore invalid blks */ 156 if (start > end || nid < 0 || nid >= MAX_NUMNODES) { 157 pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", 158 nid, start, end - 1); 159 return 0; 160 } 161 162 if (mi->nr_blks >= NR_NODE_MEMBLKS) { 163 pr_err("NUMA: too many memblk ranges\n"); 164 return -EINVAL; 165 } 166 167 mi->blk[mi->nr_blks].start = PFN_ALIGN(start); 168 mi->blk[mi->nr_blks].end = PFN_ALIGN(end - PAGE_SIZE + 1); 169 mi->blk[mi->nr_blks].nid = nid; 170 mi->nr_blks++; 171 return 0; 172 } 173 174 /** 175 * numa_add_memblk - Add one numa_memblk to numa_meminfo 176 * @nid: NUMA node ID of the new memblk 177 * @start: Start address of the new memblk 178 * @end: End address of the new memblk 179 * 180 * Add a new memblk to the default numa_meminfo. 181 * 182 * RETURNS: 183 * 0 on success, -errno on failure. 184 */ 185 int __init numa_add_memblk(int nid, u64 start, u64 end) 186 { 187 return numa_add_memblk_to(nid, start, end, &numa_meminfo); 188 } 189 190 static void __init node_mem_init(unsigned int node) 191 { 192 unsigned long start_pfn, end_pfn; 193 unsigned long node_addrspace_offset; 194 195 node_addrspace_offset = nid_to_addrbase(node); 196 pr_info("Node%d's addrspace_offset is 0x%lx\n", 197 node, node_addrspace_offset); 198 199 get_pfn_range_for_nid(node, &start_pfn, &end_pfn); 200 pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n", 201 node, start_pfn, end_pfn); 202 203 alloc_node_data(node); 204 } 205 206 #ifdef CONFIG_ACPI_NUMA 207 208 static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type) 209 { 210 static unsigned long num_physpages; 211 212 num_physpages += (size >> PAGE_SHIFT); 213 pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", 214 node, type, start, size); 215 pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n", 216 start >> PAGE_SHIFT, (start + size) >> PAGE_SHIFT, num_physpages); 217 memblock_set_node(start, size, &memblock.memory, node); 218 } 219 220 /* 221 * add_numamem_region 222 * 223 * Add a uasable memory region described by BIOS. The 224 * routine gets each intersection between BIOS's region 225 * and node's region, and adds them into node's memblock 226 * pool. 227 * 228 */ 229 static void __init add_numamem_region(u64 start, u64 end, u32 type) 230 { 231 u32 i; 232 u64 ofs = start; 233 234 if (start >= end) { 235 pr_debug("Invalid region: %016llx-%016llx\n", start, end); 236 return; 237 } 238 239 for (i = 0; i < numa_meminfo.nr_blks; i++) { 240 struct numa_memblk *mb = &numa_meminfo.blk[i]; 241 242 if (ofs > mb->end) 243 continue; 244 245 if (end > mb->end) { 246 add_node_intersection(mb->nid, ofs, mb->end - ofs, type); 247 ofs = mb->end; 248 } else { 249 add_node_intersection(mb->nid, ofs, end - ofs, type); 250 break; 251 } 252 } 253 } 254 255 static void __init init_node_memblock(void) 256 { 257 u32 mem_type; 258 u64 mem_end, mem_start, mem_size; 259 efi_memory_desc_t *md; 260 261 /* Parse memory information and activate */ 262 for_each_efi_memory_desc(md) { 263 mem_type = md->type; 264 mem_start = md->phys_addr; 265 mem_size = md->num_pages << EFI_PAGE_SHIFT; 266 mem_end = mem_start + mem_size; 267 268 switch (mem_type) { 269 case EFI_LOADER_CODE: 270 case EFI_LOADER_DATA: 271 case EFI_BOOT_SERVICES_CODE: 272 case EFI_BOOT_SERVICES_DATA: 273 case EFI_PERSISTENT_MEMORY: 274 case EFI_CONVENTIONAL_MEMORY: 275 add_numamem_region(mem_start, mem_end, mem_type); 276 break; 277 case EFI_PAL_CODE: 278 case EFI_UNUSABLE_MEMORY: 279 case EFI_ACPI_RECLAIM_MEMORY: 280 add_numamem_region(mem_start, mem_end, mem_type); 281 fallthrough; 282 case EFI_RESERVED_TYPE: 283 case EFI_RUNTIME_SERVICES_CODE: 284 case EFI_RUNTIME_SERVICES_DATA: 285 case EFI_MEMORY_MAPPED_IO: 286 case EFI_MEMORY_MAPPED_IO_PORT_SPACE: 287 pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", 288 mem_type, mem_start, mem_size); 289 break; 290 } 291 } 292 } 293 294 static void __init numa_default_distance(void) 295 { 296 int row, col; 297 298 for (row = 0; row < MAX_NUMNODES; row++) 299 for (col = 0; col < MAX_NUMNODES; col++) { 300 if (col == row) 301 node_distances[row][col] = LOCAL_DISTANCE; 302 else 303 /* We assume that one node per package here! 304 * 305 * A SLIT should be used for multiple nodes 306 * per package to override default setting. 307 */ 308 node_distances[row][col] = REMOTE_DISTANCE; 309 } 310 } 311 312 /* 313 * fake_numa_init() - For Non-ACPI systems 314 * Return: 0 on success, -errno on failure. 315 */ 316 static int __init fake_numa_init(void) 317 { 318 phys_addr_t start = memblock_start_of_DRAM(); 319 phys_addr_t end = memblock_end_of_DRAM() - 1; 320 321 node_set(0, numa_nodes_parsed); 322 pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); 323 324 return numa_add_memblk(0, start, end + 1); 325 } 326 327 int __init init_numa_memory(void) 328 { 329 int i; 330 int ret; 331 int node; 332 333 for (i = 0; i < NR_CPUS; i++) 334 set_cpuid_to_node(i, NUMA_NO_NODE); 335 336 numa_default_distance(); 337 nodes_clear(numa_nodes_parsed); 338 nodes_clear(node_possible_map); 339 nodes_clear(node_online_map); 340 memset(&numa_meminfo, 0, sizeof(numa_meminfo)); 341 342 /* Parse SRAT and SLIT if provided by firmware. */ 343 ret = acpi_disabled ? fake_numa_init() : acpi_numa_init(); 344 if (ret < 0) 345 return ret; 346 347 node_possible_map = numa_nodes_parsed; 348 if (WARN_ON(nodes_empty(node_possible_map))) 349 return -EINVAL; 350 351 init_node_memblock(); 352 if (!memblock_validate_numa_coverage(SZ_1M)) 353 return -EINVAL; 354 355 for_each_node_mask(node, node_possible_map) { 356 node_mem_init(node); 357 node_set_online(node); 358 } 359 max_low_pfn = PHYS_PFN(memblock_end_of_DRAM()); 360 361 setup_nr_node_ids(); 362 loongson_sysconf.nr_nodes = nr_node_ids; 363 loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]); 364 365 return 0; 366 } 367 368 #endif 369 370 void __init paging_init(void) 371 { 372 unsigned int node; 373 unsigned long zones_size[MAX_NR_ZONES] = {0, }; 374 375 for_each_online_node(node) { 376 unsigned long start_pfn, end_pfn; 377 378 get_pfn_range_for_nid(node, &start_pfn, &end_pfn); 379 380 if (end_pfn > max_low_pfn) 381 max_low_pfn = end_pfn; 382 } 383 #ifdef CONFIG_ZONE_DMA32 384 zones_size[ZONE_DMA32] = MAX_DMA32_PFN; 385 #endif 386 zones_size[ZONE_NORMAL] = max_low_pfn; 387 free_area_init(zones_size); 388 } 389 390 void __init mem_init(void) 391 { 392 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); 393 memblock_free_all(); 394 } 395 396 int pcibus_to_node(struct pci_bus *bus) 397 { 398 return dev_to_node(&bus->dev); 399 } 400 EXPORT_SYMBOL(pcibus_to_node); 401