1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Author: Xiang Gao <gaoxiang@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 #include <linux/init.h> 9 #include <linux/kernel.h> 10 #include <linux/mm.h> 11 #include <linux/mmzone.h> 12 #include <linux/export.h> 13 #include <linux/nodemask.h> 14 #include <linux/numa_memblks.h> 15 #include <linux/swap.h> 16 #include <linux/memblock.h> 17 #include <linux/pfn.h> 18 #include <linux/acpi.h> 19 #include <linux/efi.h> 20 #include <linux/irq.h> 21 #include <linux/pci.h> 22 #include <asm/bootinfo.h> 23 #include <asm/loongson.h> 24 #include <asm/numa.h> 25 #include <asm/page.h> 26 #include <asm/pgalloc.h> 27 #include <asm/sections.h> 28 #include <asm/time.h> 29 30 int numa_off; 31 cpumask_t cpus_on_node[MAX_NUMNODES]; 32 cpumask_t phys_cpus_on_node[MAX_NUMNODES]; 33 EXPORT_SYMBOL(cpus_on_node); 34 35 /* 36 * apicid, cpu, node mappings 37 */ 38 s16 __cpuid_to_node[CONFIG_NR_CPUS] = { 39 [0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE 40 }; 41 EXPORT_SYMBOL(__cpuid_to_node); 42 43 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 44 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 45 EXPORT_SYMBOL(__per_cpu_offset); 46 47 static int __init pcpu_cpu_to_node(int cpu) 48 { 49 return early_cpu_to_node(cpu); 50 } 51 52 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 53 { 54 if (early_cpu_to_node(from) == early_cpu_to_node(to)) 55 return LOCAL_DISTANCE; 56 else 57 return REMOTE_DISTANCE; 58 } 59 60 void __init pcpu_populate_pte(unsigned long addr) 61 { 62 populate_kernel_pte(addr); 63 } 64 65 void __init setup_per_cpu_areas(void) 66 { 67 unsigned long delta; 68 unsigned int cpu; 69 int rc = -EINVAL; 70 71 if (pcpu_chosen_fc == PCPU_FC_AUTO) { 72 if (nr_node_ids >= 8) 73 pcpu_chosen_fc = PCPU_FC_PAGE; 74 else 75 pcpu_chosen_fc = PCPU_FC_EMBED; 76 } 77 78 /* 79 * Always reserve area for module percpu variables. That's 80 * what the legacy allocator did. 81 */ 82 if (pcpu_chosen_fc != PCPU_FC_PAGE) { 83 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 84 PERCPU_DYNAMIC_RESERVE, PMD_SIZE, 85 pcpu_cpu_distance, pcpu_cpu_to_node); 86 if (rc < 0) 87 pr_warn("%s allocator failed (%d), falling back to page size\n", 88 pcpu_fc_names[pcpu_chosen_fc], rc); 89 } 90 if (rc < 0) 91 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node); 92 if (rc < 0) 93 panic("cannot initialize percpu area (err=%d)", rc); 94 95 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 96 for_each_possible_cpu(cpu) 97 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; 98 } 99 #endif 100 101 /* 102 * Get nodeid by logical cpu number. 103 * __cpuid_to_node maps phyical cpu id to node, so we 104 * should use cpu_logical_map(cpu) to index it. 105 * 106 * This routine is only used in early phase during 107 * booting, after setup_per_cpu_areas calling and numa_node 108 * initialization, cpu_to_node will be used instead. 109 */ 110 int early_cpu_to_node(int cpu) 111 { 112 int physid = cpu_logical_map(cpu); 113 114 if (physid < 0) 115 return NUMA_NO_NODE; 116 117 return __cpuid_to_node[physid]; 118 } 119 120 void __init early_numa_add_cpu(int cpuid, s16 node) 121 { 122 int cpu = __cpu_number_map[cpuid]; 123 124 if (cpu < 0) 125 return; 126 127 cpumask_set_cpu(cpu, &cpus_on_node[node]); 128 cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]); 129 } 130 131 void numa_add_cpu(unsigned int cpu) 132 { 133 int nid = cpu_to_node(cpu); 134 cpumask_set_cpu(cpu, &cpus_on_node[nid]); 135 } 136 137 void numa_remove_cpu(unsigned int cpu) 138 { 139 int nid = cpu_to_node(cpu); 140 cpumask_clear_cpu(cpu, &cpus_on_node[nid]); 141 } 142 143 static void __init node_mem_init(unsigned int node) 144 { 145 unsigned long start_pfn, end_pfn; 146 unsigned long node_addrspace_offset; 147 148 node_addrspace_offset = nid_to_addrbase(node); 149 pr_info("Node%d's addrspace_offset is 0x%lx\n", 150 node, node_addrspace_offset); 151 152 get_pfn_range_for_nid(node, &start_pfn, &end_pfn); 153 pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n", 154 node, start_pfn, end_pfn); 155 156 alloc_node_data(node); 157 } 158 159 #ifdef CONFIG_ACPI_NUMA 160 161 /* 162 * add_numamem_region 163 * 164 * Add a uasable memory region described by BIOS. The 165 * routine gets each intersection between BIOS's region 166 * and node's region, and adds them into node's memblock 167 * pool. 168 * 169 */ 170 static void __init add_numamem_region(u64 start, u64 end, u32 type) 171 { 172 u32 node = pa_to_nid(start); 173 u64 size = end - start; 174 static unsigned long num_physpages; 175 176 if (start >= end) { 177 pr_debug("Invalid region: %016llx-%016llx\n", start, end); 178 return; 179 } 180 181 num_physpages += (size >> PAGE_SHIFT); 182 pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", 183 node, type, start, size); 184 pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n", 185 start >> PAGE_SHIFT, end >> PAGE_SHIFT, num_physpages); 186 memblock_set_node(start, size, &memblock.memory, node); 187 } 188 189 static void __init init_node_memblock(void) 190 { 191 u32 mem_type; 192 u64 mem_end, mem_start, mem_size; 193 efi_memory_desc_t *md; 194 195 /* Parse memory information and activate */ 196 for_each_efi_memory_desc(md) { 197 mem_type = md->type; 198 mem_start = md->phys_addr; 199 mem_size = md->num_pages << EFI_PAGE_SHIFT; 200 mem_end = mem_start + mem_size; 201 202 switch (mem_type) { 203 case EFI_LOADER_CODE: 204 case EFI_LOADER_DATA: 205 case EFI_BOOT_SERVICES_CODE: 206 case EFI_BOOT_SERVICES_DATA: 207 case EFI_PERSISTENT_MEMORY: 208 case EFI_CONVENTIONAL_MEMORY: 209 add_numamem_region(mem_start, mem_end, mem_type); 210 break; 211 case EFI_PAL_CODE: 212 case EFI_UNUSABLE_MEMORY: 213 case EFI_ACPI_RECLAIM_MEMORY: 214 add_numamem_region(mem_start, mem_end, mem_type); 215 fallthrough; 216 case EFI_RESERVED_TYPE: 217 case EFI_RUNTIME_SERVICES_CODE: 218 case EFI_RUNTIME_SERVICES_DATA: 219 case EFI_MEMORY_MAPPED_IO: 220 case EFI_MEMORY_MAPPED_IO_PORT_SPACE: 221 pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", 222 mem_type, mem_start, mem_size); 223 break; 224 } 225 } 226 } 227 228 /* 229 * fake_numa_init() - For Non-ACPI systems 230 * Return: 0 on success, -errno on failure. 231 */ 232 static int __init fake_numa_init(void) 233 { 234 phys_addr_t start = memblock_start_of_DRAM(); 235 phys_addr_t end = memblock_end_of_DRAM() - 1; 236 237 node_set(0, numa_nodes_parsed); 238 pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); 239 240 return numa_add_memblk(0, start, end + 1); 241 } 242 243 int __init init_numa_memory(void) 244 { 245 int i; 246 int ret; 247 int node; 248 249 for (i = 0; i < NR_CPUS; i++) 250 set_cpuid_to_node(i, NUMA_NO_NODE); 251 252 numa_reset_distance(); 253 nodes_clear(numa_nodes_parsed); 254 nodes_clear(node_possible_map); 255 nodes_clear(node_online_map); 256 WARN_ON(memblock_clear_hotplug(0, PHYS_ADDR_MAX)); 257 258 /* Parse SRAT and SLIT if provided by firmware. */ 259 ret = acpi_disabled ? fake_numa_init() : acpi_numa_init(); 260 if (ret < 0) 261 return ret; 262 263 node_possible_map = numa_nodes_parsed; 264 if (WARN_ON(nodes_empty(node_possible_map))) 265 return -EINVAL; 266 267 init_node_memblock(); 268 if (!memblock_validate_numa_coverage(SZ_1M)) 269 return -EINVAL; 270 271 for_each_node_mask(node, node_possible_map) { 272 node_mem_init(node); 273 node_set_online(node); 274 } 275 max_low_pfn = PHYS_PFN(memblock_end_of_DRAM()); 276 277 setup_nr_node_ids(); 278 loongson_sysconf.nr_nodes = nr_node_ids; 279 loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]); 280 281 return 0; 282 } 283 284 #endif 285 286 void __init paging_init(void) 287 { 288 unsigned int node; 289 unsigned long zones_size[MAX_NR_ZONES] = {0, }; 290 291 for_each_online_node(node) { 292 unsigned long start_pfn, end_pfn; 293 294 get_pfn_range_for_nid(node, &start_pfn, &end_pfn); 295 296 if (end_pfn > max_low_pfn) 297 max_low_pfn = end_pfn; 298 } 299 #ifdef CONFIG_ZONE_DMA32 300 zones_size[ZONE_DMA32] = MAX_DMA32_PFN; 301 #endif 302 zones_size[ZONE_NORMAL] = max_low_pfn; 303 free_area_init(zones_size); 304 } 305 306 int pcibus_to_node(struct pci_bus *bus) 307 { 308 return dev_to_node(&bus->dev); 309 } 310 EXPORT_SYMBOL(pcibus_to_node); 311