1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Author: Xiang Gao <gaoxiang@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
5 *
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
7 */
8 #include <linux/init.h>
9 #include <linux/kernel.h>
10 #include <linux/mm.h>
11 #include <linux/mmzone.h>
12 #include <linux/export.h>
13 #include <linux/nodemask.h>
14 #include <linux/numa_memblks.h>
15 #include <linux/swap.h>
16 #include <linux/memblock.h>
17 #include <linux/pfn.h>
18 #include <linux/acpi.h>
19 #include <linux/efi.h>
20 #include <linux/irq.h>
21 #include <linux/pci.h>
22 #include <asm/bootinfo.h>
23 #include <asm/loongson.h>
24 #include <asm/numa.h>
25 #include <asm/page.h>
26 #include <asm/pgalloc.h>
27 #include <asm/sections.h>
28 #include <asm/time.h>
29
30 int numa_off;
31 cpumask_t cpus_on_node[MAX_NUMNODES];
32 cpumask_t phys_cpus_on_node[MAX_NUMNODES];
33 EXPORT_SYMBOL(cpus_on_node);
34
35 /*
36 * apicid, cpu, node mappings
37 */
38 s16 __cpuid_to_node[CONFIG_NR_CPUS] = {
39 [0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE
40 };
41 EXPORT_SYMBOL(__cpuid_to_node);
42
43 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
44 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
45 EXPORT_SYMBOL(__per_cpu_offset);
46
pcpu_cpu_to_node(int cpu)47 static int __init pcpu_cpu_to_node(int cpu)
48 {
49 return early_cpu_to_node(cpu);
50 }
51
pcpu_cpu_distance(unsigned int from,unsigned int to)52 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
53 {
54 if (early_cpu_to_node(from) == early_cpu_to_node(to))
55 return LOCAL_DISTANCE;
56 else
57 return REMOTE_DISTANCE;
58 }
59
pcpu_populate_pte(unsigned long addr)60 void __init pcpu_populate_pte(unsigned long addr)
61 {
62 populate_kernel_pte(addr);
63 }
64
setup_per_cpu_areas(void)65 void __init setup_per_cpu_areas(void)
66 {
67 unsigned long delta;
68 unsigned int cpu;
69 int rc = -EINVAL;
70
71 if (pcpu_chosen_fc == PCPU_FC_AUTO) {
72 if (nr_node_ids >= 8)
73 pcpu_chosen_fc = PCPU_FC_PAGE;
74 else
75 pcpu_chosen_fc = PCPU_FC_EMBED;
76 }
77
78 /*
79 * Always reserve area for module percpu variables. That's
80 * what the legacy allocator did.
81 */
82 if (pcpu_chosen_fc != PCPU_FC_PAGE) {
83 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
84 PERCPU_DYNAMIC_RESERVE, PMD_SIZE,
85 pcpu_cpu_distance, pcpu_cpu_to_node);
86 if (rc < 0)
87 pr_warn("%s allocator failed (%d), falling back to page size\n",
88 pcpu_fc_names[pcpu_chosen_fc], rc);
89 }
90 if (rc < 0)
91 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node);
92 if (rc < 0)
93 panic("cannot initialize percpu area (err=%d)", rc);
94
95 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
96 for_each_possible_cpu(cpu)
97 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
98 }
99 #endif
100
101 /*
102 * Get nodeid by logical cpu number.
103 * __cpuid_to_node maps phyical cpu id to node, so we
104 * should use cpu_logical_map(cpu) to index it.
105 *
106 * This routine is only used in early phase during
107 * booting, after setup_per_cpu_areas calling and numa_node
108 * initialization, cpu_to_node will be used instead.
109 */
early_cpu_to_node(int cpu)110 int early_cpu_to_node(int cpu)
111 {
112 int physid = cpu_logical_map(cpu);
113
114 if (physid < 0)
115 return NUMA_NO_NODE;
116
117 return __cpuid_to_node[physid];
118 }
119
early_numa_add_cpu(int cpuid,s16 node)120 void __init early_numa_add_cpu(int cpuid, s16 node)
121 {
122 int cpu = __cpu_number_map[cpuid];
123
124 if (cpu < 0)
125 return;
126
127 cpumask_set_cpu(cpu, &cpus_on_node[node]);
128 cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]);
129 }
130
numa_add_cpu(unsigned int cpu)131 void numa_add_cpu(unsigned int cpu)
132 {
133 int nid = cpu_to_node(cpu);
134 cpumask_set_cpu(cpu, &cpus_on_node[nid]);
135 }
136
numa_remove_cpu(unsigned int cpu)137 void numa_remove_cpu(unsigned int cpu)
138 {
139 int nid = cpu_to_node(cpu);
140 cpumask_clear_cpu(cpu, &cpus_on_node[nid]);
141 }
142
node_mem_init(unsigned int node)143 static void __init node_mem_init(unsigned int node)
144 {
145 unsigned long start_pfn, end_pfn;
146 unsigned long node_addrspace_offset;
147
148 node_addrspace_offset = nid_to_addrbase(node);
149 pr_info("Node%d's addrspace_offset is 0x%lx\n",
150 node, node_addrspace_offset);
151
152 get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
153 pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n",
154 node, start_pfn, end_pfn);
155
156 alloc_node_data(node);
157 }
158
159 #ifdef CONFIG_ACPI_NUMA
160
161 /*
162 * add_numamem_region
163 *
164 * Add a uasable memory region described by BIOS. The
165 * routine gets each intersection between BIOS's region
166 * and node's region, and adds them into node's memblock
167 * pool.
168 *
169 */
add_numamem_region(u64 start,u64 end,u32 type)170 static void __init add_numamem_region(u64 start, u64 end, u32 type)
171 {
172 u32 node = pa_to_nid(start);
173 u64 size = end - start;
174 static unsigned long num_physpages;
175
176 if (start >= end) {
177 pr_debug("Invalid region: %016llx-%016llx\n", start, end);
178 return;
179 }
180
181 num_physpages += (size >> PAGE_SHIFT);
182 pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
183 node, type, start, size);
184 pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
185 start >> PAGE_SHIFT, end >> PAGE_SHIFT, num_physpages);
186 memblock_set_node(start, size, &memblock.memory, node);
187 }
188
init_node_memblock(void)189 static void __init init_node_memblock(void)
190 {
191 u32 mem_type;
192 u64 mem_end, mem_start, mem_size;
193 efi_memory_desc_t *md;
194
195 /* Parse memory information and activate */
196 for_each_efi_memory_desc(md) {
197 mem_type = md->type;
198 mem_start = md->phys_addr;
199 mem_size = md->num_pages << EFI_PAGE_SHIFT;
200 mem_end = mem_start + mem_size;
201
202 switch (mem_type) {
203 case EFI_LOADER_CODE:
204 case EFI_LOADER_DATA:
205 case EFI_BOOT_SERVICES_CODE:
206 case EFI_BOOT_SERVICES_DATA:
207 case EFI_PERSISTENT_MEMORY:
208 case EFI_CONVENTIONAL_MEMORY:
209 add_numamem_region(mem_start, mem_end, mem_type);
210 break;
211 case EFI_PAL_CODE:
212 case EFI_UNUSABLE_MEMORY:
213 case EFI_ACPI_RECLAIM_MEMORY:
214 add_numamem_region(mem_start, mem_end, mem_type);
215 fallthrough;
216 case EFI_RESERVED_TYPE:
217 case EFI_RUNTIME_SERVICES_CODE:
218 case EFI_RUNTIME_SERVICES_DATA:
219 case EFI_MEMORY_MAPPED_IO:
220 case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
221 pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
222 mem_type, mem_start, mem_size);
223 break;
224 }
225 }
226 }
227
228 /*
229 * fake_numa_init() - For Non-ACPI systems
230 * Return: 0 on success, -errno on failure.
231 */
fake_numa_init(void)232 static int __init fake_numa_init(void)
233 {
234 phys_addr_t start = memblock_start_of_DRAM();
235 phys_addr_t end = memblock_end_of_DRAM() - 1;
236
237 node_set(0, numa_nodes_parsed);
238 pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
239
240 return numa_add_memblk(0, start, end + 1);
241 }
242
init_numa_memory(void)243 int __init init_numa_memory(void)
244 {
245 int i;
246 int ret;
247 int node;
248
249 for (i = 0; i < NR_CPUS; i++)
250 set_cpuid_to_node(i, NUMA_NO_NODE);
251
252 numa_reset_distance();
253 nodes_clear(numa_nodes_parsed);
254 nodes_clear(node_possible_map);
255 nodes_clear(node_online_map);
256 WARN_ON(memblock_clear_hotplug(0, PHYS_ADDR_MAX));
257
258 /* Parse SRAT and SLIT if provided by firmware. */
259 ret = acpi_disabled ? fake_numa_init() : acpi_numa_init();
260 if (ret < 0)
261 return ret;
262
263 node_possible_map = numa_nodes_parsed;
264 if (WARN_ON(nodes_empty(node_possible_map)))
265 return -EINVAL;
266
267 init_node_memblock();
268 if (!memblock_validate_numa_coverage(SZ_1M))
269 return -EINVAL;
270
271 for_each_node_mask(node, node_possible_map) {
272 node_mem_init(node);
273 node_set_online(node);
274 }
275 max_low_pfn = PHYS_PFN(memblock_end_of_DRAM());
276
277 setup_nr_node_ids();
278 loongson_sysconf.nr_nodes = nr_node_ids;
279 loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]);
280
281 return 0;
282 }
283
284 #endif
285
paging_init(void)286 void __init paging_init(void)
287 {
288 unsigned int node;
289 unsigned long zones_size[MAX_NR_ZONES] = {0, };
290
291 for_each_online_node(node) {
292 unsigned long start_pfn, end_pfn;
293
294 get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
295
296 if (end_pfn > max_low_pfn)
297 max_low_pfn = end_pfn;
298 }
299 #ifdef CONFIG_ZONE_DMA32
300 zones_size[ZONE_DMA32] = MAX_DMA32_PFN;
301 #endif
302 zones_size[ZONE_NORMAL] = max_low_pfn;
303 free_area_init(zones_size);
304 }
305
pcibus_to_node(struct pci_bus * bus)306 int pcibus_to_node(struct pci_bus *bus)
307 {
308 return dev_to_node(&bus->dev);
309 }
310 EXPORT_SYMBOL(pcibus_to_node);
311