1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Author: Xiang Gao <gaoxiang@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
5 *
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
7 */
8 #include <linux/init.h>
9 #include <linux/kernel.h>
10 #include <linux/mm.h>
11 #include <linux/mmzone.h>
12 #include <linux/export.h>
13 #include <linux/nodemask.h>
14 #include <linux/swap.h>
15 #include <linux/memblock.h>
16 #include <linux/pfn.h>
17 #include <linux/acpi.h>
18 #include <linux/efi.h>
19 #include <linux/irq.h>
20 #include <linux/pci.h>
21 #include <asm/bootinfo.h>
22 #include <asm/loongson.h>
23 #include <asm/numa.h>
24 #include <asm/page.h>
25 #include <asm/pgalloc.h>
26 #include <asm/sections.h>
27 #include <asm/time.h>
28
29 int numa_off;
30 unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES];
31 EXPORT_SYMBOL(node_distances);
32
33 static struct numa_meminfo numa_meminfo;
34 cpumask_t cpus_on_node[MAX_NUMNODES];
35 cpumask_t phys_cpus_on_node[MAX_NUMNODES];
36 EXPORT_SYMBOL(cpus_on_node);
37
38 /*
39 * apicid, cpu, node mappings
40 */
41 s16 __cpuid_to_node[CONFIG_NR_CPUS] = {
42 [0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE
43 };
44 EXPORT_SYMBOL(__cpuid_to_node);
45
46 nodemask_t numa_nodes_parsed __initdata;
47
48 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
49 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
50 EXPORT_SYMBOL(__per_cpu_offset);
51
pcpu_cpu_to_node(int cpu)52 static int __init pcpu_cpu_to_node(int cpu)
53 {
54 return early_cpu_to_node(cpu);
55 }
56
pcpu_cpu_distance(unsigned int from,unsigned int to)57 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
58 {
59 if (early_cpu_to_node(from) == early_cpu_to_node(to))
60 return LOCAL_DISTANCE;
61 else
62 return REMOTE_DISTANCE;
63 }
64
pcpu_populate_pte(unsigned long addr)65 void __init pcpu_populate_pte(unsigned long addr)
66 {
67 populate_kernel_pte(addr);
68 }
69
setup_per_cpu_areas(void)70 void __init setup_per_cpu_areas(void)
71 {
72 unsigned long delta;
73 unsigned int cpu;
74 int rc = -EINVAL;
75
76 if (pcpu_chosen_fc == PCPU_FC_AUTO) {
77 if (nr_node_ids >= 8)
78 pcpu_chosen_fc = PCPU_FC_PAGE;
79 else
80 pcpu_chosen_fc = PCPU_FC_EMBED;
81 }
82
83 /*
84 * Always reserve area for module percpu variables. That's
85 * what the legacy allocator did.
86 */
87 if (pcpu_chosen_fc != PCPU_FC_PAGE) {
88 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
89 PERCPU_DYNAMIC_RESERVE, PMD_SIZE,
90 pcpu_cpu_distance, pcpu_cpu_to_node);
91 if (rc < 0)
92 pr_warn("%s allocator failed (%d), falling back to page size\n",
93 pcpu_fc_names[pcpu_chosen_fc], rc);
94 }
95 if (rc < 0)
96 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node);
97 if (rc < 0)
98 panic("cannot initialize percpu area (err=%d)", rc);
99
100 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
101 for_each_possible_cpu(cpu)
102 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
103 }
104 #endif
105
106 /*
107 * Get nodeid by logical cpu number.
108 * __cpuid_to_node maps phyical cpu id to node, so we
109 * should use cpu_logical_map(cpu) to index it.
110 *
111 * This routine is only used in early phase during
112 * booting, after setup_per_cpu_areas calling and numa_node
113 * initialization, cpu_to_node will be used instead.
114 */
early_cpu_to_node(int cpu)115 int early_cpu_to_node(int cpu)
116 {
117 int physid = cpu_logical_map(cpu);
118
119 if (physid < 0)
120 return NUMA_NO_NODE;
121
122 return __cpuid_to_node[physid];
123 }
124
early_numa_add_cpu(int cpuid,s16 node)125 void __init early_numa_add_cpu(int cpuid, s16 node)
126 {
127 int cpu = __cpu_number_map[cpuid];
128
129 if (cpu < 0)
130 return;
131
132 cpumask_set_cpu(cpu, &cpus_on_node[node]);
133 cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]);
134 }
135
numa_add_cpu(unsigned int cpu)136 void numa_add_cpu(unsigned int cpu)
137 {
138 int nid = cpu_to_node(cpu);
139 cpumask_set_cpu(cpu, &cpus_on_node[nid]);
140 }
141
numa_remove_cpu(unsigned int cpu)142 void numa_remove_cpu(unsigned int cpu)
143 {
144 int nid = cpu_to_node(cpu);
145 cpumask_clear_cpu(cpu, &cpus_on_node[nid]);
146 }
147
numa_add_memblk_to(int nid,u64 start,u64 end,struct numa_meminfo * mi)148 static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
149 struct numa_meminfo *mi)
150 {
151 /* ignore zero length blks */
152 if (start == end)
153 return 0;
154
155 /* whine about and ignore invalid blks */
156 if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
157 pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
158 nid, start, end - 1);
159 return 0;
160 }
161
162 if (mi->nr_blks >= NR_NODE_MEMBLKS) {
163 pr_err("NUMA: too many memblk ranges\n");
164 return -EINVAL;
165 }
166
167 mi->blk[mi->nr_blks].start = PFN_ALIGN(start);
168 mi->blk[mi->nr_blks].end = PFN_ALIGN(end - PAGE_SIZE + 1);
169 mi->blk[mi->nr_blks].nid = nid;
170 mi->nr_blks++;
171 return 0;
172 }
173
174 /**
175 * numa_add_memblk - Add one numa_memblk to numa_meminfo
176 * @nid: NUMA node ID of the new memblk
177 * @start: Start address of the new memblk
178 * @end: End address of the new memblk
179 *
180 * Add a new memblk to the default numa_meminfo.
181 *
182 * RETURNS:
183 * 0 on success, -errno on failure.
184 */
numa_add_memblk(int nid,u64 start,u64 end)185 int __init numa_add_memblk(int nid, u64 start, u64 end)
186 {
187 return numa_add_memblk_to(nid, start, end, &numa_meminfo);
188 }
189
node_mem_init(unsigned int node)190 static void __init node_mem_init(unsigned int node)
191 {
192 unsigned long start_pfn, end_pfn;
193 unsigned long node_addrspace_offset;
194
195 node_addrspace_offset = nid_to_addrbase(node);
196 pr_info("Node%d's addrspace_offset is 0x%lx\n",
197 node, node_addrspace_offset);
198
199 get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
200 pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n",
201 node, start_pfn, end_pfn);
202
203 alloc_node_data(node);
204 }
205
206 #ifdef CONFIG_ACPI_NUMA
207
add_node_intersection(u32 node,u64 start,u64 size,u32 type)208 static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type)
209 {
210 static unsigned long num_physpages;
211
212 num_physpages += (size >> PAGE_SHIFT);
213 pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
214 node, type, start, size);
215 pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
216 start >> PAGE_SHIFT, (start + size) >> PAGE_SHIFT, num_physpages);
217 memblock_set_node(start, size, &memblock.memory, node);
218 }
219
220 /*
221 * add_numamem_region
222 *
223 * Add a uasable memory region described by BIOS. The
224 * routine gets each intersection between BIOS's region
225 * and node's region, and adds them into node's memblock
226 * pool.
227 *
228 */
add_numamem_region(u64 start,u64 end,u32 type)229 static void __init add_numamem_region(u64 start, u64 end, u32 type)
230 {
231 u32 i;
232 u64 ofs = start;
233
234 if (start >= end) {
235 pr_debug("Invalid region: %016llx-%016llx\n", start, end);
236 return;
237 }
238
239 for (i = 0; i < numa_meminfo.nr_blks; i++) {
240 struct numa_memblk *mb = &numa_meminfo.blk[i];
241
242 if (ofs > mb->end)
243 continue;
244
245 if (end > mb->end) {
246 add_node_intersection(mb->nid, ofs, mb->end - ofs, type);
247 ofs = mb->end;
248 } else {
249 add_node_intersection(mb->nid, ofs, end - ofs, type);
250 break;
251 }
252 }
253 }
254
init_node_memblock(void)255 static void __init init_node_memblock(void)
256 {
257 u32 mem_type;
258 u64 mem_end, mem_start, mem_size;
259 efi_memory_desc_t *md;
260
261 /* Parse memory information and activate */
262 for_each_efi_memory_desc(md) {
263 mem_type = md->type;
264 mem_start = md->phys_addr;
265 mem_size = md->num_pages << EFI_PAGE_SHIFT;
266 mem_end = mem_start + mem_size;
267
268 switch (mem_type) {
269 case EFI_LOADER_CODE:
270 case EFI_LOADER_DATA:
271 case EFI_BOOT_SERVICES_CODE:
272 case EFI_BOOT_SERVICES_DATA:
273 case EFI_PERSISTENT_MEMORY:
274 case EFI_CONVENTIONAL_MEMORY:
275 add_numamem_region(mem_start, mem_end, mem_type);
276 break;
277 case EFI_PAL_CODE:
278 case EFI_UNUSABLE_MEMORY:
279 case EFI_ACPI_RECLAIM_MEMORY:
280 add_numamem_region(mem_start, mem_end, mem_type);
281 fallthrough;
282 case EFI_RESERVED_TYPE:
283 case EFI_RUNTIME_SERVICES_CODE:
284 case EFI_RUNTIME_SERVICES_DATA:
285 case EFI_MEMORY_MAPPED_IO:
286 case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
287 pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
288 mem_type, mem_start, mem_size);
289 break;
290 }
291 }
292 }
293
numa_default_distance(void)294 static void __init numa_default_distance(void)
295 {
296 int row, col;
297
298 for (row = 0; row < MAX_NUMNODES; row++)
299 for (col = 0; col < MAX_NUMNODES; col++) {
300 if (col == row)
301 node_distances[row][col] = LOCAL_DISTANCE;
302 else
303 /* We assume that one node per package here!
304 *
305 * A SLIT should be used for multiple nodes
306 * per package to override default setting.
307 */
308 node_distances[row][col] = REMOTE_DISTANCE;
309 }
310 }
311
312 /*
313 * fake_numa_init() - For Non-ACPI systems
314 * Return: 0 on success, -errno on failure.
315 */
fake_numa_init(void)316 static int __init fake_numa_init(void)
317 {
318 phys_addr_t start = memblock_start_of_DRAM();
319 phys_addr_t end = memblock_end_of_DRAM() - 1;
320
321 node_set(0, numa_nodes_parsed);
322 pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
323
324 return numa_add_memblk(0, start, end + 1);
325 }
326
init_numa_memory(void)327 int __init init_numa_memory(void)
328 {
329 int i;
330 int ret;
331 int node;
332
333 for (i = 0; i < NR_CPUS; i++)
334 set_cpuid_to_node(i, NUMA_NO_NODE);
335
336 numa_default_distance();
337 nodes_clear(numa_nodes_parsed);
338 nodes_clear(node_possible_map);
339 nodes_clear(node_online_map);
340 memset(&numa_meminfo, 0, sizeof(numa_meminfo));
341
342 /* Parse SRAT and SLIT if provided by firmware. */
343 ret = acpi_disabled ? fake_numa_init() : acpi_numa_init();
344 if (ret < 0)
345 return ret;
346
347 node_possible_map = numa_nodes_parsed;
348 if (WARN_ON(nodes_empty(node_possible_map)))
349 return -EINVAL;
350
351 init_node_memblock();
352 if (!memblock_validate_numa_coverage(SZ_1M))
353 return -EINVAL;
354
355 for_each_node_mask(node, node_possible_map) {
356 node_mem_init(node);
357 node_set_online(node);
358 }
359 max_low_pfn = PHYS_PFN(memblock_end_of_DRAM());
360
361 setup_nr_node_ids();
362 loongson_sysconf.nr_nodes = nr_node_ids;
363 loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]);
364
365 return 0;
366 }
367
368 #endif
369
paging_init(void)370 void __init paging_init(void)
371 {
372 unsigned int node;
373 unsigned long zones_size[MAX_NR_ZONES] = {0, };
374
375 for_each_online_node(node) {
376 unsigned long start_pfn, end_pfn;
377
378 get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
379
380 if (end_pfn > max_low_pfn)
381 max_low_pfn = end_pfn;
382 }
383 #ifdef CONFIG_ZONE_DMA32
384 zones_size[ZONE_DMA32] = MAX_DMA32_PFN;
385 #endif
386 zones_size[ZONE_NORMAL] = max_low_pfn;
387 free_area_init(zones_size);
388 }
389
mem_init(void)390 void __init mem_init(void)
391 {
392 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
393 memblock_free_all();
394 }
395
pcibus_to_node(struct pci_bus * bus)396 int pcibus_to_node(struct pci_bus *bus)
397 {
398 return dev_to_node(&bus->dev);
399 }
400 EXPORT_SYMBOL(pcibus_to_node);
401