1 // SPDX-License-Identifier: GPL-2.0
2 #define boot_fmt(fmt) "physmem: " fmt
3 #include <linux/processor.h>
4 #include <linux/errno.h>
5 #include <linux/init.h>
6 #include <asm/physmem_info.h>
7 #include <asm/stacktrace.h>
8 #include <asm/boot_data.h>
9 #include <asm/sparsemem.h>
10 #include <asm/sections.h>
11 #include <asm/setup.h>
12 #include <asm/sclp.h>
13 #include <asm/asm.h>
14 #include <asm/uv.h>
15 #include "decompressor.h"
16 #include "boot.h"
17
18 struct physmem_info __bootdata(physmem_info);
19 static unsigned int physmem_alloc_ranges;
20 static unsigned long physmem_alloc_pos;
21
22 /* up to 256 storage elements, 1020 subincrements each */
23 #define ENTRIES_EXTENDED_MAX \
24 (256 * (1020 / 2) * sizeof(struct physmem_range))
25
__get_physmem_range_ptr(u32 n)26 static struct physmem_range *__get_physmem_range_ptr(u32 n)
27 {
28 if (n < MEM_INLINED_ENTRIES)
29 return &physmem_info.online[n];
30 if (unlikely(!physmem_info.online_extended)) {
31 physmem_info.online_extended = (struct physmem_range *)physmem_alloc_range(
32 RR_MEM_DETECT_EXT, ENTRIES_EXTENDED_MAX, sizeof(long), 0,
33 physmem_alloc_pos, true);
34 }
35 return &physmem_info.online_extended[n - MEM_INLINED_ENTRIES];
36 }
37
38 /*
39 * sequential calls to add_physmem_online_range with adjacent memory ranges
40 * are merged together into single memory range.
41 */
add_physmem_online_range(u64 start,u64 end)42 void add_physmem_online_range(u64 start, u64 end)
43 {
44 struct physmem_range *range;
45
46 if (physmem_info.range_count) {
47 range = __get_physmem_range_ptr(physmem_info.range_count - 1);
48 if (range->end == start) {
49 range->end = end;
50 return;
51 }
52 }
53
54 range = __get_physmem_range_ptr(physmem_info.range_count);
55 range->start = start;
56 range->end = end;
57 physmem_info.range_count++;
58 }
59
__diag260(unsigned long rx1,unsigned long rx2)60 static int __diag260(unsigned long rx1, unsigned long rx2)
61 {
62 unsigned long reg1, reg2, ry;
63 union register_pair rx;
64 int cc, exception;
65 psw_t old;
66
67 rx.even = rx1;
68 rx.odd = rx2;
69 ry = 0x10; /* storage configuration */
70 exception = 1;
71 asm volatile(
72 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n"
73 " epsw %[reg1],%[reg2]\n"
74 " st %[reg1],0(%[psw_pgm])\n"
75 " st %[reg2],4(%[psw_pgm])\n"
76 " larl %[reg1],1f\n"
77 " stg %[reg1],8(%[psw_pgm])\n"
78 " diag %[rx],%[ry],0x260\n"
79 " lhi %[exc],0\n"
80 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n"
81 CC_IPM(cc)
82 : CC_OUT(cc, cc),
83 [exc] "+d" (exception),
84 [reg1] "=&d" (reg1),
85 [reg2] "=&a" (reg2),
86 [ry] "+&d" (ry),
87 "+Q" (get_lowcore()->program_new_psw),
88 "=Q" (old)
89 : [rx] "d" (rx.pair),
90 [psw_old] "a" (&old),
91 [psw_pgm] "a" (&get_lowcore()->program_new_psw)
92 : CC_CLOBBER_LIST("memory"));
93 cc = exception ? -1 : CC_TRANSFORM(cc);
94 return cc == 0 ? ry : -1;
95 }
96
diag260(void)97 static int diag260(void)
98 {
99 int rc, i;
100
101 struct {
102 unsigned long start;
103 unsigned long end;
104 } storage_extents[8] __aligned(16); /* VM supports up to 8 extends */
105
106 memset(storage_extents, 0, sizeof(storage_extents));
107 rc = __diag260((unsigned long)storage_extents, sizeof(storage_extents));
108 if (rc == -1)
109 return -1;
110
111 for (i = 0; i < min_t(int, rc, ARRAY_SIZE(storage_extents)); i++)
112 add_physmem_online_range(storage_extents[i].start, storage_extents[i].end + 1);
113 return 0;
114 }
115
116 #define DIAG500_SC_STOR_LIMIT 4
117
diag500_storage_limit(unsigned long * max_physmem_end)118 static int diag500_storage_limit(unsigned long *max_physmem_end)
119 {
120 unsigned long storage_limit;
121 unsigned long reg1, reg2;
122 psw_t old;
123
124 asm volatile(
125 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n"
126 " epsw %[reg1],%[reg2]\n"
127 " st %[reg1],0(%[psw_pgm])\n"
128 " st %[reg2],4(%[psw_pgm])\n"
129 " larl %[reg1],1f\n"
130 " stg %[reg1],8(%[psw_pgm])\n"
131 " lghi 1,%[subcode]\n"
132 " lghi 2,0\n"
133 " diag 2,4,0x500\n"
134 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n"
135 " lgr %[slimit],2\n"
136 : [reg1] "=&d" (reg1),
137 [reg2] "=&a" (reg2),
138 [slimit] "=d" (storage_limit),
139 "=Q" (get_lowcore()->program_new_psw),
140 "=Q" (old)
141 : [psw_old] "a" (&old),
142 [psw_pgm] "a" (&get_lowcore()->program_new_psw),
143 [subcode] "i" (DIAG500_SC_STOR_LIMIT)
144 : "memory", "1", "2");
145 if (!storage_limit)
146 return -EINVAL;
147 /* Convert inclusive end to exclusive end */
148 *max_physmem_end = storage_limit + 1;
149 return 0;
150 }
151
tprot(unsigned long addr)152 static int tprot(unsigned long addr)
153 {
154 unsigned long reg1, reg2;
155 int cc, exception;
156 psw_t old;
157
158 exception = 1;
159 asm volatile(
160 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n"
161 " epsw %[reg1],%[reg2]\n"
162 " st %[reg1],0(%[psw_pgm])\n"
163 " st %[reg2],4(%[psw_pgm])\n"
164 " larl %[reg1],1f\n"
165 " stg %[reg1],8(%[psw_pgm])\n"
166 " tprot 0(%[addr]),0\n"
167 " lhi %[exc],0\n"
168 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n"
169 CC_IPM(cc)
170 : CC_OUT(cc, cc),
171 [exc] "+d" (exception),
172 [reg1] "=&d" (reg1),
173 [reg2] "=&a" (reg2),
174 "=Q" (get_lowcore()->program_new_psw.addr),
175 "=Q" (old)
176 : [psw_old] "a" (&old),
177 [psw_pgm] "a" (&get_lowcore()->program_new_psw),
178 [addr] "a" (addr)
179 : CC_CLOBBER_LIST("memory"));
180 cc = exception ? -EFAULT : CC_TRANSFORM(cc);
181 return cc;
182 }
183
search_mem_end(void)184 static unsigned long search_mem_end(void)
185 {
186 unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */
187 unsigned long offset = 0;
188 unsigned long pivot;
189
190 while (range > 1) {
191 range >>= 1;
192 pivot = offset + range;
193 if (!tprot(pivot << 20))
194 offset = pivot;
195 }
196 return (offset + 1) << 20;
197 }
198
detect_max_physmem_end(void)199 unsigned long detect_max_physmem_end(void)
200 {
201 unsigned long max_physmem_end = 0;
202
203 if (!diag500_storage_limit(&max_physmem_end)) {
204 physmem_info.info_source = MEM_DETECT_DIAG500_STOR_LIMIT;
205 } else if (!sclp_early_get_memsize(&max_physmem_end)) {
206 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO;
207 } else {
208 max_physmem_end = search_mem_end();
209 physmem_info.info_source = MEM_DETECT_BIN_SEARCH;
210 }
211 boot_debug("Max physical memory: 0x%016lx (info source: %s)\n", max_physmem_end,
212 get_physmem_info_source());
213 return max_physmem_end;
214 }
215
detect_physmem_online_ranges(unsigned long max_physmem_end)216 void detect_physmem_online_ranges(unsigned long max_physmem_end)
217 {
218 unsigned long start, end;
219 int i;
220
221 if (!sclp_early_read_storage_info()) {
222 physmem_info.info_source = MEM_DETECT_SCLP_STOR_INFO;
223 } else if (physmem_info.info_source == MEM_DETECT_DIAG500_STOR_LIMIT) {
224 unsigned long online_end;
225
226 if (!sclp_early_get_memsize(&online_end)) {
227 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO;
228 add_physmem_online_range(0, online_end);
229 }
230 } else if (!diag260()) {
231 physmem_info.info_source = MEM_DETECT_DIAG260;
232 } else if (max_physmem_end) {
233 add_physmem_online_range(0, max_physmem_end);
234 }
235 boot_debug("Online memory ranges (info source: %s):\n", get_physmem_info_source());
236 for_each_physmem_online_range(i, &start, &end)
237 boot_debug(" online [%d]: 0x%016lx-0x%016lx\n", i, start, end);
238 }
239
physmem_set_usable_limit(unsigned long limit)240 void physmem_set_usable_limit(unsigned long limit)
241 {
242 physmem_info.usable = limit;
243 physmem_alloc_pos = limit;
244 boot_debug("Usable memory limit: 0x%016lx\n", limit);
245 }
246
die_oom(unsigned long size,unsigned long align,unsigned long min,unsigned long max)247 static void die_oom(unsigned long size, unsigned long align, unsigned long min, unsigned long max)
248 {
249 unsigned long start, end, total_mem = 0, total_reserved_mem = 0;
250 struct reserved_range *range;
251 enum reserved_range_type t;
252 int i;
253
254 boot_emerg("Linux version %s\n", kernel_version);
255 if (!is_prot_virt_guest() && early_command_line[0])
256 boot_emerg("Kernel command line: %s\n", early_command_line);
257 boot_emerg("Out of memory allocating %lu bytes 0x%lx aligned in range %lx:%lx\n",
258 size, align, min, max);
259 boot_emerg("Reserved memory ranges:\n");
260 for_each_physmem_reserved_range(t, range, &start, &end) {
261 boot_emerg("%016lx %016lx %s\n", start, end, get_rr_type_name(t));
262 total_reserved_mem += end - start;
263 }
264 boot_emerg("Usable online memory ranges (info source: %s [%d]):\n",
265 get_physmem_info_source(), physmem_info.info_source);
266 for_each_physmem_usable_range(i, &start, &end) {
267 boot_emerg("%016lx %016lx\n", start, end);
268 total_mem += end - start;
269 }
270 boot_emerg("Usable online memory total: %lu Reserved: %lu Free: %lu\n",
271 total_mem, total_reserved_mem,
272 total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0);
273 print_stacktrace(current_frame_address());
274 boot_emerg(" -- System halted\n");
275 disabled_wait();
276 }
277
_physmem_reserve(enum reserved_range_type type,unsigned long addr,unsigned long size)278 static void _physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size)
279 {
280 physmem_info.reserved[type].start = addr;
281 physmem_info.reserved[type].end = addr + size;
282 }
283
physmem_reserve(enum reserved_range_type type,unsigned long addr,unsigned long size)284 void physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size)
285 {
286 _physmem_reserve(type, addr, size);
287 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Reserve:", addr, addr + size,
288 get_rr_type_name(type));
289 }
290
physmem_free(enum reserved_range_type type)291 void physmem_free(enum reserved_range_type type)
292 {
293 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Free:", physmem_info.reserved[type].start,
294 physmem_info.reserved[type].end, get_rr_type_name(type));
295 physmem_info.reserved[type].start = 0;
296 physmem_info.reserved[type].end = 0;
297 }
298
__physmem_alloc_intersects(unsigned long addr,unsigned long size,unsigned long * intersection_start)299 static bool __physmem_alloc_intersects(unsigned long addr, unsigned long size,
300 unsigned long *intersection_start)
301 {
302 unsigned long res_addr, res_size;
303 int t;
304
305 for (t = 0; t < RR_MAX; t++) {
306 if (!get_physmem_reserved(t, &res_addr, &res_size))
307 continue;
308 if (intersects(addr, size, res_addr, res_size)) {
309 *intersection_start = res_addr;
310 return true;
311 }
312 }
313 return ipl_report_certs_intersects(addr, size, intersection_start);
314 }
315
__physmem_alloc_range(unsigned long size,unsigned long align,unsigned long min,unsigned long max,unsigned int from_ranges,unsigned int * ranges_left,bool die_on_oom)316 static unsigned long __physmem_alloc_range(unsigned long size, unsigned long align,
317 unsigned long min, unsigned long max,
318 unsigned int from_ranges, unsigned int *ranges_left,
319 bool die_on_oom)
320 {
321 unsigned int nranges = from_ranges ?: physmem_info.range_count;
322 unsigned long range_start, range_end;
323 unsigned long intersection_start;
324 unsigned long addr, pos = max;
325
326 align = max(align, 8UL);
327 while (nranges) {
328 __get_physmem_range(nranges - 1, &range_start, &range_end, false);
329 pos = min(range_end, pos);
330
331 if (round_up(min, align) + size > pos)
332 break;
333 addr = round_down(pos - size, align);
334 if (range_start > addr) {
335 nranges--;
336 continue;
337 }
338 if (__physmem_alloc_intersects(addr, size, &intersection_start)) {
339 pos = intersection_start;
340 continue;
341 }
342
343 if (ranges_left)
344 *ranges_left = nranges;
345 return addr;
346 }
347 if (die_on_oom)
348 die_oom(size, align, min, max);
349 return 0;
350 }
351
physmem_alloc_range(enum reserved_range_type type,unsigned long size,unsigned long align,unsigned long min,unsigned long max,bool die_on_oom)352 unsigned long physmem_alloc_range(enum reserved_range_type type, unsigned long size,
353 unsigned long align, unsigned long min, unsigned long max,
354 bool die_on_oom)
355 {
356 unsigned long addr;
357
358 max = min(max, physmem_alloc_pos);
359 addr = __physmem_alloc_range(size, align, min, max, 0, NULL, die_on_oom);
360 if (addr)
361 _physmem_reserve(type, addr, size);
362 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Alloc range:", addr, addr + size,
363 get_rr_type_name(type));
364 return addr;
365 }
366
physmem_alloc(enum reserved_range_type type,unsigned long size,unsigned long align,bool die_on_oom)367 unsigned long physmem_alloc(enum reserved_range_type type, unsigned long size,
368 unsigned long align, bool die_on_oom)
369 {
370 struct reserved_range *range = &physmem_info.reserved[type];
371 struct reserved_range *new_range = NULL;
372 unsigned int ranges_left;
373 unsigned long addr;
374
375 addr = __physmem_alloc_range(size, align, 0, physmem_alloc_pos, physmem_alloc_ranges,
376 &ranges_left, die_on_oom);
377 if (!addr)
378 return 0;
379 /* if not a consecutive allocation of the same type or first allocation */
380 if (range->start != addr + size) {
381 if (range->end) {
382 addr = __physmem_alloc_range(sizeof(struct reserved_range), 0, 0,
383 physmem_alloc_pos, physmem_alloc_ranges,
384 &ranges_left, true);
385 new_range = (struct reserved_range *)addr;
386 addr = __physmem_alloc_range(size, align, 0, addr, ranges_left,
387 &ranges_left, die_on_oom);
388 if (!addr)
389 return 0;
390 *new_range = *range;
391 range->chain = new_range;
392 }
393 range->end = addr + size;
394 }
395 if (type != RR_VMEM) {
396 boot_debug("%-14s 0x%016lx-0x%016lx %-20s align 0x%lx split %d\n", "Alloc topdown:",
397 addr, addr + size, get_rr_type_name(type), align, !!new_range);
398 }
399 range->start = addr;
400 physmem_alloc_pos = addr;
401 physmem_alloc_ranges = ranges_left;
402 return addr;
403 }
404
physmem_alloc_or_die(enum reserved_range_type type,unsigned long size,unsigned long align)405 unsigned long physmem_alloc_or_die(enum reserved_range_type type, unsigned long size,
406 unsigned long align)
407 {
408 return physmem_alloc(type, size, align, true);
409 }
410
get_physmem_alloc_pos(void)411 unsigned long get_physmem_alloc_pos(void)
412 {
413 return physmem_alloc_pos;
414 }
415
dump_physmem_reserved(void)416 void dump_physmem_reserved(void)
417 {
418 struct reserved_range *range;
419 enum reserved_range_type t;
420 unsigned long start, end;
421
422 boot_debug("Reserved memory ranges:\n");
423 for_each_physmem_reserved_range(t, range, &start, &end) {
424 if (end) {
425 boot_debug("%-14s 0x%016lx-0x%016lx @%012lx chain %012lx\n",
426 get_rr_type_name(t), start, end, (unsigned long)range,
427 (unsigned long)range->chain);
428 }
429 }
430 }
431