1 // SPDX-License-Identifier: GPL-2.0 2 #define boot_fmt(fmt) "physmem: " fmt 3 #include <linux/processor.h> 4 #include <linux/errno.h> 5 #include <linux/init.h> 6 #include <asm/physmem_info.h> 7 #include <asm/stacktrace.h> 8 #include <asm/boot_data.h> 9 #include <asm/sparsemem.h> 10 #include <asm/sections.h> 11 #include <asm/setup.h> 12 #include <asm/sclp.h> 13 #include <asm/asm.h> 14 #include <asm/uv.h> 15 #include "decompressor.h" 16 #include "boot.h" 17 18 struct physmem_info __bootdata(physmem_info); 19 static unsigned int physmem_alloc_ranges; 20 static unsigned long physmem_alloc_pos; 21 22 /* up to 256 storage elements, 1020 subincrements each */ 23 #define ENTRIES_EXTENDED_MAX \ 24 (256 * (1020 / 2) * sizeof(struct physmem_range)) 25 26 static struct physmem_range *__get_physmem_range_ptr(u32 n) 27 { 28 if (n < MEM_INLINED_ENTRIES) 29 return &physmem_info.online[n]; 30 if (unlikely(!physmem_info.online_extended)) { 31 physmem_info.online_extended = (struct physmem_range *)physmem_alloc_range( 32 RR_MEM_DETECT_EXT, ENTRIES_EXTENDED_MAX, sizeof(long), 0, 33 physmem_alloc_pos, true); 34 } 35 return &physmem_info.online_extended[n - MEM_INLINED_ENTRIES]; 36 } 37 38 /* 39 * sequential calls to add_physmem_online_range with adjacent memory ranges 40 * are merged together into single memory range. 41 */ 42 void add_physmem_online_range(u64 start, u64 end) 43 { 44 struct physmem_range *range; 45 46 if (physmem_info.range_count) { 47 range = __get_physmem_range_ptr(physmem_info.range_count - 1); 48 if (range->end == start) { 49 range->end = end; 50 return; 51 } 52 } 53 54 range = __get_physmem_range_ptr(physmem_info.range_count); 55 range->start = start; 56 range->end = end; 57 physmem_info.range_count++; 58 } 59 60 static int __diag260(unsigned long rx1, unsigned long rx2) 61 { 62 unsigned long reg1, reg2, ry; 63 union register_pair rx; 64 int cc, exception; 65 psw_t old; 66 67 rx.even = rx1; 68 rx.odd = rx2; 69 ry = 0x10; /* storage configuration */ 70 exception = 1; 71 asm volatile( 72 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" 73 " epsw %[reg1],%[reg2]\n" 74 " st %[reg1],0(%[psw_pgm])\n" 75 " st %[reg2],4(%[psw_pgm])\n" 76 " larl %[reg1],1f\n" 77 " stg %[reg1],8(%[psw_pgm])\n" 78 " diag %[rx],%[ry],0x260\n" 79 " lhi %[exc],0\n" 80 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" 81 CC_IPM(cc) 82 : CC_OUT(cc, cc), 83 [exc] "+d" (exception), 84 [reg1] "=&d" (reg1), 85 [reg2] "=&a" (reg2), 86 [ry] "+&d" (ry), 87 "+Q" (get_lowcore()->program_new_psw), 88 "=Q" (old) 89 : [rx] "d" (rx.pair), 90 [psw_old] "a" (&old), 91 [psw_pgm] "a" (&get_lowcore()->program_new_psw) 92 : CC_CLOBBER_LIST("memory")); 93 cc = exception ? -1 : CC_TRANSFORM(cc); 94 return cc == 0 ? ry : -1; 95 } 96 97 static int diag260(void) 98 { 99 int rc, i; 100 101 struct { 102 unsigned long start; 103 unsigned long end; 104 } storage_extents[8] __aligned(16); /* VM supports up to 8 extends */ 105 106 memset(storage_extents, 0, sizeof(storage_extents)); 107 rc = __diag260((unsigned long)storage_extents, sizeof(storage_extents)); 108 if (rc == -1) 109 return -1; 110 111 for (i = 0; i < min_t(int, rc, ARRAY_SIZE(storage_extents)); i++) 112 add_physmem_online_range(storage_extents[i].start, storage_extents[i].end + 1); 113 return 0; 114 } 115 116 #define DIAG500_SC_STOR_LIMIT 4 117 118 static int diag500_storage_limit(unsigned long *max_physmem_end) 119 { 120 unsigned long storage_limit; 121 unsigned long reg1, reg2; 122 psw_t old; 123 124 asm volatile( 125 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" 126 " epsw %[reg1],%[reg2]\n" 127 " st %[reg1],0(%[psw_pgm])\n" 128 " st %[reg2],4(%[psw_pgm])\n" 129 " larl %[reg1],1f\n" 130 " stg %[reg1],8(%[psw_pgm])\n" 131 " lghi 1,%[subcode]\n" 132 " lghi 2,0\n" 133 " diag 2,4,0x500\n" 134 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" 135 " lgr %[slimit],2\n" 136 : [reg1] "=&d" (reg1), 137 [reg2] "=&a" (reg2), 138 [slimit] "=d" (storage_limit), 139 "=Q" (get_lowcore()->program_new_psw), 140 "=Q" (old) 141 : [psw_old] "a" (&old), 142 [psw_pgm] "a" (&get_lowcore()->program_new_psw), 143 [subcode] "i" (DIAG500_SC_STOR_LIMIT) 144 : "memory", "1", "2"); 145 if (!storage_limit) 146 return -EINVAL; 147 /* Convert inclusive end to exclusive end */ 148 *max_physmem_end = storage_limit + 1; 149 return 0; 150 } 151 152 static int tprot(unsigned long addr) 153 { 154 unsigned long reg1, reg2; 155 int cc, exception; 156 psw_t old; 157 158 exception = 1; 159 asm volatile( 160 " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" 161 " epsw %[reg1],%[reg2]\n" 162 " st %[reg1],0(%[psw_pgm])\n" 163 " st %[reg2],4(%[psw_pgm])\n" 164 " larl %[reg1],1f\n" 165 " stg %[reg1],8(%[psw_pgm])\n" 166 " tprot 0(%[addr]),0\n" 167 " lhi %[exc],0\n" 168 "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" 169 CC_IPM(cc) 170 : CC_OUT(cc, cc), 171 [exc] "+d" (exception), 172 [reg1] "=&d" (reg1), 173 [reg2] "=&a" (reg2), 174 "=Q" (get_lowcore()->program_new_psw.addr), 175 "=Q" (old) 176 : [psw_old] "a" (&old), 177 [psw_pgm] "a" (&get_lowcore()->program_new_psw), 178 [addr] "a" (addr) 179 : CC_CLOBBER_LIST("memory")); 180 cc = exception ? -EFAULT : CC_TRANSFORM(cc); 181 return cc; 182 } 183 184 static unsigned long search_mem_end(void) 185 { 186 unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ 187 unsigned long offset = 0; 188 unsigned long pivot; 189 190 while (range > 1) { 191 range >>= 1; 192 pivot = offset + range; 193 if (!tprot(pivot << 20)) 194 offset = pivot; 195 } 196 return (offset + 1) << 20; 197 } 198 199 unsigned long detect_max_physmem_end(void) 200 { 201 unsigned long max_physmem_end = 0; 202 203 if (!diag500_storage_limit(&max_physmem_end)) { 204 physmem_info.info_source = MEM_DETECT_DIAG500_STOR_LIMIT; 205 } else if (!sclp_early_get_memsize(&max_physmem_end)) { 206 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO; 207 } else { 208 max_physmem_end = search_mem_end(); 209 physmem_info.info_source = MEM_DETECT_BIN_SEARCH; 210 } 211 boot_debug("Max physical memory: 0x%016lx (info source: %s)\n", max_physmem_end, 212 get_physmem_info_source()); 213 return max_physmem_end; 214 } 215 216 void detect_physmem_online_ranges(unsigned long max_physmem_end) 217 { 218 unsigned long start, end; 219 int i; 220 221 if (!sclp_early_read_storage_info()) { 222 physmem_info.info_source = MEM_DETECT_SCLP_STOR_INFO; 223 } else if (physmem_info.info_source == MEM_DETECT_DIAG500_STOR_LIMIT) { 224 unsigned long online_end; 225 226 if (!sclp_early_get_memsize(&online_end)) { 227 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO; 228 add_physmem_online_range(0, online_end); 229 } 230 } else if (!diag260()) { 231 physmem_info.info_source = MEM_DETECT_DIAG260; 232 } else if (max_physmem_end) { 233 add_physmem_online_range(0, max_physmem_end); 234 } 235 boot_debug("Online memory ranges (info source: %s):\n", get_physmem_info_source()); 236 for_each_physmem_online_range(i, &start, &end) 237 boot_debug(" online [%d]: 0x%016lx-0x%016lx\n", i, start, end); 238 } 239 240 void physmem_set_usable_limit(unsigned long limit) 241 { 242 physmem_info.usable = limit; 243 physmem_alloc_pos = limit; 244 boot_debug("Usable memory limit: 0x%016lx\n", limit); 245 } 246 247 static void die_oom(unsigned long size, unsigned long align, unsigned long min, unsigned long max) 248 { 249 unsigned long start, end, total_mem = 0, total_reserved_mem = 0; 250 struct reserved_range *range; 251 enum reserved_range_type t; 252 int i; 253 254 boot_emerg("Linux version %s\n", kernel_version); 255 if (!is_prot_virt_guest() && early_command_line[0]) 256 boot_emerg("Kernel command line: %s\n", early_command_line); 257 boot_emerg("Out of memory allocating %lu bytes 0x%lx aligned in range %lx:%lx\n", 258 size, align, min, max); 259 boot_emerg("Reserved memory ranges:\n"); 260 for_each_physmem_reserved_range(t, range, &start, &end) { 261 boot_emerg("%016lx %016lx %s\n", start, end, get_rr_type_name(t)); 262 total_reserved_mem += end - start; 263 } 264 boot_emerg("Usable online memory ranges (info source: %s [%d]):\n", 265 get_physmem_info_source(), physmem_info.info_source); 266 for_each_physmem_usable_range(i, &start, &end) { 267 boot_emerg("%016lx %016lx\n", start, end); 268 total_mem += end - start; 269 } 270 boot_emerg("Usable online memory total: %lu Reserved: %lu Free: %lu\n", 271 total_mem, total_reserved_mem, 272 total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); 273 print_stacktrace(current_frame_address()); 274 boot_emerg(" -- System halted\n"); 275 disabled_wait(); 276 } 277 278 static void _physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size) 279 { 280 physmem_info.reserved[type].start = addr; 281 physmem_info.reserved[type].end = addr + size; 282 } 283 284 void physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size) 285 { 286 _physmem_reserve(type, addr, size); 287 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Reserve:", addr, addr + size, 288 get_rr_type_name(type)); 289 } 290 291 void physmem_free(enum reserved_range_type type) 292 { 293 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Free:", physmem_info.reserved[type].start, 294 physmem_info.reserved[type].end, get_rr_type_name(type)); 295 physmem_info.reserved[type].start = 0; 296 physmem_info.reserved[type].end = 0; 297 } 298 299 static bool __physmem_alloc_intersects(unsigned long addr, unsigned long size, 300 unsigned long *intersection_start) 301 { 302 unsigned long res_addr, res_size; 303 int t; 304 305 for (t = 0; t < RR_MAX; t++) { 306 if (!get_physmem_reserved(t, &res_addr, &res_size)) 307 continue; 308 if (intersects(addr, size, res_addr, res_size)) { 309 *intersection_start = res_addr; 310 return true; 311 } 312 } 313 return ipl_report_certs_intersects(addr, size, intersection_start); 314 } 315 316 static unsigned long __physmem_alloc_range(unsigned long size, unsigned long align, 317 unsigned long min, unsigned long max, 318 unsigned int from_ranges, unsigned int *ranges_left, 319 bool die_on_oom) 320 { 321 unsigned int nranges = from_ranges ?: physmem_info.range_count; 322 unsigned long range_start, range_end; 323 unsigned long intersection_start; 324 unsigned long addr, pos = max; 325 326 align = max(align, 8UL); 327 while (nranges) { 328 __get_physmem_range(nranges - 1, &range_start, &range_end, false); 329 pos = min(range_end, pos); 330 331 if (round_up(min, align) + size > pos) 332 break; 333 addr = round_down(pos - size, align); 334 if (range_start > addr) { 335 nranges--; 336 continue; 337 } 338 if (__physmem_alloc_intersects(addr, size, &intersection_start)) { 339 pos = intersection_start; 340 continue; 341 } 342 343 if (ranges_left) 344 *ranges_left = nranges; 345 return addr; 346 } 347 if (die_on_oom) 348 die_oom(size, align, min, max); 349 return 0; 350 } 351 352 unsigned long physmem_alloc_range(enum reserved_range_type type, unsigned long size, 353 unsigned long align, unsigned long min, unsigned long max, 354 bool die_on_oom) 355 { 356 unsigned long addr; 357 358 max = min(max, physmem_alloc_pos); 359 addr = __physmem_alloc_range(size, align, min, max, 0, NULL, die_on_oom); 360 if (addr) 361 _physmem_reserve(type, addr, size); 362 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Alloc range:", addr, addr + size, 363 get_rr_type_name(type)); 364 return addr; 365 } 366 367 unsigned long physmem_alloc(enum reserved_range_type type, unsigned long size, 368 unsigned long align, bool die_on_oom) 369 { 370 struct reserved_range *range = &physmem_info.reserved[type]; 371 struct reserved_range *new_range = NULL; 372 unsigned int ranges_left; 373 unsigned long addr; 374 375 addr = __physmem_alloc_range(size, align, 0, physmem_alloc_pos, physmem_alloc_ranges, 376 &ranges_left, die_on_oom); 377 if (!addr) 378 return 0; 379 /* if not a consecutive allocation of the same type or first allocation */ 380 if (range->start != addr + size) { 381 if (range->end) { 382 addr = __physmem_alloc_range(sizeof(struct reserved_range), 0, 0, 383 physmem_alloc_pos, physmem_alloc_ranges, 384 &ranges_left, true); 385 new_range = (struct reserved_range *)addr; 386 addr = __physmem_alloc_range(size, align, 0, addr, ranges_left, 387 &ranges_left, die_on_oom); 388 if (!addr) 389 return 0; 390 *new_range = *range; 391 range->chain = new_range; 392 } 393 range->end = addr + size; 394 } 395 if (type != RR_VMEM) { 396 boot_debug("%-14s 0x%016lx-0x%016lx %-20s align 0x%lx split %d\n", "Alloc topdown:", 397 addr, addr + size, get_rr_type_name(type), align, !!new_range); 398 } 399 range->start = addr; 400 physmem_alloc_pos = addr; 401 physmem_alloc_ranges = ranges_left; 402 return addr; 403 } 404 405 unsigned long physmem_alloc_or_die(enum reserved_range_type type, unsigned long size, 406 unsigned long align) 407 { 408 return physmem_alloc(type, size, align, true); 409 } 410 411 unsigned long get_physmem_alloc_pos(void) 412 { 413 return physmem_alloc_pos; 414 } 415 416 void dump_physmem_reserved(void) 417 { 418 struct reserved_range *range; 419 enum reserved_range_type t; 420 unsigned long start, end; 421 422 boot_debug("Reserved memory ranges:\n"); 423 for_each_physmem_reserved_range(t, range, &start, &end) { 424 if (end) { 425 boot_debug("%-14s 0x%016lx-0x%016lx @%012lx chain %012lx\n", 426 get_rr_type_name(t), start, end, (unsigned long)range, 427 (unsigned long)range->chain); 428 } 429 } 430 } 431