1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Google, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/device.h> 9 #include <linux/err.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/io.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/memblock.h> 16 #include <linux/rslib.h> 17 #include <linux/slab.h> 18 #include <linux/uaccess.h> 19 #include <linux/vmalloc.h> 20 #include <linux/mm.h> 21 #include <asm/page.h> 22 23 #include "ram_internal.h" 24 25 /** 26 * struct persistent_ram_buffer - persistent circular RAM buffer 27 * 28 * @sig: Signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value) 29 * @start: First valid byte in the buffer. 30 * @size: Number of valid bytes in the buffer. 31 * @data: The contents of the buffer. 32 */ 33 struct persistent_ram_buffer { 34 uint32_t sig; 35 atomic_t start; 36 atomic_t size; 37 uint8_t data[]; 38 }; 39 40 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ 41 42 static inline size_t buffer_size(struct persistent_ram_zone *prz) 43 { 44 return atomic_read(&prz->buffer->size); 45 } 46 47 static inline size_t buffer_start(struct persistent_ram_zone *prz) 48 { 49 return atomic_read(&prz->buffer->start); 50 } 51 52 /* increase and wrap the start pointer, returning the old value */ 53 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) 54 { 55 int old; 56 int new; 57 unsigned long flags = 0; 58 59 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 60 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 61 62 old = atomic_read(&prz->buffer->start); 63 new = old + a; 64 while (unlikely(new >= prz->buffer_size)) 65 new -= prz->buffer_size; 66 atomic_set(&prz->buffer->start, new); 67 68 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 69 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 70 71 return old; 72 } 73 74 /* increase the size counter until it hits the max size */ 75 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a) 76 { 77 size_t old; 78 size_t new; 79 unsigned long flags = 0; 80 81 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 82 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 83 84 old = atomic_read(&prz->buffer->size); 85 if (old == prz->buffer_size) 86 goto exit; 87 88 new = old + a; 89 if (new > prz->buffer_size) 90 new = prz->buffer_size; 91 atomic_set(&prz->buffer->size, new); 92 93 exit: 94 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 95 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 96 } 97 98 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, 99 uint8_t *data, size_t len, uint8_t *ecc) 100 { 101 int i; 102 103 /* Initialize the parity buffer */ 104 memset(prz->ecc_info.par, 0, 105 prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0])); 106 encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0); 107 for (i = 0; i < prz->ecc_info.ecc_size; i++) 108 ecc[i] = prz->ecc_info.par[i]; 109 } 110 111 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, 112 void *data, size_t len, uint8_t *ecc) 113 { 114 int i; 115 116 for (i = 0; i < prz->ecc_info.ecc_size; i++) 117 prz->ecc_info.par[i] = ecc[i]; 118 return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len, 119 NULL, 0, NULL, 0, NULL); 120 } 121 122 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, 123 unsigned int start, unsigned int count) 124 { 125 struct persistent_ram_buffer *buffer = prz->buffer; 126 uint8_t *buffer_end = buffer->data + prz->buffer_size; 127 uint8_t *block; 128 uint8_t *par; 129 int ecc_block_size = prz->ecc_info.block_size; 130 int ecc_size = prz->ecc_info.ecc_size; 131 int size = ecc_block_size; 132 133 if (!ecc_size) 134 return; 135 136 block = buffer->data + (start & ~(ecc_block_size - 1)); 137 par = prz->par_buffer + (start / ecc_block_size) * ecc_size; 138 139 do { 140 if (block + ecc_block_size > buffer_end) 141 size = buffer_end - block; 142 persistent_ram_encode_rs8(prz, block, size, par); 143 block += ecc_block_size; 144 par += ecc_size; 145 } while (block < buffer->data + start + count); 146 } 147 148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) 149 { 150 struct persistent_ram_buffer *buffer = prz->buffer; 151 152 if (!prz->ecc_info.ecc_size) 153 return; 154 155 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), 156 prz->par_header); 157 } 158 159 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) 160 { 161 struct persistent_ram_buffer *buffer = prz->buffer; 162 uint8_t *block; 163 uint8_t *par; 164 165 if (!prz->ecc_info.ecc_size) 166 return; 167 168 block = buffer->data; 169 par = prz->par_buffer; 170 while (block < buffer->data + buffer_size(prz)) { 171 int numerr; 172 int size = prz->ecc_info.block_size; 173 if (block + size > buffer->data + prz->buffer_size) 174 size = buffer->data + prz->buffer_size - block; 175 numerr = persistent_ram_decode_rs8(prz, block, size, par); 176 if (numerr > 0) { 177 pr_devel("error in block %p, %d\n", block, numerr); 178 prz->corrected_bytes += numerr; 179 } else if (numerr < 0) { 180 pr_devel("uncorrectable error in block %p\n", block); 181 prz->bad_blocks++; 182 } 183 block += prz->ecc_info.block_size; 184 par += prz->ecc_info.ecc_size; 185 } 186 } 187 188 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, 189 struct persistent_ram_ecc_info *ecc_info) 190 { 191 int numerr; 192 struct persistent_ram_buffer *buffer = prz->buffer; 193 size_t ecc_blocks; 194 size_t ecc_total; 195 196 if (!ecc_info || !ecc_info->ecc_size) 197 return 0; 198 199 prz->ecc_info.block_size = ecc_info->block_size ?: 128; 200 prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16; 201 prz->ecc_info.symsize = ecc_info->symsize ?: 8; 202 prz->ecc_info.poly = ecc_info->poly ?: 0x11d; 203 204 ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size, 205 prz->ecc_info.block_size + 206 prz->ecc_info.ecc_size); 207 ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size; 208 if (ecc_total >= prz->buffer_size) { 209 pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n", 210 __func__, prz->ecc_info.ecc_size, 211 ecc_total, prz->buffer_size); 212 return -EINVAL; 213 } 214 215 prz->buffer_size -= ecc_total; 216 prz->par_buffer = buffer->data + prz->buffer_size; 217 prz->par_header = prz->par_buffer + 218 ecc_blocks * prz->ecc_info.ecc_size; 219 220 /* 221 * first consecutive root is 0 222 * primitive element to generate roots = 1 223 */ 224 prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly, 225 0, 1, prz->ecc_info.ecc_size); 226 if (prz->rs_decoder == NULL) { 227 pr_info("init_rs failed\n"); 228 return -EINVAL; 229 } 230 231 /* allocate workspace instead of using stack VLA */ 232 prz->ecc_info.par = kmalloc_objs(*prz->ecc_info.par, 233 prz->ecc_info.ecc_size, GFP_KERNEL); 234 if (!prz->ecc_info.par) { 235 pr_err("cannot allocate ECC parity workspace\n"); 236 return -ENOMEM; 237 } 238 239 prz->corrected_bytes = 0; 240 prz->bad_blocks = 0; 241 242 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), 243 prz->par_header); 244 if (numerr > 0) { 245 pr_info("error in header, %d\n", numerr); 246 prz->corrected_bytes += numerr; 247 } else if (numerr < 0) { 248 pr_info_ratelimited("uncorrectable error in header\n"); 249 prz->bad_blocks++; 250 } 251 252 return 0; 253 } 254 255 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, 256 char *str, size_t len) 257 { 258 ssize_t ret; 259 260 if (!prz->ecc_info.ecc_size) 261 return 0; 262 263 if (prz->corrected_bytes || prz->bad_blocks) 264 ret = snprintf(str, len, "" 265 "\nECC: %d Corrected bytes, %d unrecoverable blocks\n", 266 prz->corrected_bytes, prz->bad_blocks); 267 else 268 ret = snprintf(str, len, "\nECC: No errors detected\n"); 269 270 return ret; 271 } 272 273 static void notrace persistent_ram_update(struct persistent_ram_zone *prz, 274 const void *s, unsigned int start, unsigned int count) 275 { 276 struct persistent_ram_buffer *buffer = prz->buffer; 277 memcpy_toio(buffer->data + start, s, count); 278 persistent_ram_update_ecc(prz, start, count); 279 } 280 281 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz, 282 const void __user *s, unsigned int start, unsigned int count) 283 { 284 struct persistent_ram_buffer *buffer = prz->buffer; 285 int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ? 286 -EFAULT : 0; 287 persistent_ram_update_ecc(prz, start, count); 288 return ret; 289 } 290 291 void persistent_ram_save_old(struct persistent_ram_zone *prz) 292 { 293 struct persistent_ram_buffer *buffer = prz->buffer; 294 size_t size = buffer_size(prz); 295 size_t start = buffer_start(prz); 296 297 if (!size) 298 return; 299 300 /* 301 * If the existing buffer is differently sized, free it so a new 302 * one is allocated. This can happen when persistent_ram_save_old() 303 * is called early in boot and later for a timer-triggered 304 * survivable crash when the crash dumps don't match in size 305 * (which would be extremely unlikely given kmsg buffers usually 306 * exceed prz buffer sizes). 307 */ 308 if (prz->old_log && prz->old_log_size != size) 309 persistent_ram_free_old(prz); 310 311 if (!prz->old_log) { 312 persistent_ram_ecc_old(prz); 313 prz->old_log = kvzalloc(size, GFP_KERNEL); 314 } 315 if (!prz->old_log) { 316 pr_err("failed to allocate buffer\n"); 317 return; 318 } 319 320 prz->old_log_size = size; 321 memcpy_fromio(prz->old_log, &buffer->data[start], size - start); 322 memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start); 323 } 324 325 int notrace persistent_ram_write(struct persistent_ram_zone *prz, 326 const void *s, unsigned int count) 327 { 328 int rem; 329 int c = count; 330 size_t start; 331 332 if (unlikely(c > prz->buffer_size)) { 333 s += c - prz->buffer_size; 334 c = prz->buffer_size; 335 } 336 337 buffer_size_add(prz, c); 338 339 start = buffer_start_add(prz, c); 340 341 rem = prz->buffer_size - start; 342 if (unlikely(rem < c)) { 343 persistent_ram_update(prz, s, start, rem); 344 s += rem; 345 c -= rem; 346 start = 0; 347 } 348 persistent_ram_update(prz, s, start, c); 349 350 persistent_ram_update_header_ecc(prz); 351 352 return count; 353 } 354 355 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz, 356 const void __user *s, unsigned int count) 357 { 358 int rem, ret = 0, c = count; 359 size_t start; 360 361 if (unlikely(c > prz->buffer_size)) { 362 s += c - prz->buffer_size; 363 c = prz->buffer_size; 364 } 365 366 buffer_size_add(prz, c); 367 368 start = buffer_start_add(prz, c); 369 370 rem = prz->buffer_size - start; 371 if (unlikely(rem < c)) { 372 ret = persistent_ram_update_user(prz, s, start, rem); 373 s += rem; 374 c -= rem; 375 start = 0; 376 } 377 if (likely(!ret)) 378 ret = persistent_ram_update_user(prz, s, start, c); 379 380 persistent_ram_update_header_ecc(prz); 381 382 return unlikely(ret) ? ret : count; 383 } 384 385 size_t persistent_ram_old_size(struct persistent_ram_zone *prz) 386 { 387 return prz->old_log_size; 388 } 389 390 void *persistent_ram_old(struct persistent_ram_zone *prz) 391 { 392 return prz->old_log; 393 } 394 395 void persistent_ram_free_old(struct persistent_ram_zone *prz) 396 { 397 kvfree(prz->old_log); 398 prz->old_log = NULL; 399 prz->old_log_size = 0; 400 } 401 402 void persistent_ram_zap(struct persistent_ram_zone *prz) 403 { 404 atomic_set(&prz->buffer->start, 0); 405 atomic_set(&prz->buffer->size, 0); 406 persistent_ram_update_header_ecc(prz); 407 } 408 409 #define MEM_TYPE_WCOMBINE 0 410 #define MEM_TYPE_NONCACHED 1 411 #define MEM_TYPE_NORMAL 2 412 413 static void *persistent_ram_vmap(phys_addr_t start, size_t size, 414 unsigned int memtype) 415 { 416 struct page **pages; 417 phys_addr_t page_start; 418 unsigned int page_count; 419 pgprot_t prot; 420 unsigned int i; 421 void *vaddr; 422 423 page_start = start - offset_in_page(start); 424 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); 425 426 switch (memtype) { 427 case MEM_TYPE_NORMAL: 428 prot = PAGE_KERNEL; 429 break; 430 case MEM_TYPE_NONCACHED: 431 prot = pgprot_noncached(PAGE_KERNEL); 432 break; 433 case MEM_TYPE_WCOMBINE: 434 prot = pgprot_writecombine(PAGE_KERNEL); 435 break; 436 default: 437 pr_err("invalid mem_type=%d\n", memtype); 438 return NULL; 439 } 440 441 pages = kmalloc_objs(struct page *, page_count); 442 if (!pages) { 443 pr_err("%s: Failed to allocate array for %u pages\n", 444 __func__, page_count); 445 return NULL; 446 } 447 448 for (i = 0; i < page_count; i++) { 449 phys_addr_t addr = page_start + i * PAGE_SIZE; 450 pages[i] = pfn_to_page(addr >> PAGE_SHIFT); 451 } 452 /* 453 * VM_IOREMAP used here to bypass this region during vread() 454 * and kmap_atomic() (i.e. kcore) to avoid __va() failures. 455 */ 456 vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); 457 kfree(pages); 458 459 /* 460 * vmap() may fail and return NULL. Do not add the offset in this 461 * case, otherwise a NULL mapping would appear successful. 462 */ 463 if (!vaddr) 464 return NULL; 465 466 /* 467 * Since vmap() uses page granularity, we must add the offset 468 * into the page here, to get the byte granularity address 469 * into the mapping to represent the actual "start" location. 470 */ 471 return vaddr + offset_in_page(start); 472 } 473 474 static void *persistent_ram_iomap(phys_addr_t start, size_t size, 475 unsigned int memtype, char *label) 476 { 477 void *va; 478 479 if (!request_mem_region(start, size, label ?: "ramoops")) { 480 pr_err("request mem region (%s 0x%llx@0x%llx) failed\n", 481 label ?: "ramoops", 482 (unsigned long long)size, (unsigned long long)start); 483 return NULL; 484 } 485 486 if (memtype) 487 va = ioremap(start, size); 488 else 489 va = ioremap_wc(start, size); 490 491 /* 492 * Since request_mem_region() and ioremap() are byte-granularity 493 * there is no need handle anything special like we do when the 494 * vmap() case in persistent_ram_vmap() above. 495 */ 496 return va; 497 } 498 499 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, 500 struct persistent_ram_zone *prz, int memtype) 501 { 502 prz->paddr = start; 503 prz->size = size; 504 505 if (pfn_valid(start >> PAGE_SHIFT)) 506 prz->vaddr = persistent_ram_vmap(start, size, memtype); 507 else 508 prz->vaddr = persistent_ram_iomap(start, size, memtype, 509 prz->label); 510 511 if (!prz->vaddr) { 512 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, 513 (unsigned long long)size, (unsigned long long)start); 514 return -ENOMEM; 515 } 516 517 prz->buffer = prz->vaddr; 518 prz->buffer_size = size - sizeof(struct persistent_ram_buffer); 519 520 return 0; 521 } 522 523 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig, 524 struct persistent_ram_ecc_info *ecc_info) 525 { 526 int ret; 527 bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD); 528 529 ret = persistent_ram_init_ecc(prz, ecc_info); 530 if (ret) { 531 pr_warn("ECC failed %s\n", prz->label); 532 return ret; 533 } 534 535 sig ^= PERSISTENT_RAM_SIG; 536 537 if (prz->buffer->sig == sig) { 538 if (buffer_size(prz) == 0 && buffer_start(prz) == 0) { 539 pr_debug("found existing empty buffer\n"); 540 return 0; 541 } 542 543 if (buffer_size(prz) > prz->buffer_size || 544 buffer_start(prz) > buffer_size(prz)) { 545 pr_info("found existing invalid buffer, size %zu, start %zu\n", 546 buffer_size(prz), buffer_start(prz)); 547 zap = true; 548 } else { 549 pr_debug("found existing buffer, size %zu, start %zu\n", 550 buffer_size(prz), buffer_start(prz)); 551 persistent_ram_save_old(prz); 552 } 553 } else { 554 pr_debug("no valid data in buffer (sig = 0x%08x)\n", 555 prz->buffer->sig); 556 prz->buffer->sig = sig; 557 zap = true; 558 } 559 560 /* Reset missing, invalid, or single-use memory area. */ 561 if (zap) 562 persistent_ram_zap(prz); 563 564 return 0; 565 } 566 567 void persistent_ram_free(struct persistent_ram_zone **_prz) 568 { 569 struct persistent_ram_zone *prz; 570 571 if (!_prz) 572 return; 573 574 prz = *_prz; 575 if (!prz) 576 return; 577 578 if (prz->vaddr) { 579 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { 580 /* We must vunmap() at page-granularity. */ 581 vunmap(prz->vaddr - offset_in_page(prz->paddr)); 582 } else { 583 iounmap(prz->vaddr); 584 release_mem_region(prz->paddr, prz->size); 585 } 586 prz->vaddr = NULL; 587 } 588 if (prz->rs_decoder) { 589 free_rs(prz->rs_decoder); 590 prz->rs_decoder = NULL; 591 } 592 kfree(prz->ecc_info.par); 593 prz->ecc_info.par = NULL; 594 595 persistent_ram_free_old(prz); 596 kfree(prz->label); 597 kfree(prz); 598 *_prz = NULL; 599 } 600 601 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, 602 u32 sig, struct persistent_ram_ecc_info *ecc_info, 603 unsigned int memtype, u32 flags, char *label) 604 { 605 struct persistent_ram_zone *prz; 606 int ret = -ENOMEM; 607 608 prz = kzalloc_obj(struct persistent_ram_zone); 609 if (!prz) { 610 pr_err("failed to allocate persistent ram zone\n"); 611 goto err; 612 } 613 614 /* Initialize general buffer state. */ 615 raw_spin_lock_init(&prz->buffer_lock); 616 prz->flags = flags; 617 prz->label = kstrdup(label, GFP_KERNEL); 618 if (!prz->label) 619 goto err; 620 621 ret = persistent_ram_buffer_map(start, size, prz, memtype); 622 if (ret) 623 goto err; 624 625 ret = persistent_ram_post_init(prz, sig, ecc_info); 626 if (ret) 627 goto err; 628 629 pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n", 630 prz->label, prz->size, (unsigned long long)prz->paddr, 631 sizeof(*prz->buffer), prz->buffer_size, 632 prz->size - sizeof(*prz->buffer) - prz->buffer_size, 633 prz->ecc_info.ecc_size, prz->ecc_info.block_size); 634 635 return prz; 636 err: 637 persistent_ram_free(&prz); 638 return ERR_PTR(ret); 639 } 640