1 /* 2 * Copyright (C) 2012 Google, Inc. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 */ 14 15 #include <linux/device.h> 16 #include <linux/err.h> 17 #include <linux/errno.h> 18 #include <linux/kernel.h> 19 #include <linux/init.h> 20 #include <linux/io.h> 21 #include <linux/list.h> 22 #include <linux/memblock.h> 23 #include <linux/rslib.h> 24 #include <linux/slab.h> 25 #include <linux/vmalloc.h> 26 #include <linux/pstore_ram.h> 27 #include <asm/page.h> 28 29 struct persistent_ram_buffer { 30 uint32_t sig; 31 atomic_t start; 32 atomic_t size; 33 uint8_t data[0]; 34 }; 35 36 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ 37 38 static __initdata LIST_HEAD(persistent_ram_list); 39 40 static inline size_t buffer_size(struct persistent_ram_zone *prz) 41 { 42 return atomic_read(&prz->buffer->size); 43 } 44 45 static inline size_t buffer_start(struct persistent_ram_zone *prz) 46 { 47 return atomic_read(&prz->buffer->start); 48 } 49 50 /* increase and wrap the start pointer, returning the old value */ 51 static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) 52 { 53 int old; 54 int new; 55 56 do { 57 old = atomic_read(&prz->buffer->start); 58 new = old + a; 59 while (unlikely(new > prz->buffer_size)) 60 new -= prz->buffer_size; 61 } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old); 62 63 return old; 64 } 65 66 /* increase the size counter until it hits the max size */ 67 static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a) 68 { 69 size_t old; 70 size_t new; 71 72 if (atomic_read(&prz->buffer->size) == prz->buffer_size) 73 return; 74 75 do { 76 old = atomic_read(&prz->buffer->size); 77 new = old + a; 78 if (new > prz->buffer_size) 79 new = prz->buffer_size; 80 } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); 81 } 82 83 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, 84 uint8_t *data, size_t len, uint8_t *ecc) 85 { 86 int i; 87 uint16_t par[prz->ecc_size]; 88 89 /* Initialize the parity buffer */ 90 memset(par, 0, sizeof(par)); 91 encode_rs8(prz->rs_decoder, data, len, par, 0); 92 for (i = 0; i < prz->ecc_size; i++) 93 ecc[i] = par[i]; 94 } 95 96 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, 97 void *data, size_t len, uint8_t *ecc) 98 { 99 int i; 100 uint16_t par[prz->ecc_size]; 101 102 for (i = 0; i < prz->ecc_size; i++) 103 par[i] = ecc[i]; 104 return decode_rs8(prz->rs_decoder, data, par, len, 105 NULL, 0, NULL, 0, NULL); 106 } 107 108 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, 109 unsigned int start, unsigned int count) 110 { 111 struct persistent_ram_buffer *buffer = prz->buffer; 112 uint8_t *buffer_end = buffer->data + prz->buffer_size; 113 uint8_t *block; 114 uint8_t *par; 115 int ecc_block_size = prz->ecc_block_size; 116 int ecc_size = prz->ecc_size; 117 int size = prz->ecc_block_size; 118 119 if (!prz->ecc) 120 return; 121 122 block = buffer->data + (start & ~(ecc_block_size - 1)); 123 par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size; 124 125 do { 126 if (block + ecc_block_size > buffer_end) 127 size = buffer_end - block; 128 persistent_ram_encode_rs8(prz, block, size, par); 129 block += ecc_block_size; 130 par += ecc_size; 131 } while (block < buffer->data + start + count); 132 } 133 134 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) 135 { 136 struct persistent_ram_buffer *buffer = prz->buffer; 137 138 if (!prz->ecc) 139 return; 140 141 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), 142 prz->par_header); 143 } 144 145 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) 146 { 147 struct persistent_ram_buffer *buffer = prz->buffer; 148 uint8_t *block; 149 uint8_t *par; 150 151 if (!prz->ecc) 152 return; 153 154 block = buffer->data; 155 par = prz->par_buffer; 156 while (block < buffer->data + buffer_size(prz)) { 157 int numerr; 158 int size = prz->ecc_block_size; 159 if (block + size > buffer->data + prz->buffer_size) 160 size = buffer->data + prz->buffer_size - block; 161 numerr = persistent_ram_decode_rs8(prz, block, size, par); 162 if (numerr > 0) { 163 pr_devel("persistent_ram: error in block %p, %d\n", 164 block, numerr); 165 prz->corrected_bytes += numerr; 166 } else if (numerr < 0) { 167 pr_devel("persistent_ram: uncorrectable error in block %p\n", 168 block); 169 prz->bad_blocks++; 170 } 171 block += prz->ecc_block_size; 172 par += prz->ecc_size; 173 } 174 } 175 176 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, 177 size_t buffer_size) 178 { 179 int numerr; 180 struct persistent_ram_buffer *buffer = prz->buffer; 181 int ecc_blocks; 182 183 if (!prz->ecc) 184 return 0; 185 186 prz->ecc_block_size = 128; 187 prz->ecc_size = 16; 188 prz->ecc_symsize = 8; 189 prz->ecc_poly = 0x11d; 190 191 ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size); 192 prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size; 193 194 if (prz->buffer_size > buffer_size) { 195 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n", 196 buffer_size, prz->buffer_size); 197 return -EINVAL; 198 } 199 200 prz->par_buffer = buffer->data + prz->buffer_size; 201 prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size; 202 203 /* 204 * first consecutive root is 0 205 * primitive element to generate roots = 1 206 */ 207 prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1, 208 prz->ecc_size); 209 if (prz->rs_decoder == NULL) { 210 pr_info("persistent_ram: init_rs failed\n"); 211 return -EINVAL; 212 } 213 214 prz->corrected_bytes = 0; 215 prz->bad_blocks = 0; 216 217 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), 218 prz->par_header); 219 if (numerr > 0) { 220 pr_info("persistent_ram: error in header, %d\n", numerr); 221 prz->corrected_bytes += numerr; 222 } else if (numerr < 0) { 223 pr_info("persistent_ram: uncorrectable error in header\n"); 224 prz->bad_blocks++; 225 } 226 227 return 0; 228 } 229 230 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, 231 char *str, size_t len) 232 { 233 ssize_t ret; 234 235 if (prz->corrected_bytes || prz->bad_blocks) 236 ret = snprintf(str, len, "" 237 "\n%d Corrected bytes, %d unrecoverable blocks\n", 238 prz->corrected_bytes, prz->bad_blocks); 239 else 240 ret = snprintf(str, len, "\nNo errors detected\n"); 241 242 return ret; 243 } 244 245 static void notrace persistent_ram_update(struct persistent_ram_zone *prz, 246 const void *s, unsigned int start, unsigned int count) 247 { 248 struct persistent_ram_buffer *buffer = prz->buffer; 249 memcpy(buffer->data + start, s, count); 250 persistent_ram_update_ecc(prz, start, count); 251 } 252 253 void persistent_ram_save_old(struct persistent_ram_zone *prz) 254 { 255 struct persistent_ram_buffer *buffer = prz->buffer; 256 size_t size = buffer_size(prz); 257 size_t start = buffer_start(prz); 258 259 if (!size) 260 return; 261 262 if (!prz->old_log) { 263 persistent_ram_ecc_old(prz); 264 prz->old_log = kmalloc(size, GFP_KERNEL); 265 } 266 if (!prz->old_log) { 267 pr_err("persistent_ram: failed to allocate buffer\n"); 268 return; 269 } 270 271 prz->old_log_size = size; 272 memcpy(prz->old_log, &buffer->data[start], size - start); 273 memcpy(prz->old_log + size - start, &buffer->data[0], start); 274 } 275 276 int notrace persistent_ram_write(struct persistent_ram_zone *prz, 277 const void *s, unsigned int count) 278 { 279 int rem; 280 int c = count; 281 size_t start; 282 283 if (unlikely(c > prz->buffer_size)) { 284 s += c - prz->buffer_size; 285 c = prz->buffer_size; 286 } 287 288 buffer_size_add(prz, c); 289 290 start = buffer_start_add(prz, c); 291 292 rem = prz->buffer_size - start; 293 if (unlikely(rem < c)) { 294 persistent_ram_update(prz, s, start, rem); 295 s += rem; 296 c -= rem; 297 start = 0; 298 } 299 persistent_ram_update(prz, s, start, c); 300 301 persistent_ram_update_header_ecc(prz); 302 303 return count; 304 } 305 306 size_t persistent_ram_old_size(struct persistent_ram_zone *prz) 307 { 308 return prz->old_log_size; 309 } 310 311 void *persistent_ram_old(struct persistent_ram_zone *prz) 312 { 313 return prz->old_log; 314 } 315 316 void persistent_ram_free_old(struct persistent_ram_zone *prz) 317 { 318 kfree(prz->old_log); 319 prz->old_log = NULL; 320 prz->old_log_size = 0; 321 } 322 323 void persistent_ram_zap(struct persistent_ram_zone *prz) 324 { 325 atomic_set(&prz->buffer->start, 0); 326 atomic_set(&prz->buffer->size, 0); 327 persistent_ram_update_header_ecc(prz); 328 } 329 330 static void *persistent_ram_vmap(phys_addr_t start, size_t size) 331 { 332 struct page **pages; 333 phys_addr_t page_start; 334 unsigned int page_count; 335 pgprot_t prot; 336 unsigned int i; 337 void *vaddr; 338 339 page_start = start - offset_in_page(start); 340 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); 341 342 prot = pgprot_noncached(PAGE_KERNEL); 343 344 pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL); 345 if (!pages) { 346 pr_err("%s: Failed to allocate array for %u pages\n", __func__, 347 page_count); 348 return NULL; 349 } 350 351 for (i = 0; i < page_count; i++) { 352 phys_addr_t addr = page_start + i * PAGE_SIZE; 353 pages[i] = pfn_to_page(addr >> PAGE_SHIFT); 354 } 355 vaddr = vmap(pages, page_count, VM_MAP, prot); 356 kfree(pages); 357 358 return vaddr; 359 } 360 361 static void *persistent_ram_iomap(phys_addr_t start, size_t size) 362 { 363 if (!request_mem_region(start, size, "persistent_ram")) { 364 pr_err("request mem region (0x%llx@0x%llx) failed\n", 365 (unsigned long long)size, (unsigned long long)start); 366 return NULL; 367 } 368 369 return ioremap(start, size); 370 } 371 372 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, 373 struct persistent_ram_zone *prz) 374 { 375 prz->paddr = start; 376 prz->size = size; 377 378 if (pfn_valid(start >> PAGE_SHIFT)) 379 prz->vaddr = persistent_ram_vmap(start, size); 380 else 381 prz->vaddr = persistent_ram_iomap(start, size); 382 383 if (!prz->vaddr) { 384 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, 385 (unsigned long long)size, (unsigned long long)start); 386 return -ENOMEM; 387 } 388 389 prz->buffer = prz->vaddr + offset_in_page(start); 390 prz->buffer_size = size - sizeof(struct persistent_ram_buffer); 391 392 return 0; 393 } 394 395 static int __init persistent_ram_post_init(struct persistent_ram_zone *prz, bool ecc) 396 { 397 int ret; 398 399 prz->ecc = ecc; 400 401 ret = persistent_ram_init_ecc(prz, prz->buffer_size); 402 if (ret) 403 return ret; 404 405 if (prz->buffer->sig == PERSISTENT_RAM_SIG) { 406 if (buffer_size(prz) > prz->buffer_size || 407 buffer_start(prz) > buffer_size(prz)) 408 pr_info("persistent_ram: found existing invalid buffer," 409 " size %zu, start %zu\n", 410 buffer_size(prz), buffer_start(prz)); 411 else { 412 pr_info("persistent_ram: found existing buffer," 413 " size %zu, start %zu\n", 414 buffer_size(prz), buffer_start(prz)); 415 persistent_ram_save_old(prz); 416 return 0; 417 } 418 } else { 419 pr_info("persistent_ram: no valid data in buffer" 420 " (sig = 0x%08x)\n", prz->buffer->sig); 421 } 422 423 prz->buffer->sig = PERSISTENT_RAM_SIG; 424 persistent_ram_zap(prz); 425 426 return 0; 427 } 428 429 void persistent_ram_free(struct persistent_ram_zone *prz) 430 { 431 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { 432 vunmap(prz->vaddr); 433 } else { 434 iounmap(prz->vaddr); 435 release_mem_region(prz->paddr, prz->size); 436 } 437 persistent_ram_free_old(prz); 438 kfree(prz); 439 } 440 441 struct persistent_ram_zone * __init persistent_ram_new(phys_addr_t start, 442 size_t size, 443 bool ecc) 444 { 445 struct persistent_ram_zone *prz; 446 int ret = -ENOMEM; 447 448 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); 449 if (!prz) { 450 pr_err("persistent_ram: failed to allocate persistent ram zone\n"); 451 goto err; 452 } 453 454 ret = persistent_ram_buffer_map(start, size, prz); 455 if (ret) 456 goto err; 457 458 persistent_ram_post_init(prz, ecc); 459 460 return prz; 461 err: 462 kfree(prz); 463 return ERR_PTR(ret); 464 } 465 466 #ifndef MODULE 467 static int __init persistent_ram_buffer_init(const char *name, 468 struct persistent_ram_zone *prz) 469 { 470 int i; 471 struct persistent_ram *ram; 472 struct persistent_ram_descriptor *desc; 473 phys_addr_t start; 474 475 list_for_each_entry(ram, &persistent_ram_list, node) { 476 start = ram->start; 477 for (i = 0; i < ram->num_descs; i++) { 478 desc = &ram->descs[i]; 479 if (!strcmp(desc->name, name)) 480 return persistent_ram_buffer_map(start, 481 desc->size, prz); 482 start += desc->size; 483 } 484 } 485 486 return -EINVAL; 487 } 488 489 static __init 490 struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc) 491 { 492 struct persistent_ram_zone *prz; 493 int ret = -ENOMEM; 494 495 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); 496 if (!prz) { 497 pr_err("persistent_ram: failed to allocate persistent ram zone\n"); 498 goto err; 499 } 500 501 ret = persistent_ram_buffer_init(dev_name(dev), prz); 502 if (ret) { 503 pr_err("persistent_ram: failed to initialize buffer\n"); 504 goto err; 505 } 506 507 persistent_ram_post_init(prz, ecc); 508 509 return prz; 510 err: 511 kfree(prz); 512 return ERR_PTR(ret); 513 } 514 515 struct persistent_ram_zone * __init 516 persistent_ram_init_ringbuffer(struct device *dev, bool ecc) 517 { 518 return __persistent_ram_init(dev, ecc); 519 } 520 521 int __init persistent_ram_early_init(struct persistent_ram *ram) 522 { 523 int ret; 524 525 ret = memblock_reserve(ram->start, ram->size); 526 if (ret) { 527 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n", 528 (long)ram->start, (long)(ram->start + ram->size - 1)); 529 return ret; 530 } 531 532 list_add_tail(&ram->node, &persistent_ram_list); 533 534 pr_info("Initialized persistent memory from %08lx-%08lx\n", 535 (long)ram->start, (long)(ram->start + ram->size - 1)); 536 537 return 0; 538 } 539 #endif 540