1 /* 2 * Memory subsystem support 3 * 4 * Written by Matt Tolentino <matthew.e.tolentino@intel.com> 5 * Dave Hansen <haveblue@us.ibm.com> 6 * 7 * This file provides the necessary infrastructure to represent 8 * a SPARSEMEM-memory-model system's physical memory in /sysfs. 9 * All arch-independent code that assumes MEMORY_HOTPLUG requires 10 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/topology.h> 16 #include <linux/capability.h> 17 #include <linux/device.h> 18 #include <linux/memory.h> 19 #include <linux/kobject.h> 20 #include <linux/memory_hotplug.h> 21 #include <linux/mm.h> 22 #include <linux/mutex.h> 23 #include <linux/stat.h> 24 #include <linux/slab.h> 25 26 #include <linux/atomic.h> 27 #include <asm/uaccess.h> 28 29 static DEFINE_MUTEX(mem_sysfs_mutex); 30 31 #define MEMORY_CLASS_NAME "memory" 32 33 static int sections_per_block; 34 35 static inline int base_memory_block_id(int section_nr) 36 { 37 return section_nr / sections_per_block; 38 } 39 40 static int memory_subsys_online(struct device *dev); 41 static int memory_subsys_offline(struct device *dev); 42 43 static struct bus_type memory_subsys = { 44 .name = MEMORY_CLASS_NAME, 45 .dev_name = MEMORY_CLASS_NAME, 46 .online = memory_subsys_online, 47 .offline = memory_subsys_offline, 48 }; 49 50 static BLOCKING_NOTIFIER_HEAD(memory_chain); 51 52 int register_memory_notifier(struct notifier_block *nb) 53 { 54 return blocking_notifier_chain_register(&memory_chain, nb); 55 } 56 EXPORT_SYMBOL(register_memory_notifier); 57 58 void unregister_memory_notifier(struct notifier_block *nb) 59 { 60 blocking_notifier_chain_unregister(&memory_chain, nb); 61 } 62 EXPORT_SYMBOL(unregister_memory_notifier); 63 64 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain); 65 66 int register_memory_isolate_notifier(struct notifier_block *nb) 67 { 68 return atomic_notifier_chain_register(&memory_isolate_chain, nb); 69 } 70 EXPORT_SYMBOL(register_memory_isolate_notifier); 71 72 void unregister_memory_isolate_notifier(struct notifier_block *nb) 73 { 74 atomic_notifier_chain_unregister(&memory_isolate_chain, nb); 75 } 76 EXPORT_SYMBOL(unregister_memory_isolate_notifier); 77 78 static void memory_block_release(struct device *dev) 79 { 80 struct memory_block *mem = container_of(dev, struct memory_block, dev); 81 82 kfree(mem); 83 } 84 85 unsigned long __weak memory_block_size_bytes(void) 86 { 87 return MIN_MEMORY_BLOCK_SIZE; 88 } 89 90 static unsigned long get_memory_block_size(void) 91 { 92 unsigned long block_sz; 93 94 block_sz = memory_block_size_bytes(); 95 96 /* Validate blk_sz is a power of 2 and not less than section size */ 97 if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) { 98 WARN_ON(1); 99 block_sz = MIN_MEMORY_BLOCK_SIZE; 100 } 101 102 return block_sz; 103 } 104 105 /* 106 * use this as the physical section index that this memsection 107 * uses. 108 */ 109 110 static ssize_t show_mem_start_phys_index(struct device *dev, 111 struct device_attribute *attr, char *buf) 112 { 113 struct memory_block *mem = 114 container_of(dev, struct memory_block, dev); 115 unsigned long phys_index; 116 117 phys_index = mem->start_section_nr / sections_per_block; 118 return sprintf(buf, "%08lx\n", phys_index); 119 } 120 121 static ssize_t show_mem_end_phys_index(struct device *dev, 122 struct device_attribute *attr, char *buf) 123 { 124 struct memory_block *mem = 125 container_of(dev, struct memory_block, dev); 126 unsigned long phys_index; 127 128 phys_index = mem->end_section_nr / sections_per_block; 129 return sprintf(buf, "%08lx\n", phys_index); 130 } 131 132 /* 133 * Show whether the section of memory is likely to be hot-removable 134 */ 135 static ssize_t show_mem_removable(struct device *dev, 136 struct device_attribute *attr, char *buf) 137 { 138 unsigned long i, pfn; 139 int ret = 1; 140 struct memory_block *mem = 141 container_of(dev, struct memory_block, dev); 142 143 for (i = 0; i < sections_per_block; i++) { 144 pfn = section_nr_to_pfn(mem->start_section_nr + i); 145 ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION); 146 } 147 148 return sprintf(buf, "%d\n", ret); 149 } 150 151 /* 152 * online, offline, going offline, etc. 153 */ 154 static ssize_t show_mem_state(struct device *dev, 155 struct device_attribute *attr, char *buf) 156 { 157 struct memory_block *mem = 158 container_of(dev, struct memory_block, dev); 159 ssize_t len = 0; 160 161 /* 162 * We can probably put these states in a nice little array 163 * so that they're not open-coded 164 */ 165 switch (mem->state) { 166 case MEM_ONLINE: 167 len = sprintf(buf, "online\n"); 168 break; 169 case MEM_OFFLINE: 170 len = sprintf(buf, "offline\n"); 171 break; 172 case MEM_GOING_OFFLINE: 173 len = sprintf(buf, "going-offline\n"); 174 break; 175 default: 176 len = sprintf(buf, "ERROR-UNKNOWN-%ld\n", 177 mem->state); 178 WARN_ON(1); 179 break; 180 } 181 182 return len; 183 } 184 185 int memory_notify(unsigned long val, void *v) 186 { 187 return blocking_notifier_call_chain(&memory_chain, val, v); 188 } 189 190 int memory_isolate_notify(unsigned long val, void *v) 191 { 192 return atomic_notifier_call_chain(&memory_isolate_chain, val, v); 193 } 194 195 /* 196 * The probe routines leave the pages reserved, just as the bootmem code does. 197 * Make sure they're still that way. 198 */ 199 static bool pages_correctly_reserved(unsigned long start_pfn) 200 { 201 int i, j; 202 struct page *page; 203 unsigned long pfn = start_pfn; 204 205 /* 206 * memmap between sections is not contiguous except with 207 * SPARSEMEM_VMEMMAP. We lookup the page once per section 208 * and assume memmap is contiguous within each section 209 */ 210 for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) { 211 if (WARN_ON_ONCE(!pfn_valid(pfn))) 212 return false; 213 page = pfn_to_page(pfn); 214 215 for (j = 0; j < PAGES_PER_SECTION; j++) { 216 if (PageReserved(page + j)) 217 continue; 218 219 printk(KERN_WARNING "section number %ld page number %d " 220 "not reserved, was it already online?\n", 221 pfn_to_section_nr(pfn), j); 222 223 return false; 224 } 225 } 226 227 return true; 228 } 229 230 /* 231 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is 232 * OK to have direct references to sparsemem variables in here. 233 */ 234 static int 235 memory_block_action(unsigned long phys_index, unsigned long action, int online_type) 236 { 237 unsigned long start_pfn; 238 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; 239 struct page *first_page; 240 int ret; 241 242 first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT); 243 start_pfn = page_to_pfn(first_page); 244 245 switch (action) { 246 case MEM_ONLINE: 247 if (!pages_correctly_reserved(start_pfn)) 248 return -EBUSY; 249 250 ret = online_pages(start_pfn, nr_pages, online_type); 251 break; 252 case MEM_OFFLINE: 253 ret = offline_pages(start_pfn, nr_pages); 254 break; 255 default: 256 WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " 257 "%ld\n", __func__, phys_index, action, action); 258 ret = -EINVAL; 259 } 260 261 return ret; 262 } 263 264 static int __memory_block_change_state(struct memory_block *mem, 265 unsigned long to_state, unsigned long from_state_req, 266 int online_type) 267 { 268 int ret = 0; 269 270 if (mem->state != from_state_req) 271 return -EINVAL; 272 273 if (to_state == MEM_OFFLINE) 274 mem->state = MEM_GOING_OFFLINE; 275 276 ret = memory_block_action(mem->start_section_nr, to_state, online_type); 277 mem->state = ret ? from_state_req : to_state; 278 return ret; 279 } 280 281 static int memory_subsys_online(struct device *dev) 282 { 283 struct memory_block *mem = container_of(dev, struct memory_block, dev); 284 int ret; 285 286 mutex_lock(&mem->state_mutex); 287 288 ret = mem->state == MEM_ONLINE ? 0 : 289 __memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE, 290 ONLINE_KEEP); 291 292 mutex_unlock(&mem->state_mutex); 293 return ret; 294 } 295 296 static int memory_subsys_offline(struct device *dev) 297 { 298 struct memory_block *mem = container_of(dev, struct memory_block, dev); 299 int ret; 300 301 mutex_lock(&mem->state_mutex); 302 303 ret = mem->state == MEM_OFFLINE ? 0 : 304 __memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE, -1); 305 306 mutex_unlock(&mem->state_mutex); 307 return ret; 308 } 309 310 static int __memory_block_change_state_uevent(struct memory_block *mem, 311 unsigned long to_state, unsigned long from_state_req, 312 int online_type) 313 { 314 int ret = __memory_block_change_state(mem, to_state, from_state_req, 315 online_type); 316 if (!ret) { 317 switch (mem->state) { 318 case MEM_OFFLINE: 319 kobject_uevent(&mem->dev.kobj, KOBJ_OFFLINE); 320 break; 321 case MEM_ONLINE: 322 kobject_uevent(&mem->dev.kobj, KOBJ_ONLINE); 323 break; 324 default: 325 break; 326 } 327 } 328 return ret; 329 } 330 331 static int memory_block_change_state(struct memory_block *mem, 332 unsigned long to_state, unsigned long from_state_req, 333 int online_type) 334 { 335 int ret; 336 337 mutex_lock(&mem->state_mutex); 338 ret = __memory_block_change_state_uevent(mem, to_state, from_state_req, 339 online_type); 340 mutex_unlock(&mem->state_mutex); 341 342 return ret; 343 } 344 static ssize_t 345 store_mem_state(struct device *dev, 346 struct device_attribute *attr, const char *buf, size_t count) 347 { 348 struct memory_block *mem; 349 bool offline; 350 int ret = -EINVAL; 351 352 mem = container_of(dev, struct memory_block, dev); 353 354 lock_device_hotplug(); 355 356 if (!strncmp(buf, "online_kernel", min_t(int, count, 13))) { 357 offline = false; 358 ret = memory_block_change_state(mem, MEM_ONLINE, 359 MEM_OFFLINE, ONLINE_KERNEL); 360 } else if (!strncmp(buf, "online_movable", min_t(int, count, 14))) { 361 offline = false; 362 ret = memory_block_change_state(mem, MEM_ONLINE, 363 MEM_OFFLINE, ONLINE_MOVABLE); 364 } else if (!strncmp(buf, "online", min_t(int, count, 6))) { 365 offline = false; 366 ret = memory_block_change_state(mem, MEM_ONLINE, 367 MEM_OFFLINE, ONLINE_KEEP); 368 } else if(!strncmp(buf, "offline", min_t(int, count, 7))) { 369 offline = true; 370 ret = memory_block_change_state(mem, MEM_OFFLINE, 371 MEM_ONLINE, -1); 372 } 373 if (!ret) 374 dev->offline = offline; 375 376 unlock_device_hotplug(); 377 378 if (ret) 379 return ret; 380 return count; 381 } 382 383 /* 384 * phys_device is a bad name for this. What I really want 385 * is a way to differentiate between memory ranges that 386 * are part of physical devices that constitute 387 * a complete removable unit or fru. 388 * i.e. do these ranges belong to the same physical device, 389 * s.t. if I offline all of these sections I can then 390 * remove the physical device? 391 */ 392 static ssize_t show_phys_device(struct device *dev, 393 struct device_attribute *attr, char *buf) 394 { 395 struct memory_block *mem = 396 container_of(dev, struct memory_block, dev); 397 return sprintf(buf, "%d\n", mem->phys_device); 398 } 399 400 static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL); 401 static DEVICE_ATTR(end_phys_index, 0444, show_mem_end_phys_index, NULL); 402 static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state); 403 static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL); 404 static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL); 405 406 /* 407 * Block size attribute stuff 408 */ 409 static ssize_t 410 print_block_size(struct device *dev, struct device_attribute *attr, 411 char *buf) 412 { 413 return sprintf(buf, "%lx\n", get_memory_block_size()); 414 } 415 416 static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL); 417 418 /* 419 * Some architectures will have custom drivers to do this, and 420 * will not need to do it from userspace. The fake hot-add code 421 * as well as ppc64 will do all of their discovery in userspace 422 * and will require this interface. 423 */ 424 #ifdef CONFIG_ARCH_MEMORY_PROBE 425 static ssize_t 426 memory_probe_store(struct device *dev, struct device_attribute *attr, 427 const char *buf, size_t count) 428 { 429 u64 phys_addr; 430 int nid; 431 int i, ret; 432 unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block; 433 434 phys_addr = simple_strtoull(buf, NULL, 0); 435 436 if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) 437 return -EINVAL; 438 439 for (i = 0; i < sections_per_block; i++) { 440 nid = memory_add_physaddr_to_nid(phys_addr); 441 ret = add_memory(nid, phys_addr, 442 PAGES_PER_SECTION << PAGE_SHIFT); 443 if (ret) 444 goto out; 445 446 phys_addr += MIN_MEMORY_BLOCK_SIZE; 447 } 448 449 ret = count; 450 out: 451 return ret; 452 } 453 454 static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store); 455 #endif 456 457 #ifdef CONFIG_MEMORY_FAILURE 458 /* 459 * Support for offlining pages of memory 460 */ 461 462 /* Soft offline a page */ 463 static ssize_t 464 store_soft_offline_page(struct device *dev, 465 struct device_attribute *attr, 466 const char *buf, size_t count) 467 { 468 int ret; 469 u64 pfn; 470 if (!capable(CAP_SYS_ADMIN)) 471 return -EPERM; 472 if (strict_strtoull(buf, 0, &pfn) < 0) 473 return -EINVAL; 474 pfn >>= PAGE_SHIFT; 475 if (!pfn_valid(pfn)) 476 return -ENXIO; 477 ret = soft_offline_page(pfn_to_page(pfn), 0); 478 return ret == 0 ? count : ret; 479 } 480 481 /* Forcibly offline a page, including killing processes. */ 482 static ssize_t 483 store_hard_offline_page(struct device *dev, 484 struct device_attribute *attr, 485 const char *buf, size_t count) 486 { 487 int ret; 488 u64 pfn; 489 if (!capable(CAP_SYS_ADMIN)) 490 return -EPERM; 491 if (strict_strtoull(buf, 0, &pfn) < 0) 492 return -EINVAL; 493 pfn >>= PAGE_SHIFT; 494 ret = memory_failure(pfn, 0, 0); 495 return ret ? ret : count; 496 } 497 498 static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page); 499 static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page); 500 #endif 501 502 /* 503 * Note that phys_device is optional. It is here to allow for 504 * differentiation between which *physical* devices each 505 * section belongs to... 506 */ 507 int __weak arch_get_memory_phys_device(unsigned long start_pfn) 508 { 509 return 0; 510 } 511 512 /* 513 * A reference for the returned object is held and the reference for the 514 * hinted object is released. 515 */ 516 struct memory_block *find_memory_block_hinted(struct mem_section *section, 517 struct memory_block *hint) 518 { 519 int block_id = base_memory_block_id(__section_nr(section)); 520 struct device *hintdev = hint ? &hint->dev : NULL; 521 struct device *dev; 522 523 dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev); 524 if (hint) 525 put_device(&hint->dev); 526 if (!dev) 527 return NULL; 528 return container_of(dev, struct memory_block, dev); 529 } 530 531 /* 532 * For now, we have a linear search to go find the appropriate 533 * memory_block corresponding to a particular phys_index. If 534 * this gets to be a real problem, we can always use a radix 535 * tree or something here. 536 * 537 * This could be made generic for all device subsystems. 538 */ 539 struct memory_block *find_memory_block(struct mem_section *section) 540 { 541 return find_memory_block_hinted(section, NULL); 542 } 543 544 static struct attribute *memory_memblk_attrs[] = { 545 &dev_attr_phys_index.attr, 546 &dev_attr_end_phys_index.attr, 547 &dev_attr_state.attr, 548 &dev_attr_phys_device.attr, 549 &dev_attr_removable.attr, 550 NULL 551 }; 552 553 static struct attribute_group memory_memblk_attr_group = { 554 .attrs = memory_memblk_attrs, 555 }; 556 557 static const struct attribute_group *memory_memblk_attr_groups[] = { 558 &memory_memblk_attr_group, 559 NULL, 560 }; 561 562 /* 563 * register_memory - Setup a sysfs device for a memory block 564 */ 565 static 566 int register_memory(struct memory_block *memory) 567 { 568 int error; 569 570 memory->dev.bus = &memory_subsys; 571 memory->dev.id = memory->start_section_nr / sections_per_block; 572 memory->dev.release = memory_block_release; 573 memory->dev.groups = memory_memblk_attr_groups; 574 memory->dev.offline = memory->state == MEM_OFFLINE; 575 576 error = device_register(&memory->dev); 577 return error; 578 } 579 580 static int init_memory_block(struct memory_block **memory, 581 struct mem_section *section, unsigned long state) 582 { 583 struct memory_block *mem; 584 unsigned long start_pfn; 585 int scn_nr; 586 int ret = 0; 587 588 mem = kzalloc(sizeof(*mem), GFP_KERNEL); 589 if (!mem) 590 return -ENOMEM; 591 592 scn_nr = __section_nr(section); 593 mem->start_section_nr = 594 base_memory_block_id(scn_nr) * sections_per_block; 595 mem->end_section_nr = mem->start_section_nr + sections_per_block - 1; 596 mem->state = state; 597 mem->section_count++; 598 mutex_init(&mem->state_mutex); 599 start_pfn = section_nr_to_pfn(mem->start_section_nr); 600 mem->phys_device = arch_get_memory_phys_device(start_pfn); 601 602 ret = register_memory(mem); 603 604 *memory = mem; 605 return ret; 606 } 607 608 static int add_memory_section(int nid, struct mem_section *section, 609 struct memory_block **mem_p, 610 unsigned long state, enum mem_add_context context) 611 { 612 struct memory_block *mem = NULL; 613 int scn_nr = __section_nr(section); 614 int ret = 0; 615 616 mutex_lock(&mem_sysfs_mutex); 617 618 if (context == BOOT) { 619 /* same memory block ? */ 620 if (mem_p && *mem_p) 621 if (scn_nr >= (*mem_p)->start_section_nr && 622 scn_nr <= (*mem_p)->end_section_nr) { 623 mem = *mem_p; 624 kobject_get(&mem->dev.kobj); 625 } 626 } else 627 mem = find_memory_block(section); 628 629 if (mem) { 630 mem->section_count++; 631 kobject_put(&mem->dev.kobj); 632 } else { 633 ret = init_memory_block(&mem, section, state); 634 /* store memory_block pointer for next loop */ 635 if (!ret && context == BOOT) 636 if (mem_p) 637 *mem_p = mem; 638 } 639 640 if (!ret) { 641 if (context == HOTPLUG && 642 mem->section_count == sections_per_block) 643 ret = register_mem_sect_under_node(mem, nid); 644 } 645 646 mutex_unlock(&mem_sysfs_mutex); 647 return ret; 648 } 649 650 /* 651 * need an interface for the VM to add new memory regions, 652 * but without onlining it. 653 */ 654 int register_new_memory(int nid, struct mem_section *section) 655 { 656 return add_memory_section(nid, section, NULL, MEM_OFFLINE, HOTPLUG); 657 } 658 659 #ifdef CONFIG_MEMORY_HOTREMOVE 660 static void 661 unregister_memory(struct memory_block *memory) 662 { 663 BUG_ON(memory->dev.bus != &memory_subsys); 664 665 /* drop the ref. we got in remove_memory_block() */ 666 kobject_put(&memory->dev.kobj); 667 device_unregister(&memory->dev); 668 } 669 670 static int remove_memory_block(unsigned long node_id, 671 struct mem_section *section, int phys_device) 672 { 673 struct memory_block *mem; 674 675 mutex_lock(&mem_sysfs_mutex); 676 mem = find_memory_block(section); 677 unregister_mem_sect_under_nodes(mem, __section_nr(section)); 678 679 mem->section_count--; 680 if (mem->section_count == 0) 681 unregister_memory(mem); 682 else 683 kobject_put(&mem->dev.kobj); 684 685 mutex_unlock(&mem_sysfs_mutex); 686 return 0; 687 } 688 689 int unregister_memory_section(struct mem_section *section) 690 { 691 if (!present_section(section)) 692 return -EINVAL; 693 694 return remove_memory_block(0, section, 0); 695 } 696 #endif /* CONFIG_MEMORY_HOTREMOVE */ 697 698 /* return true if the memory block is offlined, otherwise, return false */ 699 bool is_memblock_offlined(struct memory_block *mem) 700 { 701 return mem->state == MEM_OFFLINE; 702 } 703 704 static struct attribute *memory_root_attrs[] = { 705 #ifdef CONFIG_ARCH_MEMORY_PROBE 706 &dev_attr_probe.attr, 707 #endif 708 709 #ifdef CONFIG_MEMORY_FAILURE 710 &dev_attr_soft_offline_page.attr, 711 &dev_attr_hard_offline_page.attr, 712 #endif 713 714 &dev_attr_block_size_bytes.attr, 715 NULL 716 }; 717 718 static struct attribute_group memory_root_attr_group = { 719 .attrs = memory_root_attrs, 720 }; 721 722 static const struct attribute_group *memory_root_attr_groups[] = { 723 &memory_root_attr_group, 724 NULL, 725 }; 726 727 /* 728 * Initialize the sysfs support for memory devices... 729 */ 730 int __init memory_dev_init(void) 731 { 732 unsigned int i; 733 int ret; 734 int err; 735 unsigned long block_sz; 736 struct memory_block *mem = NULL; 737 738 ret = subsys_system_register(&memory_subsys, memory_root_attr_groups); 739 if (ret) 740 goto out; 741 742 block_sz = get_memory_block_size(); 743 sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; 744 745 /* 746 * Create entries for memory sections that were found 747 * during boot and have been initialized 748 */ 749 for (i = 0; i < NR_MEM_SECTIONS; i++) { 750 if (!present_section_nr(i)) 751 continue; 752 /* don't need to reuse memory_block if only one per block */ 753 err = add_memory_section(0, __nr_to_section(i), 754 (sections_per_block == 1) ? NULL : &mem, 755 MEM_ONLINE, 756 BOOT); 757 if (!ret) 758 ret = err; 759 } 760 761 out: 762 if (ret) 763 printk(KERN_ERR "%s() failed: %d\n", __func__, ret); 764 return ret; 765 } 766