xref: /linux/drivers/base/node.c (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Basic Node interface support
4  */
5 
6 #include <linux/module.h>
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/memory.h>
10 #include <linux/mempolicy.h>
11 #include <linux/vmstat.h>
12 #include <linux/notifier.h>
13 #include <linux/node.h>
14 #include <linux/hugetlb.h>
15 #include <linux/compaction.h>
16 #include <linux/cpumask.h>
17 #include <linux/topology.h>
18 #include <linux/nodemask.h>
19 #include <linux/cpu.h>
20 #include <linux/device.h>
21 #include <linux/pm_runtime.h>
22 #include <linux/swap.h>
23 #include <linux/slab.h>
24 #include <linux/memblock.h>
25 
26 static const struct bus_type node_subsys = {
27 	.name = "node",
28 	.dev_name = "node",
29 };
30 
31 static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj,
32 				  const struct bin_attribute *attr, char *buf,
33 				  loff_t off, size_t count)
34 {
35 	struct device *dev = kobj_to_dev(kobj);
36 	struct node *node_dev = to_node(dev);
37 	cpumask_var_t mask;
38 	ssize_t n;
39 
40 	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
41 		return 0;
42 
43 	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
44 	n = cpumap_print_bitmask_to_buf(buf, mask, off, count);
45 	free_cpumask_var(mask);
46 
47 	return n;
48 }
49 
50 static const BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES);
51 
52 static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj,
53 				   const struct bin_attribute *attr, char *buf,
54 				   loff_t off, size_t count)
55 {
56 	struct device *dev = kobj_to_dev(kobj);
57 	struct node *node_dev = to_node(dev);
58 	cpumask_var_t mask;
59 	ssize_t n;
60 
61 	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
62 		return 0;
63 
64 	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
65 	n = cpumap_print_list_to_buf(buf, mask, off, count);
66 	free_cpumask_var(mask);
67 
68 	return n;
69 }
70 
71 static const BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES);
72 
73 /**
74  * struct node_access_nodes - Access class device to hold user visible
75  * 			      relationships to other nodes.
76  * @dev:	Device for this memory access class
77  * @list_node:	List element in the node's access list
78  * @access:	The access class rank
79  * @coord:	Heterogeneous memory performance coordinates
80  */
81 struct node_access_nodes {
82 	struct device		dev;
83 	struct list_head	list_node;
84 	unsigned int		access;
85 #ifdef CONFIG_HMEM_REPORTING
86 	struct access_coordinate	coord;
87 #endif
88 };
89 #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
90 
91 static struct attribute *node_init_access_node_attrs[] = {
92 	NULL,
93 };
94 
95 static struct attribute *node_targ_access_node_attrs[] = {
96 	NULL,
97 };
98 
99 static const struct attribute_group initiators = {
100 	.name	= "initiators",
101 	.attrs	= node_init_access_node_attrs,
102 };
103 
104 static const struct attribute_group targets = {
105 	.name	= "targets",
106 	.attrs	= node_targ_access_node_attrs,
107 };
108 
109 static const struct attribute_group *node_access_node_groups[] = {
110 	&initiators,
111 	&targets,
112 	NULL,
113 };
114 
115 #ifdef CONFIG_MEMORY_HOTPLUG
116 static BLOCKING_NOTIFIER_HEAD(node_chain);
117 
118 int register_node_notifier(struct notifier_block *nb)
119 {
120 	return blocking_notifier_chain_register(&node_chain, nb);
121 }
122 EXPORT_SYMBOL(register_node_notifier);
123 
124 void unregister_node_notifier(struct notifier_block *nb)
125 {
126 	blocking_notifier_chain_unregister(&node_chain, nb);
127 }
128 EXPORT_SYMBOL(unregister_node_notifier);
129 
130 int node_notify(unsigned long val, void *v)
131 {
132 	return blocking_notifier_call_chain(&node_chain, val, v);
133 }
134 #endif
135 
136 static void node_remove_accesses(struct node *node)
137 {
138 	struct node_access_nodes *c, *cnext;
139 
140 	list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
141 		list_del(&c->list_node);
142 		device_unregister(&c->dev);
143 	}
144 }
145 
146 static void node_access_release(struct device *dev)
147 {
148 	kfree(to_access_nodes(dev));
149 }
150 
151 static struct node_access_nodes *node_init_node_access(struct node *node,
152 						       enum access_coordinate_class access)
153 {
154 	struct node_access_nodes *access_node;
155 	struct device *dev;
156 
157 	list_for_each_entry(access_node, &node->access_list, list_node)
158 		if (access_node->access == access)
159 			return access_node;
160 
161 	access_node = kzalloc_obj(*access_node);
162 	if (!access_node)
163 		return NULL;
164 
165 	access_node->access = access;
166 	dev = &access_node->dev;
167 	dev->parent = &node->dev;
168 	dev->release = node_access_release;
169 	dev->groups = node_access_node_groups;
170 	if (dev_set_name(dev, "access%u", access)) {
171 		kfree(access_node);
172 		return NULL;
173 	}
174 
175 	if (device_register(dev)) {
176 		put_device(dev);
177 		return NULL;
178 	}
179 
180 	pm_runtime_no_callbacks(dev);
181 	list_add_tail(&access_node->list_node, &node->access_list);
182 	return access_node;
183 }
184 
185 #ifdef CONFIG_HMEM_REPORTING
186 #define ACCESS_ATTR(property)						\
187 static ssize_t property##_show(struct device *dev,			\
188 			   struct device_attribute *attr,		\
189 			   char *buf)					\
190 {									\
191 	return sysfs_emit(buf, "%u\n",					\
192 			  to_access_nodes(dev)->coord.property);	\
193 }									\
194 static DEVICE_ATTR_RO(property)
195 
196 ACCESS_ATTR(read_bandwidth);
197 ACCESS_ATTR(read_latency);
198 ACCESS_ATTR(write_bandwidth);
199 ACCESS_ATTR(write_latency);
200 
201 static struct attribute *access_attrs[] = {
202 	&dev_attr_read_bandwidth.attr,
203 	&dev_attr_read_latency.attr,
204 	&dev_attr_write_bandwidth.attr,
205 	&dev_attr_write_latency.attr,
206 	NULL,
207 };
208 
209 /**
210  * node_set_perf_attrs - Set the performance values for given access class
211  * @nid: Node identifier to be set
212  * @coord: Heterogeneous memory performance coordinates
213  * @access: The access class the for the given attributes
214  */
215 void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord,
216 			 enum access_coordinate_class access)
217 {
218 	struct node_access_nodes *c;
219 	struct node *node;
220 	int i;
221 
222 	if (WARN_ON_ONCE(!node_online(nid)))
223 		return;
224 
225 	node = node_devices[nid];
226 	c = node_init_node_access(node, access);
227 	if (!c)
228 		return;
229 
230 	c->coord = *coord;
231 	for (i = 0; access_attrs[i] != NULL; i++) {
232 		if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
233 					    "initiators")) {
234 			pr_info("failed to add performance attribute to node %d\n",
235 				nid);
236 			break;
237 		}
238 	}
239 
240 	/* When setting CPU access coordinates, update mempolicy */
241 	if (access == ACCESS_COORDINATE_CPU) {
242 		if (mempolicy_set_node_perf(nid, coord)) {
243 			pr_info("failed to set mempolicy attrs for node %d\n",
244 				nid);
245 		}
246 	}
247 }
248 EXPORT_SYMBOL_GPL(node_set_perf_attrs);
249 
250 /**
251  * node_update_perf_attrs - Update the performance values for given access class
252  * @nid: Node identifier to be updated
253  * @coord: Heterogeneous memory performance coordinates
254  * @access: The access class for the given attributes
255  */
256 void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord,
257 			    enum access_coordinate_class access)
258 {
259 	struct node_access_nodes *access_node;
260 	struct node *node;
261 	int i;
262 
263 	if (WARN_ON_ONCE(!node_online(nid)))
264 		return;
265 
266 	node = node_devices[nid];
267 	list_for_each_entry(access_node, &node->access_list, list_node) {
268 		if (access_node->access != access)
269 			continue;
270 
271 		access_node->coord = *coord;
272 		for (i = 0; access_attrs[i]; i++) {
273 			sysfs_notify(&access_node->dev.kobj,
274 				     NULL, access_attrs[i]->name);
275 		}
276 		break;
277 	}
278 
279 	/* When setting CPU access coordinates, update mempolicy */
280 	if (access != ACCESS_COORDINATE_CPU)
281 		return;
282 
283 	if (mempolicy_set_node_perf(nid, coord))
284 		pr_info("failed to set mempolicy attrs for node %d\n", nid);
285 }
286 EXPORT_SYMBOL_GPL(node_update_perf_attrs);
287 
288 /**
289  * struct node_cache_info - Internal tracking for memory node caches
290  * @dev:	Device represeting the cache level
291  * @node:	List element for tracking in the node
292  * @cache_attrs:Attributes for this cache level
293  */
294 struct node_cache_info {
295 	struct device dev;
296 	struct list_head node;
297 	struct node_cache_attrs cache_attrs;
298 };
299 #define to_cache_info(device) container_of(device, struct node_cache_info, dev)
300 
301 #define CACHE_ATTR(name, fmt) 						\
302 static ssize_t name##_show(struct device *dev,				\
303 			   struct device_attribute *attr,		\
304 			   char *buf)					\
305 {									\
306 	return sysfs_emit(buf, fmt "\n",				\
307 			  to_cache_info(dev)->cache_attrs.name);	\
308 }									\
309 static DEVICE_ATTR_RO(name);
310 
311 CACHE_ATTR(size, "%llu")
312 CACHE_ATTR(line_size, "%u")
313 CACHE_ATTR(indexing, "%u")
314 CACHE_ATTR(write_policy, "%u")
315 CACHE_ATTR(address_mode, "%#x")
316 
317 static struct attribute *cache_attrs[] = {
318 	&dev_attr_indexing.attr,
319 	&dev_attr_size.attr,
320 	&dev_attr_line_size.attr,
321 	&dev_attr_write_policy.attr,
322 	&dev_attr_address_mode.attr,
323 	NULL,
324 };
325 ATTRIBUTE_GROUPS(cache);
326 
327 static void node_cache_release(struct device *dev)
328 {
329 	kfree(dev);
330 }
331 
332 static void node_cacheinfo_release(struct device *dev)
333 {
334 	struct node_cache_info *info = to_cache_info(dev);
335 	kfree(info);
336 }
337 
338 static void node_init_cache_dev(struct node *node)
339 {
340 	struct device *dev;
341 
342 	dev = kzalloc_obj(*dev);
343 	if (!dev)
344 		return;
345 
346 	device_initialize(dev);
347 	dev->parent = &node->dev;
348 	dev->release = node_cache_release;
349 	if (dev_set_name(dev, "memory_side_cache"))
350 		goto put_device;
351 
352 	if (device_add(dev))
353 		goto put_device;
354 
355 	pm_runtime_no_callbacks(dev);
356 	node->cache_dev = dev;
357 	return;
358 put_device:
359 	put_device(dev);
360 }
361 
362 /**
363  * node_add_cache() - add cache attribute to a memory node
364  * @nid: Node identifier that has new cache attributes
365  * @cache_attrs: Attributes for the cache being added
366  */
367 void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
368 {
369 	struct node_cache_info *info;
370 	struct device *dev;
371 	struct node *node;
372 
373 	if (!node_online(nid) || !node_devices[nid])
374 		return;
375 
376 	node = node_devices[nid];
377 	list_for_each_entry(info, &node->cache_attrs, node) {
378 		if (info->cache_attrs.level == cache_attrs->level) {
379 			dev_warn(&node->dev,
380 				"attempt to add duplicate cache level:%d\n",
381 				cache_attrs->level);
382 			return;
383 		}
384 	}
385 
386 	if (!node->cache_dev)
387 		node_init_cache_dev(node);
388 	if (!node->cache_dev)
389 		return;
390 
391 	info = kzalloc_obj(*info);
392 	if (!info)
393 		return;
394 
395 	dev = &info->dev;
396 	device_initialize(dev);
397 	dev->parent = node->cache_dev;
398 	dev->release = node_cacheinfo_release;
399 	dev->groups = cache_groups;
400 	if (dev_set_name(dev, "index%d", cache_attrs->level))
401 		goto put_device;
402 
403 	info->cache_attrs = *cache_attrs;
404 	if (device_add(dev)) {
405 		dev_warn(&node->dev, "failed to add cache level:%d\n",
406 			 cache_attrs->level);
407 		goto put_device;
408 	}
409 	pm_runtime_no_callbacks(dev);
410 	list_add_tail(&info->node, &node->cache_attrs);
411 	return;
412 put_device:
413 	put_device(dev);
414 }
415 
416 static void node_remove_caches(struct node *node)
417 {
418 	struct node_cache_info *info, *next;
419 
420 	if (!node->cache_dev)
421 		return;
422 
423 	list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
424 		list_del(&info->node);
425 		device_unregister(&info->dev);
426 	}
427 	device_unregister(node->cache_dev);
428 }
429 
430 static void node_init_caches(unsigned int nid)
431 {
432 	INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
433 }
434 #else
435 static void node_init_caches(unsigned int nid) { }
436 static void node_remove_caches(struct node *node) { }
437 #endif
438 
439 #define K(x) ((x) << (PAGE_SHIFT - 10))
440 static ssize_t node_read_meminfo(struct device *dev,
441 			struct device_attribute *attr, char *buf)
442 {
443 	int len = 0;
444 	int nid = dev->id;
445 	struct pglist_data *pgdat = NODE_DATA(nid);
446 	struct sysinfo i;
447 	unsigned long sreclaimable, sunreclaimable;
448 	unsigned long swapcached = 0;
449 
450 	si_meminfo_node(&i, nid);
451 	sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
452 	sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
453 #ifdef CONFIG_SWAP
454 	swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE);
455 #endif
456 	len = sysfs_emit_at(buf, len,
457 			    "Node %d MemTotal:       %8lu kB\n"
458 			    "Node %d MemFree:        %8lu kB\n"
459 			    "Node %d MemUsed:        %8lu kB\n"
460 			    "Node %d SwapCached:     %8lu kB\n"
461 			    "Node %d Active:         %8lu kB\n"
462 			    "Node %d Inactive:       %8lu kB\n"
463 			    "Node %d Active(anon):   %8lu kB\n"
464 			    "Node %d Inactive(anon): %8lu kB\n"
465 			    "Node %d Active(file):   %8lu kB\n"
466 			    "Node %d Inactive(file): %8lu kB\n"
467 			    "Node %d Unevictable:    %8lu kB\n"
468 			    "Node %d Mlocked:        %8lu kB\n",
469 			    nid, K(i.totalram),
470 			    nid, K(i.freeram),
471 			    nid, K(i.totalram - i.freeram),
472 			    nid, K(swapcached),
473 			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
474 				   node_page_state(pgdat, NR_ACTIVE_FILE)),
475 			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
476 				   node_page_state(pgdat, NR_INACTIVE_FILE)),
477 			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
478 			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
479 			    nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
480 			    nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
481 			    nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
482 			    nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
483 
484 #ifdef CONFIG_HIGHMEM
485 	len += sysfs_emit_at(buf, len,
486 			     "Node %d HighTotal:      %8lu kB\n"
487 			     "Node %d HighFree:       %8lu kB\n"
488 			     "Node %d LowTotal:       %8lu kB\n"
489 			     "Node %d LowFree:        %8lu kB\n",
490 			     nid, K(i.totalhigh),
491 			     nid, K(i.freehigh),
492 			     nid, K(i.totalram - i.totalhigh),
493 			     nid, K(i.freeram - i.freehigh));
494 #endif
495 	len += sysfs_emit_at(buf, len,
496 			     "Node %d Dirty:          %8lu kB\n"
497 			     "Node %d Writeback:      %8lu kB\n"
498 			     "Node %d FilePages:      %8lu kB\n"
499 			     "Node %d Mapped:         %8lu kB\n"
500 			     "Node %d AnonPages:      %8lu kB\n"
501 			     "Node %d Shmem:          %8lu kB\n"
502 			     "Node %d KernelStack:    %8lu kB\n"
503 #ifdef CONFIG_SHADOW_CALL_STACK
504 			     "Node %d ShadowCallStack:%8lu kB\n"
505 #endif
506 			     "Node %d PageTables:     %8lu kB\n"
507 			     "Node %d SecPageTables:  %8lu kB\n"
508 			     "Node %d NFS_Unstable:   %8lu kB\n"
509 			     "Node %d Bounce:         %8lu kB\n"
510 			     "Node %d WritebackTmp:   %8lu kB\n"
511 			     "Node %d KReclaimable:   %8lu kB\n"
512 			     "Node %d Slab:           %8lu kB\n"
513 			     "Node %d SReclaimable:   %8lu kB\n"
514 			     "Node %d SUnreclaim:     %8lu kB\n"
515 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
516 			     "Node %d AnonHugePages:  %8lu kB\n"
517 			     "Node %d ShmemHugePages: %8lu kB\n"
518 			     "Node %d ShmemPmdMapped: %8lu kB\n"
519 			     "Node %d FileHugePages:  %8lu kB\n"
520 			     "Node %d FilePmdMapped:  %8lu kB\n"
521 #endif
522 #ifdef CONFIG_UNACCEPTED_MEMORY
523 			     "Node %d Unaccepted:     %8lu kB\n"
524 #endif
525 			     "Node %d Balloon:        %8lu kB\n"
526 			     "Node %d GPUActive:      %8lu kB\n"
527 			     "Node %d GPUReclaim:     %8lu kB\n"
528 			     ,
529 			     nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
530 			     nid, K(node_page_state(pgdat, NR_WRITEBACK)),
531 			     nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
532 			     nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
533 			     nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
534 			     nid, K(i.sharedram),
535 			     nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
536 #ifdef CONFIG_SHADOW_CALL_STACK
537 			     nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
538 #endif
539 			     nid, K(node_page_state(pgdat, NR_PAGETABLE)),
540 			     nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)),
541 			     nid, 0UL,
542 			     nid, 0UL,
543 			     nid, 0UL,
544 			     nid, K(sreclaimable +
545 				    node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
546 			     nid, K(sreclaimable + sunreclaimable),
547 			     nid, K(sreclaimable),
548 			     nid, K(sunreclaimable)
549 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
550 			     ,
551 			     nid, K(node_page_state(pgdat, NR_ANON_THPS)),
552 			     nid, K(node_page_state(pgdat, NR_SHMEM_THPS)),
553 			     nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
554 			     nid, K(node_page_state(pgdat, NR_FILE_THPS)),
555 			     nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED))
556 #endif
557 #ifdef CONFIG_UNACCEPTED_MEMORY
558 			     ,
559 			     nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED))
560 #endif
561 			     ,
562 			     nid, K(node_page_state(pgdat, NR_BALLOON_PAGES)),
563 			     nid, K(node_page_state(pgdat, NR_GPU_ACTIVE)),
564 			     nid, K(node_page_state(pgdat, NR_GPU_RECLAIM))
565 			    );
566 	len += hugetlb_report_node_meminfo(buf, len, nid);
567 	return len;
568 }
569 
570 #undef K
571 static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
572 
573 static ssize_t node_read_numastat(struct device *dev,
574 				  struct device_attribute *attr, char *buf)
575 {
576 	fold_vm_numa_events();
577 	return sysfs_emit(buf,
578 			  "numa_hit %lu\n"
579 			  "numa_miss %lu\n"
580 			  "numa_foreign %lu\n"
581 			  "interleave_hit %lu\n"
582 			  "local_node %lu\n"
583 			  "other_node %lu\n",
584 			  sum_zone_numa_event_state(dev->id, NUMA_HIT),
585 			  sum_zone_numa_event_state(dev->id, NUMA_MISS),
586 			  sum_zone_numa_event_state(dev->id, NUMA_FOREIGN),
587 			  sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT),
588 			  sum_zone_numa_event_state(dev->id, NUMA_LOCAL),
589 			  sum_zone_numa_event_state(dev->id, NUMA_OTHER));
590 }
591 static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
592 
593 static ssize_t node_read_vmstat(struct device *dev,
594 				struct device_attribute *attr, char *buf)
595 {
596 	int nid = dev->id;
597 	struct pglist_data *pgdat = NODE_DATA(nid);
598 	int i;
599 	int len = 0;
600 
601 	for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
602 		len += sysfs_emit_at(buf, len, "%s %lu\n",
603 				     zone_stat_name(i),
604 				     sum_zone_node_page_state(nid, i));
605 
606 #ifdef CONFIG_NUMA
607 	fold_vm_numa_events();
608 	for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
609 		len += sysfs_emit_at(buf, len, "%s %lu\n",
610 				     numa_stat_name(i),
611 				     sum_zone_numa_event_state(nid, i));
612 
613 #endif
614 	for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
615 		unsigned long pages = node_page_state_pages(pgdat, i);
616 
617 		if (vmstat_item_print_in_thp(i))
618 			pages /= HPAGE_PMD_NR;
619 		len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i),
620 				     pages);
621 	}
622 
623 	return len;
624 }
625 static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
626 
627 static ssize_t node_read_distance(struct device *dev,
628 				  struct device_attribute *attr, char *buf)
629 {
630 	int nid = dev->id;
631 	int len = 0;
632 	int i;
633 
634 	/*
635 	 * buf is currently PAGE_SIZE in length and each node needs 4 chars
636 	 * at the most (distance + space or newline).
637 	 */
638 	BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
639 
640 	for_each_online_node(i) {
641 		len += sysfs_emit_at(buf, len, "%s%d",
642 				     i ? " " : "", node_distance(nid, i));
643 	}
644 
645 	len += sysfs_emit_at(buf, len, "\n");
646 	return len;
647 }
648 static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
649 
650 static struct attribute *node_dev_attrs[] = {
651 	&dev_attr_meminfo.attr,
652 	&dev_attr_numastat.attr,
653 	&dev_attr_distance.attr,
654 	&dev_attr_vmstat.attr,
655 	NULL
656 };
657 
658 static const struct bin_attribute *node_dev_bin_attrs[] = {
659 	&bin_attr_cpumap,
660 	&bin_attr_cpulist,
661 	NULL
662 };
663 
664 static const struct attribute_group node_dev_group = {
665 	.attrs = node_dev_attrs,
666 	.bin_attrs = node_dev_bin_attrs,
667 };
668 
669 static const struct attribute_group *node_dev_groups[] = {
670 	&node_dev_group,
671 #ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP
672 	&arch_node_dev_group,
673 #endif
674 #ifdef CONFIG_MEMORY_FAILURE
675 	&memory_failure_attr_group,
676 #endif
677 	NULL
678 };
679 
680 static void node_device_release(struct device *dev)
681 {
682 	kfree(to_node(dev));
683 }
684 
685 struct node *node_devices[MAX_NUMNODES];
686 
687 /*
688  * register cpu under node
689  */
690 int register_cpu_under_node(unsigned int cpu, unsigned int nid)
691 {
692 	int ret;
693 	struct device *obj;
694 
695 	if (!node_online(nid))
696 		return 0;
697 
698 	obj = get_cpu_device(cpu);
699 	if (!obj)
700 		return 0;
701 
702 	ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
703 				&obj->kobj,
704 				kobject_name(&obj->kobj));
705 	if (ret)
706 		return ret;
707 
708 	return sysfs_create_link(&obj->kobj,
709 				 &node_devices[nid]->dev.kobj,
710 				 kobject_name(&node_devices[nid]->dev.kobj));
711 }
712 
713 /**
714  * register_memory_node_under_compute_node - link memory node to its compute
715  *					     node for a given access class.
716  * @mem_nid:	Memory node number
717  * @cpu_nid:	Cpu  node number
718  * @access:	Access class to register
719  *
720  * Description:
721  * 	For use with platforms that may have separate memory and compute nodes.
722  * 	This function will export node relationships linking which memory
723  * 	initiator nodes can access memory targets at a given ranked access
724  * 	class.
725  */
726 int register_memory_node_under_compute_node(unsigned int mem_nid,
727 					    unsigned int cpu_nid,
728 					    enum access_coordinate_class access)
729 {
730 	struct node *init_node, *targ_node;
731 	struct node_access_nodes *initiator, *target;
732 	int ret;
733 
734 	if (!node_online(cpu_nid) || !node_online(mem_nid))
735 		return -ENODEV;
736 
737 	init_node = node_devices[cpu_nid];
738 	targ_node = node_devices[mem_nid];
739 	initiator = node_init_node_access(init_node, access);
740 	target = node_init_node_access(targ_node, access);
741 	if (!initiator || !target)
742 		return -ENOMEM;
743 
744 	ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
745 				      &targ_node->dev.kobj,
746 				      dev_name(&targ_node->dev));
747 	if (ret)
748 		return ret;
749 
750 	ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
751 				      &init_node->dev.kobj,
752 				      dev_name(&init_node->dev));
753 	if (ret)
754 		goto err;
755 
756 	return 0;
757  err:
758 	sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
759 				     dev_name(&targ_node->dev));
760 	return ret;
761 }
762 
763 int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
764 {
765 	struct device *obj;
766 
767 	if (!node_online(nid))
768 		return 0;
769 
770 	obj = get_cpu_device(cpu);
771 	if (!obj)
772 		return 0;
773 
774 	sysfs_remove_link(&node_devices[nid]->dev.kobj,
775 			  kobject_name(&obj->kobj));
776 	sysfs_remove_link(&obj->kobj,
777 			  kobject_name(&node_devices[nid]->dev.kobj));
778 
779 	return 0;
780 }
781 
782 #ifdef CONFIG_MEMORY_HOTPLUG
783 static void do_register_memory_block_under_node(int nid,
784 						struct memory_block *mem_blk)
785 {
786 	int ret;
787 
788 	ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
789 				       &mem_blk->dev.kobj,
790 				       kobject_name(&mem_blk->dev.kobj));
791 	if (ret && ret != -EEXIST)
792 		dev_err_ratelimited(&node_devices[nid]->dev,
793 				    "can't create link to %s in sysfs (%d)\n",
794 				    kobject_name(&mem_blk->dev.kobj), ret);
795 
796 	ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
797 				&node_devices[nid]->dev.kobj,
798 				kobject_name(&node_devices[nid]->dev.kobj));
799 	if (ret && ret != -EEXIST)
800 		dev_err_ratelimited(&mem_blk->dev,
801 				    "can't create link to %s in sysfs (%d)\n",
802 				    kobject_name(&node_devices[nid]->dev.kobj),
803 				    ret);
804 }
805 
806 /*
807  * During hotplug we know that all pages in the memory block belong to the same
808  * node.
809  */
810 static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
811 						 void *arg)
812 {
813 	int nid = *(int *)arg;
814 
815 	do_register_memory_block_under_node(nid, mem_blk);
816 	return 0;
817 }
818 
819 /*
820  * Unregister a memory block device under the node it spans. Memory blocks
821  * with multiple nodes cannot be offlined and therefore also never be removed.
822  */
823 void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
824 {
825 	if (mem_blk->nid == NUMA_NO_NODE)
826 		return;
827 
828 	sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
829 			  kobject_name(&mem_blk->dev.kobj));
830 	sysfs_remove_link(&mem_blk->dev.kobj,
831 			  kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
832 }
833 
834 /* register all memory blocks under the corresponding nodes */
835 static void register_memory_blocks_under_nodes(void)
836 {
837 	struct memblock_region *r;
838 
839 	for_each_mem_region(r) {
840 		const unsigned long start_block_id = phys_to_block_id(r->base);
841 		const unsigned long end_block_id = phys_to_block_id(r->base + r->size - 1);
842 		const int nid = memblock_get_region_node(r);
843 		unsigned long block_id;
844 
845 		if (!node_online(nid))
846 			continue;
847 
848 		for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
849 			struct memory_block *mem;
850 
851 			mem = memory_block_get(block_id);
852 			if (!mem)
853 				continue;
854 
855 			memory_block_add_nid_early(mem, nid);
856 			do_register_memory_block_under_node(nid, mem);
857 			memory_block_put(mem);
858 		}
859 
860 	}
861 }
862 
863 void register_memory_blocks_under_node_hotplug(int nid, unsigned long start_pfn,
864 					       unsigned long end_pfn)
865 {
866 	walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
867 			   (void *)&nid, register_mem_block_under_node_hotplug);
868 	return;
869 }
870 #endif /* CONFIG_MEMORY_HOTPLUG */
871 
872 /**
873  * register_node - Initialize and register the node device.
874  * @nid: Node number to use when creating the device.
875  *
876  * Return: 0 on success, -errno otherwise
877  */
878 int register_node(int nid)
879 {
880 	int error;
881 	int cpu;
882 	struct node *node;
883 
884 	node = kzalloc_obj(struct node);
885 	if (!node)
886 		return -ENOMEM;
887 
888 	INIT_LIST_HEAD(&node->access_list);
889 
890 	node->dev.id = nid;
891 	node->dev.bus = &node_subsys;
892 	node->dev.release = node_device_release;
893 	node->dev.groups = node_dev_groups;
894 
895 	error = device_register(&node->dev);
896 	if (error) {
897 		put_device(&node->dev);
898 		return error;
899 	}
900 
901 	node_devices[nid] = node;
902 	hugetlb_register_node(node);
903 	compaction_register_node(node);
904 	reclaim_register_node(node);
905 
906 	/* link cpu under this node */
907 	for_each_present_cpu(cpu) {
908 		if (cpu_to_node(cpu) == nid)
909 			register_cpu_under_node(cpu, nid);
910 	}
911 
912 	node_init_caches(nid);
913 
914 	return error;
915 }
916 /**
917  * unregister_node - unregister a node device
918  * @nid: nid of the node going away
919  *
920  * Unregisters the node device at node id @nid. All the devices on the
921  * node must be unregistered before calling this function.
922  */
923 void unregister_node(int nid)
924 {
925 	struct node *node = node_devices[nid];
926 
927 	if (!node)
928 		return;
929 
930 	hugetlb_unregister_node(node);
931 	compaction_unregister_node(node);
932 	reclaim_unregister_node(node);
933 	node_remove_accesses(node);
934 	node_remove_caches(node);
935 	device_unregister(&node->dev);
936 	node_devices[nid] = NULL;
937 }
938 
939 /*
940  * node states attributes
941  */
942 
943 struct node_attr {
944 	struct device_attribute attr;
945 	enum node_states state;
946 };
947 
948 static ssize_t show_node_state(struct device *dev,
949 			       struct device_attribute *attr, char *buf)
950 {
951 	struct node_attr *na = container_of(attr, struct node_attr, attr);
952 
953 	return sysfs_emit(buf, "%*pbl\n",
954 			  nodemask_pr_args(&node_states[na->state]));
955 }
956 
957 #define _NODE_ATTR(name, state) \
958 	{ __ATTR(name, 0444, show_node_state, NULL), state }
959 
960 static struct node_attr node_state_attr[] = {
961 	[N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
962 	[N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
963 	[N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
964 #ifdef CONFIG_HIGHMEM
965 	[N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
966 #endif
967 	[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
968 	[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
969 	[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
970 					   N_GENERIC_INITIATOR),
971 };
972 
973 static struct attribute *node_state_attrs[] = {
974 	&node_state_attr[N_POSSIBLE].attr.attr,
975 	&node_state_attr[N_ONLINE].attr.attr,
976 	&node_state_attr[N_NORMAL_MEMORY].attr.attr,
977 #ifdef CONFIG_HIGHMEM
978 	&node_state_attr[N_HIGH_MEMORY].attr.attr,
979 #endif
980 	&node_state_attr[N_MEMORY].attr.attr,
981 	&node_state_attr[N_CPU].attr.attr,
982 	&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
983 	NULL
984 };
985 
986 static const struct attribute_group memory_root_attr_group = {
987 	.attrs = node_state_attrs,
988 };
989 
990 static const struct attribute_group *cpu_root_attr_groups[] = {
991 	&memory_root_attr_group,
992 	NULL,
993 };
994 
995 void __init node_dev_init(void)
996 {
997 	int ret, i;
998 
999  	BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
1000  	BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
1001 
1002 	ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
1003 	if (ret)
1004 		panic("%s() failed to register subsystem: %d\n", __func__, ret);
1005 
1006 	/*
1007 	 * Create all node devices, which will properly link the node
1008 	 * to already created cpu devices.
1009 	 */
1010 	for_each_online_node(i) {
1011 		ret =  register_node(i);
1012 		if (ret)
1013 			panic("%s() failed to add node: %d\n", __func__, ret);
1014 	}
1015 
1016 	register_memory_blocks_under_nodes();
1017 }
1018