xref: /linux/drivers/acpi/numa/hmat.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019, Intel Corporation.
4  *
5  * Heterogeneous Memory Attributes Table (HMAT) representation
6  *
7  * This program parses and reports the platform's HMAT tables, and registers
8  * the applicable attributes with the node's interfaces.
9  */
10 
11 #define pr_fmt(fmt) "acpi/hmat: " fmt
12 
13 #include <linux/acpi.h>
14 #include <linux/bitops.h>
15 #include <linux/device.h>
16 #include <linux/init.h>
17 #include <linux/list.h>
18 #include <linux/mm.h>
19 #include <linux/platform_device.h>
20 #include <linux/list_sort.h>
21 #include <linux/memregion.h>
22 #include <linux/memory.h>
23 #include <linux/mutex.h>
24 #include <linux/node.h>
25 #include <linux/sysfs.h>
26 #include <linux/dax.h>
27 #include <linux/memory-tiers.h>
28 
29 static u8 hmat_revision;
30 static int hmat_disable __initdata;
31 
disable_hmat(void)32 void __init disable_hmat(void)
33 {
34 	hmat_disable = 1;
35 }
36 
37 static LIST_HEAD(targets);
38 static LIST_HEAD(initiators);
39 static LIST_HEAD(localities);
40 
41 static DEFINE_MUTEX(target_lock);
42 
43 /*
44  * The defined enum order is used to prioritize attributes to break ties when
45  * selecting the best performing node.
46  */
47 enum locality_types {
48 	WRITE_LATENCY,
49 	READ_LATENCY,
50 	WRITE_BANDWIDTH,
51 	READ_BANDWIDTH,
52 };
53 
54 static struct memory_locality *localities_types[4];
55 
56 struct target_cache {
57 	struct list_head node;
58 	struct node_cache_attrs cache_attrs;
59 };
60 
61 enum {
62 	NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,
63 	NODE_ACCESS_CLASS_GENPORT_SINK_CPU,
64 	NODE_ACCESS_CLASS_MAX,
65 };
66 
67 struct memory_target {
68 	struct list_head node;
69 	unsigned int memory_pxm;
70 	unsigned int processor_pxm;
71 	struct resource memregions;
72 	struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];
73 	struct list_head caches;
74 	struct node_cache_attrs cache_attrs;
75 	u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];
76 	bool registered;
77 	bool ext_updated;	/* externally updated */
78 };
79 
80 struct memory_initiator {
81 	struct list_head node;
82 	unsigned int processor_pxm;
83 	bool has_cpu;
84 };
85 
86 struct memory_locality {
87 	struct list_head node;
88 	struct acpi_hmat_locality *hmat_loc;
89 };
90 
find_mem_initiator(unsigned int cpu_pxm)91 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
92 {
93 	struct memory_initiator *initiator;
94 
95 	list_for_each_entry(initiator, &initiators, node)
96 		if (initiator->processor_pxm == cpu_pxm)
97 			return initiator;
98 	return NULL;
99 }
100 
find_mem_target(unsigned int mem_pxm)101 static struct memory_target *find_mem_target(unsigned int mem_pxm)
102 {
103 	struct memory_target *target;
104 
105 	list_for_each_entry(target, &targets, node)
106 		if (target->memory_pxm == mem_pxm)
107 			return target;
108 	return NULL;
109 }
110 
acpi_find_genport_target(u32 uid)111 static struct memory_target *acpi_find_genport_target(u32 uid)
112 {
113 	struct memory_target *target;
114 	u32 target_uid;
115 	u8 *uid_ptr;
116 
117 	list_for_each_entry(target, &targets, node) {
118 		uid_ptr = target->gen_port_device_handle + 8;
119 		target_uid = *(u32 *)uid_ptr;
120 		if (uid == target_uid)
121 			return target;
122 	}
123 
124 	return NULL;
125 }
126 
127 /**
128  * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port
129  * @uid: ACPI unique id
130  * @coord: The access coordinates written back out for the generic port.
131  *	   Expect 2 levels array.
132  *
133  * Return: 0 on success. Errno on failure.
134  *
135  * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.
136  */
acpi_get_genport_coordinates(u32 uid,struct access_coordinate * coord)137 int acpi_get_genport_coordinates(u32 uid,
138 				 struct access_coordinate *coord)
139 {
140 	struct memory_target *target;
141 
142 	guard(mutex)(&target_lock);
143 	target = acpi_find_genport_target(uid);
144 	if (!target)
145 		return -ENOENT;
146 
147 	coord[ACCESS_COORDINATE_LOCAL] =
148 		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];
149 	coord[ACCESS_COORDINATE_CPU] =
150 		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];
151 
152 	return 0;
153 }
154 EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL);
155 
alloc_memory_initiator(unsigned int cpu_pxm)156 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
157 {
158 	struct memory_initiator *initiator;
159 
160 	if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
161 		return;
162 
163 	initiator = find_mem_initiator(cpu_pxm);
164 	if (initiator)
165 		return;
166 
167 	initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
168 	if (!initiator)
169 		return;
170 
171 	initiator->processor_pxm = cpu_pxm;
172 	initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
173 	list_add_tail(&initiator->node, &initiators);
174 }
175 
alloc_target(unsigned int mem_pxm)176 static __init struct memory_target *alloc_target(unsigned int mem_pxm)
177 {
178 	struct memory_target *target;
179 
180 	target = find_mem_target(mem_pxm);
181 	if (!target) {
182 		target = kzalloc(sizeof(*target), GFP_KERNEL);
183 		if (!target)
184 			return NULL;
185 		target->memory_pxm = mem_pxm;
186 		target->processor_pxm = PXM_INVAL;
187 		target->memregions = (struct resource) {
188 			.name	= "ACPI mem",
189 			.start	= 0,
190 			.end	= -1,
191 			.flags	= IORESOURCE_MEM,
192 		};
193 		list_add_tail(&target->node, &targets);
194 		INIT_LIST_HEAD(&target->caches);
195 	}
196 
197 	return target;
198 }
199 
alloc_memory_target(unsigned int mem_pxm,resource_size_t start,resource_size_t len)200 static __init void alloc_memory_target(unsigned int mem_pxm,
201 				       resource_size_t start,
202 				       resource_size_t len)
203 {
204 	struct memory_target *target;
205 
206 	target = alloc_target(mem_pxm);
207 	if (!target)
208 		return;
209 
210 	/*
211 	 * There are potentially multiple ranges per PXM, so record each
212 	 * in the per-target memregions resource tree.
213 	 */
214 	if (!__request_region(&target->memregions, start, len, "memory target",
215 				IORESOURCE_MEM))
216 		pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
217 				start, start + len, mem_pxm);
218 }
219 
alloc_genport_target(unsigned int mem_pxm,u8 * handle)220 static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)
221 {
222 	struct memory_target *target;
223 
224 	target = alloc_target(mem_pxm);
225 	if (!target)
226 		return;
227 
228 	memcpy(target->gen_port_device_handle, handle,
229 	       ACPI_SRAT_DEVICE_HANDLE_SIZE);
230 }
231 
hmat_data_type(u8 type)232 static __init const char *hmat_data_type(u8 type)
233 {
234 	switch (type) {
235 	case ACPI_HMAT_ACCESS_LATENCY:
236 		return "Access Latency";
237 	case ACPI_HMAT_READ_LATENCY:
238 		return "Read Latency";
239 	case ACPI_HMAT_WRITE_LATENCY:
240 		return "Write Latency";
241 	case ACPI_HMAT_ACCESS_BANDWIDTH:
242 		return "Access Bandwidth";
243 	case ACPI_HMAT_READ_BANDWIDTH:
244 		return "Read Bandwidth";
245 	case ACPI_HMAT_WRITE_BANDWIDTH:
246 		return "Write Bandwidth";
247 	default:
248 		return "Reserved";
249 	}
250 }
251 
hmat_data_type_suffix(u8 type)252 static __init const char *hmat_data_type_suffix(u8 type)
253 {
254 	switch (type) {
255 	case ACPI_HMAT_ACCESS_LATENCY:
256 	case ACPI_HMAT_READ_LATENCY:
257 	case ACPI_HMAT_WRITE_LATENCY:
258 		return " nsec";
259 	case ACPI_HMAT_ACCESS_BANDWIDTH:
260 	case ACPI_HMAT_READ_BANDWIDTH:
261 	case ACPI_HMAT_WRITE_BANDWIDTH:
262 		return " MB/s";
263 	default:
264 		return "";
265 	}
266 }
267 
hmat_normalize(u16 entry,u64 base,u8 type)268 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
269 {
270 	u32 value;
271 
272 	/*
273 	 * Check for invalid and overflow values
274 	 */
275 	if (entry == 0xffff || !entry)
276 		return 0;
277 	else if (base > (UINT_MAX / (entry)))
278 		return 0;
279 
280 	/*
281 	 * Divide by the base unit for version 1, convert latency from
282 	 * picosenonds to nanoseconds if revision 2.
283 	 */
284 	value = entry * base;
285 	if (hmat_revision == 1) {
286 		if (value < 10)
287 			return 0;
288 		value = DIV_ROUND_UP(value, 10);
289 	} else if (hmat_revision == 2) {
290 		switch (type) {
291 		case ACPI_HMAT_ACCESS_LATENCY:
292 		case ACPI_HMAT_READ_LATENCY:
293 		case ACPI_HMAT_WRITE_LATENCY:
294 			value = DIV_ROUND_UP(value, 1000);
295 			break;
296 		default:
297 			break;
298 		}
299 	}
300 	return value;
301 }
302 
hmat_update_target_access(struct memory_target * target,u8 type,u32 value,int access)303 static void hmat_update_target_access(struct memory_target *target,
304 				      u8 type, u32 value, int access)
305 {
306 	switch (type) {
307 	case ACPI_HMAT_ACCESS_LATENCY:
308 		target->coord[access].read_latency = value;
309 		target->coord[access].write_latency = value;
310 		break;
311 	case ACPI_HMAT_READ_LATENCY:
312 		target->coord[access].read_latency = value;
313 		break;
314 	case ACPI_HMAT_WRITE_LATENCY:
315 		target->coord[access].write_latency = value;
316 		break;
317 	case ACPI_HMAT_ACCESS_BANDWIDTH:
318 		target->coord[access].read_bandwidth = value;
319 		target->coord[access].write_bandwidth = value;
320 		break;
321 	case ACPI_HMAT_READ_BANDWIDTH:
322 		target->coord[access].read_bandwidth = value;
323 		break;
324 	case ACPI_HMAT_WRITE_BANDWIDTH:
325 		target->coord[access].write_bandwidth = value;
326 		break;
327 	default:
328 		break;
329 	}
330 }
331 
hmat_update_target_coordinates(int nid,struct access_coordinate * coord,enum access_coordinate_class access)332 int hmat_update_target_coordinates(int nid, struct access_coordinate *coord,
333 				   enum access_coordinate_class access)
334 {
335 	struct memory_target *target;
336 	int pxm;
337 
338 	if (nid == NUMA_NO_NODE)
339 		return -EINVAL;
340 
341 	pxm = node_to_pxm(nid);
342 	guard(mutex)(&target_lock);
343 	target = find_mem_target(pxm);
344 	if (!target)
345 		return -ENODEV;
346 
347 	hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY,
348 				  coord->read_latency, access);
349 	hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY,
350 				  coord->write_latency, access);
351 	hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH,
352 				  coord->read_bandwidth, access);
353 	hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH,
354 				  coord->write_bandwidth, access);
355 	target->ext_updated = true;
356 
357 	return 0;
358 }
359 EXPORT_SYMBOL_GPL(hmat_update_target_coordinates);
360 
hmat_add_locality(struct acpi_hmat_locality * hmat_loc)361 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
362 {
363 	struct memory_locality *loc;
364 
365 	loc = kzalloc(sizeof(*loc), GFP_KERNEL);
366 	if (!loc) {
367 		pr_notice_once("Failed to allocate HMAT locality\n");
368 		return;
369 	}
370 
371 	loc->hmat_loc = hmat_loc;
372 	list_add_tail(&loc->node, &localities);
373 
374 	switch (hmat_loc->data_type) {
375 	case ACPI_HMAT_ACCESS_LATENCY:
376 		localities_types[READ_LATENCY] = loc;
377 		localities_types[WRITE_LATENCY] = loc;
378 		break;
379 	case ACPI_HMAT_READ_LATENCY:
380 		localities_types[READ_LATENCY] = loc;
381 		break;
382 	case ACPI_HMAT_WRITE_LATENCY:
383 		localities_types[WRITE_LATENCY] = loc;
384 		break;
385 	case ACPI_HMAT_ACCESS_BANDWIDTH:
386 		localities_types[READ_BANDWIDTH] = loc;
387 		localities_types[WRITE_BANDWIDTH] = loc;
388 		break;
389 	case ACPI_HMAT_READ_BANDWIDTH:
390 		localities_types[READ_BANDWIDTH] = loc;
391 		break;
392 	case ACPI_HMAT_WRITE_BANDWIDTH:
393 		localities_types[WRITE_BANDWIDTH] = loc;
394 		break;
395 	default:
396 		break;
397 	}
398 }
399 
hmat_update_target(unsigned int tgt_pxm,unsigned int init_pxm,u8 mem_hier,u8 type,u32 value)400 static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,
401 				      u8 mem_hier, u8 type, u32 value)
402 {
403 	struct memory_target *target = find_mem_target(tgt_pxm);
404 
405 	if (mem_hier != ACPI_HMAT_MEMORY)
406 		return;
407 
408 	if (target && target->processor_pxm == init_pxm) {
409 		hmat_update_target_access(target, type, value,
410 					  ACCESS_COORDINATE_LOCAL);
411 		/* If the node has a CPU, update access ACCESS_COORDINATE_CPU */
412 		if (node_state(pxm_to_node(init_pxm), N_CPU))
413 			hmat_update_target_access(target, type, value,
414 						  ACCESS_COORDINATE_CPU);
415 	}
416 }
417 
hmat_parse_locality(union acpi_subtable_headers * header,const unsigned long end)418 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
419 				      const unsigned long end)
420 {
421 	struct acpi_hmat_locality *hmat_loc = (void *)header;
422 	unsigned int init, targ, total_size, ipds, tpds;
423 	u32 *inits, *targs, value;
424 	u16 *entries;
425 	u8 type, mem_hier;
426 
427 	if (hmat_loc->header.length < sizeof(*hmat_loc)) {
428 		pr_notice("Unexpected locality header length: %u\n",
429 			 hmat_loc->header.length);
430 		return -EINVAL;
431 	}
432 
433 	type = hmat_loc->data_type;
434 	mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
435 	ipds = hmat_loc->number_of_initiator_Pds;
436 	tpds = hmat_loc->number_of_target_Pds;
437 	total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
438 		     sizeof(*inits) * ipds + sizeof(*targs) * tpds;
439 	if (hmat_loc->header.length < total_size) {
440 		pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
441 			 hmat_loc->header.length, total_size);
442 		return -EINVAL;
443 	}
444 
445 	pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
446 		hmat_loc->flags, hmat_data_type(type), ipds, tpds,
447 		hmat_loc->entry_base_unit);
448 
449 	inits = (u32 *)(hmat_loc + 1);
450 	targs = inits + ipds;
451 	entries = (u16 *)(targs + tpds);
452 	for (init = 0; init < ipds; init++) {
453 		alloc_memory_initiator(inits[init]);
454 		for (targ = 0; targ < tpds; targ++) {
455 			value = hmat_normalize(entries[init * tpds + targ],
456 					       hmat_loc->entry_base_unit,
457 					       type);
458 			pr_info("  Initiator-Target[%u-%u]:%u%s\n",
459 				inits[init], targs[targ], value,
460 				hmat_data_type_suffix(type));
461 
462 			hmat_update_target(targs[targ], inits[init],
463 					   mem_hier, type, value);
464 		}
465 	}
466 
467 	if (mem_hier == ACPI_HMAT_MEMORY)
468 		hmat_add_locality(hmat_loc);
469 
470 	return 0;
471 }
472 
hmat_parse_cache(union acpi_subtable_headers * header,const unsigned long end)473 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
474 				   const unsigned long end)
475 {
476 	struct acpi_hmat_cache *cache = (void *)header;
477 	struct memory_target *target;
478 	struct target_cache *tcache;
479 	u32 attrs;
480 
481 	if (cache->header.length < sizeof(*cache)) {
482 		pr_notice("Unexpected cache header length: %u\n",
483 			 cache->header.length);
484 		return -EINVAL;
485 	}
486 
487 	attrs = cache->cache_attributes;
488 	pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
489 		cache->memory_PD, cache->cache_size, attrs,
490 		cache->number_of_SMBIOShandles);
491 
492 	target = find_mem_target(cache->memory_PD);
493 	if (!target)
494 		return 0;
495 
496 	tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
497 	if (!tcache) {
498 		pr_notice_once("Failed to allocate HMAT cache info\n");
499 		return 0;
500 	}
501 
502 	tcache->cache_attrs.size = cache->cache_size;
503 	tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
504 	tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
505 
506 	switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
507 	case ACPI_HMAT_CA_DIRECT_MAPPED:
508 		tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
509 		break;
510 	case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
511 		tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
512 		break;
513 	case ACPI_HMAT_CA_NONE:
514 	default:
515 		tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
516 		break;
517 	}
518 
519 	switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
520 	case ACPI_HMAT_CP_WB:
521 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
522 		break;
523 	case ACPI_HMAT_CP_WT:
524 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
525 		break;
526 	case ACPI_HMAT_CP_NONE:
527 	default:
528 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
529 		break;
530 	}
531 	list_add_tail(&tcache->node, &target->caches);
532 
533 	return 0;
534 }
535 
hmat_parse_proximity_domain(union acpi_subtable_headers * header,const unsigned long end)536 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
537 					      const unsigned long end)
538 {
539 	struct acpi_hmat_proximity_domain *p = (void *)header;
540 	struct memory_target *target = NULL;
541 
542 	if (p->header.length != sizeof(*p)) {
543 		pr_notice("Unexpected address range header length: %u\n",
544 			 p->header.length);
545 		return -EINVAL;
546 	}
547 
548 	if (hmat_revision == 1)
549 		pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
550 			p->reserved3, p->reserved4, p->flags, p->processor_PD,
551 			p->memory_PD);
552 	else
553 		pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
554 			p->flags, p->processor_PD, p->memory_PD);
555 
556 	if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
557 	    hmat_revision > 1) {
558 		target = find_mem_target(p->memory_PD);
559 		if (!target) {
560 			pr_debug("Memory Domain missing from SRAT\n");
561 			return -EINVAL;
562 		}
563 	}
564 	if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
565 		int p_node = pxm_to_node(p->processor_PD);
566 
567 		if (p_node == NUMA_NO_NODE) {
568 			pr_debug("Invalid Processor Domain\n");
569 			return -EINVAL;
570 		}
571 		target->processor_pxm = p->processor_PD;
572 	}
573 
574 	return 0;
575 }
576 
hmat_parse_subtable(union acpi_subtable_headers * header,const unsigned long end)577 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
578 				      const unsigned long end)
579 {
580 	struct acpi_hmat_structure *hdr = (void *)header;
581 
582 	if (!hdr)
583 		return -EINVAL;
584 
585 	switch (hdr->type) {
586 	case ACPI_HMAT_TYPE_PROXIMITY:
587 		return hmat_parse_proximity_domain(header, end);
588 	case ACPI_HMAT_TYPE_LOCALITY:
589 		return hmat_parse_locality(header, end);
590 	case ACPI_HMAT_TYPE_CACHE:
591 		return hmat_parse_cache(header, end);
592 	default:
593 		return -EINVAL;
594 	}
595 }
596 
srat_parse_mem_affinity(union acpi_subtable_headers * header,const unsigned long end)597 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
598 					  const unsigned long end)
599 {
600 	struct acpi_srat_mem_affinity *ma = (void *)header;
601 
602 	if (!ma)
603 		return -EINVAL;
604 	if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
605 		return 0;
606 	alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
607 	return 0;
608 }
609 
srat_parse_genport_affinity(union acpi_subtable_headers * header,const unsigned long end)610 static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,
611 					      const unsigned long end)
612 {
613 	struct acpi_srat_generic_affinity *ga = (void *)header;
614 
615 	if (!ga)
616 		return -EINVAL;
617 
618 	if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
619 		return 0;
620 
621 	/* Skip PCI device_handle for now */
622 	if (ga->device_handle_type != 0)
623 		return 0;
624 
625 	alloc_genport_target(ga->proximity_domain,
626 			     (u8 *)ga->device_handle);
627 
628 	return 0;
629 }
630 
hmat_initiator_perf(struct memory_target * target,struct memory_initiator * initiator,struct acpi_hmat_locality * hmat_loc)631 static u32 hmat_initiator_perf(struct memory_target *target,
632 			       struct memory_initiator *initiator,
633 			       struct acpi_hmat_locality *hmat_loc)
634 {
635 	unsigned int ipds, tpds, i, idx = 0, tdx = 0;
636 	u32 *inits, *targs;
637 	u16 *entries;
638 
639 	ipds = hmat_loc->number_of_initiator_Pds;
640 	tpds = hmat_loc->number_of_target_Pds;
641 	inits = (u32 *)(hmat_loc + 1);
642 	targs = inits + ipds;
643 	entries = (u16 *)(targs + tpds);
644 
645 	for (i = 0; i < ipds; i++) {
646 		if (inits[i] == initiator->processor_pxm) {
647 			idx = i;
648 			break;
649 		}
650 	}
651 
652 	if (i == ipds)
653 		return 0;
654 
655 	for (i = 0; i < tpds; i++) {
656 		if (targs[i] == target->memory_pxm) {
657 			tdx = i;
658 			break;
659 		}
660 	}
661 	if (i == tpds)
662 		return 0;
663 
664 	return hmat_normalize(entries[idx * tpds + tdx],
665 			      hmat_loc->entry_base_unit,
666 			      hmat_loc->data_type);
667 }
668 
hmat_update_best(u8 type,u32 value,u32 * best)669 static bool hmat_update_best(u8 type, u32 value, u32 *best)
670 {
671 	bool updated = false;
672 
673 	if (!value)
674 		return false;
675 
676 	switch (type) {
677 	case ACPI_HMAT_ACCESS_LATENCY:
678 	case ACPI_HMAT_READ_LATENCY:
679 	case ACPI_HMAT_WRITE_LATENCY:
680 		if (!*best || *best > value) {
681 			*best = value;
682 			updated = true;
683 		}
684 		break;
685 	case ACPI_HMAT_ACCESS_BANDWIDTH:
686 	case ACPI_HMAT_READ_BANDWIDTH:
687 	case ACPI_HMAT_WRITE_BANDWIDTH:
688 		if (!*best || *best < value) {
689 			*best = value;
690 			updated = true;
691 		}
692 		break;
693 	}
694 
695 	return updated;
696 }
697 
initiator_cmp(void * priv,const struct list_head * a,const struct list_head * b)698 static int initiator_cmp(void *priv, const struct list_head *a,
699 			 const struct list_head *b)
700 {
701 	struct memory_initiator *ia;
702 	struct memory_initiator *ib;
703 
704 	ia = list_entry(a, struct memory_initiator, node);
705 	ib = list_entry(b, struct memory_initiator, node);
706 
707 	return ia->processor_pxm - ib->processor_pxm;
708 }
709 
initiators_to_nodemask(unsigned long * p_nodes)710 static int initiators_to_nodemask(unsigned long *p_nodes)
711 {
712 	struct memory_initiator *initiator;
713 
714 	if (list_empty(&initiators))
715 		return -ENXIO;
716 
717 	list_for_each_entry(initiator, &initiators, node)
718 		set_bit(initiator->processor_pxm, p_nodes);
719 
720 	return 0;
721 }
722 
hmat_update_target_attrs(struct memory_target * target,unsigned long * p_nodes,int access)723 static void hmat_update_target_attrs(struct memory_target *target,
724 				     unsigned long *p_nodes, int access)
725 {
726 	struct memory_initiator *initiator;
727 	unsigned int cpu_nid;
728 	struct memory_locality *loc = NULL;
729 	u32 best = 0;
730 	int i;
731 
732 	/* Don't update if an external agent has changed the data.  */
733 	if (target->ext_updated)
734 		return;
735 
736 	/* Don't update for generic port if there's no device handle */
737 	if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||
738 	     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
739 	    !(*(u16 *)target->gen_port_device_handle))
740 		return;
741 
742 	bitmap_zero(p_nodes, MAX_NUMNODES);
743 	/*
744 	 * If the Address Range Structure provides a local processor pxm, set
745 	 * only that one. Otherwise, find the best performance attributes and
746 	 * collect all initiators that match.
747 	 */
748 	if (target->processor_pxm != PXM_INVAL) {
749 		cpu_nid = pxm_to_node(target->processor_pxm);
750 		if (access == ACCESS_COORDINATE_LOCAL ||
751 		    node_state(cpu_nid, N_CPU)) {
752 			set_bit(target->processor_pxm, p_nodes);
753 			return;
754 		}
755 	}
756 
757 	if (list_empty(&localities))
758 		return;
759 
760 	/*
761 	 * We need the initiator list sorted so we can use bitmap_clear for
762 	 * previously set initiators when we find a better memory accessor.
763 	 * We'll also use the sorting to prime the candidate nodes with known
764 	 * initiators.
765 	 */
766 	list_sort(NULL, &initiators, initiator_cmp);
767 	if (initiators_to_nodemask(p_nodes) < 0)
768 		return;
769 
770 	for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
771 		loc = localities_types[i];
772 		if (!loc)
773 			continue;
774 
775 		best = 0;
776 		list_for_each_entry(initiator, &initiators, node) {
777 			u32 value;
778 
779 			if ((access == ACCESS_COORDINATE_CPU ||
780 			     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
781 			    !initiator->has_cpu) {
782 				clear_bit(initiator->processor_pxm, p_nodes);
783 				continue;
784 			}
785 			if (!test_bit(initiator->processor_pxm, p_nodes))
786 				continue;
787 
788 			value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
789 			if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
790 				bitmap_clear(p_nodes, 0, initiator->processor_pxm);
791 			if (value != best)
792 				clear_bit(initiator->processor_pxm, p_nodes);
793 		}
794 		if (best)
795 			hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);
796 	}
797 }
798 
__hmat_register_target_initiators(struct memory_target * target,unsigned long * p_nodes,int access)799 static void __hmat_register_target_initiators(struct memory_target *target,
800 					      unsigned long *p_nodes,
801 					      int access)
802 {
803 	unsigned int mem_nid, cpu_nid;
804 	int i;
805 
806 	mem_nid = pxm_to_node(target->memory_pxm);
807 	hmat_update_target_attrs(target, p_nodes, access);
808 	for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
809 		cpu_nid = pxm_to_node(i);
810 		register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
811 	}
812 }
813 
hmat_update_generic_target(struct memory_target * target)814 static void hmat_update_generic_target(struct memory_target *target)
815 {
816 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
817 
818 	hmat_update_target_attrs(target, p_nodes,
819 				 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);
820 	hmat_update_target_attrs(target, p_nodes,
821 				 NODE_ACCESS_CLASS_GENPORT_SINK_CPU);
822 }
823 
hmat_register_target_initiators(struct memory_target * target)824 static void hmat_register_target_initiators(struct memory_target *target)
825 {
826 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
827 
828 	__hmat_register_target_initiators(target, p_nodes,
829 					  ACCESS_COORDINATE_LOCAL);
830 	__hmat_register_target_initiators(target, p_nodes,
831 					  ACCESS_COORDINATE_CPU);
832 }
833 
hmat_register_target_cache(struct memory_target * target)834 static void hmat_register_target_cache(struct memory_target *target)
835 {
836 	unsigned mem_nid = pxm_to_node(target->memory_pxm);
837 	struct target_cache *tcache;
838 
839 	list_for_each_entry(tcache, &target->caches, node)
840 		node_add_cache(mem_nid, &tcache->cache_attrs);
841 }
842 
hmat_register_target_perf(struct memory_target * target,int access)843 static void hmat_register_target_perf(struct memory_target *target, int access)
844 {
845 	unsigned mem_nid = pxm_to_node(target->memory_pxm);
846 	node_set_perf_attrs(mem_nid, &target->coord[access], access);
847 }
848 
hmat_register_target_devices(struct memory_target * target)849 static void hmat_register_target_devices(struct memory_target *target)
850 {
851 	struct resource *res;
852 
853 	/*
854 	 * Do not bother creating devices if no driver is available to
855 	 * consume them.
856 	 */
857 	if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
858 		return;
859 
860 	for (res = target->memregions.child; res; res = res->sibling) {
861 		int target_nid = pxm_to_node(target->memory_pxm);
862 
863 		hmem_register_resource(target_nid, res);
864 	}
865 }
866 
hmat_register_target(struct memory_target * target)867 static void hmat_register_target(struct memory_target *target)
868 {
869 	int nid = pxm_to_node(target->memory_pxm);
870 
871 	/*
872 	 * Devices may belong to either an offline or online
873 	 * node, so unconditionally add them.
874 	 */
875 	hmat_register_target_devices(target);
876 
877 	/*
878 	 * Register generic port perf numbers. The nid may not be
879 	 * initialized and is still NUMA_NO_NODE.
880 	 */
881 	mutex_lock(&target_lock);
882 	if (*(u16 *)target->gen_port_device_handle) {
883 		hmat_update_generic_target(target);
884 		target->registered = true;
885 	}
886 	mutex_unlock(&target_lock);
887 
888 	/*
889 	 * Skip offline nodes. This can happen when memory
890 	 * marked EFI_MEMORY_SP, "specific purpose", is applied
891 	 * to all the memory in a proximity domain leading to
892 	 * the node being marked offline / unplugged, or if
893 	 * memory-only "hotplug" node is offline.
894 	 */
895 	if (nid == NUMA_NO_NODE || !node_online(nid))
896 		return;
897 
898 	mutex_lock(&target_lock);
899 	if (!target->registered) {
900 		hmat_register_target_initiators(target);
901 		hmat_register_target_cache(target);
902 		hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL);
903 		hmat_register_target_perf(target, ACCESS_COORDINATE_CPU);
904 		target->registered = true;
905 	}
906 	mutex_unlock(&target_lock);
907 }
908 
hmat_register_targets(void)909 static void hmat_register_targets(void)
910 {
911 	struct memory_target *target;
912 
913 	list_for_each_entry(target, &targets, node)
914 		hmat_register_target(target);
915 }
916 
hmat_callback(struct notifier_block * self,unsigned long action,void * arg)917 static int hmat_callback(struct notifier_block *self,
918 			 unsigned long action, void *arg)
919 {
920 	struct memory_target *target;
921 	struct memory_notify *mnb = arg;
922 	int pxm, nid = mnb->status_change_nid;
923 
924 	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
925 		return NOTIFY_OK;
926 
927 	pxm = node_to_pxm(nid);
928 	target = find_mem_target(pxm);
929 	if (!target)
930 		return NOTIFY_OK;
931 
932 	hmat_register_target(target);
933 	return NOTIFY_OK;
934 }
935 
hmat_set_default_dram_perf(void)936 static int __init hmat_set_default_dram_perf(void)
937 {
938 	int rc;
939 	int nid, pxm;
940 	struct memory_target *target;
941 	struct access_coordinate *attrs;
942 
943 	for_each_node_mask(nid, default_dram_nodes) {
944 		pxm = node_to_pxm(nid);
945 		target = find_mem_target(pxm);
946 		if (!target)
947 			continue;
948 		attrs = &target->coord[ACCESS_COORDINATE_CPU];
949 		rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");
950 		if (rc)
951 			return rc;
952 	}
953 
954 	return 0;
955 }
956 
hmat_calculate_adistance(struct notifier_block * self,unsigned long nid,void * data)957 static int hmat_calculate_adistance(struct notifier_block *self,
958 				    unsigned long nid, void *data)
959 {
960 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
961 	struct memory_target *target;
962 	struct access_coordinate *perf;
963 	int *adist = data;
964 	int pxm;
965 
966 	pxm = node_to_pxm(nid);
967 	target = find_mem_target(pxm);
968 	if (!target)
969 		return NOTIFY_OK;
970 
971 	mutex_lock(&target_lock);
972 	hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU);
973 	mutex_unlock(&target_lock);
974 
975 	perf = &target->coord[ACCESS_COORDINATE_CPU];
976 
977 	if (mt_perf_to_adistance(perf, adist))
978 		return NOTIFY_OK;
979 
980 	return NOTIFY_STOP;
981 }
982 
983 static struct notifier_block hmat_adist_nb __meminitdata = {
984 	.notifier_call = hmat_calculate_adistance,
985 	.priority = 100,
986 };
987 
hmat_free_structures(void)988 static __init void hmat_free_structures(void)
989 {
990 	struct memory_target *target, *tnext;
991 	struct memory_locality *loc, *lnext;
992 	struct memory_initiator *initiator, *inext;
993 	struct target_cache *tcache, *cnext;
994 
995 	list_for_each_entry_safe(target, tnext, &targets, node) {
996 		struct resource *res, *res_next;
997 
998 		list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
999 			list_del(&tcache->node);
1000 			kfree(tcache);
1001 		}
1002 
1003 		list_del(&target->node);
1004 		res = target->memregions.child;
1005 		while (res) {
1006 			res_next = res->sibling;
1007 			__release_region(&target->memregions, res->start,
1008 					resource_size(res));
1009 			res = res_next;
1010 		}
1011 		kfree(target);
1012 	}
1013 
1014 	list_for_each_entry_safe(initiator, inext, &initiators, node) {
1015 		list_del(&initiator->node);
1016 		kfree(initiator);
1017 	}
1018 
1019 	list_for_each_entry_safe(loc, lnext, &localities, node) {
1020 		list_del(&loc->node);
1021 		kfree(loc);
1022 	}
1023 }
1024 
hmat_init(void)1025 static __init int hmat_init(void)
1026 {
1027 	struct acpi_table_header *tbl;
1028 	enum acpi_hmat_type i;
1029 	acpi_status status;
1030 
1031 	if (srat_disabled() || hmat_disable)
1032 		return 0;
1033 
1034 	status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
1035 	if (ACPI_FAILURE(status))
1036 		return 0;
1037 
1038 	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1039 				sizeof(struct acpi_table_srat),
1040 				ACPI_SRAT_TYPE_MEMORY_AFFINITY,
1041 				srat_parse_mem_affinity, 0) < 0)
1042 		goto out_put;
1043 
1044 	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1045 				     sizeof(struct acpi_table_srat),
1046 				     ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,
1047 				     srat_parse_genport_affinity, 0) < 0)
1048 		goto out_put;
1049 
1050 	acpi_put_table(tbl);
1051 
1052 	status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
1053 	if (ACPI_FAILURE(status))
1054 		goto out_put;
1055 
1056 	hmat_revision = tbl->revision;
1057 	switch (hmat_revision) {
1058 	case 1:
1059 	case 2:
1060 		break;
1061 	default:
1062 		pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
1063 		goto out_put;
1064 	}
1065 
1066 	for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
1067 		if (acpi_table_parse_entries(ACPI_SIG_HMAT,
1068 					     sizeof(struct acpi_table_hmat), i,
1069 					     hmat_parse_subtable, 0) < 0) {
1070 			pr_notice("Ignoring: Invalid table");
1071 			goto out_put;
1072 		}
1073 	}
1074 	hmat_register_targets();
1075 
1076 	/* Keep the table and structures if the notifier may use them */
1077 	if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
1078 		goto out_put;
1079 
1080 	if (!hmat_set_default_dram_perf())
1081 		register_mt_adistance_algorithm(&hmat_adist_nb);
1082 
1083 	return 0;
1084 out_put:
1085 	hmat_free_structures();
1086 	acpi_put_table(tbl);
1087 	return 0;
1088 }
1089 subsys_initcall(hmat_init);
1090