1 // SPDX-License-Identifier: GPL-2.0
5  * Pseudo-locking support built on top of Cache Allocation Technology (CAT)
38  * resctrl_arch_get_prefetch_disable_bits - prefetch disable bits of supported
43  * pseudo-locking. This includes testing to ensure pseudo-locked regions
45  * as well as that these pseudo-locked regions can maintain their low cache
48  * After a platform has been validated to support pseudo-locking its
98  * resctrl_arch_pseudo_lock_fn - Load kernel memory into cache
99  * @_plr: the pseudo-lock region descriptor
101  * This is the core pseudo-locking flow.
106  * with class of service set to the bitmask of the pseudo-locked region.
129 	 * pseudo-locking success rate when KASAN is active.  in resctrl_arch_pseudo_lock_fn()
142 	 * will get a cache hit in below loop from outside of pseudo-locked  in resctrl_arch_pseudo_lock_fn()
158 	 * clobbering local register variables or affecting cache accesses.  in resctrl_arch_pseudo_lock_fn()
161 	 * being pseudo-locked is reached the hardware will not read beyond  in resctrl_arch_pseudo_lock_fn()
162 	 * the buffer and evict pseudo-locked memory read earlier from the  in resctrl_arch_pseudo_lock_fn()
169 	mem_r = plr->kmem;  in resctrl_arch_pseudo_lock_fn()
170 	size = plr->size;  in resctrl_arch_pseudo_lock_fn()
171 	line_size = plr->line_size;  in resctrl_arch_pseudo_lock_fn()
175 	 * pseudo-locked followed by reading of kernel memory to load it  in resctrl_arch_pseudo_lock_fn()
178 	native_wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, plr->closid);  in resctrl_arch_pseudo_lock_fn()
182 	 * into cache region associated with just activated plr->closid.  in resctrl_arch_pseudo_lock_fn()
184 	 * - In first loop the cache region is shared with the page walker  in resctrl_arch_pseudo_lock_fn()
186 	 * - In the second loop the paging structure caches are used and  in resctrl_arch_pseudo_lock_fn()
213 	 * does not overlap with pseudo-locked region.  in resctrl_arch_pseudo_lock_fn()
217 	/* Re-enable the hardware prefetcher(s) */  in resctrl_arch_pseudo_lock_fn()
221 	plr->thread_done = 1;  in resctrl_arch_pseudo_lock_fn()
222 	wake_up_interruptible(&plr->lock_thread_wq);  in resctrl_arch_pseudo_lock_fn()
227  * resctrl_arch_measure_cycles_lat_fn - Measure cycle latency to read
228  *                                      pseudo-locked memory
229  * @_plr: pseudo-lock region to measure
255 	mem_r = READ_ONCE(plr->kmem);  in resctrl_arch_measure_cycles_lat_fn()
261 	for (i = 0; i < plr->size; i += 32) {  in resctrl_arch_measure_cycles_lat_fn()
268 		trace_pseudo_lock_mem_latency((u32)(end - start));  in resctrl_arch_measure_cycles_lat_fn()
272 	plr->thread_done = 1;  in resctrl_arch_measure_cycles_lat_fn()
273 	wake_up_interruptible(&plr->lock_thread_wq);  in resctrl_arch_measure_cycles_lat_fn()
279  * be used during the performance measurement. A perf_event maintains
322 	miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu,  in measure_residency_fn()
327 	hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu,  in measure_residency_fn()
354 	 * Performance event has been validated right before this with  in measure_residency_fn()
355 	 * interrupts disabled - it is thus safe to read the counter index.  in measure_residency_fn()
359 	line_size = READ_ONCE(plr->line_size);  in measure_residency_fn()
360 	mem_r = READ_ONCE(plr->kmem);  in measure_residency_fn()
361 	size = READ_ONCE(plr->size);  in measure_residency_fn()
364 	 * Read counter variables twice - first to load the instructions  in measure_residency_fn()
371 	 * From SDM: Performing back-to-back fast reads are not guaranteed  in measure_residency_fn()
407 	/* Re-enable hardware prefetchers */  in measure_residency_fn()
418 	counts->miss_before = miss_before;  in measure_residency_fn()
419 	counts->hits_before = hits_before;  in measure_residency_fn()
420 	counts->miss_after  = miss_after;  in measure_residency_fn()
421 	counts->hits_after  = hits_after;  in measure_residency_fn()
431 	 * Non-architectural event for the Goldmont Microarchitecture  in resctrl_arch_measure_l2_residency()
455 	trace_pseudo_lock_l2(counts.hits_after - counts.hits_before,  in resctrl_arch_measure_l2_residency()
456 			     counts.miss_after - counts.miss_before);  in resctrl_arch_measure_l2_residency()
458 	plr->thread_done = 1;  in resctrl_arch_measure_l2_residency()
459 	wake_up_interruptible(&plr->lock_thread_wq);  in resctrl_arch_measure_l2_residency()
495 	counts.miss_after -= counts.miss_before;  in resctrl_arch_measure_l3_residency()
505 		counts.hits_after -= counts.hits_before;  in resctrl_arch_measure_l3_residency()
507 		counts.hits_after -= min(counts.miss_after, counts.hits_after);  in resctrl_arch_measure_l3_residency()
509 		counts.hits_after -= counts.hits_before;  in resctrl_arch_measure_l3_residency()
514 	plr->thread_done = 1;  in resctrl_arch_measure_l3_residency()
515 	wake_up_interruptible(&plr->lock_thread_wq);  in resctrl_arch_measure_l3_residency()