1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/cpu.h>
3 #include <linux/cpu_pm.h>
4 #include <linux/cpuhotplug.h>
5 #include <linux/notifier.h>
6 #include <linux/percpu-defs.h>
7 #include <linux/types.h>
8
9 #include <asm/cpufeature-macros.h>
10 #include <asm/hwcap.h>
11 #include <asm/qos.h>
12
13 /*
14 * Cached value of srmcfg csr for each cpu. Seeded to U32_MAX so the next
15 * __switch_to_srmcfg() unconditionally writes the CSR. The encoding
16 * MCID << 16 | RCID with both fields well under 16 bits can never
17 * produce this sentinel. This covers early-boot context switches that
18 * happen before riscv_srmcfg_init() runs as an arch_initcall.
19 */
20 DEFINE_PER_CPU(u32, cpu_srmcfg) = U32_MAX;
21
22 /* default srmcfg value for each cpu, set via resctrl cpu assignment */
23 DEFINE_PER_CPU(u32, cpu_srmcfg_default);
24
25 /*
26 * Invalidate the per-CPU srmcfg cache. Used as both the cpuhp startup
27 * and teardown callback. U32_MAX is not a valid srmcfg value
28 * (MCID << 16 | RCID, both fields under 16 bits), so the next
29 * __switch_to_srmcfg() always writes the CSR.
30 *
31 * Ssqosid leaves the CSR implementation-defined across hart stop/start,
32 * so the cached value cannot be trusted after online. The startup
33 * callback runs at CPUHP_AP_ONLINE_DYN, before CPUHP_AP_ACTIVE makes
34 * the CPU schedulable, so the cache is invalidated before any normal
35 * task runs and the CSR is written on that task's first switch.
36 * The teardown callback is not relied on. Idle and per-CPU kthreads keep
37 * switching as the CPU goes down and overwrite the sentinel with the CPU
38 * default, so it does not survive the offline period.
39 */
riscv_srmcfg_reset_cache(unsigned int cpu)40 static int riscv_srmcfg_reset_cache(unsigned int cpu)
41 {
42 per_cpu(cpu_srmcfg, cpu) = U32_MAX;
43 return 0;
44 }
45
46 /*
47 * CPU PM notifier: invalidate the cached srmcfg on resume from a deep
48 * idle / suspend. Ssqosid leaves CSR_SRMCFG state across low-power
49 * transitions implementation-defined, and the boot CPU never goes
50 * through the cpuhp online callback during system suspend, so without
51 * this hook __switch_to_srmcfg() would skip the CSR write when the
52 * outgoing task happens to share its srmcfg with the pre-suspend cache.
53 */
riscv_srmcfg_pm_notify(struct notifier_block * nb,unsigned long action,void * unused)54 static int riscv_srmcfg_pm_notify(struct notifier_block *nb,
55 unsigned long action, void *unused)
56 {
57 switch (action) {
58 case CPU_PM_EXIT:
59 case CPU_PM_ENTER_FAILED:
60 /*
61 * The CSR is implementation-defined across the low-power
62 * transition. Invalidate the cache and eagerly rewrite the
63 * CSR for the current task so it does not run mis-tagged
64 * until the next context switch.
65 */
66 __this_cpu_write(cpu_srmcfg, U32_MAX);
67 __switch_to_srmcfg(current);
68 break;
69 }
70 return NOTIFY_OK;
71 }
72
73 static struct notifier_block riscv_srmcfg_pm_nb = {
74 .notifier_call = riscv_srmcfg_pm_notify,
75 };
76
riscv_srmcfg_init(void)77 static int __init riscv_srmcfg_init(void)
78 {
79 int err;
80
81 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID))
82 return 0;
83
84 /*
85 * cpuhp_setup_state() invokes the startup callback locally on every
86 * already-online CPU, so no separate seed loop is needed here.
87 */
88 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/srmcfg:online",
89 riscv_srmcfg_reset_cache, riscv_srmcfg_reset_cache);
90 if (err < 0)
91 pr_warn("srmcfg: cpuhp setup failed (%d), cache not invalidated on CPU online\n",
92 err);
93
94 /*
95 * Register the PM notifier even if the cpuhp setup failed. It is
96 * independent of the cpuhp state and guards suspend/resume.
97 */
98 cpu_pm_register_notifier(&riscv_srmcfg_pm_nb);
99 return 0;
100 }
101 arch_initcall(riscv_srmcfg_init);
102