109e61dafSJames Morse // SPDX-License-Identifier: GPL-2.0 209e61dafSJames Morse // Copyright (C) 2025 Arm Ltd. 309e61dafSJames Morse 409e61dafSJames Morse #define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ 509e61dafSJames Morse 609e61dafSJames Morse #include <linux/arm_mpam.h> 709e61dafSJames Morse #include <linux/cacheinfo.h> 809e61dafSJames Morse #include <linux/cpu.h> 909e61dafSJames Morse #include <linux/cpumask.h> 1009e61dafSJames Morse #include <linux/errno.h> 119d2e1a99SJames Morse #include <linux/limits.h> 1209e61dafSJames Morse #include <linux/list.h> 13*80d147d2SDave Martin #include <linux/math.h> 1409e61dafSJames Morse #include <linux/printk.h> 1509e61dafSJames Morse #include <linux/rculist.h> 1609e61dafSJames Morse #include <linux/resctrl.h> 1709e61dafSJames Morse #include <linux/slab.h> 1809e61dafSJames Morse #include <linux/types.h> 1909e61dafSJames Morse 2009e61dafSJames Morse #include <asm/mpam.h> 2109e61dafSJames Morse 2209e61dafSJames Morse #include "mpam_internal.h" 2309e61dafSJames Morse 2409e61dafSJames Morse /* 2509e61dafSJames Morse * The classes we've picked to map to resctrl resources, wrapped 2609e61dafSJames Morse * in with their resctrl structure. 2709e61dafSJames Morse * Class pointer may be NULL. 2809e61dafSJames Morse */ 2909e61dafSJames Morse static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; 3009e61dafSJames Morse 3109e61dafSJames Morse #define for_each_mpam_resctrl_control(res, rid) \ 3209e61dafSJames Morse for (rid = 0, res = &mpam_resctrl_controls[rid]; \ 3309e61dafSJames Morse rid < RDT_NUM_RESOURCES; \ 3409e61dafSJames Morse rid++, res = &mpam_resctrl_controls[rid]) 3509e61dafSJames Morse 3609e61dafSJames Morse /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ 3709e61dafSJames Morse static DEFINE_MUTEX(domain_list_lock); 3809e61dafSJames Morse 396789fb99SJames Morse /* 406789fb99SJames Morse * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1. 416789fb99SJames Morse * This applies globally to all traffic the CPU generates. 426789fb99SJames Morse */ 439d2e1a99SJames Morse static bool cdp_enabled; 449d2e1a99SJames Morse 4509e61dafSJames Morse bool resctrl_arch_alloc_capable(void) 4609e61dafSJames Morse { 4709e61dafSJames Morse struct mpam_resctrl_res *res; 4809e61dafSJames Morse enum resctrl_res_level rid; 4909e61dafSJames Morse 5009e61dafSJames Morse for_each_mpam_resctrl_control(res, rid) { 5109e61dafSJames Morse if (res->resctrl_res.alloc_capable) 5209e61dafSJames Morse return true; 5309e61dafSJames Morse } 5409e61dafSJames Morse 5509e61dafSJames Morse return false; 5609e61dafSJames Morse } 5709e61dafSJames Morse 586789fb99SJames Morse bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) 596789fb99SJames Morse { 606789fb99SJames Morse return mpam_resctrl_controls[rid].cdp_enabled; 616789fb99SJames Morse } 626789fb99SJames Morse 636789fb99SJames Morse /** 646789fb99SJames Morse * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks. 656789fb99SJames Morse * 666789fb99SJames Morse * At boot, all existing tasks use partid zero for D and I. 676789fb99SJames Morse * To enable/disable CDP emulation, all these tasks need relabelling. 686789fb99SJames Morse */ 696789fb99SJames Morse static void resctrl_reset_task_closids(void) 706789fb99SJames Morse { 716789fb99SJames Morse struct task_struct *p, *t; 726789fb99SJames Morse 736789fb99SJames Morse read_lock(&tasklist_lock); 746789fb99SJames Morse for_each_process_thread(p, t) { 756789fb99SJames Morse resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID, 766789fb99SJames Morse RESCTRL_RESERVED_RMID); 776789fb99SJames Morse } 786789fb99SJames Morse read_unlock(&tasklist_lock); 796789fb99SJames Morse } 806789fb99SJames Morse 816789fb99SJames Morse int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) 826789fb99SJames Morse { 836789fb99SJames Morse u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; 846789fb99SJames Morse int cpu; 856789fb99SJames Morse 8601a0021fSBen Horgan if (!IS_ENABLED(CONFIG_EXPERT) && enable) { 8701a0021fSBen Horgan /* 8801a0021fSBen Horgan * If the resctrl fs is mounted more than once, sequentially, 8901a0021fSBen Horgan * then CDP can lead to the use of out of range PARTIDs. 9001a0021fSBen Horgan */ 9101a0021fSBen Horgan pr_warn("CDP not supported\n"); 9201a0021fSBen Horgan return -EOPNOTSUPP; 9301a0021fSBen Horgan } 9401a0021fSBen Horgan 9501a0021fSBen Horgan if (enable) 9601a0021fSBen Horgan pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n"); 9701a0021fSBen Horgan 986789fb99SJames Morse /* 996789fb99SJames Morse * resctrl_arch_set_cdp_enabled() is only called with enable set to 1006789fb99SJames Morse * false on error and unmount. 1016789fb99SJames Morse */ 1026789fb99SJames Morse cdp_enabled = enable; 1036789fb99SJames Morse mpam_resctrl_controls[rid].cdp_enabled = enable; 1046789fb99SJames Morse 1056789fb99SJames Morse /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ 1066789fb99SJames Morse if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) 1076789fb99SJames Morse mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; 1086789fb99SJames Morse 1096789fb99SJames Morse if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && 1106789fb99SJames Morse mpam_resctrl_controls[RDT_RESOURCE_MBA].class) 1116789fb99SJames Morse mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; 1126789fb99SJames Morse 1136789fb99SJames Morse if (enable) { 1146789fb99SJames Morse if (mpam_partid_max < 1) 1156789fb99SJames Morse return -EINVAL; 1166789fb99SJames Morse 1176789fb99SJames Morse partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA); 1186789fb99SJames Morse partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE); 1196789fb99SJames Morse } 1206789fb99SJames Morse 1216789fb99SJames Morse mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0); 1226789fb99SJames Morse WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); 1236789fb99SJames Morse 1246789fb99SJames Morse resctrl_reset_task_closids(); 1256789fb99SJames Morse 1266789fb99SJames Morse for_each_possible_cpu(cpu) 1276789fb99SJames Morse mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); 1286789fb99SJames Morse on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1); 1296789fb99SJames Morse 1306789fb99SJames Morse return 0; 1316789fb99SJames Morse } 1326789fb99SJames Morse 1336789fb99SJames Morse static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid) 1346789fb99SJames Morse { 1356789fb99SJames Morse return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid); 1366789fb99SJames Morse } 1376789fb99SJames Morse 13809e61dafSJames Morse /* 13909e61dafSJames Morse * MSC may raise an error interrupt if it sees an out or range partid/pmg, 14009e61dafSJames Morse * and go on to truncate the value. Regardless of what the hardware supports, 14109e61dafSJames Morse * only the system wide safe value is safe to use. 14209e61dafSJames Morse */ 14309e61dafSJames Morse u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) 14409e61dafSJames Morse { 14509e61dafSJames Morse return mpam_partid_max + 1; 14609e61dafSJames Morse } 14709e61dafSJames Morse 1489d2e1a99SJames Morse void resctrl_arch_sched_in(struct task_struct *tsk) 1499d2e1a99SJames Morse { 1509d2e1a99SJames Morse lockdep_assert_preemption_disabled(); 1519d2e1a99SJames Morse 1529d2e1a99SJames Morse mpam_thread_switch(tsk); 1539d2e1a99SJames Morse } 1549d2e1a99SJames Morse 1559d2e1a99SJames Morse void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid) 1569d2e1a99SJames Morse { 1579d2e1a99SJames Morse WARN_ON_ONCE(closid > U16_MAX); 1589d2e1a99SJames Morse WARN_ON_ONCE(rmid > U8_MAX); 1599d2e1a99SJames Morse 1609d2e1a99SJames Morse if (!cdp_enabled) { 1619d2e1a99SJames Morse mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid); 1629d2e1a99SJames Morse } else { 1639d2e1a99SJames Morse /* 1649d2e1a99SJames Morse * When CDP is enabled, resctrl halves the closid range and we 1659d2e1a99SJames Morse * use odd/even partid for one closid. 1669d2e1a99SJames Morse */ 1679d2e1a99SJames Morse u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); 1689d2e1a99SJames Morse u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); 1699d2e1a99SJames Morse 1709d2e1a99SJames Morse mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid); 1719d2e1a99SJames Morse } 1729d2e1a99SJames Morse } 1739d2e1a99SJames Morse 1749d2e1a99SJames Morse void resctrl_arch_sync_cpu_closid_rmid(void *info) 1759d2e1a99SJames Morse { 1769d2e1a99SJames Morse struct resctrl_cpu_defaults *r = info; 1779d2e1a99SJames Morse 1789d2e1a99SJames Morse lockdep_assert_preemption_disabled(); 1799d2e1a99SJames Morse 1809d2e1a99SJames Morse if (r) { 1819d2e1a99SJames Morse resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), 1829d2e1a99SJames Morse r->closid, r->rmid); 1839d2e1a99SJames Morse } 1849d2e1a99SJames Morse 1859d2e1a99SJames Morse resctrl_arch_sched_in(current); 1869d2e1a99SJames Morse } 1879d2e1a99SJames Morse 1889d2e1a99SJames Morse void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) 1899d2e1a99SJames Morse { 1909d2e1a99SJames Morse WARN_ON_ONCE(closid > U16_MAX); 1919d2e1a99SJames Morse WARN_ON_ONCE(rmid > U8_MAX); 1929d2e1a99SJames Morse 1939d2e1a99SJames Morse if (!cdp_enabled) { 1949d2e1a99SJames Morse mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); 1959d2e1a99SJames Morse } else { 1969d2e1a99SJames Morse u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); 1979d2e1a99SJames Morse u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); 1989d2e1a99SJames Morse 1999d2e1a99SJames Morse mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); 2009d2e1a99SJames Morse } 2019d2e1a99SJames Morse } 2029d2e1a99SJames Morse 2036789fb99SJames Morse bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) 2046789fb99SJames Morse { 2056789fb99SJames Morse u64 regval = mpam_get_regval(tsk); 2066789fb99SJames Morse u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); 2076789fb99SJames Morse 2086789fb99SJames Morse if (cdp_enabled) 2096789fb99SJames Morse tsk_closid >>= 1; 2106789fb99SJames Morse 2116789fb99SJames Morse return tsk_closid == closid; 2126789fb99SJames Morse } 2136789fb99SJames Morse 2146789fb99SJames Morse /* The task's pmg is not unique, the partid must be considered too */ 2156789fb99SJames Morse bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) 2166789fb99SJames Morse { 2176789fb99SJames Morse u64 regval = mpam_get_regval(tsk); 2186789fb99SJames Morse u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); 2196789fb99SJames Morse u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval); 2206789fb99SJames Morse 2216789fb99SJames Morse if (cdp_enabled) 2226789fb99SJames Morse tsk_closid >>= 1; 2236789fb99SJames Morse 2246789fb99SJames Morse return (tsk_closid == closid) && (tsk_rmid == rmid); 2256789fb99SJames Morse } 2266789fb99SJames Morse 22709e61dafSJames Morse struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) 22809e61dafSJames Morse { 22909e61dafSJames Morse if (l >= RDT_NUM_RESOURCES) 23009e61dafSJames Morse return NULL; 23109e61dafSJames Morse 23209e61dafSJames Morse return &mpam_resctrl_controls[l].resctrl_res; 23309e61dafSJames Morse } 23409e61dafSJames Morse 23552a4edb1SJames Morse static bool cache_has_usable_cpor(struct mpam_class *class) 23652a4edb1SJames Morse { 23752a4edb1SJames Morse struct mpam_props *cprops = &class->props; 23852a4edb1SJames Morse 23952a4edb1SJames Morse if (!mpam_has_feature(mpam_feat_cpor_part, cprops)) 24052a4edb1SJames Morse return false; 24152a4edb1SJames Morse 24252a4edb1SJames Morse /* resctrl uses u32 for all bitmap configurations */ 24352a4edb1SJames Morse return class->props.cpbm_wd <= 32; 24452a4edb1SJames Morse } 24552a4edb1SJames Morse 246*80d147d2SDave Martin /* 247*80d147d2SDave Martin * Each fixed-point hardware value architecturally represents a range 248*80d147d2SDave Martin * of values: the full range 0% - 100% is split contiguously into 249*80d147d2SDave Martin * (1 << cprops->bwa_wd) equal bands. 250*80d147d2SDave Martin * 251*80d147d2SDave Martin * Although the bwa_bwd fields have 6 bits the maximum valid value is 16 252*80d147d2SDave Martin * as it reports the width of fields that are at most 16 bits. When 253*80d147d2SDave Martin * fewer than 16 bits are valid the least significant bits are 254*80d147d2SDave Martin * ignored. The implied binary point is kept between bits 15 and 16 and 255*80d147d2SDave Martin * so the valid bits are leftmost. 256*80d147d2SDave Martin * 257*80d147d2SDave Martin * See ARM IHI0099B.a "MPAM system component specification", Section 9.3, 258*80d147d2SDave Martin * "The fixed-point fractional format" for more information. 259*80d147d2SDave Martin * 260*80d147d2SDave Martin * Find the nearest percentage value to the upper bound of the selected band: 261*80d147d2SDave Martin */ 262*80d147d2SDave Martin static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) 263*80d147d2SDave Martin { 264*80d147d2SDave Martin u32 val = mbw_max; 265*80d147d2SDave Martin 266*80d147d2SDave Martin val >>= 16 - cprops->bwa_wd; 267*80d147d2SDave Martin val += 1; 268*80d147d2SDave Martin val *= MAX_MBA_BW; 269*80d147d2SDave Martin val = DIV_ROUND_CLOSEST(val, 1 << cprops->bwa_wd); 270*80d147d2SDave Martin 271*80d147d2SDave Martin return val; 272*80d147d2SDave Martin } 273*80d147d2SDave Martin 274*80d147d2SDave Martin /* 275*80d147d2SDave Martin * Find the band whose upper bound is closest to the specified percentage. 276*80d147d2SDave Martin * 277*80d147d2SDave Martin * A round-to-nearest policy is followed here as a balanced compromise 278*80d147d2SDave Martin * between unexpected under-commit of the resource (where the total of 279*80d147d2SDave Martin * a set of resource allocations after conversion is less than the 280*80d147d2SDave Martin * expected total, due to rounding of the individual converted 281*80d147d2SDave Martin * percentages) and over-commit (where the total of the converted 282*80d147d2SDave Martin * allocations is greater than expected). 283*80d147d2SDave Martin */ 284*80d147d2SDave Martin static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) 285*80d147d2SDave Martin { 286*80d147d2SDave Martin u32 val = pc; 287*80d147d2SDave Martin 288*80d147d2SDave Martin val <<= cprops->bwa_wd; 289*80d147d2SDave Martin val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW); 290*80d147d2SDave Martin val = max(val, 1) - 1; 291*80d147d2SDave Martin val <<= 16 - cprops->bwa_wd; 292*80d147d2SDave Martin 293*80d147d2SDave Martin return val; 294*80d147d2SDave Martin } 295*80d147d2SDave Martin 29652a4edb1SJames Morse /* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ 29752a4edb1SJames Morse static void mpam_resctrl_pick_caches(void) 29852a4edb1SJames Morse { 29952a4edb1SJames Morse struct mpam_class *class; 30052a4edb1SJames Morse struct mpam_resctrl_res *res; 30152a4edb1SJames Morse 30252a4edb1SJames Morse lockdep_assert_cpus_held(); 30352a4edb1SJames Morse 30452a4edb1SJames Morse guard(srcu)(&mpam_srcu); 30552a4edb1SJames Morse list_for_each_entry_srcu(class, &mpam_classes, classes_list, 30652a4edb1SJames Morse srcu_read_lock_held(&mpam_srcu)) { 30752a4edb1SJames Morse if (class->type != MPAM_CLASS_CACHE) { 30852a4edb1SJames Morse pr_debug("class %u is not a cache\n", class->level); 30952a4edb1SJames Morse continue; 31052a4edb1SJames Morse } 31152a4edb1SJames Morse 31252a4edb1SJames Morse if (class->level != 2 && class->level != 3) { 31352a4edb1SJames Morse pr_debug("class %u is not L2 or L3\n", class->level); 31452a4edb1SJames Morse continue; 31552a4edb1SJames Morse } 31652a4edb1SJames Morse 31752a4edb1SJames Morse if (!cache_has_usable_cpor(class)) { 31852a4edb1SJames Morse pr_debug("class %u cache misses CPOR\n", class->level); 31952a4edb1SJames Morse continue; 32052a4edb1SJames Morse } 32152a4edb1SJames Morse 32252a4edb1SJames Morse if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { 32352a4edb1SJames Morse pr_debug("class %u has missing CPUs, mask %*pb != %*pb\n", class->level, 32452a4edb1SJames Morse cpumask_pr_args(&class->affinity), 32552a4edb1SJames Morse cpumask_pr_args(cpu_possible_mask)); 32652a4edb1SJames Morse continue; 32752a4edb1SJames Morse } 32852a4edb1SJames Morse 32952a4edb1SJames Morse if (class->level == 2) 33052a4edb1SJames Morse res = &mpam_resctrl_controls[RDT_RESOURCE_L2]; 33152a4edb1SJames Morse else 33252a4edb1SJames Morse res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; 33352a4edb1SJames Morse res->class = class; 33452a4edb1SJames Morse } 33552a4edb1SJames Morse } 33652a4edb1SJames Morse 33709e61dafSJames Morse static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) 33809e61dafSJames Morse { 33952a4edb1SJames Morse struct mpam_class *class = res->class; 34052a4edb1SJames Morse struct rdt_resource *r = &res->resctrl_res; 34152a4edb1SJames Morse 34252a4edb1SJames Morse switch (r->rid) { 34352a4edb1SJames Morse case RDT_RESOURCE_L2: 34452a4edb1SJames Morse case RDT_RESOURCE_L3: 34552a4edb1SJames Morse r->schema_fmt = RESCTRL_SCHEMA_BITMAP; 34652a4edb1SJames Morse r->cache.arch_has_sparse_bitmasks = true; 34752a4edb1SJames Morse 34852a4edb1SJames Morse r->cache.cbm_len = class->props.cpbm_wd; 34952a4edb1SJames Morse /* mpam_devices will reject empty bitmaps */ 35052a4edb1SJames Morse r->cache.min_cbm_bits = 1; 35152a4edb1SJames Morse 35252a4edb1SJames Morse if (r->rid == RDT_RESOURCE_L2) { 35352a4edb1SJames Morse r->name = "L2"; 35452a4edb1SJames Morse r->ctrl_scope = RESCTRL_L2_CACHE; 35552a4edb1SJames Morse r->cdp_capable = true; 35652a4edb1SJames Morse } else { 35752a4edb1SJames Morse r->name = "L3"; 35852a4edb1SJames Morse r->ctrl_scope = RESCTRL_L3_CACHE; 35952a4edb1SJames Morse r->cdp_capable = true; 36052a4edb1SJames Morse } 36152a4edb1SJames Morse 36252a4edb1SJames Morse /* 36352a4edb1SJames Morse * Which bits are shared with other ...things... Unknown 36452a4edb1SJames Morse * devices use partid-0 which uses all the bitmap fields. Until 36552a4edb1SJames Morse * we have configured the SMMU and GIC not to do this 'all the 36652a4edb1SJames Morse * bits' is the correct answer here. 36752a4edb1SJames Morse */ 36852a4edb1SJames Morse r->cache.shareable_bits = resctrl_get_default_ctrl(r); 36952a4edb1SJames Morse r->alloc_capable = true; 37052a4edb1SJames Morse break; 37152a4edb1SJames Morse default: 37252a4edb1SJames Morse return -EINVAL; 37352a4edb1SJames Morse } 37409e61dafSJames Morse 37509e61dafSJames Morse return 0; 37609e61dafSJames Morse } 37709e61dafSJames Morse 37809e61dafSJames Morse static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) 37909e61dafSJames Morse { 38009e61dafSJames Morse struct mpam_class *class = comp->class; 38109e61dafSJames Morse 38209e61dafSJames Morse if (class->type == MPAM_CLASS_CACHE) 38309e61dafSJames Morse return comp->comp_id; 38409e61dafSJames Morse 38509e61dafSJames Morse /* TODO: repaint domain ids to match the L3 domain ids */ 38609e61dafSJames Morse /* Otherwise, expose the ID used by the firmware table code. */ 38709e61dafSJames Morse return comp->comp_id; 38809e61dafSJames Morse } 38909e61dafSJames Morse 39002cc6616SJames Morse u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, 39102cc6616SJames Morse u32 closid, enum resctrl_conf_type type) 39202cc6616SJames Morse { 39302cc6616SJames Morse u32 partid; 39402cc6616SJames Morse struct mpam_config *cfg; 39502cc6616SJames Morse struct mpam_props *cprops; 39602cc6616SJames Morse struct mpam_resctrl_res *res; 39702cc6616SJames Morse struct mpam_resctrl_dom *dom; 39802cc6616SJames Morse enum mpam_device_features configured_by; 39902cc6616SJames Morse 40002cc6616SJames Morse lockdep_assert_cpus_held(); 40102cc6616SJames Morse 40202cc6616SJames Morse if (!mpam_is_enabled()) 40302cc6616SJames Morse return resctrl_get_default_ctrl(r); 40402cc6616SJames Morse 40502cc6616SJames Morse res = container_of(r, struct mpam_resctrl_res, resctrl_res); 40602cc6616SJames Morse dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); 40702cc6616SJames Morse cprops = &res->class->props; 40802cc6616SJames Morse 4096789fb99SJames Morse /* 4106789fb99SJames Morse * When CDP is enabled, but the resource doesn't support it, 4116789fb99SJames Morse * the control is cloned across both partids. 4126789fb99SJames Morse * Pick one at random to read: 4136789fb99SJames Morse */ 4146789fb99SJames Morse if (mpam_resctrl_hide_cdp(r->rid)) 4156789fb99SJames Morse type = CDP_DATA; 4166789fb99SJames Morse 41702cc6616SJames Morse partid = resctrl_get_config_index(closid, type); 41802cc6616SJames Morse cfg = &dom->ctrl_comp->cfg[partid]; 41902cc6616SJames Morse 42002cc6616SJames Morse switch (r->rid) { 42102cc6616SJames Morse case RDT_RESOURCE_L2: 42202cc6616SJames Morse case RDT_RESOURCE_L3: 42302cc6616SJames Morse configured_by = mpam_feat_cpor_part; 42402cc6616SJames Morse break; 42502cc6616SJames Morse default: 42602cc6616SJames Morse return resctrl_get_default_ctrl(r); 42702cc6616SJames Morse } 42802cc6616SJames Morse 42902cc6616SJames Morse if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) || 43002cc6616SJames Morse !mpam_has_feature(configured_by, cfg)) 43102cc6616SJames Morse return resctrl_get_default_ctrl(r); 43202cc6616SJames Morse 43302cc6616SJames Morse switch (configured_by) { 43402cc6616SJames Morse case mpam_feat_cpor_part: 43502cc6616SJames Morse return cfg->cpbm; 43602cc6616SJames Morse default: 43702cc6616SJames Morse return resctrl_get_default_ctrl(r); 43802cc6616SJames Morse } 43902cc6616SJames Morse } 44002cc6616SJames Morse 4419cd2b522SJames Morse int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, 4429cd2b522SJames Morse u32 closid, enum resctrl_conf_type t, u32 cfg_val) 4439cd2b522SJames Morse { 4446789fb99SJames Morse int err; 4459cd2b522SJames Morse u32 partid; 4469cd2b522SJames Morse struct mpam_config cfg; 4479cd2b522SJames Morse struct mpam_props *cprops; 4489cd2b522SJames Morse struct mpam_resctrl_res *res; 4499cd2b522SJames Morse struct mpam_resctrl_dom *dom; 4509cd2b522SJames Morse 4519cd2b522SJames Morse lockdep_assert_cpus_held(); 4529cd2b522SJames Morse lockdep_assert_irqs_enabled(); 4539cd2b522SJames Morse 4549cd2b522SJames Morse /* 4559cd2b522SJames Morse * No need to check the CPU as mpam_apply_config() doesn't care, and 4569cd2b522SJames Morse * resctrl_arch_update_domains() relies on this. 4579cd2b522SJames Morse */ 4589cd2b522SJames Morse res = container_of(r, struct mpam_resctrl_res, resctrl_res); 4599cd2b522SJames Morse dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); 4609cd2b522SJames Morse cprops = &res->class->props; 4619cd2b522SJames Morse 4626789fb99SJames Morse if (mpam_resctrl_hide_cdp(r->rid)) 4636789fb99SJames Morse t = CDP_DATA; 4646789fb99SJames Morse 4659cd2b522SJames Morse partid = resctrl_get_config_index(closid, t); 4669cd2b522SJames Morse if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) { 4679cd2b522SJames Morse pr_debug("Not alloc capable or computed PARTID out of range\n"); 4689cd2b522SJames Morse return -EINVAL; 4699cd2b522SJames Morse } 4709cd2b522SJames Morse 4719cd2b522SJames Morse /* 4729cd2b522SJames Morse * Copy the current config to avoid clearing other resources when the 4739cd2b522SJames Morse * same component is exposed multiple times through resctrl. 4749cd2b522SJames Morse */ 4759cd2b522SJames Morse cfg = dom->ctrl_comp->cfg[partid]; 4769cd2b522SJames Morse 4779cd2b522SJames Morse switch (r->rid) { 4789cd2b522SJames Morse case RDT_RESOURCE_L2: 4799cd2b522SJames Morse case RDT_RESOURCE_L3: 4809cd2b522SJames Morse cfg.cpbm = cfg_val; 4819cd2b522SJames Morse mpam_set_feature(mpam_feat_cpor_part, &cfg); 4829cd2b522SJames Morse break; 4839cd2b522SJames Morse default: 4849cd2b522SJames Morse return -EINVAL; 4859cd2b522SJames Morse } 4869cd2b522SJames Morse 4876789fb99SJames Morse /* 4886789fb99SJames Morse * When CDP is enabled, but the resource doesn't support it, we need to 4896789fb99SJames Morse * apply the same configuration to the other partid. 4906789fb99SJames Morse */ 4916789fb99SJames Morse if (mpam_resctrl_hide_cdp(r->rid)) { 4926789fb99SJames Morse partid = resctrl_get_config_index(closid, CDP_CODE); 4936789fb99SJames Morse err = mpam_apply_config(dom->ctrl_comp, partid, &cfg); 4946789fb99SJames Morse if (err) 4956789fb99SJames Morse return err; 4966789fb99SJames Morse 4976789fb99SJames Morse partid = resctrl_get_config_index(closid, CDP_DATA); 4986789fb99SJames Morse return mpam_apply_config(dom->ctrl_comp, partid, &cfg); 4996789fb99SJames Morse } 5006789fb99SJames Morse 5019cd2b522SJames Morse return mpam_apply_config(dom->ctrl_comp, partid, &cfg); 5029cd2b522SJames Morse } 5039cd2b522SJames Morse 5049cd2b522SJames Morse int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) 5059cd2b522SJames Morse { 5069cd2b522SJames Morse int err; 5079cd2b522SJames Morse struct rdt_ctrl_domain *d; 5089cd2b522SJames Morse 5099cd2b522SJames Morse lockdep_assert_cpus_held(); 5109cd2b522SJames Morse lockdep_assert_irqs_enabled(); 5119cd2b522SJames Morse 5129cd2b522SJames Morse list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { 5139cd2b522SJames Morse for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) { 5149cd2b522SJames Morse struct resctrl_staged_config *cfg = &d->staged_config[t]; 5159cd2b522SJames Morse 5169cd2b522SJames Morse if (!cfg->have_new_ctrl) 5179cd2b522SJames Morse continue; 5189cd2b522SJames Morse 5199cd2b522SJames Morse err = resctrl_arch_update_one(r, d, closid, t, 5209cd2b522SJames Morse cfg->new_ctrl); 5219cd2b522SJames Morse if (err) 5229cd2b522SJames Morse return err; 5239cd2b522SJames Morse } 5249cd2b522SJames Morse } 5259cd2b522SJames Morse 5269cd2b522SJames Morse return 0; 5279cd2b522SJames Morse } 5289cd2b522SJames Morse 529370d166dSJames Morse void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) 530370d166dSJames Morse { 531370d166dSJames Morse struct mpam_resctrl_res *res; 532370d166dSJames Morse 533370d166dSJames Morse lockdep_assert_cpus_held(); 534370d166dSJames Morse 535370d166dSJames Morse if (!mpam_is_enabled()) 536370d166dSJames Morse return; 537370d166dSJames Morse 538370d166dSJames Morse res = container_of(r, struct mpam_resctrl_res, resctrl_res); 539370d166dSJames Morse mpam_reset_class_locked(res->class); 540370d166dSJames Morse } 541370d166dSJames Morse 54209e61dafSJames Morse static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp, 54309e61dafSJames Morse enum resctrl_res_level rid, 54409e61dafSJames Morse struct rdt_domain_hdr *hdr) 54509e61dafSJames Morse { 54609e61dafSJames Morse lockdep_assert_cpus_held(); 54709e61dafSJames Morse 54809e61dafSJames Morse INIT_LIST_HEAD(&hdr->list); 54909e61dafSJames Morse hdr->id = mpam_resctrl_pick_domain_id(cpu, comp); 55009e61dafSJames Morse hdr->rid = rid; 55109e61dafSJames Morse cpumask_set_cpu(cpu, &hdr->cpu_mask); 55209e61dafSJames Morse } 55309e61dafSJames Morse 55409e61dafSJames Morse static void mpam_resctrl_online_domain_hdr(unsigned int cpu, 55509e61dafSJames Morse struct rdt_domain_hdr *hdr) 55609e61dafSJames Morse { 55709e61dafSJames Morse lockdep_assert_cpus_held(); 55809e61dafSJames Morse 55909e61dafSJames Morse cpumask_set_cpu(cpu, &hdr->cpu_mask); 56009e61dafSJames Morse } 56109e61dafSJames Morse 56209e61dafSJames Morse /** 56309e61dafSJames Morse * mpam_resctrl_offline_domain_hdr() - Update the domain header to remove a CPU. 56409e61dafSJames Morse * @cpu: The CPU to remove from the domain. 56509e61dafSJames Morse * @hdr: The domain's header. 56609e61dafSJames Morse * 56709e61dafSJames Morse * Removes @cpu from the header mask. If this was the last CPU in the domain, 56809e61dafSJames Morse * the domain header is removed from its parent list and true is returned, 56909e61dafSJames Morse * indicating the parent structure can be freed. 57009e61dafSJames Morse * If there are other CPUs in the domain, returns false. 57109e61dafSJames Morse */ 57209e61dafSJames Morse static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu, 57309e61dafSJames Morse struct rdt_domain_hdr *hdr) 57409e61dafSJames Morse { 57509e61dafSJames Morse lockdep_assert_held(&domain_list_lock); 57609e61dafSJames Morse 57709e61dafSJames Morse cpumask_clear_cpu(cpu, &hdr->cpu_mask); 57809e61dafSJames Morse if (cpumask_empty(&hdr->cpu_mask)) { 57909e61dafSJames Morse list_del_rcu(&hdr->list); 58009e61dafSJames Morse synchronize_rcu(); 58109e61dafSJames Morse return true; 58209e61dafSJames Morse } 58309e61dafSJames Morse 58409e61dafSJames Morse return false; 58509e61dafSJames Morse } 58609e61dafSJames Morse 58709e61dafSJames Morse static void mpam_resctrl_domain_insert(struct list_head *list, 58809e61dafSJames Morse struct rdt_domain_hdr *new) 58909e61dafSJames Morse { 59009e61dafSJames Morse struct rdt_domain_hdr *err; 59109e61dafSJames Morse struct list_head *pos = NULL; 59209e61dafSJames Morse 59309e61dafSJames Morse lockdep_assert_held(&domain_list_lock); 59409e61dafSJames Morse 59509e61dafSJames Morse err = resctrl_find_domain(list, new->id, &pos); 59609e61dafSJames Morse if (WARN_ON_ONCE(err)) 59709e61dafSJames Morse return; 59809e61dafSJames Morse 59909e61dafSJames Morse list_add_tail_rcu(&new->list, pos); 60009e61dafSJames Morse } 60109e61dafSJames Morse 60209e61dafSJames Morse static struct mpam_resctrl_dom * 60309e61dafSJames Morse mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) 60409e61dafSJames Morse { 60509e61dafSJames Morse int err; 60609e61dafSJames Morse struct mpam_resctrl_dom *dom; 60709e61dafSJames Morse struct rdt_ctrl_domain *ctrl_d; 60809e61dafSJames Morse struct mpam_class *class = res->class; 60909e61dafSJames Morse struct mpam_component *comp_iter, *ctrl_comp; 61009e61dafSJames Morse struct rdt_resource *r = &res->resctrl_res; 61109e61dafSJames Morse 61209e61dafSJames Morse lockdep_assert_held(&domain_list_lock); 61309e61dafSJames Morse 61409e61dafSJames Morse ctrl_comp = NULL; 61509e61dafSJames Morse guard(srcu)(&mpam_srcu); 61609e61dafSJames Morse list_for_each_entry_srcu(comp_iter, &class->components, class_list, 61709e61dafSJames Morse srcu_read_lock_held(&mpam_srcu)) { 61809e61dafSJames Morse if (cpumask_test_cpu(cpu, &comp_iter->affinity)) { 61909e61dafSJames Morse ctrl_comp = comp_iter; 62009e61dafSJames Morse break; 62109e61dafSJames Morse } 62209e61dafSJames Morse } 62309e61dafSJames Morse 62409e61dafSJames Morse /* class has no component for this CPU */ 62509e61dafSJames Morse if (WARN_ON_ONCE(!ctrl_comp)) 62609e61dafSJames Morse return ERR_PTR(-EINVAL); 62709e61dafSJames Morse 62809e61dafSJames Morse dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu)); 62909e61dafSJames Morse if (!dom) 63009e61dafSJames Morse return ERR_PTR(-ENOMEM); 63109e61dafSJames Morse 63209e61dafSJames Morse if (r->alloc_capable) { 63309e61dafSJames Morse dom->ctrl_comp = ctrl_comp; 63409e61dafSJames Morse 63509e61dafSJames Morse ctrl_d = &dom->resctrl_ctrl_dom; 63609e61dafSJames Morse mpam_resctrl_domain_hdr_init(cpu, ctrl_comp, r->rid, &ctrl_d->hdr); 63709e61dafSJames Morse ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN; 63809e61dafSJames Morse err = resctrl_online_ctrl_domain(r, ctrl_d); 63909e61dafSJames Morse if (err) 64009e61dafSJames Morse goto free_domain; 64109e61dafSJames Morse 64209e61dafSJames Morse mpam_resctrl_domain_insert(&r->ctrl_domains, &ctrl_d->hdr); 64309e61dafSJames Morse } else { 64409e61dafSJames Morse pr_debug("Skipped control domain online - no controls\n"); 64509e61dafSJames Morse } 64609e61dafSJames Morse return dom; 64709e61dafSJames Morse 64809e61dafSJames Morse free_domain: 64909e61dafSJames Morse kfree(dom); 65009e61dafSJames Morse dom = ERR_PTR(err); 65109e61dafSJames Morse 65209e61dafSJames Morse return dom; 65309e61dafSJames Morse } 65409e61dafSJames Morse 65509e61dafSJames Morse static struct mpam_resctrl_dom * 65609e61dafSJames Morse mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) 65709e61dafSJames Morse { 65809e61dafSJames Morse struct mpam_resctrl_dom *dom; 65909e61dafSJames Morse struct rdt_resource *r = &res->resctrl_res; 66009e61dafSJames Morse 66109e61dafSJames Morse lockdep_assert_cpus_held(); 66209e61dafSJames Morse 66309e61dafSJames Morse list_for_each_entry_rcu(dom, &r->ctrl_domains, resctrl_ctrl_dom.hdr.list) { 66409e61dafSJames Morse if (cpumask_test_cpu(cpu, &dom->ctrl_comp->affinity)) 66509e61dafSJames Morse return dom; 66609e61dafSJames Morse } 66709e61dafSJames Morse 66809e61dafSJames Morse return NULL; 66909e61dafSJames Morse } 67009e61dafSJames Morse 67109e61dafSJames Morse int mpam_resctrl_online_cpu(unsigned int cpu) 67209e61dafSJames Morse { 67309e61dafSJames Morse struct mpam_resctrl_res *res; 67409e61dafSJames Morse enum resctrl_res_level rid; 67509e61dafSJames Morse 67609e61dafSJames Morse guard(mutex)(&domain_list_lock); 67709e61dafSJames Morse for_each_mpam_resctrl_control(res, rid) { 67809e61dafSJames Morse struct mpam_resctrl_dom *dom; 67909e61dafSJames Morse struct rdt_resource *r = &res->resctrl_res; 68009e61dafSJames Morse 68109e61dafSJames Morse if (!res->class) 68209e61dafSJames Morse continue; // dummy_resource; 68309e61dafSJames Morse 68409e61dafSJames Morse dom = mpam_resctrl_get_domain_from_cpu(cpu, res); 68509e61dafSJames Morse if (!dom) { 68609e61dafSJames Morse dom = mpam_resctrl_alloc_domain(cpu, res); 68709e61dafSJames Morse if (IS_ERR(dom)) 68809e61dafSJames Morse return PTR_ERR(dom); 68909e61dafSJames Morse } else { 69009e61dafSJames Morse if (r->alloc_capable) { 69109e61dafSJames Morse struct rdt_ctrl_domain *ctrl_d = &dom->resctrl_ctrl_dom; 69209e61dafSJames Morse 69309e61dafSJames Morse mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr); 69409e61dafSJames Morse } 69509e61dafSJames Morse } 69609e61dafSJames Morse } 69709e61dafSJames Morse 69809e61dafSJames Morse resctrl_online_cpu(cpu); 69909e61dafSJames Morse 70009e61dafSJames Morse return 0; 70109e61dafSJames Morse } 70209e61dafSJames Morse 70309e61dafSJames Morse void mpam_resctrl_offline_cpu(unsigned int cpu) 70409e61dafSJames Morse { 70509e61dafSJames Morse struct mpam_resctrl_res *res; 70609e61dafSJames Morse enum resctrl_res_level rid; 70709e61dafSJames Morse 70809e61dafSJames Morse resctrl_offline_cpu(cpu); 70909e61dafSJames Morse 71009e61dafSJames Morse guard(mutex)(&domain_list_lock); 71109e61dafSJames Morse for_each_mpam_resctrl_control(res, rid) { 71209e61dafSJames Morse struct mpam_resctrl_dom *dom; 71309e61dafSJames Morse struct rdt_ctrl_domain *ctrl_d; 71409e61dafSJames Morse bool ctrl_dom_empty; 71509e61dafSJames Morse struct rdt_resource *r = &res->resctrl_res; 71609e61dafSJames Morse 71709e61dafSJames Morse if (!res->class) 71809e61dafSJames Morse continue; // dummy resource 71909e61dafSJames Morse 72009e61dafSJames Morse dom = mpam_resctrl_get_domain_from_cpu(cpu, res); 72109e61dafSJames Morse if (WARN_ON_ONCE(!dom)) 72209e61dafSJames Morse continue; 72309e61dafSJames Morse 72409e61dafSJames Morse if (r->alloc_capable) { 72509e61dafSJames Morse ctrl_d = &dom->resctrl_ctrl_dom; 72609e61dafSJames Morse ctrl_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr); 72709e61dafSJames Morse if (ctrl_dom_empty) 72809e61dafSJames Morse resctrl_offline_ctrl_domain(&res->resctrl_res, ctrl_d); 72909e61dafSJames Morse } else { 73009e61dafSJames Morse ctrl_dom_empty = true; 73109e61dafSJames Morse } 73209e61dafSJames Morse 73309e61dafSJames Morse if (ctrl_dom_empty) 73409e61dafSJames Morse kfree(dom); 73509e61dafSJames Morse } 73609e61dafSJames Morse } 73709e61dafSJames Morse 73809e61dafSJames Morse int mpam_resctrl_setup(void) 73909e61dafSJames Morse { 74009e61dafSJames Morse int err = 0; 74109e61dafSJames Morse struct mpam_resctrl_res *res; 74209e61dafSJames Morse enum resctrl_res_level rid; 74309e61dafSJames Morse 74409e61dafSJames Morse cpus_read_lock(); 74509e61dafSJames Morse for_each_mpam_resctrl_control(res, rid) { 74609e61dafSJames Morse INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains); 74709e61dafSJames Morse res->resctrl_res.rid = rid; 74809e61dafSJames Morse } 74909e61dafSJames Morse 75052a4edb1SJames Morse /* Find some classes to use for controls */ 75152a4edb1SJames Morse mpam_resctrl_pick_caches(); 75209e61dafSJames Morse 75309e61dafSJames Morse /* Initialise the resctrl structures from the classes */ 75409e61dafSJames Morse for_each_mpam_resctrl_control(res, rid) { 75509e61dafSJames Morse if (!res->class) 75609e61dafSJames Morse continue; // dummy resource 75709e61dafSJames Morse 75809e61dafSJames Morse err = mpam_resctrl_control_init(res); 75909e61dafSJames Morse if (err) { 76009e61dafSJames Morse pr_debug("Failed to initialise rid %u\n", rid); 76109e61dafSJames Morse break; 76209e61dafSJames Morse } 76309e61dafSJames Morse } 76409e61dafSJames Morse cpus_read_unlock(); 76509e61dafSJames Morse 76609e61dafSJames Morse if (err) { 76709e61dafSJames Morse pr_debug("Internal error %d - resctrl not supported\n", err); 76809e61dafSJames Morse return err; 76909e61dafSJames Morse } 77009e61dafSJames Morse 77109e61dafSJames Morse if (!resctrl_arch_alloc_capable()) { 77209e61dafSJames Morse pr_debug("No alloc(%u) found - resctrl not supported\n", 77309e61dafSJames Morse resctrl_arch_alloc_capable()); 77409e61dafSJames Morse return -EOPNOTSUPP; 77509e61dafSJames Morse } 77609e61dafSJames Morse 77709e61dafSJames Morse /* TODO: call resctrl_init() */ 77809e61dafSJames Morse 77909e61dafSJames Morse return 0; 78009e61dafSJames Morse } 781