1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Resource Director Technology(RDT) 4 * - Cache Allocation code. 5 * 6 * Copyright (C) 2016 Intel Corporation 7 * 8 * Authors: 9 * Fenghua Yu <fenghua.yu@intel.com> 10 * Tony Luck <tony.luck@intel.com> 11 * Vikas Shivappa <vikas.shivappa@intel.com> 12 * 13 * More information about RDT be found in the Intel (R) x86 Architecture 14 * Software Developer Manual June 2016, volume 3, section 17.17. 15 */ 16 17 #define pr_fmt(fmt) "resctrl: " fmt 18 19 #include <linux/cpu.h> 20 #include <linux/slab.h> 21 #include <linux/err.h> 22 #include <linux/cpuhotplug.h> 23 24 #include <asm/cpu_device_id.h> 25 #include <asm/cpuid/api.h> 26 #include <asm/msr.h> 27 #include <asm/resctrl.h> 28 #include "internal.h" 29 30 /* 31 * rdt_domain structures are kfree()d when their last CPU goes offline, 32 * and allocated when the first CPU in a new domain comes online. 33 * The rdt_resource's domain list is updated when this happens. Readers of 34 * the domain list must either take cpus_read_lock(), or rely on an RCU 35 * read-side critical section, to avoid observing concurrent modification. 36 * All writers take this mutex: 37 */ 38 static DEFINE_MUTEX(domain_list_lock); 39 40 /* 41 * The cached resctrl_pqr_state is strictly per CPU and can never be 42 * updated from a remote CPU. Functions which modify the state 43 * are called with interrupts disabled and no preemption, which 44 * is sufficient for the protection. 45 */ 46 DEFINE_PER_CPU(struct resctrl_pqr_state, pqr_state); 47 48 /* 49 * Global boolean for rdt_alloc which is true if any 50 * resource allocation is enabled. 51 */ 52 bool rdt_alloc_capable; 53 54 static void mba_wrmsr_intel(struct msr_param *m); 55 static void cat_wrmsr(struct msr_param *m); 56 static void mba_wrmsr_amd(struct msr_param *m); 57 58 #define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains) 59 #define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains) 60 61 struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { 62 [RDT_RESOURCE_L3] = 63 { 64 .r_resctrl = { 65 .name = "L3", 66 .ctrl_scope = RESCTRL_L3_CACHE, 67 .mon_scope = RESCTRL_L3_CACHE, 68 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L3), 69 .mon_domains = mon_domain_init(RDT_RESOURCE_L3), 70 .schema_fmt = RESCTRL_SCHEMA_BITMAP, 71 }, 72 .msr_base = MSR_IA32_L3_CBM_BASE, 73 .msr_update = cat_wrmsr, 74 }, 75 [RDT_RESOURCE_L2] = 76 { 77 .r_resctrl = { 78 .name = "L2", 79 .ctrl_scope = RESCTRL_L2_CACHE, 80 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L2), 81 .schema_fmt = RESCTRL_SCHEMA_BITMAP, 82 }, 83 .msr_base = MSR_IA32_L2_CBM_BASE, 84 .msr_update = cat_wrmsr, 85 }, 86 [RDT_RESOURCE_MBA] = 87 { 88 .r_resctrl = { 89 .name = "MB", 90 .ctrl_scope = RESCTRL_L3_CACHE, 91 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA), 92 .schema_fmt = RESCTRL_SCHEMA_RANGE, 93 }, 94 }, 95 [RDT_RESOURCE_SMBA] = 96 { 97 .r_resctrl = { 98 .name = "SMBA", 99 .ctrl_scope = RESCTRL_L3_CACHE, 100 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA), 101 .schema_fmt = RESCTRL_SCHEMA_RANGE, 102 }, 103 }, 104 [RDT_RESOURCE_PERF_PKG] = 105 { 106 .r_resctrl = { 107 .name = "PERF_PKG", 108 .mon_scope = RESCTRL_PACKAGE, 109 .mon_domains = mon_domain_init(RDT_RESOURCE_PERF_PKG), 110 }, 111 }, 112 }; 113 114 /** 115 * resctrl_arch_system_num_rmid_idx - Compute number of supported RMIDs 116 * (minimum across all mon_capable resource) 117 * 118 * Return: Number of supported RMIDs at time of call. Note that mount time 119 * enumeration of resources may reduce the number. 120 */ 121 u32 resctrl_arch_system_num_rmid_idx(void) 122 { 123 u32 num_rmids = U32_MAX; 124 struct rdt_resource *r; 125 126 for_each_mon_capable_rdt_resource(r) 127 num_rmids = min(num_rmids, r->mon.num_rmid); 128 129 /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */ 130 return num_rmids == U32_MAX ? 0 : num_rmids; 131 } 132 133 struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) 134 { 135 if (l >= RDT_NUM_RESOURCES) 136 return NULL; 137 138 return &rdt_resources_all[l].r_resctrl; 139 } 140 141 /* 142 * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs 143 * as they do not have CPUID enumeration support for Cache allocation. 144 * The check for Vendor/Family/Model is not enough to guarantee that 145 * the MSRs won't #GP fault because only the following SKUs support 146 * CAT: 147 * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz 148 * Intel(R) Xeon(R) CPU E5-2648L v3 @ 1.80GHz 149 * Intel(R) Xeon(R) CPU E5-2628L v3 @ 2.00GHz 150 * Intel(R) Xeon(R) CPU E5-2618L v3 @ 2.30GHz 151 * Intel(R) Xeon(R) CPU E5-2608L v3 @ 2.00GHz 152 * Intel(R) Xeon(R) CPU E5-2658A v3 @ 2.20GHz 153 * 154 * Probe by trying to write the first of the L3 cache mask registers 155 * and checking that the bits stick. Max CLOSids is always 4 and max cbm length 156 * is always 20 on hsw server parts. The minimum cache bitmask length 157 * allowed for HSW server is always 2 bits. Hardcode all of them. 158 */ 159 static inline void cache_alloc_hsw_probe(void) 160 { 161 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_L3]; 162 struct rdt_resource *r = &hw_res->r_resctrl; 163 u64 max_cbm = BIT_ULL_MASK(20) - 1, l3_cbm_0; 164 165 if (wrmsrq_safe(MSR_IA32_L3_CBM_BASE, max_cbm)) 166 return; 167 168 rdmsrq(MSR_IA32_L3_CBM_BASE, l3_cbm_0); 169 170 /* If all the bits were set in MSR, return success */ 171 if (l3_cbm_0 != max_cbm) 172 return; 173 174 hw_res->num_closid = 4; 175 r->cache.cbm_len = 20; 176 r->cache.shareable_bits = 0xc0000; 177 r->cache.min_cbm_bits = 2; 178 r->cache.arch_has_sparse_bitmasks = false; 179 r->alloc_capable = true; 180 181 rdt_alloc_capable = true; 182 } 183 184 /* 185 * rdt_get_mb_table() - get a mapping of bandwidth(b/w) percentage values 186 * exposed to user interface and the h/w understandable delay values. 187 * 188 * The non-linear delay values have the granularity of power of two 189 * and also the h/w does not guarantee a curve for configured delay 190 * values vs. actual b/w enforced. 191 * Hence we need a mapping that is pre calibrated so the user can 192 * express the memory b/w as a percentage value. 193 */ 194 static inline bool rdt_get_mb_table(struct rdt_resource *r) 195 { 196 /* 197 * There are no Intel SKUs as of now to support non-linear delay. 198 */ 199 pr_info("MBA b/w map not implemented for cpu:%d, model:%d", 200 boot_cpu_data.x86, boot_cpu_data.x86_model); 201 202 return false; 203 } 204 205 static __init bool __get_mem_config_intel(struct rdt_resource *r) 206 { 207 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 208 union cpuid_0x10_3_eax eax; 209 union cpuid_0x10_x_edx edx; 210 u32 ebx, ecx, max_delay; 211 212 cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); 213 hw_res->num_closid = edx.split.cos_max + 1; 214 max_delay = eax.split.max_delay + 1; 215 r->membw.max_bw = MAX_MBA_BW; 216 r->membw.arch_needs_linear = true; 217 if (ecx & MBA_IS_LINEAR) { 218 r->membw.delay_linear = true; 219 r->membw.min_bw = MAX_MBA_BW - max_delay; 220 r->membw.bw_gran = MAX_MBA_BW - max_delay; 221 } else { 222 if (!rdt_get_mb_table(r)) 223 return false; 224 r->membw.arch_needs_linear = false; 225 } 226 227 if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA)) 228 r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD; 229 else 230 r->membw.throttle_mode = THREAD_THROTTLE_MAX; 231 232 r->alloc_capable = true; 233 234 return true; 235 } 236 237 static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) 238 { 239 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 240 u32 eax, ebx, ecx, edx, subleaf; 241 242 /* 243 * Query CPUID_Fn80000020_EDX_x01 for MBA and 244 * CPUID_Fn80000020_EDX_x02 for SMBA 245 */ 246 subleaf = (r->rid == RDT_RESOURCE_SMBA) ? 2 : 1; 247 248 cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); 249 hw_res->num_closid = edx + 1; 250 if (BITS_PER_TYPE(r->membw.max_bw) <= eax) { 251 pr_warn("Unable to support hardware's maximum bandwidth\n"); 252 return false; 253 } 254 r->membw.max_bw = BIT(eax); 255 256 /* AMD does not use delay */ 257 r->membw.delay_linear = false; 258 r->membw.arch_needs_linear = false; 259 260 /* 261 * AMD does not use memory delay throttle model to control 262 * the allocation like Intel does. 263 */ 264 r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; 265 r->membw.min_bw = 0; 266 r->membw.bw_gran = 1; 267 268 r->alloc_capable = true; 269 270 return true; 271 } 272 273 static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r) 274 { 275 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 276 union cpuid_0x10_1_eax eax; 277 union cpuid_0x10_x_ecx ecx; 278 union cpuid_0x10_x_edx edx; 279 u32 ebx, default_ctrl; 280 281 cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx.full, &edx.full); 282 hw_res->num_closid = edx.split.cos_max + 1; 283 r->cache.cbm_len = eax.split.cbm_len + 1; 284 default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; 285 r->cache.shareable_bits = ebx & default_ctrl; 286 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 287 r->cache.arch_has_sparse_bitmasks = ecx.split.noncont; 288 r->alloc_capable = true; 289 } 290 291 static void rdt_get_cdp_config(int level) 292 { 293 /* 294 * By default, CDP is disabled. CDP can be enabled by mount parameter 295 * "cdp" during resctrl file system mount time. 296 */ 297 rdt_resources_all[level].cdp_enabled = false; 298 rdt_resources_all[level].r_resctrl.cdp_capable = true; 299 } 300 301 static void rdt_set_io_alloc_capable(struct rdt_resource *r) 302 { 303 r->cache.io_alloc_capable = true; 304 } 305 306 static void rdt_get_cdp_l3_config(void) 307 { 308 rdt_get_cdp_config(RDT_RESOURCE_L3); 309 } 310 311 static void rdt_get_cdp_l2_config(void) 312 { 313 rdt_get_cdp_config(RDT_RESOURCE_L2); 314 } 315 316 static void mba_wrmsr_amd(struct msr_param *m) 317 { 318 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 319 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 320 unsigned int i; 321 322 for (i = m->low; i < m->high; i++) 323 wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); 324 } 325 326 /* 327 * Map the memory b/w percentage value to delay values 328 * that can be written to QOS_MSRs. 329 * There are currently no SKUs which support non linear delay values. 330 */ 331 static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) 332 { 333 if (r->membw.delay_linear) 334 return MAX_MBA_BW - bw; 335 336 pr_warn_once("Non Linear delay-bw map not supported but queried\n"); 337 return MAX_MBA_BW; 338 } 339 340 static void mba_wrmsr_intel(struct msr_param *m) 341 { 342 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 343 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 344 unsigned int i; 345 346 /* Write the delay values for mba. */ 347 for (i = m->low; i < m->high; i++) 348 wrmsrq(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res)); 349 } 350 351 static void cat_wrmsr(struct msr_param *m) 352 { 353 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 354 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 355 unsigned int i; 356 357 for (i = m->low; i < m->high; i++) 358 wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); 359 } 360 361 u32 resctrl_arch_get_num_closid(struct rdt_resource *r) 362 { 363 return resctrl_to_arch_res(r)->num_closid; 364 } 365 366 void rdt_ctrl_update(void *arg) 367 { 368 struct rdt_hw_resource *hw_res; 369 struct msr_param *m = arg; 370 371 hw_res = resctrl_to_arch_res(m->res); 372 hw_res->msr_update(m); 373 } 374 375 static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) 376 { 377 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 378 int i; 379 380 /* 381 * Initialize the Control MSRs to having no control. 382 * For Cache Allocation: Set all bits in cbm 383 * For Memory Allocation: Set b/w requested to 100% 384 */ 385 for (i = 0; i < hw_res->num_closid; i++, dc++) 386 *dc = resctrl_get_default_ctrl(r); 387 } 388 389 static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) 390 { 391 kfree(hw_dom->ctrl_val); 392 kfree(hw_dom); 393 } 394 395 static void l3_mon_domain_free(struct rdt_hw_l3_mon_domain *hw_dom) 396 { 397 int idx; 398 399 for_each_mbm_idx(idx) 400 kfree(hw_dom->arch_mbm_states[idx]); 401 kfree(hw_dom); 402 } 403 404 static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *d) 405 { 406 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d); 407 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 408 struct msr_param m; 409 u32 *dc; 410 411 dc = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->ctrl_val), 412 GFP_KERNEL); 413 if (!dc) 414 return -ENOMEM; 415 416 hw_dom->ctrl_val = dc; 417 setup_default_ctrlval(r, dc); 418 419 m.res = r; 420 m.dom = d; 421 m.low = 0; 422 m.high = hw_res->num_closid; 423 hw_res->msr_update(&m); 424 return 0; 425 } 426 427 /** 428 * l3_mon_domain_mbm_alloc() - Allocate arch private storage for the MBM counters 429 * @num_rmid: The size of the MBM counter array 430 * @hw_dom: The domain that owns the allocated arrays 431 * 432 * Return: 0 for success, or -ENOMEM. 433 */ 434 static int l3_mon_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_l3_mon_domain *hw_dom) 435 { 436 size_t tsize = sizeof(*hw_dom->arch_mbm_states[0]); 437 enum resctrl_event_id eventid; 438 int idx; 439 440 for_each_mbm_event_id(eventid) { 441 if (!resctrl_is_mon_event_enabled(eventid)) 442 continue; 443 idx = MBM_STATE_IDX(eventid); 444 hw_dom->arch_mbm_states[idx] = kcalloc(num_rmid, tsize, GFP_KERNEL); 445 if (!hw_dom->arch_mbm_states[idx]) 446 goto cleanup; 447 } 448 449 return 0; 450 cleanup: 451 for_each_mbm_idx(idx) { 452 kfree(hw_dom->arch_mbm_states[idx]); 453 hw_dom->arch_mbm_states[idx] = NULL; 454 } 455 456 return -ENOMEM; 457 } 458 459 static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope) 460 { 461 switch (scope) { 462 case RESCTRL_L2_CACHE: 463 case RESCTRL_L3_CACHE: 464 return get_cpu_cacheinfo_id(cpu, scope); 465 case RESCTRL_L3_NODE: 466 return cpu_to_node(cpu); 467 case RESCTRL_PACKAGE: 468 return topology_physical_package_id(cpu); 469 default: 470 break; 471 } 472 473 return -EINVAL; 474 } 475 476 static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r) 477 { 478 int id = get_domain_id_from_scope(cpu, r->ctrl_scope); 479 struct rdt_hw_ctrl_domain *hw_dom; 480 struct list_head *add_pos = NULL; 481 struct rdt_domain_hdr *hdr; 482 struct rdt_ctrl_domain *d; 483 int err; 484 485 lockdep_assert_held(&domain_list_lock); 486 487 if (id < 0) { 488 pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", 489 cpu, r->ctrl_scope, r->name); 490 return; 491 } 492 493 hdr = resctrl_find_domain(&r->ctrl_domains, id, &add_pos); 494 if (hdr) { 495 if (!domain_header_is_valid(hdr, RESCTRL_CTRL_DOMAIN, r->rid)) 496 return; 497 d = container_of(hdr, struct rdt_ctrl_domain, hdr); 498 499 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 500 if (r->cache.arch_has_per_cpu_cfg) 501 rdt_domain_reconfigure_cdp(r); 502 return; 503 } 504 505 hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu)); 506 if (!hw_dom) 507 return; 508 509 d = &hw_dom->d_resctrl; 510 d->hdr.id = id; 511 d->hdr.type = RESCTRL_CTRL_DOMAIN; 512 d->hdr.rid = r->rid; 513 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 514 515 rdt_domain_reconfigure_cdp(r); 516 517 if (domain_setup_ctrlval(r, d)) { 518 ctrl_domain_free(hw_dom); 519 return; 520 } 521 522 err = resctrl_online_ctrl_domain(r, d); 523 if (err) { 524 ctrl_domain_free(hw_dom); 525 return; 526 } 527 list_add_tail_rcu(&d->hdr.list, add_pos); 528 } 529 530 static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct list_head *add_pos) 531 { 532 struct rdt_hw_l3_mon_domain *hw_dom; 533 struct rdt_l3_mon_domain *d; 534 struct cacheinfo *ci; 535 int err; 536 537 hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu)); 538 if (!hw_dom) 539 return; 540 541 d = &hw_dom->d_resctrl; 542 d->hdr.id = id; 543 d->hdr.type = RESCTRL_MON_DOMAIN; 544 d->hdr.rid = RDT_RESOURCE_L3; 545 ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); 546 if (!ci) { 547 pr_warn_once("Can't find L3 cache for CPU:%d resource %s\n", cpu, r->name); 548 l3_mon_domain_free(hw_dom); 549 return; 550 } 551 d->ci_id = ci->id; 552 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 553 554 arch_mon_domain_online(r, d); 555 556 if (l3_mon_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) { 557 l3_mon_domain_free(hw_dom); 558 return; 559 } 560 561 err = resctrl_online_mon_domain(r, &d->hdr); 562 if (err) { 563 l3_mon_domain_free(hw_dom); 564 return; 565 } 566 list_add_tail_rcu(&d->hdr.list, add_pos); 567 } 568 569 static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) 570 { 571 int id = get_domain_id_from_scope(cpu, r->mon_scope); 572 struct list_head *add_pos = NULL; 573 struct rdt_domain_hdr *hdr; 574 575 lockdep_assert_held(&domain_list_lock); 576 577 if (id < 0) { 578 pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", 579 cpu, r->mon_scope, r->name); 580 return; 581 } 582 583 hdr = resctrl_find_domain(&r->mon_domains, id, &add_pos); 584 if (hdr) 585 cpumask_set_cpu(cpu, &hdr->cpu_mask); 586 587 switch (r->rid) { 588 case RDT_RESOURCE_L3: 589 /* Update the mbm_assign_mode state for the CPU if supported */ 590 if (r->mon.mbm_cntr_assignable) 591 resctrl_arch_mbm_cntr_assign_set_one(r); 592 if (!hdr) 593 l3_mon_domain_setup(cpu, id, r, add_pos); 594 break; 595 case RDT_RESOURCE_PERF_PKG: 596 if (!hdr) 597 intel_aet_mon_domain_setup(cpu, id, r, add_pos); 598 break; 599 default: 600 pr_warn_once("Unknown resource rid=%d\n", r->rid); 601 break; 602 } 603 } 604 605 static void domain_add_cpu(int cpu, struct rdt_resource *r) 606 { 607 if (r->alloc_capable) 608 domain_add_cpu_ctrl(cpu, r); 609 if (r->mon_capable) 610 domain_add_cpu_mon(cpu, r); 611 } 612 613 static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) 614 { 615 int id = get_domain_id_from_scope(cpu, r->ctrl_scope); 616 struct rdt_hw_ctrl_domain *hw_dom; 617 struct rdt_domain_hdr *hdr; 618 struct rdt_ctrl_domain *d; 619 620 lockdep_assert_held(&domain_list_lock); 621 622 if (id < 0) { 623 pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", 624 cpu, r->ctrl_scope, r->name); 625 return; 626 } 627 628 hdr = resctrl_find_domain(&r->ctrl_domains, id, NULL); 629 if (!hdr) { 630 pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n", 631 id, cpu, r->name); 632 return; 633 } 634 635 cpumask_clear_cpu(cpu, &hdr->cpu_mask); 636 if (!cpumask_empty(&hdr->cpu_mask)) 637 return; 638 639 if (!domain_header_is_valid(hdr, RESCTRL_CTRL_DOMAIN, r->rid)) 640 return; 641 642 d = container_of(hdr, struct rdt_ctrl_domain, hdr); 643 hw_dom = resctrl_to_arch_ctrl_dom(d); 644 645 list_del_rcu(&hdr->list); 646 synchronize_rcu(); 647 resctrl_offline_ctrl_domain(r, d); 648 649 /* 650 * rdt_ctrl_domain "d" is going to be freed below, so clear 651 * its pointer from pseudo_lock_region struct. 652 */ 653 if (d->plr) 654 d->plr->d = NULL; 655 ctrl_domain_free(hw_dom); 656 } 657 658 static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) 659 { 660 int id = get_domain_id_from_scope(cpu, r->mon_scope); 661 struct rdt_domain_hdr *hdr; 662 663 lockdep_assert_held(&domain_list_lock); 664 665 if (id < 0) { 666 pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", 667 cpu, r->mon_scope, r->name); 668 return; 669 } 670 671 hdr = resctrl_find_domain(&r->mon_domains, id, NULL); 672 if (!hdr) { 673 pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n", 674 id, cpu, r->name); 675 return; 676 } 677 678 cpumask_clear_cpu(cpu, &hdr->cpu_mask); 679 if (!cpumask_empty(&hdr->cpu_mask)) 680 return; 681 682 switch (r->rid) { 683 case RDT_RESOURCE_L3: { 684 struct rdt_hw_l3_mon_domain *hw_dom; 685 struct rdt_l3_mon_domain *d; 686 687 if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) 688 return; 689 690 d = container_of(hdr, struct rdt_l3_mon_domain, hdr); 691 hw_dom = resctrl_to_arch_mon_dom(d); 692 list_del_rcu(&hdr->list); 693 synchronize_rcu(); 694 resctrl_offline_mon_domain(r, hdr); 695 l3_mon_domain_free(hw_dom); 696 break; 697 } 698 case RDT_RESOURCE_PERF_PKG: { 699 struct rdt_perf_pkg_mon_domain *pkgd; 700 701 if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_PERF_PKG)) 702 return; 703 704 pkgd = container_of(hdr, struct rdt_perf_pkg_mon_domain, hdr); 705 list_del_rcu(&hdr->list); 706 synchronize_rcu(); 707 resctrl_offline_mon_domain(r, hdr); 708 kfree(pkgd); 709 break; 710 } 711 default: 712 pr_warn_once("Unknown resource rid=%d\n", r->rid); 713 break; 714 } 715 } 716 717 static void domain_remove_cpu(int cpu, struct rdt_resource *r) 718 { 719 if (r->alloc_capable) 720 domain_remove_cpu_ctrl(cpu, r); 721 if (r->mon_capable) 722 domain_remove_cpu_mon(cpu, r); 723 } 724 725 static void clear_closid_rmid(int cpu) 726 { 727 struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); 728 struct msr val = { 729 .l = RESCTRL_RESERVED_RMID, 730 .h = RESCTRL_RESERVED_CLOSID 731 }; 732 733 state->default_closid = RESCTRL_RESERVED_CLOSID; 734 state->default_rmid = RESCTRL_RESERVED_RMID; 735 state->cur_closid = RESCTRL_RESERVED_CLOSID; 736 state->cur_rmid = RESCTRL_RESERVED_RMID; 737 wrmsrq(MSR_IA32_PQR_ASSOC, val.q); 738 } 739 740 static int resctrl_arch_online_cpu(unsigned int cpu) 741 { 742 struct rdt_resource *r; 743 744 mutex_lock(&domain_list_lock); 745 for_each_capable_rdt_resource(r) 746 domain_add_cpu(cpu, r); 747 mutex_unlock(&domain_list_lock); 748 749 clear_closid_rmid(cpu); 750 resctrl_online_cpu(cpu); 751 752 return 0; 753 } 754 755 static int resctrl_arch_offline_cpu(unsigned int cpu) 756 { 757 struct rdt_resource *r; 758 759 resctrl_offline_cpu(cpu); 760 761 mutex_lock(&domain_list_lock); 762 for_each_capable_rdt_resource(r) 763 domain_remove_cpu(cpu, r); 764 mutex_unlock(&domain_list_lock); 765 766 clear_closid_rmid(cpu); 767 768 return 0; 769 } 770 771 void resctrl_arch_pre_mount(void) 772 { 773 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_PERF_PKG].r_resctrl; 774 int cpu; 775 776 if (!intel_aet_get_events()) 777 return; 778 779 /* 780 * Late discovery of telemetry events means the domains for the 781 * resource were not built. Do that now. 782 */ 783 cpus_read_lock(); 784 mutex_lock(&domain_list_lock); 785 r->mon_capable = true; 786 rdt_mon_capable = true; 787 for_each_online_cpu(cpu) 788 domain_add_cpu_mon(cpu, r); 789 mutex_unlock(&domain_list_lock); 790 cpus_read_unlock(); 791 } 792 793 enum { 794 RDT_FLAG_CMT, 795 RDT_FLAG_MBM_TOTAL, 796 RDT_FLAG_MBM_LOCAL, 797 RDT_FLAG_L3_CAT, 798 RDT_FLAG_L3_CDP, 799 RDT_FLAG_L2_CAT, 800 RDT_FLAG_L2_CDP, 801 RDT_FLAG_MBA, 802 RDT_FLAG_SMBA, 803 RDT_FLAG_BMEC, 804 RDT_FLAG_ABMC, 805 RDT_FLAG_SDCIAE, 806 }; 807 808 #define RDT_OPT(idx, n, f) \ 809 [idx] = { \ 810 .name = n, \ 811 .flag = f \ 812 } 813 814 struct rdt_options { 815 char *name; 816 int flag; 817 bool force_off, force_on; 818 }; 819 820 static struct rdt_options rdt_options[] __ro_after_init = { 821 RDT_OPT(RDT_FLAG_CMT, "cmt", X86_FEATURE_CQM_OCCUP_LLC), 822 RDT_OPT(RDT_FLAG_MBM_TOTAL, "mbmtotal", X86_FEATURE_CQM_MBM_TOTAL), 823 RDT_OPT(RDT_FLAG_MBM_LOCAL, "mbmlocal", X86_FEATURE_CQM_MBM_LOCAL), 824 RDT_OPT(RDT_FLAG_L3_CAT, "l3cat", X86_FEATURE_CAT_L3), 825 RDT_OPT(RDT_FLAG_L3_CDP, "l3cdp", X86_FEATURE_CDP_L3), 826 RDT_OPT(RDT_FLAG_L2_CAT, "l2cat", X86_FEATURE_CAT_L2), 827 RDT_OPT(RDT_FLAG_L2_CDP, "l2cdp", X86_FEATURE_CDP_L2), 828 RDT_OPT(RDT_FLAG_MBA, "mba", X86_FEATURE_MBA), 829 RDT_OPT(RDT_FLAG_SMBA, "smba", X86_FEATURE_SMBA), 830 RDT_OPT(RDT_FLAG_BMEC, "bmec", X86_FEATURE_BMEC), 831 RDT_OPT(RDT_FLAG_ABMC, "abmc", X86_FEATURE_ABMC), 832 RDT_OPT(RDT_FLAG_SDCIAE, "sdciae", X86_FEATURE_SDCIAE), 833 }; 834 #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options) 835 836 static int __init set_rdt_options(char *str) 837 { 838 struct rdt_options *o; 839 bool force_off; 840 char *tok; 841 842 if (*str == '=') 843 str++; 844 while ((tok = strsep(&str, ",")) != NULL) { 845 force_off = *tok == '!'; 846 if (force_off) 847 tok++; 848 if (intel_handle_aet_option(force_off, tok)) 849 continue; 850 for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) { 851 if (strcmp(tok, o->name) == 0) { 852 if (force_off) 853 o->force_off = true; 854 else 855 o->force_on = true; 856 break; 857 } 858 } 859 } 860 return 1; 861 } 862 __setup("rdt", set_rdt_options); 863 864 bool rdt_cpu_has(int flag) 865 { 866 bool ret = boot_cpu_has(flag); 867 struct rdt_options *o; 868 869 if (!ret) 870 return ret; 871 872 for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) { 873 if (flag == o->flag) { 874 if (o->force_off) 875 ret = false; 876 if (o->force_on) 877 ret = true; 878 break; 879 } 880 } 881 return ret; 882 } 883 884 bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) 885 { 886 if (!rdt_cpu_has(X86_FEATURE_BMEC)) 887 return false; 888 889 switch (evt) { 890 case QOS_L3_MBM_TOTAL_EVENT_ID: 891 return rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL); 892 case QOS_L3_MBM_LOCAL_EVENT_ID: 893 return rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL); 894 default: 895 return false; 896 } 897 } 898 899 static __init bool get_mem_config(void) 900 { 901 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_MBA]; 902 903 if (!rdt_cpu_has(X86_FEATURE_MBA)) 904 return false; 905 906 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 907 return __get_mem_config_intel(&hw_res->r_resctrl); 908 else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 909 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 910 return __rdt_get_mem_config_amd(&hw_res->r_resctrl); 911 912 return false; 913 } 914 915 static __init bool get_slow_mem_config(void) 916 { 917 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_SMBA]; 918 919 if (!rdt_cpu_has(X86_FEATURE_SMBA)) 920 return false; 921 922 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 923 return __rdt_get_mem_config_amd(&hw_res->r_resctrl); 924 925 return false; 926 } 927 928 static __init bool get_rdt_alloc_resources(void) 929 { 930 struct rdt_resource *r; 931 bool ret = false; 932 933 if (rdt_alloc_capable) 934 return true; 935 936 if (!boot_cpu_has(X86_FEATURE_RDT_A)) 937 return false; 938 939 if (rdt_cpu_has(X86_FEATURE_CAT_L3)) { 940 r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; 941 rdt_get_cache_alloc_cfg(1, r); 942 if (rdt_cpu_has(X86_FEATURE_CDP_L3)) 943 rdt_get_cdp_l3_config(); 944 if (rdt_cpu_has(X86_FEATURE_SDCIAE)) 945 rdt_set_io_alloc_capable(r); 946 ret = true; 947 } 948 if (rdt_cpu_has(X86_FEATURE_CAT_L2)) { 949 /* CPUID 0x10.2 fields are same format at 0x10.1 */ 950 r = &rdt_resources_all[RDT_RESOURCE_L2].r_resctrl; 951 rdt_get_cache_alloc_cfg(2, r); 952 if (rdt_cpu_has(X86_FEATURE_CDP_L2)) 953 rdt_get_cdp_l2_config(); 954 ret = true; 955 } 956 957 if (get_mem_config()) 958 ret = true; 959 960 if (get_slow_mem_config()) 961 ret = true; 962 963 return ret; 964 } 965 966 static __init bool get_rdt_mon_resources(void) 967 { 968 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; 969 bool ret = false; 970 971 if (rdt_cpu_has(X86_FEATURE_CQM_OCCUP_LLC)) { 972 resctrl_enable_mon_event(QOS_L3_OCCUP_EVENT_ID, false, 0, NULL); 973 ret = true; 974 } 975 if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) { 976 resctrl_enable_mon_event(QOS_L3_MBM_TOTAL_EVENT_ID, false, 0, NULL); 977 ret = true; 978 } 979 if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) { 980 resctrl_enable_mon_event(QOS_L3_MBM_LOCAL_EVENT_ID, false, 0, NULL); 981 ret = true; 982 } 983 if (rdt_cpu_has(X86_FEATURE_ABMC)) 984 ret = true; 985 986 if (!ret) 987 return false; 988 989 return !rdt_get_l3_mon_config(r); 990 } 991 992 static __init void __check_quirks_intel(void) 993 { 994 switch (boot_cpu_data.x86_vfm) { 995 case INTEL_HASWELL_X: 996 if (!rdt_options[RDT_FLAG_L3_CAT].force_off) 997 cache_alloc_hsw_probe(); 998 break; 999 case INTEL_SKYLAKE_X: 1000 if (boot_cpu_data.x86_stepping <= 4) 1001 set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat"); 1002 else 1003 set_rdt_options("!l3cat"); 1004 fallthrough; 1005 case INTEL_BROADWELL_X: 1006 intel_rdt_mbm_apply_quirk(); 1007 break; 1008 } 1009 } 1010 1011 static __init void check_quirks(void) 1012 { 1013 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1014 __check_quirks_intel(); 1015 } 1016 1017 static __init bool get_rdt_resources(void) 1018 { 1019 rdt_alloc_capable = get_rdt_alloc_resources(); 1020 rdt_mon_capable = get_rdt_mon_resources(); 1021 1022 return (rdt_mon_capable || rdt_alloc_capable); 1023 } 1024 1025 static __init void rdt_init_res_defs_intel(void) 1026 { 1027 struct rdt_hw_resource *hw_res; 1028 struct rdt_resource *r; 1029 1030 for_each_rdt_resource(r) { 1031 hw_res = resctrl_to_arch_res(r); 1032 1033 if (r->rid == RDT_RESOURCE_L3 || 1034 r->rid == RDT_RESOURCE_L2) { 1035 r->cache.arch_has_per_cpu_cfg = false; 1036 r->cache.min_cbm_bits = 1; 1037 } else if (r->rid == RDT_RESOURCE_MBA) { 1038 hw_res->msr_base = MSR_IA32_MBA_THRTL_BASE; 1039 hw_res->msr_update = mba_wrmsr_intel; 1040 } 1041 } 1042 } 1043 1044 static __init void rdt_init_res_defs_amd(void) 1045 { 1046 struct rdt_hw_resource *hw_res; 1047 struct rdt_resource *r; 1048 1049 for_each_rdt_resource(r) { 1050 hw_res = resctrl_to_arch_res(r); 1051 1052 if (r->rid == RDT_RESOURCE_L3 || 1053 r->rid == RDT_RESOURCE_L2) { 1054 r->cache.arch_has_sparse_bitmasks = true; 1055 r->cache.arch_has_per_cpu_cfg = true; 1056 r->cache.min_cbm_bits = 0; 1057 } else if (r->rid == RDT_RESOURCE_MBA) { 1058 hw_res->msr_base = MSR_IA32_MBA_BW_BASE; 1059 hw_res->msr_update = mba_wrmsr_amd; 1060 } else if (r->rid == RDT_RESOURCE_SMBA) { 1061 hw_res->msr_base = MSR_IA32_SMBA_BW_BASE; 1062 hw_res->msr_update = mba_wrmsr_amd; 1063 } 1064 } 1065 } 1066 1067 static __init void rdt_init_res_defs(void) 1068 { 1069 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1070 rdt_init_res_defs_intel(); 1071 else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1072 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 1073 rdt_init_res_defs_amd(); 1074 } 1075 1076 static enum cpuhp_state rdt_online; 1077 1078 /* Runs once on the BSP during boot. */ 1079 void resctrl_cpu_detect(struct cpuinfo_x86 *c) 1080 { 1081 if (!cpu_has(c, X86_FEATURE_CQM_LLC) && !cpu_has(c, X86_FEATURE_ABMC)) { 1082 c->x86_cache_max_rmid = -1; 1083 c->x86_cache_occ_scale = -1; 1084 c->x86_cache_mbm_width_offset = -1; 1085 return; 1086 } 1087 1088 /* will be overridden if occupancy monitoring exists */ 1089 c->x86_cache_max_rmid = cpuid_ebx(0xf); 1090 1091 if (cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC) || 1092 cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL) || 1093 cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL) || 1094 cpu_has(c, X86_FEATURE_ABMC)) { 1095 u32 eax, ebx, ecx, edx; 1096 1097 /* QoS sub-leaf, EAX=0Fh, ECX=1 */ 1098 cpuid_count(0xf, 1, &eax, &ebx, &ecx, &edx); 1099 1100 c->x86_cache_max_rmid = ecx; 1101 c->x86_cache_occ_scale = ebx; 1102 c->x86_cache_mbm_width_offset = eax & 0xff; 1103 1104 if (!c->x86_cache_mbm_width_offset) { 1105 switch (c->x86_vendor) { 1106 case X86_VENDOR_AMD: 1107 c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; 1108 break; 1109 case X86_VENDOR_HYGON: 1110 c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_HYGON; 1111 break; 1112 default: 1113 /* Leave c->x86_cache_mbm_width_offset as 0 */ 1114 break; 1115 } 1116 } 1117 } 1118 } 1119 1120 static int __init resctrl_arch_late_init(void) 1121 { 1122 struct rdt_resource *r; 1123 int state, ret, i; 1124 1125 /* for_each_rdt_resource() requires all rid to be initialised. */ 1126 for (i = 0; i < RDT_NUM_RESOURCES; i++) 1127 rdt_resources_all[i].r_resctrl.rid = i; 1128 1129 /* 1130 * Initialize functions(or definitions) that are different 1131 * between vendors here. 1132 */ 1133 rdt_init_res_defs(); 1134 1135 check_quirks(); 1136 1137 if (!get_rdt_resources()) 1138 return -ENODEV; 1139 1140 state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 1141 "x86/resctrl/cat:online:", 1142 resctrl_arch_online_cpu, 1143 resctrl_arch_offline_cpu); 1144 if (state < 0) 1145 return state; 1146 1147 ret = resctrl_init(); 1148 if (ret) { 1149 cpuhp_remove_state(state); 1150 return ret; 1151 } 1152 rdt_online = state; 1153 1154 for_each_alloc_capable_rdt_resource(r) 1155 pr_info("%s allocation detected\n", r->name); 1156 1157 for_each_mon_capable_rdt_resource(r) 1158 pr_info("%s monitoring detected\n", r->name); 1159 1160 return 0; 1161 } 1162 1163 late_initcall(resctrl_arch_late_init); 1164 1165 static void __exit resctrl_arch_exit(void) 1166 { 1167 intel_aet_exit(); 1168 1169 cpuhp_remove_state(rdt_online); 1170 1171 resctrl_exit(); 1172 } 1173 1174 __exitcall(resctrl_arch_exit); 1175