1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Resource Director Technology(RDT) 4 * - Cache Allocation code. 5 * 6 * Copyright (C) 2016 Intel Corporation 7 * 8 * Authors: 9 * Fenghua Yu <fenghua.yu@intel.com> 10 * Tony Luck <tony.luck@intel.com> 11 * Vikas Shivappa <vikas.shivappa@intel.com> 12 * 13 * More information about RDT be found in the Intel (R) x86 Architecture 14 * Software Developer Manual June 2016, volume 3, section 17.17. 15 */ 16 17 #define pr_fmt(fmt) "resctrl: " fmt 18 19 #include <linux/cpu.h> 20 #include <linux/slab.h> 21 #include <linux/err.h> 22 #include <linux/cpuhotplug.h> 23 24 #include <asm/cpu_device_id.h> 25 #include <asm/msr.h> 26 #include <asm/resctrl.h> 27 #include "internal.h" 28 29 /* 30 * rdt_domain structures are kfree()d when their last CPU goes offline, 31 * and allocated when the first CPU in a new domain comes online. 32 * The rdt_resource's domain list is updated when this happens. Readers of 33 * the domain list must either take cpus_read_lock(), or rely on an RCU 34 * read-side critical section, to avoid observing concurrent modification. 35 * All writers take this mutex: 36 */ 37 static DEFINE_MUTEX(domain_list_lock); 38 39 /* 40 * The cached resctrl_pqr_state is strictly per CPU and can never be 41 * updated from a remote CPU. Functions which modify the state 42 * are called with interrupts disabled and no preemption, which 43 * is sufficient for the protection. 44 */ 45 DEFINE_PER_CPU(struct resctrl_pqr_state, pqr_state); 46 47 /* 48 * Global boolean for rdt_alloc which is true if any 49 * resource allocation is enabled. 50 */ 51 bool rdt_alloc_capable; 52 53 static void mba_wrmsr_intel(struct msr_param *m); 54 static void cat_wrmsr(struct msr_param *m); 55 static void mba_wrmsr_amd(struct msr_param *m); 56 57 #define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains) 58 #define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains) 59 60 struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { 61 [RDT_RESOURCE_L3] = 62 { 63 .r_resctrl = { 64 .name = "L3", 65 .ctrl_scope = RESCTRL_L3_CACHE, 66 .mon_scope = RESCTRL_L3_CACHE, 67 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L3), 68 .mon_domains = mon_domain_init(RDT_RESOURCE_L3), 69 .schema_fmt = RESCTRL_SCHEMA_BITMAP, 70 }, 71 .msr_base = MSR_IA32_L3_CBM_BASE, 72 .msr_update = cat_wrmsr, 73 }, 74 [RDT_RESOURCE_L2] = 75 { 76 .r_resctrl = { 77 .name = "L2", 78 .ctrl_scope = RESCTRL_L2_CACHE, 79 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L2), 80 .schema_fmt = RESCTRL_SCHEMA_BITMAP, 81 }, 82 .msr_base = MSR_IA32_L2_CBM_BASE, 83 .msr_update = cat_wrmsr, 84 }, 85 [RDT_RESOURCE_MBA] = 86 { 87 .r_resctrl = { 88 .name = "MB", 89 .ctrl_scope = RESCTRL_L3_CACHE, 90 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA), 91 .schema_fmt = RESCTRL_SCHEMA_RANGE, 92 }, 93 }, 94 [RDT_RESOURCE_SMBA] = 95 { 96 .r_resctrl = { 97 .name = "SMBA", 98 .ctrl_scope = RESCTRL_L3_CACHE, 99 .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA), 100 .schema_fmt = RESCTRL_SCHEMA_RANGE, 101 }, 102 }, 103 [RDT_RESOURCE_PERF_PKG] = 104 { 105 .r_resctrl = { 106 .name = "PERF_PKG", 107 .mon_scope = RESCTRL_PACKAGE, 108 .mon_domains = mon_domain_init(RDT_RESOURCE_PERF_PKG), 109 }, 110 }, 111 }; 112 113 /** 114 * resctrl_arch_system_num_rmid_idx - Compute number of supported RMIDs 115 * (minimum across all mon_capable resource) 116 * 117 * Return: Number of supported RMIDs at time of call. Note that mount time 118 * enumeration of resources may reduce the number. 119 */ 120 u32 resctrl_arch_system_num_rmid_idx(void) 121 { 122 u32 num_rmids = U32_MAX; 123 struct rdt_resource *r; 124 125 for_each_mon_capable_rdt_resource(r) 126 num_rmids = min(num_rmids, r->mon.num_rmid); 127 128 /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */ 129 return num_rmids == U32_MAX ? 0 : num_rmids; 130 } 131 132 struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) 133 { 134 if (l >= RDT_NUM_RESOURCES) 135 return NULL; 136 137 return &rdt_resources_all[l].r_resctrl; 138 } 139 140 /* 141 * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs 142 * as they do not have CPUID enumeration support for Cache allocation. 143 * The check for Vendor/Family/Model is not enough to guarantee that 144 * the MSRs won't #GP fault because only the following SKUs support 145 * CAT: 146 * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz 147 * Intel(R) Xeon(R) CPU E5-2648L v3 @ 1.80GHz 148 * Intel(R) Xeon(R) CPU E5-2628L v3 @ 2.00GHz 149 * Intel(R) Xeon(R) CPU E5-2618L v3 @ 2.30GHz 150 * Intel(R) Xeon(R) CPU E5-2608L v3 @ 2.00GHz 151 * Intel(R) Xeon(R) CPU E5-2658A v3 @ 2.20GHz 152 * 153 * Probe by trying to write the first of the L3 cache mask registers 154 * and checking that the bits stick. Max CLOSids is always 4 and max cbm length 155 * is always 20 on hsw server parts. The minimum cache bitmask length 156 * allowed for HSW server is always 2 bits. Hardcode all of them. 157 */ 158 static inline void cache_alloc_hsw_probe(void) 159 { 160 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_L3]; 161 struct rdt_resource *r = &hw_res->r_resctrl; 162 u64 max_cbm = BIT_ULL_MASK(20) - 1, l3_cbm_0; 163 164 if (wrmsrq_safe(MSR_IA32_L3_CBM_BASE, max_cbm)) 165 return; 166 167 rdmsrq(MSR_IA32_L3_CBM_BASE, l3_cbm_0); 168 169 /* If all the bits were set in MSR, return success */ 170 if (l3_cbm_0 != max_cbm) 171 return; 172 173 hw_res->num_closid = 4; 174 r->cache.cbm_len = 20; 175 r->cache.shareable_bits = 0xc0000; 176 r->cache.min_cbm_bits = 2; 177 r->cache.arch_has_sparse_bitmasks = false; 178 r->alloc_capable = true; 179 180 rdt_alloc_capable = true; 181 } 182 183 /* 184 * rdt_get_mb_table() - get a mapping of bandwidth(b/w) percentage values 185 * exposed to user interface and the h/w understandable delay values. 186 * 187 * The non-linear delay values have the granularity of power of two 188 * and also the h/w does not guarantee a curve for configured delay 189 * values vs. actual b/w enforced. 190 * Hence we need a mapping that is pre calibrated so the user can 191 * express the memory b/w as a percentage value. 192 */ 193 static inline bool rdt_get_mb_table(struct rdt_resource *r) 194 { 195 /* 196 * There are no Intel SKUs as of now to support non-linear delay. 197 */ 198 pr_info("MBA b/w map not implemented for cpu:%d, model:%d", 199 boot_cpu_data.x86, boot_cpu_data.x86_model); 200 201 return false; 202 } 203 204 static __init bool __get_mem_config_intel(struct rdt_resource *r) 205 { 206 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 207 union cpuid_0x10_3_eax eax; 208 union cpuid_0x10_x_edx edx; 209 u32 ebx, ecx, max_delay; 210 211 cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); 212 hw_res->num_closid = edx.split.cos_max + 1; 213 max_delay = eax.split.max_delay + 1; 214 r->membw.max_bw = MAX_MBA_BW; 215 r->membw.arch_needs_linear = true; 216 if (ecx & MBA_IS_LINEAR) { 217 r->membw.delay_linear = true; 218 r->membw.min_bw = MAX_MBA_BW - max_delay; 219 r->membw.bw_gran = MAX_MBA_BW - max_delay; 220 } else { 221 if (!rdt_get_mb_table(r)) 222 return false; 223 r->membw.arch_needs_linear = false; 224 } 225 226 if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA)) 227 r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD; 228 else 229 r->membw.throttle_mode = THREAD_THROTTLE_MAX; 230 231 r->alloc_capable = true; 232 233 return true; 234 } 235 236 static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) 237 { 238 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 239 u32 eax, ebx, ecx, edx, subleaf; 240 241 /* 242 * Query CPUID_Fn80000020_EDX_x01 for MBA and 243 * CPUID_Fn80000020_EDX_x02 for SMBA 244 */ 245 subleaf = (r->rid == RDT_RESOURCE_SMBA) ? 2 : 1; 246 247 cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); 248 hw_res->num_closid = edx + 1; 249 r->membw.max_bw = 1 << eax; 250 251 /* AMD does not use delay */ 252 r->membw.delay_linear = false; 253 r->membw.arch_needs_linear = false; 254 255 /* 256 * AMD does not use memory delay throttle model to control 257 * the allocation like Intel does. 258 */ 259 r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; 260 r->membw.min_bw = 0; 261 r->membw.bw_gran = 1; 262 263 r->alloc_capable = true; 264 265 return true; 266 } 267 268 static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r) 269 { 270 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 271 union cpuid_0x10_1_eax eax; 272 union cpuid_0x10_x_ecx ecx; 273 union cpuid_0x10_x_edx edx; 274 u32 ebx, default_ctrl; 275 276 cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx.full, &edx.full); 277 hw_res->num_closid = edx.split.cos_max + 1; 278 r->cache.cbm_len = eax.split.cbm_len + 1; 279 default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; 280 r->cache.shareable_bits = ebx & default_ctrl; 281 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 282 r->cache.arch_has_sparse_bitmasks = ecx.split.noncont; 283 r->alloc_capable = true; 284 } 285 286 static void rdt_get_cdp_config(int level) 287 { 288 /* 289 * By default, CDP is disabled. CDP can be enabled by mount parameter 290 * "cdp" during resctrl file system mount time. 291 */ 292 rdt_resources_all[level].cdp_enabled = false; 293 rdt_resources_all[level].r_resctrl.cdp_capable = true; 294 } 295 296 static void rdt_set_io_alloc_capable(struct rdt_resource *r) 297 { 298 r->cache.io_alloc_capable = true; 299 } 300 301 static void rdt_get_cdp_l3_config(void) 302 { 303 rdt_get_cdp_config(RDT_RESOURCE_L3); 304 } 305 306 static void rdt_get_cdp_l2_config(void) 307 { 308 rdt_get_cdp_config(RDT_RESOURCE_L2); 309 } 310 311 static void mba_wrmsr_amd(struct msr_param *m) 312 { 313 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 314 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 315 unsigned int i; 316 317 for (i = m->low; i < m->high; i++) 318 wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); 319 } 320 321 /* 322 * Map the memory b/w percentage value to delay values 323 * that can be written to QOS_MSRs. 324 * There are currently no SKUs which support non linear delay values. 325 */ 326 static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) 327 { 328 if (r->membw.delay_linear) 329 return MAX_MBA_BW - bw; 330 331 pr_warn_once("Non Linear delay-bw map not supported but queried\n"); 332 return MAX_MBA_BW; 333 } 334 335 static void mba_wrmsr_intel(struct msr_param *m) 336 { 337 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 338 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 339 unsigned int i; 340 341 /* Write the delay values for mba. */ 342 for (i = m->low; i < m->high; i++) 343 wrmsrq(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res)); 344 } 345 346 static void cat_wrmsr(struct msr_param *m) 347 { 348 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); 349 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); 350 unsigned int i; 351 352 for (i = m->low; i < m->high; i++) 353 wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); 354 } 355 356 u32 resctrl_arch_get_num_closid(struct rdt_resource *r) 357 { 358 return resctrl_to_arch_res(r)->num_closid; 359 } 360 361 void rdt_ctrl_update(void *arg) 362 { 363 struct rdt_hw_resource *hw_res; 364 struct msr_param *m = arg; 365 366 hw_res = resctrl_to_arch_res(m->res); 367 hw_res->msr_update(m); 368 } 369 370 static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) 371 { 372 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 373 int i; 374 375 /* 376 * Initialize the Control MSRs to having no control. 377 * For Cache Allocation: Set all bits in cbm 378 * For Memory Allocation: Set b/w requested to 100% 379 */ 380 for (i = 0; i < hw_res->num_closid; i++, dc++) 381 *dc = resctrl_get_default_ctrl(r); 382 } 383 384 static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) 385 { 386 kfree(hw_dom->ctrl_val); 387 kfree(hw_dom); 388 } 389 390 static void l3_mon_domain_free(struct rdt_hw_l3_mon_domain *hw_dom) 391 { 392 int idx; 393 394 for_each_mbm_idx(idx) 395 kfree(hw_dom->arch_mbm_states[idx]); 396 kfree(hw_dom); 397 } 398 399 static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *d) 400 { 401 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d); 402 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); 403 struct msr_param m; 404 u32 *dc; 405 406 dc = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->ctrl_val), 407 GFP_KERNEL); 408 if (!dc) 409 return -ENOMEM; 410 411 hw_dom->ctrl_val = dc; 412 setup_default_ctrlval(r, dc); 413 414 m.res = r; 415 m.dom = d; 416 m.low = 0; 417 m.high = hw_res->num_closid; 418 hw_res->msr_update(&m); 419 return 0; 420 } 421 422 /** 423 * l3_mon_domain_mbm_alloc() - Allocate arch private storage for the MBM counters 424 * @num_rmid: The size of the MBM counter array 425 * @hw_dom: The domain that owns the allocated arrays 426 * 427 * Return: 0 for success, or -ENOMEM. 428 */ 429 static int l3_mon_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_l3_mon_domain *hw_dom) 430 { 431 size_t tsize = sizeof(*hw_dom->arch_mbm_states[0]); 432 enum resctrl_event_id eventid; 433 int idx; 434 435 for_each_mbm_event_id(eventid) { 436 if (!resctrl_is_mon_event_enabled(eventid)) 437 continue; 438 idx = MBM_STATE_IDX(eventid); 439 hw_dom->arch_mbm_states[idx] = kcalloc(num_rmid, tsize, GFP_KERNEL); 440 if (!hw_dom->arch_mbm_states[idx]) 441 goto cleanup; 442 } 443 444 return 0; 445 cleanup: 446 for_each_mbm_idx(idx) { 447 kfree(hw_dom->arch_mbm_states[idx]); 448 hw_dom->arch_mbm_states[idx] = NULL; 449 } 450 451 return -ENOMEM; 452 } 453 454 static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope) 455 { 456 switch (scope) { 457 case RESCTRL_L2_CACHE: 458 case RESCTRL_L3_CACHE: 459 return get_cpu_cacheinfo_id(cpu, scope); 460 case RESCTRL_L3_NODE: 461 return cpu_to_node(cpu); 462 case RESCTRL_PACKAGE: 463 return topology_physical_package_id(cpu); 464 default: 465 break; 466 } 467 468 return -EINVAL; 469 } 470 471 static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r) 472 { 473 int id = get_domain_id_from_scope(cpu, r->ctrl_scope); 474 struct rdt_hw_ctrl_domain *hw_dom; 475 struct list_head *add_pos = NULL; 476 struct rdt_domain_hdr *hdr; 477 struct rdt_ctrl_domain *d; 478 int err; 479 480 lockdep_assert_held(&domain_list_lock); 481 482 if (id < 0) { 483 pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", 484 cpu, r->ctrl_scope, r->name); 485 return; 486 } 487 488 hdr = resctrl_find_domain(&r->ctrl_domains, id, &add_pos); 489 if (hdr) { 490 if (!domain_header_is_valid(hdr, RESCTRL_CTRL_DOMAIN, r->rid)) 491 return; 492 d = container_of(hdr, struct rdt_ctrl_domain, hdr); 493 494 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 495 if (r->cache.arch_has_per_cpu_cfg) 496 rdt_domain_reconfigure_cdp(r); 497 return; 498 } 499 500 hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu)); 501 if (!hw_dom) 502 return; 503 504 d = &hw_dom->d_resctrl; 505 d->hdr.id = id; 506 d->hdr.type = RESCTRL_CTRL_DOMAIN; 507 d->hdr.rid = r->rid; 508 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 509 510 rdt_domain_reconfigure_cdp(r); 511 512 if (domain_setup_ctrlval(r, d)) { 513 ctrl_domain_free(hw_dom); 514 return; 515 } 516 517 list_add_tail_rcu(&d->hdr.list, add_pos); 518 519 err = resctrl_online_ctrl_domain(r, d); 520 if (err) { 521 list_del_rcu(&d->hdr.list); 522 synchronize_rcu(); 523 ctrl_domain_free(hw_dom); 524 } 525 } 526 527 static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct list_head *add_pos) 528 { 529 struct rdt_hw_l3_mon_domain *hw_dom; 530 struct rdt_l3_mon_domain *d; 531 struct cacheinfo *ci; 532 int err; 533 534 hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu)); 535 if (!hw_dom) 536 return; 537 538 d = &hw_dom->d_resctrl; 539 d->hdr.id = id; 540 d->hdr.type = RESCTRL_MON_DOMAIN; 541 d->hdr.rid = RDT_RESOURCE_L3; 542 ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); 543 if (!ci) { 544 pr_warn_once("Can't find L3 cache for CPU:%d resource %s\n", cpu, r->name); 545 l3_mon_domain_free(hw_dom); 546 return; 547 } 548 d->ci_id = ci->id; 549 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 550 551 arch_mon_domain_online(r, d); 552 553 if (l3_mon_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) { 554 l3_mon_domain_free(hw_dom); 555 return; 556 } 557 558 list_add_tail_rcu(&d->hdr.list, add_pos); 559 560 err = resctrl_online_mon_domain(r, &d->hdr); 561 if (err) { 562 list_del_rcu(&d->hdr.list); 563 synchronize_rcu(); 564 l3_mon_domain_free(hw_dom); 565 } 566 } 567 568 static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) 569 { 570 int id = get_domain_id_from_scope(cpu, r->mon_scope); 571 struct list_head *add_pos = NULL; 572 struct rdt_domain_hdr *hdr; 573 574 lockdep_assert_held(&domain_list_lock); 575 576 if (id < 0) { 577 pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", 578 cpu, r->mon_scope, r->name); 579 return; 580 } 581 582 hdr = resctrl_find_domain(&r->mon_domains, id, &add_pos); 583 if (hdr) 584 cpumask_set_cpu(cpu, &hdr->cpu_mask); 585 586 switch (r->rid) { 587 case RDT_RESOURCE_L3: 588 /* Update the mbm_assign_mode state for the CPU if supported */ 589 if (r->mon.mbm_cntr_assignable) 590 resctrl_arch_mbm_cntr_assign_set_one(r); 591 if (!hdr) 592 l3_mon_domain_setup(cpu, id, r, add_pos); 593 break; 594 case RDT_RESOURCE_PERF_PKG: 595 if (!hdr) 596 intel_aet_mon_domain_setup(cpu, id, r, add_pos); 597 break; 598 default: 599 pr_warn_once("Unknown resource rid=%d\n", r->rid); 600 break; 601 } 602 } 603 604 static void domain_add_cpu(int cpu, struct rdt_resource *r) 605 { 606 if (r->alloc_capable) 607 domain_add_cpu_ctrl(cpu, r); 608 if (r->mon_capable) 609 domain_add_cpu_mon(cpu, r); 610 } 611 612 static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) 613 { 614 int id = get_domain_id_from_scope(cpu, r->ctrl_scope); 615 struct rdt_hw_ctrl_domain *hw_dom; 616 struct rdt_domain_hdr *hdr; 617 struct rdt_ctrl_domain *d; 618 619 lockdep_assert_held(&domain_list_lock); 620 621 if (id < 0) { 622 pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", 623 cpu, r->ctrl_scope, r->name); 624 return; 625 } 626 627 hdr = resctrl_find_domain(&r->ctrl_domains, id, NULL); 628 if (!hdr) { 629 pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n", 630 id, cpu, r->name); 631 return; 632 } 633 634 cpumask_clear_cpu(cpu, &hdr->cpu_mask); 635 if (!cpumask_empty(&hdr->cpu_mask)) 636 return; 637 638 if (!domain_header_is_valid(hdr, RESCTRL_CTRL_DOMAIN, r->rid)) 639 return; 640 641 d = container_of(hdr, struct rdt_ctrl_domain, hdr); 642 hw_dom = resctrl_to_arch_ctrl_dom(d); 643 644 resctrl_offline_ctrl_domain(r, d); 645 list_del_rcu(&hdr->list); 646 synchronize_rcu(); 647 648 /* 649 * rdt_ctrl_domain "d" is going to be freed below, so clear 650 * its pointer from pseudo_lock_region struct. 651 */ 652 if (d->plr) 653 d->plr->d = NULL; 654 ctrl_domain_free(hw_dom); 655 } 656 657 static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) 658 { 659 int id = get_domain_id_from_scope(cpu, r->mon_scope); 660 struct rdt_domain_hdr *hdr; 661 662 lockdep_assert_held(&domain_list_lock); 663 664 if (id < 0) { 665 pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", 666 cpu, r->mon_scope, r->name); 667 return; 668 } 669 670 hdr = resctrl_find_domain(&r->mon_domains, id, NULL); 671 if (!hdr) { 672 pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n", 673 id, cpu, r->name); 674 return; 675 } 676 677 cpumask_clear_cpu(cpu, &hdr->cpu_mask); 678 if (!cpumask_empty(&hdr->cpu_mask)) 679 return; 680 681 switch (r->rid) { 682 case RDT_RESOURCE_L3: { 683 struct rdt_hw_l3_mon_domain *hw_dom; 684 struct rdt_l3_mon_domain *d; 685 686 if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) 687 return; 688 689 d = container_of(hdr, struct rdt_l3_mon_domain, hdr); 690 hw_dom = resctrl_to_arch_mon_dom(d); 691 resctrl_offline_mon_domain(r, hdr); 692 list_del_rcu(&hdr->list); 693 synchronize_rcu(); 694 l3_mon_domain_free(hw_dom); 695 break; 696 } 697 case RDT_RESOURCE_PERF_PKG: { 698 struct rdt_perf_pkg_mon_domain *pkgd; 699 700 if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_PERF_PKG)) 701 return; 702 703 pkgd = container_of(hdr, struct rdt_perf_pkg_mon_domain, hdr); 704 resctrl_offline_mon_domain(r, hdr); 705 list_del_rcu(&hdr->list); 706 synchronize_rcu(); 707 kfree(pkgd); 708 break; 709 } 710 default: 711 pr_warn_once("Unknown resource rid=%d\n", r->rid); 712 break; 713 } 714 } 715 716 static void domain_remove_cpu(int cpu, struct rdt_resource *r) 717 { 718 if (r->alloc_capable) 719 domain_remove_cpu_ctrl(cpu, r); 720 if (r->mon_capable) 721 domain_remove_cpu_mon(cpu, r); 722 } 723 724 static void clear_closid_rmid(int cpu) 725 { 726 struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); 727 728 state->default_closid = RESCTRL_RESERVED_CLOSID; 729 state->default_rmid = RESCTRL_RESERVED_RMID; 730 state->cur_closid = RESCTRL_RESERVED_CLOSID; 731 state->cur_rmid = RESCTRL_RESERVED_RMID; 732 wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID, 733 RESCTRL_RESERVED_CLOSID); 734 } 735 736 static int resctrl_arch_online_cpu(unsigned int cpu) 737 { 738 struct rdt_resource *r; 739 740 mutex_lock(&domain_list_lock); 741 for_each_capable_rdt_resource(r) 742 domain_add_cpu(cpu, r); 743 mutex_unlock(&domain_list_lock); 744 745 clear_closid_rmid(cpu); 746 resctrl_online_cpu(cpu); 747 748 return 0; 749 } 750 751 static int resctrl_arch_offline_cpu(unsigned int cpu) 752 { 753 struct rdt_resource *r; 754 755 resctrl_offline_cpu(cpu); 756 757 mutex_lock(&domain_list_lock); 758 for_each_capable_rdt_resource(r) 759 domain_remove_cpu(cpu, r); 760 mutex_unlock(&domain_list_lock); 761 762 clear_closid_rmid(cpu); 763 764 return 0; 765 } 766 767 void resctrl_arch_pre_mount(void) 768 { 769 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_PERF_PKG].r_resctrl; 770 int cpu; 771 772 if (!intel_aet_get_events()) 773 return; 774 775 /* 776 * Late discovery of telemetry events means the domains for the 777 * resource were not built. Do that now. 778 */ 779 cpus_read_lock(); 780 mutex_lock(&domain_list_lock); 781 r->mon_capable = true; 782 rdt_mon_capable = true; 783 for_each_online_cpu(cpu) 784 domain_add_cpu_mon(cpu, r); 785 mutex_unlock(&domain_list_lock); 786 cpus_read_unlock(); 787 } 788 789 enum { 790 RDT_FLAG_CMT, 791 RDT_FLAG_MBM_TOTAL, 792 RDT_FLAG_MBM_LOCAL, 793 RDT_FLAG_L3_CAT, 794 RDT_FLAG_L3_CDP, 795 RDT_FLAG_L2_CAT, 796 RDT_FLAG_L2_CDP, 797 RDT_FLAG_MBA, 798 RDT_FLAG_SMBA, 799 RDT_FLAG_BMEC, 800 RDT_FLAG_ABMC, 801 RDT_FLAG_SDCIAE, 802 }; 803 804 #define RDT_OPT(idx, n, f) \ 805 [idx] = { \ 806 .name = n, \ 807 .flag = f \ 808 } 809 810 struct rdt_options { 811 char *name; 812 int flag; 813 bool force_off, force_on; 814 }; 815 816 static struct rdt_options rdt_options[] __ro_after_init = { 817 RDT_OPT(RDT_FLAG_CMT, "cmt", X86_FEATURE_CQM_OCCUP_LLC), 818 RDT_OPT(RDT_FLAG_MBM_TOTAL, "mbmtotal", X86_FEATURE_CQM_MBM_TOTAL), 819 RDT_OPT(RDT_FLAG_MBM_LOCAL, "mbmlocal", X86_FEATURE_CQM_MBM_LOCAL), 820 RDT_OPT(RDT_FLAG_L3_CAT, "l3cat", X86_FEATURE_CAT_L3), 821 RDT_OPT(RDT_FLAG_L3_CDP, "l3cdp", X86_FEATURE_CDP_L3), 822 RDT_OPT(RDT_FLAG_L2_CAT, "l2cat", X86_FEATURE_CAT_L2), 823 RDT_OPT(RDT_FLAG_L2_CDP, "l2cdp", X86_FEATURE_CDP_L2), 824 RDT_OPT(RDT_FLAG_MBA, "mba", X86_FEATURE_MBA), 825 RDT_OPT(RDT_FLAG_SMBA, "smba", X86_FEATURE_SMBA), 826 RDT_OPT(RDT_FLAG_BMEC, "bmec", X86_FEATURE_BMEC), 827 RDT_OPT(RDT_FLAG_ABMC, "abmc", X86_FEATURE_ABMC), 828 RDT_OPT(RDT_FLAG_SDCIAE, "sdciae", X86_FEATURE_SDCIAE), 829 }; 830 #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options) 831 832 static int __init set_rdt_options(char *str) 833 { 834 struct rdt_options *o; 835 bool force_off; 836 char *tok; 837 838 if (*str == '=') 839 str++; 840 while ((tok = strsep(&str, ",")) != NULL) { 841 force_off = *tok == '!'; 842 if (force_off) 843 tok++; 844 if (intel_handle_aet_option(force_off, tok)) 845 continue; 846 for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) { 847 if (strcmp(tok, o->name) == 0) { 848 if (force_off) 849 o->force_off = true; 850 else 851 o->force_on = true; 852 break; 853 } 854 } 855 } 856 return 1; 857 } 858 __setup("rdt", set_rdt_options); 859 860 bool rdt_cpu_has(int flag) 861 { 862 bool ret = boot_cpu_has(flag); 863 struct rdt_options *o; 864 865 if (!ret) 866 return ret; 867 868 for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) { 869 if (flag == o->flag) { 870 if (o->force_off) 871 ret = false; 872 if (o->force_on) 873 ret = true; 874 break; 875 } 876 } 877 return ret; 878 } 879 880 bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) 881 { 882 if (!rdt_cpu_has(X86_FEATURE_BMEC)) 883 return false; 884 885 switch (evt) { 886 case QOS_L3_MBM_TOTAL_EVENT_ID: 887 return rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL); 888 case QOS_L3_MBM_LOCAL_EVENT_ID: 889 return rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL); 890 default: 891 return false; 892 } 893 } 894 895 static __init bool get_mem_config(void) 896 { 897 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_MBA]; 898 899 if (!rdt_cpu_has(X86_FEATURE_MBA)) 900 return false; 901 902 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 903 return __get_mem_config_intel(&hw_res->r_resctrl); 904 else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 905 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 906 return __rdt_get_mem_config_amd(&hw_res->r_resctrl); 907 908 return false; 909 } 910 911 static __init bool get_slow_mem_config(void) 912 { 913 struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_SMBA]; 914 915 if (!rdt_cpu_has(X86_FEATURE_SMBA)) 916 return false; 917 918 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 919 return __rdt_get_mem_config_amd(&hw_res->r_resctrl); 920 921 return false; 922 } 923 924 static __init bool get_rdt_alloc_resources(void) 925 { 926 struct rdt_resource *r; 927 bool ret = false; 928 929 if (rdt_alloc_capable) 930 return true; 931 932 if (!boot_cpu_has(X86_FEATURE_RDT_A)) 933 return false; 934 935 if (rdt_cpu_has(X86_FEATURE_CAT_L3)) { 936 r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; 937 rdt_get_cache_alloc_cfg(1, r); 938 if (rdt_cpu_has(X86_FEATURE_CDP_L3)) 939 rdt_get_cdp_l3_config(); 940 if (rdt_cpu_has(X86_FEATURE_SDCIAE)) 941 rdt_set_io_alloc_capable(r); 942 ret = true; 943 } 944 if (rdt_cpu_has(X86_FEATURE_CAT_L2)) { 945 /* CPUID 0x10.2 fields are same format at 0x10.1 */ 946 r = &rdt_resources_all[RDT_RESOURCE_L2].r_resctrl; 947 rdt_get_cache_alloc_cfg(2, r); 948 if (rdt_cpu_has(X86_FEATURE_CDP_L2)) 949 rdt_get_cdp_l2_config(); 950 ret = true; 951 } 952 953 if (get_mem_config()) 954 ret = true; 955 956 if (get_slow_mem_config()) 957 ret = true; 958 959 return ret; 960 } 961 962 static __init bool get_rdt_mon_resources(void) 963 { 964 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; 965 bool ret = false; 966 967 if (rdt_cpu_has(X86_FEATURE_CQM_OCCUP_LLC)) { 968 resctrl_enable_mon_event(QOS_L3_OCCUP_EVENT_ID, false, 0, NULL); 969 ret = true; 970 } 971 if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) { 972 resctrl_enable_mon_event(QOS_L3_MBM_TOTAL_EVENT_ID, false, 0, NULL); 973 ret = true; 974 } 975 if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) { 976 resctrl_enable_mon_event(QOS_L3_MBM_LOCAL_EVENT_ID, false, 0, NULL); 977 ret = true; 978 } 979 if (rdt_cpu_has(X86_FEATURE_ABMC)) 980 ret = true; 981 982 if (!ret) 983 return false; 984 985 return !rdt_get_l3_mon_config(r); 986 } 987 988 static __init void __check_quirks_intel(void) 989 { 990 switch (boot_cpu_data.x86_vfm) { 991 case INTEL_HASWELL_X: 992 if (!rdt_options[RDT_FLAG_L3_CAT].force_off) 993 cache_alloc_hsw_probe(); 994 break; 995 case INTEL_SKYLAKE_X: 996 if (boot_cpu_data.x86_stepping <= 4) 997 set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat"); 998 else 999 set_rdt_options("!l3cat"); 1000 fallthrough; 1001 case INTEL_BROADWELL_X: 1002 intel_rdt_mbm_apply_quirk(); 1003 break; 1004 } 1005 } 1006 1007 static __init void check_quirks(void) 1008 { 1009 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1010 __check_quirks_intel(); 1011 } 1012 1013 static __init bool get_rdt_resources(void) 1014 { 1015 rdt_alloc_capable = get_rdt_alloc_resources(); 1016 rdt_mon_capable = get_rdt_mon_resources(); 1017 1018 return (rdt_mon_capable || rdt_alloc_capable); 1019 } 1020 1021 static __init void rdt_init_res_defs_intel(void) 1022 { 1023 struct rdt_hw_resource *hw_res; 1024 struct rdt_resource *r; 1025 1026 for_each_rdt_resource(r) { 1027 hw_res = resctrl_to_arch_res(r); 1028 1029 if (r->rid == RDT_RESOURCE_L3 || 1030 r->rid == RDT_RESOURCE_L2) { 1031 r->cache.arch_has_per_cpu_cfg = false; 1032 r->cache.min_cbm_bits = 1; 1033 } else if (r->rid == RDT_RESOURCE_MBA) { 1034 hw_res->msr_base = MSR_IA32_MBA_THRTL_BASE; 1035 hw_res->msr_update = mba_wrmsr_intel; 1036 } 1037 } 1038 } 1039 1040 static __init void rdt_init_res_defs_amd(void) 1041 { 1042 struct rdt_hw_resource *hw_res; 1043 struct rdt_resource *r; 1044 1045 for_each_rdt_resource(r) { 1046 hw_res = resctrl_to_arch_res(r); 1047 1048 if (r->rid == RDT_RESOURCE_L3 || 1049 r->rid == RDT_RESOURCE_L2) { 1050 r->cache.arch_has_sparse_bitmasks = true; 1051 r->cache.arch_has_per_cpu_cfg = true; 1052 r->cache.min_cbm_bits = 0; 1053 } else if (r->rid == RDT_RESOURCE_MBA) { 1054 hw_res->msr_base = MSR_IA32_MBA_BW_BASE; 1055 hw_res->msr_update = mba_wrmsr_amd; 1056 } else if (r->rid == RDT_RESOURCE_SMBA) { 1057 hw_res->msr_base = MSR_IA32_SMBA_BW_BASE; 1058 hw_res->msr_update = mba_wrmsr_amd; 1059 } 1060 } 1061 } 1062 1063 static __init void rdt_init_res_defs(void) 1064 { 1065 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1066 rdt_init_res_defs_intel(); 1067 else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1068 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 1069 rdt_init_res_defs_amd(); 1070 } 1071 1072 static enum cpuhp_state rdt_online; 1073 1074 /* Runs once on the BSP during boot. */ 1075 void resctrl_cpu_detect(struct cpuinfo_x86 *c) 1076 { 1077 if (!cpu_has(c, X86_FEATURE_CQM_LLC) && !cpu_has(c, X86_FEATURE_ABMC)) { 1078 c->x86_cache_max_rmid = -1; 1079 c->x86_cache_occ_scale = -1; 1080 c->x86_cache_mbm_width_offset = -1; 1081 return; 1082 } 1083 1084 /* will be overridden if occupancy monitoring exists */ 1085 c->x86_cache_max_rmid = cpuid_ebx(0xf); 1086 1087 if (cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC) || 1088 cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL) || 1089 cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL) || 1090 cpu_has(c, X86_FEATURE_ABMC)) { 1091 u32 eax, ebx, ecx, edx; 1092 1093 /* QoS sub-leaf, EAX=0Fh, ECX=1 */ 1094 cpuid_count(0xf, 1, &eax, &ebx, &ecx, &edx); 1095 1096 c->x86_cache_max_rmid = ecx; 1097 c->x86_cache_occ_scale = ebx; 1098 c->x86_cache_mbm_width_offset = eax & 0xff; 1099 1100 if (!c->x86_cache_mbm_width_offset) { 1101 switch (c->x86_vendor) { 1102 case X86_VENDOR_AMD: 1103 c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; 1104 break; 1105 case X86_VENDOR_HYGON: 1106 c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_HYGON; 1107 break; 1108 default: 1109 /* Leave c->x86_cache_mbm_width_offset as 0 */ 1110 break; 1111 } 1112 } 1113 } 1114 } 1115 1116 static int __init resctrl_arch_late_init(void) 1117 { 1118 struct rdt_resource *r; 1119 int state, ret, i; 1120 1121 /* for_each_rdt_resource() requires all rid to be initialised. */ 1122 for (i = 0; i < RDT_NUM_RESOURCES; i++) 1123 rdt_resources_all[i].r_resctrl.rid = i; 1124 1125 /* 1126 * Initialize functions(or definitions) that are different 1127 * between vendors here. 1128 */ 1129 rdt_init_res_defs(); 1130 1131 check_quirks(); 1132 1133 if (!get_rdt_resources()) 1134 return -ENODEV; 1135 1136 state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 1137 "x86/resctrl/cat:online:", 1138 resctrl_arch_online_cpu, 1139 resctrl_arch_offline_cpu); 1140 if (state < 0) 1141 return state; 1142 1143 ret = resctrl_init(); 1144 if (ret) { 1145 cpuhp_remove_state(state); 1146 return ret; 1147 } 1148 rdt_online = state; 1149 1150 for_each_alloc_capable_rdt_resource(r) 1151 pr_info("%s allocation detected\n", r->name); 1152 1153 for_each_mon_capable_rdt_resource(r) 1154 pr_info("%s monitoring detected\n", r->name); 1155 1156 return 0; 1157 } 1158 1159 late_initcall(resctrl_arch_late_init); 1160 1161 static void __exit resctrl_arch_exit(void) 1162 { 1163 intel_aet_exit(); 1164 1165 cpuhp_remove_state(rdt_online); 1166 1167 resctrl_exit(); 1168 } 1169 1170 __exitcall(resctrl_arch_exit); 1171