1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * AMD Hardware Feedback Interface Driver 4 * 5 * Copyright (C) 2025 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Authors: Perry Yuan <Perry.Yuan@amd.com> 8 * Mario Limonciello <mario.limonciello@amd.com> 9 */ 10 11 #define pr_fmt(fmt) "amd-hfi: " fmt 12 13 #include <linux/acpi.h> 14 #include <linux/cpu.h> 15 #include <linux/debugfs.h> 16 #include <linux/gfp.h> 17 #include <linux/init.h> 18 #include <linux/io.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/mailbox_client.h> 22 #include <linux/mutex.h> 23 #include <linux/percpu-defs.h> 24 #include <linux/platform_device.h> 25 #include <linux/smp.h> 26 #include <linux/topology.h> 27 #include <linux/workqueue.h> 28 29 #include <asm/cpu_device_id.h> 30 31 #include <acpi/pcc.h> 32 #include <acpi/cppc_acpi.h> 33 34 #define AMD_HFI_DRIVER "amd_hfi" 35 #define AMD_HFI_MAILBOX_COUNT 1 36 #define AMD_HETERO_RANKING_TABLE_VER 2 37 38 #define AMD_HETERO_CPUID_27 0x80000027 39 40 static struct platform_device *device; 41 42 /** 43 * struct amd_shmem_info - Shared memory table for AMD HFI 44 * 45 * @header: The PCCT table header including signature, length flags and command. 46 * @version_number: Version number of the table 47 * @n_logical_processors: Number of logical processors 48 * @n_capabilities: Number of ranking dimensions (performance, efficiency, etc) 49 * @table_update_context: Command being sent over the subspace 50 * @n_bitmaps: Number of 32-bit bitmaps to enumerate all the APIC IDs 51 * This is based on the maximum APIC ID enumerated in the system 52 * @reserved: 24 bit spare 53 * @table_data: Bit Map(s) of enabled logical processors 54 * Followed by the ranking data for each logical processor 55 */ 56 struct amd_shmem_info { 57 struct acpi_pcct_ext_pcc_shared_memory header; 58 u32 version_number :8, 59 n_logical_processors :8, 60 n_capabilities :8, 61 table_update_context :8; 62 u32 n_bitmaps :8, 63 reserved :24; 64 u32 table_data[]; 65 }; 66 67 struct amd_hfi_data { 68 const char *name; 69 struct device *dev; 70 71 /* PCCT table related */ 72 struct pcc_mbox_chan *pcc_chan; 73 void __iomem *pcc_comm_addr; 74 struct acpi_subtable_header *pcct_entry; 75 struct amd_shmem_info *shmem; 76 77 struct dentry *dbgfs_dir; 78 }; 79 80 /** 81 * struct amd_hfi_classes - HFI class capabilities per CPU 82 * @perf: Performance capability 83 * @eff: Power efficiency capability 84 * 85 * Capabilities of a logical processor in the ranking table. These capabilities 86 * are unitless and specific to each HFI class. 87 */ 88 struct amd_hfi_classes { 89 u32 perf; 90 u32 eff; 91 }; 92 93 /** 94 * struct amd_hfi_cpuinfo - HFI workload class info per CPU 95 * @cpu: CPU index 96 * @apic_id: APIC id of the current CPU 97 * @class_index: workload class ID index 98 * @nr_class: max number of workload class supported 99 * @ipcc_scores: ipcc scores for each class 100 * @amd_hfi_classes: current CPU workload class ranking data 101 * 102 * Parameters of a logical processor linked with hardware feedback class. 103 */ 104 struct amd_hfi_cpuinfo { 105 int cpu; 106 u32 apic_id; 107 s16 class_index; 108 u8 nr_class; 109 int *ipcc_scores; 110 struct amd_hfi_classes *amd_hfi_classes; 111 }; 112 113 static DEFINE_PER_CPU(struct amd_hfi_cpuinfo, amd_hfi_cpuinfo) = {.class_index = -1}; 114 115 static DEFINE_MUTEX(hfi_cpuinfo_lock); 116 117 static void amd_hfi_sched_itmt_work(struct work_struct *work) 118 { 119 sched_set_itmt_support(); 120 } 121 static DECLARE_WORK(sched_amd_hfi_itmt_work, amd_hfi_sched_itmt_work); 122 123 static int find_cpu_index_by_apicid(unsigned int target_apicid) 124 { 125 int cpu_index; 126 127 for_each_possible_cpu(cpu_index) { 128 struct cpuinfo_x86 *info = &cpu_data(cpu_index); 129 130 if (info->topo.apicid == target_apicid) { 131 pr_debug("match APIC id %u for CPU index: %d\n", 132 info->topo.apicid, cpu_index); 133 return cpu_index; 134 } 135 } 136 137 return -ENODEV; 138 } 139 140 static int amd_hfi_fill_metadata(struct amd_hfi_data *amd_hfi_data) 141 { 142 struct acpi_pcct_ext_pcc_slave *pcct_ext = 143 (struct acpi_pcct_ext_pcc_slave *)amd_hfi_data->pcct_entry; 144 void __iomem *pcc_comm_addr; 145 u32 apic_start = 0; 146 147 pcc_comm_addr = acpi_os_ioremap(amd_hfi_data->pcc_chan->shmem_base_addr, 148 amd_hfi_data->pcc_chan->shmem_size); 149 if (!pcc_comm_addr) { 150 dev_err(amd_hfi_data->dev, "failed to ioremap PCC common region mem\n"); 151 return -ENOMEM; 152 } 153 154 memcpy_fromio(amd_hfi_data->shmem, pcc_comm_addr, pcct_ext->length); 155 iounmap(pcc_comm_addr); 156 157 if (amd_hfi_data->shmem->header.signature != PCC_SIGNATURE) { 158 dev_err(amd_hfi_data->dev, "invalid signature in shared memory\n"); 159 return -EINVAL; 160 } 161 if (amd_hfi_data->shmem->version_number != AMD_HETERO_RANKING_TABLE_VER) { 162 dev_err(amd_hfi_data->dev, "invalid version %d\n", 163 amd_hfi_data->shmem->version_number); 164 return -EINVAL; 165 } 166 167 for (unsigned int i = 0; i < amd_hfi_data->shmem->n_bitmaps; i++) { 168 u32 bitmap = amd_hfi_data->shmem->table_data[i]; 169 170 for (unsigned int j = 0; j < BITS_PER_TYPE(u32); j++) { 171 u32 apic_id = i * BITS_PER_TYPE(u32) + j; 172 struct amd_hfi_cpuinfo *info; 173 int cpu_index, apic_index; 174 175 if (!(bitmap & BIT(j))) 176 continue; 177 178 cpu_index = find_cpu_index_by_apicid(apic_id); 179 if (cpu_index < 0) { 180 dev_warn(amd_hfi_data->dev, "APIC ID %u not found\n", apic_id); 181 continue; 182 } 183 184 info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index); 185 info->apic_id = apic_id; 186 187 /* Fill the ranking data for each logical processor */ 188 info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index); 189 apic_index = apic_start * info->nr_class * 2; 190 for (unsigned int k = 0; k < info->nr_class; k++) { 191 u32 *table = amd_hfi_data->shmem->table_data + 192 amd_hfi_data->shmem->n_bitmaps + 193 i * info->nr_class; 194 195 info->amd_hfi_classes[k].eff = table[apic_index + 2 * k]; 196 info->amd_hfi_classes[k].perf = table[apic_index + 2 * k + 1]; 197 } 198 apic_start++; 199 } 200 } 201 202 return 0; 203 } 204 205 static int amd_hfi_alloc_class_data(struct platform_device *pdev) 206 { 207 struct amd_hfi_cpuinfo *hfi_cpuinfo; 208 struct device *dev = &pdev->dev; 209 u32 nr_class_id; 210 int idx; 211 212 nr_class_id = cpuid_eax(AMD_HETERO_CPUID_27); 213 if (nr_class_id > 255) { 214 dev_err(dev, "number of supported classes too large: %d\n", 215 nr_class_id); 216 return -EINVAL; 217 } 218 219 for_each_possible_cpu(idx) { 220 struct amd_hfi_classes *classes; 221 int *ipcc_scores; 222 223 classes = devm_kcalloc(dev, 224 nr_class_id, 225 sizeof(struct amd_hfi_classes), 226 GFP_KERNEL); 227 if (!classes) 228 return -ENOMEM; 229 ipcc_scores = devm_kcalloc(dev, nr_class_id, sizeof(int), GFP_KERNEL); 230 if (!ipcc_scores) 231 return -ENOMEM; 232 hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, idx); 233 hfi_cpuinfo->amd_hfi_classes = classes; 234 hfi_cpuinfo->ipcc_scores = ipcc_scores; 235 hfi_cpuinfo->nr_class = nr_class_id; 236 } 237 238 return 0; 239 } 240 241 static void amd_hfi_remove(struct platform_device *pdev) 242 { 243 struct amd_hfi_data *dev = platform_get_drvdata(pdev); 244 245 debugfs_remove_recursive(dev->dbgfs_dir); 246 } 247 248 static int amd_set_hfi_ipcc_score(struct amd_hfi_cpuinfo *hfi_cpuinfo, int cpu) 249 { 250 for (int i = 0; i < hfi_cpuinfo->nr_class; i++) 251 WRITE_ONCE(hfi_cpuinfo->ipcc_scores[i], 252 hfi_cpuinfo->amd_hfi_classes[i].perf); 253 254 sched_set_itmt_core_prio(hfi_cpuinfo->ipcc_scores[0], cpu); 255 256 return 0; 257 } 258 259 static int amd_hfi_set_state(unsigned int cpu, bool state) 260 { 261 int ret; 262 263 ret = wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_CLASS_CONFIG, state ? 1 : 0); 264 if (ret) 265 return ret; 266 267 return wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_HRST, 0x1); 268 } 269 270 /** 271 * amd_hfi_online() - Enable workload classification on @cpu 272 * @cpu: CPU in which the workload classification will be enabled 273 * 274 * Return: 0 on success, negative error code on failure. 275 */ 276 static int amd_hfi_online(unsigned int cpu) 277 { 278 struct amd_hfi_cpuinfo *hfi_info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu); 279 struct amd_hfi_classes *hfi_classes; 280 int ret; 281 282 if (WARN_ON_ONCE(!hfi_info)) 283 return -EINVAL; 284 285 /* 286 * Check if @cpu as an associated, initialized and ranking data must 287 * be filled. 288 */ 289 hfi_classes = hfi_info->amd_hfi_classes; 290 if (!hfi_classes) 291 return -EINVAL; 292 293 guard(mutex)(&hfi_cpuinfo_lock); 294 295 ret = amd_hfi_set_state(cpu, true); 296 if (ret) 297 pr_err("WCT enable failed for CPU %u\n", cpu); 298 299 return ret; 300 } 301 302 /** 303 * amd_hfi_offline() - Disable workload classification on @cpu 304 * @cpu: CPU in which the workload classification will be disabled 305 * 306 * Remove @cpu from those covered by its HFI instance. 307 * 308 * Return: 0 on success, negative error code on failure 309 */ 310 static int amd_hfi_offline(unsigned int cpu) 311 { 312 struct amd_hfi_cpuinfo *hfi_info = &per_cpu(amd_hfi_cpuinfo, cpu); 313 int ret; 314 315 if (WARN_ON_ONCE(!hfi_info)) 316 return -EINVAL; 317 318 guard(mutex)(&hfi_cpuinfo_lock); 319 320 ret = amd_hfi_set_state(cpu, false); 321 if (ret) 322 pr_err("WCT disable failed for CPU %u\n", cpu); 323 324 return ret; 325 } 326 327 static int update_hfi_ipcc_scores(void) 328 { 329 int cpu; 330 int ret; 331 332 for_each_possible_cpu(cpu) { 333 struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu); 334 335 ret = amd_set_hfi_ipcc_score(hfi_cpuinfo, cpu); 336 if (ret) 337 return ret; 338 } 339 340 return 0; 341 } 342 343 static int amd_hfi_metadata_parser(struct platform_device *pdev, 344 struct amd_hfi_data *amd_hfi_data) 345 { 346 struct acpi_pcct_ext_pcc_slave *pcct_ext; 347 struct acpi_subtable_header *pcct_entry; 348 struct mbox_chan *pcc_mbox_channels; 349 struct acpi_table_header *pcct_tbl; 350 struct pcc_mbox_chan *pcc_chan; 351 acpi_status status; 352 int ret; 353 354 pcc_mbox_channels = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT, 355 sizeof(*pcc_mbox_channels), GFP_KERNEL); 356 if (!pcc_mbox_channels) 357 return -ENOMEM; 358 359 pcc_chan = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT, 360 sizeof(*pcc_chan), GFP_KERNEL); 361 if (!pcc_chan) 362 return -ENOMEM; 363 364 status = acpi_get_table(ACPI_SIG_PCCT, 0, &pcct_tbl); 365 if (ACPI_FAILURE(status) || !pcct_tbl) 366 return -ENODEV; 367 368 /* get pointer to the first PCC subspace entry */ 369 pcct_entry = (struct acpi_subtable_header *) ( 370 (unsigned long)pcct_tbl + sizeof(struct acpi_table_pcct)); 371 372 pcc_chan->mchan = &pcc_mbox_channels[0]; 373 374 amd_hfi_data->pcc_chan = pcc_chan; 375 amd_hfi_data->pcct_entry = pcct_entry; 376 pcct_ext = (struct acpi_pcct_ext_pcc_slave *)pcct_entry; 377 378 if (pcct_ext->length <= 0) { 379 ret = -EINVAL; 380 goto out; 381 } 382 383 amd_hfi_data->shmem = devm_kzalloc(amd_hfi_data->dev, pcct_ext->length, GFP_KERNEL); 384 if (!amd_hfi_data->shmem) { 385 ret = -ENOMEM; 386 goto out; 387 } 388 389 pcc_chan->shmem_base_addr = pcct_ext->base_address; 390 pcc_chan->shmem_size = pcct_ext->length; 391 392 /* parse the shared memory info from the PCCT table */ 393 ret = amd_hfi_fill_metadata(amd_hfi_data); 394 395 out: 396 /* Don't leak any ACPI memory */ 397 acpi_put_table(pcct_tbl); 398 399 return ret; 400 } 401 402 static int class_capabilities_show(struct seq_file *s, void *unused) 403 { 404 u32 cpu, idx; 405 406 seq_puts(s, "CPU #\tWLC\tPerf\tEff\n"); 407 for_each_possible_cpu(cpu) { 408 struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu); 409 410 seq_printf(s, "%d", cpu); 411 for (idx = 0; idx < hfi_cpuinfo->nr_class; idx++) { 412 seq_printf(s, "\t%u\t%u\t%u\n", idx, 413 hfi_cpuinfo->amd_hfi_classes[idx].perf, 414 hfi_cpuinfo->amd_hfi_classes[idx].eff); 415 } 416 } 417 418 return 0; 419 } 420 DEFINE_SHOW_ATTRIBUTE(class_capabilities); 421 422 static int amd_hfi_pm_resume(struct device *dev) 423 { 424 int ret, cpu; 425 426 for_each_online_cpu(cpu) { 427 ret = amd_hfi_set_state(cpu, true); 428 if (ret < 0) { 429 dev_err(dev, "failed to enable workload class config: %d\n", ret); 430 return ret; 431 } 432 } 433 434 return 0; 435 } 436 437 static int amd_hfi_pm_suspend(struct device *dev) 438 { 439 int ret, cpu; 440 441 for_each_online_cpu(cpu) { 442 ret = amd_hfi_set_state(cpu, false); 443 if (ret < 0) { 444 dev_err(dev, "failed to disable workload class config: %d\n", ret); 445 return ret; 446 } 447 } 448 449 return 0; 450 } 451 452 static DEFINE_SIMPLE_DEV_PM_OPS(amd_hfi_pm_ops, amd_hfi_pm_suspend, amd_hfi_pm_resume); 453 454 static const struct acpi_device_id amd_hfi_platform_match[] = { 455 {"AMDI0104", 0}, 456 { } 457 }; 458 MODULE_DEVICE_TABLE(acpi, amd_hfi_platform_match); 459 460 static int amd_hfi_probe(struct platform_device *pdev) 461 { 462 struct amd_hfi_data *amd_hfi_data; 463 int ret; 464 465 if (!acpi_match_device(amd_hfi_platform_match, &pdev->dev)) 466 return -ENODEV; 467 468 amd_hfi_data = devm_kzalloc(&pdev->dev, sizeof(*amd_hfi_data), GFP_KERNEL); 469 if (!amd_hfi_data) 470 return -ENOMEM; 471 472 amd_hfi_data->dev = &pdev->dev; 473 platform_set_drvdata(pdev, amd_hfi_data); 474 475 ret = amd_hfi_alloc_class_data(pdev); 476 if (ret) 477 return ret; 478 479 ret = amd_hfi_metadata_parser(pdev, amd_hfi_data); 480 if (ret) 481 return ret; 482 483 ret = update_hfi_ipcc_scores(); 484 if (ret) 485 return ret; 486 487 /* 488 * Tasks will already be running at the time this happens. This is 489 * OK because rankings will be adjusted by the callbacks. 490 */ 491 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/amd_hfi:online", 492 amd_hfi_online, amd_hfi_offline); 493 if (ret < 0) 494 return ret; 495 496 schedule_work(&sched_amd_hfi_itmt_work); 497 498 amd_hfi_data->dbgfs_dir = debugfs_create_dir("amd_hfi", arch_debugfs_dir); 499 debugfs_create_file("class_capabilities", 0644, amd_hfi_data->dbgfs_dir, pdev, 500 &class_capabilities_fops); 501 502 return 0; 503 } 504 505 static struct platform_driver amd_hfi_driver = { 506 .driver = { 507 .name = AMD_HFI_DRIVER, 508 .pm = &amd_hfi_pm_ops, 509 .acpi_match_table = ACPI_PTR(amd_hfi_platform_match), 510 }, 511 .probe = amd_hfi_probe, 512 .remove = amd_hfi_remove, 513 }; 514 515 static int __init amd_hfi_init(void) 516 { 517 int ret; 518 519 if (acpi_disabled || 520 !cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES) || 521 !cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS)) 522 return -ENODEV; 523 524 device = platform_device_register_simple(AMD_HFI_DRIVER, -1, NULL, 0); 525 if (IS_ERR(device)) { 526 pr_err("unable to register HFI platform device\n"); 527 return PTR_ERR(device); 528 } 529 530 ret = platform_driver_register(&amd_hfi_driver); 531 if (ret) 532 pr_err("failed to register HFI driver\n"); 533 534 return ret; 535 } 536 537 static __exit void amd_hfi_exit(void) 538 { 539 platform_driver_unregister(&amd_hfi_driver); 540 platform_device_unregister(device); 541 } 542 module_init(amd_hfi_init); 543 module_exit(amd_hfi_exit); 544 545 MODULE_LICENSE("GPL"); 546 MODULE_DESCRIPTION("AMD Hardware Feedback Interface Driver"); 547