1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * AMD Hardware Feedback Interface Driver
4 *
5 * Copyright (C) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
6 *
7 * Authors: Perry Yuan <Perry.Yuan@amd.com>
8 * Mario Limonciello <mario.limonciello@amd.com>
9 */
10
11 #define pr_fmt(fmt) "amd-hfi: " fmt
12
13 #include <linux/acpi.h>
14 #include <linux/cpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/debugfs.h>
17 #include <linux/gfp.h>
18 #include <linux/init.h>
19 #include <linux/io.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/mailbox_client.h>
23 #include <linux/mutex.h>
24 #include <linux/percpu-defs.h>
25 #include <linux/platform_device.h>
26 #include <linux/smp.h>
27 #include <linux/topology.h>
28 #include <linux/workqueue.h>
29
30 #include <asm/cpu_device_id.h>
31
32 #include <acpi/pcc.h>
33 #include <acpi/cppc_acpi.h>
34
35 #define AMD_HFI_DRIVER "amd_hfi"
36 #define AMD_HFI_MAILBOX_COUNT 1
37 #define AMD_HETERO_RANKING_TABLE_VER 2
38
39 #define AMD_HETERO_CPUID_27 0x80000027
40
41 static struct platform_device *device;
42
43 /**
44 * struct amd_shmem_info - Shared memory table for AMD HFI
45 *
46 * @header: The PCCT table header including signature, length flags and command.
47 * @version_number: Version number of the table
48 * @n_logical_processors: Number of logical processors
49 * @n_capabilities: Number of ranking dimensions (performance, efficiency, etc)
50 * @table_update_context: Command being sent over the subspace
51 * @n_bitmaps: Number of 32-bit bitmaps to enumerate all the APIC IDs
52 * This is based on the maximum APIC ID enumerated in the system
53 * @reserved: 24 bit spare
54 * @table_data: Bit Map(s) of enabled logical processors
55 * Followed by the ranking data for each logical processor
56 */
57 struct amd_shmem_info {
58 struct acpi_pcct_ext_pcc_shared_memory header;
59 u32 version_number :8,
60 n_logical_processors :8,
61 n_capabilities :8,
62 table_update_context :8;
63 u32 n_bitmaps :8,
64 reserved :24;
65 u32 table_data[];
66 };
67
68 struct amd_hfi_data {
69 const char *name;
70 struct device *dev;
71
72 /* PCCT table related */
73 struct pcc_mbox_chan *pcc_chan;
74 void __iomem *pcc_comm_addr;
75 struct acpi_subtable_header *pcct_entry;
76 struct amd_shmem_info *shmem;
77
78 struct dentry *dbgfs_dir;
79 };
80
81 /**
82 * struct amd_hfi_classes - HFI class capabilities per CPU
83 * @perf: Performance capability
84 * @eff: Power efficiency capability
85 *
86 * Capabilities of a logical processor in the ranking table. These capabilities
87 * are unitless and specific to each HFI class.
88 */
89 struct amd_hfi_classes {
90 u32 perf;
91 u32 eff;
92 };
93
94 /**
95 * struct amd_hfi_cpuinfo - HFI workload class info per CPU
96 * @cpu: CPU index
97 * @apic_id: APIC id of the current CPU
98 * @cpus: mask of CPUs associated with amd_hfi_cpuinfo
99 * @class_index: workload class ID index
100 * @nr_class: max number of workload class supported
101 * @ipcc_scores: ipcc scores for each class
102 * @amd_hfi_classes: current CPU workload class ranking data
103 *
104 * Parameters of a logical processor linked with hardware feedback class.
105 */
106 struct amd_hfi_cpuinfo {
107 int cpu;
108 u32 apic_id;
109 cpumask_var_t cpus;
110 s16 class_index;
111 u8 nr_class;
112 int *ipcc_scores;
113 struct amd_hfi_classes *amd_hfi_classes;
114 };
115
116 static DEFINE_PER_CPU(struct amd_hfi_cpuinfo, amd_hfi_cpuinfo) = {.class_index = -1};
117
118 static DEFINE_MUTEX(hfi_cpuinfo_lock);
119
amd_hfi_sched_itmt_work(struct work_struct * work)120 static void amd_hfi_sched_itmt_work(struct work_struct *work)
121 {
122 sched_set_itmt_support();
123 }
124 static DECLARE_WORK(sched_amd_hfi_itmt_work, amd_hfi_sched_itmt_work);
125
find_cpu_index_by_apicid(unsigned int target_apicid)126 static int find_cpu_index_by_apicid(unsigned int target_apicid)
127 {
128 int cpu_index;
129
130 for_each_possible_cpu(cpu_index) {
131 struct cpuinfo_x86 *info = &cpu_data(cpu_index);
132
133 if (info->topo.apicid == target_apicid) {
134 pr_debug("match APIC id %u for CPU index: %d\n",
135 info->topo.apicid, cpu_index);
136 return cpu_index;
137 }
138 }
139
140 return -ENODEV;
141 }
142
amd_hfi_fill_metadata(struct amd_hfi_data * amd_hfi_data)143 static int amd_hfi_fill_metadata(struct amd_hfi_data *amd_hfi_data)
144 {
145 struct acpi_pcct_ext_pcc_slave *pcct_ext =
146 (struct acpi_pcct_ext_pcc_slave *)amd_hfi_data->pcct_entry;
147 void __iomem *pcc_comm_addr;
148 u32 apic_start = 0;
149
150 pcc_comm_addr = acpi_os_ioremap(amd_hfi_data->pcc_chan->shmem_base_addr,
151 amd_hfi_data->pcc_chan->shmem_size);
152 if (!pcc_comm_addr) {
153 dev_err(amd_hfi_data->dev, "failed to ioremap PCC common region mem\n");
154 return -ENOMEM;
155 }
156
157 memcpy_fromio(amd_hfi_data->shmem, pcc_comm_addr, pcct_ext->length);
158 iounmap(pcc_comm_addr);
159
160 if (amd_hfi_data->shmem->header.signature != PCC_SIGNATURE) {
161 dev_err(amd_hfi_data->dev, "invalid signature in shared memory\n");
162 return -EINVAL;
163 }
164 if (amd_hfi_data->shmem->version_number != AMD_HETERO_RANKING_TABLE_VER) {
165 dev_err(amd_hfi_data->dev, "invalid version %d\n",
166 amd_hfi_data->shmem->version_number);
167 return -EINVAL;
168 }
169
170 for (unsigned int i = 0; i < amd_hfi_data->shmem->n_bitmaps; i++) {
171 u32 bitmap = amd_hfi_data->shmem->table_data[i];
172
173 for (unsigned int j = 0; j < BITS_PER_TYPE(u32); j++) {
174 u32 apic_id = i * BITS_PER_TYPE(u32) + j;
175 struct amd_hfi_cpuinfo *info;
176 int cpu_index, apic_index;
177
178 if (!(bitmap & BIT(j)))
179 continue;
180
181 cpu_index = find_cpu_index_by_apicid(apic_id);
182 if (cpu_index < 0) {
183 dev_warn(amd_hfi_data->dev, "APIC ID %u not found\n", apic_id);
184 continue;
185 }
186
187 info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index);
188 info->apic_id = apic_id;
189
190 /* Fill the ranking data for each logical processor */
191 info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index);
192 apic_index = apic_start * info->nr_class * 2;
193 for (unsigned int k = 0; k < info->nr_class; k++) {
194 u32 *table = amd_hfi_data->shmem->table_data +
195 amd_hfi_data->shmem->n_bitmaps +
196 i * info->nr_class;
197
198 info->amd_hfi_classes[k].eff = table[apic_index + 2 * k];
199 info->amd_hfi_classes[k].perf = table[apic_index + 2 * k + 1];
200 }
201 apic_start++;
202 }
203 }
204
205 return 0;
206 }
207
amd_hfi_alloc_class_data(struct platform_device * pdev)208 static int amd_hfi_alloc_class_data(struct platform_device *pdev)
209 {
210 struct amd_hfi_cpuinfo *hfi_cpuinfo;
211 struct device *dev = &pdev->dev;
212 u32 nr_class_id;
213 int idx;
214
215 nr_class_id = cpuid_eax(AMD_HETERO_CPUID_27);
216 if (nr_class_id > 255) {
217 dev_err(dev, "number of supported classes too large: %d\n",
218 nr_class_id);
219 return -EINVAL;
220 }
221
222 for_each_possible_cpu(idx) {
223 struct amd_hfi_classes *classes;
224 int *ipcc_scores;
225
226 classes = devm_kcalloc(dev,
227 nr_class_id,
228 sizeof(struct amd_hfi_classes),
229 GFP_KERNEL);
230 if (!classes)
231 return -ENOMEM;
232 ipcc_scores = devm_kcalloc(dev, nr_class_id, sizeof(int), GFP_KERNEL);
233 if (!ipcc_scores)
234 return -ENOMEM;
235 hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, idx);
236 hfi_cpuinfo->amd_hfi_classes = classes;
237 hfi_cpuinfo->ipcc_scores = ipcc_scores;
238 hfi_cpuinfo->nr_class = nr_class_id;
239 }
240
241 return 0;
242 }
243
amd_hfi_remove(struct platform_device * pdev)244 static void amd_hfi_remove(struct platform_device *pdev)
245 {
246 struct amd_hfi_data *dev = platform_get_drvdata(pdev);
247
248 debugfs_remove_recursive(dev->dbgfs_dir);
249 }
250
amd_set_hfi_ipcc_score(struct amd_hfi_cpuinfo * hfi_cpuinfo,int cpu)251 static int amd_set_hfi_ipcc_score(struct amd_hfi_cpuinfo *hfi_cpuinfo, int cpu)
252 {
253 for (int i = 0; i < hfi_cpuinfo->nr_class; i++)
254 WRITE_ONCE(hfi_cpuinfo->ipcc_scores[i],
255 hfi_cpuinfo->amd_hfi_classes[i].perf);
256
257 sched_set_itmt_core_prio(hfi_cpuinfo->ipcc_scores[0], cpu);
258
259 return 0;
260 }
261
amd_hfi_set_state(unsigned int cpu,bool state)262 static int amd_hfi_set_state(unsigned int cpu, bool state)
263 {
264 int ret;
265
266 ret = wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_CLASS_CONFIG, state ? 1 : 0);
267 if (ret)
268 return ret;
269
270 return wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_HRST, 0x1);
271 }
272
273 /**
274 * amd_hfi_online() - Enable workload classification on @cpu
275 * @cpu: CPU in which the workload classification will be enabled
276 *
277 * Return: 0 on success, negative error code on failure.
278 */
amd_hfi_online(unsigned int cpu)279 static int amd_hfi_online(unsigned int cpu)
280 {
281 struct amd_hfi_cpuinfo *hfi_info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
282 struct amd_hfi_classes *hfi_classes;
283 int ret;
284
285 if (WARN_ON_ONCE(!hfi_info))
286 return -EINVAL;
287
288 /*
289 * Check if @cpu as an associated, initialized and ranking data must
290 * be filled.
291 */
292 hfi_classes = hfi_info->amd_hfi_classes;
293 if (!hfi_classes)
294 return -EINVAL;
295
296 guard(mutex)(&hfi_cpuinfo_lock);
297
298 if (!zalloc_cpumask_var(&hfi_info->cpus, GFP_KERNEL))
299 return -ENOMEM;
300
301 cpumask_set_cpu(cpu, hfi_info->cpus);
302
303 ret = amd_hfi_set_state(cpu, true);
304 if (ret)
305 pr_err("WCT enable failed for CPU %u\n", cpu);
306
307 return ret;
308 }
309
310 /**
311 * amd_hfi_offline() - Disable workload classification on @cpu
312 * @cpu: CPU in which the workload classification will be disabled
313 *
314 * Remove @cpu from those covered by its HFI instance.
315 *
316 * Return: 0 on success, negative error code on failure
317 */
amd_hfi_offline(unsigned int cpu)318 static int amd_hfi_offline(unsigned int cpu)
319 {
320 struct amd_hfi_cpuinfo *hfi_info = &per_cpu(amd_hfi_cpuinfo, cpu);
321 int ret;
322
323 if (WARN_ON_ONCE(!hfi_info))
324 return -EINVAL;
325
326 guard(mutex)(&hfi_cpuinfo_lock);
327
328 ret = amd_hfi_set_state(cpu, false);
329 if (ret)
330 pr_err("WCT disable failed for CPU %u\n", cpu);
331
332 free_cpumask_var(hfi_info->cpus);
333
334 return ret;
335 }
336
update_hfi_ipcc_scores(void)337 static int update_hfi_ipcc_scores(void)
338 {
339 int cpu;
340 int ret;
341
342 for_each_possible_cpu(cpu) {
343 struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
344
345 ret = amd_set_hfi_ipcc_score(hfi_cpuinfo, cpu);
346 if (ret)
347 return ret;
348 }
349
350 return 0;
351 }
352
amd_hfi_metadata_parser(struct platform_device * pdev,struct amd_hfi_data * amd_hfi_data)353 static int amd_hfi_metadata_parser(struct platform_device *pdev,
354 struct amd_hfi_data *amd_hfi_data)
355 {
356 struct acpi_pcct_ext_pcc_slave *pcct_ext;
357 struct acpi_subtable_header *pcct_entry;
358 struct mbox_chan *pcc_mbox_channels;
359 struct acpi_table_header *pcct_tbl;
360 struct pcc_mbox_chan *pcc_chan;
361 acpi_status status;
362 int ret;
363
364 pcc_mbox_channels = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT,
365 sizeof(*pcc_mbox_channels), GFP_KERNEL);
366 if (!pcc_mbox_channels)
367 return -ENOMEM;
368
369 pcc_chan = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT,
370 sizeof(*pcc_chan), GFP_KERNEL);
371 if (!pcc_chan)
372 return -ENOMEM;
373
374 status = acpi_get_table(ACPI_SIG_PCCT, 0, &pcct_tbl);
375 if (ACPI_FAILURE(status) || !pcct_tbl)
376 return -ENODEV;
377
378 /* get pointer to the first PCC subspace entry */
379 pcct_entry = (struct acpi_subtable_header *) (
380 (unsigned long)pcct_tbl + sizeof(struct acpi_table_pcct));
381
382 pcc_chan->mchan = &pcc_mbox_channels[0];
383
384 amd_hfi_data->pcc_chan = pcc_chan;
385 amd_hfi_data->pcct_entry = pcct_entry;
386 pcct_ext = (struct acpi_pcct_ext_pcc_slave *)pcct_entry;
387
388 if (pcct_ext->length <= 0) {
389 ret = -EINVAL;
390 goto out;
391 }
392
393 amd_hfi_data->shmem = devm_kzalloc(amd_hfi_data->dev, pcct_ext->length, GFP_KERNEL);
394 if (!amd_hfi_data->shmem) {
395 ret = -ENOMEM;
396 goto out;
397 }
398
399 pcc_chan->shmem_base_addr = pcct_ext->base_address;
400 pcc_chan->shmem_size = pcct_ext->length;
401
402 /* parse the shared memory info from the PCCT table */
403 ret = amd_hfi_fill_metadata(amd_hfi_data);
404
405 out:
406 /* Don't leak any ACPI memory */
407 acpi_put_table(pcct_tbl);
408
409 return ret;
410 }
411
class_capabilities_show(struct seq_file * s,void * unused)412 static int class_capabilities_show(struct seq_file *s, void *unused)
413 {
414 u32 cpu, idx;
415
416 seq_puts(s, "CPU #\tWLC\tPerf\tEff\n");
417 for_each_possible_cpu(cpu) {
418 struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
419
420 seq_printf(s, "%d", cpu);
421 for (idx = 0; idx < hfi_cpuinfo->nr_class; idx++) {
422 seq_printf(s, "\t%u\t%u\t%u\n", idx,
423 hfi_cpuinfo->amd_hfi_classes[idx].perf,
424 hfi_cpuinfo->amd_hfi_classes[idx].eff);
425 }
426 }
427
428 return 0;
429 }
430 DEFINE_SHOW_ATTRIBUTE(class_capabilities);
431
amd_hfi_pm_resume(struct device * dev)432 static int amd_hfi_pm_resume(struct device *dev)
433 {
434 int ret, cpu;
435
436 for_each_online_cpu(cpu) {
437 ret = amd_hfi_set_state(cpu, true);
438 if (ret < 0) {
439 dev_err(dev, "failed to enable workload class config: %d\n", ret);
440 return ret;
441 }
442 }
443
444 return 0;
445 }
446
amd_hfi_pm_suspend(struct device * dev)447 static int amd_hfi_pm_suspend(struct device *dev)
448 {
449 int ret, cpu;
450
451 for_each_online_cpu(cpu) {
452 ret = amd_hfi_set_state(cpu, false);
453 if (ret < 0) {
454 dev_err(dev, "failed to disable workload class config: %d\n", ret);
455 return ret;
456 }
457 }
458
459 return 0;
460 }
461
462 static DEFINE_SIMPLE_DEV_PM_OPS(amd_hfi_pm_ops, amd_hfi_pm_suspend, amd_hfi_pm_resume);
463
464 static const struct acpi_device_id amd_hfi_platform_match[] = {
465 {"AMDI0104", 0},
466 { }
467 };
468 MODULE_DEVICE_TABLE(acpi, amd_hfi_platform_match);
469
amd_hfi_probe(struct platform_device * pdev)470 static int amd_hfi_probe(struct platform_device *pdev)
471 {
472 struct amd_hfi_data *amd_hfi_data;
473 int ret;
474
475 if (!acpi_match_device(amd_hfi_platform_match, &pdev->dev))
476 return -ENODEV;
477
478 amd_hfi_data = devm_kzalloc(&pdev->dev, sizeof(*amd_hfi_data), GFP_KERNEL);
479 if (!amd_hfi_data)
480 return -ENOMEM;
481
482 amd_hfi_data->dev = &pdev->dev;
483 platform_set_drvdata(pdev, amd_hfi_data);
484
485 ret = amd_hfi_alloc_class_data(pdev);
486 if (ret)
487 return ret;
488
489 ret = amd_hfi_metadata_parser(pdev, amd_hfi_data);
490 if (ret)
491 return ret;
492
493 ret = update_hfi_ipcc_scores();
494 if (ret)
495 return ret;
496
497 /*
498 * Tasks will already be running at the time this happens. This is
499 * OK because rankings will be adjusted by the callbacks.
500 */
501 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/amd_hfi:online",
502 amd_hfi_online, amd_hfi_offline);
503 if (ret < 0)
504 return ret;
505
506 schedule_work(&sched_amd_hfi_itmt_work);
507
508 amd_hfi_data->dbgfs_dir = debugfs_create_dir("amd_hfi", arch_debugfs_dir);
509 debugfs_create_file("class_capabilities", 0644, amd_hfi_data->dbgfs_dir, pdev,
510 &class_capabilities_fops);
511
512 return 0;
513 }
514
515 static struct platform_driver amd_hfi_driver = {
516 .driver = {
517 .name = AMD_HFI_DRIVER,
518 .owner = THIS_MODULE,
519 .pm = &amd_hfi_pm_ops,
520 .acpi_match_table = ACPI_PTR(amd_hfi_platform_match),
521 },
522 .probe = amd_hfi_probe,
523 .remove = amd_hfi_remove,
524 };
525
amd_hfi_init(void)526 static int __init amd_hfi_init(void)
527 {
528 int ret;
529
530 if (acpi_disabled ||
531 !cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES) ||
532 !cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS))
533 return -ENODEV;
534
535 device = platform_device_register_simple(AMD_HFI_DRIVER, -1, NULL, 0);
536 if (IS_ERR(device)) {
537 pr_err("unable to register HFI platform device\n");
538 return PTR_ERR(device);
539 }
540
541 ret = platform_driver_register(&amd_hfi_driver);
542 if (ret)
543 pr_err("failed to register HFI driver\n");
544
545 return ret;
546 }
547
amd_hfi_exit(void)548 static __exit void amd_hfi_exit(void)
549 {
550 platform_driver_unregister(&amd_hfi_driver);
551 platform_device_unregister(device);
552 }
553 module_init(amd_hfi_init);
554 module_exit(amd_hfi_exit);
555
556 MODULE_LICENSE("GPL");
557 MODULE_DESCRIPTION("AMD Hardware Feedback Interface Driver");
558