1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * HiSilicon uncore frequency scaling driver 4 * 5 * Copyright (c) 2025 HiSilicon Co., Ltd 6 */ 7 8 #include <linux/acpi.h> 9 #include <linux/bits.h> 10 #include <linux/cleanup.h> 11 #include <linux/devfreq.h> 12 #include <linux/device.h> 13 #include <linux/dev_printk.h> 14 #include <linux/errno.h> 15 #include <linux/iopoll.h> 16 #include <linux/kernel.h> 17 #include <linux/ktime.h> 18 #include <linux/mailbox_client.h> 19 #include <linux/module.h> 20 #include <linux/mod_devicetable.h> 21 #include <linux/mutex.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_opp.h> 24 #include <linux/property.h> 25 #include <linux/topology.h> 26 #include <linux/units.h> 27 #include <acpi/pcc.h> 28 29 #include "governor.h" 30 31 struct hisi_uncore_pcc_data { 32 u16 status; 33 u16 resv; 34 u32 data; 35 }; 36 37 struct hisi_uncore_pcc_shmem { 38 struct acpi_pcct_shared_memory head; 39 struct hisi_uncore_pcc_data pcc_data; 40 }; 41 42 enum hisi_uncore_pcc_cmd_type { 43 HUCF_PCC_CMD_GET_CAP = 0, 44 HUCF_PCC_CMD_GET_FREQ, 45 HUCF_PCC_CMD_SET_FREQ, 46 HUCF_PCC_CMD_GET_MODE, 47 HUCF_PCC_CMD_SET_MODE, 48 HUCF_PCC_CMD_GET_PLAT_FREQ_NUM, 49 HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX, 50 HUCF_PCC_CMD_MAX = 256 51 }; 52 53 static int hisi_platform_gov_usage; 54 static DEFINE_MUTEX(hisi_platform_gov_usage_lock); 55 56 enum hisi_uncore_freq_mode { 57 HUCF_MODE_PLATFORM = 0, 58 HUCF_MODE_OS, 59 HUCF_MODE_MAX 60 }; 61 62 #define HUCF_CAP_PLATFORM_CTRL BIT(0) 63 64 /** 65 * struct hisi_uncore_freq - hisi uncore frequency scaling device data 66 * @dev: device of this frequency scaling driver 67 * @cl: mailbox client object 68 * @pchan: PCC mailbox channel 69 * @chan_id: PCC channel ID 70 * @last_cmd_cmpl_time: timestamp of the last completed PCC command 71 * @pcc_lock: PCC channel lock 72 * @devfreq: devfreq data of this hisi_uncore_freq device 73 * @related_cpus: CPUs whose performance is majorly affected by this 74 * uncore frequency domain 75 * @cap: capability flag 76 */ 77 struct hisi_uncore_freq { 78 struct device *dev; 79 struct mbox_client cl; 80 struct pcc_mbox_chan *pchan; 81 int chan_id; 82 ktime_t last_cmd_cmpl_time; 83 struct mutex pcc_lock; 84 struct devfreq *devfreq; 85 struct cpumask related_cpus; 86 u32 cap; 87 }; 88 89 /* PCC channel timeout = PCC nominal latency * NUM */ 90 #define HUCF_PCC_POLL_TIMEOUT_NUM 1000 91 #define HUCF_PCC_POLL_INTERVAL_US 5 92 93 /* Default polling interval in ms for devfreq governors*/ 94 #define HUCF_DEFAULT_POLLING_MS 100 95 96 static void hisi_uncore_free_pcc_chan(struct hisi_uncore_freq *uncore) 97 { 98 guard(mutex)(&uncore->pcc_lock); 99 pcc_mbox_free_channel(uncore->pchan); 100 uncore->pchan = NULL; 101 } 102 103 static void devm_hisi_uncore_free_pcc_chan(void *data) 104 { 105 hisi_uncore_free_pcc_chan(data); 106 } 107 108 static int hisi_uncore_request_pcc_chan(struct hisi_uncore_freq *uncore) 109 { 110 struct device *dev = uncore->dev; 111 struct pcc_mbox_chan *pcc_chan; 112 113 uncore->cl = (struct mbox_client) { 114 .dev = dev, 115 .tx_block = false, 116 .knows_txdone = true, 117 }; 118 119 pcc_chan = pcc_mbox_request_channel(&uncore->cl, uncore->chan_id); 120 if (IS_ERR(pcc_chan)) 121 return dev_err_probe(dev, PTR_ERR(pcc_chan), 122 "Failed to request PCC channel %u\n", uncore->chan_id); 123 124 if (!pcc_chan->shmem_base_addr) { 125 pcc_mbox_free_channel(pcc_chan); 126 return dev_err_probe(dev, -EINVAL, 127 "Invalid PCC shared memory address\n"); 128 } 129 130 if (pcc_chan->shmem_size < sizeof(struct hisi_uncore_pcc_shmem)) { 131 pcc_mbox_free_channel(pcc_chan); 132 return dev_err_probe(dev, -EINVAL, 133 "Invalid PCC shared memory size (%lluB)\n", 134 pcc_chan->shmem_size); 135 } 136 137 uncore->pchan = pcc_chan; 138 139 return devm_add_action_or_reset(uncore->dev, 140 devm_hisi_uncore_free_pcc_chan, uncore); 141 } 142 143 static acpi_status hisi_uncore_pcc_reg_scan(struct acpi_resource *res, 144 void *ctx) 145 { 146 struct acpi_resource_generic_register *reg; 147 struct hisi_uncore_freq *uncore; 148 149 if (!res || res->type != ACPI_RESOURCE_TYPE_GENERIC_REGISTER) 150 return AE_OK; 151 152 reg = &res->data.generic_reg; 153 if (reg->space_id != ACPI_ADR_SPACE_PLATFORM_COMM) 154 return AE_OK; 155 156 if (!ctx) 157 return AE_ERROR; 158 159 uncore = ctx; 160 /* PCC subspace ID stored in Access Size */ 161 uncore->chan_id = reg->access_size; 162 163 return AE_CTRL_TERMINATE; 164 } 165 166 static int hisi_uncore_init_pcc_chan(struct hisi_uncore_freq *uncore) 167 { 168 acpi_handle handle = ACPI_HANDLE(uncore->dev); 169 acpi_status status; 170 int rc; 171 172 uncore->chan_id = -1; 173 status = acpi_walk_resources(handle, METHOD_NAME__CRS, 174 hisi_uncore_pcc_reg_scan, uncore); 175 if (ACPI_FAILURE(status) || uncore->chan_id < 0) 176 return dev_err_probe(uncore->dev, -ENODEV, 177 "Failed to get a PCC channel\n"); 178 179 180 rc = devm_mutex_init(uncore->dev, &uncore->pcc_lock); 181 if (rc) 182 return rc; 183 184 return hisi_uncore_request_pcc_chan(uncore); 185 } 186 187 static int hisi_uncore_cmd_send(struct hisi_uncore_freq *uncore, 188 u8 cmd, u32 *data) 189 { 190 struct hisi_uncore_pcc_shmem __iomem *addr; 191 struct hisi_uncore_pcc_shmem shmem; 192 struct pcc_mbox_chan *pchan; 193 unsigned int mrtt; 194 s64 time_delta; 195 u16 status; 196 int rc; 197 198 guard(mutex)(&uncore->pcc_lock); 199 200 pchan = uncore->pchan; 201 if (!pchan) 202 return -ENODEV; 203 204 addr = (struct hisi_uncore_pcc_shmem __iomem *)pchan->shmem; 205 if (!addr) 206 return -EINVAL; 207 208 /* Handle the Minimum Request Turnaround Time (MRTT) */ 209 mrtt = pchan->min_turnaround_time; 210 time_delta = ktime_us_delta(ktime_get(), uncore->last_cmd_cmpl_time); 211 if (mrtt > time_delta) 212 udelay(mrtt - time_delta); 213 214 /* Copy data */ 215 shmem.head = (struct acpi_pcct_shared_memory) { 216 .signature = PCC_SIGNATURE | uncore->chan_id, 217 .command = cmd, 218 }; 219 shmem.pcc_data.data = *data; 220 memcpy_toio(addr, &shmem, sizeof(shmem)); 221 222 /* Ring doorbell */ 223 rc = mbox_send_message(pchan->mchan, &cmd); 224 if (rc < 0) { 225 dev_err(uncore->dev, "Failed to send mbox message, %d\n", rc); 226 return rc; 227 } 228 229 /* Wait status */ 230 rc = readw_poll_timeout(&addr->head.status, status, 231 status & (PCC_STATUS_CMD_COMPLETE | 232 PCC_STATUS_ERROR), 233 HUCF_PCC_POLL_INTERVAL_US, 234 pchan->latency * HUCF_PCC_POLL_TIMEOUT_NUM); 235 if (rc) { 236 dev_err(uncore->dev, "PCC channel response timeout, cmd=%u\n", cmd); 237 } else if (status & PCC_STATUS_ERROR) { 238 dev_err(uncore->dev, "PCC cmd error, cmd=%u\n", cmd); 239 rc = -EIO; 240 } 241 242 uncore->last_cmd_cmpl_time = ktime_get(); 243 244 /* Copy data back */ 245 memcpy_fromio(data, &addr->pcc_data.data, sizeof(*data)); 246 247 /* Clear mailbox active req */ 248 mbox_client_txdone(pchan->mchan, rc); 249 250 return rc; 251 } 252 253 static int hisi_uncore_target(struct device *dev, unsigned long *freq, 254 u32 flags) 255 { 256 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev); 257 struct dev_pm_opp *opp; 258 u32 data; 259 260 if (WARN_ON(!uncore || !uncore->pchan)) 261 return -ENODEV; 262 263 opp = devfreq_recommended_opp(dev, freq, flags); 264 if (IS_ERR(opp)) { 265 dev_err(dev, "Failed to get opp for freq %lu hz\n", *freq); 266 return PTR_ERR(opp); 267 } 268 dev_pm_opp_put(opp); 269 270 data = (u32)(dev_pm_opp_get_freq(opp) / HZ_PER_MHZ); 271 272 return hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_FREQ, &data); 273 } 274 275 static int hisi_uncore_get_dev_status(struct device *dev, 276 struct devfreq_dev_status *stat) 277 { 278 /* Not used */ 279 return 0; 280 } 281 282 static int hisi_uncore_get_cur_freq(struct device *dev, unsigned long *freq) 283 { 284 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev); 285 u32 data = 0; 286 int rc; 287 288 if (WARN_ON(!uncore || !uncore->pchan)) 289 return -ENODEV; 290 291 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_FREQ, &data); 292 293 /* 294 * Upon a failure, 'data' remains 0 and 'freq' is set to 0 rather than a 295 * random value. devfreq shouldn't use 'freq' in that case though. 296 */ 297 *freq = data * HZ_PER_MHZ; 298 299 return rc; 300 } 301 302 static void devm_hisi_uncore_remove_opp(void *data) 303 { 304 struct hisi_uncore_freq *uncore = data; 305 306 dev_pm_opp_remove_all_dynamic(uncore->dev); 307 } 308 309 static int hisi_uncore_init_opp(struct hisi_uncore_freq *uncore) 310 { 311 struct device *dev = uncore->dev; 312 unsigned long freq_mhz; 313 u32 num, index; 314 u32 data = 0; 315 int rc; 316 317 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_PLAT_FREQ_NUM, 318 &data); 319 if (rc) 320 return dev_err_probe(dev, rc, "Failed to get plat freq num\n"); 321 322 num = data; 323 324 for (index = 0; index < num; index++) { 325 data = index; 326 rc = hisi_uncore_cmd_send(uncore, 327 HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX, 328 &data); 329 if (rc) { 330 dev_pm_opp_remove_all_dynamic(dev); 331 return dev_err_probe(dev, rc, 332 "Failed to get plat freq at index %u\n", index); 333 } 334 freq_mhz = data; 335 336 /* Don't care OPP voltage, take 1V as default */ 337 rc = dev_pm_opp_add(dev, freq_mhz * HZ_PER_MHZ, 1000000); 338 if (rc) { 339 dev_pm_opp_remove_all_dynamic(dev); 340 return dev_err_probe(dev, rc, 341 "Add OPP %lu failed\n", freq_mhz); 342 } 343 } 344 345 return devm_add_action_or_reset(dev, devm_hisi_uncore_remove_opp, 346 uncore); 347 } 348 349 static int hisi_platform_gov_func(struct devfreq *df, unsigned long *freq) 350 { 351 /* 352 * Platform-controlled mode doesn't care the frequency issued from 353 * devfreq, so just pick the max freq. 354 */ 355 *freq = DEVFREQ_MAX_FREQ; 356 357 return 0; 358 } 359 360 static int hisi_platform_gov_handler(struct devfreq *df, unsigned int event, 361 void *val) 362 { 363 struct hisi_uncore_freq *uncore = dev_get_drvdata(df->dev.parent); 364 int rc = 0; 365 u32 data; 366 367 if (WARN_ON(!uncore || !uncore->pchan)) 368 return -ENODEV; 369 370 switch (event) { 371 case DEVFREQ_GOV_START: 372 data = HUCF_MODE_PLATFORM; 373 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 374 if (rc) 375 dev_err(uncore->dev, "Failed to set platform mode (%d)\n", rc); 376 break; 377 case DEVFREQ_GOV_STOP: 378 data = HUCF_MODE_OS; 379 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 380 if (rc) 381 dev_err(uncore->dev, "Failed to set os mode (%d)\n", rc); 382 break; 383 default: 384 break; 385 } 386 387 return rc; 388 } 389 390 /* 391 * In the platform-controlled mode, the platform decides the uncore frequency 392 * and ignores the frequency issued from the driver. 393 * Thus, create a pseudo 'hisi_platform' governor that stops devfreq monitor 394 * from working so as to save meaningless overhead. 395 */ 396 static struct devfreq_governor hisi_platform_governor = { 397 .name = "hisi_platform", 398 /* 399 * Set interrupt_driven to skip the devfreq monitor mechanism, though 400 * this governor is not interrupt-driven. 401 */ 402 .flags = DEVFREQ_GOV_FLAG_IRQ_DRIVEN, 403 .get_target_freq = hisi_platform_gov_func, 404 .event_handler = hisi_platform_gov_handler, 405 }; 406 407 static void hisi_uncore_remove_platform_gov(struct hisi_uncore_freq *uncore) 408 { 409 u32 data = HUCF_MODE_PLATFORM; 410 int rc; 411 412 if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL)) 413 return; 414 415 guard(mutex)(&hisi_platform_gov_usage_lock); 416 417 if (--hisi_platform_gov_usage == 0) { 418 rc = devfreq_remove_governor(&hisi_platform_governor); 419 if (rc) 420 dev_err(uncore->dev, "Failed to remove hisi_platform gov (%d)\n", rc); 421 } 422 423 /* 424 * Set to the platform-controlled mode on exit if supported, so as to 425 * have a certain behaviour when the driver is detached. 426 */ 427 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 428 if (rc) 429 dev_err(uncore->dev, "Failed to set platform mode on exit (%d)\n", rc); 430 } 431 432 static void devm_hisi_uncore_remove_platform_gov(void *data) 433 { 434 hisi_uncore_remove_platform_gov(data); 435 } 436 437 static int hisi_uncore_add_platform_gov(struct hisi_uncore_freq *uncore) 438 { 439 if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL)) 440 return 0; 441 442 guard(mutex)(&hisi_platform_gov_usage_lock); 443 444 if (hisi_platform_gov_usage == 0) { 445 int rc = devfreq_add_governor(&hisi_platform_governor); 446 if (rc) 447 return rc; 448 } 449 hisi_platform_gov_usage++; 450 451 return devm_add_action_or_reset(uncore->dev, 452 devm_hisi_uncore_remove_platform_gov, 453 uncore); 454 } 455 456 /* 457 * Returns: 458 * 0 if success, uncore->related_cpus is set. 459 * -EINVAL if property not found, or property found but without elements in it, 460 * or invalid arguments received in any of the subroutine. 461 * Other error codes if it goes wrong. 462 */ 463 static int hisi_uncore_mark_related_cpus(struct hisi_uncore_freq *uncore, 464 char *property, int (*get_topo_id)(int cpu), 465 const struct cpumask *(*get_cpumask)(int cpu)) 466 { 467 unsigned int i, cpu; 468 size_t len; 469 int rc; 470 471 rc = device_property_count_u32(uncore->dev, property); 472 if (rc < 0) 473 return rc; 474 if (rc == 0) 475 return -EINVAL; 476 477 len = rc; 478 u32 *num __free(kfree) = kcalloc(len, sizeof(*num), GFP_KERNEL); 479 if (!num) 480 return -ENOMEM; 481 482 rc = device_property_read_u32_array(uncore->dev, property, num, len); 483 if (rc) 484 return rc; 485 486 for (i = 0; i < len; i++) { 487 for_each_possible_cpu(cpu) { 488 if (get_topo_id(cpu) != num[i]) 489 continue; 490 491 cpumask_or(&uncore->related_cpus, 492 &uncore->related_cpus, get_cpumask(cpu)); 493 break; 494 } 495 } 496 497 return 0; 498 } 499 500 static int get_package_id(int cpu) 501 { 502 return topology_physical_package_id(cpu); 503 } 504 505 static const struct cpumask *get_package_cpumask(int cpu) 506 { 507 return topology_core_cpumask(cpu); 508 } 509 510 static int get_cluster_id(int cpu) 511 { 512 return topology_cluster_id(cpu); 513 } 514 515 static const struct cpumask *get_cluster_cpumask(int cpu) 516 { 517 return topology_cluster_cpumask(cpu); 518 } 519 520 static int hisi_uncore_mark_related_cpus_wrap(struct hisi_uncore_freq *uncore) 521 { 522 int rc; 523 524 cpumask_clear(&uncore->related_cpus); 525 526 rc = hisi_uncore_mark_related_cpus(uncore, "related-package", 527 get_package_id, 528 get_package_cpumask); 529 /* Success, or firmware probably broken */ 530 if (!rc || rc != -EINVAL) 531 return rc; 532 533 /* Try another property name if rc == -EINVAL */ 534 return hisi_uncore_mark_related_cpus(uncore, "related-cluster", 535 get_cluster_id, 536 get_cluster_cpumask); 537 } 538 539 static ssize_t related_cpus_show(struct device *dev, 540 struct device_attribute *attr, char *buf) 541 { 542 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev->parent); 543 544 return cpumap_print_to_pagebuf(true, buf, &uncore->related_cpus); 545 } 546 547 static DEVICE_ATTR_RO(related_cpus); 548 549 static struct attribute *hisi_uncore_freq_attrs[] = { 550 &dev_attr_related_cpus.attr, 551 NULL 552 }; 553 ATTRIBUTE_GROUPS(hisi_uncore_freq); 554 555 static int hisi_uncore_devfreq_register(struct hisi_uncore_freq *uncore) 556 { 557 struct devfreq_dev_profile *profile; 558 struct device *dev = uncore->dev; 559 unsigned long freq; 560 u32 data; 561 int rc; 562 563 rc = hisi_uncore_get_cur_freq(dev, &freq); 564 if (rc) 565 return dev_err_probe(dev, rc, "Failed to get plat init freq\n"); 566 567 profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); 568 if (!profile) 569 return -ENOMEM; 570 571 *profile = (struct devfreq_dev_profile) { 572 .initial_freq = freq, 573 .polling_ms = HUCF_DEFAULT_POLLING_MS, 574 .timer = DEVFREQ_TIMER_DELAYED, 575 .target = hisi_uncore_target, 576 .get_dev_status = hisi_uncore_get_dev_status, 577 .get_cur_freq = hisi_uncore_get_cur_freq, 578 .dev_groups = hisi_uncore_freq_groups, 579 }; 580 581 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_MODE, &data); 582 if (rc) 583 return dev_err_probe(dev, rc, "Failed to get operate mode\n"); 584 585 if (data == HUCF_MODE_PLATFORM) 586 uncore->devfreq = devm_devfreq_add_device(dev, profile, 587 hisi_platform_governor.name, NULL); 588 else 589 uncore->devfreq = devm_devfreq_add_device(dev, profile, 590 DEVFREQ_GOV_PERFORMANCE, NULL); 591 if (IS_ERR(uncore->devfreq)) 592 return dev_err_probe(dev, PTR_ERR(uncore->devfreq), 593 "Failed to add devfreq device\n"); 594 595 return 0; 596 } 597 598 static int hisi_uncore_freq_probe(struct platform_device *pdev) 599 { 600 struct hisi_uncore_freq *uncore; 601 struct device *dev = &pdev->dev; 602 u32 cap; 603 int rc; 604 605 uncore = devm_kzalloc(dev, sizeof(*uncore), GFP_KERNEL); 606 if (!uncore) 607 return -ENOMEM; 608 609 uncore->dev = dev; 610 platform_set_drvdata(pdev, uncore); 611 612 rc = hisi_uncore_init_pcc_chan(uncore); 613 if (rc) 614 return dev_err_probe(dev, rc, "Failed to init PCC channel\n"); 615 616 rc = hisi_uncore_init_opp(uncore); 617 if (rc) 618 return dev_err_probe(dev, rc, "Failed to init OPP\n"); 619 620 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_CAP, &cap); 621 if (rc) 622 return dev_err_probe(dev, rc, "Failed to get capability\n"); 623 624 uncore->cap = cap; 625 626 rc = hisi_uncore_add_platform_gov(uncore); 627 if (rc) 628 return dev_err_probe(dev, rc, "Failed to add hisi_platform governor\n"); 629 630 rc = hisi_uncore_mark_related_cpus_wrap(uncore); 631 if (rc) 632 return dev_err_probe(dev, rc, "Failed to mark related cpus\n"); 633 634 rc = hisi_uncore_devfreq_register(uncore); 635 if (rc) 636 return dev_err_probe(dev, rc, "Failed to register devfreq\n"); 637 638 return 0; 639 } 640 641 static const struct acpi_device_id hisi_uncore_freq_acpi_match[] = { 642 { "HISI04F1", }, 643 { } 644 }; 645 MODULE_DEVICE_TABLE(acpi, hisi_uncore_freq_acpi_match); 646 647 static struct platform_driver hisi_uncore_freq_drv = { 648 .probe = hisi_uncore_freq_probe, 649 .driver = { 650 .name = "hisi_uncore_freq", 651 .acpi_match_table = hisi_uncore_freq_acpi_match, 652 }, 653 }; 654 module_platform_driver(hisi_uncore_freq_drv); 655 656 MODULE_DESCRIPTION("HiSilicon uncore frequency scaling driver"); 657 MODULE_AUTHOR("Jie Zhan <zhanjie9@hisilicon.com>"); 658 MODULE_LICENSE("GPL"); 659