1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * HiSilicon uncore frequency scaling driver 4 * 5 * Copyright (c) 2025 HiSilicon Co., Ltd 6 */ 7 8 #include <linux/acpi.h> 9 #include <linux/bits.h> 10 #include <linux/cleanup.h> 11 #include <linux/devfreq.h> 12 #include <linux/devfreq-governor.h> 13 #include <linux/device.h> 14 #include <linux/dev_printk.h> 15 #include <linux/errno.h> 16 #include <linux/iopoll.h> 17 #include <linux/kernel.h> 18 #include <linux/ktime.h> 19 #include <linux/mailbox_client.h> 20 #include <linux/module.h> 21 #include <linux/mutex.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_opp.h> 24 #include <linux/property.h> 25 #include <linux/topology.h> 26 #include <linux/units.h> 27 #include <acpi/pcc.h> 28 29 struct hisi_uncore_pcc_data { 30 u16 status; 31 u16 resv; 32 u32 data; 33 }; 34 35 struct hisi_uncore_pcc_shmem { 36 struct acpi_pcct_shared_memory head; 37 struct hisi_uncore_pcc_data pcc_data; 38 }; 39 40 enum hisi_uncore_pcc_cmd_type { 41 HUCF_PCC_CMD_GET_CAP = 0, 42 HUCF_PCC_CMD_GET_FREQ, 43 HUCF_PCC_CMD_SET_FREQ, 44 HUCF_PCC_CMD_GET_MODE, 45 HUCF_PCC_CMD_SET_MODE, 46 HUCF_PCC_CMD_GET_PLAT_FREQ_NUM, 47 HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX, 48 HUCF_PCC_CMD_MAX = 256 49 }; 50 51 static int hisi_platform_gov_usage; 52 static DEFINE_MUTEX(hisi_platform_gov_usage_lock); 53 54 enum hisi_uncore_freq_mode { 55 HUCF_MODE_PLATFORM = 0, 56 HUCF_MODE_OS, 57 HUCF_MODE_MAX 58 }; 59 60 #define HUCF_CAP_PLATFORM_CTRL BIT(0) 61 62 /** 63 * struct hisi_uncore_freq - hisi uncore frequency scaling device data 64 * @dev: device of this frequency scaling driver 65 * @cl: mailbox client object 66 * @pchan: PCC mailbox channel 67 * @chan_id: PCC channel ID 68 * @last_cmd_cmpl_time: timestamp of the last completed PCC command 69 * @pcc_lock: PCC channel lock 70 * @devfreq: devfreq data of this hisi_uncore_freq device 71 * @related_cpus: CPUs whose performance is majorly affected by this 72 * uncore frequency domain 73 * @cap: capability flag 74 */ 75 struct hisi_uncore_freq { 76 struct device *dev; 77 struct mbox_client cl; 78 struct pcc_mbox_chan *pchan; 79 int chan_id; 80 ktime_t last_cmd_cmpl_time; 81 struct mutex pcc_lock; 82 struct devfreq *devfreq; 83 struct cpumask related_cpus; 84 u32 cap; 85 }; 86 87 /* PCC channel timeout = PCC nominal latency * NUM */ 88 #define HUCF_PCC_POLL_TIMEOUT_NUM 1000 89 #define HUCF_PCC_POLL_INTERVAL_US 5 90 91 /* Default polling interval in ms for devfreq governors*/ 92 #define HUCF_DEFAULT_POLLING_MS 100 93 94 static void hisi_uncore_free_pcc_chan(struct hisi_uncore_freq *uncore) 95 { 96 guard(mutex)(&uncore->pcc_lock); 97 pcc_mbox_free_channel(uncore->pchan); 98 uncore->pchan = NULL; 99 } 100 101 static void devm_hisi_uncore_free_pcc_chan(void *data) 102 { 103 hisi_uncore_free_pcc_chan(data); 104 } 105 106 static int hisi_uncore_request_pcc_chan(struct hisi_uncore_freq *uncore) 107 { 108 struct device *dev = uncore->dev; 109 struct pcc_mbox_chan *pcc_chan; 110 111 uncore->cl = (struct mbox_client) { 112 .dev = dev, 113 .tx_block = false, 114 .knows_txdone = true, 115 }; 116 117 pcc_chan = pcc_mbox_request_channel(&uncore->cl, uncore->chan_id); 118 if (IS_ERR(pcc_chan)) 119 return dev_err_probe(dev, PTR_ERR(pcc_chan), 120 "Failed to request PCC channel %u\n", uncore->chan_id); 121 122 if (!pcc_chan->shmem_base_addr) { 123 pcc_mbox_free_channel(pcc_chan); 124 return dev_err_probe(dev, -EINVAL, 125 "Invalid PCC shared memory address\n"); 126 } 127 128 if (pcc_chan->shmem_size < sizeof(struct hisi_uncore_pcc_shmem)) { 129 pcc_mbox_free_channel(pcc_chan); 130 return dev_err_probe(dev, -EINVAL, 131 "Invalid PCC shared memory size (%lluB)\n", 132 pcc_chan->shmem_size); 133 } 134 135 uncore->pchan = pcc_chan; 136 137 return devm_add_action_or_reset(uncore->dev, 138 devm_hisi_uncore_free_pcc_chan, uncore); 139 } 140 141 static acpi_status hisi_uncore_pcc_reg_scan(struct acpi_resource *res, 142 void *ctx) 143 { 144 struct acpi_resource_generic_register *reg; 145 struct hisi_uncore_freq *uncore; 146 147 if (!res || res->type != ACPI_RESOURCE_TYPE_GENERIC_REGISTER) 148 return AE_OK; 149 150 reg = &res->data.generic_reg; 151 if (reg->space_id != ACPI_ADR_SPACE_PLATFORM_COMM) 152 return AE_OK; 153 154 if (!ctx) 155 return AE_ERROR; 156 157 uncore = ctx; 158 /* PCC subspace ID stored in Access Size */ 159 uncore->chan_id = reg->access_size; 160 161 return AE_CTRL_TERMINATE; 162 } 163 164 static int hisi_uncore_init_pcc_chan(struct hisi_uncore_freq *uncore) 165 { 166 acpi_handle handle = ACPI_HANDLE(uncore->dev); 167 acpi_status status; 168 int rc; 169 170 uncore->chan_id = -1; 171 status = acpi_walk_resources(handle, METHOD_NAME__CRS, 172 hisi_uncore_pcc_reg_scan, uncore); 173 if (ACPI_FAILURE(status) || uncore->chan_id < 0) 174 return dev_err_probe(uncore->dev, -ENODEV, 175 "Failed to get a PCC channel\n"); 176 177 178 rc = devm_mutex_init(uncore->dev, &uncore->pcc_lock); 179 if (rc) 180 return rc; 181 182 return hisi_uncore_request_pcc_chan(uncore); 183 } 184 185 static int hisi_uncore_cmd_send(struct hisi_uncore_freq *uncore, 186 u8 cmd, u32 *data) 187 { 188 struct hisi_uncore_pcc_shmem __iomem *addr; 189 struct hisi_uncore_pcc_shmem shmem; 190 struct pcc_mbox_chan *pchan; 191 unsigned int mrtt; 192 s64 time_delta; 193 u16 status; 194 int rc; 195 196 guard(mutex)(&uncore->pcc_lock); 197 198 pchan = uncore->pchan; 199 if (!pchan) 200 return -ENODEV; 201 202 addr = (struct hisi_uncore_pcc_shmem __iomem *)pchan->shmem; 203 if (!addr) 204 return -EINVAL; 205 206 /* Handle the Minimum Request Turnaround Time (MRTT) */ 207 mrtt = pchan->min_turnaround_time; 208 time_delta = ktime_us_delta(ktime_get(), uncore->last_cmd_cmpl_time); 209 if (mrtt > time_delta) 210 udelay(mrtt - time_delta); 211 212 /* Copy data */ 213 shmem.head = (struct acpi_pcct_shared_memory) { 214 .signature = PCC_SIGNATURE | uncore->chan_id, 215 .command = cmd, 216 }; 217 shmem.pcc_data.data = *data; 218 memcpy_toio(addr, &shmem, sizeof(shmem)); 219 220 /* Ring doorbell */ 221 rc = mbox_send_message(pchan->mchan, &cmd); 222 if (rc < 0) { 223 dev_err(uncore->dev, "Failed to send mbox message, %d\n", rc); 224 return rc; 225 } 226 227 /* Wait status */ 228 rc = readw_poll_timeout(&addr->head.status, status, 229 status & (PCC_STATUS_CMD_COMPLETE | 230 PCC_STATUS_ERROR), 231 HUCF_PCC_POLL_INTERVAL_US, 232 pchan->latency * HUCF_PCC_POLL_TIMEOUT_NUM); 233 if (rc) { 234 dev_err(uncore->dev, "PCC channel response timeout, cmd=%u\n", cmd); 235 } else if (status & PCC_STATUS_ERROR) { 236 dev_err(uncore->dev, "PCC cmd error, cmd=%u\n", cmd); 237 rc = -EIO; 238 } 239 240 uncore->last_cmd_cmpl_time = ktime_get(); 241 242 /* Copy data back */ 243 memcpy_fromio(data, &addr->pcc_data.data, sizeof(*data)); 244 245 /* Clear mailbox active req */ 246 mbox_client_txdone(pchan->mchan, rc); 247 248 return rc; 249 } 250 251 static int hisi_uncore_target(struct device *dev, unsigned long *freq, 252 u32 flags) 253 { 254 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev); 255 struct dev_pm_opp *opp; 256 u32 data; 257 258 if (WARN_ON(!uncore || !uncore->pchan)) 259 return -ENODEV; 260 261 opp = devfreq_recommended_opp(dev, freq, flags); 262 if (IS_ERR(opp)) { 263 dev_err(dev, "Failed to get opp for freq %lu hz\n", *freq); 264 return PTR_ERR(opp); 265 } 266 267 data = (u32)(dev_pm_opp_get_freq(opp) / HZ_PER_MHZ); 268 269 dev_pm_opp_put(opp); 270 271 return hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_FREQ, &data); 272 } 273 274 static int hisi_uncore_get_dev_status(struct device *dev, 275 struct devfreq_dev_status *stat) 276 { 277 /* Not used */ 278 return 0; 279 } 280 281 static int hisi_uncore_get_cur_freq(struct device *dev, unsigned long *freq) 282 { 283 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev); 284 u32 data = 0; 285 int rc; 286 287 if (WARN_ON(!uncore || !uncore->pchan)) 288 return -ENODEV; 289 290 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_FREQ, &data); 291 292 /* 293 * Upon a failure, 'data' remains 0 and 'freq' is set to 0 rather than a 294 * random value. devfreq shouldn't use 'freq' in that case though. 295 */ 296 *freq = data * HZ_PER_MHZ; 297 298 return rc; 299 } 300 301 static void devm_hisi_uncore_remove_opp(void *data) 302 { 303 struct hisi_uncore_freq *uncore = data; 304 305 dev_pm_opp_remove_all_dynamic(uncore->dev); 306 } 307 308 static int hisi_uncore_init_opp(struct hisi_uncore_freq *uncore) 309 { 310 struct device *dev = uncore->dev; 311 unsigned long freq_mhz; 312 u32 num, index; 313 u32 data = 0; 314 int rc; 315 316 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_PLAT_FREQ_NUM, 317 &data); 318 if (rc) 319 return dev_err_probe(dev, rc, "Failed to get plat freq num\n"); 320 321 num = data; 322 323 for (index = 0; index < num; index++) { 324 data = index; 325 rc = hisi_uncore_cmd_send(uncore, 326 HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX, 327 &data); 328 if (rc) { 329 dev_pm_opp_remove_all_dynamic(dev); 330 return dev_err_probe(dev, rc, 331 "Failed to get plat freq at index %u\n", index); 332 } 333 freq_mhz = data; 334 335 /* Don't care OPP voltage, take 1V as default */ 336 rc = dev_pm_opp_add(dev, freq_mhz * HZ_PER_MHZ, 1000000); 337 if (rc) { 338 dev_pm_opp_remove_all_dynamic(dev); 339 return dev_err_probe(dev, rc, 340 "Add OPP %lu failed\n", freq_mhz); 341 } 342 } 343 344 return devm_add_action_or_reset(dev, devm_hisi_uncore_remove_opp, 345 uncore); 346 } 347 348 static int hisi_platform_gov_func(struct devfreq *df, unsigned long *freq) 349 { 350 /* 351 * Platform-controlled mode doesn't care the frequency issued from 352 * devfreq, so just pick the max freq. 353 */ 354 *freq = DEVFREQ_MAX_FREQ; 355 356 return 0; 357 } 358 359 static int hisi_platform_gov_handler(struct devfreq *df, unsigned int event, 360 void *val) 361 { 362 struct hisi_uncore_freq *uncore = dev_get_drvdata(df->dev.parent); 363 int rc = 0; 364 u32 data; 365 366 if (WARN_ON(!uncore || !uncore->pchan)) 367 return -ENODEV; 368 369 switch (event) { 370 case DEVFREQ_GOV_START: 371 data = HUCF_MODE_PLATFORM; 372 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 373 if (rc) 374 dev_err(uncore->dev, "Failed to set platform mode (%d)\n", rc); 375 break; 376 case DEVFREQ_GOV_STOP: 377 data = HUCF_MODE_OS; 378 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 379 if (rc) 380 dev_err(uncore->dev, "Failed to set os mode (%d)\n", rc); 381 break; 382 default: 383 break; 384 } 385 386 return rc; 387 } 388 389 /* 390 * In the platform-controlled mode, the platform decides the uncore frequency 391 * and ignores the frequency issued from the driver. 392 * Thus, create a pseudo 'hisi_platform' governor that stops devfreq monitor 393 * from working so as to save meaningless overhead. 394 */ 395 static struct devfreq_governor hisi_platform_governor = { 396 .name = "hisi_platform", 397 /* 398 * Set interrupt_driven to skip the devfreq monitor mechanism, though 399 * this governor is not interrupt-driven. 400 */ 401 .flags = DEVFREQ_GOV_FLAG_IRQ_DRIVEN, 402 .get_target_freq = hisi_platform_gov_func, 403 .event_handler = hisi_platform_gov_handler, 404 }; 405 406 static void hisi_uncore_remove_platform_gov(struct hisi_uncore_freq *uncore) 407 { 408 u32 data = HUCF_MODE_PLATFORM; 409 int rc; 410 411 if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL)) 412 return; 413 414 guard(mutex)(&hisi_platform_gov_usage_lock); 415 416 if (--hisi_platform_gov_usage == 0) { 417 rc = devfreq_remove_governor(&hisi_platform_governor); 418 if (rc) 419 dev_err(uncore->dev, "Failed to remove hisi_platform gov (%d)\n", rc); 420 } 421 422 /* 423 * Set to the platform-controlled mode on exit if supported, so as to 424 * have a certain behaviour when the driver is detached. 425 */ 426 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data); 427 if (rc) 428 dev_err(uncore->dev, "Failed to set platform mode on exit (%d)\n", rc); 429 } 430 431 static void devm_hisi_uncore_remove_platform_gov(void *data) 432 { 433 hisi_uncore_remove_platform_gov(data); 434 } 435 436 static int hisi_uncore_add_platform_gov(struct hisi_uncore_freq *uncore) 437 { 438 if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL)) 439 return 0; 440 441 guard(mutex)(&hisi_platform_gov_usage_lock); 442 443 if (hisi_platform_gov_usage == 0) { 444 int rc = devfreq_add_governor(&hisi_platform_governor); 445 if (rc) 446 return rc; 447 } 448 hisi_platform_gov_usage++; 449 450 return devm_add_action_or_reset(uncore->dev, 451 devm_hisi_uncore_remove_platform_gov, 452 uncore); 453 } 454 455 /* 456 * Returns: 457 * 0 if success, uncore->related_cpus is set. 458 * -EINVAL if property not found, or property found but without elements in it, 459 * or invalid arguments received in any of the subroutine. 460 * Other error codes if it goes wrong. 461 */ 462 static int hisi_uncore_mark_related_cpus(struct hisi_uncore_freq *uncore, 463 char *property, int (*get_topo_id)(int cpu), 464 const struct cpumask *(*get_cpumask)(int cpu)) 465 { 466 unsigned int i, cpu; 467 size_t len; 468 int rc; 469 470 rc = device_property_count_u32(uncore->dev, property); 471 if (rc < 0) 472 return rc; 473 if (rc == 0) 474 return -EINVAL; 475 476 len = rc; 477 u32 *num __free(kfree) = kcalloc(len, sizeof(*num), GFP_KERNEL); 478 if (!num) 479 return -ENOMEM; 480 481 rc = device_property_read_u32_array(uncore->dev, property, num, len); 482 if (rc) 483 return rc; 484 485 for (i = 0; i < len; i++) { 486 for_each_possible_cpu(cpu) { 487 if (get_topo_id(cpu) != num[i]) 488 continue; 489 490 cpumask_or(&uncore->related_cpus, 491 &uncore->related_cpus, get_cpumask(cpu)); 492 break; 493 } 494 } 495 496 return 0; 497 } 498 499 static int get_package_id(int cpu) 500 { 501 return topology_physical_package_id(cpu); 502 } 503 504 static const struct cpumask *get_package_cpumask(int cpu) 505 { 506 return topology_core_cpumask(cpu); 507 } 508 509 static int get_cluster_id(int cpu) 510 { 511 return topology_cluster_id(cpu); 512 } 513 514 static const struct cpumask *get_cluster_cpumask(int cpu) 515 { 516 return topology_cluster_cpumask(cpu); 517 } 518 519 static int hisi_uncore_mark_related_cpus_wrap(struct hisi_uncore_freq *uncore) 520 { 521 int rc; 522 523 cpumask_clear(&uncore->related_cpus); 524 525 rc = hisi_uncore_mark_related_cpus(uncore, "related-package", 526 get_package_id, 527 get_package_cpumask); 528 /* Success, or firmware probably broken */ 529 if (!rc || rc != -EINVAL) 530 return rc; 531 532 /* Try another property name if rc == -EINVAL */ 533 return hisi_uncore_mark_related_cpus(uncore, "related-cluster", 534 get_cluster_id, 535 get_cluster_cpumask); 536 } 537 538 static ssize_t related_cpus_show(struct device *dev, 539 struct device_attribute *attr, char *buf) 540 { 541 struct hisi_uncore_freq *uncore = dev_get_drvdata(dev->parent); 542 543 return cpumap_print_to_pagebuf(true, buf, &uncore->related_cpus); 544 } 545 546 static DEVICE_ATTR_RO(related_cpus); 547 548 static struct attribute *hisi_uncore_freq_attrs[] = { 549 &dev_attr_related_cpus.attr, 550 NULL 551 }; 552 ATTRIBUTE_GROUPS(hisi_uncore_freq); 553 554 static int hisi_uncore_devfreq_register(struct hisi_uncore_freq *uncore) 555 { 556 struct devfreq_dev_profile *profile; 557 struct device *dev = uncore->dev; 558 unsigned long freq; 559 u32 data; 560 int rc; 561 562 rc = hisi_uncore_get_cur_freq(dev, &freq); 563 if (rc) 564 return dev_err_probe(dev, rc, "Failed to get plat init freq\n"); 565 566 profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); 567 if (!profile) 568 return -ENOMEM; 569 570 *profile = (struct devfreq_dev_profile) { 571 .initial_freq = freq, 572 .polling_ms = HUCF_DEFAULT_POLLING_MS, 573 .timer = DEVFREQ_TIMER_DELAYED, 574 .target = hisi_uncore_target, 575 .get_dev_status = hisi_uncore_get_dev_status, 576 .get_cur_freq = hisi_uncore_get_cur_freq, 577 .dev_groups = hisi_uncore_freq_groups, 578 }; 579 580 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_MODE, &data); 581 if (rc) 582 return dev_err_probe(dev, rc, "Failed to get operate mode\n"); 583 584 if (data == HUCF_MODE_PLATFORM) 585 uncore->devfreq = devm_devfreq_add_device(dev, profile, 586 hisi_platform_governor.name, NULL); 587 else 588 uncore->devfreq = devm_devfreq_add_device(dev, profile, 589 DEVFREQ_GOV_PERFORMANCE, NULL); 590 if (IS_ERR(uncore->devfreq)) 591 return dev_err_probe(dev, PTR_ERR(uncore->devfreq), 592 "Failed to add devfreq device\n"); 593 594 return 0; 595 } 596 597 static int hisi_uncore_freq_probe(struct platform_device *pdev) 598 { 599 struct hisi_uncore_freq *uncore; 600 struct device *dev = &pdev->dev; 601 u32 cap; 602 int rc; 603 604 uncore = devm_kzalloc(dev, sizeof(*uncore), GFP_KERNEL); 605 if (!uncore) 606 return -ENOMEM; 607 608 uncore->dev = dev; 609 platform_set_drvdata(pdev, uncore); 610 611 rc = hisi_uncore_init_pcc_chan(uncore); 612 if (rc) 613 return dev_err_probe(dev, rc, "Failed to init PCC channel\n"); 614 615 rc = hisi_uncore_init_opp(uncore); 616 if (rc) 617 return dev_err_probe(dev, rc, "Failed to init OPP\n"); 618 619 rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_CAP, &cap); 620 if (rc) 621 return dev_err_probe(dev, rc, "Failed to get capability\n"); 622 623 uncore->cap = cap; 624 625 rc = hisi_uncore_add_platform_gov(uncore); 626 if (rc) 627 return dev_err_probe(dev, rc, "Failed to add hisi_platform governor\n"); 628 629 rc = hisi_uncore_mark_related_cpus_wrap(uncore); 630 if (rc) 631 return dev_err_probe(dev, rc, "Failed to mark related cpus\n"); 632 633 rc = hisi_uncore_devfreq_register(uncore); 634 if (rc) 635 return dev_err_probe(dev, rc, "Failed to register devfreq\n"); 636 637 return 0; 638 } 639 640 static const struct acpi_device_id hisi_uncore_freq_acpi_match[] = { 641 { "HISI04F1", }, 642 { } 643 }; 644 MODULE_DEVICE_TABLE(acpi, hisi_uncore_freq_acpi_match); 645 646 static struct platform_driver hisi_uncore_freq_drv = { 647 .probe = hisi_uncore_freq_probe, 648 .driver = { 649 .name = "hisi_uncore_freq", 650 .acpi_match_table = hisi_uncore_freq_acpi_match, 651 }, 652 }; 653 module_platform_driver(hisi_uncore_freq_drv); 654 655 MODULE_DESCRIPTION("HiSilicon uncore frequency scaling driver"); 656 MODULE_AUTHOR("Jie Zhan <zhanjie9@hisilicon.com>"); 657 MODULE_LICENSE("GPL"); 658