1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI 4 * 5 * Copyright (c) 2023, Intel Corporation. 6 * All Rights Reserved. 7 * 8 * The hardware interface to read/write is basically substitution of 9 * MSR 0x620 and 0x621. 10 * There are specific MMIO offset and bits to get/set minimum and 11 * maximum uncore ratio, similar to MSRs. 12 * The scope of the uncore MSRs was package scope. But TPMI allows 13 * new gen CPUs to have multiple uncore controls at uncore-cluster 14 * level. Each package can have multiple power domains which further 15 * can have multiple clusters. 16 * Here number of power domains = number of resources in this aux 17 * device. There are offsets and bits to discover number of clusters 18 * and offset for each cluster level controls. 19 * 20 */ 21 22 #include <linux/auxiliary_bus.h> 23 #include <linux/bitfield.h> 24 #include <linux/bits.h> 25 #include <linux/intel_tpmi.h> 26 #include <linux/intel_vsec.h> 27 #include <linux/io.h> 28 #include <linux/module.h> 29 30 #include "../tpmi_power_domains.h" 31 #include "uncore-frequency-common.h" 32 33 #define UNCORE_MAJOR_VERSION 0 34 #define UNCORE_MINOR_VERSION 3 35 #define UNCORE_ELC_SUPPORTED_VERSION 2 36 #define UNCORE_HEADER_INDEX 0 37 #define UNCORE_FABRIC_CLUSTER_OFFSET 8 38 39 /* status + control + adv_ctl1 + adv_ctl2 */ 40 #define UNCORE_FABRIC_CLUSTER_SIZE (4 * 8) 41 42 #define UNCORE_STATUS_INDEX 0 43 #define UNCORE_CONTROL_INDEX 8 44 45 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000 46 47 struct tpmi_uncore_struct; 48 49 /* Information for each cluster */ 50 struct tpmi_uncore_cluster_info { 51 bool root_domain; 52 bool elc_supported; 53 u8 __iomem *cluster_base; 54 u16 cdie_id; 55 struct uncore_data uncore_data; 56 struct tpmi_uncore_struct *uncore_root; 57 }; 58 59 /* Information for each power domain */ 60 struct tpmi_uncore_power_domain_info { 61 u8 __iomem *uncore_base; 62 int ufs_header_ver; 63 int cluster_count; 64 struct tpmi_uncore_cluster_info *cluster_infos; 65 }; 66 67 /* Information for all power domains in a package */ 68 struct tpmi_uncore_struct { 69 int power_domain_count; 70 int max_ratio; 71 int min_ratio; 72 struct tpmi_uncore_power_domain_info *pd_info; 73 struct tpmi_uncore_cluster_info root_cluster; 74 bool write_blocked; 75 }; 76 77 /* Bit definitions for STATUS register */ 78 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0) 79 80 /* Bit definitions for CONTROL register */ 81 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(14, 8) 82 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(21, 15) 83 #define UNCORE_EFF_LAT_CTRL_RATIO_MASK GENMASK_ULL(28, 22) 84 #define UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK GENMASK_ULL(38, 32) 85 #define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE BIT(39) 86 #define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK GENMASK_ULL(46, 40) 87 88 /* Helper function to read MMIO offset for max/min control frequency */ 89 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info, 90 unsigned int *value, enum uncore_index index) 91 { 92 u64 control; 93 94 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 95 if (index == UNCORE_INDEX_MAX_FREQ) 96 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 97 else 98 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 99 } 100 101 /* Helper function to read efficiency latency control values over MMIO */ 102 static int read_eff_lat_ctrl(struct uncore_data *data, unsigned int *val, enum uncore_index index) 103 { 104 struct tpmi_uncore_cluster_info *cluster_info; 105 u64 ctrl; 106 107 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 108 if (cluster_info->root_domain) 109 return -ENODATA; 110 111 if (!cluster_info->elc_supported) 112 return -EOPNOTSUPP; 113 114 ctrl = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 115 116 switch (index) { 117 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD: 118 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, ctrl); 119 *val *= 100; 120 *val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK)); 121 break; 122 123 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD: 124 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, ctrl); 125 *val *= 100; 126 *val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK)); 127 break; 128 129 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE: 130 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, ctrl); 131 break; 132 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ: 133 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_RATIO_MASK, ctrl) * UNCORE_FREQ_KHZ_MULTIPLIER; 134 break; 135 136 default: 137 return -EOPNOTSUPP; 138 } 139 140 return 0; 141 } 142 143 #define UNCORE_MAX_RATIO FIELD_MAX(UNCORE_MAX_RATIO_MASK) 144 145 /* Helper for sysfs read for max/min frequencies. Called under mutex locks */ 146 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value, 147 enum uncore_index index) 148 { 149 struct tpmi_uncore_cluster_info *cluster_info; 150 151 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 152 153 if (cluster_info->root_domain) { 154 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 155 unsigned int min, max, v; 156 int i; 157 158 min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER; 159 max = 0; 160 161 /* 162 * Get the max/min by looking at each cluster. Get the lowest 163 * min and highest max. 164 */ 165 for (i = 0; i < uncore_root->power_domain_count; ++i) { 166 int j; 167 168 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) { 169 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 170 &v, index); 171 if (v < min) 172 min = v; 173 if (v > max) 174 max = v; 175 } 176 } 177 178 if (index == UNCORE_INDEX_MIN_FREQ) 179 *value = min; 180 else 181 *value = max; 182 183 return 0; 184 } 185 186 read_control_freq(cluster_info, value, index); 187 188 return 0; 189 } 190 191 /* Helper function for writing efficiency latency control values over MMIO */ 192 static int write_eff_lat_ctrl(struct uncore_data *data, unsigned int val, enum uncore_index index) 193 { 194 struct tpmi_uncore_cluster_info *cluster_info; 195 struct tpmi_uncore_struct *uncore_root; 196 u64 control; 197 198 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 199 uncore_root = cluster_info->uncore_root; 200 201 if (uncore_root->write_blocked) 202 return -EPERM; 203 204 if (cluster_info->root_domain) 205 return -ENODATA; 206 207 if (!cluster_info->elc_supported) 208 return -EOPNOTSUPP; 209 210 switch (index) { 211 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD: 212 if (val > 100) 213 return -EINVAL; 214 break; 215 216 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD: 217 if (val > 100) 218 return -EINVAL; 219 break; 220 221 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE: 222 if (val > 1) 223 return -EINVAL; 224 break; 225 226 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ: 227 val /= UNCORE_FREQ_KHZ_MULTIPLIER; 228 if (val > FIELD_MAX(UNCORE_EFF_LAT_CTRL_RATIO_MASK)) 229 return -EINVAL; 230 break; 231 232 default: 233 return -EOPNOTSUPP; 234 } 235 236 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 237 238 switch (index) { 239 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD: 240 val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK); 241 val /= 100; 242 control &= ~UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK; 243 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, val); 244 break; 245 246 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD: 247 val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK); 248 val /= 100; 249 control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK; 250 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, val); 251 break; 252 253 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE: 254 control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE; 255 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, val); 256 break; 257 258 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ: 259 control &= ~UNCORE_EFF_LAT_CTRL_RATIO_MASK; 260 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_RATIO_MASK, val); 261 break; 262 263 default: 264 break; 265 } 266 267 writeq(control, cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 268 269 return 0; 270 } 271 272 /* Helper function to write MMIO offset for max/min control frequency */ 273 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input, 274 unsigned int index) 275 { 276 u64 control; 277 278 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 279 280 if (index == UNCORE_INDEX_MAX_FREQ) { 281 control &= ~UNCORE_MAX_RATIO_MASK; 282 control |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input); 283 } else { 284 control &= ~UNCORE_MIN_RATIO_MASK; 285 control |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input); 286 } 287 288 writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX)); 289 } 290 291 /* Helper for sysfs write for max/min frequencies. Called under mutex locks */ 292 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input, 293 enum uncore_index index) 294 { 295 struct tpmi_uncore_cluster_info *cluster_info; 296 struct tpmi_uncore_struct *uncore_root; 297 298 input /= UNCORE_FREQ_KHZ_MULTIPLIER; 299 if (!input || input > UNCORE_MAX_RATIO) 300 return -EINVAL; 301 302 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 303 uncore_root = cluster_info->uncore_root; 304 305 if (uncore_root->write_blocked) 306 return -EPERM; 307 308 /* Update each cluster in a package */ 309 if (cluster_info->root_domain) { 310 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 311 int i; 312 313 for (i = 0; i < uncore_root->power_domain_count; ++i) { 314 int j; 315 316 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) 317 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 318 input, index); 319 } 320 321 if (index == UNCORE_INDEX_MAX_FREQ) 322 uncore_root->max_ratio = input; 323 else 324 uncore_root->min_ratio = input; 325 326 return 0; 327 } 328 329 if (index == UNCORE_INDEX_MAX_FREQ && uncore_root->max_ratio && 330 uncore_root->max_ratio < input) 331 return -EINVAL; 332 333 if (index == UNCORE_INDEX_MIN_FREQ && uncore_root->min_ratio && 334 uncore_root->min_ratio > input) 335 return -EINVAL; 336 337 write_control_freq(cluster_info, input, index); 338 339 return 0; 340 } 341 342 /* Helper for sysfs read for the current uncore frequency. Called under mutex locks */ 343 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq) 344 { 345 struct tpmi_uncore_cluster_info *cluster_info; 346 u64 status; 347 348 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 349 if (cluster_info->root_domain) 350 return -ENODATA; 351 352 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX); 353 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, status) * UNCORE_FREQ_KHZ_MULTIPLIER; 354 355 return 0; 356 } 357 358 /* 359 * Agent types as per the TPMI UFS Specification for UFS_STATUS 360 * Agent Type - Core Bit: 23 361 * Agent Type - Cache Bit: 24 362 * Agent Type - Memory Bit: 25 363 * Agent Type - IO Bit: 26 364 */ 365 366 #define UNCORE_AGENT_TYPES GENMASK_ULL(26, 23) 367 368 /* Helper function to read agent type over MMIO and set the agent type mask */ 369 static void uncore_set_agent_type(struct tpmi_uncore_cluster_info *cluster_info) 370 { 371 u64 status; 372 373 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX); 374 cluster_info->uncore_data.agent_type_mask = FIELD_GET(UNCORE_AGENT_TYPES, status); 375 } 376 377 #define MAX_PARTITIONS 2 378 379 /* IO domain ID start index for a partition */ 380 static u8 io_die_start[MAX_PARTITIONS]; 381 382 /* Next IO domain ID index after the current partition IO die IDs */ 383 static u8 io_die_index_next; 384 385 /* Lock to protect io_die_start, io_die_index_next */ 386 static DEFINE_MUTEX(domain_lock); 387 388 static inline void set_instance_id(int id, struct tpmi_uncore_cluster_info *cluster_info) 389 { 390 /* 391 * On non-partitioned systems domain_id can be used for mapping both 392 * CPUs to compute die IDs and physical die indexes to MMIO mapped 393 * memory. However on partitioned systems domain_id loses the second 394 * association. Therefore instance_id should be used for that instead, 395 * while domain_id should still be used to match CPUs to compute dies. 396 */ 397 cluster_info->uncore_data.instance_id = id; 398 } 399 400 static void set_domain_id(int id, int num_resources, 401 struct oobmsm_plat_info *plat_info, 402 struct tpmi_uncore_cluster_info *cluster_info) 403 { 404 u8 part_io_index, cdie_range, pkg_io_index, max_dies; 405 406 if (plat_info->partition >= MAX_PARTITIONS) { 407 cluster_info->uncore_data.domain_id = id; 408 return; 409 } 410 411 if (cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE) { 412 cluster_info->uncore_data.domain_id = cluster_info->cdie_id; 413 return; 414 } 415 416 /* Unlikely but cdie_mask may have holes, so take range */ 417 cdie_range = fls(plat_info->cdie_mask) - ffs(plat_info->cdie_mask) + 1; 418 max_dies = topology_max_dies_per_package(); 419 420 /* 421 * If the CPU doesn't enumerate dies, then use current cdie range 422 * as the max. 423 */ 424 if (cdie_range > max_dies) 425 max_dies = cdie_range; 426 427 guard(mutex)(&domain_lock); 428 429 if (!io_die_index_next) 430 io_die_index_next = max_dies; 431 432 if (!io_die_start[plat_info->partition]) { 433 io_die_start[plat_info->partition] = io_die_index_next; 434 /* 435 * number of IO dies = num_resources - cdie_range. Hence 436 * next partition io_die_index_next is set after IO dies 437 * in the current partition. 438 */ 439 io_die_index_next += (num_resources - cdie_range); 440 } 441 442 /* 443 * Index from IO die start within the partition: 444 * This is the first valid domain after the cdies. 445 * For example the current resource index 5 and cdies end at 446 * index 3 (cdie_cnt = 4). Then the IO only index 5 - 4 = 1. 447 */ 448 part_io_index = id - cdie_range; 449 450 /* 451 * Add to the IO die start index for this partition in this package 452 * to make unique in the package. 453 */ 454 pkg_io_index = io_die_start[plat_info->partition] + part_io_index; 455 456 /* Assign this to domain ID */ 457 cluster_info->uncore_data.domain_id = pkg_io_index; 458 } 459 460 /* Callback for sysfs read for TPMI uncore values. Called under mutex locks. */ 461 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index) 462 { 463 struct tpmi_uncore_cluster_info *cluster_info; 464 int ret; 465 466 switch (index) { 467 case UNCORE_INDEX_MIN_FREQ: 468 case UNCORE_INDEX_MAX_FREQ: 469 return uncore_read_control_freq(data, value, index); 470 471 case UNCORE_INDEX_CURRENT_FREQ: 472 return uncore_read_freq(data, value); 473 474 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD: 475 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD: 476 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE: 477 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ: 478 return read_eff_lat_ctrl(data, value, index); 479 480 case UNCORE_INDEX_DIE_ID: 481 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 482 ret = tpmi_get_linux_die_id(cluster_info->uncore_data.package_id, 483 cluster_info->cdie_id); 484 if (ret < 0) 485 return ret; 486 487 *value = ret; 488 return 0; 489 490 default: 491 break; 492 } 493 494 return -EOPNOTSUPP; 495 } 496 497 /* Callback for sysfs write for TPMI uncore data. Called under mutex locks. */ 498 static int uncore_write(struct uncore_data *data, unsigned int value, enum uncore_index index) 499 { 500 switch (index) { 501 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD: 502 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD: 503 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE: 504 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ: 505 return write_eff_lat_ctrl(data, value, index); 506 507 case UNCORE_INDEX_MIN_FREQ: 508 case UNCORE_INDEX_MAX_FREQ: 509 return uncore_write_control_freq(data, value, index); 510 511 default: 512 break; 513 } 514 515 return -EOPNOTSUPP; 516 } 517 518 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore) 519 { 520 int i; 521 522 for (i = 0; i < tpmi_uncore->power_domain_count; ++i) { 523 struct tpmi_uncore_power_domain_info *pd_info; 524 int j; 525 526 pd_info = &tpmi_uncore->pd_info[i]; 527 if (!pd_info->uncore_base) 528 continue; 529 530 for (j = 0; j < pd_info->cluster_count; ++j) { 531 struct tpmi_uncore_cluster_info *cluster_info; 532 533 cluster_info = &pd_info->cluster_infos[j]; 534 uncore_freq_remove_die_entry(&cluster_info->uncore_data); 535 } 536 } 537 } 538 539 static void set_cdie_id(int domain_id, struct tpmi_uncore_cluster_info *cluster_info, 540 struct oobmsm_plat_info *plat_info) 541 { 542 543 cluster_info->cdie_id = domain_id; 544 545 if (plat_info->cdie_mask && cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE) 546 cluster_info->cdie_id = domain_id + ffs(plat_info->cdie_mask) - 1; 547 } 548 549 #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0) 550 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8) 551 #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0) 552 #define UNCORE_AUTONOMOUS_UFS_DISABLED BIT(32) 553 #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8 554 555 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) 556 { 557 bool read_blocked = 0, write_blocked = 0; 558 struct oobmsm_plat_info *plat_info; 559 struct tpmi_uncore_struct *tpmi_uncore; 560 bool uncore_sysfs_added = false; 561 int ret, i, pkg = 0; 562 int num_resources; 563 564 ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked); 565 if (ret) 566 dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n"); 567 568 if (read_blocked) { 569 dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n"); 570 return -ENODEV; 571 } 572 573 /* Get number of power domains, which is equal to number of resources */ 574 num_resources = tpmi_get_resource_count(auxdev); 575 if (!num_resources) 576 return -EINVAL; 577 578 /* Register callbacks to uncore core */ 579 ret = uncore_freq_common_init(uncore_read, uncore_write); 580 if (ret) 581 return ret; 582 583 /* Allocate uncore instance per package */ 584 tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL); 585 if (!tpmi_uncore) { 586 ret = -ENOMEM; 587 goto err_rem_common; 588 } 589 590 /* Allocate memory for all power domains in a package */ 591 tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources, 592 sizeof(*tpmi_uncore->pd_info), 593 GFP_KERNEL); 594 if (!tpmi_uncore->pd_info) { 595 ret = -ENOMEM; 596 goto err_rem_common; 597 } 598 599 tpmi_uncore->power_domain_count = num_resources; 600 tpmi_uncore->write_blocked = write_blocked; 601 602 /* Get the package ID from the TPMI core */ 603 plat_info = tpmi_get_platform_data(auxdev); 604 if (unlikely(!plat_info)) { 605 dev_info(&auxdev->dev, "Platform information is NULL\n"); 606 ret = -ENODEV; 607 goto err_rem_common; 608 } 609 610 pkg = plat_info->package_id; 611 612 for (i = 0; i < num_resources; ++i) { 613 struct tpmi_uncore_power_domain_info *pd_info; 614 bool auto_ufs_enabled; 615 struct resource *res; 616 u64 cluster_offset; 617 u8 cluster_mask; 618 int mask, j; 619 u64 header; 620 621 res = tpmi_get_resource_at_index(auxdev, i); 622 if (!res) 623 continue; 624 625 pd_info = &tpmi_uncore->pd_info[i]; 626 627 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res); 628 if (IS_ERR(pd_info->uncore_base)) { 629 ret = PTR_ERR(pd_info->uncore_base); 630 /* 631 * Set to NULL so that clean up can still remove other 632 * entries already created if any by 633 * remove_cluster_entries() 634 */ 635 pd_info->uncore_base = NULL; 636 goto remove_clusters; 637 } 638 639 /* Check for version and skip this resource if there is mismatch */ 640 header = readq(pd_info->uncore_base); 641 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK; 642 643 if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID) 644 continue; 645 646 if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) { 647 dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n", 648 TPMI_MAJOR_VERSION(pd_info->ufs_header_ver)); 649 ret = -ENODEV; 650 goto remove_clusters; 651 } 652 653 if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) > UNCORE_MINOR_VERSION) 654 dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n", 655 TPMI_MINOR_VERSION(pd_info->ufs_header_ver)); 656 657 /* Get Cluster ID Mask */ 658 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header); 659 if (!cluster_mask) { 660 dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask); 661 continue; 662 } 663 664 auto_ufs_enabled = !(header & UNCORE_AUTONOMOUS_UFS_DISABLED); 665 666 /* Find out number of clusters in this resource */ 667 pd_info->cluster_count = hweight8(cluster_mask); 668 669 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count, 670 sizeof(struct tpmi_uncore_cluster_info), 671 GFP_KERNEL); 672 if (!pd_info->cluster_infos) { 673 ret = -ENOMEM; 674 goto remove_clusters; 675 } 676 /* 677 * Each byte in the register point to status and control 678 * registers belonging to cluster id 0-8. 679 */ 680 cluster_offset = readq(pd_info->uncore_base + 681 UNCORE_FABRIC_CLUSTER_OFFSET); 682 683 for (j = 0; j < pd_info->cluster_count; ++j) { 684 struct tpmi_uncore_cluster_info *cluster_info; 685 686 /* Get the offset for this cluster */ 687 mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK); 688 /* Offset in QWORD, so change to bytes */ 689 mask <<= 3; 690 691 cluster_info = &pd_info->cluster_infos[j]; 692 693 cluster_info->cluster_base = pd_info->uncore_base + mask; 694 695 uncore_set_agent_type(cluster_info); 696 697 cluster_info->uncore_data.package_id = pkg; 698 /* There are no dies like Cascade Lake */ 699 cluster_info->uncore_data.die_id = 0; 700 cluster_info->uncore_data.cluster_id = j; 701 702 set_cdie_id(i, cluster_info, plat_info); 703 704 set_domain_id(i, num_resources, plat_info, cluster_info); 705 set_instance_id(i, cluster_info); 706 707 cluster_info->uncore_root = tpmi_uncore; 708 709 if ((TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >= 710 UNCORE_ELC_SUPPORTED_VERSION) && 711 auto_ufs_enabled) 712 cluster_info->elc_supported = true; 713 714 ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0); 715 if (ret) { 716 cluster_info->cluster_base = NULL; 717 goto remove_clusters; 718 } 719 /* Point to next cluster offset */ 720 cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN; 721 uncore_sysfs_added = true; 722 } 723 } 724 725 if (!uncore_sysfs_added) { 726 ret = -ENODEV; 727 goto remove_clusters; 728 } 729 730 auxiliary_set_drvdata(auxdev, tpmi_uncore); 731 732 if (topology_max_dies_per_package() > 1 || plat_info->partition) 733 return 0; 734 735 tpmi_uncore->root_cluster.root_domain = true; 736 tpmi_uncore->root_cluster.uncore_root = tpmi_uncore; 737 738 tpmi_uncore->root_cluster.uncore_data.package_id = pkg; 739 tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID; 740 ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0); 741 if (ret) 742 goto remove_clusters; 743 744 return 0; 745 746 remove_clusters: 747 remove_cluster_entries(tpmi_uncore); 748 err_rem_common: 749 uncore_freq_common_exit(); 750 751 return ret; 752 } 753 754 static void uncore_remove(struct auxiliary_device *auxdev) 755 { 756 struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev); 757 758 if (tpmi_uncore->root_cluster.root_domain) 759 uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data); 760 761 remove_cluster_entries(tpmi_uncore); 762 763 uncore_freq_common_exit(); 764 } 765 766 static const struct auxiliary_device_id intel_uncore_id_table[] = { 767 { .name = "intel_vsec.tpmi-uncore" }, 768 {} 769 }; 770 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table); 771 772 static struct auxiliary_driver intel_uncore_aux_driver = { 773 .id_table = intel_uncore_id_table, 774 .remove = uncore_remove, 775 .probe = uncore_probe, 776 }; 777 778 module_auxiliary_driver(intel_uncore_aux_driver); 779 780 MODULE_IMPORT_NS("INTEL_TPMI"); 781 MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY"); 782 MODULE_IMPORT_NS("INTEL_TPMI_POWER_DOMAIN"); 783 MODULE_DESCRIPTION("Intel TPMI UFS Driver"); 784 MODULE_LICENSE("GPL"); 785