1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
4 *
5 * Copyright (c) 2023, Intel Corporation.
6 * All Rights Reserved.
7 *
8 * The hardware interface to read/write is basically substitution of
9 * MSR 0x620 and 0x621.
10 * There are specific MMIO offset and bits to get/set minimum and
11 * maximum uncore ratio, similar to MSRs.
12 * The scope of the uncore MSRs was package scope. But TPMI allows
13 * new gen CPUs to have multiple uncore controls at uncore-cluster
14 * level. Each package can have multiple power domains which further
15 * can have multiple clusters.
16 * Here number of power domains = number of resources in this aux
17 * device. There are offsets and bits to discover number of clusters
18 * and offset for each cluster level controls.
19 *
20 */
21
22 #include <linux/auxiliary_bus.h>
23 #include <linux/bitfield.h>
24 #include <linux/bits.h>
25 #include <linux/intel_tpmi.h>
26 #include <linux/intel_vsec.h>
27 #include <linux/io.h>
28 #include <linux/module.h>
29
30 #include "../tpmi_power_domains.h"
31 #include "uncore-frequency-common.h"
32
33 #define UNCORE_MAJOR_VERSION 0
34 #define UNCORE_MINOR_VERSION 3
35 #define UNCORE_ELC_SUPPORTED_VERSION 2
36 #define UNCORE_HEADER_INDEX 0
37 #define UNCORE_FABRIC_CLUSTER_OFFSET 8
38
39 /* status + control + adv_ctl1 + adv_ctl2 */
40 #define UNCORE_FABRIC_CLUSTER_SIZE (4 * 8)
41
42 #define UNCORE_STATUS_INDEX 0
43 #define UNCORE_CONTROL_INDEX 8
44
45 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
46
47 struct tpmi_uncore_struct;
48
49 /* Information for each cluster */
50 struct tpmi_uncore_cluster_info {
51 bool root_domain;
52 bool elc_supported;
53 u8 __iomem *cluster_base;
54 u16 cdie_id;
55 struct uncore_data uncore_data;
56 struct tpmi_uncore_struct *uncore_root;
57 };
58
59 /* Information for each power domain */
60 struct tpmi_uncore_power_domain_info {
61 u8 __iomem *uncore_base;
62 int ufs_header_ver;
63 int cluster_count;
64 struct tpmi_uncore_cluster_info *cluster_infos;
65 };
66
67 /* Information for all power domains in a package */
68 struct tpmi_uncore_struct {
69 int power_domain_count;
70 int max_ratio;
71 int min_ratio;
72 struct tpmi_uncore_power_domain_info *pd_info;
73 struct tpmi_uncore_cluster_info root_cluster;
74 bool write_blocked;
75 };
76
77 /* Bit definitions for STATUS register */
78 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0)
79
80 /* Bit definitions for CONTROL register */
81 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(14, 8)
82 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(21, 15)
83 #define UNCORE_EFF_LAT_CTRL_RATIO_MASK GENMASK_ULL(28, 22)
84 #define UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK GENMASK_ULL(38, 32)
85 #define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE BIT(39)
86 #define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK GENMASK_ULL(46, 40)
87
88 /* Helper function to read MMIO offset for max/min control frequency */
read_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int * value,enum uncore_index index)89 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
90 unsigned int *value, enum uncore_index index)
91 {
92 u64 control;
93
94 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
95 if (index == UNCORE_INDEX_MAX_FREQ)
96 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
97 else
98 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
99 }
100
101 /* Helper function to read efficiency latency control values over MMIO */
read_eff_lat_ctrl(struct uncore_data * data,unsigned int * val,enum uncore_index index)102 static int read_eff_lat_ctrl(struct uncore_data *data, unsigned int *val, enum uncore_index index)
103 {
104 struct tpmi_uncore_cluster_info *cluster_info;
105 u64 ctrl;
106
107 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
108 if (cluster_info->root_domain)
109 return -ENODATA;
110
111 if (!cluster_info->elc_supported)
112 return -EOPNOTSUPP;
113
114 ctrl = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
115
116 switch (index) {
117 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
118 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, ctrl);
119 *val *= 100;
120 *val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK));
121 break;
122
123 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
124 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, ctrl);
125 *val *= 100;
126 *val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK));
127 break;
128
129 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
130 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, ctrl);
131 break;
132 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
133 *val = FIELD_GET(UNCORE_EFF_LAT_CTRL_RATIO_MASK, ctrl) * UNCORE_FREQ_KHZ_MULTIPLIER;
134 break;
135
136 default:
137 return -EOPNOTSUPP;
138 }
139
140 return 0;
141 }
142
143 #define UNCORE_MAX_RATIO FIELD_MAX(UNCORE_MAX_RATIO_MASK)
144
145 /* Helper for sysfs read for max/min frequencies. Called under mutex locks */
uncore_read_control_freq(struct uncore_data * data,unsigned int * value,enum uncore_index index)146 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
147 enum uncore_index index)
148 {
149 struct tpmi_uncore_cluster_info *cluster_info;
150
151 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
152
153 if (cluster_info->root_domain) {
154 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
155 unsigned int min, max, v;
156 int i;
157
158 min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
159 max = 0;
160
161 /*
162 * Get the max/min by looking at each cluster. Get the lowest
163 * min and highest max.
164 */
165 for (i = 0; i < uncore_root->power_domain_count; ++i) {
166 int j;
167
168 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
169 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
170 &v, index);
171 if (v < min)
172 min = v;
173 if (v > max)
174 max = v;
175 }
176 }
177
178 if (index == UNCORE_INDEX_MIN_FREQ)
179 *value = min;
180 else
181 *value = max;
182
183 return 0;
184 }
185
186 read_control_freq(cluster_info, value, index);
187
188 return 0;
189 }
190
191 /* Helper function for writing efficiency latency control values over MMIO */
write_eff_lat_ctrl(struct uncore_data * data,unsigned int val,enum uncore_index index)192 static int write_eff_lat_ctrl(struct uncore_data *data, unsigned int val, enum uncore_index index)
193 {
194 struct tpmi_uncore_cluster_info *cluster_info;
195 struct tpmi_uncore_struct *uncore_root;
196 u64 control;
197
198 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
199 uncore_root = cluster_info->uncore_root;
200
201 if (uncore_root->write_blocked)
202 return -EPERM;
203
204 if (cluster_info->root_domain)
205 return -ENODATA;
206
207 if (!cluster_info->elc_supported)
208 return -EOPNOTSUPP;
209
210 switch (index) {
211 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
212 if (val > 100)
213 return -EINVAL;
214 break;
215
216 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
217 if (val > 100)
218 return -EINVAL;
219 break;
220
221 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
222 if (val > 1)
223 return -EINVAL;
224 break;
225
226 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
227 val /= UNCORE_FREQ_KHZ_MULTIPLIER;
228 if (val > FIELD_MAX(UNCORE_EFF_LAT_CTRL_RATIO_MASK))
229 return -EINVAL;
230 break;
231
232 default:
233 return -EOPNOTSUPP;
234 }
235
236 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
237
238 switch (index) {
239 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
240 val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK);
241 val /= 100;
242 control &= ~UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK;
243 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, val);
244 break;
245
246 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
247 val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK);
248 val /= 100;
249 control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK;
250 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, val);
251 break;
252
253 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
254 control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE;
255 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, val);
256 break;
257
258 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
259 control &= ~UNCORE_EFF_LAT_CTRL_RATIO_MASK;
260 control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_RATIO_MASK, val);
261 break;
262
263 default:
264 break;
265 }
266
267 writeq(control, cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
268
269 return 0;
270 }
271
272 /* Helper function to write MMIO offset for max/min control frequency */
write_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int input,unsigned int index)273 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
274 unsigned int index)
275 {
276 u64 control;
277
278 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
279
280 if (index == UNCORE_INDEX_MAX_FREQ) {
281 control &= ~UNCORE_MAX_RATIO_MASK;
282 control |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
283 } else {
284 control &= ~UNCORE_MIN_RATIO_MASK;
285 control |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
286 }
287
288 writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
289 }
290
291 /* Helper for sysfs write for max/min frequencies. Called under mutex locks */
uncore_write_control_freq(struct uncore_data * data,unsigned int input,enum uncore_index index)292 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
293 enum uncore_index index)
294 {
295 struct tpmi_uncore_cluster_info *cluster_info;
296 struct tpmi_uncore_struct *uncore_root;
297
298 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
299 if (!input || input > UNCORE_MAX_RATIO)
300 return -EINVAL;
301
302 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
303 uncore_root = cluster_info->uncore_root;
304
305 if (uncore_root->write_blocked)
306 return -EPERM;
307
308 /* Update each cluster in a package */
309 if (cluster_info->root_domain) {
310 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
311 int i;
312
313 for (i = 0; i < uncore_root->power_domain_count; ++i) {
314 int j;
315
316 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
317 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
318 input, index);
319 }
320
321 if (index == UNCORE_INDEX_MAX_FREQ)
322 uncore_root->max_ratio = input;
323 else
324 uncore_root->min_ratio = input;
325
326 return 0;
327 }
328
329 if (index == UNCORE_INDEX_MAX_FREQ && uncore_root->max_ratio &&
330 uncore_root->max_ratio < input)
331 return -EINVAL;
332
333 if (index == UNCORE_INDEX_MIN_FREQ && uncore_root->min_ratio &&
334 uncore_root->min_ratio > input)
335 return -EINVAL;
336
337 write_control_freq(cluster_info, input, index);
338
339 return 0;
340 }
341
342 /* Helper for sysfs read for the current uncore frequency. Called under mutex locks */
uncore_read_freq(struct uncore_data * data,unsigned int * freq)343 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
344 {
345 struct tpmi_uncore_cluster_info *cluster_info;
346 u64 status;
347
348 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
349 if (cluster_info->root_domain)
350 return -ENODATA;
351
352 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
353 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
354
355 return 0;
356 }
357
358 /*
359 * Agent types as per the TPMI UFS Specification for UFS_STATUS
360 * Agent Type - Core Bit: 23
361 * Agent Type - Cache Bit: 24
362 * Agent Type - Memory Bit: 25
363 * Agent Type - IO Bit: 26
364 */
365
366 #define UNCORE_AGENT_TYPES GENMASK_ULL(26, 23)
367
368 /* Helper function to read agent type over MMIO and set the agent type mask */
uncore_set_agent_type(struct tpmi_uncore_cluster_info * cluster_info)369 static void uncore_set_agent_type(struct tpmi_uncore_cluster_info *cluster_info)
370 {
371 u64 status;
372
373 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
374 cluster_info->uncore_data.agent_type_mask = FIELD_GET(UNCORE_AGENT_TYPES, status);
375 }
376
377 #define MAX_PARTITIONS 2
378
379 /* IO domain ID start index for a partition */
380 static u8 io_die_start[MAX_PARTITIONS];
381
382 /* Next IO domain ID index after the current partition IO die IDs */
383 static u8 io_die_index_next;
384
385 /* Lock to protect io_die_start, io_die_index_next */
386 static DEFINE_MUTEX(domain_lock);
387
set_domain_id(int id,int num_resources,struct oobmsm_plat_info * plat_info,struct tpmi_uncore_cluster_info * cluster_info)388 static void set_domain_id(int id, int num_resources,
389 struct oobmsm_plat_info *plat_info,
390 struct tpmi_uncore_cluster_info *cluster_info)
391 {
392 u8 part_io_index, cdie_range, pkg_io_index, max_dies;
393
394 if (plat_info->partition >= MAX_PARTITIONS) {
395 cluster_info->uncore_data.domain_id = id;
396 return;
397 }
398
399 if (cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE) {
400 cluster_info->uncore_data.domain_id = cluster_info->cdie_id;
401 return;
402 }
403
404 /* Unlikely but cdie_mask may have holes, so take range */
405 cdie_range = fls(plat_info->cdie_mask) - ffs(plat_info->cdie_mask) + 1;
406 max_dies = topology_max_dies_per_package();
407
408 /*
409 * If the CPU doesn't enumerate dies, then use current cdie range
410 * as the max.
411 */
412 if (cdie_range > max_dies)
413 max_dies = cdie_range;
414
415 guard(mutex)(&domain_lock);
416
417 if (!io_die_index_next)
418 io_die_index_next = max_dies;
419
420 if (!io_die_start[plat_info->partition]) {
421 io_die_start[plat_info->partition] = io_die_index_next;
422 /*
423 * number of IO dies = num_resources - cdie_range. Hence
424 * next partition io_die_index_next is set after IO dies
425 * in the current partition.
426 */
427 io_die_index_next += (num_resources - cdie_range);
428 }
429
430 /*
431 * Index from IO die start within the partition:
432 * This is the first valid domain after the cdies.
433 * For example the current resource index 5 and cdies end at
434 * index 3 (cdie_cnt = 4). Then the IO only index 5 - 4 = 1.
435 */
436 part_io_index = id - cdie_range;
437
438 /*
439 * Add to the IO die start index for this partition in this package
440 * to make unique in the package.
441 */
442 pkg_io_index = io_die_start[plat_info->partition] + part_io_index;
443
444 /* Assign this to domain ID */
445 cluster_info->uncore_data.domain_id = pkg_io_index;
446 }
447
448 /* Callback for sysfs read for TPMI uncore values. Called under mutex locks. */
uncore_read(struct uncore_data * data,unsigned int * value,enum uncore_index index)449 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
450 {
451 struct tpmi_uncore_cluster_info *cluster_info;
452 int ret;
453
454 switch (index) {
455 case UNCORE_INDEX_MIN_FREQ:
456 case UNCORE_INDEX_MAX_FREQ:
457 return uncore_read_control_freq(data, value, index);
458
459 case UNCORE_INDEX_CURRENT_FREQ:
460 return uncore_read_freq(data, value);
461
462 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
463 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
464 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
465 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
466 return read_eff_lat_ctrl(data, value, index);
467
468 case UNCORE_INDEX_DIE_ID:
469 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
470 ret = tpmi_get_linux_die_id(cluster_info->uncore_data.package_id,
471 cluster_info->cdie_id);
472 if (ret < 0)
473 return ret;
474
475 *value = ret;
476 return 0;
477
478 default:
479 break;
480 }
481
482 return -EOPNOTSUPP;
483 }
484
485 /* Callback for sysfs write for TPMI uncore data. Called under mutex locks. */
uncore_write(struct uncore_data * data,unsigned int value,enum uncore_index index)486 static int uncore_write(struct uncore_data *data, unsigned int value, enum uncore_index index)
487 {
488 switch (index) {
489 case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
490 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
491 case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
492 case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
493 return write_eff_lat_ctrl(data, value, index);
494
495 case UNCORE_INDEX_MIN_FREQ:
496 case UNCORE_INDEX_MAX_FREQ:
497 return uncore_write_control_freq(data, value, index);
498
499 default:
500 break;
501 }
502
503 return -EOPNOTSUPP;
504 }
505
remove_cluster_entries(struct tpmi_uncore_struct * tpmi_uncore)506 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
507 {
508 int i;
509
510 for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
511 struct tpmi_uncore_power_domain_info *pd_info;
512 int j;
513
514 pd_info = &tpmi_uncore->pd_info[i];
515 if (!pd_info->uncore_base)
516 continue;
517
518 for (j = 0; j < pd_info->cluster_count; ++j) {
519 struct tpmi_uncore_cluster_info *cluster_info;
520
521 cluster_info = &pd_info->cluster_infos[j];
522 uncore_freq_remove_die_entry(&cluster_info->uncore_data);
523 }
524 }
525 }
526
set_cdie_id(int domain_id,struct tpmi_uncore_cluster_info * cluster_info,struct oobmsm_plat_info * plat_info)527 static void set_cdie_id(int domain_id, struct tpmi_uncore_cluster_info *cluster_info,
528 struct oobmsm_plat_info *plat_info)
529 {
530
531 cluster_info->cdie_id = domain_id;
532
533 if (plat_info->cdie_mask && cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE)
534 cluster_info->cdie_id = domain_id + ffs(plat_info->cdie_mask) - 1;
535 }
536
537 #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0)
538 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8)
539 #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0)
540 #define UNCORE_AUTONOMOUS_UFS_DISABLED BIT(32)
541 #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8
542
uncore_probe(struct auxiliary_device * auxdev,const struct auxiliary_device_id * id)543 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
544 {
545 bool read_blocked = 0, write_blocked = 0;
546 struct oobmsm_plat_info *plat_info;
547 struct tpmi_uncore_struct *tpmi_uncore;
548 bool uncore_sysfs_added = false;
549 int ret, i, pkg = 0;
550 int num_resources;
551
552 ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked);
553 if (ret)
554 dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n");
555
556 if (read_blocked) {
557 dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n");
558 return -ENODEV;
559 }
560
561 /* Get number of power domains, which is equal to number of resources */
562 num_resources = tpmi_get_resource_count(auxdev);
563 if (!num_resources)
564 return -EINVAL;
565
566 /* Register callbacks to uncore core */
567 ret = uncore_freq_common_init(uncore_read, uncore_write);
568 if (ret)
569 return ret;
570
571 /* Allocate uncore instance per package */
572 tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
573 if (!tpmi_uncore) {
574 ret = -ENOMEM;
575 goto err_rem_common;
576 }
577
578 /* Allocate memory for all power domains in a package */
579 tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
580 sizeof(*tpmi_uncore->pd_info),
581 GFP_KERNEL);
582 if (!tpmi_uncore->pd_info) {
583 ret = -ENOMEM;
584 goto err_rem_common;
585 }
586
587 tpmi_uncore->power_domain_count = num_resources;
588 tpmi_uncore->write_blocked = write_blocked;
589
590 /* Get the package ID from the TPMI core */
591 plat_info = tpmi_get_platform_data(auxdev);
592 if (unlikely(!plat_info)) {
593 dev_info(&auxdev->dev, "Platform information is NULL\n");
594 ret = -ENODEV;
595 goto err_rem_common;
596 }
597
598 pkg = plat_info->package_id;
599
600 for (i = 0; i < num_resources; ++i) {
601 struct tpmi_uncore_power_domain_info *pd_info;
602 bool auto_ufs_enabled;
603 struct resource *res;
604 u64 cluster_offset;
605 u8 cluster_mask;
606 int mask, j;
607 u64 header;
608
609 res = tpmi_get_resource_at_index(auxdev, i);
610 if (!res)
611 continue;
612
613 pd_info = &tpmi_uncore->pd_info[i];
614
615 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
616 if (IS_ERR(pd_info->uncore_base)) {
617 ret = PTR_ERR(pd_info->uncore_base);
618 /*
619 * Set to NULL so that clean up can still remove other
620 * entries already created if any by
621 * remove_cluster_entries()
622 */
623 pd_info->uncore_base = NULL;
624 goto remove_clusters;
625 }
626
627 /* Check for version and skip this resource if there is mismatch */
628 header = readq(pd_info->uncore_base);
629 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
630
631 if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID)
632 continue;
633
634 if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) {
635 dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n",
636 TPMI_MAJOR_VERSION(pd_info->ufs_header_ver));
637 ret = -ENODEV;
638 goto remove_clusters;
639 }
640
641 if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) > UNCORE_MINOR_VERSION)
642 dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n",
643 TPMI_MINOR_VERSION(pd_info->ufs_header_ver));
644
645 /* Get Cluster ID Mask */
646 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
647 if (!cluster_mask) {
648 dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
649 continue;
650 }
651
652 auto_ufs_enabled = !(header & UNCORE_AUTONOMOUS_UFS_DISABLED);
653
654 /* Find out number of clusters in this resource */
655 pd_info->cluster_count = hweight8(cluster_mask);
656
657 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
658 sizeof(struct tpmi_uncore_cluster_info),
659 GFP_KERNEL);
660 if (!pd_info->cluster_infos) {
661 ret = -ENOMEM;
662 goto remove_clusters;
663 }
664 /*
665 * Each byte in the register point to status and control
666 * registers belonging to cluster id 0-8.
667 */
668 cluster_offset = readq(pd_info->uncore_base +
669 UNCORE_FABRIC_CLUSTER_OFFSET);
670
671 for (j = 0; j < pd_info->cluster_count; ++j) {
672 struct tpmi_uncore_cluster_info *cluster_info;
673
674 /* Get the offset for this cluster */
675 mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
676 /* Offset in QWORD, so change to bytes */
677 mask <<= 3;
678
679 cluster_info = &pd_info->cluster_infos[j];
680
681 cluster_info->cluster_base = pd_info->uncore_base + mask;
682
683 uncore_set_agent_type(cluster_info);
684
685 cluster_info->uncore_data.package_id = pkg;
686 /* There are no dies like Cascade Lake */
687 cluster_info->uncore_data.die_id = 0;
688 cluster_info->uncore_data.cluster_id = j;
689
690 set_cdie_id(i, cluster_info, plat_info);
691
692 set_domain_id(i, num_resources, plat_info, cluster_info);
693
694 cluster_info->uncore_root = tpmi_uncore;
695
696 if ((TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >=
697 UNCORE_ELC_SUPPORTED_VERSION) &&
698 auto_ufs_enabled)
699 cluster_info->elc_supported = true;
700
701 ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
702 if (ret) {
703 cluster_info->cluster_base = NULL;
704 goto remove_clusters;
705 }
706 /* Point to next cluster offset */
707 cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
708 uncore_sysfs_added = true;
709 }
710 }
711
712 if (!uncore_sysfs_added) {
713 ret = -ENODEV;
714 goto remove_clusters;
715 }
716
717 auxiliary_set_drvdata(auxdev, tpmi_uncore);
718
719 if (topology_max_dies_per_package() > 1 || plat_info->partition)
720 return 0;
721
722 tpmi_uncore->root_cluster.root_domain = true;
723 tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
724
725 tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
726 tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
727 ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
728 if (ret)
729 goto remove_clusters;
730
731 return 0;
732
733 remove_clusters:
734 remove_cluster_entries(tpmi_uncore);
735 err_rem_common:
736 uncore_freq_common_exit();
737
738 return ret;
739 }
740
uncore_remove(struct auxiliary_device * auxdev)741 static void uncore_remove(struct auxiliary_device *auxdev)
742 {
743 struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
744
745 if (tpmi_uncore->root_cluster.root_domain)
746 uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
747
748 remove_cluster_entries(tpmi_uncore);
749
750 uncore_freq_common_exit();
751 }
752
753 static const struct auxiliary_device_id intel_uncore_id_table[] = {
754 { .name = "intel_vsec.tpmi-uncore" },
755 {}
756 };
757 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
758
759 static struct auxiliary_driver intel_uncore_aux_driver = {
760 .id_table = intel_uncore_id_table,
761 .remove = uncore_remove,
762 .probe = uncore_probe,
763 };
764
765 module_auxiliary_driver(intel_uncore_aux_driver);
766
767 MODULE_IMPORT_NS("INTEL_TPMI");
768 MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY");
769 MODULE_IMPORT_NS("INTEL_TPMI_POWER_DOMAIN");
770 MODULE_DESCRIPTION("Intel TPMI UFS Driver");
771 MODULE_LICENSE("GPL");
772