1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Resource Director Technology(RDT)
4 * - Cache Allocation code.
5 *
6 * Copyright (C) 2016 Intel Corporation
7 *
8 * Authors:
9 * Fenghua Yu <fenghua.yu@intel.com>
10 * Tony Luck <tony.luck@intel.com>
11 *
12 * More information about RDT be found in the Intel (R) x86 Architecture
13 * Software Developer Manual June 2016, volume 3, section 17.17.
14 */
15
16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17
18 #include <linux/cpu.h>
19 #include <linux/kernfs.h>
20 #include <linux/seq_file.h>
21 #include <linux/slab.h>
22 #include <linux/tick.h>
23
24 #include "internal.h"
25
26 /*
27 * Check whether MBA bandwidth percentage value is correct. The value is
28 * checked against the minimum and max bandwidth values specified by the
29 * hardware. The allocated bandwidth percentage is rounded to the next
30 * control step available on the hardware.
31 */
bw_validate(char * buf,u32 * data,struct rdt_resource * r)32 static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r)
33 {
34 int ret;
35 u32 bw;
36
37 /*
38 * Only linear delay values is supported for current Intel SKUs.
39 */
40 if (!r->membw.delay_linear && r->membw.arch_needs_linear) {
41 rdt_last_cmd_puts("No support for non-linear MB domains\n");
42 return false;
43 }
44
45 ret = kstrtou32(buf, 10, &bw);
46 if (ret) {
47 rdt_last_cmd_printf("Invalid MB value %s\n", buf);
48 return false;
49 }
50
51 /* Nothing else to do if software controller is enabled. */
52 if (is_mba_sc(r)) {
53 *data = bw;
54 return true;
55 }
56
57 if (bw < r->membw.min_bw || bw > r->default_ctrl) {
58 rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n",
59 bw, r->membw.min_bw, r->default_ctrl);
60 return false;
61 }
62
63 *data = roundup(bw, (unsigned long)r->membw.bw_gran);
64 return true;
65 }
66
parse_bw(struct rdt_parse_data * data,struct resctrl_schema * s,struct rdt_ctrl_domain * d)67 int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
68 struct rdt_ctrl_domain *d)
69 {
70 struct resctrl_staged_config *cfg;
71 u32 closid = data->rdtgrp->closid;
72 struct rdt_resource *r = s->res;
73 u32 bw_val;
74
75 cfg = &d->staged_config[s->conf_type];
76 if (cfg->have_new_ctrl) {
77 rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id);
78 return -EINVAL;
79 }
80
81 if (!bw_validate(data->buf, &bw_val, r))
82 return -EINVAL;
83
84 if (is_mba_sc(r)) {
85 d->mbps_val[closid] = bw_val;
86 return 0;
87 }
88
89 cfg->new_ctrl = bw_val;
90 cfg->have_new_ctrl = true;
91
92 return 0;
93 }
94
95 /*
96 * Check whether a cache bit mask is valid.
97 * On Intel CPUs, non-contiguous 1s value support is indicated by CPUID:
98 * - CPUID.0x10.1:ECX[3]: L3 non-contiguous 1s value supported if 1
99 * - CPUID.0x10.2:ECX[3]: L2 non-contiguous 1s value supported if 1
100 *
101 * Haswell does not support a non-contiguous 1s value and additionally
102 * requires at least two bits set.
103 * AMD allows non-contiguous bitmasks.
104 */
cbm_validate(char * buf,u32 * data,struct rdt_resource * r)105 static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
106 {
107 unsigned long first_bit, zero_bit, val;
108 unsigned int cbm_len = r->cache.cbm_len;
109 int ret;
110
111 ret = kstrtoul(buf, 16, &val);
112 if (ret) {
113 rdt_last_cmd_printf("Non-hex character in the mask %s\n", buf);
114 return false;
115 }
116
117 if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) {
118 rdt_last_cmd_puts("Mask out of range\n");
119 return false;
120 }
121
122 first_bit = find_first_bit(&val, cbm_len);
123 zero_bit = find_next_zero_bit(&val, cbm_len, first_bit);
124
125 /* Are non-contiguous bitmasks allowed? */
126 if (!r->cache.arch_has_sparse_bitmasks &&
127 (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) {
128 rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val);
129 return false;
130 }
131
132 if ((zero_bit - first_bit) < r->cache.min_cbm_bits) {
133 rdt_last_cmd_printf("Need at least %d bits in the mask\n",
134 r->cache.min_cbm_bits);
135 return false;
136 }
137
138 *data = val;
139 return true;
140 }
141
142 /*
143 * Read one cache bit mask (hex). Check that it is valid for the current
144 * resource type.
145 */
parse_cbm(struct rdt_parse_data * data,struct resctrl_schema * s,struct rdt_ctrl_domain * d)146 int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s,
147 struct rdt_ctrl_domain *d)
148 {
149 struct rdtgroup *rdtgrp = data->rdtgrp;
150 struct resctrl_staged_config *cfg;
151 struct rdt_resource *r = s->res;
152 u32 cbm_val;
153
154 cfg = &d->staged_config[s->conf_type];
155 if (cfg->have_new_ctrl) {
156 rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id);
157 return -EINVAL;
158 }
159
160 /*
161 * Cannot set up more than one pseudo-locked region in a cache
162 * hierarchy.
163 */
164 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP &&
165 rdtgroup_pseudo_locked_in_hierarchy(d)) {
166 rdt_last_cmd_puts("Pseudo-locked region in hierarchy\n");
167 return -EINVAL;
168 }
169
170 if (!cbm_validate(data->buf, &cbm_val, r))
171 return -EINVAL;
172
173 if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE ||
174 rdtgrp->mode == RDT_MODE_SHAREABLE) &&
175 rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) {
176 rdt_last_cmd_puts("CBM overlaps with pseudo-locked region\n");
177 return -EINVAL;
178 }
179
180 /*
181 * The CBM may not overlap with the CBM of another closid if
182 * either is exclusive.
183 */
184 if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, true)) {
185 rdt_last_cmd_puts("Overlaps with exclusive group\n");
186 return -EINVAL;
187 }
188
189 if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, false)) {
190 if (rdtgrp->mode == RDT_MODE_EXCLUSIVE ||
191 rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
192 rdt_last_cmd_puts("Overlaps with other group\n");
193 return -EINVAL;
194 }
195 }
196
197 cfg->new_ctrl = cbm_val;
198 cfg->have_new_ctrl = true;
199
200 return 0;
201 }
202
203 /*
204 * For each domain in this resource we expect to find a series of:
205 * id=mask
206 * separated by ";". The "id" is in decimal, and must match one of
207 * the "id"s for this resource.
208 */
parse_line(char * line,struct resctrl_schema * s,struct rdtgroup * rdtgrp)209 static int parse_line(char *line, struct resctrl_schema *s,
210 struct rdtgroup *rdtgrp)
211 {
212 enum resctrl_conf_type t = s->conf_type;
213 struct resctrl_staged_config *cfg;
214 struct rdt_resource *r = s->res;
215 struct rdt_parse_data data;
216 struct rdt_ctrl_domain *d;
217 char *dom = NULL, *id;
218 unsigned long dom_id;
219
220 /* Walking r->domains, ensure it can't race with cpuhp */
221 lockdep_assert_cpus_held();
222
223 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP &&
224 (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA)) {
225 rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n");
226 return -EINVAL;
227 }
228
229 next:
230 if (!line || line[0] == '\0')
231 return 0;
232 dom = strsep(&line, ";");
233 id = strsep(&dom, "=");
234 if (!dom || kstrtoul(id, 10, &dom_id)) {
235 rdt_last_cmd_puts("Missing '=' or non-numeric domain\n");
236 return -EINVAL;
237 }
238 dom = strim(dom);
239 list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
240 if (d->hdr.id == dom_id) {
241 data.buf = dom;
242 data.rdtgrp = rdtgrp;
243 if (r->parse_ctrlval(&data, s, d))
244 return -EINVAL;
245 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
246 cfg = &d->staged_config[t];
247 /*
248 * In pseudo-locking setup mode and just
249 * parsed a valid CBM that should be
250 * pseudo-locked. Only one locked region per
251 * resource group and domain so just do
252 * the required initialization for single
253 * region and return.
254 */
255 rdtgrp->plr->s = s;
256 rdtgrp->plr->d = d;
257 rdtgrp->plr->cbm = cfg->new_ctrl;
258 d->plr = rdtgrp->plr;
259 return 0;
260 }
261 goto next;
262 }
263 }
264 return -EINVAL;
265 }
266
get_config_index(u32 closid,enum resctrl_conf_type type)267 static u32 get_config_index(u32 closid, enum resctrl_conf_type type)
268 {
269 switch (type) {
270 default:
271 case CDP_NONE:
272 return closid;
273 case CDP_CODE:
274 return closid * 2 + 1;
275 case CDP_DATA:
276 return closid * 2;
277 }
278 }
279
resctrl_arch_update_one(struct rdt_resource * r,struct rdt_ctrl_domain * d,u32 closid,enum resctrl_conf_type t,u32 cfg_val)280 int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
281 u32 closid, enum resctrl_conf_type t, u32 cfg_val)
282 {
283 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d);
284 struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
285 u32 idx = get_config_index(closid, t);
286 struct msr_param msr_param;
287
288 if (!cpumask_test_cpu(smp_processor_id(), &d->hdr.cpu_mask))
289 return -EINVAL;
290
291 hw_dom->ctrl_val[idx] = cfg_val;
292
293 msr_param.res = r;
294 msr_param.dom = d;
295 msr_param.low = idx;
296 msr_param.high = idx + 1;
297 hw_res->msr_update(&msr_param);
298
299 return 0;
300 }
301
resctrl_arch_update_domains(struct rdt_resource * r,u32 closid)302 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
303 {
304 struct resctrl_staged_config *cfg;
305 struct rdt_hw_ctrl_domain *hw_dom;
306 struct msr_param msr_param;
307 struct rdt_ctrl_domain *d;
308 enum resctrl_conf_type t;
309 u32 idx;
310
311 /* Walking r->domains, ensure it can't race with cpuhp */
312 lockdep_assert_cpus_held();
313
314 list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
315 hw_dom = resctrl_to_arch_ctrl_dom(d);
316 msr_param.res = NULL;
317 for (t = 0; t < CDP_NUM_TYPES; t++) {
318 cfg = &hw_dom->d_resctrl.staged_config[t];
319 if (!cfg->have_new_ctrl)
320 continue;
321
322 idx = get_config_index(closid, t);
323 if (cfg->new_ctrl == hw_dom->ctrl_val[idx])
324 continue;
325 hw_dom->ctrl_val[idx] = cfg->new_ctrl;
326
327 if (!msr_param.res) {
328 msr_param.low = idx;
329 msr_param.high = msr_param.low + 1;
330 msr_param.res = r;
331 msr_param.dom = d;
332 } else {
333 msr_param.low = min(msr_param.low, idx);
334 msr_param.high = max(msr_param.high, idx + 1);
335 }
336 }
337 if (msr_param.res)
338 smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1);
339 }
340
341 return 0;
342 }
343
rdtgroup_parse_resource(char * resname,char * tok,struct rdtgroup * rdtgrp)344 static int rdtgroup_parse_resource(char *resname, char *tok,
345 struct rdtgroup *rdtgrp)
346 {
347 struct resctrl_schema *s;
348
349 list_for_each_entry(s, &resctrl_schema_all, list) {
350 if (!strcmp(resname, s->name) && rdtgrp->closid < s->num_closid)
351 return parse_line(tok, s, rdtgrp);
352 }
353 rdt_last_cmd_printf("Unknown or unsupported resource name '%s'\n", resname);
354 return -EINVAL;
355 }
356
rdtgroup_schemata_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)357 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
358 char *buf, size_t nbytes, loff_t off)
359 {
360 struct resctrl_schema *s;
361 struct rdtgroup *rdtgrp;
362 struct rdt_resource *r;
363 char *tok, *resname;
364 int ret = 0;
365
366 /* Valid input requires a trailing newline */
367 if (nbytes == 0 || buf[nbytes - 1] != '\n')
368 return -EINVAL;
369 buf[nbytes - 1] = '\0';
370
371 rdtgrp = rdtgroup_kn_lock_live(of->kn);
372 if (!rdtgrp) {
373 rdtgroup_kn_unlock(of->kn);
374 return -ENOENT;
375 }
376 rdt_last_cmd_clear();
377
378 /*
379 * No changes to pseudo-locked region allowed. It has to be removed
380 * and re-created instead.
381 */
382 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
383 ret = -EINVAL;
384 rdt_last_cmd_puts("Resource group is pseudo-locked\n");
385 goto out;
386 }
387
388 rdt_staged_configs_clear();
389
390 while ((tok = strsep(&buf, "\n")) != NULL) {
391 resname = strim(strsep(&tok, ":"));
392 if (!tok) {
393 rdt_last_cmd_puts("Missing ':'\n");
394 ret = -EINVAL;
395 goto out;
396 }
397 if (tok[0] == '\0') {
398 rdt_last_cmd_printf("Missing '%s' value\n", resname);
399 ret = -EINVAL;
400 goto out;
401 }
402 ret = rdtgroup_parse_resource(resname, tok, rdtgrp);
403 if (ret)
404 goto out;
405 }
406
407 list_for_each_entry(s, &resctrl_schema_all, list) {
408 r = s->res;
409
410 /*
411 * Writes to mba_sc resources update the software controller,
412 * not the control MSR.
413 */
414 if (is_mba_sc(r))
415 continue;
416
417 ret = resctrl_arch_update_domains(r, rdtgrp->closid);
418 if (ret)
419 goto out;
420 }
421
422 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
423 /*
424 * If pseudo-locking fails we keep the resource group in
425 * mode RDT_MODE_PSEUDO_LOCKSETUP with its class of service
426 * active and updated for just the domain the pseudo-locked
427 * region was requested for.
428 */
429 ret = rdtgroup_pseudo_lock_create(rdtgrp);
430 }
431
432 out:
433 rdt_staged_configs_clear();
434 rdtgroup_kn_unlock(of->kn);
435 return ret ?: nbytes;
436 }
437
resctrl_arch_get_config(struct rdt_resource * r,struct rdt_ctrl_domain * d,u32 closid,enum resctrl_conf_type type)438 u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d,
439 u32 closid, enum resctrl_conf_type type)
440 {
441 struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d);
442 u32 idx = get_config_index(closid, type);
443
444 return hw_dom->ctrl_val[idx];
445 }
446
show_doms(struct seq_file * s,struct resctrl_schema * schema,int closid)447 static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int closid)
448 {
449 struct rdt_resource *r = schema->res;
450 struct rdt_ctrl_domain *dom;
451 bool sep = false;
452 u32 ctrl_val;
453
454 /* Walking r->domains, ensure it can't race with cpuhp */
455 lockdep_assert_cpus_held();
456
457 seq_printf(s, "%*s:", max_name_width, schema->name);
458 list_for_each_entry(dom, &r->ctrl_domains, hdr.list) {
459 if (sep)
460 seq_puts(s, ";");
461
462 if (is_mba_sc(r))
463 ctrl_val = dom->mbps_val[closid];
464 else
465 ctrl_val = resctrl_arch_get_config(r, dom, closid,
466 schema->conf_type);
467
468 seq_printf(s, r->format_str, dom->hdr.id, max_data_width,
469 ctrl_val);
470 sep = true;
471 }
472 seq_puts(s, "\n");
473 }
474
rdtgroup_schemata_show(struct kernfs_open_file * of,struct seq_file * s,void * v)475 int rdtgroup_schemata_show(struct kernfs_open_file *of,
476 struct seq_file *s, void *v)
477 {
478 struct resctrl_schema *schema;
479 struct rdtgroup *rdtgrp;
480 int ret = 0;
481 u32 closid;
482
483 rdtgrp = rdtgroup_kn_lock_live(of->kn);
484 if (rdtgrp) {
485 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
486 list_for_each_entry(schema, &resctrl_schema_all, list) {
487 seq_printf(s, "%s:uninitialized\n", schema->name);
488 }
489 } else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
490 if (!rdtgrp->plr->d) {
491 rdt_last_cmd_clear();
492 rdt_last_cmd_puts("Cache domain offline\n");
493 ret = -ENODEV;
494 } else {
495 seq_printf(s, "%s:%d=%x\n",
496 rdtgrp->plr->s->res->name,
497 rdtgrp->plr->d->hdr.id,
498 rdtgrp->plr->cbm);
499 }
500 } else {
501 closid = rdtgrp->closid;
502 list_for_each_entry(schema, &resctrl_schema_all, list) {
503 if (closid < schema->num_closid)
504 show_doms(s, schema, closid);
505 }
506 }
507 } else {
508 ret = -ENOENT;
509 }
510 rdtgroup_kn_unlock(of->kn);
511 return ret;
512 }
513
smp_mon_event_count(void * arg)514 static int smp_mon_event_count(void *arg)
515 {
516 mon_event_count(arg);
517
518 return 0;
519 }
520
mon_event_read(struct rmid_read * rr,struct rdt_resource * r,struct rdt_mon_domain * d,struct rdtgroup * rdtgrp,cpumask_t * cpumask,int evtid,int first)521 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
522 struct rdt_mon_domain *d, struct rdtgroup *rdtgrp,
523 cpumask_t *cpumask, int evtid, int first)
524 {
525 int cpu;
526
527 /* When picking a CPU from cpu_mask, ensure it can't race with cpuhp */
528 lockdep_assert_cpus_held();
529
530 /*
531 * Setup the parameters to pass to mon_event_count() to read the data.
532 */
533 rr->rgrp = rdtgrp;
534 rr->evtid = evtid;
535 rr->r = r;
536 rr->d = d;
537 rr->first = first;
538 rr->arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, evtid);
539 if (IS_ERR(rr->arch_mon_ctx)) {
540 rr->err = -EINVAL;
541 return;
542 }
543
544 cpu = cpumask_any_housekeeping(cpumask, RESCTRL_PICK_ANY_CPU);
545
546 /*
547 * cpumask_any_housekeeping() prefers housekeeping CPUs, but
548 * are all the CPUs nohz_full? If yes, pick a CPU to IPI.
549 * MPAM's resctrl_arch_rmid_read() is unable to read the
550 * counters on some platforms if its called in IRQ context.
551 */
552 if (tick_nohz_full_cpu(cpu))
553 smp_call_function_any(cpumask, mon_event_count, rr, 1);
554 else
555 smp_call_on_cpu(cpu, smp_mon_event_count, rr, false);
556
557 resctrl_arch_mon_ctx_free(r, evtid, rr->arch_mon_ctx);
558 }
559
rdtgroup_mondata_show(struct seq_file * m,void * arg)560 int rdtgroup_mondata_show(struct seq_file *m, void *arg)
561 {
562 struct kernfs_open_file *of = m->private;
563 struct rdt_domain_hdr *hdr;
564 struct rmid_read rr = {0};
565 struct rdt_mon_domain *d;
566 u32 resid, evtid, domid;
567 struct rdtgroup *rdtgrp;
568 struct rdt_resource *r;
569 union mon_data_bits md;
570 int ret = 0;
571
572 rdtgrp = rdtgroup_kn_lock_live(of->kn);
573 if (!rdtgrp) {
574 ret = -ENOENT;
575 goto out;
576 }
577
578 md.priv = of->kn->priv;
579 resid = md.u.rid;
580 domid = md.u.domid;
581 evtid = md.u.evtid;
582 r = &rdt_resources_all[resid].r_resctrl;
583
584 if (md.u.sum) {
585 /*
586 * This file requires summing across all domains that share
587 * the L3 cache id that was provided in the "domid" field of the
588 * mon_data_bits union. Search all domains in the resource for
589 * one that matches this cache id.
590 */
591 list_for_each_entry(d, &r->mon_domains, hdr.list) {
592 if (d->ci->id == domid) {
593 rr.ci = d->ci;
594 mon_event_read(&rr, r, NULL, rdtgrp,
595 &d->ci->shared_cpu_map, evtid, false);
596 goto checkresult;
597 }
598 }
599 ret = -ENOENT;
600 goto out;
601 } else {
602 /*
603 * This file provides data from a single domain. Search
604 * the resource to find the domain with "domid".
605 */
606 hdr = rdt_find_domain(&r->mon_domains, domid, NULL);
607 if (!hdr || WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) {
608 ret = -ENOENT;
609 goto out;
610 }
611 d = container_of(hdr, struct rdt_mon_domain, hdr);
612 mon_event_read(&rr, r, d, rdtgrp, &d->hdr.cpu_mask, evtid, false);
613 }
614
615 checkresult:
616
617 if (rr.err == -EIO)
618 seq_puts(m, "Error\n");
619 else if (rr.err == -EINVAL)
620 seq_puts(m, "Unavailable\n");
621 else
622 seq_printf(m, "%llu\n", rr.val);
623
624 out:
625 rdtgroup_kn_unlock(of->kn);
626 return ret;
627 }
628