Lines Matching +full:interrupt +full:- +full:affinity
1 // SPDX-License-Identifier: GPL-2.0
17 #include <linux/interrupt.h>
44 * mpam_enabled key is enabled these lists are read-only,
45 * unless the error interrupt disables the driver.
88 * mpam_is_enabled(). The disable call after an error interrupt makes
96 * their RIS index. These share a base-address, interrupts and some MMIO
102 * are from different MSC, so have different base-address, interrupts etc.
114 * e.g. The L2 may have one MSC and two RIS, one for cache-controls another
126 * The same MSC may exist under different class->component->vmsc paths, but the
136 init_llist_node(&garbage->llist); in init_garbage()
142 _x->garbage.to_free = _x; \
143 llist_add(&_x->garbage.llist, &mpam_garbage); \
157 if (iter->pdev) in mpam_free_garbage()
158 devm_kfree(&iter->pdev->dev, iter->to_free); in mpam_free_garbage()
160 kfree(iter->to_free); in mpam_free_garbage()
182 WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); in __mpam_read_reg()
184 return readl_relaxed(msc->mapped_hwpage + reg); in __mpam_read_reg()
189 lockdep_assert_held_once(&msc->part_sel_lock); in _mpam_read_partsel_reg()
197 WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz); in __mpam_write_reg()
198 WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); in __mpam_write_reg()
200 writel_relaxed(val, msc->mapped_hwpage + reg); in __mpam_write_reg()
205 lockdep_assert_held_once(&msc->part_sel_lock); in _mpam_write_partsel_reg()
231 lockdep_assert_held(&msc->part_sel_lock); in mpam_msc_read_idr()
253 if (msc->has_extd_esr) in mpam_msc_clear_esr()
263 if (msc->has_extd_esr) in mpam_msc_read_esr()
271 lockdep_assert_held(&msc->part_sel_lock); in __mpam_part_sel_raw()
306 return -EBUSY; in mpam_register_requestor()
322 return ERR_PTR(-ENOMEM); in mpam_class_alloc()
323 init_garbage(&class->garbage); in mpam_class_alloc()
325 INIT_LIST_HEAD_RCU(&class->components); in mpam_class_alloc()
326 /* Affinity is updated when ris are added */ in mpam_class_alloc()
327 class->level = level_idx; in mpam_class_alloc()
328 class->type = type; in mpam_class_alloc()
329 INIT_LIST_HEAD_RCU(&class->classes_list); in mpam_class_alloc()
330 ida_init(&class->ida_csu_mon); in mpam_class_alloc()
331 ida_init(&class->ida_mbwu_mon); in mpam_class_alloc()
333 list_add_rcu(&class->classes_list, &mpam_classes); in mpam_class_alloc()
342 list_del_rcu(&class->classes_list); in mpam_class_destroy()
354 if (class->type == type && class->level == level_idx) in mpam_class_find()
370 return ERR_PTR(-ENOMEM); in mpam_component_alloc()
371 init_garbage(&comp->garbage); in mpam_component_alloc()
373 comp->comp_id = id; in mpam_component_alloc()
374 INIT_LIST_HEAD_RCU(&comp->vmsc); in mpam_component_alloc()
375 /* Affinity is updated when RIS are added */ in mpam_component_alloc()
376 INIT_LIST_HEAD_RCU(&comp->class_list); in mpam_component_alloc()
377 comp->class = class; in mpam_component_alloc()
379 list_add_rcu(&comp->class_list, &class->components); in mpam_component_alloc()
388 struct mpam_class *class = comp->class; in mpam_component_destroy()
394 list_del_rcu(&comp->class_list); in mpam_component_destroy()
397 if (list_empty(&class->components)) in mpam_component_destroy()
408 list_for_each_entry(comp, &class->components, class_list) { in mpam_component_find()
409 if (comp->comp_id == id) in mpam_component_find()
425 return ERR_PTR(-ENOMEM); in mpam_vmsc_alloc()
426 init_garbage(&vmsc->garbage); in mpam_vmsc_alloc()
428 INIT_LIST_HEAD_RCU(&vmsc->ris); in mpam_vmsc_alloc()
429 INIT_LIST_HEAD_RCU(&vmsc->comp_list); in mpam_vmsc_alloc()
430 vmsc->comp = comp; in mpam_vmsc_alloc()
431 vmsc->msc = msc; in mpam_vmsc_alloc()
433 list_add_rcu(&vmsc->comp_list, &comp->vmsc); in mpam_vmsc_alloc()
440 struct mpam_component *comp = vmsc->comp; in mpam_vmsc_destroy()
444 list_del_rcu(&vmsc->comp_list); in mpam_vmsc_destroy()
447 if (list_empty(&comp->vmsc)) in mpam_vmsc_destroy()
458 list_for_each_entry(vmsc, &comp->vmsc, comp_list) { in mpam_vmsc_find()
459 if (vmsc->msc->id == msc->id) in mpam_vmsc_find()
471 cpumask_t *affinity) in mpam_get_cpumask_from_cache_id() argument
473 return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity); in mpam_get_cpumask_from_cache_id()
480 static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity) in get_cpumask_from_node_id() argument
486 cpumask_set_cpu(cpu, affinity); in get_cpumask_from_node_id()
490 static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity, in mpam_ris_get_affinity() argument
499 err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level, in mpam_ris_get_affinity()
500 affinity); in mpam_ris_get_affinity()
502 dev_warn_once(&msc->pdev->dev, in mpam_ris_get_affinity()
503 "Failed to determine CPU affinity\n"); in mpam_ris_get_affinity()
507 if (cpumask_empty(affinity)) in mpam_ris_get_affinity()
508 dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n"); in mpam_ris_get_affinity()
512 get_cpumask_from_node_id(comp->comp_id, affinity); in mpam_ris_get_affinity()
513 /* affinity may be empty for CPU-less memory nodes */ in mpam_ris_get_affinity()
519 cpumask_and(affinity, affinity, &msc->accessibility); in mpam_ris_get_affinity()
533 struct platform_device *pdev = msc->pdev; in mpam_ris_create_locked()
538 return -EINVAL; in mpam_ris_create_locked()
540 if (test_and_set_bit(ris_idx, &msc->ris_idxs)) in mpam_ris_create_locked()
541 return -EBUSY; in mpam_ris_create_locked()
543 ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL); in mpam_ris_create_locked()
545 return -ENOMEM; in mpam_ris_create_locked()
546 init_garbage(&ris->garbage); in mpam_ris_create_locked()
547 ris->garbage.pdev = pdev; in mpam_ris_create_locked()
555 if (list_empty(&class->components)) in mpam_ris_create_locked()
562 if (list_empty(&comp->vmsc)) in mpam_ris_create_locked()
567 err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp); in mpam_ris_create_locked()
569 if (list_empty(&vmsc->ris)) in mpam_ris_create_locked()
574 ris->ris_idx = ris_idx; in mpam_ris_create_locked()
575 INIT_LIST_HEAD_RCU(&ris->msc_list); in mpam_ris_create_locked()
576 INIT_LIST_HEAD_RCU(&ris->vmsc_list); in mpam_ris_create_locked()
577 ris->vmsc = vmsc; in mpam_ris_create_locked()
579 cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity); in mpam_ris_create_locked()
580 cpumask_or(&class->affinity, &class->affinity, &ris->affinity); in mpam_ris_create_locked()
581 list_add_rcu(&ris->vmsc_list, &vmsc->ris); in mpam_ris_create_locked()
582 list_add_rcu(&ris->msc_list, &msc->ris); in mpam_ris_create_locked()
589 struct mpam_vmsc *vmsc = ris->vmsc; in mpam_ris_destroy()
590 struct mpam_msc *msc = vmsc->msc; in mpam_ris_destroy()
591 struct mpam_component *comp = vmsc->comp; in mpam_ris_destroy()
592 struct mpam_class *class = comp->class; in mpam_ris_destroy()
600 cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity); in mpam_ris_destroy()
601 cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity); in mpam_ris_destroy()
602 clear_bit(ris->ris_idx, &msc->ris_idxs); in mpam_ris_destroy()
603 list_del_rcu(&ris->msc_list); in mpam_ris_destroy()
604 list_del_rcu(&ris->vmsc_list); in mpam_ris_destroy()
607 if (list_empty(&vmsc->ris)) in mpam_ris_destroy()
634 if (!test_bit(ris_idx, &msc->ris_idxs)) { in mpam_get_or_create_ris()
641 list_for_each_entry(ris, &msc->ris, msc_list) { in mpam_get_or_create_ris()
642 if (ris->ris_idx == ris_idx) in mpam_get_or_create_ris()
646 return ERR_PTR(-ENOENT); in mpam_get_or_create_ris()
661 return -EINVAL; in mpam_enable_quirk_nvidia_t241_1()
663 r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0); in mpam_enable_quirk_nvidia_t241_1()
665 return -EINVAL; in mpam_enable_quirk_nvidia_t241_1()
668 msc->t241_id = T241_CHIP_ID(r->start); in mpam_enable_quirk_nvidia_t241_1()
669 phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) | 0x19000000ULL; in mpam_enable_quirk_nvidia_t241_1()
671 t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M); in mpam_enable_quirk_nvidia_t241_1()
672 if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id])) in mpam_enable_quirk_nvidia_t241_1()
673 return -EINVAL; in mpam_enable_quirk_nvidia_t241_1()
675 pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n"); in mpam_enable_quirk_nvidia_t241_1()
682 /* NVIDIA t241 erratum T241-MPAM-1 */
689 /* NVIDIA t241 erratum T241-MPAM-4 */
695 /* NVIDIA t241 erratum T241-MPAM-6 */
701 /* ARM CMN-650 CSU erratum 3642720 */
713 for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) { in mpam_enable_quirks()
716 if (quirk->iidr != (msc->iidr & quirk->iidr_mask)) in mpam_enable_quirks()
719 if (quirk->init) in mpam_enable_quirks()
720 err = quirk->init(msc, quirk); in mpam_enable_quirks()
725 mpam_set_quirk(quirk->workaround, msc); in mpam_enable_quirks()
731 * of NRDY, software can use this bit for any purpose" - so hardware might not
732 * implement this - but it isn't RES0.
741 struct mpam_msc *msc = ris->vmsc->msc; in mpam_ris_hw_probe_csu_nrdy()
747 FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); in mpam_ris_hw_probe_csu_nrdy()
775 struct mpam_msc *msc = ris->vmsc->msc; in mpam_ris_hw_probe()
776 struct device *dev = &msc->pdev->dev; in mpam_ris_hw_probe()
777 struct mpam_props *props = &ris->props; in mpam_ris_hw_probe()
778 struct mpam_class *class = ris->vmsc->comp->class; in mpam_ris_hw_probe()
780 lockdep_assert_held(&msc->probe_lock); in mpam_ris_hw_probe()
781 lockdep_assert_held(&msc->part_sel_lock); in mpam_ris_hw_probe()
784 if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) { in mpam_ris_hw_probe()
787 props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features); in mpam_ris_hw_probe()
788 if (props->cmax_wd && in mpam_ris_hw_probe()
792 if (props->cmax_wd && in mpam_ris_hw_probe()
796 if (props->cmax_wd && in mpam_ris_hw_probe()
800 props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features); in mpam_ris_hw_probe()
801 if (props->cassoc_wd && in mpam_ris_hw_probe()
807 if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) { in mpam_ris_hw_probe()
810 props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features); in mpam_ris_hw_probe()
811 if (props->cpbm_wd) in mpam_ris_hw_probe()
816 if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) { in mpam_ris_hw_probe()
820 props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features); in mpam_ris_hw_probe()
821 if (props->mbw_pbm_bits && in mpam_ris_hw_probe()
825 props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features); in mpam_ris_hw_probe()
828 * The BWA_WD field can represent 0-63, but the control fields it in mpam_ris_hw_probe()
831 props->bwa_wd = min(props->bwa_wd, 16); in mpam_ris_hw_probe()
833 if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) in mpam_ris_hw_probe()
836 if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features)) in mpam_ris_hw_probe()
839 if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features)) in mpam_ris_hw_probe()
844 if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) { in mpam_ris_hw_probe()
847 props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features); in mpam_ris_hw_probe()
848 if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) { in mpam_ris_hw_probe()
854 props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features); in mpam_ris_hw_probe()
855 if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) { in mpam_ris_hw_probe()
863 if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) { in mpam_ris_hw_probe()
867 * If the firmware max-nrdy-us property is missing, the in mpam_ris_hw_probe()
870 err = device_property_read_u32(&msc->pdev->dev, in mpam_ris_hw_probe()
871 "arm,not-ready-us", in mpam_ris_hw_probe()
872 &msc->nrdy_usec); in mpam_ris_hw_probe()
878 props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr); in mpam_ris_hw_probe()
879 if (props->num_csu_mon) { in mpam_ris_hw_probe()
892 * un-implemented. in mpam_ris_hw_probe()
895 …dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.… in mpam_ris_hw_probe()
902 props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr); in mpam_ris_hw_probe()
903 if (props->num_mbwu_mon) { in mpam_ris_hw_probe()
928 if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) && in mpam_ris_hw_probe()
929 class->type != MPAM_CLASS_UNKNOWN) { in mpam_ris_hw_probe()
934 msc->partid_max = min(msc->partid_max, partid_max); in mpam_ris_hw_probe()
944 struct device *dev = &msc->pdev->dev; in mpam_msc_hw_probe()
946 lockdep_assert_held(&msc->probe_lock); in mpam_msc_hw_probe()
951 return -EIO; in mpam_msc_hw_probe()
955 mutex_lock(&msc->part_sel_lock); in mpam_msc_hw_probe()
957 msc->iidr = mpam_read_partsel_reg(msc, IIDR); in mpam_msc_hw_probe()
958 mutex_unlock(&msc->part_sel_lock); in mpam_msc_hw_probe()
962 msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr); in mpam_msc_hw_probe()
965 msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); in mpam_msc_hw_probe()
966 msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); in mpam_msc_hw_probe()
968 for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) { in mpam_msc_hw_probe()
969 mutex_lock(&msc->part_sel_lock); in mpam_msc_hw_probe()
972 mutex_unlock(&msc->part_sel_lock); in mpam_msc_hw_probe()
976 msc->partid_max = min(msc->partid_max, partid_max); in mpam_msc_hw_probe()
977 msc->pmg_max = min(msc->pmg_max, pmg_max); in mpam_msc_hw_probe()
978 msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr); in mpam_msc_hw_probe()
985 ris->idr = idr; in mpam_msc_hw_probe()
987 mutex_lock(&msc->part_sel_lock); in mpam_msc_hw_probe()
990 mutex_unlock(&msc->part_sel_lock); in mpam_msc_hw_probe()
997 mpam_partid_max = min(mpam_partid_max, msc->partid_max); in mpam_msc_hw_probe()
998 mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max); in mpam_msc_hw_probe()
1001 msc->probed = true; in mpam_msc_hw_probe()
1017 return (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props) || in mpam_ris_has_mbwu_long_counter()
1018 mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props)); in mpam_ris_has_mbwu_long_counter()
1029 WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); in mpam_msc_read_mbwu_l()
1030 WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); in mpam_msc_read_mbwu_l()
1038 retry--; in mpam_msc_read_mbwu_l()
1052 WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); in mpam_msc_zero_mbwu_l()
1053 WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); in mpam_msc_zero_mbwu_l()
1062 struct mon_cfg *ctx = m->ctx; in gen_msmon_ctl_flt_vals()
1065 * For CSU counters its implementation-defined what happens when not in gen_msmon_ctl_flt_vals()
1070 *flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid); in gen_msmon_ctl_flt_vals()
1072 if (m->ctx->match_pmg) { in gen_msmon_ctl_flt_vals()
1074 *flt_val |= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg); in gen_msmon_ctl_flt_vals()
1077 switch (m->type) { in gen_msmon_ctl_flt_vals()
1081 if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props)) in gen_msmon_ctl_flt_vals()
1082 *flt_val |= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean); in gen_msmon_ctl_flt_vals()
1090 if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props)) in gen_msmon_ctl_flt_vals()
1091 *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts); in gen_msmon_ctl_flt_vals()
1095 pr_warn("Unexpected monitor type %d\n", m->type); in gen_msmon_ctl_flt_vals()
1102 struct mpam_msc *msc = m->ris->vmsc->msc; in read_msmon_ctl_flt_vals()
1104 switch (m->type) { in read_msmon_ctl_flt_vals()
1116 pr_warn("Unexpected monitor type %d\n", m->type); in read_msmon_ctl_flt_vals()
1132 struct mpam_msc *msc = m->ris->vmsc->msc; in write_msmon_ctl_flt_vals()
1138 switch (m->type) { in write_msmon_ctl_flt_vals()
1152 if (m->type == mpam_feat_msmon_mbwu_31counter) in write_msmon_ctl_flt_vals()
1155 mpam_msc_zero_mbwu_l(m->ris->vmsc->msc); in write_msmon_ctl_flt_vals()
1158 pr_warn("Unexpected monitor type %d\n", m->type); in write_msmon_ctl_flt_vals()
1196 struct mon_cfg *ctx = m->ctx; in __ris_msmon_read()
1198 struct mpam_msc_ris *ris = m->ris; in __ris_msmon_read()
1200 struct mpam_msc *msc = m->ris->vmsc->msc; in __ris_msmon_read()
1204 m->err = -EIO; in __ris_msmon_read()
1207 mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) | in __ris_msmon_read()
1208 FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); in __ris_msmon_read()
1211 switch (m->type) { in __ris_msmon_read()
1215 mbwu_state = &ris->mbwu_state[ctx->mon]; in __ris_msmon_read()
1217 reset_on_next_read = mbwu_state->reset_on_next_read; in __ris_msmon_read()
1218 mbwu_state->reset_on_next_read = false; in __ris_msmon_read()
1226 * Read the existing configuration to avoid re-writing the same values. in __ris_msmon_read()
1231 if (mpam_feat_msmon_mbwu_31counter == m->type) in __ris_msmon_read()
1233 else if (mpam_feat_msmon_mbwu_44counter == m->type || in __ris_msmon_read()
1234 mpam_feat_msmon_mbwu_63counter == m->type) in __ris_msmon_read()
1252 switch (m->type) { in __ris_msmon_read()
1258 if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout) in __ris_msmon_read()
1265 if (m->type != mpam_feat_msmon_mbwu_31counter) { in __ris_msmon_read()
1269 if (m->type == mpam_feat_msmon_mbwu_63counter) in __ris_msmon_read()
1280 m->type != mpam_feat_msmon_mbwu_63counter) in __ris_msmon_read()
1286 mbwu_state = &ris->mbwu_state[ctx->mon]; in __ris_msmon_read()
1289 mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc); in __ris_msmon_read()
1295 now += mbwu_state->correction; in __ris_msmon_read()
1298 m->err = -EINVAL; in __ris_msmon_read()
1303 m->err = -EBUSY; in __ris_msmon_read()
1305 if (m->err) in __ris_msmon_read()
1308 *m->val += now; in __ris_msmon_read()
1317 list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, in _msmon_read()
1319 struct mpam_msc *msc = vmsc->msc; in _msmon_read()
1322 list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, in _msmon_read()
1324 arg->ris = ris; in _msmon_read()
1326 err = smp_call_function_any(&msc->accessibility, in _msmon_read()
1329 if (!err && arg->err) in _msmon_read()
1330 err = arg->err; in _msmon_read()
1347 struct mpam_props *cprops = &class->props; in mpam_msmon_choose_counter()
1363 struct mpam_class *class = comp->class; in mpam_msmon_read()
1364 struct mpam_props *cprops = &class->props; in mpam_msmon_read()
1369 return -EIO; in mpam_msmon_read()
1372 return -EOPNOTSUPP; in mpam_msmon_read()
1385 if (err == -EBUSY && class->nrdy_usec) in mpam_msmon_read()
1386 wait_jiffies = usecs_to_jiffies(class->nrdy_usec); in mpam_msmon_read()
1391 if (err == -EBUSY) { in mpam_msmon_read()
1416 list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, in mpam_msmon_reset_mbwu()
1418 if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props)) in mpam_msmon_reset_mbwu()
1421 msc = vmsc->msc; in mpam_msmon_reset_mbwu()
1422 list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, in mpam_msmon_reset_mbwu()
1424 if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) in mpam_msmon_reset_mbwu()
1430 ris->mbwu_state[ctx->mon].correction = 0; in mpam_msmon_reset_mbwu()
1431 ris->mbwu_state[ctx->mon].reset_on_next_read = true; in mpam_msmon_reset_mbwu()
1443 lockdep_assert_held(&msc->part_sel_lock); in mpam_reset_msc_bitmap()
1449 * Write all ~0 to all but the last 32bit-word, which may in mpam_reset_msc_bitmap()
1453 for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) in mpam_reset_msc_bitmap()
1460 msb = (wd - 1) % 32; in mpam_reset_msc_bitmap()
1471 regs = t241_scratch_regs[ris->vmsc->msc->t241_id]; in mpam_apply_t241_erratum()
1499 if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc)) in mpam_quirk_post_config_change()
1507 res0_bits = 16 - props->bwa_wd; in mpam_wa_t241_force_mbw_min_to_one()
1508 max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits; in mpam_wa_t241_force_mbw_min_to_one()
1519 u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1; in mpam_wa_t241_calc_min_from_max()
1522 max = cfg->mbw_max; in mpam_wa_t241_calc_min_from_max()
1525 max = GENMASK(15, 16 - props->bwa_wd); in mpam_wa_t241_calc_min_from_max()
1529 val = max - delta; in mpam_wa_t241_calc_min_from_max()
1540 struct mpam_msc *msc = ris->vmsc->msc; in mpam_reprogram_ris_partid()
1541 struct mpam_props *rprops = &ris->props; in mpam_reprogram_ris_partid()
1542 u16 dspri = GENMASK(rprops->dspri_wd, 0); in mpam_reprogram_ris_partid()
1543 u16 intpri = GENMASK(rprops->intpri_wd, 0); in mpam_reprogram_ris_partid()
1545 mutex_lock(&msc->part_sel_lock); in mpam_reprogram_ris_partid()
1546 __mpam_part_sel(ris->ris_idx, partid, msc); in mpam_reprogram_ris_partid()
1557 __mpam_intpart_sel(ris->ris_idx, partid, msc); in mpam_reprogram_ris_partid()
1562 mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); in mpam_reprogram_ris_partid()
1564 mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); in mpam_reprogram_ris_partid()
1569 mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); in mpam_reprogram_ris_partid()
1571 mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); in mpam_reprogram_ris_partid()
1589 mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); in mpam_reprogram_ris_partid()
1624 mutex_unlock(&msc->part_sel_lock); in mpam_reprogram_ris_partid()
1634 struct mpam_class *class = ris->vmsc->comp->class; in mpam_restore_mbwu_state()
1636 for (i = 0; i < ris->props.num_mbwu_mon; i++) { in mpam_restore_mbwu_state()
1637 if (ris->mbwu_state[i].enabled) { in mpam_restore_mbwu_state()
1639 mwbu_arg.ctx = &ris->mbwu_state[i].cfg; in mpam_restore_mbwu_state()
1659 struct mpam_msc *msc = ris->vmsc->msc; in mpam_save_mbwu_state()
1661 for (i = 0; i < ris->props.num_mbwu_mon; i++) { in mpam_save_mbwu_state()
1662 mbwu_state = &ris->mbwu_state[i]; in mpam_save_mbwu_state()
1663 cfg = &mbwu_state->cfg; in mpam_save_mbwu_state()
1666 return -EIO; in mpam_save_mbwu_state()
1669 FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); in mpam_save_mbwu_state()
1684 cfg->mon = i; in mpam_save_mbwu_state()
1685 cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt); in mpam_save_mbwu_state()
1686 cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl); in mpam_save_mbwu_state()
1687 cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt); in mpam_save_mbwu_state()
1688 mbwu_state->correction += val; in mpam_save_mbwu_state()
1689 mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl); in mpam_save_mbwu_state()
1698 * pre-emptible. Caller must hold mpam_srcu.
1706 if (ris->in_reset_state) in mpam_reset_ris()
1727 if (cpumask_test_cpu(cpu, &msc->accessibility)) in mpam_get_msc_preferred_cpu()
1730 return cpumask_first_and(&msc->accessibility, cpu_online_mask); in mpam_get_msc_preferred_cpu()
1752 mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]); in __write_config()
1771 mutex_lock(&msc->cfg_lock); in mpam_reprogram_msc()
1772 list_for_each_entry_srcu(ris, &msc->ris, msc_list, in mpam_reprogram_msc()
1774 if (!mpam_is_enabled() && !ris->in_reset_state) { in mpam_reprogram_msc()
1776 ris->in_reset_state = true; in mpam_reprogram_msc()
1780 arg.comp = ris->vmsc->comp; in mpam_reprogram_msc()
1784 cfg = &ris->vmsc->comp->cfg[partid]; in mpam_reprogram_msc()
1785 if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST)) in mpam_reprogram_msc()
1791 ris->in_reset_state = reset; in mpam_reprogram_msc()
1793 if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) in mpam_reprogram_msc()
1796 mutex_unlock(&msc->cfg_lock); in mpam_reprogram_msc()
1813 if (!cpumask_test_cpu(cpu, &msc->accessibility)) in mpam_cpu_online()
1816 if (msc->reenable_error_ppi) in mpam_cpu_online()
1817 _enable_percpu_irq(&msc->reenable_error_ppi); in mpam_cpu_online()
1819 if (atomic_fetch_inc(&msc->online_refs) == 0) in mpam_cpu_online()
1842 if (!cpumask_test_cpu(cpu, &msc->accessibility)) in mpam_discovery_cpu_online()
1845 mutex_lock(&msc->probe_lock); in mpam_discovery_cpu_online()
1846 if (!msc->probed) in mpam_discovery_cpu_online()
1848 mutex_unlock(&msc->probe_lock); in mpam_discovery_cpu_online()
1875 if (!cpumask_test_cpu(cpu, &msc->accessibility)) in mpam_cpu_offline()
1878 if (msc->reenable_error_ppi) in mpam_cpu_offline()
1879 disable_percpu_irq(msc->reenable_error_ppi); in mpam_cpu_offline()
1881 if (atomic_dec_and_test(&msc->online_refs)) { in mpam_cpu_offline()
1884 mutex_lock(&msc->cfg_lock); in mpam_cpu_offline()
1885 list_for_each_entry_srcu(ris, &msc->ris, msc_list, in mpam_cpu_offline()
1890 * The reset state for non-zero partid may be in mpam_cpu_offline()
1893 ris->in_reset_state = false; in mpam_cpu_offline()
1898 mutex_unlock(&msc->cfg_lock); in mpam_cpu_offline()
1928 msc->error_dev_id = alloc_percpu(struct mpam_msc *); in __setup_ppi()
1929 if (!msc->error_dev_id) in __setup_ppi()
1930 return -ENOMEM; in __setup_ppi()
1932 for_each_cpu(cpu, &msc->accessibility) in __setup_ppi()
1933 *per_cpu_ptr(msc->error_dev_id, cpu) = msc; in __setup_ppi()
1942 irq = platform_get_irq_byname_optional(msc->pdev, "error"); in mpam_msc_setup_error_irq()
1951 if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) { in mpam_msc_setup_error_irq()
1952 pr_err_once("msc:%u is a private resource with a shared error interrupt", in mpam_msc_setup_error_irq()
1953 msc->id); in mpam_msc_setup_error_irq()
1954 return -EINVAL; in mpam_msc_setup_error_irq()
1972 err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity", in update_msc_accessibility()
1975 cpumask_copy(&msc->accessibility, cpu_possible_mask); in update_msc_accessibility()
1977 acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility); in update_msc_accessibility()
1982 * class->component->vmsc->ris, or via the msc.
1987 struct platform_device *pdev = msc->pdev; in mpam_msc_destroy()
1992 list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list) in mpam_msc_destroy()
1995 list_del_rcu(&msc->all_msc_list); in mpam_msc_destroy()
2018 struct device *dev = &pdev->dev; in do_mpam_msc_drv_probe()
2022 msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL); in do_mpam_msc_drv_probe()
2024 return ERR_PTR(-ENOMEM); in do_mpam_msc_drv_probe()
2025 init_garbage(&msc->garbage); in do_mpam_msc_drv_probe()
2026 msc->garbage.pdev = pdev; in do_mpam_msc_drv_probe()
2028 err = devm_mutex_init(dev, &msc->probe_lock); in do_mpam_msc_drv_probe()
2032 err = devm_mutex_init(dev, &msc->part_sel_lock); in do_mpam_msc_drv_probe()
2036 err = devm_mutex_init(dev, &msc->error_irq_lock); in do_mpam_msc_drv_probe()
2040 err = devm_mutex_init(dev, &msc->cfg_lock); in do_mpam_msc_drv_probe()
2045 msc->id = pdev->id; in do_mpam_msc_drv_probe()
2046 msc->pdev = pdev; in do_mpam_msc_drv_probe()
2047 INIT_LIST_HEAD_RCU(&msc->all_msc_list); in do_mpam_msc_drv_probe()
2048 INIT_LIST_HEAD_RCU(&msc->ris); in do_mpam_msc_drv_probe()
2051 if (cpumask_empty(&msc->accessibility)) { in do_mpam_msc_drv_probe()
2053 return ERR_PTR(-EINVAL); in do_mpam_msc_drv_probe()
2060 if (device_property_read_u32(&pdev->dev, "pcc-channel", &tmp)) in do_mpam_msc_drv_probe()
2061 msc->iface = MPAM_IFACE_MMIO; in do_mpam_msc_drv_probe()
2063 msc->iface = MPAM_IFACE_PCC; in do_mpam_msc_drv_probe()
2065 if (msc->iface == MPAM_IFACE_MMIO) { in do_mpam_msc_drv_probe()
2074 msc->mapped_hwpage_sz = msc_res->end - msc_res->start; in do_mpam_msc_drv_probe()
2075 msc->mapped_hwpage = io; in do_mpam_msc_drv_probe()
2077 return ERR_PTR(-EINVAL); in do_mpam_msc_drv_probe()
2080 list_add_rcu(&msc->all_msc_list, &mpam_all_msc); in do_mpam_msc_drv_probe()
2092 void *plat_data = pdev->dev.platform_data; in mpam_msc_drv_probe()
2147 ((helper(child) && (parent)->field != (child)->field) || \
2152 ((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) || \
2164 * For non-aliasing controls, these control different resources, and the
2174 parent->cpbm_wd = child->cpbm_wd; in __props_mismatch()
2179 parent->cpbm_wd = 0; in __props_mismatch()
2183 parent->mbw_pbm_bits = child->mbw_pbm_bits; in __props_mismatch()
2188 parent->mbw_pbm_bits = 0; in __props_mismatch()
2194 parent->bwa_wd = child->bwa_wd; in __props_mismatch()
2198 parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd); in __props_mismatch()
2202 parent->cmax_wd = child->cmax_wd; in __props_mismatch()
2206 parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd); in __props_mismatch()
2210 parent->cassoc_wd = child->cassoc_wd; in __props_mismatch()
2215 parent->cassoc_wd = 0; in __props_mismatch()
2220 parent->num_csu_mon = child->num_csu_mon; in __props_mismatch()
2224 parent->num_csu_mon = min(parent->num_csu_mon, in __props_mismatch()
2225 child->num_csu_mon); in __props_mismatch()
2229 parent->num_mbwu_mon = child->num_mbwu_mon; in __props_mismatch()
2233 parent->num_mbwu_mon = min(parent->num_mbwu_mon, in __props_mismatch()
2234 child->num_mbwu_mon); in __props_mismatch()
2238 parent->intpri_wd = child->intpri_wd; in __props_mismatch()
2242 parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd); in __props_mismatch()
2246 parent->dspri_wd = child->dspri_wd; in __props_mismatch()
2250 parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd); in __props_mismatch()
2254 /* {int,ds}pri may not have differing 0-low behaviour */ in __props_mismatch()
2274 bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); in __props_mismatch()
2277 bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); in __props_mismatch()
2293 struct mpam_props *cprops = &class->props; in __class_props_mismatch()
2294 struct mpam_props *vprops = &vmsc->props; in __class_props_mismatch()
2295 struct device *dev = &vmsc->msc->pdev->dev; in __class_props_mismatch()
2300 (long)cprops->features, (long)vprops->features); in __class_props_mismatch()
2303 class->quirks |= vmsc->msc->quirks; in __class_props_mismatch()
2312 struct mpam_props *rprops = &ris->props; in __vmsc_props_mismatch()
2313 struct mpam_props *vprops = &vmsc->props; in __vmsc_props_mismatch()
2314 struct device *dev = &vmsc->msc->pdev->dev; in __vmsc_props_mismatch()
2319 (long)vprops->features, (long)rprops->features); in __vmsc_props_mismatch()
2322 * Merge mismatched features - Copy any features that aren't common, in __vmsc_props_mismatch()
2339 comp = list_first_entry(&class->components, in mpam_enable_init_class_features()
2341 vmsc = list_first_entry(&comp->vmsc, in mpam_enable_init_class_features()
2344 class->props = vmsc->props; in mpam_enable_init_class_features()
2351 struct mpam_class *class = comp->class; in mpam_enable_merge_vmsc_features()
2353 list_for_each_entry(vmsc, &comp->vmsc, comp_list) { in mpam_enable_merge_vmsc_features()
2354 list_for_each_entry(ris, &vmsc->ris, vmsc_list) { in mpam_enable_merge_vmsc_features()
2356 class->nrdy_usec = max(class->nrdy_usec, in mpam_enable_merge_vmsc_features()
2357 vmsc->msc->nrdy_usec); in mpam_enable_merge_vmsc_features()
2365 struct mpam_class *class = comp->class; in mpam_enable_merge_class_features()
2367 list_for_each_entry(vmsc, &comp->vmsc, comp_list) in mpam_enable_merge_class_features()
2371 mpam_clear_feature(mpam_feat_mbw_min, &class->props); in mpam_enable_merge_class_features()
2376 * vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features()
2379 * Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc
2383 * To avoid walking the whole tree twice, the class->nrdy_usec property is
2395 list_for_each_entry(comp, &class->components, class_list) in mpam_enable_merge_features()
2400 list_for_each_entry(comp, &class->components, class_list) in mpam_enable_merge_features()
2430 /* This can run in mpam_disable(), and the interrupt handler on the same CPU */
2448 &msc->accessibility))) in __mpam_irq_handler()
2465 msc->id, mpam_errcode_names[errcode], partid, pmg, in __mpam_irq_handler()
2468 /* Disable this interrupt. */ in __mpam_irq_handler()
2477 * unregister the interrupt from the threaded part of the handler. in __mpam_irq_handler()
2479 mpam_disable_reason = "hardware error interrupt"; in __mpam_irq_handler()
2509 irq = platform_get_irq_byname_optional(msc->pdev, "error"); in mpam_register_irqs()
2513 /* The MPAM spec says the interrupt can be SPI, PPI or LPI */ in mpam_register_irqs()
2514 /* We anticipate sharing the interrupt with other MSCs */ in mpam_register_irqs()
2518 msc->error_dev_id); in mpam_register_irqs()
2522 msc->reenable_error_ppi = irq; in mpam_register_irqs()
2523 smp_call_function_many(&msc->accessibility, in mpam_register_irqs()
2527 err = devm_request_irq(&msc->pdev->dev, irq, in mpam_register_irqs()
2534 mutex_lock(&msc->error_irq_lock); in mpam_register_irqs()
2535 msc->error_irq_req = true; in mpam_register_irqs()
2537 msc->error_irq_hw_enabled = true; in mpam_register_irqs()
2538 mutex_unlock(&msc->error_irq_lock); in mpam_register_irqs()
2553 irq = platform_get_irq_byname_optional(msc->pdev, "error"); in mpam_unregister_irqs()
2557 mutex_lock(&msc->error_irq_lock); in mpam_unregister_irqs()
2558 if (msc->error_irq_hw_enabled) { in mpam_unregister_irqs()
2560 msc->error_irq_hw_enabled = false; in mpam_unregister_irqs()
2563 if (msc->error_irq_req) { in mpam_unregister_irqs()
2565 msc->reenable_error_ppi = 0; in mpam_unregister_irqs()
2566 free_percpu_irq(irq, msc->error_dev_id); in mpam_unregister_irqs()
2568 devm_free_irq(&msc->pdev->dev, irq, msc); in mpam_unregister_irqs()
2570 msc->error_irq_req = false; in mpam_unregister_irqs()
2572 mutex_unlock(&msc->error_irq_lock); in mpam_unregister_irqs()
2584 if (!comp->cfg) in __destroy_component_cfg()
2587 add_to_garbage(comp->cfg); in __destroy_component_cfg()
2588 list_for_each_entry(vmsc, &comp->vmsc, comp_list) { in __destroy_component_cfg()
2589 msc = vmsc->msc; in __destroy_component_cfg()
2592 list_for_each_entry(ris, &vmsc->ris, vmsc_list) in __destroy_component_cfg()
2593 add_to_garbage(ris->mbwu_state); in __destroy_component_cfg()
2602 struct mpam_props *cprops = &comp->class->props; in mpam_reset_component_cfg()
2606 if (!comp->cfg) in mpam_reset_component_cfg()
2610 comp->cfg[i] = (struct mpam_config) {}; in mpam_reset_component_cfg()
2611 if (cprops->cpbm_wd) in mpam_reset_component_cfg()
2612 comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0); in mpam_reset_component_cfg()
2613 if (cprops->mbw_pbm_bits) in mpam_reset_component_cfg()
2614 comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); in mpam_reset_component_cfg()
2615 if (cprops->bwa_wd) in mpam_reset_component_cfg()
2616 comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); in mpam_reset_component_cfg()
2626 if (comp->cfg) in __allocate_component_cfg()
2629 comp->cfg = kzalloc_objs(*comp->cfg, mpam_partid_max + 1); in __allocate_component_cfg()
2630 if (!comp->cfg) in __allocate_component_cfg()
2631 return -ENOMEM; in __allocate_component_cfg()
2637 init_garbage(&comp->cfg[0].garbage); in __allocate_component_cfg()
2641 list_for_each_entry(vmsc, &comp->vmsc, comp_list) { in __allocate_component_cfg()
2646 if (!vmsc->props.num_mbwu_mon) in __allocate_component_cfg()
2649 msc = vmsc->msc; in __allocate_component_cfg()
2650 list_for_each_entry(ris, &vmsc->ris, vmsc_list) { in __allocate_component_cfg()
2651 if (!ris->props.num_mbwu_mon) in __allocate_component_cfg()
2654 mbwu_state = kzalloc_objs(*ris->mbwu_state, in __allocate_component_cfg()
2655 ris->props.num_mbwu_mon); in __allocate_component_cfg()
2658 return -ENOMEM; in __allocate_component_cfg()
2664 ris->mbwu_state = mbwu_state; in __allocate_component_cfg()
2681 list_for_each_entry(comp, &class->components, class_list) { in mpam_allocate_config()
2705 * That involves cross-calling to a CPU that can reach the MSC, and in mpam_enable_once()
2760 list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, in mpam_reset_component_locked()
2762 struct mpam_msc *msc = vmsc->msc; in mpam_reset_component_locked()
2765 list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, in mpam_reset_component_locked()
2767 if (!ris->in_reset_state) in mpam_reset_component_locked()
2769 ris->in_reset_state = true; in mpam_reset_component_locked()
2781 list_for_each_entry_srcu(comp, &class->components, class_list, in mpam_reset_class_locked()
2860 mutex_lock(&msc->probe_lock); in mpam_enable()
2861 if (!msc->probed) in mpam_enable()
2863 mutex_unlock(&msc->probe_lock); in mpam_enable()
2875 (newcfg)->member != (cfg)->member) { \
2876 (cfg)->member = (newcfg)->member; \
2906 WARN_ON_ONCE(&comp->cfg[partid] == cfg); in mpam_apply_config()
2908 if (!mpam_update_config(&comp->cfg[partid], cfg)) in mpam_apply_config()
2915 list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, in mpam_apply_config()
2917 msc = vmsc->msc; in mpam_apply_config()
2919 mutex_lock(&msc->cfg_lock); in mpam_apply_config()
2920 list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, in mpam_apply_config()
2924 ris->in_reset_state = false; in mpam_apply_config()
2926 mutex_unlock(&msc->cfg_lock); in mpam_apply_config()
2935 return -EOPNOTSUPP; in mpam_msc_driver_init()
2942 return -EINVAL; in mpam_msc_driver_init()