xref: /linux/tools/testing/selftests/kvm/aarch64/get-reg-list.c (revision 114143a595895c03fbefccfd8346fc51fb4908ed) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Check for KVM_GET_REG_LIST regressions.
4  *
5  * Copyright (C) 2020, Red Hat, Inc.
6  *
7  * While the blessed list should be created from the oldest possible
8  * kernel, we can't go older than v5.2, though, because that's the first
9  * release which includes df205b5c6328 ("KVM: arm64: Filter out invalid
10  * core register IDs in KVM_GET_REG_LIST"). Without that commit the core
11  * registers won't match expectations.
12  */
13 #include <stdio.h>
14 #include "kvm_util.h"
15 #include "test_util.h"
16 #include "processor.h"
17 
18 struct feature_id_reg {
19 	__u64 reg;
20 	__u64 id_reg;
21 	__u64 feat_shift;
22 	__u64 feat_min;
23 };
24 
25 static struct feature_id_reg feat_id_regs[] = {
26 	{
27 		ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
28 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
29 		0,
30 		1
31 	},
32 	{
33 		ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
34 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
35 		8,
36 		1
37 	},
38 	{
39 		ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
40 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
41 		8,
42 		1
43 	},
44 	{
45 		ARM64_SYS_REG(3, 0, 10, 2, 4),	/* POR_EL1 */
46 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
47 		16,
48 		1
49 	},
50 	{
51 		ARM64_SYS_REG(3, 3, 10, 2, 4),	/* POR_EL0 */
52 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
53 		16,
54 		1
55 	}
56 };
57 
filter_reg(__u64 reg)58 bool filter_reg(__u64 reg)
59 {
60 	/*
61 	 * DEMUX register presence depends on the host's CLIDR_EL1.
62 	 * This means there's no set of them that we can bless.
63 	 */
64 	if ((reg & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
65 		return true;
66 
67 	return false;
68 }
69 
check_supported_feat_reg(struct kvm_vcpu * vcpu,__u64 reg)70 static bool check_supported_feat_reg(struct kvm_vcpu *vcpu, __u64 reg)
71 {
72 	int i, ret;
73 	__u64 data, feat_val;
74 
75 	for (i = 0; i < ARRAY_SIZE(feat_id_regs); i++) {
76 		if (feat_id_regs[i].reg == reg) {
77 			ret = __vcpu_get_reg(vcpu, feat_id_regs[i].id_reg, &data);
78 			if (ret < 0)
79 				return false;
80 
81 			feat_val = ((data >> feat_id_regs[i].feat_shift) & 0xf);
82 			return feat_val >= feat_id_regs[i].feat_min;
83 		}
84 	}
85 
86 	return true;
87 }
88 
check_supported_reg(struct kvm_vcpu * vcpu,__u64 reg)89 bool check_supported_reg(struct kvm_vcpu *vcpu, __u64 reg)
90 {
91 	return check_supported_feat_reg(vcpu, reg);
92 }
93 
check_reject_set(int err)94 bool check_reject_set(int err)
95 {
96 	return err == EPERM;
97 }
98 
finalize_vcpu(struct kvm_vcpu * vcpu,struct vcpu_reg_list * c)99 void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)
100 {
101 	struct vcpu_reg_sublist *s;
102 	int feature;
103 
104 	for_each_sublist(c, s) {
105 		if (s->finalize) {
106 			feature = s->feature;
107 			vcpu_ioctl(vcpu, KVM_ARM_VCPU_FINALIZE, &feature);
108 		}
109 	}
110 }
111 
112 #define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_COPROC_MASK)
113 
114 #define CORE_REGS_XX_NR_WORDS	2
115 #define CORE_SPSR_XX_NR_WORDS	2
116 #define CORE_FPREGS_XX_NR_WORDS	4
117 
core_id_to_str(const char * prefix,__u64 id)118 static const char *core_id_to_str(const char *prefix, __u64 id)
119 {
120 	__u64 core_off = id & ~REG_MASK, idx;
121 
122 	/*
123 	 * core_off is the offset into struct kvm_regs
124 	 */
125 	switch (core_off) {
126 	case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
127 	     KVM_REG_ARM_CORE_REG(regs.regs[30]):
128 		idx = (core_off - KVM_REG_ARM_CORE_REG(regs.regs[0])) / CORE_REGS_XX_NR_WORDS;
129 		TEST_ASSERT(idx < 31, "%s: Unexpected regs.regs index: %lld", prefix, idx);
130 		return strdup_printf("KVM_REG_ARM_CORE_REG(regs.regs[%lld])", idx);
131 	case KVM_REG_ARM_CORE_REG(regs.sp):
132 		return "KVM_REG_ARM_CORE_REG(regs.sp)";
133 	case KVM_REG_ARM_CORE_REG(regs.pc):
134 		return "KVM_REG_ARM_CORE_REG(regs.pc)";
135 	case KVM_REG_ARM_CORE_REG(regs.pstate):
136 		return "KVM_REG_ARM_CORE_REG(regs.pstate)";
137 	case KVM_REG_ARM_CORE_REG(sp_el1):
138 		return "KVM_REG_ARM_CORE_REG(sp_el1)";
139 	case KVM_REG_ARM_CORE_REG(elr_el1):
140 		return "KVM_REG_ARM_CORE_REG(elr_el1)";
141 	case KVM_REG_ARM_CORE_REG(spsr[0]) ...
142 	     KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
143 		idx = (core_off - KVM_REG_ARM_CORE_REG(spsr[0])) / CORE_SPSR_XX_NR_WORDS;
144 		TEST_ASSERT(idx < KVM_NR_SPSR, "%s: Unexpected spsr index: %lld", prefix, idx);
145 		return strdup_printf("KVM_REG_ARM_CORE_REG(spsr[%lld])", idx);
146 	case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
147 	     KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
148 		idx = (core_off - KVM_REG_ARM_CORE_REG(fp_regs.vregs[0])) / CORE_FPREGS_XX_NR_WORDS;
149 		TEST_ASSERT(idx < 32, "%s: Unexpected fp_regs.vregs index: %lld", prefix, idx);
150 		return strdup_printf("KVM_REG_ARM_CORE_REG(fp_regs.vregs[%lld])", idx);
151 	case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
152 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpsr)";
153 	case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
154 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpcr)";
155 	}
156 
157 	TEST_FAIL("%s: Unknown core reg id: 0x%llx", prefix, id);
158 	return NULL;
159 }
160 
sve_id_to_str(const char * prefix,__u64 id)161 static const char *sve_id_to_str(const char *prefix, __u64 id)
162 {
163 	__u64 sve_off, n, i;
164 
165 	if (id == KVM_REG_ARM64_SVE_VLS)
166 		return "KVM_REG_ARM64_SVE_VLS";
167 
168 	sve_off = id & ~(REG_MASK | ((1ULL << 5) - 1));
169 	i = id & (KVM_ARM64_SVE_MAX_SLICES - 1);
170 
171 	TEST_ASSERT(i == 0, "%s: Currently we don't expect slice > 0, reg id 0x%llx", prefix, id);
172 
173 	switch (sve_off) {
174 	case KVM_REG_ARM64_SVE_ZREG_BASE ...
175 	     KVM_REG_ARM64_SVE_ZREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_ZREGS - 1:
176 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_ZREGS - 1);
177 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_ZREG(n, 0),
178 			    "%s: Unexpected bits set in SVE ZREG id: 0x%llx", prefix, id);
179 		return strdup_printf("KVM_REG_ARM64_SVE_ZREG(%lld, 0)", n);
180 	case KVM_REG_ARM64_SVE_PREG_BASE ...
181 	     KVM_REG_ARM64_SVE_PREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_PREGS - 1:
182 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_PREGS - 1);
183 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_PREG(n, 0),
184 			    "%s: Unexpected bits set in SVE PREG id: 0x%llx", prefix, id);
185 		return strdup_printf("KVM_REG_ARM64_SVE_PREG(%lld, 0)", n);
186 	case KVM_REG_ARM64_SVE_FFR_BASE:
187 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_FFR(0),
188 			    "%s: Unexpected bits set in SVE FFR id: 0x%llx", prefix, id);
189 		return "KVM_REG_ARM64_SVE_FFR(0)";
190 	}
191 
192 	return NULL;
193 }
194 
print_reg(const char * prefix,__u64 id)195 void print_reg(const char *prefix, __u64 id)
196 {
197 	unsigned op0, op1, crn, crm, op2;
198 	const char *reg_size = NULL;
199 
200 	TEST_ASSERT((id & KVM_REG_ARCH_MASK) == KVM_REG_ARM64,
201 		    "%s: KVM_REG_ARM64 missing in reg id: 0x%llx", prefix, id);
202 
203 	switch (id & KVM_REG_SIZE_MASK) {
204 	case KVM_REG_SIZE_U8:
205 		reg_size = "KVM_REG_SIZE_U8";
206 		break;
207 	case KVM_REG_SIZE_U16:
208 		reg_size = "KVM_REG_SIZE_U16";
209 		break;
210 	case KVM_REG_SIZE_U32:
211 		reg_size = "KVM_REG_SIZE_U32";
212 		break;
213 	case KVM_REG_SIZE_U64:
214 		reg_size = "KVM_REG_SIZE_U64";
215 		break;
216 	case KVM_REG_SIZE_U128:
217 		reg_size = "KVM_REG_SIZE_U128";
218 		break;
219 	case KVM_REG_SIZE_U256:
220 		reg_size = "KVM_REG_SIZE_U256";
221 		break;
222 	case KVM_REG_SIZE_U512:
223 		reg_size = "KVM_REG_SIZE_U512";
224 		break;
225 	case KVM_REG_SIZE_U1024:
226 		reg_size = "KVM_REG_SIZE_U1024";
227 		break;
228 	case KVM_REG_SIZE_U2048:
229 		reg_size = "KVM_REG_SIZE_U2048";
230 		break;
231 	default:
232 		TEST_FAIL("%s: Unexpected reg size: 0x%llx in reg id: 0x%llx",
233 			  prefix, (id & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT, id);
234 	}
235 
236 	switch (id & KVM_REG_ARM_COPROC_MASK) {
237 	case KVM_REG_ARM_CORE:
238 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_CORE | %s,\n", reg_size, core_id_to_str(prefix, id));
239 		break;
240 	case KVM_REG_ARM_DEMUX:
241 		TEST_ASSERT(!(id & ~(REG_MASK | KVM_REG_ARM_DEMUX_ID_MASK | KVM_REG_ARM_DEMUX_VAL_MASK)),
242 			    "%s: Unexpected bits set in DEMUX reg id: 0x%llx", prefix, id);
243 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_DEMUX | KVM_REG_ARM_DEMUX_ID_CCSIDR | %lld,\n",
244 		       reg_size, id & KVM_REG_ARM_DEMUX_VAL_MASK);
245 		break;
246 	case KVM_REG_ARM64_SYSREG:
247 		op0 = (id & KVM_REG_ARM64_SYSREG_OP0_MASK) >> KVM_REG_ARM64_SYSREG_OP0_SHIFT;
248 		op1 = (id & KVM_REG_ARM64_SYSREG_OP1_MASK) >> KVM_REG_ARM64_SYSREG_OP1_SHIFT;
249 		crn = (id & KVM_REG_ARM64_SYSREG_CRN_MASK) >> KVM_REG_ARM64_SYSREG_CRN_SHIFT;
250 		crm = (id & KVM_REG_ARM64_SYSREG_CRM_MASK) >> KVM_REG_ARM64_SYSREG_CRM_SHIFT;
251 		op2 = (id & KVM_REG_ARM64_SYSREG_OP2_MASK) >> KVM_REG_ARM64_SYSREG_OP2_SHIFT;
252 		TEST_ASSERT(id == ARM64_SYS_REG(op0, op1, crn, crm, op2),
253 			    "%s: Unexpected bits set in SYSREG reg id: 0x%llx", prefix, id);
254 		printf("\tARM64_SYS_REG(%d, %d, %d, %d, %d),\n", op0, op1, crn, crm, op2);
255 		break;
256 	case KVM_REG_ARM_FW:
257 		TEST_ASSERT(id == KVM_REG_ARM_FW_REG(id & 0xffff),
258 			    "%s: Unexpected bits set in FW reg id: 0x%llx", prefix, id);
259 		printf("\tKVM_REG_ARM_FW_REG(%lld),\n", id & 0xffff);
260 		break;
261 	case KVM_REG_ARM_FW_FEAT_BMAP:
262 		TEST_ASSERT(id == KVM_REG_ARM_FW_FEAT_BMAP_REG(id & 0xffff),
263 			    "%s: Unexpected bits set in the bitmap feature FW reg id: 0x%llx", prefix, id);
264 		printf("\tKVM_REG_ARM_FW_FEAT_BMAP_REG(%lld),\n", id & 0xffff);
265 		break;
266 	case KVM_REG_ARM64_SVE:
267 		printf("\t%s,\n", sve_id_to_str(prefix, id));
268 		break;
269 	default:
270 		TEST_FAIL("%s: Unexpected coproc type: 0x%llx in reg id: 0x%llx",
271 			  prefix, (id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT, id);
272 	}
273 }
274 
275 /*
276  * The original blessed list was primed with the output of kernel version
277  * v4.15 with --core-reg-fixup and then later updated with new registers.
278  * (The --core-reg-fixup option and it's fixup function have been removed
279  * from the test, as it's unlikely to use this type of test on a kernel
280  * older than v5.2.)
281  *
282  * The blessed list is up to date with kernel version v6.4 (or so we hope)
283  */
284 static __u64 base_regs[] = {
285 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[0]),
286 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[1]),
287 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[2]),
288 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[3]),
289 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[4]),
290 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[5]),
291 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[6]),
292 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[7]),
293 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[8]),
294 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[9]),
295 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[10]),
296 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[11]),
297 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[12]),
298 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[13]),
299 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[14]),
300 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[15]),
301 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[16]),
302 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[17]),
303 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[18]),
304 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[19]),
305 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[20]),
306 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[21]),
307 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[22]),
308 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[23]),
309 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[24]),
310 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[25]),
311 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[26]),
312 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[27]),
313 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[28]),
314 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[29]),
315 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[30]),
316 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.sp),
317 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pc),
318 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pstate),
319 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(sp_el1),
320 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(elr_el1),
321 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[0]),
322 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[1]),
323 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[2]),
324 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[3]),
325 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[4]),
326 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpsr),
327 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpcr),
328 	KVM_REG_ARM_FW_REG(0),		/* KVM_REG_ARM_PSCI_VERSION */
329 	KVM_REG_ARM_FW_REG(1),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1 */
330 	KVM_REG_ARM_FW_REG(2),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 */
331 	KVM_REG_ARM_FW_REG(3),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 */
332 	KVM_REG_ARM_FW_FEAT_BMAP_REG(0),	/* KVM_REG_ARM_STD_BMAP */
333 	KVM_REG_ARM_FW_FEAT_BMAP_REG(1),	/* KVM_REG_ARM_STD_HYP_BMAP */
334 	KVM_REG_ARM_FW_FEAT_BMAP_REG(2),	/* KVM_REG_ARM_VENDOR_HYP_BMAP */
335 	ARM64_SYS_REG(3, 3, 14, 3, 1),	/* CNTV_CTL_EL0 */
336 	ARM64_SYS_REG(3, 3, 14, 3, 2),	/* CNTV_CVAL_EL0 */
337 	ARM64_SYS_REG(3, 3, 14, 0, 2),
338 	ARM64_SYS_REG(3, 0, 0, 0, 0),	/* MIDR_EL1 */
339 	ARM64_SYS_REG(3, 0, 0, 0, 6),	/* REVIDR_EL1 */
340 	ARM64_SYS_REG(3, 1, 0, 0, 1),	/* CLIDR_EL1 */
341 	ARM64_SYS_REG(3, 1, 0, 0, 7),	/* AIDR_EL1 */
342 	ARM64_SYS_REG(3, 3, 0, 0, 1),	/* CTR_EL0 */
343 	ARM64_SYS_REG(2, 0, 0, 0, 4),
344 	ARM64_SYS_REG(2, 0, 0, 0, 5),
345 	ARM64_SYS_REG(2, 0, 0, 0, 6),
346 	ARM64_SYS_REG(2, 0, 0, 0, 7),
347 	ARM64_SYS_REG(2, 0, 0, 1, 4),
348 	ARM64_SYS_REG(2, 0, 0, 1, 5),
349 	ARM64_SYS_REG(2, 0, 0, 1, 6),
350 	ARM64_SYS_REG(2, 0, 0, 1, 7),
351 	ARM64_SYS_REG(2, 0, 0, 2, 0),	/* MDCCINT_EL1 */
352 	ARM64_SYS_REG(2, 0, 0, 2, 2),	/* MDSCR_EL1 */
353 	ARM64_SYS_REG(2, 0, 0, 2, 4),
354 	ARM64_SYS_REG(2, 0, 0, 2, 5),
355 	ARM64_SYS_REG(2, 0, 0, 2, 6),
356 	ARM64_SYS_REG(2, 0, 0, 2, 7),
357 	ARM64_SYS_REG(2, 0, 0, 3, 4),
358 	ARM64_SYS_REG(2, 0, 0, 3, 5),
359 	ARM64_SYS_REG(2, 0, 0, 3, 6),
360 	ARM64_SYS_REG(2, 0, 0, 3, 7),
361 	ARM64_SYS_REG(2, 0, 0, 4, 4),
362 	ARM64_SYS_REG(2, 0, 0, 4, 5),
363 	ARM64_SYS_REG(2, 0, 0, 4, 6),
364 	ARM64_SYS_REG(2, 0, 0, 4, 7),
365 	ARM64_SYS_REG(2, 0, 0, 5, 4),
366 	ARM64_SYS_REG(2, 0, 0, 5, 5),
367 	ARM64_SYS_REG(2, 0, 0, 5, 6),
368 	ARM64_SYS_REG(2, 0, 0, 5, 7),
369 	ARM64_SYS_REG(2, 0, 0, 6, 4),
370 	ARM64_SYS_REG(2, 0, 0, 6, 5),
371 	ARM64_SYS_REG(2, 0, 0, 6, 6),
372 	ARM64_SYS_REG(2, 0, 0, 6, 7),
373 	ARM64_SYS_REG(2, 0, 0, 7, 4),
374 	ARM64_SYS_REG(2, 0, 0, 7, 5),
375 	ARM64_SYS_REG(2, 0, 0, 7, 6),
376 	ARM64_SYS_REG(2, 0, 0, 7, 7),
377 	ARM64_SYS_REG(2, 0, 0, 8, 4),
378 	ARM64_SYS_REG(2, 0, 0, 8, 5),
379 	ARM64_SYS_REG(2, 0, 0, 8, 6),
380 	ARM64_SYS_REG(2, 0, 0, 8, 7),
381 	ARM64_SYS_REG(2, 0, 0, 9, 4),
382 	ARM64_SYS_REG(2, 0, 0, 9, 5),
383 	ARM64_SYS_REG(2, 0, 0, 9, 6),
384 	ARM64_SYS_REG(2, 0, 0, 9, 7),
385 	ARM64_SYS_REG(2, 0, 0, 10, 4),
386 	ARM64_SYS_REG(2, 0, 0, 10, 5),
387 	ARM64_SYS_REG(2, 0, 0, 10, 6),
388 	ARM64_SYS_REG(2, 0, 0, 10, 7),
389 	ARM64_SYS_REG(2, 0, 0, 11, 4),
390 	ARM64_SYS_REG(2, 0, 0, 11, 5),
391 	ARM64_SYS_REG(2, 0, 0, 11, 6),
392 	ARM64_SYS_REG(2, 0, 0, 11, 7),
393 	ARM64_SYS_REG(2, 0, 0, 12, 4),
394 	ARM64_SYS_REG(2, 0, 0, 12, 5),
395 	ARM64_SYS_REG(2, 0, 0, 12, 6),
396 	ARM64_SYS_REG(2, 0, 0, 12, 7),
397 	ARM64_SYS_REG(2, 0, 0, 13, 4),
398 	ARM64_SYS_REG(2, 0, 0, 13, 5),
399 	ARM64_SYS_REG(2, 0, 0, 13, 6),
400 	ARM64_SYS_REG(2, 0, 0, 13, 7),
401 	ARM64_SYS_REG(2, 0, 0, 14, 4),
402 	ARM64_SYS_REG(2, 0, 0, 14, 5),
403 	ARM64_SYS_REG(2, 0, 0, 14, 6),
404 	ARM64_SYS_REG(2, 0, 0, 14, 7),
405 	ARM64_SYS_REG(2, 0, 0, 15, 4),
406 	ARM64_SYS_REG(2, 0, 0, 15, 5),
407 	ARM64_SYS_REG(2, 0, 0, 15, 6),
408 	ARM64_SYS_REG(2, 0, 0, 15, 7),
409 	ARM64_SYS_REG(2, 0, 1, 1, 4),	/* OSLSR_EL1 */
410 	ARM64_SYS_REG(2, 4, 0, 7, 0),	/* DBGVCR32_EL2 */
411 	ARM64_SYS_REG(3, 0, 0, 0, 5),	/* MPIDR_EL1 */
412 	ARM64_SYS_REG(3, 0, 0, 1, 0),	/* ID_PFR0_EL1 */
413 	ARM64_SYS_REG(3, 0, 0, 1, 1),	/* ID_PFR1_EL1 */
414 	ARM64_SYS_REG(3, 0, 0, 1, 2),	/* ID_DFR0_EL1 */
415 	ARM64_SYS_REG(3, 0, 0, 1, 3),	/* ID_AFR0_EL1 */
416 	ARM64_SYS_REG(3, 0, 0, 1, 4),	/* ID_MMFR0_EL1 */
417 	ARM64_SYS_REG(3, 0, 0, 1, 5),	/* ID_MMFR1_EL1 */
418 	ARM64_SYS_REG(3, 0, 0, 1, 6),	/* ID_MMFR2_EL1 */
419 	ARM64_SYS_REG(3, 0, 0, 1, 7),	/* ID_MMFR3_EL1 */
420 	ARM64_SYS_REG(3, 0, 0, 2, 0),	/* ID_ISAR0_EL1 */
421 	ARM64_SYS_REG(3, 0, 0, 2, 1),	/* ID_ISAR1_EL1 */
422 	ARM64_SYS_REG(3, 0, 0, 2, 2),	/* ID_ISAR2_EL1 */
423 	ARM64_SYS_REG(3, 0, 0, 2, 3),	/* ID_ISAR3_EL1 */
424 	ARM64_SYS_REG(3, 0, 0, 2, 4),	/* ID_ISAR4_EL1 */
425 	ARM64_SYS_REG(3, 0, 0, 2, 5),	/* ID_ISAR5_EL1 */
426 	ARM64_SYS_REG(3, 0, 0, 2, 6),	/* ID_MMFR4_EL1 */
427 	ARM64_SYS_REG(3, 0, 0, 2, 7),	/* ID_ISAR6_EL1 */
428 	ARM64_SYS_REG(3, 0, 0, 3, 0),	/* MVFR0_EL1 */
429 	ARM64_SYS_REG(3, 0, 0, 3, 1),	/* MVFR1_EL1 */
430 	ARM64_SYS_REG(3, 0, 0, 3, 2),	/* MVFR2_EL1 */
431 	ARM64_SYS_REG(3, 0, 0, 3, 3),
432 	ARM64_SYS_REG(3, 0, 0, 3, 4),	/* ID_PFR2_EL1 */
433 	ARM64_SYS_REG(3, 0, 0, 3, 5),	/* ID_DFR1_EL1 */
434 	ARM64_SYS_REG(3, 0, 0, 3, 6),	/* ID_MMFR5_EL1 */
435 	ARM64_SYS_REG(3, 0, 0, 3, 7),
436 	ARM64_SYS_REG(3, 0, 0, 4, 0),	/* ID_AA64PFR0_EL1 */
437 	ARM64_SYS_REG(3, 0, 0, 4, 1),	/* ID_AA64PFR1_EL1 */
438 	ARM64_SYS_REG(3, 0, 0, 4, 2),	/* ID_AA64PFR2_EL1 */
439 	ARM64_SYS_REG(3, 0, 0, 4, 3),
440 	ARM64_SYS_REG(3, 0, 0, 4, 4),	/* ID_AA64ZFR0_EL1 */
441 	ARM64_SYS_REG(3, 0, 0, 4, 5),	/* ID_AA64SMFR0_EL1 */
442 	ARM64_SYS_REG(3, 0, 0, 4, 6),
443 	ARM64_SYS_REG(3, 0, 0, 4, 7),
444 	ARM64_SYS_REG(3, 0, 0, 5, 0),	/* ID_AA64DFR0_EL1 */
445 	ARM64_SYS_REG(3, 0, 0, 5, 1),	/* ID_AA64DFR1_EL1 */
446 	ARM64_SYS_REG(3, 0, 0, 5, 2),
447 	ARM64_SYS_REG(3, 0, 0, 5, 3),
448 	ARM64_SYS_REG(3, 0, 0, 5, 4),	/* ID_AA64AFR0_EL1 */
449 	ARM64_SYS_REG(3, 0, 0, 5, 5),	/* ID_AA64AFR1_EL1 */
450 	ARM64_SYS_REG(3, 0, 0, 5, 6),
451 	ARM64_SYS_REG(3, 0, 0, 5, 7),
452 	ARM64_SYS_REG(3, 0, 0, 6, 0),	/* ID_AA64ISAR0_EL1 */
453 	ARM64_SYS_REG(3, 0, 0, 6, 1),	/* ID_AA64ISAR1_EL1 */
454 	ARM64_SYS_REG(3, 0, 0, 6, 2),	/* ID_AA64ISAR2_EL1 */
455 	ARM64_SYS_REG(3, 0, 0, 6, 3),
456 	ARM64_SYS_REG(3, 0, 0, 6, 4),
457 	ARM64_SYS_REG(3, 0, 0, 6, 5),
458 	ARM64_SYS_REG(3, 0, 0, 6, 6),
459 	ARM64_SYS_REG(3, 0, 0, 6, 7),
460 	ARM64_SYS_REG(3, 0, 0, 7, 0),	/* ID_AA64MMFR0_EL1 */
461 	ARM64_SYS_REG(3, 0, 0, 7, 1),	/* ID_AA64MMFR1_EL1 */
462 	ARM64_SYS_REG(3, 0, 0, 7, 2),	/* ID_AA64MMFR2_EL1 */
463 	ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
464 	ARM64_SYS_REG(3, 0, 0, 7, 4),	/* ID_AA64MMFR4_EL1 */
465 	ARM64_SYS_REG(3, 0, 0, 7, 5),
466 	ARM64_SYS_REG(3, 0, 0, 7, 6),
467 	ARM64_SYS_REG(3, 0, 0, 7, 7),
468 	ARM64_SYS_REG(3, 0, 1, 0, 0),	/* SCTLR_EL1 */
469 	ARM64_SYS_REG(3, 0, 1, 0, 1),	/* ACTLR_EL1 */
470 	ARM64_SYS_REG(3, 0, 1, 0, 2),	/* CPACR_EL1 */
471 	ARM64_SYS_REG(3, 0, 2, 0, 0),	/* TTBR0_EL1 */
472 	ARM64_SYS_REG(3, 0, 2, 0, 1),	/* TTBR1_EL1 */
473 	ARM64_SYS_REG(3, 0, 2, 0, 2),	/* TCR_EL1 */
474 	ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
475 	ARM64_SYS_REG(3, 0, 5, 1, 0),	/* AFSR0_EL1 */
476 	ARM64_SYS_REG(3, 0, 5, 1, 1),	/* AFSR1_EL1 */
477 	ARM64_SYS_REG(3, 0, 5, 2, 0),	/* ESR_EL1 */
478 	ARM64_SYS_REG(3, 0, 6, 0, 0),	/* FAR_EL1 */
479 	ARM64_SYS_REG(3, 0, 7, 4, 0),	/* PAR_EL1 */
480 	ARM64_SYS_REG(3, 0, 10, 2, 0),	/* MAIR_EL1 */
481 	ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
482 	ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
483 	ARM64_SYS_REG(3, 0, 10, 2, 4),	/* POR_EL1 */
484 	ARM64_SYS_REG(3, 0, 10, 3, 0),	/* AMAIR_EL1 */
485 	ARM64_SYS_REG(3, 0, 12, 0, 0),	/* VBAR_EL1 */
486 	ARM64_SYS_REG(3, 0, 12, 1, 1),	/* DISR_EL1 */
487 	ARM64_SYS_REG(3, 0, 13, 0, 1),	/* CONTEXTIDR_EL1 */
488 	ARM64_SYS_REG(3, 0, 13, 0, 4),	/* TPIDR_EL1 */
489 	ARM64_SYS_REG(3, 0, 14, 1, 0),	/* CNTKCTL_EL1 */
490 	ARM64_SYS_REG(3, 2, 0, 0, 0),	/* CSSELR_EL1 */
491 	ARM64_SYS_REG(3, 3, 10, 2, 4),	/* POR_EL0 */
492 	ARM64_SYS_REG(3, 3, 13, 0, 2),	/* TPIDR_EL0 */
493 	ARM64_SYS_REG(3, 3, 13, 0, 3),	/* TPIDRRO_EL0 */
494 	ARM64_SYS_REG(3, 3, 14, 0, 1),	/* CNTPCT_EL0 */
495 	ARM64_SYS_REG(3, 3, 14, 2, 1),	/* CNTP_CTL_EL0 */
496 	ARM64_SYS_REG(3, 3, 14, 2, 2),	/* CNTP_CVAL_EL0 */
497 	ARM64_SYS_REG(3, 4, 3, 0, 0),	/* DACR32_EL2 */
498 	ARM64_SYS_REG(3, 4, 5, 0, 1),	/* IFSR32_EL2 */
499 	ARM64_SYS_REG(3, 4, 5, 3, 0),	/* FPEXC32_EL2 */
500 };
501 
502 static __u64 pmu_regs[] = {
503 	ARM64_SYS_REG(3, 0, 9, 14, 1),	/* PMINTENSET_EL1 */
504 	ARM64_SYS_REG(3, 0, 9, 14, 2),	/* PMINTENCLR_EL1 */
505 	ARM64_SYS_REG(3, 3, 9, 12, 0),	/* PMCR_EL0 */
506 	ARM64_SYS_REG(3, 3, 9, 12, 1),	/* PMCNTENSET_EL0 */
507 	ARM64_SYS_REG(3, 3, 9, 12, 2),	/* PMCNTENCLR_EL0 */
508 	ARM64_SYS_REG(3, 3, 9, 12, 3),	/* PMOVSCLR_EL0 */
509 	ARM64_SYS_REG(3, 3, 9, 12, 4),	/* PMSWINC_EL0 */
510 	ARM64_SYS_REG(3, 3, 9, 12, 5),	/* PMSELR_EL0 */
511 	ARM64_SYS_REG(3, 3, 9, 13, 0),	/* PMCCNTR_EL0 */
512 	ARM64_SYS_REG(3, 3, 9, 14, 0),	/* PMUSERENR_EL0 */
513 	ARM64_SYS_REG(3, 3, 9, 14, 3),	/* PMOVSSET_EL0 */
514 	ARM64_SYS_REG(3, 3, 14, 8, 0),
515 	ARM64_SYS_REG(3, 3, 14, 8, 1),
516 	ARM64_SYS_REG(3, 3, 14, 8, 2),
517 	ARM64_SYS_REG(3, 3, 14, 8, 3),
518 	ARM64_SYS_REG(3, 3, 14, 8, 4),
519 	ARM64_SYS_REG(3, 3, 14, 8, 5),
520 	ARM64_SYS_REG(3, 3, 14, 8, 6),
521 	ARM64_SYS_REG(3, 3, 14, 8, 7),
522 	ARM64_SYS_REG(3, 3, 14, 9, 0),
523 	ARM64_SYS_REG(3, 3, 14, 9, 1),
524 	ARM64_SYS_REG(3, 3, 14, 9, 2),
525 	ARM64_SYS_REG(3, 3, 14, 9, 3),
526 	ARM64_SYS_REG(3, 3, 14, 9, 4),
527 	ARM64_SYS_REG(3, 3, 14, 9, 5),
528 	ARM64_SYS_REG(3, 3, 14, 9, 6),
529 	ARM64_SYS_REG(3, 3, 14, 9, 7),
530 	ARM64_SYS_REG(3, 3, 14, 10, 0),
531 	ARM64_SYS_REG(3, 3, 14, 10, 1),
532 	ARM64_SYS_REG(3, 3, 14, 10, 2),
533 	ARM64_SYS_REG(3, 3, 14, 10, 3),
534 	ARM64_SYS_REG(3, 3, 14, 10, 4),
535 	ARM64_SYS_REG(3, 3, 14, 10, 5),
536 	ARM64_SYS_REG(3, 3, 14, 10, 6),
537 	ARM64_SYS_REG(3, 3, 14, 10, 7),
538 	ARM64_SYS_REG(3, 3, 14, 11, 0),
539 	ARM64_SYS_REG(3, 3, 14, 11, 1),
540 	ARM64_SYS_REG(3, 3, 14, 11, 2),
541 	ARM64_SYS_REG(3, 3, 14, 11, 3),
542 	ARM64_SYS_REG(3, 3, 14, 11, 4),
543 	ARM64_SYS_REG(3, 3, 14, 11, 5),
544 	ARM64_SYS_REG(3, 3, 14, 11, 6),
545 	ARM64_SYS_REG(3, 3, 14, 12, 0),
546 	ARM64_SYS_REG(3, 3, 14, 12, 1),
547 	ARM64_SYS_REG(3, 3, 14, 12, 2),
548 	ARM64_SYS_REG(3, 3, 14, 12, 3),
549 	ARM64_SYS_REG(3, 3, 14, 12, 4),
550 	ARM64_SYS_REG(3, 3, 14, 12, 5),
551 	ARM64_SYS_REG(3, 3, 14, 12, 6),
552 	ARM64_SYS_REG(3, 3, 14, 12, 7),
553 	ARM64_SYS_REG(3, 3, 14, 13, 0),
554 	ARM64_SYS_REG(3, 3, 14, 13, 1),
555 	ARM64_SYS_REG(3, 3, 14, 13, 2),
556 	ARM64_SYS_REG(3, 3, 14, 13, 3),
557 	ARM64_SYS_REG(3, 3, 14, 13, 4),
558 	ARM64_SYS_REG(3, 3, 14, 13, 5),
559 	ARM64_SYS_REG(3, 3, 14, 13, 6),
560 	ARM64_SYS_REG(3, 3, 14, 13, 7),
561 	ARM64_SYS_REG(3, 3, 14, 14, 0),
562 	ARM64_SYS_REG(3, 3, 14, 14, 1),
563 	ARM64_SYS_REG(3, 3, 14, 14, 2),
564 	ARM64_SYS_REG(3, 3, 14, 14, 3),
565 	ARM64_SYS_REG(3, 3, 14, 14, 4),
566 	ARM64_SYS_REG(3, 3, 14, 14, 5),
567 	ARM64_SYS_REG(3, 3, 14, 14, 6),
568 	ARM64_SYS_REG(3, 3, 14, 14, 7),
569 	ARM64_SYS_REG(3, 3, 14, 15, 0),
570 	ARM64_SYS_REG(3, 3, 14, 15, 1),
571 	ARM64_SYS_REG(3, 3, 14, 15, 2),
572 	ARM64_SYS_REG(3, 3, 14, 15, 3),
573 	ARM64_SYS_REG(3, 3, 14, 15, 4),
574 	ARM64_SYS_REG(3, 3, 14, 15, 5),
575 	ARM64_SYS_REG(3, 3, 14, 15, 6),
576 	ARM64_SYS_REG(3, 3, 14, 15, 7),	/* PMCCFILTR_EL0 */
577 };
578 
579 static __u64 vregs[] = {
580 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]),
581 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[1]),
582 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[2]),
583 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[3]),
584 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[4]),
585 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[5]),
586 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[6]),
587 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[7]),
588 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[8]),
589 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[9]),
590 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[10]),
591 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[11]),
592 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[12]),
593 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[13]),
594 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[14]),
595 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[15]),
596 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[16]),
597 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[17]),
598 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[18]),
599 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[19]),
600 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[20]),
601 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[21]),
602 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[22]),
603 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[23]),
604 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[24]),
605 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[25]),
606 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[26]),
607 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[27]),
608 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[28]),
609 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[29]),
610 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[30]),
611 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]),
612 };
613 
614 static __u64 sve_regs[] = {
615 	KVM_REG_ARM64_SVE_VLS,
616 	KVM_REG_ARM64_SVE_ZREG(0, 0),
617 	KVM_REG_ARM64_SVE_ZREG(1, 0),
618 	KVM_REG_ARM64_SVE_ZREG(2, 0),
619 	KVM_REG_ARM64_SVE_ZREG(3, 0),
620 	KVM_REG_ARM64_SVE_ZREG(4, 0),
621 	KVM_REG_ARM64_SVE_ZREG(5, 0),
622 	KVM_REG_ARM64_SVE_ZREG(6, 0),
623 	KVM_REG_ARM64_SVE_ZREG(7, 0),
624 	KVM_REG_ARM64_SVE_ZREG(8, 0),
625 	KVM_REG_ARM64_SVE_ZREG(9, 0),
626 	KVM_REG_ARM64_SVE_ZREG(10, 0),
627 	KVM_REG_ARM64_SVE_ZREG(11, 0),
628 	KVM_REG_ARM64_SVE_ZREG(12, 0),
629 	KVM_REG_ARM64_SVE_ZREG(13, 0),
630 	KVM_REG_ARM64_SVE_ZREG(14, 0),
631 	KVM_REG_ARM64_SVE_ZREG(15, 0),
632 	KVM_REG_ARM64_SVE_ZREG(16, 0),
633 	KVM_REG_ARM64_SVE_ZREG(17, 0),
634 	KVM_REG_ARM64_SVE_ZREG(18, 0),
635 	KVM_REG_ARM64_SVE_ZREG(19, 0),
636 	KVM_REG_ARM64_SVE_ZREG(20, 0),
637 	KVM_REG_ARM64_SVE_ZREG(21, 0),
638 	KVM_REG_ARM64_SVE_ZREG(22, 0),
639 	KVM_REG_ARM64_SVE_ZREG(23, 0),
640 	KVM_REG_ARM64_SVE_ZREG(24, 0),
641 	KVM_REG_ARM64_SVE_ZREG(25, 0),
642 	KVM_REG_ARM64_SVE_ZREG(26, 0),
643 	KVM_REG_ARM64_SVE_ZREG(27, 0),
644 	KVM_REG_ARM64_SVE_ZREG(28, 0),
645 	KVM_REG_ARM64_SVE_ZREG(29, 0),
646 	KVM_REG_ARM64_SVE_ZREG(30, 0),
647 	KVM_REG_ARM64_SVE_ZREG(31, 0),
648 	KVM_REG_ARM64_SVE_PREG(0, 0),
649 	KVM_REG_ARM64_SVE_PREG(1, 0),
650 	KVM_REG_ARM64_SVE_PREG(2, 0),
651 	KVM_REG_ARM64_SVE_PREG(3, 0),
652 	KVM_REG_ARM64_SVE_PREG(4, 0),
653 	KVM_REG_ARM64_SVE_PREG(5, 0),
654 	KVM_REG_ARM64_SVE_PREG(6, 0),
655 	KVM_REG_ARM64_SVE_PREG(7, 0),
656 	KVM_REG_ARM64_SVE_PREG(8, 0),
657 	KVM_REG_ARM64_SVE_PREG(9, 0),
658 	KVM_REG_ARM64_SVE_PREG(10, 0),
659 	KVM_REG_ARM64_SVE_PREG(11, 0),
660 	KVM_REG_ARM64_SVE_PREG(12, 0),
661 	KVM_REG_ARM64_SVE_PREG(13, 0),
662 	KVM_REG_ARM64_SVE_PREG(14, 0),
663 	KVM_REG_ARM64_SVE_PREG(15, 0),
664 	KVM_REG_ARM64_SVE_FFR(0),
665 	ARM64_SYS_REG(3, 0, 1, 2, 0),   /* ZCR_EL1 */
666 };
667 
668 static __u64 sve_rejects_set[] = {
669 	KVM_REG_ARM64_SVE_VLS,
670 };
671 
672 static __u64 pauth_addr_regs[] = {
673 	ARM64_SYS_REG(3, 0, 2, 1, 0),	/* APIAKEYLO_EL1 */
674 	ARM64_SYS_REG(3, 0, 2, 1, 1),	/* APIAKEYHI_EL1 */
675 	ARM64_SYS_REG(3, 0, 2, 1, 2),	/* APIBKEYLO_EL1 */
676 	ARM64_SYS_REG(3, 0, 2, 1, 3),	/* APIBKEYHI_EL1 */
677 	ARM64_SYS_REG(3, 0, 2, 2, 0),	/* APDAKEYLO_EL1 */
678 	ARM64_SYS_REG(3, 0, 2, 2, 1),	/* APDAKEYHI_EL1 */
679 	ARM64_SYS_REG(3, 0, 2, 2, 2),	/* APDBKEYLO_EL1 */
680 	ARM64_SYS_REG(3, 0, 2, 2, 3)	/* APDBKEYHI_EL1 */
681 };
682 
683 static __u64 pauth_generic_regs[] = {
684 	ARM64_SYS_REG(3, 0, 2, 3, 0),	/* APGAKEYLO_EL1 */
685 	ARM64_SYS_REG(3, 0, 2, 3, 1),	/* APGAKEYHI_EL1 */
686 };
687 
688 #define BASE_SUBLIST \
689 	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
690 #define VREGS_SUBLIST \
691 	{ "vregs", .regs = vregs, .regs_n = ARRAY_SIZE(vregs), }
692 #define PMU_SUBLIST \
693 	{ "pmu", .capability = KVM_CAP_ARM_PMU_V3, .feature = KVM_ARM_VCPU_PMU_V3, \
694 	  .regs = pmu_regs, .regs_n = ARRAY_SIZE(pmu_regs), }
695 #define SVE_SUBLIST \
696 	{ "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \
697 	  .regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \
698 	  .rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }
699 #define PAUTH_SUBLIST							\
700 	{								\
701 		.name 		= "pauth_address",			\
702 		.capability	= KVM_CAP_ARM_PTRAUTH_ADDRESS,		\
703 		.feature	= KVM_ARM_VCPU_PTRAUTH_ADDRESS,		\
704 		.regs		= pauth_addr_regs,			\
705 		.regs_n		= ARRAY_SIZE(pauth_addr_regs),		\
706 	},								\
707 	{								\
708 		.name 		= "pauth_generic",			\
709 		.capability	= KVM_CAP_ARM_PTRAUTH_GENERIC,		\
710 		.feature	= KVM_ARM_VCPU_PTRAUTH_GENERIC,		\
711 		.regs		= pauth_generic_regs,			\
712 		.regs_n		= ARRAY_SIZE(pauth_generic_regs),	\
713 	}
714 
715 static struct vcpu_reg_list vregs_config = {
716 	.sublists = {
717 	BASE_SUBLIST,
718 	VREGS_SUBLIST,
719 	{0},
720 	},
721 };
722 static struct vcpu_reg_list vregs_pmu_config = {
723 	.sublists = {
724 	BASE_SUBLIST,
725 	VREGS_SUBLIST,
726 	PMU_SUBLIST,
727 	{0},
728 	},
729 };
730 static struct vcpu_reg_list sve_config = {
731 	.sublists = {
732 	BASE_SUBLIST,
733 	SVE_SUBLIST,
734 	{0},
735 	},
736 };
737 static struct vcpu_reg_list sve_pmu_config = {
738 	.sublists = {
739 	BASE_SUBLIST,
740 	SVE_SUBLIST,
741 	PMU_SUBLIST,
742 	{0},
743 	},
744 };
745 static struct vcpu_reg_list pauth_config = {
746 	.sublists = {
747 	BASE_SUBLIST,
748 	VREGS_SUBLIST,
749 	PAUTH_SUBLIST,
750 	{0},
751 	},
752 };
753 static struct vcpu_reg_list pauth_pmu_config = {
754 	.sublists = {
755 	BASE_SUBLIST,
756 	VREGS_SUBLIST,
757 	PAUTH_SUBLIST,
758 	PMU_SUBLIST,
759 	{0},
760 	},
761 };
762 
763 struct vcpu_reg_list *vcpu_configs[] = {
764 	&vregs_config,
765 	&vregs_pmu_config,
766 	&sve_config,
767 	&sve_pmu_config,
768 	&pauth_config,
769 	&pauth_pmu_config,
770 };
771 int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
772