1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * Copyright (C) 2025 Google LLC
4
 * Author: Marc Zyngier <[email protected]>
5
 */
6

7
#include <linux/kvm_host.h>
8
#include <asm/sysreg.h>
9

10
/*
11
 * Describes the dependencies between a set of bits (or the negation
12
 * of a set of RES0 bits) and a feature. The flags indicate how the
13
 * data is interpreted.
14
 */
15
struct reg_bits_to_feat_map {
16
	union {
17
		u64	bits;
18
		u64	*res0p;
19
	};
20

21
#define	NEVER_FGU	BIT(0)	/* Can trap, but never UNDEF */
22
#define	CALL_FUNC	BIT(1)	/* Needs to evaluate tons of crap */
23
#define	FIXED_VALUE	BIT(2)	/* RAZ/WI or RAO/WI in KVM */
24
#define	RES0_POINTER	BIT(3)	/* Pointer to RES0 value instead of bits */
25

26
	unsigned long	flags;
27

28
	union {
29
		struct {
30
			u8	regidx;
31
			u8	shift;
32
			u8	width;
33
			bool	sign;
34
			s8	lo_lim;
35
		};
36
		bool	(*match)(struct kvm *);
37
		bool	(*fval)(struct kvm *, u64 *);
38
	};
39
};
40

41
/*
42
 * Describes the dependencies for a given register:
43
 *
44
 * @feat_map describes the dependency for the whole register. If the
45
 * features the register depends on are not present, the whole
46
 * register is effectively RES0.
47
 *
48
 * @bit_feat_map describes the dependencies for a set of bits in that
49
 * register. If the features these bits depend on are not present, the
50
 * bits are effectively RES0.
51
 */
52
struct reg_feat_map_desc {
53
	const char			  *name;
54
	const struct reg_bits_to_feat_map feat_map;
55
	const struct reg_bits_to_feat_map *bit_feat_map;
56
	const unsigned int		  bit_feat_map_sz;
57
};
58

59
#define __NEEDS_FEAT_3(m, f, w, id, fld, lim)		\
60
	{						\
61
		.w	= (m),				\
62
		.flags = (f),				\
63
		.regidx	= IDREG_IDX(SYS_ ## id),	\
64
		.shift	= id ##_## fld ## _SHIFT,	\
65
		.width	= id ##_## fld ## _WIDTH,	\
66
		.sign	= id ##_## fld ## _SIGNED,	\
67
		.lo_lim	= id ##_## fld ##_## lim	\
68
	}
69

70
#define __NEEDS_FEAT_2(m, f, w, fun, dummy)		\
71
	{						\
72
		.w	= (m),				\
73
		.flags = (f) | CALL_FUNC,		\
74
		.fval = (fun),				\
75
	}
76

77
#define __NEEDS_FEAT_1(m, f, w, fun)			\
78
	{						\
79
		.w	= (m),				\
80
		.flags = (f) | CALL_FUNC,		\
81
		.match = (fun),				\
82
	}
83

84
#define __NEEDS_FEAT_FLAG(m, f, w, ...)			\
85
	CONCATENATE(__NEEDS_FEAT_, COUNT_ARGS(__VA_ARGS__))(m, f, w, __VA_ARGS__)
86

87
#define NEEDS_FEAT_FLAG(m, f, ...)			\
88
	__NEEDS_FEAT_FLAG(m, f, bits, __VA_ARGS__)
89

90
#define NEEDS_FEAT_FIXED(m, ...)			\
91
	__NEEDS_FEAT_FLAG(m, FIXED_VALUE, bits, __VA_ARGS__, 0)
92

93
#define NEEDS_FEAT_RES0(p, ...)				\
94
	__NEEDS_FEAT_FLAG(p, RES0_POINTER, res0p, __VA_ARGS__)
95

96
/*
97
 * Declare the dependency between a set of bits and a set of features,
98
 * generating a struct reg_bit_to_feat_map.
99
 */
100
#define NEEDS_FEAT(m, ...)	NEEDS_FEAT_FLAG(m, 0, __VA_ARGS__)
101

102
/*
103
 * Declare the dependency between a non-FGT register, a set of
104
 * feature, and the set of individual bits it contains. This generates
105
 * a struct reg_feat_map_desc.
106
 */
107
#define DECLARE_FEAT_MAP(n, r, m, f)					\
108
	struct reg_feat_map_desc n = {					\
109
		.name			= #r,				\
110
		.feat_map		= NEEDS_FEAT(~r##_RES0, f), 	\
111
		.bit_feat_map		= m,				\
112
		.bit_feat_map_sz	= ARRAY_SIZE(m),		\
113
	}
114

115
/*
116
 * Specialised version of the above for FGT registers that have their
117
 * RES0 masks described as struct fgt_masks.
118
 */
119
#define DECLARE_FEAT_MAP_FGT(n, msk, m, f)				\
120
	struct reg_feat_map_desc n = {					\
121
		.name			= #msk,				\
122
		.feat_map		= NEEDS_FEAT_RES0(&msk.res0, f),\
123
		.bit_feat_map		= m,				\
124
		.bit_feat_map_sz	= ARRAY_SIZE(m),		\
125
	}
126

127
#define FEAT_SPE		ID_AA64DFR0_EL1, PMSVer, IMP
128
#define FEAT_SPE_FnE		ID_AA64DFR0_EL1, PMSVer, V1P2
129
#define FEAT_BRBE		ID_AA64DFR0_EL1, BRBE, IMP
130
#define FEAT_TRC_SR		ID_AA64DFR0_EL1, TraceVer, IMP
131
#define FEAT_PMUv3		ID_AA64DFR0_EL1, PMUVer, IMP
132
#define FEAT_TRBE		ID_AA64DFR0_EL1, TraceBuffer, IMP
133
#define FEAT_TRBEv1p1		ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1
134
#define FEAT_DoubleLock		ID_AA64DFR0_EL1, DoubleLock, IMP
135
#define FEAT_TRF		ID_AA64DFR0_EL1, TraceFilt, IMP
136
#define FEAT_AA32EL0		ID_AA64PFR0_EL1, EL0, AARCH32
137
#define FEAT_AA32EL1		ID_AA64PFR0_EL1, EL1, AARCH32
138
#define FEAT_AA64EL1		ID_AA64PFR0_EL1, EL1, IMP
139
#define FEAT_AA64EL2		ID_AA64PFR0_EL1, EL2, IMP
140
#define FEAT_AA64EL3		ID_AA64PFR0_EL1, EL3, IMP
141
#define FEAT_AIE		ID_AA64MMFR3_EL1, AIE, IMP
142
#define FEAT_S2POE		ID_AA64MMFR3_EL1, S2POE, IMP
143
#define FEAT_S1POE		ID_AA64MMFR3_EL1, S1POE, IMP
144
#define FEAT_S1PIE		ID_AA64MMFR3_EL1, S1PIE, IMP
145
#define FEAT_THE		ID_AA64PFR1_EL1, THE, IMP
146
#define FEAT_SME		ID_AA64PFR1_EL1, SME, IMP
147
#define FEAT_GCS		ID_AA64PFR1_EL1, GCS, IMP
148
#define FEAT_LS64		ID_AA64ISAR1_EL1, LS64, LS64
149
#define FEAT_LS64_V		ID_AA64ISAR1_EL1, LS64, LS64_V
150
#define FEAT_LS64_ACCDATA	ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA
151
#define FEAT_RAS		ID_AA64PFR0_EL1, RAS, IMP
152
#define FEAT_RASv2		ID_AA64PFR0_EL1, RAS, V2
153
#define FEAT_GICv3		ID_AA64PFR0_EL1, GIC, IMP
154
#define FEAT_LOR		ID_AA64MMFR1_EL1, LO, IMP
155
#define FEAT_SPEv1p2		ID_AA64DFR0_EL1, PMSVer, V1P2
156
#define FEAT_SPEv1p4		ID_AA64DFR0_EL1, PMSVer, V1P4
157
#define FEAT_SPEv1p5		ID_AA64DFR0_EL1, PMSVer, V1P5
158
#define FEAT_ATS1A		ID_AA64ISAR2_EL1, ATS1A, IMP
159
#define FEAT_SPECRES2		ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX
160
#define FEAT_SPECRES		ID_AA64ISAR1_EL1, SPECRES, IMP
161
#define FEAT_TLBIRANGE		ID_AA64ISAR0_EL1, TLB, RANGE
162
#define FEAT_TLBIOS		ID_AA64ISAR0_EL1, TLB, OS
163
#define FEAT_PAN2		ID_AA64MMFR1_EL1, PAN, PAN2
164
#define FEAT_DPB2		ID_AA64ISAR1_EL1, DPB, DPB2
165
#define FEAT_AMUv1		ID_AA64PFR0_EL1, AMU, IMP
166
#define FEAT_AMUv1p1		ID_AA64PFR0_EL1, AMU, V1P1
167
#define FEAT_CMOW		ID_AA64MMFR1_EL1, CMOW, IMP
168
#define FEAT_D128		ID_AA64MMFR3_EL1, D128, IMP
169
#define FEAT_DoubleFault2	ID_AA64PFR1_EL1, DF2, IMP
170
#define FEAT_FPMR		ID_AA64PFR2_EL1, FPMR, IMP
171
#define FEAT_MOPS		ID_AA64ISAR2_EL1, MOPS, IMP
172
#define FEAT_NMI		ID_AA64PFR1_EL1, NMI, IMP
173
#define FEAT_SCTLR2		ID_AA64MMFR3_EL1, SCTLRX, IMP
174
#define FEAT_SYSREG128		ID_AA64ISAR2_EL1, SYSREG_128, IMP
175
#define FEAT_TCR2		ID_AA64MMFR3_EL1, TCRX, IMP
176
#define FEAT_XS			ID_AA64ISAR1_EL1, XS, IMP
177
#define FEAT_EVT		ID_AA64MMFR2_EL1, EVT, IMP
178
#define FEAT_EVT_TTLBxS		ID_AA64MMFR2_EL1, EVT, TTLBxS
179
#define FEAT_MTE2		ID_AA64PFR1_EL1, MTE, MTE2
180
#define FEAT_RME		ID_AA64PFR0_EL1, RME, IMP
181
#define FEAT_MPAM		ID_AA64PFR0_EL1, MPAM, 1
182
#define FEAT_S2FWB		ID_AA64MMFR2_EL1, FWB, IMP
183
#define FEAT_TME		ID_AA64ISAR0_EL1, TME, IMP
184
#define FEAT_TWED		ID_AA64MMFR1_EL1, TWED, IMP
185
#define FEAT_E2H0		ID_AA64MMFR4_EL1, E2H0, IMP
186
#define FEAT_SRMASK		ID_AA64MMFR4_EL1, SRMASK, IMP
187
#define FEAT_PoPS		ID_AA64MMFR4_EL1, PoPS, IMP
188
#define FEAT_PFAR		ID_AA64PFR1_EL1, PFAR, IMP
189
#define FEAT_Debugv8p9		ID_AA64DFR0_EL1, PMUVer, V3P9
190
#define FEAT_PMUv3_SS		ID_AA64DFR0_EL1, PMSS, IMP
191
#define FEAT_SEBEP		ID_AA64DFR0_EL1, SEBEP, IMP
192
#define FEAT_EBEP		ID_AA64DFR1_EL1, EBEP, IMP
193
#define FEAT_ITE		ID_AA64DFR1_EL1, ITE, IMP
194
#define FEAT_PMUv3_ICNTR	ID_AA64DFR1_EL1, PMICNTR, IMP
195
#define FEAT_SPMU		ID_AA64DFR1_EL1, SPMU, IMP
196
#define FEAT_SPE_nVM		ID_AA64DFR2_EL1, SPE_nVM, IMP
197
#define FEAT_STEP2		ID_AA64DFR2_EL1, STEP, IMP
198
#define FEAT_CPA2		ID_AA64ISAR3_EL1, CPA, CPA2
199
#define FEAT_ASID2		ID_AA64MMFR4_EL1, ASID2, IMP
200
#define FEAT_MEC		ID_AA64MMFR3_EL1, MEC, IMP
201
#define FEAT_HAFT		ID_AA64MMFR1_EL1, HAFDBS, HAFT
202
#define FEAT_BTI		ID_AA64PFR1_EL1, BT, IMP
203
#define FEAT_ExS		ID_AA64MMFR0_EL1, EXS, IMP
204
#define FEAT_IESB		ID_AA64MMFR2_EL1, IESB, IMP
205
#define FEAT_LSE2		ID_AA64MMFR2_EL1, AT, IMP
206
#define FEAT_LSMAOC		ID_AA64MMFR2_EL1, LSM, IMP
207
#define FEAT_MixedEnd		ID_AA64MMFR0_EL1, BIGEND, IMP
208
#define FEAT_MixedEndEL0	ID_AA64MMFR0_EL1, BIGENDEL0, IMP
209
#define FEAT_MTE_ASYNC		ID_AA64PFR1_EL1, MTE_frac, ASYNC
210
#define FEAT_MTE_STORE_ONLY	ID_AA64PFR2_EL1, MTESTOREONLY, IMP
211
#define FEAT_PAN		ID_AA64MMFR1_EL1, PAN, IMP
212
#define FEAT_PAN3		ID_AA64MMFR1_EL1, PAN, PAN3
213
#define FEAT_SSBS		ID_AA64PFR1_EL1, SSBS, IMP
214
#define FEAT_TIDCP1		ID_AA64MMFR1_EL1, TIDCP1, IMP
215
#define FEAT_FGT		ID_AA64MMFR0_EL1, FGT, IMP
216
#define FEAT_FGT2		ID_AA64MMFR0_EL1, FGT, FGT2
217
#define FEAT_MTPMU		ID_AA64DFR0_EL1, MTPMU, IMP
218
#define FEAT_HCX		ID_AA64MMFR1_EL1, HCX, IMP
219

220
static bool not_feat_aa64el3(struct kvm *kvm)
221
{
222
	return !kvm_has_feat(kvm, FEAT_AA64EL3);
223
}
224

225
static bool feat_nv2(struct kvm *kvm)
226
{
227
	return ((kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY) &&
228
		 kvm_has_feat_enum(kvm, ID_AA64MMFR2_EL1, NV, NI)) ||
229
		kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2));
230
}
231

232
static bool feat_nv2_e2h0_ni(struct kvm *kvm)
233
{
234
	return feat_nv2(kvm) && !kvm_has_feat(kvm, FEAT_E2H0);
235
}
236

237
static bool feat_rasv1p1(struct kvm *kvm)
238
{
239
	return (kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1) ||
240
		(kvm_has_feat_enum(kvm, ID_AA64PFR0_EL1, RAS, IMP) &&
241
		 kvm_has_feat(kvm, ID_AA64PFR1_EL1, RAS_frac, RASv1p1)));
242
}
243

244
static bool feat_csv2_2_csv2_1p2(struct kvm *kvm)
245
{
246
	return (kvm_has_feat(kvm,  ID_AA64PFR0_EL1, CSV2, CSV2_2) ||
247
		(kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2) &&
248
		 kvm_has_feat_enum(kvm,  ID_AA64PFR0_EL1, CSV2, IMP)));
249
}
250

251
static bool feat_pauth(struct kvm *kvm)
252
{
253
	return kvm_has_pauth(kvm, PAuth);
254
}
255

256
static bool feat_pauth_lr(struct kvm *kvm)
257
{
258
	return kvm_has_pauth(kvm, PAuth_LR);
259
}
260

261
static bool feat_aderr(struct kvm *kvm)
262
{
263
	return (kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, FEAT_ADERR) &&
264
		kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SDERR, FEAT_ADERR));
265
}
266

267
static bool feat_anerr(struct kvm *kvm)
268
{
269
	return (kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ANERR, FEAT_ANERR) &&
270
		kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SNERR, FEAT_ANERR));
271
}
272

273
static bool feat_sme_smps(struct kvm *kvm)
274
{
275
	/*
276
	 * Revists this if KVM ever supports SME -- this really should
277
	 * look at the guest's view of SMIDR_EL1. Funnily enough, this
278
	 * is not captured in the JSON file, but only as a note in the
279
	 * ARM ARM.
280
	 */
281
	return (kvm_has_feat(kvm, FEAT_SME) &&
282
		(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS));
283
}
284

285
static bool feat_spe_fds(struct kvm *kvm)
286
{
287
	/*
288
	 * Revists this if KVM ever supports SPE -- this really should
289
	 * look at the guest's view of PMSIDR_EL1.
290
	 */
291
	return (kvm_has_feat(kvm, FEAT_SPEv1p4) &&
292
		(read_sysreg_s(SYS_PMSIDR_EL1) & PMSIDR_EL1_FDS));
293
}
294

295
static bool feat_trbe_mpam(struct kvm *kvm)
296
{
297
	/*
298
	 * Revists this if KVM ever supports both MPAM and TRBE --
299
	 * this really should look at the guest's view of TRBIDR_EL1.
300
	 */
301
	return (kvm_has_feat(kvm, FEAT_TRBE) &&
302
		kvm_has_feat(kvm, FEAT_MPAM) &&
303
		(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_EL1_MPAM));
304
}
305

306
static bool feat_asid2_e2h1(struct kvm *kvm)
307
{
308
	return kvm_has_feat(kvm, FEAT_ASID2) && !kvm_has_feat(kvm, FEAT_E2H0);
309
}
310

311
static bool feat_d128_e2h1(struct kvm *kvm)
312
{
313
	return kvm_has_feat(kvm, FEAT_D128) && !kvm_has_feat(kvm, FEAT_E2H0);
314
}
315

316
static bool feat_mec_e2h1(struct kvm *kvm)
317
{
318
	return kvm_has_feat(kvm, FEAT_MEC) && !kvm_has_feat(kvm, FEAT_E2H0);
319
}
320

321
static bool feat_ebep_pmuv3_ss(struct kvm *kvm)
322
{
323
	return kvm_has_feat(kvm, FEAT_EBEP) || kvm_has_feat(kvm, FEAT_PMUv3_SS);
324
}
325

326
static bool feat_mixedendel0(struct kvm *kvm)
327
{
328
	return kvm_has_feat(kvm, FEAT_MixedEnd) || kvm_has_feat(kvm, FEAT_MixedEndEL0);
329
}
330

331
static bool feat_mte_async(struct kvm *kvm)
332
{
333
	return kvm_has_feat(kvm, FEAT_MTE2) && kvm_has_feat_enum(kvm, FEAT_MTE_ASYNC);
334
}
335

336
#define check_pmu_revision(k, r)					\
337
	({								\
338
		(kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, r) &&	\
339
		 !kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, IMP_DEF));	\
340
	})
341

342
static bool feat_pmuv3p1(struct kvm *kvm)
343
{
344
	return check_pmu_revision(kvm, V3P1);
345
}
346

347
static bool feat_pmuv3p5(struct kvm *kvm)
348
{
349
	return check_pmu_revision(kvm, V3P5);
350
}
351

352
static bool feat_pmuv3p7(struct kvm *kvm)
353
{
354
	return check_pmu_revision(kvm, V3P7);
355
}
356

357
static bool feat_pmuv3p9(struct kvm *kvm)
358
{
359
	return check_pmu_revision(kvm, V3P9);
360
}
361

362
static bool compute_hcr_rw(struct kvm *kvm, u64 *bits)
363
{
364
	/* This is purely academic: AArch32 and NV are mutually exclusive */
365
	if (bits) {
366
		if (kvm_has_feat(kvm, FEAT_AA32EL1))
367
			*bits &= ~HCR_EL2_RW;
368
		else
369
			*bits |= HCR_EL2_RW;
370
	}
371

372
	return true;
373
}
374

375
static bool compute_hcr_e2h(struct kvm *kvm, u64 *bits)
376
{
377
	if (bits) {
378
		if (kvm_has_feat(kvm, FEAT_E2H0))
379
			*bits &= ~HCR_EL2_E2H;
380
		else
381
			*bits |= HCR_EL2_E2H;
382
	}
383

384
	return true;
385
}
386

387
static const struct reg_bits_to_feat_map hfgrtr_feat_map[] = {
388
	NEEDS_FEAT(HFGRTR_EL2_nAMAIR2_EL1	|
389
		   HFGRTR_EL2_nMAIR2_EL1,
390
		   FEAT_AIE),
391
	NEEDS_FEAT(HFGRTR_EL2_nS2POR_EL1, FEAT_S2POE),
392
	NEEDS_FEAT(HFGRTR_EL2_nPOR_EL1		|
393
		   HFGRTR_EL2_nPOR_EL0,
394
		   FEAT_S1POE),
395
	NEEDS_FEAT(HFGRTR_EL2_nPIR_EL1		|
396
		   HFGRTR_EL2_nPIRE0_EL1,
397
		   FEAT_S1PIE),
398
	NEEDS_FEAT(HFGRTR_EL2_nRCWMASK_EL1, FEAT_THE),
399
	NEEDS_FEAT(HFGRTR_EL2_nTPIDR2_EL0	|
400
		   HFGRTR_EL2_nSMPRI_EL1,
401
		   FEAT_SME),
402
	NEEDS_FEAT(HFGRTR_EL2_nGCS_EL1		|
403
		   HFGRTR_EL2_nGCS_EL0,
404
		   FEAT_GCS),
405
	NEEDS_FEAT(HFGRTR_EL2_nACCDATA_EL1, FEAT_LS64_ACCDATA),
406
	NEEDS_FEAT(HFGRTR_EL2_ERXADDR_EL1	|
407
		   HFGRTR_EL2_ERXMISCn_EL1	|
408
		   HFGRTR_EL2_ERXSTATUS_EL1	|
409
		   HFGRTR_EL2_ERXCTLR_EL1	|
410
		   HFGRTR_EL2_ERXFR_EL1		|
411
		   HFGRTR_EL2_ERRSELR_EL1	|
412
		   HFGRTR_EL2_ERRIDR_EL1,
413
		   FEAT_RAS),
414
	NEEDS_FEAT(HFGRTR_EL2_ERXPFGCDN_EL1	|
415
		   HFGRTR_EL2_ERXPFGCTL_EL1	|
416
		   HFGRTR_EL2_ERXPFGF_EL1,
417
		   feat_rasv1p1),
418
	NEEDS_FEAT(HFGRTR_EL2_ICC_IGRPENn_EL1, FEAT_GICv3),
419
	NEEDS_FEAT(HFGRTR_EL2_SCXTNUM_EL0	|
420
		   HFGRTR_EL2_SCXTNUM_EL1,
421
		   feat_csv2_2_csv2_1p2),
422
	NEEDS_FEAT(HFGRTR_EL2_LORSA_EL1		|
423
		   HFGRTR_EL2_LORN_EL1		|
424
		   HFGRTR_EL2_LORID_EL1		|
425
		   HFGRTR_EL2_LOREA_EL1		|
426
		   HFGRTR_EL2_LORC_EL1,
427
		   FEAT_LOR),
428
	NEEDS_FEAT(HFGRTR_EL2_APIBKey		|
429
		   HFGRTR_EL2_APIAKey		|
430
		   HFGRTR_EL2_APGAKey		|
431
		   HFGRTR_EL2_APDBKey		|
432
		   HFGRTR_EL2_APDAKey,
433
		   feat_pauth),
434
	NEEDS_FEAT_FLAG(HFGRTR_EL2_VBAR_EL1	|
435
			HFGRTR_EL2_TTBR1_EL1	|
436
			HFGRTR_EL2_TTBR0_EL1	|
437
			HFGRTR_EL2_TPIDR_EL0	|
438
			HFGRTR_EL2_TPIDRRO_EL0	|
439
			HFGRTR_EL2_TPIDR_EL1	|
440
			HFGRTR_EL2_TCR_EL1	|
441
			HFGRTR_EL2_SCTLR_EL1	|
442
			HFGRTR_EL2_REVIDR_EL1	|
443
			HFGRTR_EL2_PAR_EL1	|
444
			HFGRTR_EL2_MPIDR_EL1	|
445
			HFGRTR_EL2_MIDR_EL1	|
446
			HFGRTR_EL2_MAIR_EL1	|
447
			HFGRTR_EL2_ISR_EL1	|
448
			HFGRTR_EL2_FAR_EL1	|
449
			HFGRTR_EL2_ESR_EL1	|
450
			HFGRTR_EL2_DCZID_EL0	|
451
			HFGRTR_EL2_CTR_EL0	|
452
			HFGRTR_EL2_CSSELR_EL1	|
453
			HFGRTR_EL2_CPACR_EL1	|
454
			HFGRTR_EL2_CONTEXTIDR_EL1|
455
			HFGRTR_EL2_CLIDR_EL1	|
456
			HFGRTR_EL2_CCSIDR_EL1	|
457
			HFGRTR_EL2_AMAIR_EL1	|
458
			HFGRTR_EL2_AIDR_EL1	|
459
			HFGRTR_EL2_AFSR1_EL1	|
460
			HFGRTR_EL2_AFSR0_EL1,
461
			NEVER_FGU, FEAT_AA64EL1),
462
};
463

464

465
static const DECLARE_FEAT_MAP_FGT(hfgrtr_desc, hfgrtr_masks,
466
				  hfgrtr_feat_map, FEAT_FGT);
467

468
static const struct reg_bits_to_feat_map hfgwtr_feat_map[] = {
469
	NEEDS_FEAT(HFGWTR_EL2_nAMAIR2_EL1	|
470
		   HFGWTR_EL2_nMAIR2_EL1,
471
		   FEAT_AIE),
472
	NEEDS_FEAT(HFGWTR_EL2_nS2POR_EL1, FEAT_S2POE),
473
	NEEDS_FEAT(HFGWTR_EL2_nPOR_EL1		|
474
		   HFGWTR_EL2_nPOR_EL0,
475
		   FEAT_S1POE),
476
	NEEDS_FEAT(HFGWTR_EL2_nPIR_EL1		|
477
		   HFGWTR_EL2_nPIRE0_EL1,
478
		   FEAT_S1PIE),
479
	NEEDS_FEAT(HFGWTR_EL2_nRCWMASK_EL1, FEAT_THE),
480
	NEEDS_FEAT(HFGWTR_EL2_nTPIDR2_EL0	|
481
		   HFGWTR_EL2_nSMPRI_EL1,
482
		   FEAT_SME),
483
	NEEDS_FEAT(HFGWTR_EL2_nGCS_EL1		|
484
		   HFGWTR_EL2_nGCS_EL0,
485
		   FEAT_GCS),
486
	NEEDS_FEAT(HFGWTR_EL2_nACCDATA_EL1, FEAT_LS64_ACCDATA),
487
	NEEDS_FEAT(HFGWTR_EL2_ERXADDR_EL1	|
488
		   HFGWTR_EL2_ERXMISCn_EL1	|
489
		   HFGWTR_EL2_ERXSTATUS_EL1	|
490
		   HFGWTR_EL2_ERXCTLR_EL1	|
491
		   HFGWTR_EL2_ERRSELR_EL1,
492
		   FEAT_RAS),
493
	NEEDS_FEAT(HFGWTR_EL2_ERXPFGCDN_EL1	|
494
		   HFGWTR_EL2_ERXPFGCTL_EL1,
495
		   feat_rasv1p1),
496
	NEEDS_FEAT(HFGWTR_EL2_ICC_IGRPENn_EL1, FEAT_GICv3),
497
	NEEDS_FEAT(HFGWTR_EL2_SCXTNUM_EL0	|
498
		   HFGWTR_EL2_SCXTNUM_EL1,
499
		   feat_csv2_2_csv2_1p2),
500
	NEEDS_FEAT(HFGWTR_EL2_LORSA_EL1		|
501
		   HFGWTR_EL2_LORN_EL1		|
502
		   HFGWTR_EL2_LOREA_EL1		|
503
		   HFGWTR_EL2_LORC_EL1,
504
		   FEAT_LOR),
505
	NEEDS_FEAT(HFGWTR_EL2_APIBKey		|
506
		   HFGWTR_EL2_APIAKey		|
507
		   HFGWTR_EL2_APGAKey		|
508
		   HFGWTR_EL2_APDBKey		|
509
		   HFGWTR_EL2_APDAKey,
510
		   feat_pauth),
511
	NEEDS_FEAT_FLAG(HFGWTR_EL2_VBAR_EL1	|
512
			HFGWTR_EL2_TTBR1_EL1	|
513
			HFGWTR_EL2_TTBR0_EL1	|
514
			HFGWTR_EL2_TPIDR_EL0	|
515
			HFGWTR_EL2_TPIDRRO_EL0	|
516
			HFGWTR_EL2_TPIDR_EL1	|
517
			HFGWTR_EL2_TCR_EL1	|
518
			HFGWTR_EL2_SCTLR_EL1	|
519
			HFGWTR_EL2_PAR_EL1	|
520
			HFGWTR_EL2_MAIR_EL1	|
521
			HFGWTR_EL2_FAR_EL1	|
522
			HFGWTR_EL2_ESR_EL1	|
523
			HFGWTR_EL2_CSSELR_EL1	|
524
			HFGWTR_EL2_CPACR_EL1	|
525
			HFGWTR_EL2_CONTEXTIDR_EL1|
526
			HFGWTR_EL2_AMAIR_EL1	|
527
			HFGWTR_EL2_AFSR1_EL1	|
528
			HFGWTR_EL2_AFSR0_EL1,
529
			NEVER_FGU, FEAT_AA64EL1),
530
};
531

532
static const DECLARE_FEAT_MAP_FGT(hfgwtr_desc, hfgwtr_masks,
533
				  hfgwtr_feat_map, FEAT_FGT);
534

535
static const struct reg_bits_to_feat_map hdfgrtr_feat_map[] = {
536
	NEEDS_FEAT(HDFGRTR_EL2_PMBIDR_EL1	|
537
		   HDFGRTR_EL2_PMSLATFR_EL1	|
538
		   HDFGRTR_EL2_PMSIRR_EL1	|
539
		   HDFGRTR_EL2_PMSIDR_EL1	|
540
		   HDFGRTR_EL2_PMSICR_EL1	|
541
		   HDFGRTR_EL2_PMSFCR_EL1	|
542
		   HDFGRTR_EL2_PMSEVFR_EL1	|
543
		   HDFGRTR_EL2_PMSCR_EL1	|
544
		   HDFGRTR_EL2_PMBSR_EL1	|
545
		   HDFGRTR_EL2_PMBPTR_EL1	|
546
		   HDFGRTR_EL2_PMBLIMITR_EL1,
547
		   FEAT_SPE),
548
	NEEDS_FEAT(HDFGRTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE),
549
	NEEDS_FEAT(HDFGRTR_EL2_nBRBDATA		|
550
		   HDFGRTR_EL2_nBRBCTL		|
551
		   HDFGRTR_EL2_nBRBIDR,
552
		   FEAT_BRBE),
553
	NEEDS_FEAT(HDFGRTR_EL2_TRCVICTLR	|
554
		   HDFGRTR_EL2_TRCSTATR		|
555
		   HDFGRTR_EL2_TRCSSCSRn	|
556
		   HDFGRTR_EL2_TRCSEQSTR	|
557
		   HDFGRTR_EL2_TRCPRGCTLR	|
558
		   HDFGRTR_EL2_TRCOSLSR		|
559
		   HDFGRTR_EL2_TRCIMSPECn	|
560
		   HDFGRTR_EL2_TRCID		|
561
		   HDFGRTR_EL2_TRCCNTVRn	|
562
		   HDFGRTR_EL2_TRCCLAIM		|
563
		   HDFGRTR_EL2_TRCAUXCTLR	|
564
		   HDFGRTR_EL2_TRCAUTHSTATUS	|
565
		   HDFGRTR_EL2_TRC,
566
		   FEAT_TRC_SR),
567
	NEEDS_FEAT(HDFGRTR_EL2_PMCEIDn_EL0	|
568
		   HDFGRTR_EL2_PMUSERENR_EL0	|
569
		   HDFGRTR_EL2_PMMIR_EL1	|
570
		   HDFGRTR_EL2_PMSELR_EL0	|
571
		   HDFGRTR_EL2_PMOVS		|
572
		   HDFGRTR_EL2_PMINTEN		|
573
		   HDFGRTR_EL2_PMCNTEN		|
574
		   HDFGRTR_EL2_PMCCNTR_EL0	|
575
		   HDFGRTR_EL2_PMCCFILTR_EL0	|
576
		   HDFGRTR_EL2_PMEVTYPERn_EL0	|
577
		   HDFGRTR_EL2_PMEVCNTRn_EL0,
578
		   FEAT_PMUv3),
579
	NEEDS_FEAT(HDFGRTR_EL2_TRBTRG_EL1	|
580
		   HDFGRTR_EL2_TRBSR_EL1	|
581
		   HDFGRTR_EL2_TRBPTR_EL1	|
582
		   HDFGRTR_EL2_TRBMAR_EL1	|
583
		   HDFGRTR_EL2_TRBLIMITR_EL1	|
584
		   HDFGRTR_EL2_TRBIDR_EL1	|
585
		   HDFGRTR_EL2_TRBBASER_EL1,
586
		   FEAT_TRBE),
587
	NEEDS_FEAT_FLAG(HDFGRTR_EL2_OSDLR_EL1, NEVER_FGU,
588
			FEAT_DoubleLock),
589
	NEEDS_FEAT_FLAG(HDFGRTR_EL2_OSECCR_EL1	|
590
			HDFGRTR_EL2_OSLSR_EL1	|
591
			HDFGRTR_EL2_DBGPRCR_EL1	|
592
			HDFGRTR_EL2_DBGAUTHSTATUS_EL1|
593
			HDFGRTR_EL2_DBGCLAIM	|
594
			HDFGRTR_EL2_MDSCR_EL1	|
595
			HDFGRTR_EL2_DBGWVRn_EL1	|
596
			HDFGRTR_EL2_DBGWCRn_EL1	|
597
			HDFGRTR_EL2_DBGBVRn_EL1	|
598
			HDFGRTR_EL2_DBGBCRn_EL1,
599
			NEVER_FGU, FEAT_AA64EL1)
600
};
601

602
static const DECLARE_FEAT_MAP_FGT(hdfgrtr_desc, hdfgrtr_masks,
603
				  hdfgrtr_feat_map, FEAT_FGT);
604

605
static const struct reg_bits_to_feat_map hdfgwtr_feat_map[] = {
606
	NEEDS_FEAT(HDFGWTR_EL2_PMSLATFR_EL1	|
607
		   HDFGWTR_EL2_PMSIRR_EL1	|
608
		   HDFGWTR_EL2_PMSICR_EL1	|
609
		   HDFGWTR_EL2_PMSFCR_EL1	|
610
		   HDFGWTR_EL2_PMSEVFR_EL1	|
611
		   HDFGWTR_EL2_PMSCR_EL1	|
612
		   HDFGWTR_EL2_PMBSR_EL1	|
613
		   HDFGWTR_EL2_PMBPTR_EL1	|
614
		   HDFGWTR_EL2_PMBLIMITR_EL1,
615
		   FEAT_SPE),
616
	NEEDS_FEAT(HDFGWTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE),
617
	NEEDS_FEAT(HDFGWTR_EL2_nBRBDATA		|
618
		   HDFGWTR_EL2_nBRBCTL,
619
		   FEAT_BRBE),
620
	NEEDS_FEAT(HDFGWTR_EL2_TRCVICTLR	|
621
		   HDFGWTR_EL2_TRCSSCSRn	|
622
		   HDFGWTR_EL2_TRCSEQSTR	|
623
		   HDFGWTR_EL2_TRCPRGCTLR	|
624
		   HDFGWTR_EL2_TRCOSLAR		|
625
		   HDFGWTR_EL2_TRCIMSPECn	|
626
		   HDFGWTR_EL2_TRCCNTVRn	|
627
		   HDFGWTR_EL2_TRCCLAIM		|
628
		   HDFGWTR_EL2_TRCAUXCTLR	|
629
		   HDFGWTR_EL2_TRC,
630
		   FEAT_TRC_SR),
631
	NEEDS_FEAT(HDFGWTR_EL2_PMUSERENR_EL0	|
632
		   HDFGWTR_EL2_PMCR_EL0		|
633
		   HDFGWTR_EL2_PMSWINC_EL0	|
634
		   HDFGWTR_EL2_PMSELR_EL0	|
635
		   HDFGWTR_EL2_PMOVS		|
636
		   HDFGWTR_EL2_PMINTEN		|
637
		   HDFGWTR_EL2_PMCNTEN		|
638
		   HDFGWTR_EL2_PMCCNTR_EL0	|
639
		   HDFGWTR_EL2_PMCCFILTR_EL0	|
640
		   HDFGWTR_EL2_PMEVTYPERn_EL0	|
641
		   HDFGWTR_EL2_PMEVCNTRn_EL0,
642
		   FEAT_PMUv3),
643
	NEEDS_FEAT(HDFGWTR_EL2_TRBTRG_EL1	|
644
		   HDFGWTR_EL2_TRBSR_EL1	|
645
		   HDFGWTR_EL2_TRBPTR_EL1	|
646
		   HDFGWTR_EL2_TRBMAR_EL1	|
647
		   HDFGWTR_EL2_TRBLIMITR_EL1	|
648
		   HDFGWTR_EL2_TRBBASER_EL1,
649
		   FEAT_TRBE),
650
	NEEDS_FEAT_FLAG(HDFGWTR_EL2_OSDLR_EL1,
651
			NEVER_FGU, FEAT_DoubleLock),
652
	NEEDS_FEAT_FLAG(HDFGWTR_EL2_OSECCR_EL1	|
653
			HDFGWTR_EL2_OSLAR_EL1	|
654
			HDFGWTR_EL2_DBGPRCR_EL1	|
655
			HDFGWTR_EL2_DBGCLAIM	|
656
			HDFGWTR_EL2_MDSCR_EL1	|
657
			HDFGWTR_EL2_DBGWVRn_EL1	|
658
			HDFGWTR_EL2_DBGWCRn_EL1	|
659
			HDFGWTR_EL2_DBGBVRn_EL1	|
660
			HDFGWTR_EL2_DBGBCRn_EL1,
661
			NEVER_FGU, FEAT_AA64EL1),
662
	NEEDS_FEAT(HDFGWTR_EL2_TRFCR_EL1, FEAT_TRF),
663
};
664

665
static const DECLARE_FEAT_MAP_FGT(hdfgwtr_desc, hdfgwtr_masks,
666
				  hdfgwtr_feat_map, FEAT_FGT);
667

668
static const struct reg_bits_to_feat_map hfgitr_feat_map[] = {
669
	NEEDS_FEAT(HFGITR_EL2_PSBCSYNC, FEAT_SPEv1p5),
670
	NEEDS_FEAT(HFGITR_EL2_ATS1E1A, FEAT_ATS1A),
671
	NEEDS_FEAT(HFGITR_EL2_COSPRCTX, FEAT_SPECRES2),
672
	NEEDS_FEAT(HFGITR_EL2_nGCSEPP		|
673
		   HFGITR_EL2_nGCSSTR_EL1	|
674
		   HFGITR_EL2_nGCSPUSHM_EL1,
675
		   FEAT_GCS),
676
	NEEDS_FEAT(HFGITR_EL2_nBRBIALL		|
677
		   HFGITR_EL2_nBRBINJ,
678
		   FEAT_BRBE),
679
	NEEDS_FEAT(HFGITR_EL2_CPPRCTX		|
680
		   HFGITR_EL2_DVPRCTX		|
681
		   HFGITR_EL2_CFPRCTX,
682
		   FEAT_SPECRES),
683
	NEEDS_FEAT(HFGITR_EL2_TLBIRVAALE1	|
684
		   HFGITR_EL2_TLBIRVALE1	|
685
		   HFGITR_EL2_TLBIRVAAE1	|
686
		   HFGITR_EL2_TLBIRVAE1		|
687
		   HFGITR_EL2_TLBIRVAALE1IS	|
688
		   HFGITR_EL2_TLBIRVALE1IS	|
689
		   HFGITR_EL2_TLBIRVAAE1IS	|
690
		   HFGITR_EL2_TLBIRVAE1IS	|
691
		   HFGITR_EL2_TLBIRVAALE1OS	|
692
		   HFGITR_EL2_TLBIRVALE1OS	|
693
		   HFGITR_EL2_TLBIRVAAE1OS	|
694
		   HFGITR_EL2_TLBIRVAE1OS,
695
		   FEAT_TLBIRANGE),
696
	NEEDS_FEAT(HFGITR_EL2_TLBIVAALE1OS	|
697
		   HFGITR_EL2_TLBIVALE1OS	|
698
		   HFGITR_EL2_TLBIVAAE1OS	|
699
		   HFGITR_EL2_TLBIASIDE1OS	|
700
		   HFGITR_EL2_TLBIVAE1OS	|
701
		   HFGITR_EL2_TLBIVMALLE1OS,
702
		   FEAT_TLBIOS),
703
	NEEDS_FEAT(HFGITR_EL2_ATS1E1WP		|
704
		   HFGITR_EL2_ATS1E1RP,
705
		   FEAT_PAN2),
706
	NEEDS_FEAT(HFGITR_EL2_DCCVADP, FEAT_DPB2),
707
	NEEDS_FEAT_FLAG(HFGITR_EL2_DCCVAC	|
708
			HFGITR_EL2_SVC_EL1	|
709
			HFGITR_EL2_SVC_EL0	|
710
			HFGITR_EL2_ERET		|
711
			HFGITR_EL2_TLBIVAALE1	|
712
			HFGITR_EL2_TLBIVALE1	|
713
			HFGITR_EL2_TLBIVAAE1	|
714
			HFGITR_EL2_TLBIASIDE1	|
715
			HFGITR_EL2_TLBIVAE1	|
716
			HFGITR_EL2_TLBIVMALLE1	|
717
			HFGITR_EL2_TLBIVAALE1IS	|
718
			HFGITR_EL2_TLBIVALE1IS	|
719
			HFGITR_EL2_TLBIVAAE1IS	|
720
			HFGITR_EL2_TLBIASIDE1IS	|
721
			HFGITR_EL2_TLBIVAE1IS	|
722
			HFGITR_EL2_TLBIVMALLE1IS|
723
			HFGITR_EL2_ATS1E0W	|
724
			HFGITR_EL2_ATS1E0R	|
725
			HFGITR_EL2_ATS1E1W	|
726
			HFGITR_EL2_ATS1E1R	|
727
			HFGITR_EL2_DCZVA	|
728
			HFGITR_EL2_DCCIVAC	|
729
			HFGITR_EL2_DCCVAP	|
730
			HFGITR_EL2_DCCVAU	|
731
			HFGITR_EL2_DCCISW	|
732
			HFGITR_EL2_DCCSW	|
733
			HFGITR_EL2_DCISW	|
734
			HFGITR_EL2_DCIVAC	|
735
			HFGITR_EL2_ICIVAU	|
736
			HFGITR_EL2_ICIALLU	|
737
			HFGITR_EL2_ICIALLUIS,
738
			NEVER_FGU, FEAT_AA64EL1),
739
};
740

741
static const DECLARE_FEAT_MAP_FGT(hfgitr_desc, hfgitr_masks,
742
				  hfgitr_feat_map, FEAT_FGT);
743

744
static const struct reg_bits_to_feat_map hafgrtr_feat_map[] = {
745
	NEEDS_FEAT(HAFGRTR_EL2_AMEVTYPER115_EL0	|
746
		   HAFGRTR_EL2_AMEVTYPER114_EL0	|
747
		   HAFGRTR_EL2_AMEVTYPER113_EL0	|
748
		   HAFGRTR_EL2_AMEVTYPER112_EL0	|
749
		   HAFGRTR_EL2_AMEVTYPER111_EL0	|
750
		   HAFGRTR_EL2_AMEVTYPER110_EL0	|
751
		   HAFGRTR_EL2_AMEVTYPER19_EL0	|
752
		   HAFGRTR_EL2_AMEVTYPER18_EL0	|
753
		   HAFGRTR_EL2_AMEVTYPER17_EL0	|
754
		   HAFGRTR_EL2_AMEVTYPER16_EL0	|
755
		   HAFGRTR_EL2_AMEVTYPER15_EL0	|
756
		   HAFGRTR_EL2_AMEVTYPER14_EL0	|
757
		   HAFGRTR_EL2_AMEVTYPER13_EL0	|
758
		   HAFGRTR_EL2_AMEVTYPER12_EL0	|
759
		   HAFGRTR_EL2_AMEVTYPER11_EL0	|
760
		   HAFGRTR_EL2_AMEVTYPER10_EL0	|
761
		   HAFGRTR_EL2_AMEVCNTR115_EL0	|
762
		   HAFGRTR_EL2_AMEVCNTR114_EL0	|
763
		   HAFGRTR_EL2_AMEVCNTR113_EL0	|
764
		   HAFGRTR_EL2_AMEVCNTR112_EL0	|
765
		   HAFGRTR_EL2_AMEVCNTR111_EL0	|
766
		   HAFGRTR_EL2_AMEVCNTR110_EL0	|
767
		   HAFGRTR_EL2_AMEVCNTR19_EL0	|
768
		   HAFGRTR_EL2_AMEVCNTR18_EL0	|
769
		   HAFGRTR_EL2_AMEVCNTR17_EL0	|
770
		   HAFGRTR_EL2_AMEVCNTR16_EL0	|
771
		   HAFGRTR_EL2_AMEVCNTR15_EL0	|
772
		   HAFGRTR_EL2_AMEVCNTR14_EL0	|
773
		   HAFGRTR_EL2_AMEVCNTR13_EL0	|
774
		   HAFGRTR_EL2_AMEVCNTR12_EL0	|
775
		   HAFGRTR_EL2_AMEVCNTR11_EL0	|
776
		   HAFGRTR_EL2_AMEVCNTR10_EL0	|
777
		   HAFGRTR_EL2_AMCNTEN1		|
778
		   HAFGRTR_EL2_AMCNTEN0		|
779
		   HAFGRTR_EL2_AMEVCNTR03_EL0	|
780
		   HAFGRTR_EL2_AMEVCNTR02_EL0	|
781
		   HAFGRTR_EL2_AMEVCNTR01_EL0	|
782
		   HAFGRTR_EL2_AMEVCNTR00_EL0,
783
		   FEAT_AMUv1),
784
};
785

786
static const DECLARE_FEAT_MAP_FGT(hafgrtr_desc, hafgrtr_masks,
787
				  hafgrtr_feat_map, FEAT_FGT);
788

789
static const struct reg_bits_to_feat_map hfgitr2_feat_map[] = {
790
	NEEDS_FEAT(HFGITR2_EL2_nDCCIVAPS, FEAT_PoPS),
791
	NEEDS_FEAT(HFGITR2_EL2_TSBCSYNC, FEAT_TRBEv1p1)
792
};
793

794
static const DECLARE_FEAT_MAP_FGT(hfgitr2_desc, hfgitr2_masks,
795
				  hfgitr2_feat_map, FEAT_FGT2);
796

797
static const struct reg_bits_to_feat_map hfgrtr2_feat_map[] = {
798
	NEEDS_FEAT(HFGRTR2_EL2_nPFAR_EL1, FEAT_PFAR),
799
	NEEDS_FEAT(HFGRTR2_EL2_nERXGSR_EL1, FEAT_RASv2),
800
	NEEDS_FEAT(HFGRTR2_EL2_nACTLRALIAS_EL1	|
801
		   HFGRTR2_EL2_nACTLRMASK_EL1	|
802
		   HFGRTR2_EL2_nCPACRALIAS_EL1	|
803
		   HFGRTR2_EL2_nCPACRMASK_EL1	|
804
		   HFGRTR2_EL2_nSCTLR2MASK_EL1	|
805
		   HFGRTR2_EL2_nSCTLRALIAS2_EL1	|
806
		   HFGRTR2_EL2_nSCTLRALIAS_EL1	|
807
		   HFGRTR2_EL2_nSCTLRMASK_EL1	|
808
		   HFGRTR2_EL2_nTCR2ALIAS_EL1	|
809
		   HFGRTR2_EL2_nTCR2MASK_EL1	|
810
		   HFGRTR2_EL2_nTCRALIAS_EL1	|
811
		   HFGRTR2_EL2_nTCRMASK_EL1,
812
		   FEAT_SRMASK),
813
	NEEDS_FEAT(HFGRTR2_EL2_nRCWSMASK_EL1, FEAT_THE),
814
};
815

816
static const DECLARE_FEAT_MAP_FGT(hfgrtr2_desc, hfgrtr2_masks,
817
				  hfgrtr2_feat_map, FEAT_FGT2);
818

819
static const struct reg_bits_to_feat_map hfgwtr2_feat_map[] = {
820
	NEEDS_FEAT(HFGWTR2_EL2_nPFAR_EL1, FEAT_PFAR),
821
	NEEDS_FEAT(HFGWTR2_EL2_nACTLRALIAS_EL1	|
822
		   HFGWTR2_EL2_nACTLRMASK_EL1	|
823
		   HFGWTR2_EL2_nCPACRALIAS_EL1	|
824
		   HFGWTR2_EL2_nCPACRMASK_EL1	|
825
		   HFGWTR2_EL2_nSCTLR2MASK_EL1	|
826
		   HFGWTR2_EL2_nSCTLRALIAS2_EL1	|
827
		   HFGWTR2_EL2_nSCTLRALIAS_EL1	|
828
		   HFGWTR2_EL2_nSCTLRMASK_EL1	|
829
		   HFGWTR2_EL2_nTCR2ALIAS_EL1	|
830
		   HFGWTR2_EL2_nTCR2MASK_EL1	|
831
		   HFGWTR2_EL2_nTCRALIAS_EL1	|
832
		   HFGWTR2_EL2_nTCRMASK_EL1,
833
		   FEAT_SRMASK),
834
	NEEDS_FEAT(HFGWTR2_EL2_nRCWSMASK_EL1, FEAT_THE),
835
};
836

837
static const DECLARE_FEAT_MAP_FGT(hfgwtr2_desc, hfgwtr2_masks,
838
				  hfgwtr2_feat_map, FEAT_FGT2);
839

840
static const struct reg_bits_to_feat_map hdfgrtr2_feat_map[] = {
841
	NEEDS_FEAT(HDFGRTR2_EL2_nMDSELR_EL1, FEAT_Debugv8p9),
842
	NEEDS_FEAT(HDFGRTR2_EL2_nPMECR_EL1, feat_ebep_pmuv3_ss),
843
	NEEDS_FEAT(HDFGRTR2_EL2_nTRCITECR_EL1, FEAT_ITE),
844
	NEEDS_FEAT(HDFGRTR2_EL2_nPMICFILTR_EL0	|
845
		   HDFGRTR2_EL2_nPMICNTR_EL0,
846
		   FEAT_PMUv3_ICNTR),
847
	NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, feat_pmuv3p9),
848
	NEEDS_FEAT(HDFGRTR2_EL2_nPMSSCR_EL1	|
849
		   HDFGRTR2_EL2_nPMSSDATA,
850
		   FEAT_PMUv3_SS),
851
	NEEDS_FEAT(HDFGRTR2_EL2_nPMIAR_EL1, FEAT_SEBEP),
852
	NEEDS_FEAT(HDFGRTR2_EL2_nPMSDSFR_EL1, feat_spe_fds),
853
	NEEDS_FEAT(HDFGRTR2_EL2_nPMBMAR_EL1, FEAT_SPE_nVM),
854
	NEEDS_FEAT(HDFGRTR2_EL2_nSPMACCESSR_EL1	|
855
		   HDFGRTR2_EL2_nSPMCNTEN	|
856
		   HDFGRTR2_EL2_nSPMCR_EL0	|
857
		   HDFGRTR2_EL2_nSPMDEVAFF_EL1	|
858
		   HDFGRTR2_EL2_nSPMEVCNTRn_EL0	|
859
		   HDFGRTR2_EL2_nSPMEVTYPERn_EL0|
860
		   HDFGRTR2_EL2_nSPMID		|
861
		   HDFGRTR2_EL2_nSPMINTEN	|
862
		   HDFGRTR2_EL2_nSPMOVS		|
863
		   HDFGRTR2_EL2_nSPMSCR_EL1	|
864
		   HDFGRTR2_EL2_nSPMSELR_EL0,
865
		   FEAT_SPMU),
866
	NEEDS_FEAT(HDFGRTR2_EL2_nMDSTEPOP_EL1, FEAT_STEP2),
867
	NEEDS_FEAT(HDFGRTR2_EL2_nTRBMPAM_EL1, feat_trbe_mpam),
868
};
869

870
static const DECLARE_FEAT_MAP_FGT(hdfgrtr2_desc, hdfgrtr2_masks,
871
				  hdfgrtr2_feat_map, FEAT_FGT2);
872

873
static const struct reg_bits_to_feat_map hdfgwtr2_feat_map[] = {
874
	NEEDS_FEAT(HDFGWTR2_EL2_nMDSELR_EL1, FEAT_Debugv8p9),
875
	NEEDS_FEAT(HDFGWTR2_EL2_nPMECR_EL1, feat_ebep_pmuv3_ss),
876
	NEEDS_FEAT(HDFGWTR2_EL2_nTRCITECR_EL1, FEAT_ITE),
877
	NEEDS_FEAT(HDFGWTR2_EL2_nPMICFILTR_EL0	|
878
		   HDFGWTR2_EL2_nPMICNTR_EL0,
879
		   FEAT_PMUv3_ICNTR),
880
	NEEDS_FEAT(HDFGWTR2_EL2_nPMUACR_EL1	|
881
		   HDFGWTR2_EL2_nPMZR_EL0,
882
		   feat_pmuv3p9),
883
	NEEDS_FEAT(HDFGWTR2_EL2_nPMSSCR_EL1, FEAT_PMUv3_SS),
884
	NEEDS_FEAT(HDFGWTR2_EL2_nPMIAR_EL1, FEAT_SEBEP),
885
	NEEDS_FEAT(HDFGWTR2_EL2_nPMSDSFR_EL1, feat_spe_fds),
886
	NEEDS_FEAT(HDFGWTR2_EL2_nPMBMAR_EL1, FEAT_SPE_nVM),
887
	NEEDS_FEAT(HDFGWTR2_EL2_nSPMACCESSR_EL1	|
888
		   HDFGWTR2_EL2_nSPMCNTEN	|
889
		   HDFGWTR2_EL2_nSPMCR_EL0	|
890
		   HDFGWTR2_EL2_nSPMEVCNTRn_EL0	|
891
		   HDFGWTR2_EL2_nSPMEVTYPERn_EL0|
892
		   HDFGWTR2_EL2_nSPMINTEN	|
893
		   HDFGWTR2_EL2_nSPMOVS		|
894
		   HDFGWTR2_EL2_nSPMSCR_EL1	|
895
		   HDFGWTR2_EL2_nSPMSELR_EL0,
896
		   FEAT_SPMU),
897
	NEEDS_FEAT(HDFGWTR2_EL2_nMDSTEPOP_EL1, FEAT_STEP2),
898
	NEEDS_FEAT(HDFGWTR2_EL2_nTRBMPAM_EL1, feat_trbe_mpam),
899
};
900

901
static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks,
902
				  hdfgwtr2_feat_map, FEAT_FGT2);
903

904

905
static const struct reg_bits_to_feat_map hcrx_feat_map[] = {
906
	NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr),
907
	NEEDS_FEAT(HCRX_EL2_EnFPM, FEAT_FPMR),
908
	NEEDS_FEAT(HCRX_EL2_GCSEn, FEAT_GCS),
909
	NEEDS_FEAT(HCRX_EL2_EnIDCP128, FEAT_SYSREG128),
910
	NEEDS_FEAT(HCRX_EL2_EnSDERR, feat_aderr),
911
	NEEDS_FEAT(HCRX_EL2_TMEA, FEAT_DoubleFault2),
912
	NEEDS_FEAT(HCRX_EL2_EnSNERR, feat_anerr),
913
	NEEDS_FEAT(HCRX_EL2_D128En, FEAT_D128),
914
	NEEDS_FEAT(HCRX_EL2_PTTWI, FEAT_THE),
915
	NEEDS_FEAT(HCRX_EL2_SCTLR2En, FEAT_SCTLR2),
916
	NEEDS_FEAT(HCRX_EL2_TCR2En, FEAT_TCR2),
917
	NEEDS_FEAT(HCRX_EL2_MSCEn		|
918
		   HCRX_EL2_MCE2,
919
		   FEAT_MOPS),
920
	NEEDS_FEAT(HCRX_EL2_CMOW, FEAT_CMOW),
921
	NEEDS_FEAT(HCRX_EL2_VFNMI		|
922
		   HCRX_EL2_VINMI		|
923
		   HCRX_EL2_TALLINT,
924
		   FEAT_NMI),
925
	NEEDS_FEAT(HCRX_EL2_SMPME, feat_sme_smps),
926
	NEEDS_FEAT(HCRX_EL2_FGTnXS		|
927
		   HCRX_EL2_FnXS,
928
		   FEAT_XS),
929
	NEEDS_FEAT(HCRX_EL2_EnASR, FEAT_LS64_V),
930
	NEEDS_FEAT(HCRX_EL2_EnALS, FEAT_LS64),
931
	NEEDS_FEAT(HCRX_EL2_EnAS0, FEAT_LS64_ACCDATA),
932
};
933

934

935
static const DECLARE_FEAT_MAP(hcrx_desc, __HCRX_EL2,
936
			      hcrx_feat_map, FEAT_HCX);
937

938
static const struct reg_bits_to_feat_map hcr_feat_map[] = {
939
	NEEDS_FEAT(HCR_EL2_TID0, FEAT_AA32EL0),
940
	NEEDS_FEAT_FIXED(HCR_EL2_RW, compute_hcr_rw),
941
	NEEDS_FEAT(HCR_EL2_HCD, not_feat_aa64el3),
942
	NEEDS_FEAT(HCR_EL2_AMO		|
943
		   HCR_EL2_BSU		|
944
		   HCR_EL2_CD		|
945
		   HCR_EL2_DC		|
946
		   HCR_EL2_FB		|
947
		   HCR_EL2_FMO		|
948
		   HCR_EL2_ID		|
949
		   HCR_EL2_IMO		|
950
		   HCR_EL2_MIOCNCE	|
951
		   HCR_EL2_PTW		|
952
		   HCR_EL2_SWIO		|
953
		   HCR_EL2_TACR		|
954
		   HCR_EL2_TDZ		|
955
		   HCR_EL2_TGE		|
956
		   HCR_EL2_TID1		|
957
		   HCR_EL2_TID2		|
958
		   HCR_EL2_TID3		|
959
		   HCR_EL2_TIDCP	|
960
		   HCR_EL2_TPCP		|
961
		   HCR_EL2_TPU		|
962
		   HCR_EL2_TRVM		|
963
		   HCR_EL2_TSC		|
964
		   HCR_EL2_TSW		|
965
		   HCR_EL2_TTLB		|
966
		   HCR_EL2_TVM		|
967
		   HCR_EL2_TWE		|
968
		   HCR_EL2_TWI		|
969
		   HCR_EL2_VF		|
970
		   HCR_EL2_VI		|
971
		   HCR_EL2_VM		|
972
		   HCR_EL2_VSE,
973
		   FEAT_AA64EL1),
974
	NEEDS_FEAT(HCR_EL2_AMVOFFEN, FEAT_AMUv1p1),
975
	NEEDS_FEAT(HCR_EL2_EnSCXT, feat_csv2_2_csv2_1p2),
976
	NEEDS_FEAT(HCR_EL2_TICAB	|
977
		   HCR_EL2_TID4		|
978
		   HCR_EL2_TOCU,
979
		   FEAT_EVT),
980
	NEEDS_FEAT(HCR_EL2_TTLBIS	|
981
		   HCR_EL2_TTLBOS,
982
		   FEAT_EVT_TTLBxS),
983
	NEEDS_FEAT(HCR_EL2_TLOR, FEAT_LOR),
984
	NEEDS_FEAT(HCR_EL2_ATA		|
985
		   HCR_EL2_DCT		|
986
		   HCR_EL2_TID5,
987
		   FEAT_MTE2),
988
	NEEDS_FEAT(HCR_EL2_AT		| /* Ignore the original FEAT_NV */
989
		   HCR_EL2_NV2		|
990
		   HCR_EL2_NV,
991
		   feat_nv2),
992
	NEEDS_FEAT(HCR_EL2_NV1, feat_nv2_e2h0_ni), /* Missing from JSON */
993
	NEEDS_FEAT(HCR_EL2_API		|
994
		   HCR_EL2_APK,
995
		   feat_pauth),
996
	NEEDS_FEAT(HCR_EL2_TEA		|
997
		   HCR_EL2_TERR,
998
		   FEAT_RAS),
999
	NEEDS_FEAT(HCR_EL2_FIEN, feat_rasv1p1),
1000
	NEEDS_FEAT(HCR_EL2_GPF, FEAT_RME),
1001
	NEEDS_FEAT(HCR_EL2_FWB, FEAT_S2FWB),
1002
	NEEDS_FEAT(HCR_EL2_TME, FEAT_TME),
1003
	NEEDS_FEAT(HCR_EL2_TWEDEL	|
1004
		   HCR_EL2_TWEDEn,
1005
		   FEAT_TWED),
1006
	NEEDS_FEAT_FIXED(HCR_EL2_E2H, compute_hcr_e2h),
1007
};
1008

1009
static const DECLARE_FEAT_MAP(hcr_desc, HCR_EL2,
1010
			      hcr_feat_map, FEAT_AA64EL2);
1011

1012
static const struct reg_bits_to_feat_map sctlr2_feat_map[] = {
1013
	NEEDS_FEAT(SCTLR2_EL1_NMEA	|
1014
		   SCTLR2_EL1_EASE,
1015
		   FEAT_DoubleFault2),
1016
	NEEDS_FEAT(SCTLR2_EL1_EnADERR, feat_aderr),
1017
	NEEDS_FEAT(SCTLR2_EL1_EnANERR, feat_anerr),
1018
	NEEDS_FEAT(SCTLR2_EL1_EnIDCP128, FEAT_SYSREG128),
1019
	NEEDS_FEAT(SCTLR2_EL1_EnPACM	|
1020
		   SCTLR2_EL1_EnPACM0,
1021
		   feat_pauth_lr),
1022
	NEEDS_FEAT(SCTLR2_EL1_CPTA	|
1023
		   SCTLR2_EL1_CPTA0	|
1024
		   SCTLR2_EL1_CPTM	|
1025
		   SCTLR2_EL1_CPTM0,
1026
		   FEAT_CPA2),
1027
};
1028

1029
static const DECLARE_FEAT_MAP(sctlr2_desc, SCTLR2_EL1,
1030
			      sctlr2_feat_map, FEAT_SCTLR2);
1031

1032
static const struct reg_bits_to_feat_map tcr2_el2_feat_map[] = {
1033
	NEEDS_FEAT(TCR2_EL2_FNG1	|
1034
		   TCR2_EL2_FNG0	|
1035
		   TCR2_EL2_A2,
1036
		   feat_asid2_e2h1),
1037
	NEEDS_FEAT(TCR2_EL2_DisCH1	|
1038
		   TCR2_EL2_DisCH0	|
1039
		   TCR2_EL2_D128,
1040
		   feat_d128_e2h1),
1041
	NEEDS_FEAT(TCR2_EL2_AMEC1, feat_mec_e2h1),
1042
	NEEDS_FEAT(TCR2_EL2_AMEC0, FEAT_MEC),
1043
	NEEDS_FEAT(TCR2_EL2_HAFT, FEAT_HAFT),
1044
	NEEDS_FEAT(TCR2_EL2_PTTWI	|
1045
		   TCR2_EL2_PnCH,
1046
		   FEAT_THE),
1047
	NEEDS_FEAT(TCR2_EL2_AIE, FEAT_AIE),
1048
	NEEDS_FEAT(TCR2_EL2_POE		|
1049
		   TCR2_EL2_E0POE,
1050
		   FEAT_S1POE),
1051
	NEEDS_FEAT(TCR2_EL2_PIE, FEAT_S1PIE),
1052
};
1053

1054
static const DECLARE_FEAT_MAP(tcr2_el2_desc, TCR2_EL2,
1055
			      tcr2_el2_feat_map, FEAT_TCR2);
1056

1057
static const struct reg_bits_to_feat_map sctlr_el1_feat_map[] = {
1058
	NEEDS_FEAT(SCTLR_EL1_CP15BEN	|
1059
		   SCTLR_EL1_ITD	|
1060
		   SCTLR_EL1_SED,
1061
		   FEAT_AA32EL0),
1062
	NEEDS_FEAT(SCTLR_EL1_BT0	|
1063
		   SCTLR_EL1_BT1,
1064
		   FEAT_BTI),
1065
	NEEDS_FEAT(SCTLR_EL1_CMOW, FEAT_CMOW),
1066
	NEEDS_FEAT(SCTLR_EL1_TSCXT, feat_csv2_2_csv2_1p2),
1067
	NEEDS_FEAT(SCTLR_EL1_EIS	|
1068
		   SCTLR_EL1_EOS,
1069
		   FEAT_ExS),
1070
	NEEDS_FEAT(SCTLR_EL1_EnFPM, FEAT_FPMR),
1071
	NEEDS_FEAT(SCTLR_EL1_IESB, FEAT_IESB),
1072
	NEEDS_FEAT(SCTLR_EL1_EnALS, FEAT_LS64),
1073
	NEEDS_FEAT(SCTLR_EL1_EnAS0, FEAT_LS64_ACCDATA),
1074
	NEEDS_FEAT(SCTLR_EL1_EnASR, FEAT_LS64_V),
1075
	NEEDS_FEAT(SCTLR_EL1_nAA, FEAT_LSE2),
1076
	NEEDS_FEAT(SCTLR_EL1_LSMAOE	|
1077
		   SCTLR_EL1_nTLSMD,
1078
		   FEAT_LSMAOC),
1079
	NEEDS_FEAT(SCTLR_EL1_EE, FEAT_MixedEnd),
1080
	NEEDS_FEAT(SCTLR_EL1_E0E, feat_mixedendel0),
1081
	NEEDS_FEAT(SCTLR_EL1_MSCEn, FEAT_MOPS),
1082
	NEEDS_FEAT(SCTLR_EL1_ATA0	|
1083
		   SCTLR_EL1_ATA	|
1084
		   SCTLR_EL1_TCF0	|
1085
		   SCTLR_EL1_TCF,
1086
		   FEAT_MTE2),
1087
	NEEDS_FEAT(SCTLR_EL1_ITFSB, feat_mte_async),
1088
	NEEDS_FEAT(SCTLR_EL1_TCSO0	|
1089
		   SCTLR_EL1_TCSO,
1090
		   FEAT_MTE_STORE_ONLY),
1091
	NEEDS_FEAT(SCTLR_EL1_NMI	|
1092
		   SCTLR_EL1_SPINTMASK,
1093
		   FEAT_NMI),
1094
	NEEDS_FEAT(SCTLR_EL1_SPAN, FEAT_PAN),
1095
	NEEDS_FEAT(SCTLR_EL1_EPAN, FEAT_PAN3),
1096
	NEEDS_FEAT(SCTLR_EL1_EnDA	|
1097
		   SCTLR_EL1_EnDB	|
1098
		   SCTLR_EL1_EnIA	|
1099
		   SCTLR_EL1_EnIB,
1100
		   feat_pauth),
1101
	NEEDS_FEAT(SCTLR_EL1_EnTP2, FEAT_SME),
1102
	NEEDS_FEAT(SCTLR_EL1_EnRCTX, FEAT_SPECRES),
1103
	NEEDS_FEAT(SCTLR_EL1_DSSBS, FEAT_SSBS),
1104
	NEEDS_FEAT(SCTLR_EL1_TIDCP, FEAT_TIDCP1),
1105
	NEEDS_FEAT(SCTLR_EL1_TME0	|
1106
		   SCTLR_EL1_TME	|
1107
		   SCTLR_EL1_TMT0	|
1108
		   SCTLR_EL1_TMT,
1109
		   FEAT_TME),
1110
	NEEDS_FEAT(SCTLR_EL1_TWEDEL	|
1111
		   SCTLR_EL1_TWEDEn,
1112
		   FEAT_TWED),
1113
	NEEDS_FEAT(SCTLR_EL1_UCI	|
1114
		   SCTLR_EL1_EE		|
1115
		   SCTLR_EL1_E0E	|
1116
		   SCTLR_EL1_WXN	|
1117
		   SCTLR_EL1_nTWE	|
1118
		   SCTLR_EL1_nTWI	|
1119
		   SCTLR_EL1_UCT	|
1120
		   SCTLR_EL1_DZE	|
1121
		   SCTLR_EL1_I		|
1122
		   SCTLR_EL1_UMA	|
1123
		   SCTLR_EL1_SA0	|
1124
		   SCTLR_EL1_SA		|
1125
		   SCTLR_EL1_C		|
1126
		   SCTLR_EL1_A		|
1127
		   SCTLR_EL1_M,
1128
		   FEAT_AA64EL1),
1129
};
1130

1131
static const DECLARE_FEAT_MAP(sctlr_el1_desc, SCTLR_EL1,
1132
			      sctlr_el1_feat_map, FEAT_AA64EL1);
1133

1134
static const struct reg_bits_to_feat_map mdcr_el2_feat_map[] = {
1135
	NEEDS_FEAT(MDCR_EL2_EBWE, FEAT_Debugv8p9),
1136
	NEEDS_FEAT(MDCR_EL2_TDOSA, FEAT_DoubleLock),
1137
	NEEDS_FEAT(MDCR_EL2_PMEE, FEAT_EBEP),
1138
	NEEDS_FEAT(MDCR_EL2_TDCC, FEAT_FGT),
1139
	NEEDS_FEAT(MDCR_EL2_MTPME, FEAT_MTPMU),
1140
	NEEDS_FEAT(MDCR_EL2_HPME	|
1141
		   MDCR_EL2_HPMN	|
1142
		   MDCR_EL2_TPMCR	|
1143
		   MDCR_EL2_TPM,
1144
		   FEAT_PMUv3),
1145
	NEEDS_FEAT(MDCR_EL2_HPMD, feat_pmuv3p1),
1146
	NEEDS_FEAT(MDCR_EL2_HCCD	|
1147
		   MDCR_EL2_HLP,
1148
		   feat_pmuv3p5),
1149
	NEEDS_FEAT(MDCR_EL2_HPMFZO, feat_pmuv3p7),
1150
	NEEDS_FEAT(MDCR_EL2_PMSSE, FEAT_PMUv3_SS),
1151
	NEEDS_FEAT(MDCR_EL2_E2PB	|
1152
		   MDCR_EL2_TPMS,
1153
		   FEAT_SPE),
1154
	NEEDS_FEAT(MDCR_EL2_HPMFZS, FEAT_SPEv1p2),
1155
	NEEDS_FEAT(MDCR_EL2_EnSPM, FEAT_SPMU),
1156
	NEEDS_FEAT(MDCR_EL2_EnSTEPOP, FEAT_STEP2),
1157
	NEEDS_FEAT(MDCR_EL2_E2TB, FEAT_TRBE),
1158
	NEEDS_FEAT(MDCR_EL2_TTRF, FEAT_TRF),
1159
	NEEDS_FEAT(MDCR_EL2_TDA		|
1160
		   MDCR_EL2_TDE		|
1161
		   MDCR_EL2_TDRA,
1162
		   FEAT_AA64EL1),
1163
};
1164

1165
static const DECLARE_FEAT_MAP(mdcr_el2_desc, MDCR_EL2,
1166
			      mdcr_el2_feat_map, FEAT_AA64EL2);
1167

1168
static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
1169
				  int map_size, u64 res0, const char *str)
1170
{
1171
	u64 mask = 0;
1172

1173
	for (int i = 0; i < map_size; i++)
1174
		mask |= map[i].bits;
1175

1176
	if (mask != ~res0)
1177
		kvm_err("Undefined %s behaviour, bits %016llx\n",
1178
			str, mask ^ ~res0);
1179
}
1180

1181
static u64 reg_feat_map_bits(const struct reg_bits_to_feat_map *map)
1182
{
1183
	return map->flags & RES0_POINTER ? ~(*map->res0p) : map->bits;
1184
}
1185

1186
static void __init check_reg_desc(const struct reg_feat_map_desc *r)
1187
{
1188
	check_feat_map(r->bit_feat_map, r->bit_feat_map_sz,
1189
		       ~reg_feat_map_bits(&r->feat_map), r->name);
1190
}
1191

1192
void __init check_feature_map(void)
1193
{
1194
	check_reg_desc(&hfgrtr_desc);
1195
	check_reg_desc(&hfgwtr_desc);
1196
	check_reg_desc(&hfgitr_desc);
1197
	check_reg_desc(&hdfgrtr_desc);
1198
	check_reg_desc(&hdfgwtr_desc);
1199
	check_reg_desc(&hafgrtr_desc);
1200
	check_reg_desc(&hfgrtr2_desc);
1201
	check_reg_desc(&hfgwtr2_desc);
1202
	check_reg_desc(&hfgitr2_desc);
1203
	check_reg_desc(&hdfgrtr2_desc);
1204
	check_reg_desc(&hdfgwtr2_desc);
1205
	check_reg_desc(&hcrx_desc);
1206
	check_reg_desc(&hcr_desc);
1207
	check_reg_desc(&sctlr2_desc);
1208
	check_reg_desc(&tcr2_el2_desc);
1209
	check_reg_desc(&sctlr_el1_desc);
1210
	check_reg_desc(&mdcr_el2_desc);
1211
}
1212

1213
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
1214
{
1215
	u64 regval = kvm->arch.id_regs[map->regidx];
1216
	u64 regfld = (regval >> map->shift) & GENMASK(map->width - 1, 0);
1217

1218
	if (map->sign) {
1219
		s64 sfld = sign_extend64(regfld, map->width - 1);
1220
		s64 slim = sign_extend64(map->lo_lim, map->width - 1);
1221
		return sfld >= slim;
1222
	} else {
1223
		return regfld >= map->lo_lim;
1224
	}
1225
}
1226

1227
static u64 __compute_fixed_bits(struct kvm *kvm,
1228
				const struct reg_bits_to_feat_map *map,
1229
				int map_size,
1230
				u64 *fixed_bits,
1231
				unsigned long require,
1232
				unsigned long exclude)
1233
{
1234
	u64 val = 0;
1235

1236
	for (int i = 0; i < map_size; i++) {
1237
		bool match;
1238

1239
		if ((map[i].flags & require) != require)
1240
			continue;
1241

1242
		if (map[i].flags & exclude)
1243
			continue;
1244

1245
		if (map[i].flags & CALL_FUNC)
1246
			match = (map[i].flags & FIXED_VALUE) ?
1247
				map[i].fval(kvm, fixed_bits) :
1248
				map[i].match(kvm);
1249
		else
1250
			match = idreg_feat_match(kvm, &map[i]);
1251

1252
		if (!match || (map[i].flags & FIXED_VALUE))
1253
			val |= reg_feat_map_bits(&map[i]);
1254
	}
1255

1256
	return val;
1257
}
1258

1259
static u64 compute_res0_bits(struct kvm *kvm,
1260
			     const struct reg_bits_to_feat_map *map,
1261
			     int map_size,
1262
			     unsigned long require,
1263
			     unsigned long exclude)
1264
{
1265
	return __compute_fixed_bits(kvm, map, map_size, NULL,
1266
				    require, exclude | FIXED_VALUE);
1267
}
1268

1269
static u64 compute_reg_res0_bits(struct kvm *kvm,
1270
				 const struct reg_feat_map_desc *r,
1271
				 unsigned long require, unsigned long exclude)
1272

1273
{
1274
	u64 res0;
1275

1276
	res0 = compute_res0_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz,
1277
				 require, exclude);
1278

1279
	/*
1280
	 * If computing FGUs, don't take RES0 or register existence
1281
	 * into account -- we're not computing bits for the register
1282
	 * itself.
1283
	 */
1284
	if (!(exclude & NEVER_FGU)) {
1285
		res0 |= compute_res0_bits(kvm, &r->feat_map, 1, require, exclude);
1286
		res0 |= ~reg_feat_map_bits(&r->feat_map);
1287
	}
1288

1289
	return res0;
1290
}
1291

1292
static u64 compute_reg_fixed_bits(struct kvm *kvm,
1293
				  const struct reg_feat_map_desc *r,
1294
				  u64 *fixed_bits, unsigned long require,
1295
				  unsigned long exclude)
1296
{
1297
	return __compute_fixed_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz,
1298
				    fixed_bits, require | FIXED_VALUE, exclude);
1299
}
1300

1301
void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt)
1302
{
1303
	u64 val = 0;
1304

1305
	switch (fgt) {
1306
	case HFGRTR_GROUP:
1307
		val |= compute_reg_res0_bits(kvm, &hfgrtr_desc,
1308
					     0, NEVER_FGU);
1309
		val |= compute_reg_res0_bits(kvm, &hfgwtr_desc,
1310
					     0, NEVER_FGU);
1311
		break;
1312
	case HFGITR_GROUP:
1313
		val |= compute_reg_res0_bits(kvm, &hfgitr_desc,
1314
					     0, NEVER_FGU);
1315
		break;
1316
	case HDFGRTR_GROUP:
1317
		val |= compute_reg_res0_bits(kvm, &hdfgrtr_desc,
1318
					     0, NEVER_FGU);
1319
		val |= compute_reg_res0_bits(kvm, &hdfgwtr_desc,
1320
					     0, NEVER_FGU);
1321
		break;
1322
	case HAFGRTR_GROUP:
1323
		val |= compute_reg_res0_bits(kvm, &hafgrtr_desc,
1324
					     0, NEVER_FGU);
1325
		break;
1326
	case HFGRTR2_GROUP:
1327
		val |= compute_reg_res0_bits(kvm, &hfgrtr2_desc,
1328
					     0, NEVER_FGU);
1329
		val |= compute_reg_res0_bits(kvm, &hfgwtr2_desc,
1330
					     0, NEVER_FGU);
1331
		break;
1332
	case HFGITR2_GROUP:
1333
		val |= compute_reg_res0_bits(kvm, &hfgitr2_desc,
1334
					     0, NEVER_FGU);
1335
		break;
1336
	case HDFGRTR2_GROUP:
1337
		val |= compute_reg_res0_bits(kvm, &hdfgrtr2_desc,
1338
					     0, NEVER_FGU);
1339
		val |= compute_reg_res0_bits(kvm, &hdfgwtr2_desc,
1340
					     0, NEVER_FGU);
1341
		break;
1342
	default:
1343
		BUG();
1344
	}
1345

1346
	kvm->arch.fgu[fgt] = val;
1347
}
1348

1349
void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *res1)
1350
{
1351
	u64 fixed = 0, mask;
1352

1353
	switch (reg) {
1354
	case HFGRTR_EL2:
1355
		*res0 = compute_reg_res0_bits(kvm, &hfgrtr_desc, 0, 0);
1356
		*res1 = HFGRTR_EL2_RES1;
1357
		break;
1358
	case HFGWTR_EL2:
1359
		*res0 = compute_reg_res0_bits(kvm, &hfgwtr_desc, 0, 0);
1360
		*res1 = HFGWTR_EL2_RES1;
1361
		break;
1362
	case HFGITR_EL2:
1363
		*res0 = compute_reg_res0_bits(kvm, &hfgitr_desc, 0, 0);
1364
		*res1 = HFGITR_EL2_RES1;
1365
		break;
1366
	case HDFGRTR_EL2:
1367
		*res0 = compute_reg_res0_bits(kvm, &hdfgrtr_desc, 0, 0);
1368
		*res1 = HDFGRTR_EL2_RES1;
1369
		break;
1370
	case HDFGWTR_EL2:
1371
		*res0 = compute_reg_res0_bits(kvm, &hdfgwtr_desc, 0, 0);
1372
		*res1 = HDFGWTR_EL2_RES1;
1373
		break;
1374
	case HAFGRTR_EL2:
1375
		*res0 = compute_reg_res0_bits(kvm, &hafgrtr_desc, 0, 0);
1376
		*res1 = HAFGRTR_EL2_RES1;
1377
		break;
1378
	case HFGRTR2_EL2:
1379
		*res0 = compute_reg_res0_bits(kvm, &hfgrtr2_desc, 0, 0);
1380
		*res1 = HFGRTR2_EL2_RES1;
1381
		break;
1382
	case HFGWTR2_EL2:
1383
		*res0 = compute_reg_res0_bits(kvm, &hfgwtr2_desc, 0, 0);
1384
		*res1 = HFGWTR2_EL2_RES1;
1385
		break;
1386
	case HFGITR2_EL2:
1387
		*res0 = compute_reg_res0_bits(kvm, &hfgitr2_desc, 0, 0);
1388
		*res1 = HFGITR2_EL2_RES1;
1389
		break;
1390
	case HDFGRTR2_EL2:
1391
		*res0 = compute_reg_res0_bits(kvm, &hdfgrtr2_desc, 0, 0);
1392
		*res1 = HDFGRTR2_EL2_RES1;
1393
		break;
1394
	case HDFGWTR2_EL2:
1395
		*res0 = compute_reg_res0_bits(kvm, &hdfgwtr2_desc, 0, 0);
1396
		*res1 = HDFGWTR2_EL2_RES1;
1397
		break;
1398
	case HCRX_EL2:
1399
		*res0 = compute_reg_res0_bits(kvm, &hcrx_desc, 0, 0);
1400
		*res1 = __HCRX_EL2_RES1;
1401
		break;
1402
	case HCR_EL2:
1403
		mask = compute_reg_fixed_bits(kvm, &hcr_desc, &fixed, 0, 0);
1404
		*res0 = compute_reg_res0_bits(kvm, &hcr_desc, 0, 0);
1405
		*res0 |= (mask & ~fixed);
1406
		*res1 = HCR_EL2_RES1 | (mask & fixed);
1407
		break;
1408
	case SCTLR2_EL1:
1409
	case SCTLR2_EL2:
1410
		*res0 = compute_reg_res0_bits(kvm, &sctlr2_desc, 0, 0);
1411
		*res1 = SCTLR2_EL1_RES1;
1412
		break;
1413
	case TCR2_EL2:
1414
		*res0 = compute_reg_res0_bits(kvm, &tcr2_el2_desc, 0, 0);
1415
		*res1 = TCR2_EL2_RES1;
1416
		break;
1417
	case SCTLR_EL1:
1418
		*res0 = compute_reg_res0_bits(kvm, &sctlr_el1_desc, 0, 0);
1419
		*res1 = SCTLR_EL1_RES1;
1420
		break;
1421
	case MDCR_EL2:
1422
		*res0 = compute_reg_res0_bits(kvm, &mdcr_el2_desc, 0, 0);
1423
		*res1 = MDCR_EL2_RES1;
1424
		break;
1425
	default:
1426
		WARN_ON_ONCE(1);
1427
		*res0 = *res1 = 0;
1428
		break;
1429
	}
1430
}
1431

1432