1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * Copyright (C) 2016 - Linaro and Columbia University
4
 * Author: Jintack Lim <[email protected]>
5
 */
6

7
#include <linux/kvm.h>
8
#include <linux/kvm_host.h>
9

10
#include <asm/kvm_emulate.h>
11
#include <asm/kvm_nested.h>
12

13
#include "hyp/include/hyp/adjust_pc.h"
14

15
#include "trace.h"
16

17
enum trap_behaviour {
18
	BEHAVE_HANDLE_LOCALLY	= 0,
19

20
	BEHAVE_FORWARD_READ	= BIT(0),
21
	BEHAVE_FORWARD_WRITE	= BIT(1),
22
	BEHAVE_FORWARD_RW	= BEHAVE_FORWARD_READ | BEHAVE_FORWARD_WRITE,
23

24
	/* Traps that take effect in Host EL0, this is rare! */
25
	BEHAVE_FORWARD_IN_HOST_EL0	= BIT(2),
26
};
27

28
struct trap_bits {
29
	const enum vcpu_sysreg		index;
30
	const enum trap_behaviour	behaviour;
31
	const u64			value;
32
	const u64			mask;
33
};
34

35
/* Coarse Grained Trap definitions */
36
enum cgt_group_id {
37
	/* Indicates no coarse trap control */
38
	__RESERVED__,
39

40
	/*
41
	 * The first batch of IDs denote coarse trapping that are used
42
	 * on their own instead of being part of a combination of
43
	 * trap controls.
44
	 */
45
	CGT_HCR_TID1,
46
	CGT_HCR_TID2,
47
	CGT_HCR_TID3,
48
	CGT_HCR_IMO,
49
	CGT_HCR_FMO,
50
	CGT_HCR_TIDCP,
51
	CGT_HCR_TACR,
52
	CGT_HCR_TSW,
53
	CGT_HCR_TPC,
54
	CGT_HCR_TPU,
55
	CGT_HCR_TTLB,
56
	CGT_HCR_TVM,
57
	CGT_HCR_TDZ,
58
	CGT_HCR_TRVM,
59
	CGT_HCR_TLOR,
60
	CGT_HCR_TERR,
61
	CGT_HCR_APK,
62
	CGT_HCR_NV,
63
	CGT_HCR_NV_nNV2,
64
	CGT_HCR_NV1_nNV2,
65
	CGT_HCR_AT,
66
	CGT_HCR_nFIEN,
67
	CGT_HCR_TID4,
68
	CGT_HCR_TICAB,
69
	CGT_HCR_TOCU,
70
	CGT_HCR_ENSCXT,
71
	CGT_HCR_TTLBIS,
72
	CGT_HCR_TTLBOS,
73

74
	CGT_MDCR_TPMCR,
75
	CGT_MDCR_TPM,
76
	CGT_MDCR_TDE,
77
	CGT_MDCR_TDA,
78
	CGT_MDCR_TDOSA,
79
	CGT_MDCR_TDRA,
80
	CGT_MDCR_E2PB,
81
	CGT_MDCR_TPMS,
82
	CGT_MDCR_TTRF,
83
	CGT_MDCR_E2TB,
84
	CGT_MDCR_TDCC,
85

86
	CGT_CPTR_TAM,
87
	CGT_CPTR_TCPAC,
88

89
	CGT_HCRX_EnFPM,
90
	CGT_HCRX_TCR2En,
91
	CGT_HCRX_SCTLR2En,
92

93
	CGT_CNTHCTL_EL1TVT,
94
	CGT_CNTHCTL_EL1TVCT,
95

96
	CGT_ICH_HCR_TC,
97
	CGT_ICH_HCR_TALL0,
98
	CGT_ICH_HCR_TALL1,
99
	CGT_ICH_HCR_TDIR,
100

101
	/*
102
	 * Anything after this point is a combination of coarse trap
103
	 * controls, which must all be evaluated to decide what to do.
104
	 */
105
	__MULTIPLE_CONTROL_BITS__,
106
	CGT_HCR_IMO_FMO_ICH_HCR_TC = __MULTIPLE_CONTROL_BITS__,
107
	CGT_HCR_TID2_TID4,
108
	CGT_HCR_TTLB_TTLBIS,
109
	CGT_HCR_TTLB_TTLBOS,
110
	CGT_HCR_TVM_TRVM,
111
	CGT_HCR_TVM_TRVM_HCRX_TCR2En,
112
	CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
113
	CGT_HCR_TPU_TICAB,
114
	CGT_HCR_TPU_TOCU,
115
	CGT_HCR_NV1_nNV2_ENSCXT,
116
	CGT_MDCR_TPM_TPMCR,
117
	CGT_MDCR_TPM_HPMN,
118
	CGT_MDCR_TDE_TDA,
119
	CGT_MDCR_TDE_TDOSA,
120
	CGT_MDCR_TDE_TDRA,
121
	CGT_MDCR_TDCC_TDE_TDA,
122

123
	CGT_ICH_HCR_TC_TDIR,
124

125
	/*
126
	 * Anything after this point requires a callback evaluating a
127
	 * complex trap condition. Ugly stuff.
128
	 */
129
	__COMPLEX_CONDITIONS__,
130
	CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,
131
	CGT_CNTHCTL_EL1PTEN,
132
	CGT_CNTHCTL_EL1NVPCT,
133
	CGT_CNTHCTL_EL1NVVCT,
134

135
	CGT_CPTR_TTA,
136
	CGT_MDCR_HPMN,
137

138
	/* Must be last */
139
	__NR_CGT_GROUP_IDS__
140
};
141

142
static const struct trap_bits coarse_trap_bits[] = {
143
	[CGT_HCR_TID1] = {
144
		.index		= HCR_EL2,
145
		.value 		= HCR_TID1,
146
		.mask		= HCR_TID1,
147
		.behaviour	= BEHAVE_FORWARD_READ,
148
	},
149
	[CGT_HCR_TID2] = {
150
		.index		= HCR_EL2,
151
		.value 		= HCR_TID2,
152
		.mask		= HCR_TID2,
153
		.behaviour	= BEHAVE_FORWARD_RW,
154
	},
155
	[CGT_HCR_TID3] = {
156
		.index		= HCR_EL2,
157
		.value 		= HCR_TID3,
158
		.mask		= HCR_TID3,
159
		.behaviour	= BEHAVE_FORWARD_READ,
160
	},
161
	[CGT_HCR_IMO] = {
162
		.index		= HCR_EL2,
163
		.value 		= HCR_IMO,
164
		.mask		= HCR_IMO,
165
		.behaviour	= BEHAVE_FORWARD_WRITE,
166
	},
167
	[CGT_HCR_FMO] = {
168
		.index		= HCR_EL2,
169
		.value 		= HCR_FMO,
170
		.mask		= HCR_FMO,
171
		.behaviour	= BEHAVE_FORWARD_WRITE,
172
	},
173
	[CGT_HCR_TIDCP] = {
174
		.index		= HCR_EL2,
175
		.value		= HCR_TIDCP,
176
		.mask		= HCR_TIDCP,
177
		.behaviour	= BEHAVE_FORWARD_RW,
178
	},
179
	[CGT_HCR_TACR] = {
180
		.index		= HCR_EL2,
181
		.value		= HCR_TACR,
182
		.mask		= HCR_TACR,
183
		.behaviour	= BEHAVE_FORWARD_RW,
184
	},
185
	[CGT_HCR_TSW] = {
186
		.index		= HCR_EL2,
187
		.value		= HCR_TSW,
188
		.mask		= HCR_TSW,
189
		.behaviour	= BEHAVE_FORWARD_RW,
190
	},
191
	[CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */
192
		.index		= HCR_EL2,
193
		.value		= HCR_TPC,
194
		.mask		= HCR_TPC,
195
		.behaviour	= BEHAVE_FORWARD_RW,
196
	},
197
	[CGT_HCR_TPU] = {
198
		.index		= HCR_EL2,
199
		.value		= HCR_TPU,
200
		.mask		= HCR_TPU,
201
		.behaviour	= BEHAVE_FORWARD_RW,
202
	},
203
	[CGT_HCR_TTLB] = {
204
		.index		= HCR_EL2,
205
		.value		= HCR_TTLB,
206
		.mask		= HCR_TTLB,
207
		.behaviour	= BEHAVE_FORWARD_RW,
208
	},
209
	[CGT_HCR_TVM] = {
210
		.index		= HCR_EL2,
211
		.value		= HCR_TVM,
212
		.mask		= HCR_TVM,
213
		.behaviour	= BEHAVE_FORWARD_WRITE,
214
	},
215
	[CGT_HCR_TDZ] = {
216
		.index		= HCR_EL2,
217
		.value		= HCR_TDZ,
218
		.mask		= HCR_TDZ,
219
		.behaviour	= BEHAVE_FORWARD_RW,
220
	},
221
	[CGT_HCR_TRVM] = {
222
		.index		= HCR_EL2,
223
		.value		= HCR_TRVM,
224
		.mask		= HCR_TRVM,
225
		.behaviour	= BEHAVE_FORWARD_READ,
226
	},
227
	[CGT_HCR_TLOR] = {
228
		.index		= HCR_EL2,
229
		.value		= HCR_TLOR,
230
		.mask		= HCR_TLOR,
231
		.behaviour	= BEHAVE_FORWARD_RW,
232
	},
233
	[CGT_HCR_TERR] = {
234
		.index		= HCR_EL2,
235
		.value		= HCR_TERR,
236
		.mask		= HCR_TERR,
237
		.behaviour	= BEHAVE_FORWARD_RW,
238
	},
239
	[CGT_HCR_APK] = {
240
		.index		= HCR_EL2,
241
		.value		= 0,
242
		.mask		= HCR_APK,
243
		.behaviour	= BEHAVE_FORWARD_RW,
244
	},
245
	[CGT_HCR_NV] = {
246
		.index		= HCR_EL2,
247
		.value		= HCR_NV,
248
		.mask		= HCR_NV,
249
		.behaviour	= BEHAVE_FORWARD_RW,
250
	},
251
	[CGT_HCR_NV_nNV2] = {
252
		.index		= HCR_EL2,
253
		.value		= HCR_NV,
254
		.mask		= HCR_NV | HCR_NV2,
255
		.behaviour	= BEHAVE_FORWARD_RW,
256
	},
257
	[CGT_HCR_NV1_nNV2] = {
258
		.index		= HCR_EL2,
259
		.value		= HCR_NV | HCR_NV1,
260
		.mask		= HCR_NV | HCR_NV1 | HCR_NV2,
261
		.behaviour	= BEHAVE_FORWARD_RW,
262
	},
263
	[CGT_HCR_AT] = {
264
		.index		= HCR_EL2,
265
		.value		= HCR_AT,
266
		.mask		= HCR_AT,
267
		.behaviour	= BEHAVE_FORWARD_RW,
268
	},
269
	[CGT_HCR_nFIEN] = {
270
		.index		= HCR_EL2,
271
		.value		= 0,
272
		.mask		= HCR_FIEN,
273
		.behaviour	= BEHAVE_FORWARD_RW,
274
	},
275
	[CGT_HCR_TID4] = {
276
		.index		= HCR_EL2,
277
		.value 		= HCR_TID4,
278
		.mask		= HCR_TID4,
279
		.behaviour	= BEHAVE_FORWARD_RW,
280
	},
281
	[CGT_HCR_TICAB] = {
282
		.index		= HCR_EL2,
283
		.value 		= HCR_TICAB,
284
		.mask		= HCR_TICAB,
285
		.behaviour	= BEHAVE_FORWARD_RW,
286
	},
287
	[CGT_HCR_TOCU] = {
288
		.index		= HCR_EL2,
289
		.value 		= HCR_TOCU,
290
		.mask		= HCR_TOCU,
291
		.behaviour	= BEHAVE_FORWARD_RW,
292
	},
293
	[CGT_HCR_ENSCXT] = {
294
		.index		= HCR_EL2,
295
		.value 		= 0,
296
		.mask		= HCR_ENSCXT,
297
		.behaviour	= BEHAVE_FORWARD_RW,
298
	},
299
	[CGT_HCR_TTLBIS] = {
300
		.index		= HCR_EL2,
301
		.value		= HCR_TTLBIS,
302
		.mask		= HCR_TTLBIS,
303
		.behaviour	= BEHAVE_FORWARD_RW,
304
	},
305
	[CGT_HCR_TTLBOS] = {
306
		.index		= HCR_EL2,
307
		.value		= HCR_TTLBOS,
308
		.mask		= HCR_TTLBOS,
309
		.behaviour	= BEHAVE_FORWARD_RW,
310
	},
311
	[CGT_MDCR_TPMCR] = {
312
		.index		= MDCR_EL2,
313
		.value		= MDCR_EL2_TPMCR,
314
		.mask		= MDCR_EL2_TPMCR,
315
		.behaviour	= BEHAVE_FORWARD_RW |
316
				  BEHAVE_FORWARD_IN_HOST_EL0,
317
	},
318
	[CGT_MDCR_TPM] = {
319
		.index		= MDCR_EL2,
320
		.value		= MDCR_EL2_TPM,
321
		.mask		= MDCR_EL2_TPM,
322
		.behaviour	= BEHAVE_FORWARD_RW |
323
				  BEHAVE_FORWARD_IN_HOST_EL0,
324
	},
325
	[CGT_MDCR_TDE] = {
326
		.index		= MDCR_EL2,
327
		.value		= MDCR_EL2_TDE,
328
		.mask		= MDCR_EL2_TDE,
329
		.behaviour	= BEHAVE_FORWARD_RW,
330
	},
331
	[CGT_MDCR_TDA] = {
332
		.index		= MDCR_EL2,
333
		.value		= MDCR_EL2_TDA,
334
		.mask		= MDCR_EL2_TDA,
335
		.behaviour	= BEHAVE_FORWARD_RW,
336
	},
337
	[CGT_MDCR_TDOSA] = {
338
		.index		= MDCR_EL2,
339
		.value		= MDCR_EL2_TDOSA,
340
		.mask		= MDCR_EL2_TDOSA,
341
		.behaviour	= BEHAVE_FORWARD_RW,
342
	},
343
	[CGT_MDCR_TDRA] = {
344
		.index		= MDCR_EL2,
345
		.value		= MDCR_EL2_TDRA,
346
		.mask		= MDCR_EL2_TDRA,
347
		.behaviour	= BEHAVE_FORWARD_RW,
348
	},
349
	[CGT_MDCR_E2PB] = {
350
		.index		= MDCR_EL2,
351
		.value		= 0,
352
		.mask		= BIT(MDCR_EL2_E2PB_SHIFT),
353
		.behaviour	= BEHAVE_FORWARD_RW,
354
	},
355
	[CGT_MDCR_TPMS] = {
356
		.index		= MDCR_EL2,
357
		.value		= MDCR_EL2_TPMS,
358
		.mask		= MDCR_EL2_TPMS,
359
		.behaviour	= BEHAVE_FORWARD_RW,
360
	},
361
	[CGT_MDCR_TTRF] = {
362
		.index		= MDCR_EL2,
363
		.value		= MDCR_EL2_TTRF,
364
		.mask		= MDCR_EL2_TTRF,
365
		.behaviour	= BEHAVE_FORWARD_RW,
366
	},
367
	[CGT_MDCR_E2TB] = {
368
		.index		= MDCR_EL2,
369
		.value		= 0,
370
		.mask		= BIT(MDCR_EL2_E2TB_SHIFT),
371
		.behaviour	= BEHAVE_FORWARD_RW,
372
	},
373
	[CGT_MDCR_TDCC] = {
374
		.index		= MDCR_EL2,
375
		.value		= MDCR_EL2_TDCC,
376
		.mask		= MDCR_EL2_TDCC,
377
		.behaviour	= BEHAVE_FORWARD_RW,
378
	},
379
	[CGT_CPTR_TAM] = {
380
		.index		= CPTR_EL2,
381
		.value		= CPTR_EL2_TAM,
382
		.mask		= CPTR_EL2_TAM,
383
		.behaviour	= BEHAVE_FORWARD_RW,
384
	},
385
	[CGT_CPTR_TCPAC] = {
386
		.index		= CPTR_EL2,
387
		.value		= CPTR_EL2_TCPAC,
388
		.mask		= CPTR_EL2_TCPAC,
389
		.behaviour	= BEHAVE_FORWARD_RW,
390
	},
391
	[CGT_HCRX_EnFPM] = {
392
		.index		= HCRX_EL2,
393
		.value 		= 0,
394
		.mask		= HCRX_EL2_EnFPM,
395
		.behaviour	= BEHAVE_FORWARD_RW,
396
	},
397
	[CGT_HCRX_TCR2En] = {
398
		.index		= HCRX_EL2,
399
		.value 		= 0,
400
		.mask		= HCRX_EL2_TCR2En,
401
		.behaviour	= BEHAVE_FORWARD_RW,
402
	},
403
	[CGT_HCRX_SCTLR2En] = {
404
		.index		= HCRX_EL2,
405
		.value		= 0,
406
		.mask		= HCRX_EL2_SCTLR2En,
407
		.behaviour	= BEHAVE_FORWARD_RW,
408
	},
409
	[CGT_CNTHCTL_EL1TVT] = {
410
		.index		= CNTHCTL_EL2,
411
		.value		= CNTHCTL_EL1TVT,
412
		.mask		= CNTHCTL_EL1TVT,
413
		.behaviour	= BEHAVE_FORWARD_RW,
414
	},
415
	[CGT_CNTHCTL_EL1TVCT] = {
416
		.index		= CNTHCTL_EL2,
417
		.value		= CNTHCTL_EL1TVCT,
418
		.mask		= CNTHCTL_EL1TVCT,
419
		.behaviour	= BEHAVE_FORWARD_READ,
420
	},
421
	[CGT_ICH_HCR_TC] = {
422
		.index		= ICH_HCR_EL2,
423
		.value		= ICH_HCR_EL2_TC,
424
		.mask		= ICH_HCR_EL2_TC,
425
		.behaviour	= BEHAVE_FORWARD_RW,
426
	},
427
	[CGT_ICH_HCR_TALL0] = {
428
		.index		= ICH_HCR_EL2,
429
		.value		= ICH_HCR_EL2_TALL0,
430
		.mask		= ICH_HCR_EL2_TALL0,
431
		.behaviour	= BEHAVE_FORWARD_RW,
432
	},
433
	[CGT_ICH_HCR_TALL1] = {
434
		.index		= ICH_HCR_EL2,
435
		.value		= ICH_HCR_EL2_TALL1,
436
		.mask		= ICH_HCR_EL2_TALL1,
437
		.behaviour	= BEHAVE_FORWARD_RW,
438
	},
439
	[CGT_ICH_HCR_TDIR] = {
440
		.index		= ICH_HCR_EL2,
441
		.value		= ICH_HCR_EL2_TDIR,
442
		.mask		= ICH_HCR_EL2_TDIR,
443
		.behaviour	= BEHAVE_FORWARD_RW,
444
	},
445
};
446

447
#define MCB(id, ...)						\
448
	[id - __MULTIPLE_CONTROL_BITS__]	=		\
449
		(const enum cgt_group_id[]){			\
450
		__VA_ARGS__, __RESERVED__			\
451
		}
452

453
static const enum cgt_group_id *coarse_control_combo[] = {
454
	MCB(CGT_HCR_TID2_TID4,		CGT_HCR_TID2, CGT_HCR_TID4),
455
	MCB(CGT_HCR_TTLB_TTLBIS,	CGT_HCR_TTLB, CGT_HCR_TTLBIS),
456
	MCB(CGT_HCR_TTLB_TTLBOS,	CGT_HCR_TTLB, CGT_HCR_TTLBOS),
457
	MCB(CGT_HCR_TVM_TRVM,		CGT_HCR_TVM, CGT_HCR_TRVM),
458
	MCB(CGT_HCR_TVM_TRVM_HCRX_TCR2En,
459
					CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_TCR2En),
460
	MCB(CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
461
					CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_SCTLR2En),
462
	MCB(CGT_HCR_TPU_TICAB,		CGT_HCR_TPU, CGT_HCR_TICAB),
463
	MCB(CGT_HCR_TPU_TOCU,		CGT_HCR_TPU, CGT_HCR_TOCU),
464
	MCB(CGT_HCR_NV1_nNV2_ENSCXT,	CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
465
	MCB(CGT_MDCR_TPM_TPMCR,		CGT_MDCR_TPM, CGT_MDCR_TPMCR),
466
	MCB(CGT_MDCR_TPM_HPMN,		CGT_MDCR_TPM, CGT_MDCR_HPMN),
467
	MCB(CGT_MDCR_TDE_TDA,		CGT_MDCR_TDE, CGT_MDCR_TDA),
468
	MCB(CGT_MDCR_TDE_TDOSA,		CGT_MDCR_TDE, CGT_MDCR_TDOSA),
469
	MCB(CGT_MDCR_TDE_TDRA,		CGT_MDCR_TDE, CGT_MDCR_TDRA),
470
	MCB(CGT_MDCR_TDCC_TDE_TDA,	CGT_MDCR_TDCC, CGT_MDCR_TDE, CGT_MDCR_TDA),
471

472
	MCB(CGT_HCR_IMO_FMO_ICH_HCR_TC,	CGT_HCR_IMO, CGT_HCR_FMO, CGT_ICH_HCR_TC),
473
	MCB(CGT_ICH_HCR_TC_TDIR,	CGT_ICH_HCR_TC, CGT_ICH_HCR_TDIR),
474
};
475

476
typedef enum trap_behaviour (*complex_condition_check)(struct kvm_vcpu *);
477

478
/*
479
 * Warning, maximum confusion ahead.
480
 *
481
 * When E2H=0, CNTHCTL_EL2[1:0] are defined as EL1PCEN:EL1PCTEN
482
 * When E2H=1, CNTHCTL_EL2[11:10] are defined as EL1PTEN:EL1PCTEN
483
 *
484
 * Note the single letter difference? Yet, the bits have the same
485
 * function despite a different layout and a different name.
486
 *
487
 * We don't try to reconcile this mess. We just use the E2H=0 bits
488
 * to generate something that is in the E2H=1 format, and live with
489
 * it. You're welcome.
490
 */
491
static u64 get_sanitized_cnthctl(struct kvm_vcpu *vcpu)
492
{
493
	u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
494

495
	if (!vcpu_el2_e2h_is_set(vcpu))
496
		val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
497

498
	return val & ((CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN) << 10);
499
}
500

501
static enum trap_behaviour check_cnthctl_el1pcten(struct kvm_vcpu *vcpu)
502
{
503
	if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCTEN << 10))
504
		return BEHAVE_HANDLE_LOCALLY;
505

506
	return BEHAVE_FORWARD_RW;
507
}
508

509
static enum trap_behaviour check_cnthctl_el1pten(struct kvm_vcpu *vcpu)
510
{
511
	if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCEN << 10))
512
		return BEHAVE_HANDLE_LOCALLY;
513

514
	return BEHAVE_FORWARD_RW;
515
}
516

517
static bool is_nested_nv2_guest(struct kvm_vcpu *vcpu)
518
{
519
	u64 val;
520

521
	val = __vcpu_sys_reg(vcpu, HCR_EL2);
522
	return ((val & (HCR_E2H | HCR_TGE | HCR_NV2 | HCR_NV1 | HCR_NV)) == (HCR_E2H | HCR_NV2 | HCR_NV));
523
}
524

525
static enum trap_behaviour check_cnthctl_el1nvpct(struct kvm_vcpu *vcpu)
526
{
527
	if (!is_nested_nv2_guest(vcpu) ||
528
	    !(__vcpu_sys_reg(vcpu, CNTHCTL_EL2) & CNTHCTL_EL1NVPCT))
529
		return BEHAVE_HANDLE_LOCALLY;
530

531
	return BEHAVE_FORWARD_RW;
532
}
533

534
static enum trap_behaviour check_cnthctl_el1nvvct(struct kvm_vcpu *vcpu)
535
{
536
	if (!is_nested_nv2_guest(vcpu) ||
537
	    !(__vcpu_sys_reg(vcpu, CNTHCTL_EL2) & CNTHCTL_EL1NVVCT))
538
		return BEHAVE_HANDLE_LOCALLY;
539

540
	return BEHAVE_FORWARD_RW;
541
}
542

543
static enum trap_behaviour check_cptr_tta(struct kvm_vcpu *vcpu)
544
{
545
	u64 val = __vcpu_sys_reg(vcpu, CPTR_EL2);
546

547
	if (!vcpu_el2_e2h_is_set(vcpu))
548
		val = translate_cptr_el2_to_cpacr_el1(val);
549

550
	if (val & CPACR_EL1_TTA)
551
		return BEHAVE_FORWARD_RW;
552

553
	return BEHAVE_HANDLE_LOCALLY;
554
}
555

556
static enum trap_behaviour check_mdcr_hpmn(struct kvm_vcpu *vcpu)
557
{
558
	u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));
559
	unsigned int idx;
560

561

562
	switch (sysreg) {
563
	case SYS_PMEVTYPERn_EL0(0) ... SYS_PMEVTYPERn_EL0(30):
564
	case SYS_PMEVCNTRn_EL0(0) ... SYS_PMEVCNTRn_EL0(30):
565
		idx = (sys_reg_CRm(sysreg) & 0x3) << 3 | sys_reg_Op2(sysreg);
566
		break;
567
	case SYS_PMXEVTYPER_EL0:
568
	case SYS_PMXEVCNTR_EL0:
569
		idx = SYS_FIELD_GET(PMSELR_EL0, SEL,
570
				    __vcpu_sys_reg(vcpu, PMSELR_EL0));
571
		break;
572
	default:
573
		/* Someone used this trap helper for something else... */
574
		KVM_BUG_ON(1, vcpu->kvm);
575
		return BEHAVE_HANDLE_LOCALLY;
576
	}
577

578
	if (kvm_pmu_counter_is_hyp(vcpu, idx))
579
		return BEHAVE_FORWARD_RW | BEHAVE_FORWARD_IN_HOST_EL0;
580

581
	return BEHAVE_HANDLE_LOCALLY;
582
}
583

584
#define CCC(id, fn)				\
585
	[id - __COMPLEX_CONDITIONS__] = fn
586

587
static const complex_condition_check ccc[] = {
588
	CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),
589
	CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),
590
	CCC(CGT_CNTHCTL_EL1NVPCT, check_cnthctl_el1nvpct),
591
	CCC(CGT_CNTHCTL_EL1NVVCT, check_cnthctl_el1nvvct),
592
	CCC(CGT_CPTR_TTA, check_cptr_tta),
593
	CCC(CGT_MDCR_HPMN, check_mdcr_hpmn),
594
};
595

596
/*
597
 * Bit assignment for the trap controls. We use a 64bit word with the
598
 * following layout for each trapped sysreg:
599
 *
600
 * [9:0]	enum cgt_group_id (10 bits)
601
 * [13:10]	enum fgt_group_id (4 bits)
602
 * [19:14]	bit number in the FGT register (6 bits)
603
 * [20]		trap polarity (1 bit)
604
 * [25:21]	FG filter (5 bits)
605
 * [35:26]	Main SysReg table index (10 bits)
606
 * [62:36]	Unused (27 bits)
607
 * [63]		RES0 - Must be zero, as lost on insertion in the xarray
608
 */
609
#define TC_CGT_BITS	10
610
#define TC_FGT_BITS	4
611
#define TC_FGF_BITS	5
612
#define TC_SRI_BITS	10
613

614
union trap_config {
615
	u64	val;
616
	struct {
617
		unsigned long	cgt:TC_CGT_BITS; /* Coarse Grained Trap id */
618
		unsigned long	fgt:TC_FGT_BITS; /* Fine Grained Trap id */
619
		unsigned long	bit:6;		 /* Bit number */
620
		unsigned long	pol:1;		 /* Polarity */
621
		unsigned long	fgf:TC_FGF_BITS; /* Fine Grained Filter */
622
		unsigned long	sri:TC_SRI_BITS; /* SysReg Index */
623
		unsigned long	unused:27;	 /* Unused, should be zero */
624
		unsigned long	mbz:1;		 /* Must Be Zero */
625
	};
626
};
627

628
struct encoding_to_trap_config {
629
	const u32			encoding;
630
	const u32			end;
631
	const union trap_config		tc;
632
	const unsigned int		line;
633
};
634

635
/*
636
 * WARNING: using ranges is a treacherous endeavour, as sysregs that
637
 * are part of an architectural range are not necessarily contiguous
638
 * in the [Op0,Op1,CRn,CRm,Ops] space. Tread carefully.
639
 */
640
#define SR_RANGE_TRAP(sr_start, sr_end, trap_id)			\
641
	{								\
642
		.encoding	= sr_start,				\
643
		.end		= sr_end,				\
644
		.tc		= {					\
645
			.cgt		= trap_id,			\
646
		},							\
647
		.line = __LINE__,					\
648
	}
649

650
#define SR_TRAP(sr, trap_id)		SR_RANGE_TRAP(sr, sr, trap_id)
651

652
/*
653
 * Map encoding to trap bits for exception reported with EC=0x18.
654
 * These must only be evaluated when running a nested hypervisor, but
655
 * that the current context is not a hypervisor context. When the
656
 * trapped access matches one of the trap controls, the exception is
657
 * re-injected in the nested hypervisor.
658
 */
659
static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
660
	SR_TRAP(SYS_REVIDR_EL1,		CGT_HCR_TID1),
661
	SR_TRAP(SYS_AIDR_EL1,		CGT_HCR_TID1),
662
	SR_TRAP(SYS_SMIDR_EL1,		CGT_HCR_TID1),
663
	SR_TRAP(SYS_CTR_EL0,		CGT_HCR_TID2),
664
	SR_TRAP(SYS_CCSIDR_EL1,		CGT_HCR_TID2_TID4),
665
	SR_TRAP(SYS_CCSIDR2_EL1,	CGT_HCR_TID2_TID4),
666
	SR_TRAP(SYS_CLIDR_EL1,		CGT_HCR_TID2_TID4),
667
	SR_TRAP(SYS_CSSELR_EL1,		CGT_HCR_TID2_TID4),
668
	SR_RANGE_TRAP(SYS_ID_PFR0_EL1,
669
		      sys_reg(3, 0, 0, 7, 7), CGT_HCR_TID3),
670
	SR_TRAP(SYS_ICC_SGI0R_EL1,	CGT_HCR_IMO_FMO_ICH_HCR_TC),
671
	SR_TRAP(SYS_ICC_ASGI1R_EL1,	CGT_HCR_IMO_FMO_ICH_HCR_TC),
672
	SR_TRAP(SYS_ICC_SGI1R_EL1,	CGT_HCR_IMO_FMO_ICH_HCR_TC),
673
	SR_RANGE_TRAP(sys_reg(3, 0, 11, 0, 0),
674
		      sys_reg(3, 0, 11, 15, 7), CGT_HCR_TIDCP),
675
	SR_RANGE_TRAP(sys_reg(3, 1, 11, 0, 0),
676
		      sys_reg(3, 1, 11, 15, 7), CGT_HCR_TIDCP),
677
	SR_RANGE_TRAP(sys_reg(3, 2, 11, 0, 0),
678
		      sys_reg(3, 2, 11, 15, 7), CGT_HCR_TIDCP),
679
	SR_RANGE_TRAP(sys_reg(3, 3, 11, 0, 0),
680
		      sys_reg(3, 3, 11, 15, 7), CGT_HCR_TIDCP),
681
	SR_RANGE_TRAP(sys_reg(3, 4, 11, 0, 0),
682
		      sys_reg(3, 4, 11, 15, 7), CGT_HCR_TIDCP),
683
	SR_RANGE_TRAP(sys_reg(3, 5, 11, 0, 0),
684
		      sys_reg(3, 5, 11, 15, 7), CGT_HCR_TIDCP),
685
	SR_RANGE_TRAP(sys_reg(3, 6, 11, 0, 0),
686
		      sys_reg(3, 6, 11, 15, 7), CGT_HCR_TIDCP),
687
	SR_RANGE_TRAP(sys_reg(3, 7, 11, 0, 0),
688
		      sys_reg(3, 7, 11, 15, 7), CGT_HCR_TIDCP),
689
	SR_RANGE_TRAP(sys_reg(3, 0, 15, 0, 0),
690
		      sys_reg(3, 0, 15, 15, 7), CGT_HCR_TIDCP),
691
	SR_RANGE_TRAP(sys_reg(3, 1, 15, 0, 0),
692
		      sys_reg(3, 1, 15, 15, 7), CGT_HCR_TIDCP),
693
	SR_RANGE_TRAP(sys_reg(3, 2, 15, 0, 0),
694
		      sys_reg(3, 2, 15, 15, 7), CGT_HCR_TIDCP),
695
	SR_RANGE_TRAP(sys_reg(3, 3, 15, 0, 0),
696
		      sys_reg(3, 3, 15, 15, 7), CGT_HCR_TIDCP),
697
	SR_RANGE_TRAP(sys_reg(3, 4, 15, 0, 0),
698
		      sys_reg(3, 4, 15, 15, 7), CGT_HCR_TIDCP),
699
	SR_RANGE_TRAP(sys_reg(3, 5, 15, 0, 0),
700
		      sys_reg(3, 5, 15, 15, 7), CGT_HCR_TIDCP),
701
	SR_RANGE_TRAP(sys_reg(3, 6, 15, 0, 0),
702
		      sys_reg(3, 6, 15, 15, 7), CGT_HCR_TIDCP),
703
	SR_RANGE_TRAP(sys_reg(3, 7, 15, 0, 0),
704
		      sys_reg(3, 7, 15, 15, 7), CGT_HCR_TIDCP),
705
	SR_TRAP(SYS_ACTLR_EL1,		CGT_HCR_TACR),
706
	SR_TRAP(SYS_DC_ISW,		CGT_HCR_TSW),
707
	SR_TRAP(SYS_DC_CSW,		CGT_HCR_TSW),
708
	SR_TRAP(SYS_DC_CISW,		CGT_HCR_TSW),
709
	SR_TRAP(SYS_DC_IGSW,		CGT_HCR_TSW),
710
	SR_TRAP(SYS_DC_IGDSW,		CGT_HCR_TSW),
711
	SR_TRAP(SYS_DC_CGSW,		CGT_HCR_TSW),
712
	SR_TRAP(SYS_DC_CGDSW,		CGT_HCR_TSW),
713
	SR_TRAP(SYS_DC_CIGSW,		CGT_HCR_TSW),
714
	SR_TRAP(SYS_DC_CIGDSW,		CGT_HCR_TSW),
715
	SR_TRAP(SYS_DC_CIVAC,		CGT_HCR_TPC),
716
	SR_TRAP(SYS_DC_CVAC,		CGT_HCR_TPC),
717
	SR_TRAP(SYS_DC_CVAP,		CGT_HCR_TPC),
718
	SR_TRAP(SYS_DC_CVADP,		CGT_HCR_TPC),
719
	SR_TRAP(SYS_DC_IVAC,		CGT_HCR_TPC),
720
	SR_TRAP(SYS_DC_CIGVAC,		CGT_HCR_TPC),
721
	SR_TRAP(SYS_DC_CIGDVAC,		CGT_HCR_TPC),
722
	SR_TRAP(SYS_DC_IGVAC,		CGT_HCR_TPC),
723
	SR_TRAP(SYS_DC_IGDVAC,		CGT_HCR_TPC),
724
	SR_TRAP(SYS_DC_CGVAC,		CGT_HCR_TPC),
725
	SR_TRAP(SYS_DC_CGDVAC,		CGT_HCR_TPC),
726
	SR_TRAP(SYS_DC_CGVAP,		CGT_HCR_TPC),
727
	SR_TRAP(SYS_DC_CGDVAP,		CGT_HCR_TPC),
728
	SR_TRAP(SYS_DC_CGVADP,		CGT_HCR_TPC),
729
	SR_TRAP(SYS_DC_CGDVADP,		CGT_HCR_TPC),
730
	SR_TRAP(SYS_IC_IVAU,		CGT_HCR_TPU_TOCU),
731
	SR_TRAP(SYS_IC_IALLU,		CGT_HCR_TPU_TOCU),
732
	SR_TRAP(SYS_IC_IALLUIS,		CGT_HCR_TPU_TICAB),
733
	SR_TRAP(SYS_DC_CVAU,		CGT_HCR_TPU_TOCU),
734
	SR_TRAP(OP_TLBI_RVAE1,		CGT_HCR_TTLB),
735
	SR_TRAP(OP_TLBI_RVAAE1,		CGT_HCR_TTLB),
736
	SR_TRAP(OP_TLBI_RVALE1,		CGT_HCR_TTLB),
737
	SR_TRAP(OP_TLBI_RVAALE1,	CGT_HCR_TTLB),
738
	SR_TRAP(OP_TLBI_VMALLE1,	CGT_HCR_TTLB),
739
	SR_TRAP(OP_TLBI_VAE1,		CGT_HCR_TTLB),
740
	SR_TRAP(OP_TLBI_ASIDE1,		CGT_HCR_TTLB),
741
	SR_TRAP(OP_TLBI_VAAE1,		CGT_HCR_TTLB),
742
	SR_TRAP(OP_TLBI_VALE1,		CGT_HCR_TTLB),
743
	SR_TRAP(OP_TLBI_VAALE1,		CGT_HCR_TTLB),
744
	SR_TRAP(OP_TLBI_RVAE1NXS,	CGT_HCR_TTLB),
745
	SR_TRAP(OP_TLBI_RVAAE1NXS,	CGT_HCR_TTLB),
746
	SR_TRAP(OP_TLBI_RVALE1NXS,	CGT_HCR_TTLB),
747
	SR_TRAP(OP_TLBI_RVAALE1NXS,	CGT_HCR_TTLB),
748
	SR_TRAP(OP_TLBI_VMALLE1NXS,	CGT_HCR_TTLB),
749
	SR_TRAP(OP_TLBI_VAE1NXS,	CGT_HCR_TTLB),
750
	SR_TRAP(OP_TLBI_ASIDE1NXS,	CGT_HCR_TTLB),
751
	SR_TRAP(OP_TLBI_VAAE1NXS,	CGT_HCR_TTLB),
752
	SR_TRAP(OP_TLBI_VALE1NXS,	CGT_HCR_TTLB),
753
	SR_TRAP(OP_TLBI_VAALE1NXS,	CGT_HCR_TTLB),
754
	SR_TRAP(OP_TLBI_RVAE1IS,	CGT_HCR_TTLB_TTLBIS),
755
	SR_TRAP(OP_TLBI_RVAAE1IS,	CGT_HCR_TTLB_TTLBIS),
756
	SR_TRAP(OP_TLBI_RVALE1IS,	CGT_HCR_TTLB_TTLBIS),
757
	SR_TRAP(OP_TLBI_RVAALE1IS,	CGT_HCR_TTLB_TTLBIS),
758
	SR_TRAP(OP_TLBI_VMALLE1IS,	CGT_HCR_TTLB_TTLBIS),
759
	SR_TRAP(OP_TLBI_VAE1IS,		CGT_HCR_TTLB_TTLBIS),
760
	SR_TRAP(OP_TLBI_ASIDE1IS,	CGT_HCR_TTLB_TTLBIS),
761
	SR_TRAP(OP_TLBI_VAAE1IS,	CGT_HCR_TTLB_TTLBIS),
762
	SR_TRAP(OP_TLBI_VALE1IS,	CGT_HCR_TTLB_TTLBIS),
763
	SR_TRAP(OP_TLBI_VAALE1IS,	CGT_HCR_TTLB_TTLBIS),
764
	SR_TRAP(OP_TLBI_RVAE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
765
	SR_TRAP(OP_TLBI_RVAAE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
766
	SR_TRAP(OP_TLBI_RVALE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
767
	SR_TRAP(OP_TLBI_RVAALE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
768
	SR_TRAP(OP_TLBI_VMALLE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
769
	SR_TRAP(OP_TLBI_VAE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
770
	SR_TRAP(OP_TLBI_ASIDE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
771
	SR_TRAP(OP_TLBI_VAAE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
772
	SR_TRAP(OP_TLBI_VALE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
773
	SR_TRAP(OP_TLBI_VAALE1ISNXS,	CGT_HCR_TTLB_TTLBIS),
774
	SR_TRAP(OP_TLBI_VMALLE1OS,	CGT_HCR_TTLB_TTLBOS),
775
	SR_TRAP(OP_TLBI_VAE1OS,		CGT_HCR_TTLB_TTLBOS),
776
	SR_TRAP(OP_TLBI_ASIDE1OS,	CGT_HCR_TTLB_TTLBOS),
777
	SR_TRAP(OP_TLBI_VAAE1OS,	CGT_HCR_TTLB_TTLBOS),
778
	SR_TRAP(OP_TLBI_VALE1OS,	CGT_HCR_TTLB_TTLBOS),
779
	SR_TRAP(OP_TLBI_VAALE1OS,	CGT_HCR_TTLB_TTLBOS),
780
	SR_TRAP(OP_TLBI_RVAE1OS,	CGT_HCR_TTLB_TTLBOS),
781
	SR_TRAP(OP_TLBI_RVAAE1OS,	CGT_HCR_TTLB_TTLBOS),
782
	SR_TRAP(OP_TLBI_RVALE1OS,	CGT_HCR_TTLB_TTLBOS),
783
	SR_TRAP(OP_TLBI_RVAALE1OS,	CGT_HCR_TTLB_TTLBOS),
784
	SR_TRAP(OP_TLBI_VMALLE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
785
	SR_TRAP(OP_TLBI_VAE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
786
	SR_TRAP(OP_TLBI_ASIDE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
787
	SR_TRAP(OP_TLBI_VAAE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
788
	SR_TRAP(OP_TLBI_VALE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
789
	SR_TRAP(OP_TLBI_VAALE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
790
	SR_TRAP(OP_TLBI_RVAE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
791
	SR_TRAP(OP_TLBI_RVAAE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
792
	SR_TRAP(OP_TLBI_RVALE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
793
	SR_TRAP(OP_TLBI_RVAALE1OSNXS,	CGT_HCR_TTLB_TTLBOS),
794
	SR_TRAP(SYS_SCTLR_EL1,		CGT_HCR_TVM_TRVM),
795
	SR_TRAP(SYS_SCTLR2_EL1,		CGT_HCR_TVM_TRVM_HCRX_SCTLR2En),
796
	SR_TRAP(SYS_TTBR0_EL1,		CGT_HCR_TVM_TRVM),
797
	SR_TRAP(SYS_TTBR1_EL1,		CGT_HCR_TVM_TRVM),
798
	SR_TRAP(SYS_TCR_EL1,		CGT_HCR_TVM_TRVM),
799
	SR_TRAP(SYS_ESR_EL1,		CGT_HCR_TVM_TRVM),
800
	SR_TRAP(SYS_FAR_EL1,		CGT_HCR_TVM_TRVM),
801
	SR_TRAP(SYS_AFSR0_EL1,		CGT_HCR_TVM_TRVM),
802
	SR_TRAP(SYS_AFSR1_EL1,		CGT_HCR_TVM_TRVM),
803
	SR_TRAP(SYS_MAIR_EL1,		CGT_HCR_TVM_TRVM),
804
	SR_TRAP(SYS_AMAIR_EL1,		CGT_HCR_TVM_TRVM),
805
	SR_TRAP(SYS_CONTEXTIDR_EL1,	CGT_HCR_TVM_TRVM),
806
	SR_TRAP(SYS_PIR_EL1,		CGT_HCR_TVM_TRVM),
807
	SR_TRAP(SYS_PIRE0_EL1,		CGT_HCR_TVM_TRVM),
808
	SR_TRAP(SYS_POR_EL0,		CGT_HCR_TVM_TRVM),
809
	SR_TRAP(SYS_POR_EL1,		CGT_HCR_TVM_TRVM),
810
	SR_TRAP(SYS_TCR2_EL1,		CGT_HCR_TVM_TRVM_HCRX_TCR2En),
811
	SR_TRAP(SYS_DC_ZVA,		CGT_HCR_TDZ),
812
	SR_TRAP(SYS_DC_GVA,		CGT_HCR_TDZ),
813
	SR_TRAP(SYS_DC_GZVA,		CGT_HCR_TDZ),
814
	SR_TRAP(SYS_LORSA_EL1,		CGT_HCR_TLOR),
815
	SR_TRAP(SYS_LOREA_EL1, 		CGT_HCR_TLOR),
816
	SR_TRAP(SYS_LORN_EL1, 		CGT_HCR_TLOR),
817
	SR_TRAP(SYS_LORC_EL1, 		CGT_HCR_TLOR),
818
	SR_TRAP(SYS_LORID_EL1,		CGT_HCR_TLOR),
819
	SR_TRAP(SYS_ERRIDR_EL1,		CGT_HCR_TERR),
820
	SR_TRAP(SYS_ERRSELR_EL1,	CGT_HCR_TERR),
821
	SR_TRAP(SYS_ERXADDR_EL1,	CGT_HCR_TERR),
822
	SR_TRAP(SYS_ERXCTLR_EL1,	CGT_HCR_TERR),
823
	SR_TRAP(SYS_ERXFR_EL1,		CGT_HCR_TERR),
824
	SR_TRAP(SYS_ERXMISC0_EL1,	CGT_HCR_TERR),
825
	SR_TRAP(SYS_ERXMISC1_EL1,	CGT_HCR_TERR),
826
	SR_TRAP(SYS_ERXMISC2_EL1,	CGT_HCR_TERR),
827
	SR_TRAP(SYS_ERXMISC3_EL1,	CGT_HCR_TERR),
828
	SR_TRAP(SYS_ERXSTATUS_EL1,	CGT_HCR_TERR),
829
	SR_TRAP(SYS_APIAKEYLO_EL1,	CGT_HCR_APK),
830
	SR_TRAP(SYS_APIAKEYHI_EL1,	CGT_HCR_APK),
831
	SR_TRAP(SYS_APIBKEYLO_EL1,	CGT_HCR_APK),
832
	SR_TRAP(SYS_APIBKEYHI_EL1,	CGT_HCR_APK),
833
	SR_TRAP(SYS_APDAKEYLO_EL1,	CGT_HCR_APK),
834
	SR_TRAP(SYS_APDAKEYHI_EL1,	CGT_HCR_APK),
835
	SR_TRAP(SYS_APDBKEYLO_EL1,	CGT_HCR_APK),
836
	SR_TRAP(SYS_APDBKEYHI_EL1,	CGT_HCR_APK),
837
	SR_TRAP(SYS_APGAKEYLO_EL1,	CGT_HCR_APK),
838
	SR_TRAP(SYS_APGAKEYHI_EL1,	CGT_HCR_APK),
839
	/* All _EL2 registers */
840
	SR_TRAP(SYS_BRBCR_EL2,		CGT_HCR_NV),
841
	SR_TRAP(SYS_VPIDR_EL2,		CGT_HCR_NV),
842
	SR_TRAP(SYS_VMPIDR_EL2,		CGT_HCR_NV),
843
	SR_TRAP(SYS_SCTLR_EL2,		CGT_HCR_NV),
844
	SR_TRAP(SYS_ACTLR_EL2,		CGT_HCR_NV),
845
	SR_TRAP(SYS_SCTLR2_EL2,		CGT_HCR_NV),
846
	SR_RANGE_TRAP(SYS_HCR_EL2,
847
		      SYS_HCRX_EL2,	CGT_HCR_NV),
848
	SR_TRAP(SYS_SMPRIMAP_EL2,	CGT_HCR_NV),
849
	SR_TRAP(SYS_SMCR_EL2,		CGT_HCR_NV),
850
	SR_RANGE_TRAP(SYS_TTBR0_EL2,
851
		      SYS_TCR2_EL2,	CGT_HCR_NV),
852
	SR_TRAP(SYS_VTTBR_EL2,		CGT_HCR_NV),
853
	SR_TRAP(SYS_VTCR_EL2,		CGT_HCR_NV),
854
	SR_TRAP(SYS_VNCR_EL2,		CGT_HCR_NV),
855
	SR_RANGE_TRAP(SYS_HDFGRTR_EL2,
856
		      SYS_HAFGRTR_EL2,	CGT_HCR_NV),
857
	/* Skip the SP_EL1 encoding... */
858
	SR_TRAP(SYS_SPSR_EL2,		CGT_HCR_NV),
859
	SR_TRAP(SYS_ELR_EL2,		CGT_HCR_NV),
860
	/* Skip SPSR_irq, SPSR_abt, SPSR_und, SPSR_fiq */
861
	SR_TRAP(SYS_AFSR0_EL2,		CGT_HCR_NV),
862
	SR_TRAP(SYS_AFSR1_EL2,		CGT_HCR_NV),
863
	SR_TRAP(SYS_ESR_EL2,		CGT_HCR_NV),
864
	SR_TRAP(SYS_VSESR_EL2,		CGT_HCR_NV),
865
	SR_TRAP(SYS_TFSR_EL2,		CGT_HCR_NV),
866
	SR_TRAP(SYS_FAR_EL2,		CGT_HCR_NV),
867
	SR_TRAP(SYS_HPFAR_EL2,		CGT_HCR_NV),
868
	SR_TRAP(SYS_PMSCR_EL2,		CGT_HCR_NV),
869
	SR_TRAP(SYS_MAIR_EL2,		CGT_HCR_NV),
870
	SR_TRAP(SYS_AMAIR_EL2,		CGT_HCR_NV),
871
	SR_TRAP(SYS_MPAMHCR_EL2,	CGT_HCR_NV),
872
	SR_TRAP(SYS_MPAMVPMV_EL2,	CGT_HCR_NV),
873
	SR_TRAP(SYS_MPAM2_EL2,		CGT_HCR_NV),
874
	SR_RANGE_TRAP(SYS_MPAMVPM0_EL2,
875
		      SYS_MPAMVPM7_EL2,	CGT_HCR_NV),
876
	/*
877
	 * Note that the spec. describes a group of MEC registers
878
	 * whose access should not trap, therefore skip the following:
879
	 * MECID_A0_EL2, MECID_A1_EL2, MECID_P0_EL2,
880
	 * MECID_P1_EL2, MECIDR_EL2, VMECID_A_EL2,
881
	 * VMECID_P_EL2.
882
	 */
883
	SR_RANGE_TRAP(SYS_VBAR_EL2,
884
		      SYS_RMR_EL2,	CGT_HCR_NV),
885
	SR_TRAP(SYS_VDISR_EL2,		CGT_HCR_NV),
886
	/* ICH_AP0R<m>_EL2 */
887
	SR_RANGE_TRAP(SYS_ICH_AP0R0_EL2,
888
		      SYS_ICH_AP0R3_EL2, CGT_HCR_NV),
889
	/* ICH_AP1R<m>_EL2 */
890
	SR_RANGE_TRAP(SYS_ICH_AP1R0_EL2,
891
		      SYS_ICH_AP1R3_EL2, CGT_HCR_NV),
892
	SR_TRAP(SYS_ICC_SRE_EL2,	CGT_HCR_NV),
893
	SR_RANGE_TRAP(SYS_ICH_HCR_EL2,
894
		      SYS_ICH_EISR_EL2,	CGT_HCR_NV),
895
	SR_TRAP(SYS_ICH_ELRSR_EL2,	CGT_HCR_NV),
896
	SR_TRAP(SYS_ICH_VMCR_EL2,	CGT_HCR_NV),
897
	/* ICH_LR<m>_EL2 */
898
	SR_RANGE_TRAP(SYS_ICH_LR0_EL2,
899
		      SYS_ICH_LR15_EL2, CGT_HCR_NV),
900
	SR_TRAP(SYS_CONTEXTIDR_EL2,	CGT_HCR_NV),
901
	SR_TRAP(SYS_TPIDR_EL2,		CGT_HCR_NV),
902
	SR_TRAP(SYS_SCXTNUM_EL2,	CGT_HCR_NV),
903
	/* AMEVCNTVOFF0<n>_EL2, AMEVCNTVOFF1<n>_EL2  */
904
	SR_RANGE_TRAP(SYS_AMEVCNTVOFF0n_EL2(0),
905
		      SYS_AMEVCNTVOFF1n_EL2(15), CGT_HCR_NV),
906
	/* CNT*_EL2 */
907
	SR_TRAP(SYS_CNTVOFF_EL2,	CGT_HCR_NV),
908
	SR_TRAP(SYS_CNTPOFF_EL2,	CGT_HCR_NV),
909
	SR_TRAP(SYS_CNTHCTL_EL2,	CGT_HCR_NV),
910
	SR_RANGE_TRAP(SYS_CNTHP_TVAL_EL2,
911
		      SYS_CNTHP_CVAL_EL2, CGT_HCR_NV),
912
	SR_RANGE_TRAP(SYS_CNTHV_TVAL_EL2,
913
		      SYS_CNTHV_CVAL_EL2, CGT_HCR_NV),
914
	/* All _EL02, _EL12 registers up to CNTKCTL_EL12*/
915
	SR_RANGE_TRAP(sys_reg(3, 5, 0, 0, 0),
916
		      sys_reg(3, 5, 10, 15, 7), CGT_HCR_NV),
917
	SR_RANGE_TRAP(sys_reg(3, 5, 12, 0, 0),
918
		      sys_reg(3, 5, 14, 1, 0), CGT_HCR_NV),
919
	SR_TRAP(SYS_CNTP_CTL_EL02,	CGT_CNTHCTL_EL1NVPCT),
920
	SR_TRAP(SYS_CNTP_CVAL_EL02,	CGT_CNTHCTL_EL1NVPCT),
921
	SR_TRAP(SYS_CNTV_CTL_EL02,	CGT_CNTHCTL_EL1NVVCT),
922
	SR_TRAP(SYS_CNTV_CVAL_EL02,	CGT_CNTHCTL_EL1NVVCT),
923
	SR_TRAP(OP_AT_S1E2R,		CGT_HCR_NV),
924
	SR_TRAP(OP_AT_S1E2W,		CGT_HCR_NV),
925
	SR_TRAP(OP_AT_S12E1R,		CGT_HCR_NV),
926
	SR_TRAP(OP_AT_S12E1W,		CGT_HCR_NV),
927
	SR_TRAP(OP_AT_S12E0R,		CGT_HCR_NV),
928
	SR_TRAP(OP_AT_S12E0W,		CGT_HCR_NV),
929
	SR_TRAP(OP_AT_S1E2A,		CGT_HCR_NV),
930
	SR_TRAP(OP_TLBI_IPAS2E1,	CGT_HCR_NV),
931
	SR_TRAP(OP_TLBI_RIPAS2E1,	CGT_HCR_NV),
932
	SR_TRAP(OP_TLBI_IPAS2LE1,	CGT_HCR_NV),
933
	SR_TRAP(OP_TLBI_RIPAS2LE1,	CGT_HCR_NV),
934
	SR_TRAP(OP_TLBI_RVAE2,		CGT_HCR_NV),
935
	SR_TRAP(OP_TLBI_RVALE2,		CGT_HCR_NV),
936
	SR_TRAP(OP_TLBI_ALLE2,		CGT_HCR_NV),
937
	SR_TRAP(OP_TLBI_VAE2,		CGT_HCR_NV),
938
	SR_TRAP(OP_TLBI_ALLE1,		CGT_HCR_NV),
939
	SR_TRAP(OP_TLBI_VALE2,		CGT_HCR_NV),
940
	SR_TRAP(OP_TLBI_VMALLS12E1,	CGT_HCR_NV),
941
	SR_TRAP(OP_TLBI_IPAS2E1NXS,	CGT_HCR_NV),
942
	SR_TRAP(OP_TLBI_RIPAS2E1NXS,	CGT_HCR_NV),
943
	SR_TRAP(OP_TLBI_IPAS2LE1NXS,	CGT_HCR_NV),
944
	SR_TRAP(OP_TLBI_RIPAS2LE1NXS,	CGT_HCR_NV),
945
	SR_TRAP(OP_TLBI_RVAE2NXS,	CGT_HCR_NV),
946
	SR_TRAP(OP_TLBI_RVALE2NXS,	CGT_HCR_NV),
947
	SR_TRAP(OP_TLBI_ALLE2NXS,	CGT_HCR_NV),
948
	SR_TRAP(OP_TLBI_VAE2NXS,	CGT_HCR_NV),
949
	SR_TRAP(OP_TLBI_ALLE1NXS,	CGT_HCR_NV),
950
	SR_TRAP(OP_TLBI_VALE2NXS,	CGT_HCR_NV),
951
	SR_TRAP(OP_TLBI_VMALLS12E1NXS,	CGT_HCR_NV),
952
	SR_TRAP(OP_TLBI_IPAS2E1IS,	CGT_HCR_NV),
953
	SR_TRAP(OP_TLBI_RIPAS2E1IS,	CGT_HCR_NV),
954
	SR_TRAP(OP_TLBI_IPAS2LE1IS,	CGT_HCR_NV),
955
	SR_TRAP(OP_TLBI_RIPAS2LE1IS,	CGT_HCR_NV),
956
	SR_TRAP(OP_TLBI_RVAE2IS,	CGT_HCR_NV),
957
	SR_TRAP(OP_TLBI_RVALE2IS,	CGT_HCR_NV),
958
	SR_TRAP(OP_TLBI_ALLE2IS,	CGT_HCR_NV),
959
	SR_TRAP(OP_TLBI_VAE2IS,		CGT_HCR_NV),
960
	SR_TRAP(OP_TLBI_ALLE1IS,	CGT_HCR_NV),
961
	SR_TRAP(OP_TLBI_VALE2IS,	CGT_HCR_NV),
962
	SR_TRAP(OP_TLBI_VMALLS12E1IS,	CGT_HCR_NV),
963
	SR_TRAP(OP_TLBI_IPAS2E1ISNXS,	CGT_HCR_NV),
964
	SR_TRAP(OP_TLBI_RIPAS2E1ISNXS,	CGT_HCR_NV),
965
	SR_TRAP(OP_TLBI_IPAS2LE1ISNXS,	CGT_HCR_NV),
966
	SR_TRAP(OP_TLBI_RIPAS2LE1ISNXS,	CGT_HCR_NV),
967
	SR_TRAP(OP_TLBI_RVAE2ISNXS,	CGT_HCR_NV),
968
	SR_TRAP(OP_TLBI_RVALE2ISNXS,	CGT_HCR_NV),
969
	SR_TRAP(OP_TLBI_ALLE2ISNXS,	CGT_HCR_NV),
970
	SR_TRAP(OP_TLBI_VAE2ISNXS,	CGT_HCR_NV),
971
	SR_TRAP(OP_TLBI_ALLE1ISNXS,	CGT_HCR_NV),
972
	SR_TRAP(OP_TLBI_VALE2ISNXS,	CGT_HCR_NV),
973
	SR_TRAP(OP_TLBI_VMALLS12E1ISNXS,CGT_HCR_NV),
974
	SR_TRAP(OP_TLBI_ALLE2OS,	CGT_HCR_NV),
975
	SR_TRAP(OP_TLBI_VAE2OS,		CGT_HCR_NV),
976
	SR_TRAP(OP_TLBI_ALLE1OS,	CGT_HCR_NV),
977
	SR_TRAP(OP_TLBI_VALE2OS,	CGT_HCR_NV),
978
	SR_TRAP(OP_TLBI_VMALLS12E1OS,	CGT_HCR_NV),
979
	SR_TRAP(OP_TLBI_IPAS2E1OS,	CGT_HCR_NV),
980
	SR_TRAP(OP_TLBI_RIPAS2E1OS,	CGT_HCR_NV),
981
	SR_TRAP(OP_TLBI_IPAS2LE1OS,	CGT_HCR_NV),
982
	SR_TRAP(OP_TLBI_RIPAS2LE1OS,	CGT_HCR_NV),
983
	SR_TRAP(OP_TLBI_RVAE2OS,	CGT_HCR_NV),
984
	SR_TRAP(OP_TLBI_RVALE2OS,	CGT_HCR_NV),
985
	SR_TRAP(OP_TLBI_ALLE2OSNXS,	CGT_HCR_NV),
986
	SR_TRAP(OP_TLBI_VAE2OSNXS,	CGT_HCR_NV),
987
	SR_TRAP(OP_TLBI_ALLE1OSNXS,	CGT_HCR_NV),
988
	SR_TRAP(OP_TLBI_VALE2OSNXS,	CGT_HCR_NV),
989
	SR_TRAP(OP_TLBI_VMALLS12E1OSNXS,CGT_HCR_NV),
990
	SR_TRAP(OP_TLBI_IPAS2E1OSNXS,	CGT_HCR_NV),
991
	SR_TRAP(OP_TLBI_RIPAS2E1OSNXS,	CGT_HCR_NV),
992
	SR_TRAP(OP_TLBI_IPAS2LE1OSNXS,	CGT_HCR_NV),
993
	SR_TRAP(OP_TLBI_RIPAS2LE1OSNXS,	CGT_HCR_NV),
994
	SR_TRAP(OP_TLBI_RVAE2OSNXS,	CGT_HCR_NV),
995
	SR_TRAP(OP_TLBI_RVALE2OSNXS,	CGT_HCR_NV),
996
	SR_TRAP(OP_CPP_RCTX, 		CGT_HCR_NV),
997
	SR_TRAP(OP_DVP_RCTX, 		CGT_HCR_NV),
998
	SR_TRAP(OP_CFP_RCTX, 		CGT_HCR_NV),
999
	SR_TRAP(SYS_SP_EL1,		CGT_HCR_NV_nNV2),
1000
	SR_TRAP(SYS_VBAR_EL1,		CGT_HCR_NV1_nNV2),
1001
	SR_TRAP(SYS_ELR_EL1,		CGT_HCR_NV1_nNV2),
1002
	SR_TRAP(SYS_SPSR_EL1,		CGT_HCR_NV1_nNV2),
1003
	SR_TRAP(SYS_SCXTNUM_EL1,	CGT_HCR_NV1_nNV2_ENSCXT),
1004
	SR_TRAP(SYS_SCXTNUM_EL0,	CGT_HCR_ENSCXT),
1005
	SR_TRAP(OP_AT_S1E1R, 		CGT_HCR_AT),
1006
	SR_TRAP(OP_AT_S1E1W, 		CGT_HCR_AT),
1007
	SR_TRAP(OP_AT_S1E0R, 		CGT_HCR_AT),
1008
	SR_TRAP(OP_AT_S1E0W, 		CGT_HCR_AT),
1009
	SR_TRAP(OP_AT_S1E1RP, 		CGT_HCR_AT),
1010
	SR_TRAP(OP_AT_S1E1WP, 		CGT_HCR_AT),
1011
	SR_TRAP(OP_AT_S1E1A,		CGT_HCR_AT),
1012
	SR_TRAP(SYS_ERXPFGF_EL1,	CGT_HCR_nFIEN),
1013
	SR_TRAP(SYS_ERXPFGCTL_EL1,	CGT_HCR_nFIEN),
1014
	SR_TRAP(SYS_ERXPFGCDN_EL1,	CGT_HCR_nFIEN),
1015
	SR_TRAP(SYS_PMCR_EL0,		CGT_MDCR_TPM_TPMCR),
1016
	SR_TRAP(SYS_PMCNTENSET_EL0,	CGT_MDCR_TPM),
1017
	SR_TRAP(SYS_PMCNTENCLR_EL0,	CGT_MDCR_TPM),
1018
	SR_TRAP(SYS_PMOVSSET_EL0,	CGT_MDCR_TPM),
1019
	SR_TRAP(SYS_PMOVSCLR_EL0,	CGT_MDCR_TPM),
1020
	SR_TRAP(SYS_PMCEID0_EL0,	CGT_MDCR_TPM),
1021
	SR_TRAP(SYS_PMCEID1_EL0,	CGT_MDCR_TPM),
1022
	SR_TRAP(SYS_PMXEVTYPER_EL0,	CGT_MDCR_TPM_HPMN),
1023
	SR_TRAP(SYS_PMSWINC_EL0,	CGT_MDCR_TPM),
1024
	SR_TRAP(SYS_PMSELR_EL0,		CGT_MDCR_TPM),
1025
	SR_TRAP(SYS_PMXEVCNTR_EL0,	CGT_MDCR_TPM_HPMN),
1026
	SR_TRAP(SYS_PMCCNTR_EL0,	CGT_MDCR_TPM),
1027
	SR_TRAP(SYS_PMUSERENR_EL0,	CGT_MDCR_TPM),
1028
	SR_TRAP(SYS_PMINTENSET_EL1,	CGT_MDCR_TPM),
1029
	SR_TRAP(SYS_PMINTENCLR_EL1,	CGT_MDCR_TPM),
1030
	SR_TRAP(SYS_PMMIR_EL1,		CGT_MDCR_TPM),
1031
	SR_TRAP(SYS_PMEVCNTRn_EL0(0),	CGT_MDCR_TPM_HPMN),
1032
	SR_TRAP(SYS_PMEVCNTRn_EL0(1),	CGT_MDCR_TPM_HPMN),
1033
	SR_TRAP(SYS_PMEVCNTRn_EL0(2),	CGT_MDCR_TPM_HPMN),
1034
	SR_TRAP(SYS_PMEVCNTRn_EL0(3),	CGT_MDCR_TPM_HPMN),
1035
	SR_TRAP(SYS_PMEVCNTRn_EL0(4),	CGT_MDCR_TPM_HPMN),
1036
	SR_TRAP(SYS_PMEVCNTRn_EL0(5),	CGT_MDCR_TPM_HPMN),
1037
	SR_TRAP(SYS_PMEVCNTRn_EL0(6),	CGT_MDCR_TPM_HPMN),
1038
	SR_TRAP(SYS_PMEVCNTRn_EL0(7),	CGT_MDCR_TPM_HPMN),
1039
	SR_TRAP(SYS_PMEVCNTRn_EL0(8),	CGT_MDCR_TPM_HPMN),
1040
	SR_TRAP(SYS_PMEVCNTRn_EL0(9),	CGT_MDCR_TPM_HPMN),
1041
	SR_TRAP(SYS_PMEVCNTRn_EL0(10),	CGT_MDCR_TPM_HPMN),
1042
	SR_TRAP(SYS_PMEVCNTRn_EL0(11),	CGT_MDCR_TPM_HPMN),
1043
	SR_TRAP(SYS_PMEVCNTRn_EL0(12),	CGT_MDCR_TPM_HPMN),
1044
	SR_TRAP(SYS_PMEVCNTRn_EL0(13),	CGT_MDCR_TPM_HPMN),
1045
	SR_TRAP(SYS_PMEVCNTRn_EL0(14),	CGT_MDCR_TPM_HPMN),
1046
	SR_TRAP(SYS_PMEVCNTRn_EL0(15),	CGT_MDCR_TPM_HPMN),
1047
	SR_TRAP(SYS_PMEVCNTRn_EL0(16),	CGT_MDCR_TPM_HPMN),
1048
	SR_TRAP(SYS_PMEVCNTRn_EL0(17),	CGT_MDCR_TPM_HPMN),
1049
	SR_TRAP(SYS_PMEVCNTRn_EL0(18),	CGT_MDCR_TPM_HPMN),
1050
	SR_TRAP(SYS_PMEVCNTRn_EL0(19),	CGT_MDCR_TPM_HPMN),
1051
	SR_TRAP(SYS_PMEVCNTRn_EL0(20),	CGT_MDCR_TPM_HPMN),
1052
	SR_TRAP(SYS_PMEVCNTRn_EL0(21),	CGT_MDCR_TPM_HPMN),
1053
	SR_TRAP(SYS_PMEVCNTRn_EL0(22),	CGT_MDCR_TPM_HPMN),
1054
	SR_TRAP(SYS_PMEVCNTRn_EL0(23),	CGT_MDCR_TPM_HPMN),
1055
	SR_TRAP(SYS_PMEVCNTRn_EL0(24),	CGT_MDCR_TPM_HPMN),
1056
	SR_TRAP(SYS_PMEVCNTRn_EL0(25),	CGT_MDCR_TPM_HPMN),
1057
	SR_TRAP(SYS_PMEVCNTRn_EL0(26),	CGT_MDCR_TPM_HPMN),
1058
	SR_TRAP(SYS_PMEVCNTRn_EL0(27),	CGT_MDCR_TPM_HPMN),
1059
	SR_TRAP(SYS_PMEVCNTRn_EL0(28),	CGT_MDCR_TPM_HPMN),
1060
	SR_TRAP(SYS_PMEVCNTRn_EL0(29),	CGT_MDCR_TPM_HPMN),
1061
	SR_TRAP(SYS_PMEVCNTRn_EL0(30),	CGT_MDCR_TPM_HPMN),
1062
	SR_TRAP(SYS_PMEVTYPERn_EL0(0),	CGT_MDCR_TPM_HPMN),
1063
	SR_TRAP(SYS_PMEVTYPERn_EL0(1),	CGT_MDCR_TPM_HPMN),
1064
	SR_TRAP(SYS_PMEVTYPERn_EL0(2),	CGT_MDCR_TPM_HPMN),
1065
	SR_TRAP(SYS_PMEVTYPERn_EL0(3),	CGT_MDCR_TPM_HPMN),
1066
	SR_TRAP(SYS_PMEVTYPERn_EL0(4),	CGT_MDCR_TPM_HPMN),
1067
	SR_TRAP(SYS_PMEVTYPERn_EL0(5),	CGT_MDCR_TPM_HPMN),
1068
	SR_TRAP(SYS_PMEVTYPERn_EL0(6),	CGT_MDCR_TPM_HPMN),
1069
	SR_TRAP(SYS_PMEVTYPERn_EL0(7),	CGT_MDCR_TPM_HPMN),
1070
	SR_TRAP(SYS_PMEVTYPERn_EL0(8),	CGT_MDCR_TPM_HPMN),
1071
	SR_TRAP(SYS_PMEVTYPERn_EL0(9),	CGT_MDCR_TPM_HPMN),
1072
	SR_TRAP(SYS_PMEVTYPERn_EL0(10),	CGT_MDCR_TPM_HPMN),
1073
	SR_TRAP(SYS_PMEVTYPERn_EL0(11),	CGT_MDCR_TPM_HPMN),
1074
	SR_TRAP(SYS_PMEVTYPERn_EL0(12),	CGT_MDCR_TPM_HPMN),
1075
	SR_TRAP(SYS_PMEVTYPERn_EL0(13),	CGT_MDCR_TPM_HPMN),
1076
	SR_TRAP(SYS_PMEVTYPERn_EL0(14),	CGT_MDCR_TPM_HPMN),
1077
	SR_TRAP(SYS_PMEVTYPERn_EL0(15),	CGT_MDCR_TPM_HPMN),
1078
	SR_TRAP(SYS_PMEVTYPERn_EL0(16),	CGT_MDCR_TPM_HPMN),
1079
	SR_TRAP(SYS_PMEVTYPERn_EL0(17),	CGT_MDCR_TPM_HPMN),
1080
	SR_TRAP(SYS_PMEVTYPERn_EL0(18),	CGT_MDCR_TPM_HPMN),
1081
	SR_TRAP(SYS_PMEVTYPERn_EL0(19),	CGT_MDCR_TPM_HPMN),
1082
	SR_TRAP(SYS_PMEVTYPERn_EL0(20),	CGT_MDCR_TPM_HPMN),
1083
	SR_TRAP(SYS_PMEVTYPERn_EL0(21),	CGT_MDCR_TPM_HPMN),
1084
	SR_TRAP(SYS_PMEVTYPERn_EL0(22),	CGT_MDCR_TPM_HPMN),
1085
	SR_TRAP(SYS_PMEVTYPERn_EL0(23),	CGT_MDCR_TPM_HPMN),
1086
	SR_TRAP(SYS_PMEVTYPERn_EL0(24),	CGT_MDCR_TPM_HPMN),
1087
	SR_TRAP(SYS_PMEVTYPERn_EL0(25),	CGT_MDCR_TPM_HPMN),
1088
	SR_TRAP(SYS_PMEVTYPERn_EL0(26),	CGT_MDCR_TPM_HPMN),
1089
	SR_TRAP(SYS_PMEVTYPERn_EL0(27),	CGT_MDCR_TPM_HPMN),
1090
	SR_TRAP(SYS_PMEVTYPERn_EL0(28),	CGT_MDCR_TPM_HPMN),
1091
	SR_TRAP(SYS_PMEVTYPERn_EL0(29),	CGT_MDCR_TPM_HPMN),
1092
	SR_TRAP(SYS_PMEVTYPERn_EL0(30),	CGT_MDCR_TPM_HPMN),
1093
	SR_TRAP(SYS_PMCCFILTR_EL0,	CGT_MDCR_TPM),
1094
	SR_TRAP(SYS_MDCCSR_EL0,		CGT_MDCR_TDCC_TDE_TDA),
1095
	SR_TRAP(SYS_MDCCINT_EL1,	CGT_MDCR_TDCC_TDE_TDA),
1096
	SR_TRAP(SYS_OSDTRRX_EL1,	CGT_MDCR_TDCC_TDE_TDA),
1097
	SR_TRAP(SYS_OSDTRTX_EL1,	CGT_MDCR_TDCC_TDE_TDA),
1098
	SR_TRAP(SYS_DBGDTR_EL0,		CGT_MDCR_TDCC_TDE_TDA),
1099
	/*
1100
	 * Also covers DBGDTRRX_EL0, which has the same encoding as
1101
	 * SYS_DBGDTRTX_EL0...
1102
	 */
1103
	SR_TRAP(SYS_DBGDTRTX_EL0,	CGT_MDCR_TDCC_TDE_TDA),
1104
	SR_TRAP(SYS_MDSCR_EL1,		CGT_MDCR_TDE_TDA),
1105
	SR_TRAP(SYS_OSECCR_EL1,		CGT_MDCR_TDE_TDA),
1106
	SR_TRAP(SYS_DBGBVRn_EL1(0),	CGT_MDCR_TDE_TDA),
1107
	SR_TRAP(SYS_DBGBVRn_EL1(1),	CGT_MDCR_TDE_TDA),
1108
	SR_TRAP(SYS_DBGBVRn_EL1(2),	CGT_MDCR_TDE_TDA),
1109
	SR_TRAP(SYS_DBGBVRn_EL1(3),	CGT_MDCR_TDE_TDA),
1110
	SR_TRAP(SYS_DBGBVRn_EL1(4),	CGT_MDCR_TDE_TDA),
1111
	SR_TRAP(SYS_DBGBVRn_EL1(5),	CGT_MDCR_TDE_TDA),
1112
	SR_TRAP(SYS_DBGBVRn_EL1(6),	CGT_MDCR_TDE_TDA),
1113
	SR_TRAP(SYS_DBGBVRn_EL1(7),	CGT_MDCR_TDE_TDA),
1114
	SR_TRAP(SYS_DBGBVRn_EL1(8),	CGT_MDCR_TDE_TDA),
1115
	SR_TRAP(SYS_DBGBVRn_EL1(9),	CGT_MDCR_TDE_TDA),
1116
	SR_TRAP(SYS_DBGBVRn_EL1(10),	CGT_MDCR_TDE_TDA),
1117
	SR_TRAP(SYS_DBGBVRn_EL1(11),	CGT_MDCR_TDE_TDA),
1118
	SR_TRAP(SYS_DBGBVRn_EL1(12),	CGT_MDCR_TDE_TDA),
1119
	SR_TRAP(SYS_DBGBVRn_EL1(13),	CGT_MDCR_TDE_TDA),
1120
	SR_TRAP(SYS_DBGBVRn_EL1(14),	CGT_MDCR_TDE_TDA),
1121
	SR_TRAP(SYS_DBGBVRn_EL1(15),	CGT_MDCR_TDE_TDA),
1122
	SR_TRAP(SYS_DBGBCRn_EL1(0),	CGT_MDCR_TDE_TDA),
1123
	SR_TRAP(SYS_DBGBCRn_EL1(1),	CGT_MDCR_TDE_TDA),
1124
	SR_TRAP(SYS_DBGBCRn_EL1(2),	CGT_MDCR_TDE_TDA),
1125
	SR_TRAP(SYS_DBGBCRn_EL1(3),	CGT_MDCR_TDE_TDA),
1126
	SR_TRAP(SYS_DBGBCRn_EL1(4),	CGT_MDCR_TDE_TDA),
1127
	SR_TRAP(SYS_DBGBCRn_EL1(5),	CGT_MDCR_TDE_TDA),
1128
	SR_TRAP(SYS_DBGBCRn_EL1(6),	CGT_MDCR_TDE_TDA),
1129
	SR_TRAP(SYS_DBGBCRn_EL1(7),	CGT_MDCR_TDE_TDA),
1130
	SR_TRAP(SYS_DBGBCRn_EL1(8),	CGT_MDCR_TDE_TDA),
1131
	SR_TRAP(SYS_DBGBCRn_EL1(9),	CGT_MDCR_TDE_TDA),
1132
	SR_TRAP(SYS_DBGBCRn_EL1(10),	CGT_MDCR_TDE_TDA),
1133
	SR_TRAP(SYS_DBGBCRn_EL1(11),	CGT_MDCR_TDE_TDA),
1134
	SR_TRAP(SYS_DBGBCRn_EL1(12),	CGT_MDCR_TDE_TDA),
1135
	SR_TRAP(SYS_DBGBCRn_EL1(13),	CGT_MDCR_TDE_TDA),
1136
	SR_TRAP(SYS_DBGBCRn_EL1(14),	CGT_MDCR_TDE_TDA),
1137
	SR_TRAP(SYS_DBGBCRn_EL1(15),	CGT_MDCR_TDE_TDA),
1138
	SR_TRAP(SYS_DBGWVRn_EL1(0),	CGT_MDCR_TDE_TDA),
1139
	SR_TRAP(SYS_DBGWVRn_EL1(1),	CGT_MDCR_TDE_TDA),
1140
	SR_TRAP(SYS_DBGWVRn_EL1(2),	CGT_MDCR_TDE_TDA),
1141
	SR_TRAP(SYS_DBGWVRn_EL1(3),	CGT_MDCR_TDE_TDA),
1142
	SR_TRAP(SYS_DBGWVRn_EL1(4),	CGT_MDCR_TDE_TDA),
1143
	SR_TRAP(SYS_DBGWVRn_EL1(5),	CGT_MDCR_TDE_TDA),
1144
	SR_TRAP(SYS_DBGWVRn_EL1(6),	CGT_MDCR_TDE_TDA),
1145
	SR_TRAP(SYS_DBGWVRn_EL1(7),	CGT_MDCR_TDE_TDA),
1146
	SR_TRAP(SYS_DBGWVRn_EL1(8),	CGT_MDCR_TDE_TDA),
1147
	SR_TRAP(SYS_DBGWVRn_EL1(9),	CGT_MDCR_TDE_TDA),
1148
	SR_TRAP(SYS_DBGWVRn_EL1(10),	CGT_MDCR_TDE_TDA),
1149
	SR_TRAP(SYS_DBGWVRn_EL1(11),	CGT_MDCR_TDE_TDA),
1150
	SR_TRAP(SYS_DBGWVRn_EL1(12),	CGT_MDCR_TDE_TDA),
1151
	SR_TRAP(SYS_DBGWVRn_EL1(13),	CGT_MDCR_TDE_TDA),
1152
	SR_TRAP(SYS_DBGWVRn_EL1(14),	CGT_MDCR_TDE_TDA),
1153
	SR_TRAP(SYS_DBGWVRn_EL1(15),	CGT_MDCR_TDE_TDA),
1154
	SR_TRAP(SYS_DBGWCRn_EL1(0),	CGT_MDCR_TDE_TDA),
1155
	SR_TRAP(SYS_DBGWCRn_EL1(1),	CGT_MDCR_TDE_TDA),
1156
	SR_TRAP(SYS_DBGWCRn_EL1(2),	CGT_MDCR_TDE_TDA),
1157
	SR_TRAP(SYS_DBGWCRn_EL1(3),	CGT_MDCR_TDE_TDA),
1158
	SR_TRAP(SYS_DBGWCRn_EL1(4),	CGT_MDCR_TDE_TDA),
1159
	SR_TRAP(SYS_DBGWCRn_EL1(5),	CGT_MDCR_TDE_TDA),
1160
	SR_TRAP(SYS_DBGWCRn_EL1(6),	CGT_MDCR_TDE_TDA),
1161
	SR_TRAP(SYS_DBGWCRn_EL1(7),	CGT_MDCR_TDE_TDA),
1162
	SR_TRAP(SYS_DBGWCRn_EL1(8),	CGT_MDCR_TDE_TDA),
1163
	SR_TRAP(SYS_DBGWCRn_EL1(9),	CGT_MDCR_TDE_TDA),
1164
	SR_TRAP(SYS_DBGWCRn_EL1(10),	CGT_MDCR_TDE_TDA),
1165
	SR_TRAP(SYS_DBGWCRn_EL1(11),	CGT_MDCR_TDE_TDA),
1166
	SR_TRAP(SYS_DBGWCRn_EL1(12),	CGT_MDCR_TDE_TDA),
1167
	SR_TRAP(SYS_DBGWCRn_EL1(13),	CGT_MDCR_TDE_TDA),
1168
	SR_TRAP(SYS_DBGWCRn_EL1(14),	CGT_MDCR_TDE_TDA),
1169
	SR_TRAP(SYS_DBGCLAIMSET_EL1,	CGT_MDCR_TDE_TDA),
1170
	SR_TRAP(SYS_DBGCLAIMCLR_EL1,	CGT_MDCR_TDE_TDA),
1171
	SR_TRAP(SYS_DBGAUTHSTATUS_EL1,	CGT_MDCR_TDE_TDA),
1172
	SR_TRAP(SYS_OSLAR_EL1,		CGT_MDCR_TDE_TDOSA),
1173
	SR_TRAP(SYS_OSLSR_EL1,		CGT_MDCR_TDE_TDOSA),
1174
	SR_TRAP(SYS_OSDLR_EL1,		CGT_MDCR_TDE_TDOSA),
1175
	SR_TRAP(SYS_DBGPRCR_EL1,	CGT_MDCR_TDE_TDOSA),
1176
	SR_TRAP(SYS_MDRAR_EL1,		CGT_MDCR_TDE_TDRA),
1177
	SR_TRAP(SYS_PMBLIMITR_EL1,	CGT_MDCR_E2PB),
1178
	SR_TRAP(SYS_PMBPTR_EL1,		CGT_MDCR_E2PB),
1179
	SR_TRAP(SYS_PMBSR_EL1,		CGT_MDCR_E2PB),
1180
	SR_TRAP(SYS_PMSCR_EL1,		CGT_MDCR_TPMS),
1181
	SR_TRAP(SYS_PMSEVFR_EL1,	CGT_MDCR_TPMS),
1182
	SR_TRAP(SYS_PMSFCR_EL1,		CGT_MDCR_TPMS),
1183
	SR_TRAP(SYS_PMSICR_EL1,		CGT_MDCR_TPMS),
1184
	SR_TRAP(SYS_PMSIDR_EL1,		CGT_MDCR_TPMS),
1185
	SR_TRAP(SYS_PMSIRR_EL1,		CGT_MDCR_TPMS),
1186
	SR_TRAP(SYS_PMSLATFR_EL1,	CGT_MDCR_TPMS),
1187
	SR_TRAP(SYS_PMSNEVFR_EL1,	CGT_MDCR_TPMS),
1188
	SR_TRAP(SYS_PMSDSFR_EL1,	CGT_MDCR_TPMS),
1189
	SR_TRAP(SYS_TRFCR_EL1,		CGT_MDCR_TTRF),
1190
	SR_TRAP(SYS_TRBBASER_EL1,	CGT_MDCR_E2TB),
1191
	SR_TRAP(SYS_TRBLIMITR_EL1,	CGT_MDCR_E2TB),
1192
	SR_TRAP(SYS_TRBMAR_EL1, 	CGT_MDCR_E2TB),
1193
	SR_TRAP(SYS_TRBPTR_EL1, 	CGT_MDCR_E2TB),
1194
	SR_TRAP(SYS_TRBSR_EL1, 		CGT_MDCR_E2TB),
1195
	SR_TRAP(SYS_TRBTRG_EL1,		CGT_MDCR_E2TB),
1196
	SR_TRAP(SYS_CPACR_EL1,		CGT_CPTR_TCPAC),
1197
	SR_TRAP(SYS_AMUSERENR_EL0,	CGT_CPTR_TAM),
1198
	SR_TRAP(SYS_AMCFGR_EL0,		CGT_CPTR_TAM),
1199
	SR_TRAP(SYS_AMCGCR_EL0,		CGT_CPTR_TAM),
1200
	SR_TRAP(SYS_AMCNTENCLR0_EL0,	CGT_CPTR_TAM),
1201
	SR_TRAP(SYS_AMCNTENCLR1_EL0,	CGT_CPTR_TAM),
1202
	SR_TRAP(SYS_AMCNTENSET0_EL0,	CGT_CPTR_TAM),
1203
	SR_TRAP(SYS_AMCNTENSET1_EL0,	CGT_CPTR_TAM),
1204
	SR_TRAP(SYS_AMCR_EL0,		CGT_CPTR_TAM),
1205
	SR_TRAP(SYS_AMEVCNTR0_EL0(0),	CGT_CPTR_TAM),
1206
	SR_TRAP(SYS_AMEVCNTR0_EL0(1),	CGT_CPTR_TAM),
1207
	SR_TRAP(SYS_AMEVCNTR0_EL0(2),	CGT_CPTR_TAM),
1208
	SR_TRAP(SYS_AMEVCNTR0_EL0(3),	CGT_CPTR_TAM),
1209
	SR_TRAP(SYS_AMEVCNTR1_EL0(0),	CGT_CPTR_TAM),
1210
	SR_TRAP(SYS_AMEVCNTR1_EL0(1),	CGT_CPTR_TAM),
1211
	SR_TRAP(SYS_AMEVCNTR1_EL0(2),	CGT_CPTR_TAM),
1212
	SR_TRAP(SYS_AMEVCNTR1_EL0(3),	CGT_CPTR_TAM),
1213
	SR_TRAP(SYS_AMEVCNTR1_EL0(4),	CGT_CPTR_TAM),
1214
	SR_TRAP(SYS_AMEVCNTR1_EL0(5),	CGT_CPTR_TAM),
1215
	SR_TRAP(SYS_AMEVCNTR1_EL0(6),	CGT_CPTR_TAM),
1216
	SR_TRAP(SYS_AMEVCNTR1_EL0(7),	CGT_CPTR_TAM),
1217
	SR_TRAP(SYS_AMEVCNTR1_EL0(8),	CGT_CPTR_TAM),
1218
	SR_TRAP(SYS_AMEVCNTR1_EL0(9),	CGT_CPTR_TAM),
1219
	SR_TRAP(SYS_AMEVCNTR1_EL0(10),	CGT_CPTR_TAM),
1220
	SR_TRAP(SYS_AMEVCNTR1_EL0(11),	CGT_CPTR_TAM),
1221
	SR_TRAP(SYS_AMEVCNTR1_EL0(12),	CGT_CPTR_TAM),
1222
	SR_TRAP(SYS_AMEVCNTR1_EL0(13),	CGT_CPTR_TAM),
1223
	SR_TRAP(SYS_AMEVCNTR1_EL0(14),	CGT_CPTR_TAM),
1224
	SR_TRAP(SYS_AMEVCNTR1_EL0(15),	CGT_CPTR_TAM),
1225
	SR_TRAP(SYS_AMEVTYPER0_EL0(0),	CGT_CPTR_TAM),
1226
	SR_TRAP(SYS_AMEVTYPER0_EL0(1),	CGT_CPTR_TAM),
1227
	SR_TRAP(SYS_AMEVTYPER0_EL0(2),	CGT_CPTR_TAM),
1228
	SR_TRAP(SYS_AMEVTYPER0_EL0(3),	CGT_CPTR_TAM),
1229
	SR_TRAP(SYS_AMEVTYPER1_EL0(0),	CGT_CPTR_TAM),
1230
	SR_TRAP(SYS_AMEVTYPER1_EL0(1),	CGT_CPTR_TAM),
1231
	SR_TRAP(SYS_AMEVTYPER1_EL0(2),	CGT_CPTR_TAM),
1232
	SR_TRAP(SYS_AMEVTYPER1_EL0(3),	CGT_CPTR_TAM),
1233
	SR_TRAP(SYS_AMEVTYPER1_EL0(4),	CGT_CPTR_TAM),
1234
	SR_TRAP(SYS_AMEVTYPER1_EL0(5),	CGT_CPTR_TAM),
1235
	SR_TRAP(SYS_AMEVTYPER1_EL0(6),	CGT_CPTR_TAM),
1236
	SR_TRAP(SYS_AMEVTYPER1_EL0(7),	CGT_CPTR_TAM),
1237
	SR_TRAP(SYS_AMEVTYPER1_EL0(8),	CGT_CPTR_TAM),
1238
	SR_TRAP(SYS_AMEVTYPER1_EL0(9),	CGT_CPTR_TAM),
1239
	SR_TRAP(SYS_AMEVTYPER1_EL0(10),	CGT_CPTR_TAM),
1240
	SR_TRAP(SYS_AMEVTYPER1_EL0(11),	CGT_CPTR_TAM),
1241
	SR_TRAP(SYS_AMEVTYPER1_EL0(12),	CGT_CPTR_TAM),
1242
	SR_TRAP(SYS_AMEVTYPER1_EL0(13),	CGT_CPTR_TAM),
1243
	SR_TRAP(SYS_AMEVTYPER1_EL0(14),	CGT_CPTR_TAM),
1244
	SR_TRAP(SYS_AMEVTYPER1_EL0(15),	CGT_CPTR_TAM),
1245
	/* op0=2, op1=1, and CRn<0b1000 */
1246
	SR_RANGE_TRAP(sys_reg(2, 1, 0, 0, 0),
1247
		      sys_reg(2, 1, 7, 15, 7), CGT_CPTR_TTA),
1248
	SR_TRAP(SYS_CNTP_TVAL_EL0,	CGT_CNTHCTL_EL1PTEN),
1249
	SR_TRAP(SYS_CNTP_CVAL_EL0,	CGT_CNTHCTL_EL1PTEN),
1250
	SR_TRAP(SYS_CNTP_CTL_EL0,	CGT_CNTHCTL_EL1PTEN),
1251
	SR_TRAP(SYS_CNTPCT_EL0,		CGT_CNTHCTL_EL1PCTEN),
1252
	SR_TRAP(SYS_CNTPCTSS_EL0,	CGT_CNTHCTL_EL1PCTEN),
1253
	SR_TRAP(SYS_CNTV_TVAL_EL0,	CGT_CNTHCTL_EL1TVT),
1254
	SR_TRAP(SYS_CNTV_CVAL_EL0,	CGT_CNTHCTL_EL1TVT),
1255
	SR_TRAP(SYS_CNTV_CTL_EL0,	CGT_CNTHCTL_EL1TVT),
1256
	SR_TRAP(SYS_CNTVCT_EL0,		CGT_CNTHCTL_EL1TVCT),
1257
	SR_TRAP(SYS_CNTVCTSS_EL0,	CGT_CNTHCTL_EL1TVCT),
1258
	SR_TRAP(SYS_FPMR,		CGT_HCRX_EnFPM),
1259
	/*
1260
	 * IMPDEF choice:
1261
	 * We treat ICC_SRE_EL2.{SRE,Enable) and ICV_SRE_EL1.SRE as
1262
	 * RAO/WI. We therefore never consider ICC_SRE_EL2.Enable for
1263
	 * ICC_SRE_EL1 access, and always handle it locally.
1264
	 */
1265
	SR_TRAP(SYS_ICC_AP0R0_EL1,	CGT_ICH_HCR_TALL0),
1266
	SR_TRAP(SYS_ICC_AP0R1_EL1,	CGT_ICH_HCR_TALL0),
1267
	SR_TRAP(SYS_ICC_AP0R2_EL1,	CGT_ICH_HCR_TALL0),
1268
	SR_TRAP(SYS_ICC_AP0R3_EL1,	CGT_ICH_HCR_TALL0),
1269
	SR_TRAP(SYS_ICC_AP1R0_EL1,	CGT_ICH_HCR_TALL1),
1270
	SR_TRAP(SYS_ICC_AP1R1_EL1,	CGT_ICH_HCR_TALL1),
1271
	SR_TRAP(SYS_ICC_AP1R2_EL1,	CGT_ICH_HCR_TALL1),
1272
	SR_TRAP(SYS_ICC_AP1R3_EL1,	CGT_ICH_HCR_TALL1),
1273
	SR_TRAP(SYS_ICC_BPR0_EL1,	CGT_ICH_HCR_TALL0),
1274
	SR_TRAP(SYS_ICC_BPR1_EL1,	CGT_ICH_HCR_TALL1),
1275
	SR_TRAP(SYS_ICC_CTLR_EL1,	CGT_ICH_HCR_TC),
1276
	SR_TRAP(SYS_ICC_DIR_EL1,	CGT_ICH_HCR_TC_TDIR),
1277
	SR_TRAP(SYS_ICC_EOIR0_EL1,	CGT_ICH_HCR_TALL0),
1278
	SR_TRAP(SYS_ICC_EOIR1_EL1,	CGT_ICH_HCR_TALL1),
1279
	SR_TRAP(SYS_ICC_HPPIR0_EL1,	CGT_ICH_HCR_TALL0),
1280
	SR_TRAP(SYS_ICC_HPPIR1_EL1,	CGT_ICH_HCR_TALL1),
1281
	SR_TRAP(SYS_ICC_IAR0_EL1,	CGT_ICH_HCR_TALL0),
1282
	SR_TRAP(SYS_ICC_IAR1_EL1,	CGT_ICH_HCR_TALL1),
1283
	SR_TRAP(SYS_ICC_IGRPEN0_EL1,	CGT_ICH_HCR_TALL0),
1284
	SR_TRAP(SYS_ICC_IGRPEN1_EL1,	CGT_ICH_HCR_TALL1),
1285
	SR_TRAP(SYS_ICC_PMR_EL1,	CGT_ICH_HCR_TC),
1286
	SR_TRAP(SYS_ICC_RPR_EL1,	CGT_ICH_HCR_TC),
1287
};
1288

1289
static DEFINE_XARRAY(sr_forward_xa);
1290

1291
enum fg_filter_id {
1292
	__NO_FGF__,
1293
	HCRX_FGTnXS,
1294

1295
	/* Must be last */
1296
	__NR_FG_FILTER_IDS__
1297
};
1298

1299
#define __FGT(g, b, p, f)					\
1300
		{						\
1301
			.fgt = g ## _GROUP,			\
1302
			.bit = g ## _EL2_ ## b ## _SHIFT,	\
1303
			.pol = p,				\
1304
			.fgf = f,				\
1305
		}
1306

1307
#define FGT(g, b, p)		__FGT(g, b, p, __NO_FGF__)
1308

1309
/*
1310
 * See the warning next to SR_RANGE_TRAP(), and apply the same
1311
 * level of caution.
1312
 */
1313
#define SR_FGF_RANGE(sr, e, g, b, p, f)				\
1314
	{							\
1315
		.encoding	= sr,				\
1316
		.end		= e,				\
1317
		.tc		= __FGT(g, b, p, f),		\
1318
		.line = __LINE__,				\
1319
	}
1320

1321
#define SR_FGF(sr, g, b, p, f) 	SR_FGF_RANGE(sr, sr, g, b, p, f)
1322
#define SR_FGT(sr, g, b, p)	SR_FGF_RANGE(sr, sr, g, b, p, __NO_FGF__)
1323
#define SR_FGT_RANGE(sr, end, g, b, p)	\
1324
				SR_FGF_RANGE(sr, end, g, b, p, __NO_FGF__)
1325

1326
static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
1327
	/* HFGRTR_EL2, HFGWTR_EL2 */
1328
	SR_FGT(SYS_AMAIR2_EL1,		HFGRTR, nAMAIR2_EL1, 0),
1329
	SR_FGT(SYS_MAIR2_EL1,		HFGRTR, nMAIR2_EL1, 0),
1330
	SR_FGT(SYS_S2POR_EL1,		HFGRTR, nS2POR_EL1, 0),
1331
	SR_FGT(SYS_POR_EL1,		HFGRTR, nPOR_EL1, 0),
1332
	SR_FGT(SYS_POR_EL0,		HFGRTR, nPOR_EL0, 0),
1333
	SR_FGT(SYS_PIR_EL1,		HFGRTR, nPIR_EL1, 0),
1334
	SR_FGT(SYS_PIRE0_EL1,		HFGRTR, nPIRE0_EL1, 0),
1335
	SR_FGT(SYS_RCWMASK_EL1,		HFGRTR, nRCWMASK_EL1, 0),
1336
	SR_FGT(SYS_TPIDR2_EL0,		HFGRTR, nTPIDR2_EL0, 0),
1337
	SR_FGT(SYS_SMPRI_EL1,		HFGRTR, nSMPRI_EL1, 0),
1338
	SR_FGT(SYS_GCSCR_EL1,		HFGRTR, nGCS_EL1, 0),
1339
	SR_FGT(SYS_GCSPR_EL1,		HFGRTR, nGCS_EL1, 0),
1340
	SR_FGT(SYS_GCSCRE0_EL1,		HFGRTR, nGCS_EL0, 0),
1341
	SR_FGT(SYS_GCSPR_EL0,		HFGRTR, nGCS_EL0, 0),
1342
	SR_FGT(SYS_ACCDATA_EL1,		HFGRTR, nACCDATA_EL1, 0),
1343
	SR_FGT(SYS_ERXADDR_EL1,		HFGRTR, ERXADDR_EL1, 1),
1344
	SR_FGT(SYS_ERXPFGCDN_EL1,	HFGRTR, ERXPFGCDN_EL1, 1),
1345
	SR_FGT(SYS_ERXPFGCTL_EL1,	HFGRTR, ERXPFGCTL_EL1, 1),
1346
	SR_FGT(SYS_ERXPFGF_EL1,		HFGRTR, ERXPFGF_EL1, 1),
1347
	SR_FGT(SYS_ERXMISC0_EL1,	HFGRTR, ERXMISCn_EL1, 1),
1348
	SR_FGT(SYS_ERXMISC1_EL1,	HFGRTR, ERXMISCn_EL1, 1),
1349
	SR_FGT(SYS_ERXMISC2_EL1,	HFGRTR, ERXMISCn_EL1, 1),
1350
	SR_FGT(SYS_ERXMISC3_EL1,	HFGRTR, ERXMISCn_EL1, 1),
1351
	SR_FGT(SYS_ERXSTATUS_EL1,	HFGRTR, ERXSTATUS_EL1, 1),
1352
	SR_FGT(SYS_ERXCTLR_EL1,		HFGRTR, ERXCTLR_EL1, 1),
1353
	SR_FGT(SYS_ERXFR_EL1,		HFGRTR, ERXFR_EL1, 1),
1354
	SR_FGT(SYS_ERRSELR_EL1,		HFGRTR, ERRSELR_EL1, 1),
1355
	SR_FGT(SYS_ERRIDR_EL1,		HFGRTR, ERRIDR_EL1, 1),
1356
	SR_FGT(SYS_ICC_IGRPEN0_EL1,	HFGRTR, ICC_IGRPENn_EL1, 1),
1357
	SR_FGT(SYS_ICC_IGRPEN1_EL1,	HFGRTR, ICC_IGRPENn_EL1, 1),
1358
	SR_FGT(SYS_VBAR_EL1,		HFGRTR, VBAR_EL1, 1),
1359
	SR_FGT(SYS_TTBR1_EL1,		HFGRTR, TTBR1_EL1, 1),
1360
	SR_FGT(SYS_TTBR0_EL1,		HFGRTR, TTBR0_EL1, 1),
1361
	SR_FGT(SYS_TPIDR_EL0,		HFGRTR, TPIDR_EL0, 1),
1362
	SR_FGT(SYS_TPIDRRO_EL0,		HFGRTR, TPIDRRO_EL0, 1),
1363
	SR_FGT(SYS_TPIDR_EL1,		HFGRTR, TPIDR_EL1, 1),
1364
	SR_FGT(SYS_TCR_EL1,		HFGRTR, TCR_EL1, 1),
1365
	SR_FGT(SYS_TCR2_EL1,		HFGRTR, TCR_EL1, 1),
1366
	SR_FGT(SYS_SCXTNUM_EL0,		HFGRTR, SCXTNUM_EL0, 1),
1367
	SR_FGT(SYS_SCXTNUM_EL1, 	HFGRTR, SCXTNUM_EL1, 1),
1368
	SR_FGT(SYS_SCTLR_EL1, 		HFGRTR, SCTLR_EL1, 1),
1369
	SR_FGT(SYS_SCTLR2_EL1,		HFGRTR, SCTLR_EL1, 1),
1370
	SR_FGT(SYS_REVIDR_EL1, 		HFGRTR, REVIDR_EL1, 1),
1371
	SR_FGT(SYS_PAR_EL1, 		HFGRTR, PAR_EL1, 1),
1372
	SR_FGT(SYS_MPIDR_EL1, 		HFGRTR, MPIDR_EL1, 1),
1373
	SR_FGT(SYS_MIDR_EL1, 		HFGRTR, MIDR_EL1, 1),
1374
	SR_FGT(SYS_MAIR_EL1, 		HFGRTR, MAIR_EL1, 1),
1375
	SR_FGT(SYS_LORSA_EL1, 		HFGRTR, LORSA_EL1, 1),
1376
	SR_FGT(SYS_LORN_EL1, 		HFGRTR, LORN_EL1, 1),
1377
	SR_FGT(SYS_LORID_EL1, 		HFGRTR, LORID_EL1, 1),
1378
	SR_FGT(SYS_LOREA_EL1, 		HFGRTR, LOREA_EL1, 1),
1379
	SR_FGT(SYS_LORC_EL1, 		HFGRTR, LORC_EL1, 1),
1380
	SR_FGT(SYS_ISR_EL1, 		HFGRTR, ISR_EL1, 1),
1381
	SR_FGT(SYS_FAR_EL1, 		HFGRTR, FAR_EL1, 1),
1382
	SR_FGT(SYS_ESR_EL1, 		HFGRTR, ESR_EL1, 1),
1383
	SR_FGT(SYS_DCZID_EL0, 		HFGRTR, DCZID_EL0, 1),
1384
	SR_FGT(SYS_CTR_EL0, 		HFGRTR, CTR_EL0, 1),
1385
	SR_FGT(SYS_CSSELR_EL1, 		HFGRTR, CSSELR_EL1, 1),
1386
	SR_FGT(SYS_CPACR_EL1, 		HFGRTR, CPACR_EL1, 1),
1387
	SR_FGT(SYS_CONTEXTIDR_EL1, 	HFGRTR, CONTEXTIDR_EL1, 1),
1388
	SR_FGT(SYS_CLIDR_EL1, 		HFGRTR, CLIDR_EL1, 1),
1389
	SR_FGT(SYS_CCSIDR_EL1, 		HFGRTR, CCSIDR_EL1, 1),
1390
	SR_FGT(SYS_APIBKEYLO_EL1, 	HFGRTR, APIBKey, 1),
1391
	SR_FGT(SYS_APIBKEYHI_EL1, 	HFGRTR, APIBKey, 1),
1392
	SR_FGT(SYS_APIAKEYLO_EL1, 	HFGRTR, APIAKey, 1),
1393
	SR_FGT(SYS_APIAKEYHI_EL1, 	HFGRTR, APIAKey, 1),
1394
	SR_FGT(SYS_APGAKEYLO_EL1, 	HFGRTR, APGAKey, 1),
1395
	SR_FGT(SYS_APGAKEYHI_EL1, 	HFGRTR, APGAKey, 1),
1396
	SR_FGT(SYS_APDBKEYLO_EL1, 	HFGRTR, APDBKey, 1),
1397
	SR_FGT(SYS_APDBKEYHI_EL1, 	HFGRTR, APDBKey, 1),
1398
	SR_FGT(SYS_APDAKEYLO_EL1, 	HFGRTR, APDAKey, 1),
1399
	SR_FGT(SYS_APDAKEYHI_EL1, 	HFGRTR, APDAKey, 1),
1400
	SR_FGT(SYS_AMAIR_EL1, 		HFGRTR, AMAIR_EL1, 1),
1401
	SR_FGT(SYS_AIDR_EL1, 		HFGRTR, AIDR_EL1, 1),
1402
	SR_FGT(SYS_AFSR1_EL1, 		HFGRTR, AFSR1_EL1, 1),
1403
	SR_FGT(SYS_AFSR0_EL1, 		HFGRTR, AFSR0_EL1, 1),
1404

1405
	/* HFGRTR2_EL2, HFGWTR2_EL2 */
1406
	SR_FGT(SYS_ACTLRALIAS_EL1,	HFGRTR2, nACTLRALIAS_EL1, 0),
1407
	SR_FGT(SYS_ACTLRMASK_EL1,	HFGRTR2, nACTLRMASK_EL1, 0),
1408
	SR_FGT(SYS_CPACRALIAS_EL1,	HFGRTR2, nCPACRALIAS_EL1, 0),
1409
	SR_FGT(SYS_CPACRMASK_EL1,	HFGRTR2, nCPACRMASK_EL1, 0),
1410
	SR_FGT(SYS_PFAR_EL1,		HFGRTR2, nPFAR_EL1, 0),
1411
	SR_FGT(SYS_RCWSMASK_EL1,	HFGRTR2, nRCWSMASK_EL1, 0),
1412
	SR_FGT(SYS_SCTLR2ALIAS_EL1,	HFGRTR2, nSCTLRALIAS2_EL1, 0),
1413
	SR_FGT(SYS_SCTLR2MASK_EL1,	HFGRTR2, nSCTLR2MASK_EL1, 0),
1414
	SR_FGT(SYS_SCTLRALIAS_EL1,	HFGRTR2, nSCTLRALIAS_EL1, 0),
1415
	SR_FGT(SYS_SCTLRMASK_EL1,	HFGRTR2, nSCTLRMASK_EL1, 0),
1416
	SR_FGT(SYS_TCR2ALIAS_EL1,	HFGRTR2, nTCR2ALIAS_EL1, 0),
1417
	SR_FGT(SYS_TCR2MASK_EL1,	HFGRTR2, nTCR2MASK_EL1, 0),
1418
	SR_FGT(SYS_TCRALIAS_EL1,	HFGRTR2, nTCRALIAS_EL1, 0),
1419
	SR_FGT(SYS_TCRMASK_EL1,		HFGRTR2, nTCRMASK_EL1, 0),
1420
	SR_FGT(SYS_ERXGSR_EL1,		HFGRTR2, nERXGSR_EL1, 0),
1421

1422
	/* HFGITR_EL2 */
1423
	SR_FGT(OP_AT_S1E1A, 		HFGITR, ATS1E1A, 1),
1424
	SR_FGT(OP_COSP_RCTX, 		HFGITR, COSPRCTX, 1),
1425
	SR_FGT(OP_GCSPUSHX, 		HFGITR, nGCSEPP, 0),
1426
	SR_FGT(OP_GCSPOPX, 		HFGITR, nGCSEPP, 0),
1427
	SR_FGT(OP_GCSPUSHM, 		HFGITR, nGCSPUSHM_EL1, 0),
1428
	SR_FGT(OP_BRB_IALL, 		HFGITR, nBRBIALL, 0),
1429
	SR_FGT(OP_BRB_INJ, 		HFGITR, nBRBINJ, 0),
1430
	SR_FGT(SYS_DC_CVAC, 		HFGITR, DCCVAC, 1),
1431
	SR_FGT(SYS_DC_CGVAC, 		HFGITR, DCCVAC, 1),
1432
	SR_FGT(SYS_DC_CGDVAC, 		HFGITR, DCCVAC, 1),
1433
	SR_FGT(OP_CPP_RCTX, 		HFGITR, CPPRCTX, 1),
1434
	SR_FGT(OP_DVP_RCTX, 		HFGITR, DVPRCTX, 1),
1435
	SR_FGT(OP_CFP_RCTX, 		HFGITR, CFPRCTX, 1),
1436
	SR_FGT(OP_TLBI_VAALE1, 		HFGITR, TLBIVAALE1, 1),
1437
	SR_FGT(OP_TLBI_VALE1, 		HFGITR, TLBIVALE1, 1),
1438
	SR_FGT(OP_TLBI_VAAE1, 		HFGITR, TLBIVAAE1, 1),
1439
	SR_FGT(OP_TLBI_ASIDE1, 		HFGITR, TLBIASIDE1, 1),
1440
	SR_FGT(OP_TLBI_VAE1, 		HFGITR, TLBIVAE1, 1),
1441
	SR_FGT(OP_TLBI_VMALLE1, 	HFGITR, TLBIVMALLE1, 1),
1442
	SR_FGT(OP_TLBI_RVAALE1, 	HFGITR, TLBIRVAALE1, 1),
1443
	SR_FGT(OP_TLBI_RVALE1, 		HFGITR, TLBIRVALE1, 1),
1444
	SR_FGT(OP_TLBI_RVAAE1, 		HFGITR, TLBIRVAAE1, 1),
1445
	SR_FGT(OP_TLBI_RVAE1, 		HFGITR, TLBIRVAE1, 1),
1446
	SR_FGT(OP_TLBI_RVAALE1IS, 	HFGITR, TLBIRVAALE1IS, 1),
1447
	SR_FGT(OP_TLBI_RVALE1IS, 	HFGITR, TLBIRVALE1IS, 1),
1448
	SR_FGT(OP_TLBI_RVAAE1IS, 	HFGITR, TLBIRVAAE1IS, 1),
1449
	SR_FGT(OP_TLBI_RVAE1IS, 	HFGITR, TLBIRVAE1IS, 1),
1450
	SR_FGT(OP_TLBI_VAALE1IS, 	HFGITR, TLBIVAALE1IS, 1),
1451
	SR_FGT(OP_TLBI_VALE1IS, 	HFGITR, TLBIVALE1IS, 1),
1452
	SR_FGT(OP_TLBI_VAAE1IS, 	HFGITR, TLBIVAAE1IS, 1),
1453
	SR_FGT(OP_TLBI_ASIDE1IS, 	HFGITR, TLBIASIDE1IS, 1),
1454
	SR_FGT(OP_TLBI_VAE1IS, 		HFGITR, TLBIVAE1IS, 1),
1455
	SR_FGT(OP_TLBI_VMALLE1IS, 	HFGITR, TLBIVMALLE1IS, 1),
1456
	SR_FGT(OP_TLBI_RVAALE1OS, 	HFGITR, TLBIRVAALE1OS, 1),
1457
	SR_FGT(OP_TLBI_RVALE1OS, 	HFGITR, TLBIRVALE1OS, 1),
1458
	SR_FGT(OP_TLBI_RVAAE1OS, 	HFGITR, TLBIRVAAE1OS, 1),
1459
	SR_FGT(OP_TLBI_RVAE1OS, 	HFGITR, TLBIRVAE1OS, 1),
1460
	SR_FGT(OP_TLBI_VAALE1OS, 	HFGITR, TLBIVAALE1OS, 1),
1461
	SR_FGT(OP_TLBI_VALE1OS, 	HFGITR, TLBIVALE1OS, 1),
1462
	SR_FGT(OP_TLBI_VAAE1OS, 	HFGITR, TLBIVAAE1OS, 1),
1463
	SR_FGT(OP_TLBI_ASIDE1OS, 	HFGITR, TLBIASIDE1OS, 1),
1464
	SR_FGT(OP_TLBI_VAE1OS, 		HFGITR, TLBIVAE1OS, 1),
1465
	SR_FGT(OP_TLBI_VMALLE1OS, 	HFGITR, TLBIVMALLE1OS, 1),
1466
	/* nXS variants must be checked against HCRX_EL2.FGTnXS */
1467
	SR_FGF(OP_TLBI_VAALE1NXS, 	HFGITR, TLBIVAALE1, 1, HCRX_FGTnXS),
1468
	SR_FGF(OP_TLBI_VALE1NXS, 	HFGITR, TLBIVALE1, 1, HCRX_FGTnXS),
1469
	SR_FGF(OP_TLBI_VAAE1NXS, 	HFGITR, TLBIVAAE1, 1, HCRX_FGTnXS),
1470
	SR_FGF(OP_TLBI_ASIDE1NXS, 	HFGITR, TLBIASIDE1, 1, HCRX_FGTnXS),
1471
	SR_FGF(OP_TLBI_VAE1NXS, 	HFGITR, TLBIVAE1, 1, HCRX_FGTnXS),
1472
	SR_FGF(OP_TLBI_VMALLE1NXS, 	HFGITR, TLBIVMALLE1, 1, HCRX_FGTnXS),
1473
	SR_FGF(OP_TLBI_RVAALE1NXS, 	HFGITR, TLBIRVAALE1, 1, HCRX_FGTnXS),
1474
	SR_FGF(OP_TLBI_RVALE1NXS, 	HFGITR, TLBIRVALE1, 1, HCRX_FGTnXS),
1475
	SR_FGF(OP_TLBI_RVAAE1NXS, 	HFGITR, TLBIRVAAE1, 1, HCRX_FGTnXS),
1476
	SR_FGF(OP_TLBI_RVAE1NXS, 	HFGITR, TLBIRVAE1, 1, HCRX_FGTnXS),
1477
	SR_FGF(OP_TLBI_RVAALE1ISNXS, 	HFGITR, TLBIRVAALE1IS, 1, HCRX_FGTnXS),
1478
	SR_FGF(OP_TLBI_RVALE1ISNXS, 	HFGITR, TLBIRVALE1IS, 1, HCRX_FGTnXS),
1479
	SR_FGF(OP_TLBI_RVAAE1ISNXS, 	HFGITR, TLBIRVAAE1IS, 1, HCRX_FGTnXS),
1480
	SR_FGF(OP_TLBI_RVAE1ISNXS, 	HFGITR, TLBIRVAE1IS, 1, HCRX_FGTnXS),
1481
	SR_FGF(OP_TLBI_VAALE1ISNXS, 	HFGITR, TLBIVAALE1IS, 1, HCRX_FGTnXS),
1482
	SR_FGF(OP_TLBI_VALE1ISNXS, 	HFGITR, TLBIVALE1IS, 1, HCRX_FGTnXS),
1483
	SR_FGF(OP_TLBI_VAAE1ISNXS, 	HFGITR, TLBIVAAE1IS, 1, HCRX_FGTnXS),
1484
	SR_FGF(OP_TLBI_ASIDE1ISNXS, 	HFGITR, TLBIASIDE1IS, 1, HCRX_FGTnXS),
1485
	SR_FGF(OP_TLBI_VAE1ISNXS, 	HFGITR, TLBIVAE1IS, 1, HCRX_FGTnXS),
1486
	SR_FGF(OP_TLBI_VMALLE1ISNXS, 	HFGITR, TLBIVMALLE1IS, 1, HCRX_FGTnXS),
1487
	SR_FGF(OP_TLBI_RVAALE1OSNXS, 	HFGITR, TLBIRVAALE1OS, 1, HCRX_FGTnXS),
1488
	SR_FGF(OP_TLBI_RVALE1OSNXS, 	HFGITR, TLBIRVALE1OS, 1, HCRX_FGTnXS),
1489
	SR_FGF(OP_TLBI_RVAAE1OSNXS, 	HFGITR, TLBIRVAAE1OS, 1, HCRX_FGTnXS),
1490
	SR_FGF(OP_TLBI_RVAE1OSNXS, 	HFGITR, TLBIRVAE1OS, 1, HCRX_FGTnXS),
1491
	SR_FGF(OP_TLBI_VAALE1OSNXS, 	HFGITR, TLBIVAALE1OS, 1, HCRX_FGTnXS),
1492
	SR_FGF(OP_TLBI_VALE1OSNXS, 	HFGITR, TLBIVALE1OS, 1, HCRX_FGTnXS),
1493
	SR_FGF(OP_TLBI_VAAE1OSNXS, 	HFGITR, TLBIVAAE1OS, 1, HCRX_FGTnXS),
1494
	SR_FGF(OP_TLBI_ASIDE1OSNXS, 	HFGITR, TLBIASIDE1OS, 1, HCRX_FGTnXS),
1495
	SR_FGF(OP_TLBI_VAE1OSNXS, 	HFGITR, TLBIVAE1OS, 1, HCRX_FGTnXS),
1496
	SR_FGF(OP_TLBI_VMALLE1OSNXS, 	HFGITR, TLBIVMALLE1OS, 1, HCRX_FGTnXS),
1497
	SR_FGT(OP_AT_S1E1WP, 		HFGITR, ATS1E1WP, 1),
1498
	SR_FGT(OP_AT_S1E1RP, 		HFGITR, ATS1E1RP, 1),
1499
	SR_FGT(OP_AT_S1E0W, 		HFGITR, ATS1E0W, 1),
1500
	SR_FGT(OP_AT_S1E0R, 		HFGITR, ATS1E0R, 1),
1501
	SR_FGT(OP_AT_S1E1W, 		HFGITR, ATS1E1W, 1),
1502
	SR_FGT(OP_AT_S1E1R, 		HFGITR, ATS1E1R, 1),
1503
	SR_FGT(SYS_DC_ZVA, 		HFGITR, DCZVA, 1),
1504
	SR_FGT(SYS_DC_GVA, 		HFGITR, DCZVA, 1),
1505
	SR_FGT(SYS_DC_GZVA, 		HFGITR, DCZVA, 1),
1506
	SR_FGT(SYS_DC_CIVAC, 		HFGITR, DCCIVAC, 1),
1507
	SR_FGT(SYS_DC_CIGVAC, 		HFGITR, DCCIVAC, 1),
1508
	SR_FGT(SYS_DC_CIGDVAC, 		HFGITR, DCCIVAC, 1),
1509
	SR_FGT(SYS_DC_CVADP, 		HFGITR, DCCVADP, 1),
1510
	SR_FGT(SYS_DC_CGVADP, 		HFGITR, DCCVADP, 1),
1511
	SR_FGT(SYS_DC_CGDVADP, 		HFGITR, DCCVADP, 1),
1512
	SR_FGT(SYS_DC_CVAP, 		HFGITR, DCCVAP, 1),
1513
	SR_FGT(SYS_DC_CGVAP, 		HFGITR, DCCVAP, 1),
1514
	SR_FGT(SYS_DC_CGDVAP, 		HFGITR, DCCVAP, 1),
1515
	SR_FGT(SYS_DC_CVAU, 		HFGITR, DCCVAU, 1),
1516
	SR_FGT(SYS_DC_CISW, 		HFGITR, DCCISW, 1),
1517
	SR_FGT(SYS_DC_CIGSW, 		HFGITR, DCCISW, 1),
1518
	SR_FGT(SYS_DC_CIGDSW, 		HFGITR, DCCISW, 1),
1519
	SR_FGT(SYS_DC_CSW, 		HFGITR, DCCSW, 1),
1520
	SR_FGT(SYS_DC_CGSW, 		HFGITR, DCCSW, 1),
1521
	SR_FGT(SYS_DC_CGDSW, 		HFGITR, DCCSW, 1),
1522
	SR_FGT(SYS_DC_ISW, 		HFGITR, DCISW, 1),
1523
	SR_FGT(SYS_DC_IGSW, 		HFGITR, DCISW, 1),
1524
	SR_FGT(SYS_DC_IGDSW, 		HFGITR, DCISW, 1),
1525
	SR_FGT(SYS_DC_IVAC, 		HFGITR, DCIVAC, 1),
1526
	SR_FGT(SYS_DC_IGVAC, 		HFGITR, DCIVAC, 1),
1527
	SR_FGT(SYS_DC_IGDVAC, 		HFGITR, DCIVAC, 1),
1528
	SR_FGT(SYS_IC_IVAU, 		HFGITR, ICIVAU, 1),
1529
	SR_FGT(SYS_IC_IALLU, 		HFGITR, ICIALLU, 1),
1530
	SR_FGT(SYS_IC_IALLUIS, 		HFGITR, ICIALLUIS, 1),
1531

1532
	/* HFGITR2_EL2 */
1533
	SR_FGT(SYS_DC_CIGDVAPS,		HFGITR2, nDCCIVAPS, 0),
1534
	SR_FGT(SYS_DC_CIVAPS,		HFGITR2, nDCCIVAPS, 0),
1535

1536
	/* HDFGRTR_EL2 */
1537
	SR_FGT(SYS_PMBIDR_EL1, 		HDFGRTR, PMBIDR_EL1, 1),
1538
	SR_FGT(SYS_PMSNEVFR_EL1, 	HDFGRTR, nPMSNEVFR_EL1, 0),
1539
	SR_FGT(SYS_BRBINF_EL1(0), 	HDFGRTR, nBRBDATA, 0),
1540
	SR_FGT(SYS_BRBINF_EL1(1), 	HDFGRTR, nBRBDATA, 0),
1541
	SR_FGT(SYS_BRBINF_EL1(2), 	HDFGRTR, nBRBDATA, 0),
1542
	SR_FGT(SYS_BRBINF_EL1(3), 	HDFGRTR, nBRBDATA, 0),
1543
	SR_FGT(SYS_BRBINF_EL1(4), 	HDFGRTR, nBRBDATA, 0),
1544
	SR_FGT(SYS_BRBINF_EL1(5), 	HDFGRTR, nBRBDATA, 0),
1545
	SR_FGT(SYS_BRBINF_EL1(6), 	HDFGRTR, nBRBDATA, 0),
1546
	SR_FGT(SYS_BRBINF_EL1(7), 	HDFGRTR, nBRBDATA, 0),
1547
	SR_FGT(SYS_BRBINF_EL1(8), 	HDFGRTR, nBRBDATA, 0),
1548
	SR_FGT(SYS_BRBINF_EL1(9), 	HDFGRTR, nBRBDATA, 0),
1549
	SR_FGT(SYS_BRBINF_EL1(10), 	HDFGRTR, nBRBDATA, 0),
1550
	SR_FGT(SYS_BRBINF_EL1(11), 	HDFGRTR, nBRBDATA, 0),
1551
	SR_FGT(SYS_BRBINF_EL1(12), 	HDFGRTR, nBRBDATA, 0),
1552
	SR_FGT(SYS_BRBINF_EL1(13), 	HDFGRTR, nBRBDATA, 0),
1553
	SR_FGT(SYS_BRBINF_EL1(14), 	HDFGRTR, nBRBDATA, 0),
1554
	SR_FGT(SYS_BRBINF_EL1(15), 	HDFGRTR, nBRBDATA, 0),
1555
	SR_FGT(SYS_BRBINF_EL1(16), 	HDFGRTR, nBRBDATA, 0),
1556
	SR_FGT(SYS_BRBINF_EL1(17), 	HDFGRTR, nBRBDATA, 0),
1557
	SR_FGT(SYS_BRBINF_EL1(18), 	HDFGRTR, nBRBDATA, 0),
1558
	SR_FGT(SYS_BRBINF_EL1(19), 	HDFGRTR, nBRBDATA, 0),
1559
	SR_FGT(SYS_BRBINF_EL1(20), 	HDFGRTR, nBRBDATA, 0),
1560
	SR_FGT(SYS_BRBINF_EL1(21), 	HDFGRTR, nBRBDATA, 0),
1561
	SR_FGT(SYS_BRBINF_EL1(22), 	HDFGRTR, nBRBDATA, 0),
1562
	SR_FGT(SYS_BRBINF_EL1(23), 	HDFGRTR, nBRBDATA, 0),
1563
	SR_FGT(SYS_BRBINF_EL1(24), 	HDFGRTR, nBRBDATA, 0),
1564
	SR_FGT(SYS_BRBINF_EL1(25), 	HDFGRTR, nBRBDATA, 0),
1565
	SR_FGT(SYS_BRBINF_EL1(26), 	HDFGRTR, nBRBDATA, 0),
1566
	SR_FGT(SYS_BRBINF_EL1(27), 	HDFGRTR, nBRBDATA, 0),
1567
	SR_FGT(SYS_BRBINF_EL1(28), 	HDFGRTR, nBRBDATA, 0),
1568
	SR_FGT(SYS_BRBINF_EL1(29), 	HDFGRTR, nBRBDATA, 0),
1569
	SR_FGT(SYS_BRBINF_EL1(30), 	HDFGRTR, nBRBDATA, 0),
1570
	SR_FGT(SYS_BRBINF_EL1(31), 	HDFGRTR, nBRBDATA, 0),
1571
	SR_FGT(SYS_BRBINFINJ_EL1, 	HDFGRTR, nBRBDATA, 0),
1572
	SR_FGT(SYS_BRBSRC_EL1(0), 	HDFGRTR, nBRBDATA, 0),
1573
	SR_FGT(SYS_BRBSRC_EL1(1), 	HDFGRTR, nBRBDATA, 0),
1574
	SR_FGT(SYS_BRBSRC_EL1(2), 	HDFGRTR, nBRBDATA, 0),
1575
	SR_FGT(SYS_BRBSRC_EL1(3), 	HDFGRTR, nBRBDATA, 0),
1576
	SR_FGT(SYS_BRBSRC_EL1(4), 	HDFGRTR, nBRBDATA, 0),
1577
	SR_FGT(SYS_BRBSRC_EL1(5), 	HDFGRTR, nBRBDATA, 0),
1578
	SR_FGT(SYS_BRBSRC_EL1(6), 	HDFGRTR, nBRBDATA, 0),
1579
	SR_FGT(SYS_BRBSRC_EL1(7), 	HDFGRTR, nBRBDATA, 0),
1580
	SR_FGT(SYS_BRBSRC_EL1(8), 	HDFGRTR, nBRBDATA, 0),
1581
	SR_FGT(SYS_BRBSRC_EL1(9), 	HDFGRTR, nBRBDATA, 0),
1582
	SR_FGT(SYS_BRBSRC_EL1(10), 	HDFGRTR, nBRBDATA, 0),
1583
	SR_FGT(SYS_BRBSRC_EL1(11), 	HDFGRTR, nBRBDATA, 0),
1584
	SR_FGT(SYS_BRBSRC_EL1(12), 	HDFGRTR, nBRBDATA, 0),
1585
	SR_FGT(SYS_BRBSRC_EL1(13), 	HDFGRTR, nBRBDATA, 0),
1586
	SR_FGT(SYS_BRBSRC_EL1(14), 	HDFGRTR, nBRBDATA, 0),
1587
	SR_FGT(SYS_BRBSRC_EL1(15), 	HDFGRTR, nBRBDATA, 0),
1588
	SR_FGT(SYS_BRBSRC_EL1(16), 	HDFGRTR, nBRBDATA, 0),
1589
	SR_FGT(SYS_BRBSRC_EL1(17), 	HDFGRTR, nBRBDATA, 0),
1590
	SR_FGT(SYS_BRBSRC_EL1(18), 	HDFGRTR, nBRBDATA, 0),
1591
	SR_FGT(SYS_BRBSRC_EL1(19), 	HDFGRTR, nBRBDATA, 0),
1592
	SR_FGT(SYS_BRBSRC_EL1(20), 	HDFGRTR, nBRBDATA, 0),
1593
	SR_FGT(SYS_BRBSRC_EL1(21), 	HDFGRTR, nBRBDATA, 0),
1594
	SR_FGT(SYS_BRBSRC_EL1(22), 	HDFGRTR, nBRBDATA, 0),
1595
	SR_FGT(SYS_BRBSRC_EL1(23), 	HDFGRTR, nBRBDATA, 0),
1596
	SR_FGT(SYS_BRBSRC_EL1(24), 	HDFGRTR, nBRBDATA, 0),
1597
	SR_FGT(SYS_BRBSRC_EL1(25), 	HDFGRTR, nBRBDATA, 0),
1598
	SR_FGT(SYS_BRBSRC_EL1(26), 	HDFGRTR, nBRBDATA, 0),
1599
	SR_FGT(SYS_BRBSRC_EL1(27), 	HDFGRTR, nBRBDATA, 0),
1600
	SR_FGT(SYS_BRBSRC_EL1(28), 	HDFGRTR, nBRBDATA, 0),
1601
	SR_FGT(SYS_BRBSRC_EL1(29), 	HDFGRTR, nBRBDATA, 0),
1602
	SR_FGT(SYS_BRBSRC_EL1(30), 	HDFGRTR, nBRBDATA, 0),
1603
	SR_FGT(SYS_BRBSRC_EL1(31), 	HDFGRTR, nBRBDATA, 0),
1604
	SR_FGT(SYS_BRBSRCINJ_EL1, 	HDFGRTR, nBRBDATA, 0),
1605
	SR_FGT(SYS_BRBTGT_EL1(0), 	HDFGRTR, nBRBDATA, 0),
1606
	SR_FGT(SYS_BRBTGT_EL1(1), 	HDFGRTR, nBRBDATA, 0),
1607
	SR_FGT(SYS_BRBTGT_EL1(2), 	HDFGRTR, nBRBDATA, 0),
1608
	SR_FGT(SYS_BRBTGT_EL1(3), 	HDFGRTR, nBRBDATA, 0),
1609
	SR_FGT(SYS_BRBTGT_EL1(4), 	HDFGRTR, nBRBDATA, 0),
1610
	SR_FGT(SYS_BRBTGT_EL1(5), 	HDFGRTR, nBRBDATA, 0),
1611
	SR_FGT(SYS_BRBTGT_EL1(6), 	HDFGRTR, nBRBDATA, 0),
1612
	SR_FGT(SYS_BRBTGT_EL1(7), 	HDFGRTR, nBRBDATA, 0),
1613
	SR_FGT(SYS_BRBTGT_EL1(8), 	HDFGRTR, nBRBDATA, 0),
1614
	SR_FGT(SYS_BRBTGT_EL1(9), 	HDFGRTR, nBRBDATA, 0),
1615
	SR_FGT(SYS_BRBTGT_EL1(10), 	HDFGRTR, nBRBDATA, 0),
1616
	SR_FGT(SYS_BRBTGT_EL1(11), 	HDFGRTR, nBRBDATA, 0),
1617
	SR_FGT(SYS_BRBTGT_EL1(12), 	HDFGRTR, nBRBDATA, 0),
1618
	SR_FGT(SYS_BRBTGT_EL1(13), 	HDFGRTR, nBRBDATA, 0),
1619
	SR_FGT(SYS_BRBTGT_EL1(14), 	HDFGRTR, nBRBDATA, 0),
1620
	SR_FGT(SYS_BRBTGT_EL1(15), 	HDFGRTR, nBRBDATA, 0),
1621
	SR_FGT(SYS_BRBTGT_EL1(16), 	HDFGRTR, nBRBDATA, 0),
1622
	SR_FGT(SYS_BRBTGT_EL1(17), 	HDFGRTR, nBRBDATA, 0),
1623
	SR_FGT(SYS_BRBTGT_EL1(18), 	HDFGRTR, nBRBDATA, 0),
1624
	SR_FGT(SYS_BRBTGT_EL1(19), 	HDFGRTR, nBRBDATA, 0),
1625
	SR_FGT(SYS_BRBTGT_EL1(20), 	HDFGRTR, nBRBDATA, 0),
1626
	SR_FGT(SYS_BRBTGT_EL1(21), 	HDFGRTR, nBRBDATA, 0),
1627
	SR_FGT(SYS_BRBTGT_EL1(22), 	HDFGRTR, nBRBDATA, 0),
1628
	SR_FGT(SYS_BRBTGT_EL1(23), 	HDFGRTR, nBRBDATA, 0),
1629
	SR_FGT(SYS_BRBTGT_EL1(24), 	HDFGRTR, nBRBDATA, 0),
1630
	SR_FGT(SYS_BRBTGT_EL1(25), 	HDFGRTR, nBRBDATA, 0),
1631
	SR_FGT(SYS_BRBTGT_EL1(26), 	HDFGRTR, nBRBDATA, 0),
1632
	SR_FGT(SYS_BRBTGT_EL1(27), 	HDFGRTR, nBRBDATA, 0),
1633
	SR_FGT(SYS_BRBTGT_EL1(28), 	HDFGRTR, nBRBDATA, 0),
1634
	SR_FGT(SYS_BRBTGT_EL1(29), 	HDFGRTR, nBRBDATA, 0),
1635
	SR_FGT(SYS_BRBTGT_EL1(30), 	HDFGRTR, nBRBDATA, 0),
1636
	SR_FGT(SYS_BRBTGT_EL1(31), 	HDFGRTR, nBRBDATA, 0),
1637
	SR_FGT(SYS_BRBTGTINJ_EL1, 	HDFGRTR, nBRBDATA, 0),
1638
	SR_FGT(SYS_BRBTS_EL1, 		HDFGRTR, nBRBDATA, 0),
1639
	SR_FGT(SYS_BRBCR_EL1, 		HDFGRTR, nBRBCTL, 0),
1640
	SR_FGT(SYS_BRBFCR_EL1, 		HDFGRTR, nBRBCTL, 0),
1641
	SR_FGT(SYS_BRBIDR0_EL1, 	HDFGRTR, nBRBIDR, 0),
1642
	SR_FGT(SYS_PMCEID0_EL0, 	HDFGRTR, PMCEIDn_EL0, 1),
1643
	SR_FGT(SYS_PMCEID1_EL0, 	HDFGRTR, PMCEIDn_EL0, 1),
1644
	SR_FGT(SYS_PMUSERENR_EL0, 	HDFGRTR, PMUSERENR_EL0, 1),
1645
	SR_FGT(SYS_TRBTRG_EL1, 		HDFGRTR, TRBTRG_EL1, 1),
1646
	SR_FGT(SYS_TRBSR_EL1, 		HDFGRTR, TRBSR_EL1, 1),
1647
	SR_FGT(SYS_TRBPTR_EL1, 		HDFGRTR, TRBPTR_EL1, 1),
1648
	SR_FGT(SYS_TRBMAR_EL1, 		HDFGRTR, TRBMAR_EL1, 1),
1649
	SR_FGT(SYS_TRBLIMITR_EL1, 	HDFGRTR, TRBLIMITR_EL1, 1),
1650
	SR_FGT(SYS_TRBIDR_EL1, 		HDFGRTR, TRBIDR_EL1, 1),
1651
	SR_FGT(SYS_TRBBASER_EL1, 	HDFGRTR, TRBBASER_EL1, 1),
1652
	SR_FGT(SYS_TRCVICTLR, 		HDFGRTR, TRCVICTLR, 1),
1653
	SR_FGT(SYS_TRCSTATR, 		HDFGRTR, TRCSTATR, 1),
1654
	SR_FGT(SYS_TRCSSCSR(0), 	HDFGRTR, TRCSSCSRn, 1),
1655
	SR_FGT(SYS_TRCSSCSR(1), 	HDFGRTR, TRCSSCSRn, 1),
1656
	SR_FGT(SYS_TRCSSCSR(2), 	HDFGRTR, TRCSSCSRn, 1),
1657
	SR_FGT(SYS_TRCSSCSR(3), 	HDFGRTR, TRCSSCSRn, 1),
1658
	SR_FGT(SYS_TRCSSCSR(4), 	HDFGRTR, TRCSSCSRn, 1),
1659
	SR_FGT(SYS_TRCSSCSR(5), 	HDFGRTR, TRCSSCSRn, 1),
1660
	SR_FGT(SYS_TRCSSCSR(6), 	HDFGRTR, TRCSSCSRn, 1),
1661
	SR_FGT(SYS_TRCSSCSR(7), 	HDFGRTR, TRCSSCSRn, 1),
1662
	SR_FGT(SYS_TRCSEQSTR, 		HDFGRTR, TRCSEQSTR, 1),
1663
	SR_FGT(SYS_TRCPRGCTLR, 		HDFGRTR, TRCPRGCTLR, 1),
1664
	SR_FGT(SYS_TRCOSLSR, 		HDFGRTR, TRCOSLSR, 1),
1665
	SR_FGT(SYS_TRCIMSPEC(0), 	HDFGRTR, TRCIMSPECn, 1),
1666
	SR_FGT(SYS_TRCIMSPEC(1), 	HDFGRTR, TRCIMSPECn, 1),
1667
	SR_FGT(SYS_TRCIMSPEC(2), 	HDFGRTR, TRCIMSPECn, 1),
1668
	SR_FGT(SYS_TRCIMSPEC(3), 	HDFGRTR, TRCIMSPECn, 1),
1669
	SR_FGT(SYS_TRCIMSPEC(4), 	HDFGRTR, TRCIMSPECn, 1),
1670
	SR_FGT(SYS_TRCIMSPEC(5), 	HDFGRTR, TRCIMSPECn, 1),
1671
	SR_FGT(SYS_TRCIMSPEC(6), 	HDFGRTR, TRCIMSPECn, 1),
1672
	SR_FGT(SYS_TRCIMSPEC(7), 	HDFGRTR, TRCIMSPECn, 1),
1673
	SR_FGT(SYS_TRCDEVARCH, 		HDFGRTR, TRCID, 1),
1674
	SR_FGT(SYS_TRCDEVID, 		HDFGRTR, TRCID, 1),
1675
	SR_FGT(SYS_TRCIDR0, 		HDFGRTR, TRCID, 1),
1676
	SR_FGT(SYS_TRCIDR1, 		HDFGRTR, TRCID, 1),
1677
	SR_FGT(SYS_TRCIDR2, 		HDFGRTR, TRCID, 1),
1678
	SR_FGT(SYS_TRCIDR3, 		HDFGRTR, TRCID, 1),
1679
	SR_FGT(SYS_TRCIDR4, 		HDFGRTR, TRCID, 1),
1680
	SR_FGT(SYS_TRCIDR5, 		HDFGRTR, TRCID, 1),
1681
	SR_FGT(SYS_TRCIDR6, 		HDFGRTR, TRCID, 1),
1682
	SR_FGT(SYS_TRCIDR7, 		HDFGRTR, TRCID, 1),
1683
	SR_FGT(SYS_TRCIDR8, 		HDFGRTR, TRCID, 1),
1684
	SR_FGT(SYS_TRCIDR9, 		HDFGRTR, TRCID, 1),
1685
	SR_FGT(SYS_TRCIDR10, 		HDFGRTR, TRCID, 1),
1686
	SR_FGT(SYS_TRCIDR11, 		HDFGRTR, TRCID, 1),
1687
	SR_FGT(SYS_TRCIDR12, 		HDFGRTR, TRCID, 1),
1688
	SR_FGT(SYS_TRCIDR13, 		HDFGRTR, TRCID, 1),
1689
	SR_FGT(SYS_TRCCNTVR(0), 	HDFGRTR, TRCCNTVRn, 1),
1690
	SR_FGT(SYS_TRCCNTVR(1), 	HDFGRTR, TRCCNTVRn, 1),
1691
	SR_FGT(SYS_TRCCNTVR(2), 	HDFGRTR, TRCCNTVRn, 1),
1692
	SR_FGT(SYS_TRCCNTVR(3), 	HDFGRTR, TRCCNTVRn, 1),
1693
	SR_FGT(SYS_TRCCLAIMCLR, 	HDFGRTR, TRCCLAIM, 1),
1694
	SR_FGT(SYS_TRCCLAIMSET, 	HDFGRTR, TRCCLAIM, 1),
1695
	SR_FGT(SYS_TRCAUXCTLR, 		HDFGRTR, TRCAUXCTLR, 1),
1696
	SR_FGT(SYS_TRCAUTHSTATUS, 	HDFGRTR, TRCAUTHSTATUS, 1),
1697
	SR_FGT(SYS_TRCACATR(0), 	HDFGRTR, TRC, 1),
1698
	SR_FGT(SYS_TRCACATR(1), 	HDFGRTR, TRC, 1),
1699
	SR_FGT(SYS_TRCACATR(2), 	HDFGRTR, TRC, 1),
1700
	SR_FGT(SYS_TRCACATR(3), 	HDFGRTR, TRC, 1),
1701
	SR_FGT(SYS_TRCACATR(4), 	HDFGRTR, TRC, 1),
1702
	SR_FGT(SYS_TRCACATR(5), 	HDFGRTR, TRC, 1),
1703
	SR_FGT(SYS_TRCACATR(6), 	HDFGRTR, TRC, 1),
1704
	SR_FGT(SYS_TRCACATR(7), 	HDFGRTR, TRC, 1),
1705
	SR_FGT(SYS_TRCACATR(8), 	HDFGRTR, TRC, 1),
1706
	SR_FGT(SYS_TRCACATR(9), 	HDFGRTR, TRC, 1),
1707
	SR_FGT(SYS_TRCACATR(10), 	HDFGRTR, TRC, 1),
1708
	SR_FGT(SYS_TRCACATR(11), 	HDFGRTR, TRC, 1),
1709
	SR_FGT(SYS_TRCACATR(12), 	HDFGRTR, TRC, 1),
1710
	SR_FGT(SYS_TRCACATR(13), 	HDFGRTR, TRC, 1),
1711
	SR_FGT(SYS_TRCACATR(14), 	HDFGRTR, TRC, 1),
1712
	SR_FGT(SYS_TRCACATR(15), 	HDFGRTR, TRC, 1),
1713
	SR_FGT(SYS_TRCACVR(0), 		HDFGRTR, TRC, 1),
1714
	SR_FGT(SYS_TRCACVR(1), 		HDFGRTR, TRC, 1),
1715
	SR_FGT(SYS_TRCACVR(2), 		HDFGRTR, TRC, 1),
1716
	SR_FGT(SYS_TRCACVR(3), 		HDFGRTR, TRC, 1),
1717
	SR_FGT(SYS_TRCACVR(4), 		HDFGRTR, TRC, 1),
1718
	SR_FGT(SYS_TRCACVR(5), 		HDFGRTR, TRC, 1),
1719
	SR_FGT(SYS_TRCACVR(6), 		HDFGRTR, TRC, 1),
1720
	SR_FGT(SYS_TRCACVR(7), 		HDFGRTR, TRC, 1),
1721
	SR_FGT(SYS_TRCACVR(8), 		HDFGRTR, TRC, 1),
1722
	SR_FGT(SYS_TRCACVR(9), 		HDFGRTR, TRC, 1),
1723
	SR_FGT(SYS_TRCACVR(10), 	HDFGRTR, TRC, 1),
1724
	SR_FGT(SYS_TRCACVR(11), 	HDFGRTR, TRC, 1),
1725
	SR_FGT(SYS_TRCACVR(12), 	HDFGRTR, TRC, 1),
1726
	SR_FGT(SYS_TRCACVR(13), 	HDFGRTR, TRC, 1),
1727
	SR_FGT(SYS_TRCACVR(14), 	HDFGRTR, TRC, 1),
1728
	SR_FGT(SYS_TRCACVR(15), 	HDFGRTR, TRC, 1),
1729
	SR_FGT(SYS_TRCBBCTLR, 		HDFGRTR, TRC, 1),
1730
	SR_FGT(SYS_TRCCCCTLR, 		HDFGRTR, TRC, 1),
1731
	SR_FGT(SYS_TRCCIDCCTLR0, 	HDFGRTR, TRC, 1),
1732
	SR_FGT(SYS_TRCCIDCCTLR1, 	HDFGRTR, TRC, 1),
1733
	SR_FGT(SYS_TRCCIDCVR(0), 	HDFGRTR, TRC, 1),
1734
	SR_FGT(SYS_TRCCIDCVR(1), 	HDFGRTR, TRC, 1),
1735
	SR_FGT(SYS_TRCCIDCVR(2), 	HDFGRTR, TRC, 1),
1736
	SR_FGT(SYS_TRCCIDCVR(3), 	HDFGRTR, TRC, 1),
1737
	SR_FGT(SYS_TRCCIDCVR(4), 	HDFGRTR, TRC, 1),
1738
	SR_FGT(SYS_TRCCIDCVR(5), 	HDFGRTR, TRC, 1),
1739
	SR_FGT(SYS_TRCCIDCVR(6), 	HDFGRTR, TRC, 1),
1740
	SR_FGT(SYS_TRCCIDCVR(7), 	HDFGRTR, TRC, 1),
1741
	SR_FGT(SYS_TRCCNTCTLR(0), 	HDFGRTR, TRC, 1),
1742
	SR_FGT(SYS_TRCCNTCTLR(1), 	HDFGRTR, TRC, 1),
1743
	SR_FGT(SYS_TRCCNTCTLR(2), 	HDFGRTR, TRC, 1),
1744
	SR_FGT(SYS_TRCCNTCTLR(3), 	HDFGRTR, TRC, 1),
1745
	SR_FGT(SYS_TRCCNTRLDVR(0), 	HDFGRTR, TRC, 1),
1746
	SR_FGT(SYS_TRCCNTRLDVR(1), 	HDFGRTR, TRC, 1),
1747
	SR_FGT(SYS_TRCCNTRLDVR(2), 	HDFGRTR, TRC, 1),
1748
	SR_FGT(SYS_TRCCNTRLDVR(3), 	HDFGRTR, TRC, 1),
1749
	SR_FGT(SYS_TRCCONFIGR, 		HDFGRTR, TRC, 1),
1750
	SR_FGT(SYS_TRCEVENTCTL0R, 	HDFGRTR, TRC, 1),
1751
	SR_FGT(SYS_TRCEVENTCTL1R, 	HDFGRTR, TRC, 1),
1752
	SR_FGT(SYS_TRCEXTINSELR(0), 	HDFGRTR, TRC, 1),
1753
	SR_FGT(SYS_TRCEXTINSELR(1), 	HDFGRTR, TRC, 1),
1754
	SR_FGT(SYS_TRCEXTINSELR(2), 	HDFGRTR, TRC, 1),
1755
	SR_FGT(SYS_TRCEXTINSELR(3), 	HDFGRTR, TRC, 1),
1756
	SR_FGT(SYS_TRCQCTLR, 		HDFGRTR, TRC, 1),
1757
	SR_FGT(SYS_TRCRSCTLR(2), 	HDFGRTR, TRC, 1),
1758
	SR_FGT(SYS_TRCRSCTLR(3), 	HDFGRTR, TRC, 1),
1759
	SR_FGT(SYS_TRCRSCTLR(4), 	HDFGRTR, TRC, 1),
1760
	SR_FGT(SYS_TRCRSCTLR(5), 	HDFGRTR, TRC, 1),
1761
	SR_FGT(SYS_TRCRSCTLR(6), 	HDFGRTR, TRC, 1),
1762
	SR_FGT(SYS_TRCRSCTLR(7), 	HDFGRTR, TRC, 1),
1763
	SR_FGT(SYS_TRCRSCTLR(8), 	HDFGRTR, TRC, 1),
1764
	SR_FGT(SYS_TRCRSCTLR(9), 	HDFGRTR, TRC, 1),
1765
	SR_FGT(SYS_TRCRSCTLR(10), 	HDFGRTR, TRC, 1),
1766
	SR_FGT(SYS_TRCRSCTLR(11), 	HDFGRTR, TRC, 1),
1767
	SR_FGT(SYS_TRCRSCTLR(12), 	HDFGRTR, TRC, 1),
1768
	SR_FGT(SYS_TRCRSCTLR(13), 	HDFGRTR, TRC, 1),
1769
	SR_FGT(SYS_TRCRSCTLR(14), 	HDFGRTR, TRC, 1),
1770
	SR_FGT(SYS_TRCRSCTLR(15), 	HDFGRTR, TRC, 1),
1771
	SR_FGT(SYS_TRCRSCTLR(16), 	HDFGRTR, TRC, 1),
1772
	SR_FGT(SYS_TRCRSCTLR(17), 	HDFGRTR, TRC, 1),
1773
	SR_FGT(SYS_TRCRSCTLR(18), 	HDFGRTR, TRC, 1),
1774
	SR_FGT(SYS_TRCRSCTLR(19), 	HDFGRTR, TRC, 1),
1775
	SR_FGT(SYS_TRCRSCTLR(20), 	HDFGRTR, TRC, 1),
1776
	SR_FGT(SYS_TRCRSCTLR(21), 	HDFGRTR, TRC, 1),
1777
	SR_FGT(SYS_TRCRSCTLR(22), 	HDFGRTR, TRC, 1),
1778
	SR_FGT(SYS_TRCRSCTLR(23), 	HDFGRTR, TRC, 1),
1779
	SR_FGT(SYS_TRCRSCTLR(24), 	HDFGRTR, TRC, 1),
1780
	SR_FGT(SYS_TRCRSCTLR(25), 	HDFGRTR, TRC, 1),
1781
	SR_FGT(SYS_TRCRSCTLR(26), 	HDFGRTR, TRC, 1),
1782
	SR_FGT(SYS_TRCRSCTLR(27), 	HDFGRTR, TRC, 1),
1783
	SR_FGT(SYS_TRCRSCTLR(28), 	HDFGRTR, TRC, 1),
1784
	SR_FGT(SYS_TRCRSCTLR(29), 	HDFGRTR, TRC, 1),
1785
	SR_FGT(SYS_TRCRSCTLR(30), 	HDFGRTR, TRC, 1),
1786
	SR_FGT(SYS_TRCRSCTLR(31), 	HDFGRTR, TRC, 1),
1787
	SR_FGT(SYS_TRCRSR, 		HDFGRTR, TRC, 1),
1788
	SR_FGT(SYS_TRCSEQEVR(0), 	HDFGRTR, TRC, 1),
1789
	SR_FGT(SYS_TRCSEQEVR(1), 	HDFGRTR, TRC, 1),
1790
	SR_FGT(SYS_TRCSEQEVR(2), 	HDFGRTR, TRC, 1),
1791
	SR_FGT(SYS_TRCSEQRSTEVR, 	HDFGRTR, TRC, 1),
1792
	SR_FGT(SYS_TRCSSCCR(0), 	HDFGRTR, TRC, 1),
1793
	SR_FGT(SYS_TRCSSCCR(1), 	HDFGRTR, TRC, 1),
1794
	SR_FGT(SYS_TRCSSCCR(2), 	HDFGRTR, TRC, 1),
1795
	SR_FGT(SYS_TRCSSCCR(3), 	HDFGRTR, TRC, 1),
1796
	SR_FGT(SYS_TRCSSCCR(4), 	HDFGRTR, TRC, 1),
1797
	SR_FGT(SYS_TRCSSCCR(5), 	HDFGRTR, TRC, 1),
1798
	SR_FGT(SYS_TRCSSCCR(6), 	HDFGRTR, TRC, 1),
1799
	SR_FGT(SYS_TRCSSCCR(7), 	HDFGRTR, TRC, 1),
1800
	SR_FGT(SYS_TRCSSPCICR(0), 	HDFGRTR, TRC, 1),
1801
	SR_FGT(SYS_TRCSSPCICR(1), 	HDFGRTR, TRC, 1),
1802
	SR_FGT(SYS_TRCSSPCICR(2), 	HDFGRTR, TRC, 1),
1803
	SR_FGT(SYS_TRCSSPCICR(3), 	HDFGRTR, TRC, 1),
1804
	SR_FGT(SYS_TRCSSPCICR(4), 	HDFGRTR, TRC, 1),
1805
	SR_FGT(SYS_TRCSSPCICR(5), 	HDFGRTR, TRC, 1),
1806
	SR_FGT(SYS_TRCSSPCICR(6), 	HDFGRTR, TRC, 1),
1807
	SR_FGT(SYS_TRCSSPCICR(7), 	HDFGRTR, TRC, 1),
1808
	SR_FGT(SYS_TRCSTALLCTLR, 	HDFGRTR, TRC, 1),
1809
	SR_FGT(SYS_TRCSYNCPR, 		HDFGRTR, TRC, 1),
1810
	SR_FGT(SYS_TRCTRACEIDR, 	HDFGRTR, TRC, 1),
1811
	SR_FGT(SYS_TRCTSCTLR, 		HDFGRTR, TRC, 1),
1812
	SR_FGT(SYS_TRCVIIECTLR, 	HDFGRTR, TRC, 1),
1813
	SR_FGT(SYS_TRCVIPCSSCTLR, 	HDFGRTR, TRC, 1),
1814
	SR_FGT(SYS_TRCVISSCTLR, 	HDFGRTR, TRC, 1),
1815
	SR_FGT(SYS_TRCVMIDCCTLR0, 	HDFGRTR, TRC, 1),
1816
	SR_FGT(SYS_TRCVMIDCCTLR1, 	HDFGRTR, TRC, 1),
1817
	SR_FGT(SYS_TRCVMIDCVR(0), 	HDFGRTR, TRC, 1),
1818
	SR_FGT(SYS_TRCVMIDCVR(1), 	HDFGRTR, TRC, 1),
1819
	SR_FGT(SYS_TRCVMIDCVR(2), 	HDFGRTR, TRC, 1),
1820
	SR_FGT(SYS_TRCVMIDCVR(3), 	HDFGRTR, TRC, 1),
1821
	SR_FGT(SYS_TRCVMIDCVR(4), 	HDFGRTR, TRC, 1),
1822
	SR_FGT(SYS_TRCVMIDCVR(5), 	HDFGRTR, TRC, 1),
1823
	SR_FGT(SYS_TRCVMIDCVR(6), 	HDFGRTR, TRC, 1),
1824
	SR_FGT(SYS_TRCVMIDCVR(7), 	HDFGRTR, TRC, 1),
1825
	SR_FGT(SYS_PMSLATFR_EL1, 	HDFGRTR, PMSLATFR_EL1, 1),
1826
	SR_FGT(SYS_PMSIRR_EL1, 		HDFGRTR, PMSIRR_EL1, 1),
1827
	SR_FGT(SYS_PMSIDR_EL1, 		HDFGRTR, PMSIDR_EL1, 1),
1828
	SR_FGT(SYS_PMSICR_EL1, 		HDFGRTR, PMSICR_EL1, 1),
1829
	SR_FGT(SYS_PMSFCR_EL1, 		HDFGRTR, PMSFCR_EL1, 1),
1830
	SR_FGT(SYS_PMSEVFR_EL1, 	HDFGRTR, PMSEVFR_EL1, 1),
1831
	SR_FGT(SYS_PMSCR_EL1, 		HDFGRTR, PMSCR_EL1, 1),
1832
	SR_FGT(SYS_PMBSR_EL1, 		HDFGRTR, PMBSR_EL1, 1),
1833
	SR_FGT(SYS_PMBPTR_EL1, 		HDFGRTR, PMBPTR_EL1, 1),
1834
	SR_FGT(SYS_PMBLIMITR_EL1, 	HDFGRTR, PMBLIMITR_EL1, 1),
1835
	SR_FGT(SYS_PMMIR_EL1, 		HDFGRTR, PMMIR_EL1, 1),
1836
	SR_FGT(SYS_PMSELR_EL0, 		HDFGRTR, PMSELR_EL0, 1),
1837
	SR_FGT(SYS_PMOVSCLR_EL0, 	HDFGRTR, PMOVS, 1),
1838
	SR_FGT(SYS_PMOVSSET_EL0, 	HDFGRTR, PMOVS, 1),
1839
	SR_FGT(SYS_PMINTENCLR_EL1, 	HDFGRTR, PMINTEN, 1),
1840
	SR_FGT(SYS_PMINTENSET_EL1, 	HDFGRTR, PMINTEN, 1),
1841
	SR_FGT(SYS_PMCNTENCLR_EL0, 	HDFGRTR, PMCNTEN, 1),
1842
	SR_FGT(SYS_PMCNTENSET_EL0, 	HDFGRTR, PMCNTEN, 1),
1843
	SR_FGT(SYS_PMCCNTR_EL0, 	HDFGRTR, PMCCNTR_EL0, 1),
1844
	SR_FGT(SYS_PMCCFILTR_EL0, 	HDFGRTR, PMCCFILTR_EL0, 1),
1845
	SR_FGT_RANGE(SYS_PMEVTYPERn_EL0(0),
1846
		     SYS_PMEVTYPERn_EL0(30),
1847
		     HDFGRTR, PMEVTYPERn_EL0, 1),
1848
	SR_FGT_RANGE(SYS_PMEVCNTRn_EL0(0),
1849
		     SYS_PMEVCNTRn_EL0(30),
1850
		     HDFGRTR, PMEVCNTRn_EL0, 1),
1851
	SR_FGT(SYS_OSDLR_EL1, 		HDFGRTR, OSDLR_EL1, 1),
1852
	SR_FGT(SYS_OSECCR_EL1, 		HDFGRTR, OSECCR_EL1, 1),
1853
	SR_FGT(SYS_OSLSR_EL1, 		HDFGRTR, OSLSR_EL1, 1),
1854
	SR_FGT(SYS_DBGPRCR_EL1, 	HDFGRTR, DBGPRCR_EL1, 1),
1855
	SR_FGT(SYS_DBGAUTHSTATUS_EL1, 	HDFGRTR, DBGAUTHSTATUS_EL1, 1),
1856
	SR_FGT(SYS_DBGCLAIMSET_EL1, 	HDFGRTR, DBGCLAIM, 1),
1857
	SR_FGT(SYS_DBGCLAIMCLR_EL1, 	HDFGRTR, DBGCLAIM, 1),
1858
	SR_FGT(SYS_MDSCR_EL1, 		HDFGRTR, MDSCR_EL1, 1),
1859
	/*
1860
	 * The trap bits capture *64* debug registers per bit, but the
1861
	 * ARM ARM only describes the encoding for the first 16, and
1862
	 * we don't really support more than that anyway.
1863
	 */
1864
	SR_FGT(SYS_DBGWVRn_EL1(0), 	HDFGRTR, DBGWVRn_EL1, 1),
1865
	SR_FGT(SYS_DBGWVRn_EL1(1), 	HDFGRTR, DBGWVRn_EL1, 1),
1866
	SR_FGT(SYS_DBGWVRn_EL1(2), 	HDFGRTR, DBGWVRn_EL1, 1),
1867
	SR_FGT(SYS_DBGWVRn_EL1(3), 	HDFGRTR, DBGWVRn_EL1, 1),
1868
	SR_FGT(SYS_DBGWVRn_EL1(4), 	HDFGRTR, DBGWVRn_EL1, 1),
1869
	SR_FGT(SYS_DBGWVRn_EL1(5), 	HDFGRTR, DBGWVRn_EL1, 1),
1870
	SR_FGT(SYS_DBGWVRn_EL1(6), 	HDFGRTR, DBGWVRn_EL1, 1),
1871
	SR_FGT(SYS_DBGWVRn_EL1(7), 	HDFGRTR, DBGWVRn_EL1, 1),
1872
	SR_FGT(SYS_DBGWVRn_EL1(8), 	HDFGRTR, DBGWVRn_EL1, 1),
1873
	SR_FGT(SYS_DBGWVRn_EL1(9), 	HDFGRTR, DBGWVRn_EL1, 1),
1874
	SR_FGT(SYS_DBGWVRn_EL1(10), 	HDFGRTR, DBGWVRn_EL1, 1),
1875
	SR_FGT(SYS_DBGWVRn_EL1(11), 	HDFGRTR, DBGWVRn_EL1, 1),
1876
	SR_FGT(SYS_DBGWVRn_EL1(12), 	HDFGRTR, DBGWVRn_EL1, 1),
1877
	SR_FGT(SYS_DBGWVRn_EL1(13), 	HDFGRTR, DBGWVRn_EL1, 1),
1878
	SR_FGT(SYS_DBGWVRn_EL1(14), 	HDFGRTR, DBGWVRn_EL1, 1),
1879
	SR_FGT(SYS_DBGWVRn_EL1(15), 	HDFGRTR, DBGWVRn_EL1, 1),
1880
	SR_FGT(SYS_DBGWCRn_EL1(0), 	HDFGRTR, DBGWCRn_EL1, 1),
1881
	SR_FGT(SYS_DBGWCRn_EL1(1), 	HDFGRTR, DBGWCRn_EL1, 1),
1882
	SR_FGT(SYS_DBGWCRn_EL1(2), 	HDFGRTR, DBGWCRn_EL1, 1),
1883
	SR_FGT(SYS_DBGWCRn_EL1(3), 	HDFGRTR, DBGWCRn_EL1, 1),
1884
	SR_FGT(SYS_DBGWCRn_EL1(4), 	HDFGRTR, DBGWCRn_EL1, 1),
1885
	SR_FGT(SYS_DBGWCRn_EL1(5), 	HDFGRTR, DBGWCRn_EL1, 1),
1886
	SR_FGT(SYS_DBGWCRn_EL1(6), 	HDFGRTR, DBGWCRn_EL1, 1),
1887
	SR_FGT(SYS_DBGWCRn_EL1(7), 	HDFGRTR, DBGWCRn_EL1, 1),
1888
	SR_FGT(SYS_DBGWCRn_EL1(8), 	HDFGRTR, DBGWCRn_EL1, 1),
1889
	SR_FGT(SYS_DBGWCRn_EL1(9), 	HDFGRTR, DBGWCRn_EL1, 1),
1890
	SR_FGT(SYS_DBGWCRn_EL1(10), 	HDFGRTR, DBGWCRn_EL1, 1),
1891
	SR_FGT(SYS_DBGWCRn_EL1(11), 	HDFGRTR, DBGWCRn_EL1, 1),
1892
	SR_FGT(SYS_DBGWCRn_EL1(12), 	HDFGRTR, DBGWCRn_EL1, 1),
1893
	SR_FGT(SYS_DBGWCRn_EL1(13), 	HDFGRTR, DBGWCRn_EL1, 1),
1894
	SR_FGT(SYS_DBGWCRn_EL1(14), 	HDFGRTR, DBGWCRn_EL1, 1),
1895
	SR_FGT(SYS_DBGWCRn_EL1(15), 	HDFGRTR, DBGWCRn_EL1, 1),
1896
	SR_FGT(SYS_DBGBVRn_EL1(0), 	HDFGRTR, DBGBVRn_EL1, 1),
1897
	SR_FGT(SYS_DBGBVRn_EL1(1), 	HDFGRTR, DBGBVRn_EL1, 1),
1898
	SR_FGT(SYS_DBGBVRn_EL1(2), 	HDFGRTR, DBGBVRn_EL1, 1),
1899
	SR_FGT(SYS_DBGBVRn_EL1(3), 	HDFGRTR, DBGBVRn_EL1, 1),
1900
	SR_FGT(SYS_DBGBVRn_EL1(4), 	HDFGRTR, DBGBVRn_EL1, 1),
1901
	SR_FGT(SYS_DBGBVRn_EL1(5), 	HDFGRTR, DBGBVRn_EL1, 1),
1902
	SR_FGT(SYS_DBGBVRn_EL1(6), 	HDFGRTR, DBGBVRn_EL1, 1),
1903
	SR_FGT(SYS_DBGBVRn_EL1(7), 	HDFGRTR, DBGBVRn_EL1, 1),
1904
	SR_FGT(SYS_DBGBVRn_EL1(8), 	HDFGRTR, DBGBVRn_EL1, 1),
1905
	SR_FGT(SYS_DBGBVRn_EL1(9), 	HDFGRTR, DBGBVRn_EL1, 1),
1906
	SR_FGT(SYS_DBGBVRn_EL1(10), 	HDFGRTR, DBGBVRn_EL1, 1),
1907
	SR_FGT(SYS_DBGBVRn_EL1(11), 	HDFGRTR, DBGBVRn_EL1, 1),
1908
	SR_FGT(SYS_DBGBVRn_EL1(12), 	HDFGRTR, DBGBVRn_EL1, 1),
1909
	SR_FGT(SYS_DBGBVRn_EL1(13), 	HDFGRTR, DBGBVRn_EL1, 1),
1910
	SR_FGT(SYS_DBGBVRn_EL1(14), 	HDFGRTR, DBGBVRn_EL1, 1),
1911
	SR_FGT(SYS_DBGBVRn_EL1(15), 	HDFGRTR, DBGBVRn_EL1, 1),
1912
	SR_FGT(SYS_DBGBCRn_EL1(0), 	HDFGRTR, DBGBCRn_EL1, 1),
1913
	SR_FGT(SYS_DBGBCRn_EL1(1), 	HDFGRTR, DBGBCRn_EL1, 1),
1914
	SR_FGT(SYS_DBGBCRn_EL1(2), 	HDFGRTR, DBGBCRn_EL1, 1),
1915
	SR_FGT(SYS_DBGBCRn_EL1(3), 	HDFGRTR, DBGBCRn_EL1, 1),
1916
	SR_FGT(SYS_DBGBCRn_EL1(4), 	HDFGRTR, DBGBCRn_EL1, 1),
1917
	SR_FGT(SYS_DBGBCRn_EL1(5), 	HDFGRTR, DBGBCRn_EL1, 1),
1918
	SR_FGT(SYS_DBGBCRn_EL1(6), 	HDFGRTR, DBGBCRn_EL1, 1),
1919
	SR_FGT(SYS_DBGBCRn_EL1(7), 	HDFGRTR, DBGBCRn_EL1, 1),
1920
	SR_FGT(SYS_DBGBCRn_EL1(8), 	HDFGRTR, DBGBCRn_EL1, 1),
1921
	SR_FGT(SYS_DBGBCRn_EL1(9), 	HDFGRTR, DBGBCRn_EL1, 1),
1922
	SR_FGT(SYS_DBGBCRn_EL1(10), 	HDFGRTR, DBGBCRn_EL1, 1),
1923
	SR_FGT(SYS_DBGBCRn_EL1(11), 	HDFGRTR, DBGBCRn_EL1, 1),
1924
	SR_FGT(SYS_DBGBCRn_EL1(12), 	HDFGRTR, DBGBCRn_EL1, 1),
1925
	SR_FGT(SYS_DBGBCRn_EL1(13), 	HDFGRTR, DBGBCRn_EL1, 1),
1926
	SR_FGT(SYS_DBGBCRn_EL1(14), 	HDFGRTR, DBGBCRn_EL1, 1),
1927
	SR_FGT(SYS_DBGBCRn_EL1(15), 	HDFGRTR, DBGBCRn_EL1, 1),
1928

1929
	/* HDFGRTR2_EL2 */
1930
	SR_FGT(SYS_MDSELR_EL1,		HDFGRTR2, nMDSELR_EL1, 0),
1931
	SR_FGT(SYS_MDSTEPOP_EL1,	HDFGRTR2, nMDSTEPOP_EL1, 0),
1932
	SR_FGT(SYS_PMCCNTSVR_EL1,	HDFGRTR2, nPMSSDATA, 0),
1933
	SR_FGT_RANGE(SYS_PMEVCNTSVRn_EL1(0),
1934
		     SYS_PMEVCNTSVRn_EL1(30),
1935
		     HDFGRTR2, nPMSSDATA, 0),
1936
	SR_FGT(SYS_PMICNTSVR_EL1,	HDFGRTR2, nPMSSDATA, 0),
1937
	SR_FGT(SYS_PMECR_EL1,		HDFGRTR2, nPMECR_EL1, 0),
1938
	SR_FGT(SYS_PMIAR_EL1,		HDFGRTR2, nPMIAR_EL1, 0),
1939
	SR_FGT(SYS_PMICFILTR_EL0,	HDFGRTR2, nPMICFILTR_EL0, 0),
1940
	SR_FGT(SYS_PMICNTR_EL0,		HDFGRTR2, nPMICNTR_EL0, 0),
1941
	SR_FGT(SYS_PMSSCR_EL1,		HDFGRTR2, nPMSSCR_EL1, 0),
1942
	SR_FGT(SYS_PMUACR_EL1,		HDFGRTR2, nPMUACR_EL1, 0),
1943
	SR_FGT(SYS_SPMACCESSR_EL1,	HDFGRTR2, nSPMACCESSR_EL1, 0),
1944
	SR_FGT(SYS_SPMCFGR_EL1,		HDFGRTR2, nSPMID, 0),
1945
	SR_FGT(SYS_SPMDEVARCH_EL1,	HDFGRTR2, nSPMID, 0),
1946
	SR_FGT(SYS_SPMCGCRn_EL1(0),	HDFGRTR2, nSPMID, 0),
1947
	SR_FGT(SYS_SPMCGCRn_EL1(1),	HDFGRTR2, nSPMID, 0),
1948
	SR_FGT(SYS_SPMIIDR_EL1,		HDFGRTR2, nSPMID, 0),
1949
	SR_FGT(SYS_SPMCNTENCLR_EL0,	HDFGRTR2, nSPMCNTEN, 0),
1950
	SR_FGT(SYS_SPMCNTENSET_EL0,	HDFGRTR2, nSPMCNTEN, 0),
1951
	SR_FGT(SYS_SPMCR_EL0,		HDFGRTR2, nSPMCR_EL0, 0),
1952
	SR_FGT(SYS_SPMDEVAFF_EL1,	HDFGRTR2, nSPMDEVAFF_EL1, 0),
1953
	/*
1954
	 * We have up to 64 of these registers in ranges of 16, banked via
1955
	 * SPMSELR_EL0.BANK. We're only concerned with the accessors here,
1956
	 * not the architectural registers.
1957
	 */
1958
	SR_FGT_RANGE(SYS_SPMEVCNTRn_EL0(0),
1959
		     SYS_SPMEVCNTRn_EL0(15),
1960
		     HDFGRTR2, nSPMEVCNTRn_EL0, 0),
1961
	SR_FGT_RANGE(SYS_SPMEVFILT2Rn_EL0(0),
1962
		     SYS_SPMEVFILT2Rn_EL0(15),
1963
		     HDFGRTR2, nSPMEVTYPERn_EL0, 0),
1964
	SR_FGT_RANGE(SYS_SPMEVFILTRn_EL0(0),
1965
		     SYS_SPMEVFILTRn_EL0(15),
1966
		     HDFGRTR2, nSPMEVTYPERn_EL0, 0),
1967
	SR_FGT_RANGE(SYS_SPMEVTYPERn_EL0(0),
1968
		     SYS_SPMEVTYPERn_EL0(15),
1969
		     HDFGRTR2, nSPMEVTYPERn_EL0, 0),
1970
	SR_FGT(SYS_SPMINTENCLR_EL1,	HDFGRTR2, nSPMINTEN, 0),
1971
	SR_FGT(SYS_SPMINTENSET_EL1,	HDFGRTR2, nSPMINTEN, 0),
1972
	SR_FGT(SYS_SPMOVSCLR_EL0,	HDFGRTR2, nSPMOVS, 0),
1973
	SR_FGT(SYS_SPMOVSSET_EL0,	HDFGRTR2, nSPMOVS, 0),
1974
	SR_FGT(SYS_SPMSCR_EL1,		HDFGRTR2, nSPMSCR_EL1, 0),
1975
	SR_FGT(SYS_SPMSELR_EL0,		HDFGRTR2, nSPMSELR_EL0, 0),
1976
	SR_FGT(SYS_TRCITECR_EL1,	HDFGRTR2, nTRCITECR_EL1, 0),
1977
	SR_FGT(SYS_PMBMAR_EL1,		HDFGRTR2, nPMBMAR_EL1, 0),
1978
	SR_FGT(SYS_PMSDSFR_EL1,		HDFGRTR2, nPMSDSFR_EL1, 0),
1979
	SR_FGT(SYS_TRBMPAM_EL1,		HDFGRTR2, nTRBMPAM_EL1, 0),
1980

1981
	/*
1982
	 * HDFGWTR_EL2
1983
	 *
1984
	 * Although HDFGRTR_EL2 and HDFGWTR_EL2 registers largely
1985
	 * overlap in their bit assignment, there are a number of bits
1986
	 * that are RES0 on one side, and an actual trap bit on the
1987
	 * other.  The policy chosen here is to describe all the
1988
	 * read-side mappings, and only the write-side mappings that
1989
	 * differ from the read side, and the trap handler will pick
1990
	 * the correct shadow register based on the access type.
1991
	 *
1992
	 * Same model applies to the FEAT_FGT2 registers.
1993
	 */
1994
	SR_FGT(SYS_TRFCR_EL1,		HDFGWTR, TRFCR_EL1, 1),
1995
	SR_FGT(SYS_TRCOSLAR,		HDFGWTR, TRCOSLAR, 1),
1996
	SR_FGT(SYS_PMCR_EL0,		HDFGWTR, PMCR_EL0, 1),
1997
	SR_FGT(SYS_PMSWINC_EL0,		HDFGWTR, PMSWINC_EL0, 1),
1998
	SR_FGT(SYS_OSLAR_EL1,		HDFGWTR, OSLAR_EL1, 1),
1999

2000
	/* HDFGWTR2_EL2 */
2001
	SR_FGT(SYS_PMZR_EL0,		HDFGWTR2, nPMZR_EL0, 0),
2002
	SR_FGT(SYS_SPMZR_EL0,		HDFGWTR2, nSPMEVCNTRn_EL0, 0),
2003

2004
	/*
2005
	 * HAFGRTR_EL2
2006
	 */
2007
	SR_FGT(SYS_AMEVTYPER1_EL0(15),	HAFGRTR, AMEVTYPER115_EL0, 1),
2008
	SR_FGT(SYS_AMEVTYPER1_EL0(14),	HAFGRTR, AMEVTYPER114_EL0, 1),
2009
	SR_FGT(SYS_AMEVTYPER1_EL0(13),	HAFGRTR, AMEVTYPER113_EL0, 1),
2010
	SR_FGT(SYS_AMEVTYPER1_EL0(12),	HAFGRTR, AMEVTYPER112_EL0, 1),
2011
	SR_FGT(SYS_AMEVTYPER1_EL0(11),	HAFGRTR, AMEVTYPER111_EL0, 1),
2012
	SR_FGT(SYS_AMEVTYPER1_EL0(10),	HAFGRTR, AMEVTYPER110_EL0, 1),
2013
	SR_FGT(SYS_AMEVTYPER1_EL0(9),	HAFGRTR, AMEVTYPER19_EL0, 1),
2014
	SR_FGT(SYS_AMEVTYPER1_EL0(8),	HAFGRTR, AMEVTYPER18_EL0, 1),
2015
	SR_FGT(SYS_AMEVTYPER1_EL0(7),	HAFGRTR, AMEVTYPER17_EL0, 1),
2016
	SR_FGT(SYS_AMEVTYPER1_EL0(6),	HAFGRTR, AMEVTYPER16_EL0, 1),
2017
	SR_FGT(SYS_AMEVTYPER1_EL0(5),	HAFGRTR, AMEVTYPER15_EL0, 1),
2018
	SR_FGT(SYS_AMEVTYPER1_EL0(4),	HAFGRTR, AMEVTYPER14_EL0, 1),
2019
	SR_FGT(SYS_AMEVTYPER1_EL0(3),	HAFGRTR, AMEVTYPER13_EL0, 1),
2020
	SR_FGT(SYS_AMEVTYPER1_EL0(2),	HAFGRTR, AMEVTYPER12_EL0, 1),
2021
	SR_FGT(SYS_AMEVTYPER1_EL0(1),	HAFGRTR, AMEVTYPER11_EL0, 1),
2022
	SR_FGT(SYS_AMEVTYPER1_EL0(0),	HAFGRTR, AMEVTYPER10_EL0, 1),
2023
	SR_FGT(SYS_AMEVCNTR1_EL0(15),	HAFGRTR, AMEVCNTR115_EL0, 1),
2024
	SR_FGT(SYS_AMEVCNTR1_EL0(14),	HAFGRTR, AMEVCNTR114_EL0, 1),
2025
	SR_FGT(SYS_AMEVCNTR1_EL0(13),	HAFGRTR, AMEVCNTR113_EL0, 1),
2026
	SR_FGT(SYS_AMEVCNTR1_EL0(12),	HAFGRTR, AMEVCNTR112_EL0, 1),
2027
	SR_FGT(SYS_AMEVCNTR1_EL0(11),	HAFGRTR, AMEVCNTR111_EL0, 1),
2028
	SR_FGT(SYS_AMEVCNTR1_EL0(10),	HAFGRTR, AMEVCNTR110_EL0, 1),
2029
	SR_FGT(SYS_AMEVCNTR1_EL0(9),	HAFGRTR, AMEVCNTR19_EL0, 1),
2030
	SR_FGT(SYS_AMEVCNTR1_EL0(8),	HAFGRTR, AMEVCNTR18_EL0, 1),
2031
	SR_FGT(SYS_AMEVCNTR1_EL0(7),	HAFGRTR, AMEVCNTR17_EL0, 1),
2032
	SR_FGT(SYS_AMEVCNTR1_EL0(6),	HAFGRTR, AMEVCNTR16_EL0, 1),
2033
	SR_FGT(SYS_AMEVCNTR1_EL0(5),	HAFGRTR, AMEVCNTR15_EL0, 1),
2034
	SR_FGT(SYS_AMEVCNTR1_EL0(4),	HAFGRTR, AMEVCNTR14_EL0, 1),
2035
	SR_FGT(SYS_AMEVCNTR1_EL0(3),	HAFGRTR, AMEVCNTR13_EL0, 1),
2036
	SR_FGT(SYS_AMEVCNTR1_EL0(2),	HAFGRTR, AMEVCNTR12_EL0, 1),
2037
	SR_FGT(SYS_AMEVCNTR1_EL0(1),	HAFGRTR, AMEVCNTR11_EL0, 1),
2038
	SR_FGT(SYS_AMEVCNTR1_EL0(0),	HAFGRTR, AMEVCNTR10_EL0, 1),
2039
	SR_FGT(SYS_AMCNTENCLR1_EL0,	HAFGRTR, AMCNTEN1, 1),
2040
	SR_FGT(SYS_AMCNTENSET1_EL0,	HAFGRTR, AMCNTEN1, 1),
2041
	SR_FGT(SYS_AMCNTENCLR0_EL0,	HAFGRTR, AMCNTEN0, 1),
2042
	SR_FGT(SYS_AMCNTENSET0_EL0,	HAFGRTR, AMCNTEN0, 1),
2043
	SR_FGT(SYS_AMEVCNTR0_EL0(3),	HAFGRTR, AMEVCNTR03_EL0, 1),
2044
	SR_FGT(SYS_AMEVCNTR0_EL0(2),	HAFGRTR, AMEVCNTR02_EL0, 1),
2045
	SR_FGT(SYS_AMEVCNTR0_EL0(1),	HAFGRTR, AMEVCNTR01_EL0, 1),
2046
	SR_FGT(SYS_AMEVCNTR0_EL0(0),	HAFGRTR, AMEVCNTR00_EL0, 1),
2047
};
2048

2049
/*
2050
 * Additional FGTs that do not fire with ESR_EL2.EC==0x18. This table
2051
 * isn't used for exception routing, but only as a promise that the
2052
 * trap is handled somewhere else.
2053
 */
2054
static const union trap_config non_0x18_fgt[] __initconst = {
2055
	FGT(HFGITR, PSBCSYNC, 1),
2056
	FGT(HFGITR, nGCSSTR_EL1, 0),
2057
	FGT(HFGITR, SVC_EL1, 1),
2058
	FGT(HFGITR, SVC_EL0, 1),
2059
	FGT(HFGITR, ERET, 1),
2060
	FGT(HFGITR2, TSBCSYNC, 1),
2061
};
2062

2063
static union trap_config get_trap_config(u32 sysreg)
2064
{
2065
	return (union trap_config) {
2066
		.val = xa_to_value(xa_load(&sr_forward_xa, sysreg)),
2067
	};
2068
}
2069

2070
static __init void print_nv_trap_error(const struct encoding_to_trap_config *tc,
2071
				       const char *type, int err)
2072
{
2073
	kvm_err("%s line %d encoding range "
2074
		"(%d, %d, %d, %d, %d) - (%d, %d, %d, %d, %d) (err=%d)\n",
2075
		type, tc->line,
2076
		sys_reg_Op0(tc->encoding), sys_reg_Op1(tc->encoding),
2077
		sys_reg_CRn(tc->encoding), sys_reg_CRm(tc->encoding),
2078
		sys_reg_Op2(tc->encoding),
2079
		sys_reg_Op0(tc->end), sys_reg_Op1(tc->end),
2080
		sys_reg_CRn(tc->end), sys_reg_CRm(tc->end),
2081
		sys_reg_Op2(tc->end),
2082
		err);
2083
}
2084

2085
static u32 encoding_next(u32 encoding)
2086
{
2087
	u8 op0, op1, crn, crm, op2;
2088

2089
	op0 = sys_reg_Op0(encoding);
2090
	op1 = sys_reg_Op1(encoding);
2091
	crn = sys_reg_CRn(encoding);
2092
	crm = sys_reg_CRm(encoding);
2093
	op2 = sys_reg_Op2(encoding);
2094

2095
	if (op2 < Op2_mask)
2096
		return sys_reg(op0, op1, crn, crm, op2 + 1);
2097
	if (crm < CRm_mask)
2098
		return sys_reg(op0, op1, crn, crm + 1, 0);
2099
	if (crn < CRn_mask)
2100
		return sys_reg(op0, op1, crn + 1, 0, 0);
2101
	if (op1 < Op1_mask)
2102
		return sys_reg(op0, op1 + 1, 0, 0, 0);
2103

2104
	return sys_reg(op0 + 1, 0, 0, 0, 0);
2105
}
2106

2107
#define FGT_MASKS(__n, __m)						\
2108
	struct fgt_masks __n = { .str = #__m, .res0 = __m, }
2109

2110
FGT_MASKS(hfgrtr_masks, HFGRTR_EL2_RES0);
2111
FGT_MASKS(hfgwtr_masks, HFGWTR_EL2_RES0);
2112
FGT_MASKS(hfgitr_masks, HFGITR_EL2_RES0);
2113
FGT_MASKS(hdfgrtr_masks, HDFGRTR_EL2_RES0);
2114
FGT_MASKS(hdfgwtr_masks, HDFGWTR_EL2_RES0);
2115
FGT_MASKS(hafgrtr_masks, HAFGRTR_EL2_RES0);
2116
FGT_MASKS(hfgrtr2_masks, HFGRTR2_EL2_RES0);
2117
FGT_MASKS(hfgwtr2_masks, HFGWTR2_EL2_RES0);
2118
FGT_MASKS(hfgitr2_masks, HFGITR2_EL2_RES0);
2119
FGT_MASKS(hdfgrtr2_masks, HDFGRTR2_EL2_RES0);
2120
FGT_MASKS(hdfgwtr2_masks, HDFGWTR2_EL2_RES0);
2121

2122
static __init bool aggregate_fgt(union trap_config tc)
2123
{
2124
	struct fgt_masks *rmasks, *wmasks;
2125

2126
	switch (tc.fgt) {
2127
	case HFGRTR_GROUP:
2128
		rmasks = &hfgrtr_masks;
2129
		wmasks = &hfgwtr_masks;
2130
		break;
2131
	case HDFGRTR_GROUP:
2132
		rmasks = &hdfgrtr_masks;
2133
		wmasks = &hdfgwtr_masks;
2134
		break;
2135
	case HAFGRTR_GROUP:
2136
		rmasks = &hafgrtr_masks;
2137
		wmasks = NULL;
2138
		break;
2139
	case HFGITR_GROUP:
2140
		rmasks = &hfgitr_masks;
2141
		wmasks = NULL;
2142
		break;
2143
	case HFGRTR2_GROUP:
2144
		rmasks = &hfgrtr2_masks;
2145
		wmasks = &hfgwtr2_masks;
2146
		break;
2147
	case HDFGRTR2_GROUP:
2148
		rmasks = &hdfgrtr2_masks;
2149
		wmasks = &hdfgwtr2_masks;
2150
		break;
2151
	case HFGITR2_GROUP:
2152
		rmasks = &hfgitr2_masks;
2153
		wmasks = NULL;
2154
		break;
2155
	}
2156

2157
	/*
2158
	 * A bit can be reserved in either the R or W register, but
2159
	 * not both.
2160
	 */
2161
	if ((BIT(tc.bit) & rmasks->res0) &&
2162
	    (!wmasks || (BIT(tc.bit) & wmasks->res0)))
2163
		return false;
2164

2165
	if (tc.pol)
2166
		rmasks->mask |= BIT(tc.bit) & ~rmasks->res0;
2167
	else
2168
		rmasks->nmask |= BIT(tc.bit) & ~rmasks->res0;
2169

2170
	if (wmasks) {
2171
		if (tc.pol)
2172
			wmasks->mask |= BIT(tc.bit) & ~wmasks->res0;
2173
		else
2174
			wmasks->nmask |= BIT(tc.bit) & ~wmasks->res0;
2175
	}
2176

2177
	return true;
2178
}
2179

2180
static __init int check_fgt_masks(struct fgt_masks *masks)
2181
{
2182
	unsigned long duplicate = masks->mask & masks->nmask;
2183
	u64 res0 = masks->res0;
2184
	int ret = 0;
2185

2186
	if (duplicate) {
2187
		int i;
2188

2189
		for_each_set_bit(i, &duplicate, 64) {
2190
			kvm_err("%s[%d] bit has both polarities\n",
2191
				masks->str, i);
2192
		}
2193

2194
		ret = -EINVAL;
2195
	}
2196

2197
	masks->res0 = ~(masks->mask | masks->nmask);
2198
	if (masks->res0 != res0)
2199
		kvm_info("Implicit %s = %016llx, expecting %016llx\n",
2200
			 masks->str, masks->res0, res0);
2201

2202
	return ret;
2203
}
2204

2205
static __init int check_all_fgt_masks(int ret)
2206
{
2207
	static struct fgt_masks * const masks[] __initconst = {
2208
		&hfgrtr_masks,
2209
		&hfgwtr_masks,
2210
		&hfgitr_masks,
2211
		&hdfgrtr_masks,
2212
		&hdfgwtr_masks,
2213
		&hafgrtr_masks,
2214
		&hfgrtr2_masks,
2215
		&hfgwtr2_masks,
2216
		&hfgitr2_masks,
2217
		&hdfgrtr2_masks,
2218
		&hdfgwtr2_masks,
2219
	};
2220
	int err = 0;
2221

2222
	for (int i = 0; i < ARRAY_SIZE(masks); i++)
2223
		err |= check_fgt_masks(masks[i]);
2224

2225
	return ret ?: err;
2226
}
2227

2228
#define for_each_encoding_in(__x, __s, __e)	\
2229
	for (u32 __x = __s; __x <= __e; __x = encoding_next(__x))
2230

2231
int __init populate_nv_trap_config(void)
2232
{
2233
	int ret = 0;
2234

2235
	BUILD_BUG_ON(sizeof(union trap_config) != sizeof(void *));
2236
	BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS));
2237
	BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS));
2238
	BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS));
2239
	BUILD_BUG_ON(__HCRX_EL2_MASK & __HCRX_EL2_nMASK);
2240

2241
	for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) {
2242
		const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i];
2243
		void *prev;
2244

2245
		if (cgt->tc.val & BIT(63)) {
2246
			kvm_err("CGT[%d] has MBZ bit set\n", i);
2247
			ret = -EINVAL;
2248
		}
2249

2250
		for_each_encoding_in(enc, cgt->encoding, cgt->end) {
2251
			prev = xa_store(&sr_forward_xa, enc,
2252
					xa_mk_value(cgt->tc.val), GFP_KERNEL);
2253
			if (prev && !xa_is_err(prev)) {
2254
				ret = -EINVAL;
2255
				print_nv_trap_error(cgt, "Duplicate CGT", ret);
2256
			}
2257

2258
			if (xa_is_err(prev)) {
2259
				ret = xa_err(prev);
2260
				print_nv_trap_error(cgt, "Failed CGT insertion", ret);
2261
			}
2262
		}
2263
	}
2264

2265
	if (__HCRX_EL2_RES0 != HCRX_EL2_RES0)
2266
		kvm_info("Sanitised HCR_EL2_RES0 = %016llx, expecting %016llx\n",
2267
			 __HCRX_EL2_RES0, HCRX_EL2_RES0);
2268

2269
	kvm_info("nv: %ld coarse grained trap handlers\n",
2270
		 ARRAY_SIZE(encoding_to_cgt));
2271

2272
	if (!cpus_have_final_cap(ARM64_HAS_FGT))
2273
		goto check_mcb;
2274

2275
	for (int i = 0; i < ARRAY_SIZE(encoding_to_fgt); i++) {
2276
		const struct encoding_to_trap_config *fgt = &encoding_to_fgt[i];
2277
		union trap_config tc;
2278
		void *prev;
2279

2280
		if (fgt->tc.fgt >= __NR_FGT_GROUP_IDS__) {
2281
			ret = -EINVAL;
2282
			print_nv_trap_error(fgt, "Invalid FGT", ret);
2283
		}
2284

2285
		for_each_encoding_in(enc, fgt->encoding, fgt->end) {
2286
			tc = get_trap_config(enc);
2287

2288
			if (tc.fgt) {
2289
				ret = -EINVAL;
2290
				print_nv_trap_error(fgt, "Duplicate FGT", ret);
2291
			}
2292

2293
			tc.val |= fgt->tc.val;
2294
			prev = xa_store(&sr_forward_xa, enc,
2295
					xa_mk_value(tc.val), GFP_KERNEL);
2296

2297
			if (xa_is_err(prev)) {
2298
				ret = xa_err(prev);
2299
				print_nv_trap_error(fgt, "Failed FGT insertion", ret);
2300
			}
2301

2302
			if (!aggregate_fgt(tc)) {
2303
				ret = -EINVAL;
2304
				print_nv_trap_error(fgt, "FGT bit is reserved", ret);
2305
			}
2306
		}
2307
	}
2308

2309
	for (int i = 0; i < ARRAY_SIZE(non_0x18_fgt); i++) {
2310
		if (!aggregate_fgt(non_0x18_fgt[i])) {
2311
			ret = -EINVAL;
2312
			kvm_err("non_0x18_fgt[%d] is reserved\n", i);
2313
		}
2314
	}
2315

2316
	ret = check_all_fgt_masks(ret);
2317

2318
	kvm_info("nv: %ld fine grained trap handlers\n",
2319
		 ARRAY_SIZE(encoding_to_fgt));
2320

2321
check_mcb:
2322
	for (int id = __MULTIPLE_CONTROL_BITS__; id < __COMPLEX_CONDITIONS__; id++) {
2323
		const enum cgt_group_id *cgids;
2324

2325
		cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
2326

2327
		for (int i = 0; cgids[i] != __RESERVED__; i++) {
2328
			if (cgids[i] >= __MULTIPLE_CONTROL_BITS__ &&
2329
			    cgids[i] < __COMPLEX_CONDITIONS__) {
2330
				kvm_err("Recursive MCB %d/%d\n", id, cgids[i]);
2331
				ret = -EINVAL;
2332
			}
2333
		}
2334
	}
2335

2336
	if (ret)
2337
		xa_destroy(&sr_forward_xa);
2338

2339
	return ret;
2340
}
2341

2342
int __init populate_sysreg_config(const struct sys_reg_desc *sr,
2343
				  unsigned int idx)
2344
{
2345
	union trap_config tc;
2346
	u32 encoding;
2347
	void *ret;
2348

2349
	/*
2350
	 * 0 is a valid value for the index, but not for the storage.
2351
	 * We'll store (idx+1), so check against an offset'd limit.
2352
	 */
2353
	if (idx >= (BIT(TC_SRI_BITS) - 1)) {
2354
		kvm_err("sysreg %s (%d) out of range\n", sr->name, idx);
2355
		return -EINVAL;
2356
	}
2357

2358
	encoding = sys_reg(sr->Op0, sr->Op1, sr->CRn, sr->CRm, sr->Op2);
2359
	tc = get_trap_config(encoding);
2360

2361
	if (tc.sri) {
2362
		kvm_err("sysreg %s (%d) duplicate entry (%d)\n",
2363
			sr->name, idx - 1, tc.sri);
2364
		return -EINVAL;
2365
	}
2366

2367
	tc.sri = idx + 1;
2368
	ret = xa_store(&sr_forward_xa, encoding,
2369
		       xa_mk_value(tc.val), GFP_KERNEL);
2370

2371
	return xa_err(ret);
2372
}
2373

2374
static enum trap_behaviour get_behaviour(struct kvm_vcpu *vcpu,
2375
					 const struct trap_bits *tb)
2376
{
2377
	enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
2378
	u64 val;
2379

2380
	val = __vcpu_sys_reg(vcpu, tb->index);
2381
	if ((val & tb->mask) == tb->value)
2382
		b |= tb->behaviour;
2383

2384
	return b;
2385
}
2386

2387
static enum trap_behaviour __compute_trap_behaviour(struct kvm_vcpu *vcpu,
2388
						    const enum cgt_group_id id,
2389
						    enum trap_behaviour b)
2390
{
2391
	switch (id) {
2392
		const enum cgt_group_id *cgids;
2393

2394
	case __RESERVED__ ... __MULTIPLE_CONTROL_BITS__ - 1:
2395
		if (likely(id != __RESERVED__))
2396
			b |= get_behaviour(vcpu, &coarse_trap_bits[id]);
2397
		break;
2398
	case __MULTIPLE_CONTROL_BITS__ ... __COMPLEX_CONDITIONS__ - 1:
2399
		/* Yes, this is recursive. Don't do anything stupid. */
2400
		cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
2401
		for (int i = 0; cgids[i] != __RESERVED__; i++)
2402
			b |= __compute_trap_behaviour(vcpu, cgids[i], b);
2403
		break;
2404
	default:
2405
		if (ARRAY_SIZE(ccc))
2406
			b |= ccc[id -  __COMPLEX_CONDITIONS__](vcpu);
2407
		break;
2408
	}
2409

2410
	return b;
2411
}
2412

2413
static enum trap_behaviour compute_trap_behaviour(struct kvm_vcpu *vcpu,
2414
						  const union trap_config tc)
2415
{
2416
	enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
2417

2418
	return __compute_trap_behaviour(vcpu, tc.cgt, b);
2419
}
2420

2421
static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr)
2422
{
2423
	struct kvm_sysreg_masks *masks;
2424

2425
	/* Only handle the VNCR-backed regs for now */
2426
	if (sr < __VNCR_START__)
2427
		return 0;
2428

2429
	masks = kvm->arch.sysreg_masks;
2430

2431
	return masks->mask[sr - __VNCR_START__].res0;
2432
}
2433

2434
static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr,
2435
			  const union trap_config tc)
2436
{
2437
	struct kvm *kvm = vcpu->kvm;
2438
	u64 val;
2439

2440
	/*
2441
	 * KVM doesn't know about any FGTs that apply to the host, and hopefully
2442
	 * that'll remain the case.
2443
	 */
2444
	if (is_hyp_ctxt(vcpu))
2445
		return false;
2446

2447
	val = __vcpu_sys_reg(vcpu, sr);
2448

2449
	if (tc.pol)
2450
		return (val & BIT(tc.bit));
2451

2452
	/*
2453
	 * FGTs with negative polarities are an absolute nightmare, as
2454
	 * we need to evaluate the bit in the light of the feature
2455
	 * that defines it. WTF were they thinking?
2456
	 *
2457
	 * So let's check if the bit has been earmarked as RES0, as
2458
	 * this indicates an unimplemented feature.
2459
	 */
2460
	if (val & BIT(tc.bit))
2461
		return false;
2462

2463
	return !(kvm_get_sysreg_res0(kvm, sr) & BIT(tc.bit));
2464
}
2465

2466
bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index)
2467
{
2468
	enum vcpu_sysreg fgtreg;
2469
	union trap_config tc;
2470
	enum trap_behaviour b;
2471
	bool is_read;
2472
	u32 sysreg;
2473
	u64 esr;
2474

2475
	esr = kvm_vcpu_get_esr(vcpu);
2476
	sysreg = esr_sys64_to_sysreg(esr);
2477
	is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
2478

2479
	tc = get_trap_config(sysreg);
2480

2481
	/*
2482
	 * A value of 0 for the whole entry means that we know nothing
2483
	 * for this sysreg, and that it cannot be re-injected into the
2484
	 * nested hypervisor. In this situation, let's cut it short.
2485
	 */
2486
	if (!tc.val)
2487
		goto local;
2488

2489
	/*
2490
	 * If a sysreg can be trapped using a FGT, first check whether we
2491
	 * trap for the purpose of forbidding the feature. In that case,
2492
	 * inject an UNDEF.
2493
	 */
2494
	if (tc.fgt != __NO_FGT_GROUP__ &&
2495
	    (vcpu->kvm->arch.fgu[tc.fgt] & BIT(tc.bit))) {
2496
		kvm_inject_undefined(vcpu);
2497
		return true;
2498
	}
2499

2500
	/*
2501
	 * If we're not nesting, immediately return to the caller, with the
2502
	 * sysreg index, should we have it.
2503
	 */
2504
	if (!vcpu_has_nv(vcpu))
2505
		goto local;
2506

2507
	/*
2508
	 * There are a few traps that take effect InHost, but are constrained
2509
	 * to EL0. Don't bother with computing the trap behaviour if the vCPU
2510
	 * isn't in EL0.
2511
	 */
2512
	if (is_hyp_ctxt(vcpu) && !vcpu_is_host_el0(vcpu))
2513
		goto local;
2514

2515
	switch ((enum fgt_group_id)tc.fgt) {
2516
	case __NO_FGT_GROUP__:
2517
		break;
2518

2519
	case HFGRTR_GROUP:
2520
		fgtreg = is_read ? HFGRTR_EL2 : HFGWTR_EL2;
2521
		break;
2522

2523
	case HDFGRTR_GROUP:
2524
		fgtreg = is_read ? HDFGRTR_EL2 : HDFGWTR_EL2;
2525
		break;
2526

2527
	case HAFGRTR_GROUP:
2528
		fgtreg = HAFGRTR_EL2;
2529
		break;
2530

2531
	case HFGITR_GROUP:
2532
		fgtreg = HFGITR_EL2;
2533
		switch (tc.fgf) {
2534
			u64 tmp;
2535

2536
		case __NO_FGF__:
2537
			break;
2538

2539
		case HCRX_FGTnXS:
2540
			tmp = __vcpu_sys_reg(vcpu, HCRX_EL2);
2541
			if (tmp & HCRX_EL2_FGTnXS)
2542
				tc.fgt = __NO_FGT_GROUP__;
2543
		}
2544
		break;
2545

2546
	case HFGRTR2_GROUP:
2547
		fgtreg = is_read ? HFGRTR2_EL2 : HFGWTR2_EL2;
2548
		break;
2549

2550
	case HDFGRTR2_GROUP:
2551
		fgtreg = is_read ? HDFGRTR2_EL2 : HDFGWTR2_EL2;
2552
		break;
2553

2554
	case HFGITR2_GROUP:
2555
		fgtreg = HFGITR2_EL2;
2556
		break;
2557

2558
	default:
2559
		/* Something is really wrong, bail out */
2560
		WARN_ONCE(1, "Bad FGT group (encoding %08x, config %016llx)\n",
2561
			  sysreg, tc.val);
2562
		goto local;
2563
	}
2564

2565
	if (tc.fgt != __NO_FGT_GROUP__ && check_fgt_bit(vcpu, fgtreg, tc))
2566
		goto inject;
2567

2568
	b = compute_trap_behaviour(vcpu, tc);
2569

2570
	if (!(b & BEHAVE_FORWARD_IN_HOST_EL0) && vcpu_is_host_el0(vcpu))
2571
		goto local;
2572

2573
	if (((b & BEHAVE_FORWARD_READ) && is_read) ||
2574
	    ((b & BEHAVE_FORWARD_WRITE) && !is_read))
2575
		goto inject;
2576

2577
local:
2578
	if (!tc.sri) {
2579
		struct sys_reg_params params;
2580

2581
		params = esr_sys64_to_params(esr);
2582

2583
		/*
2584
		 * Check for the IMPDEF range, as per DDI0487 J.a,
2585
		 * D18.3.2 Reserved encodings for IMPLEMENTATION
2586
		 * DEFINED registers.
2587
		 */
2588
		if (!(params.Op0 == 3 && (params.CRn & 0b1011) == 0b1011))
2589
			print_sys_reg_msg(&params,
2590
					  "Unsupported guest access at: %lx\n",
2591
					  *vcpu_pc(vcpu));
2592
		kvm_inject_undefined(vcpu);
2593
		return true;
2594
	}
2595

2596
	*sr_index = tc.sri - 1;
2597
	return false;
2598

2599
inject:
2600
	trace_kvm_forward_sysreg_trap(vcpu, sysreg, is_read);
2601

2602
	kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
2603
	return true;
2604
}
2605

2606
static bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit)
2607
{
2608
	if (is_nested_ctxt(vcpu) &&
2609
	    (__vcpu_sys_reg(vcpu, reg) & control_bit)) {
2610
		kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
2611
		return true;
2612
	}
2613
	return false;
2614
}
2615

2616
static bool forward_hcr_traps(struct kvm_vcpu *vcpu, u64 control_bit)
2617
{
2618
	return __forward_traps(vcpu, HCR_EL2, control_bit);
2619
}
2620

2621
bool forward_smc_trap(struct kvm_vcpu *vcpu)
2622
{
2623
	return forward_hcr_traps(vcpu, HCR_TSC);
2624
}
2625

2626
static bool forward_mdcr_traps(struct kvm_vcpu *vcpu, u64 control_bit)
2627
{
2628
	return __forward_traps(vcpu, MDCR_EL2, control_bit);
2629
}
2630

2631
bool forward_debug_exception(struct kvm_vcpu *vcpu)
2632
{
2633
	return forward_mdcr_traps(vcpu, MDCR_EL2_TDE);
2634
}
2635

2636
static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr)
2637
{
2638
	u64 mode = spsr & PSR_MODE_MASK;
2639

2640
	/*
2641
	 * Possible causes for an Illegal Exception Return from EL2:
2642
	 * - trying to return to EL3
2643
	 * - trying to return to an illegal M value
2644
	 * - trying to return to a 32bit EL
2645
	 * - trying to return to EL1 with HCR_EL2.TGE set
2646
	 */
2647
	if (mode == PSR_MODE_EL3t   || mode == PSR_MODE_EL3h ||
2648
	    mode == 0b00001         || (mode & BIT(1))       ||
2649
	    (spsr & PSR_MODE32_BIT) ||
2650
	    (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
2651
					   mode == PSR_MODE_EL1h))) {
2652
		/*
2653
		 * The guest is playing with our nerves. Preserve EL, SP,
2654
		 * masks, flags from the existing PSTATE, and set IL.
2655
		 * The HW will then generate an Illegal State Exception
2656
		 * immediately after ERET.
2657
		 */
2658
		spsr = *vcpu_cpsr(vcpu);
2659

2660
		spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
2661
			 PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
2662
			 PSR_MODE_MASK | PSR_MODE32_BIT);
2663
		spsr |= PSR_IL_BIT;
2664
	}
2665

2666
	return spsr;
2667
}
2668

2669
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)
2670
{
2671
	u64 spsr, elr, esr;
2672

2673
	spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
2674
	spsr = kvm_check_illegal_exception_return(vcpu, spsr);
2675

2676
	/* Check for an ERETAx */
2677
	esr = kvm_vcpu_get_esr(vcpu);
2678
	if (esr_iss_is_eretax(esr) && !kvm_auth_eretax(vcpu, &elr)) {
2679
		/*
2680
		 * Oh no, ERETAx failed to authenticate.
2681
		 *
2682
		 * If we have FPACCOMBINE and we don't have a pending
2683
		 * Illegal Execution State exception (which has priority
2684
		 * over FPAC), deliver an exception right away.
2685
		 *
2686
		 * Otherwise, let the mangled ELR value trickle down the
2687
		 * ERET handling, and the guest will have a little surprise.
2688
		 */
2689
		if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) {
2690
			esr &= ESR_ELx_ERET_ISS_ERETA;
2691
			esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC);
2692
			kvm_inject_nested_sync(vcpu, esr);
2693
			return;
2694
		}
2695
	}
2696

2697
	preempt_disable();
2698
	vcpu_set_flag(vcpu, IN_NESTED_ERET);
2699
	kvm_arch_vcpu_put(vcpu);
2700

2701
	if (!esr_iss_is_eretax(esr))
2702
		elr = __vcpu_sys_reg(vcpu, ELR_EL2);
2703

2704
	trace_kvm_nested_eret(vcpu, elr, spsr);
2705

2706
	*vcpu_pc(vcpu) = elr;
2707
	*vcpu_cpsr(vcpu) = spsr;
2708

2709
	kvm_arch_vcpu_load(vcpu, smp_processor_id());
2710
	vcpu_clear_flag(vcpu, IN_NESTED_ERET);
2711
	preempt_enable();
2712

2713
	if (kvm_vcpu_has_pmu(vcpu))
2714
		kvm_pmu_nested_transition(vcpu);
2715
}
2716

2717
static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
2718
				     enum exception_type type)
2719
{
2720
	trace_kvm_inject_nested_exception(vcpu, esr_el2, type);
2721

2722
	switch (type) {
2723
	case except_type_sync:
2724
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
2725
		vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
2726
		break;
2727
	case except_type_irq:
2728
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
2729
		break;
2730
	case except_type_serror:
2731
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
2732
		break;
2733
	default:
2734
		WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
2735
	}
2736
}
2737

2738
/*
2739
 * Emulate taking an exception to EL2.
2740
 * See ARM ARM J8.1.2 AArch64.TakeException()
2741
 */
2742
static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
2743
			     enum exception_type type)
2744
{
2745
	u64 pstate, mode;
2746
	bool direct_inject;
2747

2748
	if (!vcpu_has_nv(vcpu)) {
2749
		kvm_err("Unexpected call to %s for the non-nesting configuration\n",
2750
				__func__);
2751
		return -EINVAL;
2752
	}
2753

2754
	/*
2755
	 * As for ERET, we can avoid doing too much on the injection path by
2756
	 * checking that we either took the exception from a VHE host
2757
	 * userspace or from vEL2. In these cases, there is no change in
2758
	 * translation regime (or anything else), so let's do as little as
2759
	 * possible.
2760
	 */
2761
	pstate = *vcpu_cpsr(vcpu);
2762
	mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT);
2763

2764
	direct_inject  = (mode == PSR_MODE_EL0t &&
2765
			  vcpu_el2_e2h_is_set(vcpu) &&
2766
			  vcpu_el2_tge_is_set(vcpu));
2767
	direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
2768

2769
	if (direct_inject) {
2770
		kvm_inject_el2_exception(vcpu, esr_el2, type);
2771
		return 1;
2772
	}
2773

2774
	preempt_disable();
2775

2776
	/*
2777
	 * We may have an exception or PC update in the EL0/EL1 context.
2778
	 * Commit it before entering EL2.
2779
	 */
2780
	__kvm_adjust_pc(vcpu);
2781

2782
	kvm_arch_vcpu_put(vcpu);
2783

2784
	kvm_inject_el2_exception(vcpu, esr_el2, type);
2785

2786
	/*
2787
	 * A hard requirement is that a switch between EL1 and EL2
2788
	 * contexts has to happen between a put/load, so that we can
2789
	 * pick the correct timer and interrupt configuration, among
2790
	 * other things.
2791
	 *
2792
	 * Make sure the exception actually took place before we load
2793
	 * the new context.
2794
	 */
2795
	__kvm_adjust_pc(vcpu);
2796

2797
	kvm_arch_vcpu_load(vcpu, smp_processor_id());
2798
	preempt_enable();
2799

2800
	if (kvm_vcpu_has_pmu(vcpu))
2801
		kvm_pmu_nested_transition(vcpu);
2802

2803
	return 1;
2804
}
2805

2806
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2)
2807
{
2808
	return kvm_inject_nested(vcpu, esr_el2, except_type_sync);
2809
}
2810

2811
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
2812
{
2813
	/*
2814
	 * Do not inject an irq if the:
2815
	 *  - Current exception level is EL2, and
2816
	 *  - virtual HCR_EL2.TGE == 0
2817
	 *  - virtual HCR_EL2.IMO == 0
2818
	 *
2819
	 * See Table D1-17 "Physical interrupt target and masking when EL3 is
2820
	 * not implemented and EL2 is implemented" in ARM DDI 0487C.a.
2821
	 */
2822

2823
	if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
2824
	    !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
2825
		return 1;
2826

2827
	/* esr_el2 value doesn't matter for exits due to irqs. */
2828
	return kvm_inject_nested(vcpu, 0, except_type_irq);
2829
}
2830

2831
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
2832
{
2833
	u64 esr = FIELD_PREP(ESR_ELx_EC_MASK,
2834
			     iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW);
2835
	esr |= ESR_ELx_FSC_EXTABT | ESR_ELx_IL;
2836

2837
	vcpu_write_sys_reg(vcpu, addr, FAR_EL2);
2838

2839
	if (__vcpu_sys_reg(vcpu, SCTLR2_EL2) & SCTLR2_EL1_EASE)
2840
		return kvm_inject_nested(vcpu, esr, except_type_serror);
2841

2842
	return kvm_inject_nested_sync(vcpu, esr);
2843
}
2844

2845
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr)
2846
{
2847
	/*
2848
	 * Hardware sets up the EC field when propagating ESR as a result of
2849
	 * vSError injection. Manually populate EC for an emulated SError
2850
	 * exception.
2851
	 */
2852
	esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
2853
	return kvm_inject_nested(vcpu, esr, except_type_serror);
2854
}
2855

2856