Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
torvalds
GitHub Repository: torvalds/linux
Path: blob/master/arch/arm64/kvm/hyp/vgic-v3-sr.c
29524 views
1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
* Copyright (C) 2012-2015 - ARM Ltd
4
* Author: Marc Zyngier <[email protected]>
5
*/
6
7
#include <hyp/adjust_pc.h>
8
9
#include <linux/compiler.h>
10
#include <linux/irqchip/arm-gic-v3.h>
11
#include <linux/kvm_host.h>
12
13
#include <asm/kvm_emulate.h>
14
#include <asm/kvm_hyp.h>
15
#include <asm/kvm_mmu.h>
16
17
#define vtr_to_max_lr_idx(v) ((v) & 0xf)
18
#define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
19
#define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
20
21
u64 __gic_v3_get_lr(unsigned int lr)
22
{
23
switch (lr & 0xf) {
24
case 0:
25
return read_gicreg(ICH_LR0_EL2);
26
case 1:
27
return read_gicreg(ICH_LR1_EL2);
28
case 2:
29
return read_gicreg(ICH_LR2_EL2);
30
case 3:
31
return read_gicreg(ICH_LR3_EL2);
32
case 4:
33
return read_gicreg(ICH_LR4_EL2);
34
case 5:
35
return read_gicreg(ICH_LR5_EL2);
36
case 6:
37
return read_gicreg(ICH_LR6_EL2);
38
case 7:
39
return read_gicreg(ICH_LR7_EL2);
40
case 8:
41
return read_gicreg(ICH_LR8_EL2);
42
case 9:
43
return read_gicreg(ICH_LR9_EL2);
44
case 10:
45
return read_gicreg(ICH_LR10_EL2);
46
case 11:
47
return read_gicreg(ICH_LR11_EL2);
48
case 12:
49
return read_gicreg(ICH_LR12_EL2);
50
case 13:
51
return read_gicreg(ICH_LR13_EL2);
52
case 14:
53
return read_gicreg(ICH_LR14_EL2);
54
case 15:
55
return read_gicreg(ICH_LR15_EL2);
56
}
57
58
unreachable();
59
}
60
61
static void __gic_v3_set_lr(u64 val, int lr)
62
{
63
switch (lr & 0xf) {
64
case 0:
65
write_gicreg(val, ICH_LR0_EL2);
66
break;
67
case 1:
68
write_gicreg(val, ICH_LR1_EL2);
69
break;
70
case 2:
71
write_gicreg(val, ICH_LR2_EL2);
72
break;
73
case 3:
74
write_gicreg(val, ICH_LR3_EL2);
75
break;
76
case 4:
77
write_gicreg(val, ICH_LR4_EL2);
78
break;
79
case 5:
80
write_gicreg(val, ICH_LR5_EL2);
81
break;
82
case 6:
83
write_gicreg(val, ICH_LR6_EL2);
84
break;
85
case 7:
86
write_gicreg(val, ICH_LR7_EL2);
87
break;
88
case 8:
89
write_gicreg(val, ICH_LR8_EL2);
90
break;
91
case 9:
92
write_gicreg(val, ICH_LR9_EL2);
93
break;
94
case 10:
95
write_gicreg(val, ICH_LR10_EL2);
96
break;
97
case 11:
98
write_gicreg(val, ICH_LR11_EL2);
99
break;
100
case 12:
101
write_gicreg(val, ICH_LR12_EL2);
102
break;
103
case 13:
104
write_gicreg(val, ICH_LR13_EL2);
105
break;
106
case 14:
107
write_gicreg(val, ICH_LR14_EL2);
108
break;
109
case 15:
110
write_gicreg(val, ICH_LR15_EL2);
111
break;
112
}
113
}
114
115
static void __vgic_v3_write_ap0rn(u32 val, int n)
116
{
117
switch (n) {
118
case 0:
119
write_gicreg(val, ICH_AP0R0_EL2);
120
break;
121
case 1:
122
write_gicreg(val, ICH_AP0R1_EL2);
123
break;
124
case 2:
125
write_gicreg(val, ICH_AP0R2_EL2);
126
break;
127
case 3:
128
write_gicreg(val, ICH_AP0R3_EL2);
129
break;
130
}
131
}
132
133
static void __vgic_v3_write_ap1rn(u32 val, int n)
134
{
135
switch (n) {
136
case 0:
137
write_gicreg(val, ICH_AP1R0_EL2);
138
break;
139
case 1:
140
write_gicreg(val, ICH_AP1R1_EL2);
141
break;
142
case 2:
143
write_gicreg(val, ICH_AP1R2_EL2);
144
break;
145
case 3:
146
write_gicreg(val, ICH_AP1R3_EL2);
147
break;
148
}
149
}
150
151
static u32 __vgic_v3_read_ap0rn(int n)
152
{
153
u32 val;
154
155
switch (n) {
156
case 0:
157
val = read_gicreg(ICH_AP0R0_EL2);
158
break;
159
case 1:
160
val = read_gicreg(ICH_AP0R1_EL2);
161
break;
162
case 2:
163
val = read_gicreg(ICH_AP0R2_EL2);
164
break;
165
case 3:
166
val = read_gicreg(ICH_AP0R3_EL2);
167
break;
168
default:
169
unreachable();
170
}
171
172
return val;
173
}
174
175
static u32 __vgic_v3_read_ap1rn(int n)
176
{
177
u32 val;
178
179
switch (n) {
180
case 0:
181
val = read_gicreg(ICH_AP1R0_EL2);
182
break;
183
case 1:
184
val = read_gicreg(ICH_AP1R1_EL2);
185
break;
186
case 2:
187
val = read_gicreg(ICH_AP1R2_EL2);
188
break;
189
case 3:
190
val = read_gicreg(ICH_AP1R3_EL2);
191
break;
192
default:
193
unreachable();
194
}
195
196
return val;
197
}
198
199
void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
200
{
201
u64 used_lrs = cpu_if->used_lrs;
202
203
/*
204
* Make sure stores to the GIC via the memory mapped interface
205
* are now visible to the system register interface when reading the
206
* LRs, and when reading back the VMCR on non-VHE systems.
207
*/
208
if (used_lrs || !has_vhe()) {
209
if (!cpu_if->vgic_sre) {
210
dsb(sy);
211
isb();
212
}
213
}
214
215
if (used_lrs || cpu_if->its_vpe.its_vm) {
216
int i;
217
u32 elrsr;
218
219
elrsr = read_gicreg(ICH_ELRSR_EL2);
220
221
write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2);
222
223
for (i = 0; i < used_lrs; i++) {
224
if (elrsr & (1 << i))
225
cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
226
else
227
cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
228
229
__gic_v3_set_lr(0, i);
230
}
231
}
232
}
233
234
void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
235
{
236
u64 used_lrs = cpu_if->used_lrs;
237
int i;
238
239
if (used_lrs || cpu_if->its_vpe.its_vm) {
240
write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
241
242
for (i = 0; i < used_lrs; i++)
243
__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
244
}
245
246
/*
247
* Ensure that writes to the LRs, and on non-VHE systems ensure that
248
* the write to the VMCR in __vgic_v3_activate_traps(), will have
249
* reached the (re)distributors. This ensure the guest will read the
250
* correct values from the memory-mapped interface.
251
*/
252
if (used_lrs || !has_vhe()) {
253
if (!cpu_if->vgic_sre) {
254
isb();
255
dsb(sy);
256
}
257
}
258
}
259
260
void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
261
{
262
/*
263
* VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264
* Group0 interrupt (as generated in GICv2 mode) to be
265
* delivered as a FIQ to the guest, with potentially fatal
266
* consequences. So we must make sure that ICC_SRE_EL1 has
267
* been actually programmed with the value we want before
268
* starting to mess with the rest of the GIC, and VMCR_EL2 in
269
* particular. This logic must be called before
270
* __vgic_v3_restore_state().
271
*
272
* However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
273
* provisioned at all. In order to prevent illegal accesses to the
274
* system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
275
* so that the trap bits can take effect. Yes, we *loves* the GIC.
276
*/
277
if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) {
278
write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
279
isb();
280
} else if (!cpu_if->vgic_sre) {
281
write_gicreg(0, ICC_SRE_EL1);
282
isb();
283
write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
284
285
286
if (has_vhe()) {
287
/*
288
* Ensure that the write to the VMCR will have reached
289
* the (re)distributors. This ensure the guest will
290
* read the correct values from the memory-mapped
291
* interface.
292
*/
293
isb();
294
dsb(sy);
295
}
296
}
297
298
/* Only disable SRE if the host implements the GICv2 interface */
299
if (static_branch_unlikely(&vgic_v3_has_v2_compat)) {
300
/*
301
* Prevent the guest from touching the ICC_SRE_EL1 system
302
* register. Note that this may not have any effect, as
303
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
304
*/
305
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
306
ICC_SRE_EL2);
307
}
308
309
/*
310
* If we need to trap system registers, we must write
311
* ICH_HCR_EL2 anyway, even if no interrupts are being
312
* injected. Note that this also applies if we don't expect
313
* any system register access (no vgic at all).
314
*/
315
if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
316
cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
317
write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
318
}
319
320
void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
321
{
322
u64 val;
323
324
if (!cpu_if->vgic_sre) {
325
cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
326
}
327
328
/* Only restore SRE if the host implements the GICv2 interface */
329
if (static_branch_unlikely(&vgic_v3_has_v2_compat)) {
330
val = read_gicreg(ICC_SRE_EL2);
331
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
332
333
if (!cpu_if->vgic_sre) {
334
/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
335
isb();
336
write_gicreg(1, ICC_SRE_EL1);
337
}
338
}
339
340
/*
341
* If we were trapping system registers, we enabled the VGIC even if
342
* no interrupts were being injected, and we disable it again here.
343
*/
344
if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
345
cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
346
write_gicreg(0, ICH_HCR_EL2);
347
}
348
349
static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
350
{
351
u64 val;
352
u32 nr_pre_bits;
353
354
val = read_gicreg(ICH_VTR_EL2);
355
nr_pre_bits = vtr_to_nr_pre_bits(val);
356
357
switch (nr_pre_bits) {
358
case 7:
359
cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
360
cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
361
fallthrough;
362
case 6:
363
cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
364
fallthrough;
365
default:
366
cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
367
}
368
369
switch (nr_pre_bits) {
370
case 7:
371
cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
372
cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
373
fallthrough;
374
case 6:
375
cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
376
fallthrough;
377
default:
378
cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
379
}
380
}
381
382
static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
383
{
384
u64 val;
385
u32 nr_pre_bits;
386
387
val = read_gicreg(ICH_VTR_EL2);
388
nr_pre_bits = vtr_to_nr_pre_bits(val);
389
390
switch (nr_pre_bits) {
391
case 7:
392
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
393
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
394
fallthrough;
395
case 6:
396
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
397
fallthrough;
398
default:
399
__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
400
}
401
402
switch (nr_pre_bits) {
403
case 7:
404
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
405
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
406
fallthrough;
407
case 6:
408
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
409
fallthrough;
410
default:
411
__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
412
}
413
}
414
415
void __vgic_v3_init_lrs(void)
416
{
417
int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
418
int i;
419
420
for (i = 0; i <= max_lr_idx; i++)
421
__gic_v3_set_lr(0, i);
422
}
423
424
/*
425
* Return the GIC CPU configuration:
426
* - [31:0] ICH_VTR_EL2
427
* - [62:32] RES0
428
* - [63] MMIO (GICv2) capable
429
*/
430
u64 __vgic_v3_get_gic_config(void)
431
{
432
u64 val, sre;
433
unsigned long flags = 0;
434
435
/*
436
* In compat mode, we cannot access ICC_SRE_EL1 at any EL
437
* other than EL1 itself; just return the
438
* ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
439
* system, so we first check if we have GICv5 support.
440
*/
441
if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
442
return read_gicreg(ICH_VTR_EL2);
443
444
sre = read_gicreg(ICC_SRE_EL1);
445
/*
446
* To check whether we have a MMIO-based (GICv2 compatible)
447
* CPU interface, we need to disable the system register
448
* view.
449
*
450
* Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
451
* that to be able to set ICC_SRE_EL1.SRE to 0, all the
452
* interrupt overrides must be set. You've got to love this.
453
*
454
* As we always run VHE with HCR_xMO set, no extra xMO
455
* manipulation is required in that case.
456
*
457
* To safely disable SRE, we have to prevent any interrupt
458
* from firing (which would be deadly). This only makes sense
459
* on VHE, as interrupts are already masked for nVHE as part
460
* of the exception entry to EL2.
461
*/
462
if (has_vhe()) {
463
flags = local_daif_save();
464
} else {
465
sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
466
isb();
467
}
468
469
write_gicreg(0, ICC_SRE_EL1);
470
isb();
471
472
val = read_gicreg(ICC_SRE_EL1);
473
474
write_gicreg(sre, ICC_SRE_EL1);
475
isb();
476
477
if (has_vhe()) {
478
local_daif_restore(flags);
479
} else {
480
sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
481
isb();
482
}
483
484
val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
485
val |= read_gicreg(ICH_VTR_EL2);
486
487
return val;
488
}
489
490
static void __vgic_v3_compat_mode_enable(void)
491
{
492
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
493
return;
494
495
sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
496
/* Wait for V3 to become enabled */
497
isb();
498
}
499
500
static u64 __vgic_v3_read_vmcr(void)
501
{
502
return read_gicreg(ICH_VMCR_EL2);
503
}
504
505
static void __vgic_v3_write_vmcr(u32 vmcr)
506
{
507
write_gicreg(vmcr, ICH_VMCR_EL2);
508
}
509
510
void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
511
{
512
__vgic_v3_save_aprs(cpu_if);
513
if (cpu_if->vgic_sre)
514
cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
515
}
516
517
void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
518
{
519
__vgic_v3_compat_mode_enable();
520
521
/*
522
* If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
523
* is dependent on ICC_SRE_EL1.SRE, and we have to perform the
524
* VMCR_EL2 save/restore in the world switch.
525
*/
526
if (cpu_if->vgic_sre)
527
__vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
528
__vgic_v3_restore_aprs(cpu_if);
529
}
530
531
static int __vgic_v3_bpr_min(void)
532
{
533
/* See Pseudocode for VPriorityGroup */
534
return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
535
}
536
537
static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
538
{
539
u64 esr = kvm_vcpu_get_esr(vcpu);
540
u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
541
542
return crm != 8;
543
}
544
545
#define GICv3_IDLE_PRIORITY 0xff
546
547
static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
548
u64 *lr_val)
549
{
550
unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
551
u8 priority = GICv3_IDLE_PRIORITY;
552
int i, lr = -1;
553
554
for (i = 0; i < used_lrs; i++) {
555
u64 val = __gic_v3_get_lr(i);
556
u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
557
558
/* Not pending in the state? */
559
if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
560
continue;
561
562
/* Group-0 interrupt, but Group-0 disabled? */
563
if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
564
continue;
565
566
/* Group-1 interrupt, but Group-1 disabled? */
567
if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
568
continue;
569
570
/* Not the highest priority? */
571
if (lr_prio >= priority)
572
continue;
573
574
/* This is a candidate */
575
priority = lr_prio;
576
*lr_val = val;
577
lr = i;
578
}
579
580
if (lr == -1)
581
*lr_val = ICC_IAR1_EL1_SPURIOUS;
582
583
return lr;
584
}
585
586
static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
587
u64 *lr_val)
588
{
589
unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
590
int i;
591
592
for (i = 0; i < used_lrs; i++) {
593
u64 val = __gic_v3_get_lr(i);
594
595
if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
596
(val & ICH_LR_ACTIVE_BIT)) {
597
*lr_val = val;
598
return i;
599
}
600
}
601
602
*lr_val = ICC_IAR1_EL1_SPURIOUS;
603
return -1;
604
}
605
606
static int __vgic_v3_get_highest_active_priority(void)
607
{
608
u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
609
u32 hap = 0;
610
int i;
611
612
for (i = 0; i < nr_apr_regs; i++) {
613
u32 val;
614
615
/*
616
* The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
617
* contain the active priority levels for this VCPU
618
* for the maximum number of supported priority
619
* levels, and we return the full priority level only
620
* if the BPR is programmed to its minimum, otherwise
621
* we return a combination of the priority level and
622
* subpriority, as determined by the setting of the
623
* BPR, but without the full subpriority.
624
*/
625
val = __vgic_v3_read_ap0rn(i);
626
val |= __vgic_v3_read_ap1rn(i);
627
if (!val) {
628
hap += 32;
629
continue;
630
}
631
632
return (hap + __ffs(val)) << __vgic_v3_bpr_min();
633
}
634
635
return GICv3_IDLE_PRIORITY;
636
}
637
638
static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
639
{
640
return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
641
}
642
643
static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
644
{
645
unsigned int bpr;
646
647
if (vmcr & ICH_VMCR_CBPR_MASK) {
648
bpr = __vgic_v3_get_bpr0(vmcr);
649
if (bpr < 7)
650
bpr++;
651
} else {
652
bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
653
}
654
655
return bpr;
656
}
657
658
/*
659
* Convert a priority to a preemption level, taking the relevant BPR
660
* into account by zeroing the sub-priority bits.
661
*/
662
static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
663
{
664
unsigned int bpr;
665
666
if (!grp)
667
bpr = __vgic_v3_get_bpr0(vmcr) + 1;
668
else
669
bpr = __vgic_v3_get_bpr1(vmcr);
670
671
return pri & (GENMASK(7, 0) << bpr);
672
}
673
674
/*
675
* The priority value is independent of any of the BPR values, so we
676
* normalize it using the minimal BPR value. This guarantees that no
677
* matter what the guest does with its BPR, we can always set/get the
678
* same value of a priority.
679
*/
680
static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
681
{
682
u8 pre, ap;
683
u32 val;
684
int apr;
685
686
pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
687
ap = pre >> __vgic_v3_bpr_min();
688
apr = ap / 32;
689
690
if (!grp) {
691
val = __vgic_v3_read_ap0rn(apr);
692
__vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
693
} else {
694
val = __vgic_v3_read_ap1rn(apr);
695
__vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
696
}
697
}
698
699
static int __vgic_v3_clear_highest_active_priority(void)
700
{
701
u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
702
u32 hap = 0;
703
int i;
704
705
for (i = 0; i < nr_apr_regs; i++) {
706
u32 ap0, ap1;
707
int c0, c1;
708
709
ap0 = __vgic_v3_read_ap0rn(i);
710
ap1 = __vgic_v3_read_ap1rn(i);
711
if (!ap0 && !ap1) {
712
hap += 32;
713
continue;
714
}
715
716
c0 = ap0 ? __ffs(ap0) : 32;
717
c1 = ap1 ? __ffs(ap1) : 32;
718
719
/* Always clear the LSB, which is the highest priority */
720
if (c0 < c1) {
721
ap0 &= ~BIT(c0);
722
__vgic_v3_write_ap0rn(ap0, i);
723
hap += c0;
724
} else {
725
ap1 &= ~BIT(c1);
726
__vgic_v3_write_ap1rn(ap1, i);
727
hap += c1;
728
}
729
730
/* Rescale to 8 bits of priority */
731
return hap << __vgic_v3_bpr_min();
732
}
733
734
return GICv3_IDLE_PRIORITY;
735
}
736
737
static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
738
{
739
u64 lr_val;
740
u8 lr_prio, pmr;
741
int lr, grp;
742
743
grp = __vgic_v3_get_group(vcpu);
744
745
lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
746
if (lr < 0)
747
goto spurious;
748
749
if (grp != !!(lr_val & ICH_LR_GROUP))
750
goto spurious;
751
752
pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
753
lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
754
if (pmr <= lr_prio)
755
goto spurious;
756
757
if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
758
goto spurious;
759
760
lr_val &= ~ICH_LR_STATE;
761
lr_val |= ICH_LR_ACTIVE_BIT;
762
__gic_v3_set_lr(lr_val, lr);
763
__vgic_v3_set_active_priority(lr_prio, vmcr, grp);
764
vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
765
return;
766
767
spurious:
768
vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
769
}
770
771
static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
772
{
773
lr_val &= ~ICH_LR_ACTIVE_BIT;
774
if (lr_val & ICH_LR_HW) {
775
u32 pid;
776
777
pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
778
gic_write_dir(pid);
779
}
780
781
__gic_v3_set_lr(lr_val, lr);
782
}
783
784
static void __vgic_v3_bump_eoicount(void)
785
{
786
u32 hcr;
787
788
hcr = read_gicreg(ICH_HCR_EL2);
789
hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT;
790
write_gicreg(hcr, ICH_HCR_EL2);
791
}
792
793
static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
794
{
795
u32 vid = vcpu_get_reg(vcpu, rt);
796
u64 lr_val;
797
int lr;
798
799
/* EOImode == 0, nothing to be done here */
800
if (!(vmcr & ICH_VMCR_EOIM_MASK))
801
return;
802
803
/* No deactivate to be performed on an LPI */
804
if (vid >= VGIC_MIN_LPI)
805
return;
806
807
lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
808
if (lr == -1) {
809
__vgic_v3_bump_eoicount();
810
return;
811
}
812
813
__vgic_v3_clear_active_lr(lr, lr_val);
814
}
815
816
static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
817
{
818
u32 vid = vcpu_get_reg(vcpu, rt);
819
u64 lr_val;
820
u8 lr_prio, act_prio;
821
int lr, grp;
822
823
grp = __vgic_v3_get_group(vcpu);
824
825
/* Drop priority in any case */
826
act_prio = __vgic_v3_clear_highest_active_priority();
827
828
lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
829
if (lr == -1) {
830
/* Do not bump EOIcount for LPIs that aren't in the LRs */
831
if (!(vid >= VGIC_MIN_LPI))
832
__vgic_v3_bump_eoicount();
833
return;
834
}
835
836
/* EOImode == 1 and not an LPI, nothing to be done here */
837
if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
838
return;
839
840
lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
841
842
/* If priorities or group do not match, the guest has fscked-up. */
843
if (grp != !!(lr_val & ICH_LR_GROUP) ||
844
__vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
845
return;
846
847
/* Let's now perform the deactivation */
848
__vgic_v3_clear_active_lr(lr, lr_val);
849
}
850
851
static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
852
{
853
vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
854
}
855
856
static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
857
{
858
vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
859
}
860
861
static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
862
{
863
u64 val = vcpu_get_reg(vcpu, rt);
864
865
if (val & 1)
866
vmcr |= ICH_VMCR_ENG0_MASK;
867
else
868
vmcr &= ~ICH_VMCR_ENG0_MASK;
869
870
__vgic_v3_write_vmcr(vmcr);
871
}
872
873
static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
874
{
875
u64 val = vcpu_get_reg(vcpu, rt);
876
877
if (val & 1)
878
vmcr |= ICH_VMCR_ENG1_MASK;
879
else
880
vmcr &= ~ICH_VMCR_ENG1_MASK;
881
882
__vgic_v3_write_vmcr(vmcr);
883
}
884
885
static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
886
{
887
vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
888
}
889
890
static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
891
{
892
vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
893
}
894
895
static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
896
{
897
u64 val = vcpu_get_reg(vcpu, rt);
898
u8 bpr_min = __vgic_v3_bpr_min() - 1;
899
900
/* Enforce BPR limiting */
901
if (val < bpr_min)
902
val = bpr_min;
903
904
val <<= ICH_VMCR_BPR0_SHIFT;
905
val &= ICH_VMCR_BPR0_MASK;
906
vmcr &= ~ICH_VMCR_BPR0_MASK;
907
vmcr |= val;
908
909
__vgic_v3_write_vmcr(vmcr);
910
}
911
912
static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
913
{
914
u64 val = vcpu_get_reg(vcpu, rt);
915
u8 bpr_min = __vgic_v3_bpr_min();
916
917
if (vmcr & ICH_VMCR_CBPR_MASK)
918
return;
919
920
/* Enforce BPR limiting */
921
if (val < bpr_min)
922
val = bpr_min;
923
924
val <<= ICH_VMCR_BPR1_SHIFT;
925
val &= ICH_VMCR_BPR1_MASK;
926
vmcr &= ~ICH_VMCR_BPR1_MASK;
927
vmcr |= val;
928
929
__vgic_v3_write_vmcr(vmcr);
930
}
931
932
static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
933
{
934
u32 val;
935
936
if (!__vgic_v3_get_group(vcpu))
937
val = __vgic_v3_read_ap0rn(n);
938
else
939
val = __vgic_v3_read_ap1rn(n);
940
941
vcpu_set_reg(vcpu, rt, val);
942
}
943
944
static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
945
{
946
u32 val = vcpu_get_reg(vcpu, rt);
947
948
if (!__vgic_v3_get_group(vcpu))
949
__vgic_v3_write_ap0rn(val, n);
950
else
951
__vgic_v3_write_ap1rn(val, n);
952
}
953
954
static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
955
u32 vmcr, int rt)
956
{
957
__vgic_v3_read_apxrn(vcpu, rt, 0);
958
}
959
960
static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
961
u32 vmcr, int rt)
962
{
963
__vgic_v3_read_apxrn(vcpu, rt, 1);
964
}
965
966
static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
967
{
968
__vgic_v3_read_apxrn(vcpu, rt, 2);
969
}
970
971
static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
972
{
973
__vgic_v3_read_apxrn(vcpu, rt, 3);
974
}
975
976
static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
977
{
978
__vgic_v3_write_apxrn(vcpu, rt, 0);
979
}
980
981
static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
982
{
983
__vgic_v3_write_apxrn(vcpu, rt, 1);
984
}
985
986
static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
987
{
988
__vgic_v3_write_apxrn(vcpu, rt, 2);
989
}
990
991
static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
992
{
993
__vgic_v3_write_apxrn(vcpu, rt, 3);
994
}
995
996
static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
997
{
998
u64 lr_val;
999
int lr, lr_grp, grp;
1000
1001
grp = __vgic_v3_get_group(vcpu);
1002
1003
lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
1004
if (lr == -1)
1005
goto spurious;
1006
1007
lr_grp = !!(lr_val & ICH_LR_GROUP);
1008
if (lr_grp != grp)
1009
lr_val = ICC_IAR1_EL1_SPURIOUS;
1010
1011
spurious:
1012
vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
1013
}
1014
1015
static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1016
{
1017
vmcr &= ICH_VMCR_PMR_MASK;
1018
vmcr >>= ICH_VMCR_PMR_SHIFT;
1019
vcpu_set_reg(vcpu, rt, vmcr);
1020
}
1021
1022
static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1023
{
1024
u32 val = vcpu_get_reg(vcpu, rt);
1025
1026
val <<= ICH_VMCR_PMR_SHIFT;
1027
val &= ICH_VMCR_PMR_MASK;
1028
vmcr &= ~ICH_VMCR_PMR_MASK;
1029
vmcr |= val;
1030
1031
write_gicreg(vmcr, ICH_VMCR_EL2);
1032
}
1033
1034
static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1035
{
1036
u32 val = __vgic_v3_get_highest_active_priority();
1037
vcpu_set_reg(vcpu, rt, val);
1038
}
1039
1040
static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1041
{
1042
u32 vtr, val;
1043
1044
vtr = read_gicreg(ICH_VTR_EL2);
1045
/* PRIbits */
1046
val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1047
/* IDbits */
1048
val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1049
/* A3V */
1050
val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1051
/* EOImode */
1052
val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1053
/* CBPR */
1054
val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1055
1056
vcpu_set_reg(vcpu, rt, val);
1057
}
1058
1059
static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1060
{
1061
u32 val = vcpu_get_reg(vcpu, rt);
1062
1063
if (val & ICC_CTLR_EL1_CBPR_MASK)
1064
vmcr |= ICH_VMCR_CBPR_MASK;
1065
else
1066
vmcr &= ~ICH_VMCR_CBPR_MASK;
1067
1068
if (val & ICC_CTLR_EL1_EOImode_MASK)
1069
vmcr |= ICH_VMCR_EOIM_MASK;
1070
else
1071
vmcr &= ~ICH_VMCR_EOIM_MASK;
1072
1073
write_gicreg(vmcr, ICH_VMCR_EL2);
1074
}
1075
1076
static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
1077
u32 sysreg, bool is_read)
1078
{
1079
u64 ich_hcr;
1080
1081
if (!is_nested_ctxt(vcpu))
1082
return false;
1083
1084
ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
1085
1086
switch (sysreg) {
1087
case SYS_ICC_IGRPEN0_EL1:
1088
if (is_read &&
1089
(__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1090
return true;
1091
1092
if (!is_read &&
1093
(__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1094
return true;
1095
1096
fallthrough;
1097
1098
case SYS_ICC_AP0Rn_EL1(0):
1099
case SYS_ICC_AP0Rn_EL1(1):
1100
case SYS_ICC_AP0Rn_EL1(2):
1101
case SYS_ICC_AP0Rn_EL1(3):
1102
case SYS_ICC_BPR0_EL1:
1103
case SYS_ICC_EOIR0_EL1:
1104
case SYS_ICC_HPPIR0_EL1:
1105
case SYS_ICC_IAR0_EL1:
1106
return ich_hcr & ICH_HCR_EL2_TALL0;
1107
1108
case SYS_ICC_IGRPEN1_EL1:
1109
if (is_read &&
1110
(__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1111
return true;
1112
1113
if (!is_read &&
1114
(__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1115
return true;
1116
1117
fallthrough;
1118
1119
case SYS_ICC_AP1Rn_EL1(0):
1120
case SYS_ICC_AP1Rn_EL1(1):
1121
case SYS_ICC_AP1Rn_EL1(2):
1122
case SYS_ICC_AP1Rn_EL1(3):
1123
case SYS_ICC_BPR1_EL1:
1124
case SYS_ICC_EOIR1_EL1:
1125
case SYS_ICC_HPPIR1_EL1:
1126
case SYS_ICC_IAR1_EL1:
1127
return ich_hcr & ICH_HCR_EL2_TALL1;
1128
1129
case SYS_ICC_DIR_EL1:
1130
if (ich_hcr & ICH_HCR_EL2_TDIR)
1131
return true;
1132
1133
fallthrough;
1134
1135
case SYS_ICC_RPR_EL1:
1136
case SYS_ICC_CTLR_EL1:
1137
case SYS_ICC_PMR_EL1:
1138
return ich_hcr & ICH_HCR_EL2_TC;
1139
1140
default:
1141
return false;
1142
}
1143
}
1144
1145
int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1146
{
1147
int rt;
1148
u64 esr;
1149
u32 vmcr;
1150
void (*fn)(struct kvm_vcpu *, u32, int);
1151
bool is_read;
1152
u32 sysreg;
1153
1154
if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
1155
return 0;
1156
1157
esr = kvm_vcpu_get_esr(vcpu);
1158
if (vcpu_mode_is_32bit(vcpu)) {
1159
if (!kvm_condition_valid(vcpu)) {
1160
__kvm_skip_instr(vcpu);
1161
return 1;
1162
}
1163
1164
sysreg = esr_cp15_to_sysreg(esr);
1165
} else {
1166
sysreg = esr_sys64_to_sysreg(esr);
1167
}
1168
1169
is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1170
1171
if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
1172
return 0;
1173
1174
switch (sysreg) {
1175
case SYS_ICC_IAR0_EL1:
1176
case SYS_ICC_IAR1_EL1:
1177
if (unlikely(!is_read))
1178
return 0;
1179
fn = __vgic_v3_read_iar;
1180
break;
1181
case SYS_ICC_EOIR0_EL1:
1182
case SYS_ICC_EOIR1_EL1:
1183
if (unlikely(is_read))
1184
return 0;
1185
fn = __vgic_v3_write_eoir;
1186
break;
1187
case SYS_ICC_IGRPEN1_EL1:
1188
if (is_read)
1189
fn = __vgic_v3_read_igrpen1;
1190
else
1191
fn = __vgic_v3_write_igrpen1;
1192
break;
1193
case SYS_ICC_BPR1_EL1:
1194
if (is_read)
1195
fn = __vgic_v3_read_bpr1;
1196
else
1197
fn = __vgic_v3_write_bpr1;
1198
break;
1199
case SYS_ICC_AP0Rn_EL1(0):
1200
case SYS_ICC_AP1Rn_EL1(0):
1201
if (is_read)
1202
fn = __vgic_v3_read_apxr0;
1203
else
1204
fn = __vgic_v3_write_apxr0;
1205
break;
1206
case SYS_ICC_AP0Rn_EL1(1):
1207
case SYS_ICC_AP1Rn_EL1(1):
1208
if (is_read)
1209
fn = __vgic_v3_read_apxr1;
1210
else
1211
fn = __vgic_v3_write_apxr1;
1212
break;
1213
case SYS_ICC_AP0Rn_EL1(2):
1214
case SYS_ICC_AP1Rn_EL1(2):
1215
if (is_read)
1216
fn = __vgic_v3_read_apxr2;
1217
else
1218
fn = __vgic_v3_write_apxr2;
1219
break;
1220
case SYS_ICC_AP0Rn_EL1(3):
1221
case SYS_ICC_AP1Rn_EL1(3):
1222
if (is_read)
1223
fn = __vgic_v3_read_apxr3;
1224
else
1225
fn = __vgic_v3_write_apxr3;
1226
break;
1227
case SYS_ICC_HPPIR0_EL1:
1228
case SYS_ICC_HPPIR1_EL1:
1229
if (unlikely(!is_read))
1230
return 0;
1231
fn = __vgic_v3_read_hppir;
1232
break;
1233
case SYS_ICC_IGRPEN0_EL1:
1234
if (is_read)
1235
fn = __vgic_v3_read_igrpen0;
1236
else
1237
fn = __vgic_v3_write_igrpen0;
1238
break;
1239
case SYS_ICC_BPR0_EL1:
1240
if (is_read)
1241
fn = __vgic_v3_read_bpr0;
1242
else
1243
fn = __vgic_v3_write_bpr0;
1244
break;
1245
case SYS_ICC_DIR_EL1:
1246
if (unlikely(is_read))
1247
return 0;
1248
fn = __vgic_v3_write_dir;
1249
break;
1250
case SYS_ICC_RPR_EL1:
1251
if (unlikely(!is_read))
1252
return 0;
1253
fn = __vgic_v3_read_rpr;
1254
break;
1255
case SYS_ICC_CTLR_EL1:
1256
if (is_read)
1257
fn = __vgic_v3_read_ctlr;
1258
else
1259
fn = __vgic_v3_write_ctlr;
1260
break;
1261
case SYS_ICC_PMR_EL1:
1262
if (is_read)
1263
fn = __vgic_v3_read_pmr;
1264
else
1265
fn = __vgic_v3_write_pmr;
1266
break;
1267
default:
1268
return 0;
1269
}
1270
1271
vmcr = __vgic_v3_read_vmcr();
1272
rt = kvm_vcpu_sys_get_rt(vcpu);
1273
fn(vcpu, vmcr, rt);
1274
1275
__kvm_skip_instr(vcpu);
1276
1277
return 1;
1278
}
1279
1280