1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * Fault injection for both 32 and 64bit guests.
4
 *
5
 * Copyright (C) 2012,2013 - ARM Ltd
6
 * Author: Marc Zyngier <[email protected]>
7
 *
8
 * Based on arch/arm/kvm/emulate.c
9
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10
 * Author: Christoffer Dall <[email protected]>
11
 */
12

13
#include <linux/kvm_host.h>
14
#include <asm/kvm_emulate.h>
15
#include <asm/kvm_nested.h>
16
#include <asm/esr.h>
17

18
static unsigned int exception_target_el(struct kvm_vcpu *vcpu)
19
{
20
	/* If not nesting, EL1 is the only possible exception target */
21
	if (likely(!vcpu_has_nv(vcpu)))
22
		return PSR_MODE_EL1h;
23

24
	/*
25
	 * With NV, we need to pick between EL1 and EL2. Note that we
26
	 * never deal with a nesting exception here, hence never
27
	 * changing context, and the exception itself can be delayed
28
	 * until the next entry.
29
	 */
30
	switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) {
31
	case PSR_MODE_EL2h:
32
	case PSR_MODE_EL2t:
33
		return PSR_MODE_EL2h;
34
	case PSR_MODE_EL1h:
35
	case PSR_MODE_EL1t:
36
		return PSR_MODE_EL1h;
37
	case PSR_MODE_EL0t:
38
		return vcpu_el2_tge_is_set(vcpu) ? PSR_MODE_EL2h : PSR_MODE_EL1h;
39
	default:
40
		BUG();
41
	}
42
}
43

44
static enum vcpu_sysreg exception_esr_elx(struct kvm_vcpu *vcpu)
45
{
46
	if (exception_target_el(vcpu) == PSR_MODE_EL2h)
47
		return ESR_EL2;
48

49
	return ESR_EL1;
50
}
51

52
static enum vcpu_sysreg exception_far_elx(struct kvm_vcpu *vcpu)
53
{
54
	if (exception_target_el(vcpu) == PSR_MODE_EL2h)
55
		return FAR_EL2;
56

57
	return FAR_EL1;
58
}
59

60
static void pend_sync_exception(struct kvm_vcpu *vcpu)
61
{
62
	if (exception_target_el(vcpu) == PSR_MODE_EL1h)
63
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
64
	else
65
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
66
}
67

68
static void pend_serror_exception(struct kvm_vcpu *vcpu)
69
{
70
	if (exception_target_el(vcpu) == PSR_MODE_EL1h)
71
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SERR);
72
	else
73
		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
74
}
75

76
static bool __effective_sctlr2_bit(struct kvm_vcpu *vcpu, unsigned int idx)
77
{
78
	u64 sctlr2;
79

80
	if (!kvm_has_sctlr2(vcpu->kvm))
81
		return false;
82

83
	if (is_nested_ctxt(vcpu) &&
84
	    !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_SCTLR2En))
85
		return false;
86

87
	if (exception_target_el(vcpu) == PSR_MODE_EL1h)
88
		sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL1);
89
	else
90
		sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL2);
91

92
	return sctlr2 & BIT(idx);
93
}
94

95
static bool effective_sctlr2_ease(struct kvm_vcpu *vcpu)
96
{
97
	return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_EASE_SHIFT);
98
}
99

100
static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
101
{
102
	return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
103
}
104

105
static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
106
{
107
	unsigned long cpsr = *vcpu_cpsr(vcpu);
108
	bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
109
	u64 esr = 0, fsc;
110
	int level;
111

112
	/*
113
	 * If injecting an abort from a failed S1PTW, rewalk the S1 PTs to
114
	 * find the failing level. If we can't find it, assume the error was
115
	 * transient and restart without changing the state.
116
	 */
117
	if (kvm_vcpu_abt_iss1tw(vcpu)) {
118
		u64 hpfar = kvm_vcpu_get_fault_ipa(vcpu);
119
		int ret;
120

121
		if (hpfar == INVALID_GPA)
122
			return;
123

124
		ret = __kvm_find_s1_desc_level(vcpu, addr, hpfar, &level);
125
		if (ret)
126
			return;
127

128
		WARN_ON_ONCE(level < -1 || level > 3);
129
		fsc = ESR_ELx_FSC_SEA_TTW(level);
130
	} else {
131
		fsc = ESR_ELx_FSC_EXTABT;
132
	}
133

134
	/* This delight is brought to you by FEAT_DoubleFault2. */
135
	if (effective_sctlr2_ease(vcpu))
136
		pend_serror_exception(vcpu);
137
	else
138
		pend_sync_exception(vcpu);
139

140
	/*
141
	 * Build an {i,d}abort, depending on the level and the
142
	 * instruction set. Report an external synchronous abort.
143
	 */
144
	if (kvm_vcpu_trap_il_is32bit(vcpu))
145
		esr |= ESR_ELx_IL;
146

147
	/*
148
	 * Here, the guest runs in AArch64 mode when in EL1. If we get
149
	 * an AArch32 fault, it means we managed to trap an EL0 fault.
150
	 */
151
	if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
152
		esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
153
	else
154
		esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
155

156
	if (!is_iabt)
157
		esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT;
158

159
	esr |= fsc;
160

161
	vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
162
	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
163
}
164

165
static void inject_undef64(struct kvm_vcpu *vcpu)
166
{
167
	u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
168

169
	pend_sync_exception(vcpu);
170

171
	/*
172
	 * Build an unknown exception, depending on the instruction
173
	 * set.
174
	 */
175
	if (kvm_vcpu_trap_il_is32bit(vcpu))
176
		esr |= ESR_ELx_IL;
177

178
	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
179
}
180

181
#define DFSR_FSC_EXTABT_LPAE	0x10
182
#define DFSR_FSC_EXTABT_nLPAE	0x08
183
#define DFSR_LPAE		BIT(9)
184
#define TTBCR_EAE		BIT(31)
185

186
static void inject_undef32(struct kvm_vcpu *vcpu)
187
{
188
	kvm_pend_exception(vcpu, EXCEPT_AA32_UND);
189
}
190

191
/*
192
 * Modelled after TakeDataAbortException() and TakePrefetchAbortException
193
 * pseudocode.
194
 */
195
static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
196
{
197
	u64 far;
198
	u32 fsr;
199

200
	/* Give the guest an IMPLEMENTATION DEFINED exception */
201
	if (vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE) {
202
		fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE;
203
	} else {
204
		/* no need to shuffle FS[4] into DFSR[10] as it's 0 */
205
		fsr = DFSR_FSC_EXTABT_nLPAE;
206
	}
207

208
	far = vcpu_read_sys_reg(vcpu, FAR_EL1);
209

210
	if (is_pabt) {
211
		kvm_pend_exception(vcpu, EXCEPT_AA32_IABT);
212
		far &= GENMASK(31, 0);
213
		far |= (u64)addr << 32;
214
		vcpu_write_sys_reg(vcpu, fsr, IFSR32_EL2);
215
	} else { /* !iabt */
216
		kvm_pend_exception(vcpu, EXCEPT_AA32_DABT);
217
		far &= GENMASK(63, 32);
218
		far |= addr;
219
		vcpu_write_sys_reg(vcpu, fsr, ESR_EL1);
220
	}
221

222
	vcpu_write_sys_reg(vcpu, far, FAR_EL1);
223
}
224

225
static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
226
{
227
	if (vcpu_el1_is_32bit(vcpu))
228
		inject_abt32(vcpu, iabt, addr);
229
	else
230
		inject_abt64(vcpu, iabt, addr);
231
}
232

233
static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
234
{
235
	if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA))
236
		return true;
237

238
	if (!vcpu_mode_priv(vcpu))
239
		return false;
240

241
	return (*vcpu_cpsr(vcpu) & PSR_A_BIT) &&
242
	       (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
243
}
244

245
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
246
{
247
	lockdep_assert_held(&vcpu->mutex);
248

249
	if (is_nested_ctxt(vcpu) && kvm_sea_target_is_el2(vcpu))
250
		return kvm_inject_nested_sea(vcpu, iabt, addr);
251

252
	__kvm_inject_sea(vcpu, iabt, addr);
253
	return 1;
254
}
255

256
void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
257
{
258
	unsigned long addr, esr;
259

260
	addr  = kvm_vcpu_get_fault_ipa(vcpu);
261
	addr |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
262

263
	__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
264

265
	/*
266
	 * If AArch64 or LPAE, set FSC to 0 to indicate an Address
267
	 * Size Fault at level 0, as if exceeding PARange.
268
	 *
269
	 * Non-LPAE guests will only get the external abort, as there
270
	 * is no way to describe the ASF.
271
	 */
272
	if (vcpu_el1_is_32bit(vcpu) &&
273
	    !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
274
		return;
275

276
	esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
277
	esr &= ~GENMASK_ULL(5, 0);
278
	vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
279
}
280

281
/**
282
 * kvm_inject_undefined - inject an undefined instruction into the guest
283
 * @vcpu: The vCPU in which to inject the exception
284
 *
285
 * It is assumed that this code is called from the VCPU thread and that the
286
 * VCPU therefore is not currently executing guest code.
287
 */
288
void kvm_inject_undefined(struct kvm_vcpu *vcpu)
289
{
290
	if (vcpu_el1_is_32bit(vcpu))
291
		inject_undef32(vcpu);
292
	else
293
		inject_undef64(vcpu);
294
}
295

296
static bool serror_is_masked(struct kvm_vcpu *vcpu)
297
{
298
	return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu);
299
}
300

301
static bool kvm_serror_target_is_el2(struct kvm_vcpu *vcpu)
302
{
303
	if (is_hyp_ctxt(vcpu) || vcpu_el2_amo_is_set(vcpu))
304
		return true;
305

306
	if (!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA))
307
		return false;
308

309
	/*
310
	 * In another example where FEAT_DoubleFault2 is entirely backwards,
311
	 * "masked" as it relates to the routing effects of HCRX_EL2.TMEA
312
	 * doesn't consider SCTLR2_EL1.NMEA. That is to say, even if EL1 asked
313
	 * for non-maskable SErrors, the EL2 bit takes priority if A is set.
314
	 */
315
	if (vcpu_mode_priv(vcpu))
316
		return *vcpu_cpsr(vcpu) & PSR_A_BIT;
317

318
	/*
319
	 * Otherwise SErrors are considered unmasked when taken from EL0 and
320
	 * NMEA is set.
321
	 */
322
	return serror_is_masked(vcpu);
323
}
324

325
static bool kvm_serror_undeliverable_at_el2(struct kvm_vcpu *vcpu)
326
{
327
	return !(vcpu_el2_tge_is_set(vcpu) || vcpu_el2_amo_is_set(vcpu));
328
}
329

330
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
331
{
332
	lockdep_assert_held(&vcpu->mutex);
333

334
	if (is_nested_ctxt(vcpu) && kvm_serror_target_is_el2(vcpu))
335
		return kvm_inject_nested_serror(vcpu, esr);
336

337
	if (vcpu_is_el2(vcpu) && kvm_serror_undeliverable_at_el2(vcpu)) {
338
		vcpu_set_vsesr(vcpu, esr);
339
		vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
340
		return 1;
341
	}
342

343
	/*
344
	 * Emulate the exception entry if SErrors are unmasked. This is useful if
345
	 * the vCPU is in a nested context w/ vSErrors enabled then we've already
346
	 * delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2,
347
	 * VDISR_EL2) to the guest hypervisor.
348
	 *
349
	 * As we're emulating the SError injection we need to explicitly populate
350
	 * ESR_ELx.EC because hardware will not do it on our behalf.
351
	 */
352
	if (!serror_is_masked(vcpu)) {
353
		pend_serror_exception(vcpu);
354
		esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
355
		vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
356
		return 1;
357
	}
358

359
	vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
360
	*vcpu_hcr(vcpu) |= HCR_VSE;
361
	return 1;
362
}
363

364