1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4
 *
5
 * Authors:
6
 *     Anup Patel <[email protected]>
7
 */
8

9
#include <linux/bitops.h>
10
#include <linux/entry-kvm.h>
11
#include <linux/errno.h>
12
#include <linux/err.h>
13
#include <linux/kdebug.h>
14
#include <linux/module.h>
15
#include <linux/percpu.h>
16
#include <linux/vmalloc.h>
17
#include <linux/sched/signal.h>
18
#include <linux/fs.h>
19
#include <linux/kvm_host.h>
20
#include <asm/cacheflush.h>
21
#include <asm/kvm_mmu.h>
22
#include <asm/kvm_nacl.h>
23
#include <asm/kvm_vcpu_vector.h>
24

25
#define CREATE_TRACE_POINTS
26
#include "trace.h"
27

28
const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
29
	KVM_GENERIC_VCPU_STATS(),
30
	STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
31
	STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
32
	STATS_DESC_COUNTER(VCPU, wrs_exit_stat),
33
	STATS_DESC_COUNTER(VCPU, mmio_exit_user),
34
	STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
35
	STATS_DESC_COUNTER(VCPU, csr_exit_user),
36
	STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
37
	STATS_DESC_COUNTER(VCPU, signal_exits),
38
	STATS_DESC_COUNTER(VCPU, exits),
39
	STATS_DESC_COUNTER(VCPU, instr_illegal_exits),
40
	STATS_DESC_COUNTER(VCPU, load_misaligned_exits),
41
	STATS_DESC_COUNTER(VCPU, store_misaligned_exits),
42
	STATS_DESC_COUNTER(VCPU, load_access_exits),
43
	STATS_DESC_COUNTER(VCPU, store_access_exits),
44
};
45

46
const struct kvm_stats_header kvm_vcpu_stats_header = {
47
	.name_size = KVM_STATS_NAME_SIZE,
48
	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
49
	.id_offset = sizeof(struct kvm_stats_header),
50
	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
51
	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
52
		       sizeof(kvm_vcpu_stats_desc),
53
};
54

55
static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu,
56
					 bool kvm_sbi_reset)
57
{
58
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
59
	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
60
	void *vector_datap = cntx->vector.datap;
61

62
	memset(cntx, 0, sizeof(*cntx));
63
	memset(csr, 0, sizeof(*csr));
64
	memset(&vcpu->arch.smstateen_csr, 0, sizeof(vcpu->arch.smstateen_csr));
65

66
	/* Restore datap as it's not a part of the guest context. */
67
	cntx->vector.datap = vector_datap;
68

69
	if (kvm_sbi_reset)
70
		kvm_riscv_vcpu_sbi_load_reset_state(vcpu);
71

72
	/* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
73
	cntx->sstatus = SR_SPP | SR_SPIE;
74

75
	cntx->hstatus |= HSTATUS_VTW;
76
	cntx->hstatus |= HSTATUS_SPVP;
77
	cntx->hstatus |= HSTATUS_SPV;
78
}
79

80
static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
81
{
82
	bool loaded;
83

84
	/**
85
	 * The preemption should be disabled here because it races with
86
	 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which
87
	 * also calls vcpu_load/put.
88
	 */
89
	get_cpu();
90
	loaded = (vcpu->cpu != -1);
91
	if (loaded)
92
		kvm_arch_vcpu_put(vcpu);
93

94
	vcpu->arch.last_exit_cpu = -1;
95

96
	kvm_riscv_vcpu_context_reset(vcpu, kvm_sbi_reset);
97

98
	kvm_riscv_vcpu_fp_reset(vcpu);
99

100
	kvm_riscv_vcpu_vector_reset(vcpu);
101

102
	kvm_riscv_vcpu_timer_reset(vcpu);
103

104
	kvm_riscv_vcpu_aia_reset(vcpu);
105

106
	bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
107
	bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
108

109
	kvm_riscv_vcpu_pmu_reset(vcpu);
110

111
	vcpu->arch.hfence_head = 0;
112
	vcpu->arch.hfence_tail = 0;
113
	memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
114

115
	kvm_riscv_vcpu_sbi_reset(vcpu);
116

117
	/* Reset the guest CSRs for hotplug usecase */
118
	if (loaded)
119
		kvm_arch_vcpu_load(vcpu, smp_processor_id());
120
	put_cpu();
121
}
122

123
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
124
{
125
	return 0;
126
}
127

128
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
129
{
130
	int rc;
131

132
	spin_lock_init(&vcpu->arch.mp_state_lock);
133

134
	/* Mark this VCPU never ran */
135
	vcpu->arch.ran_atleast_once = false;
136

137
	vcpu->arch.cfg.hedeleg = KVM_HEDELEG_DEFAULT;
138
	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
139
	bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX);
140

141
	/* Setup ISA features available to VCPU */
142
	kvm_riscv_vcpu_setup_isa(vcpu);
143

144
	/* Setup vendor, arch, and implementation details */
145
	vcpu->arch.mvendorid = sbi_get_mvendorid();
146
	vcpu->arch.marchid = sbi_get_marchid();
147
	vcpu->arch.mimpid = sbi_get_mimpid();
148

149
	/* Setup VCPU hfence queue */
150
	spin_lock_init(&vcpu->arch.hfence_lock);
151

152
	spin_lock_init(&vcpu->arch.reset_state.lock);
153

154
	rc = kvm_riscv_vcpu_alloc_vector_context(vcpu);
155
	if (rc)
156
		return rc;
157

158
	/* Setup VCPU timer */
159
	kvm_riscv_vcpu_timer_init(vcpu);
160

161
	/* setup performance monitoring */
162
	kvm_riscv_vcpu_pmu_init(vcpu);
163

164
	/* Setup VCPU AIA */
165
	kvm_riscv_vcpu_aia_init(vcpu);
166

167
	/*
168
	 * Setup SBI extensions
169
	 * NOTE: This must be the last thing to be initialized.
170
	 */
171
	kvm_riscv_vcpu_sbi_init(vcpu);
172

173
	/* Reset VCPU */
174
	kvm_riscv_reset_vcpu(vcpu, false);
175

176
	return 0;
177
}
178

179
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
180
{
181
	/**
182
	 * vcpu with id 0 is the designated boot cpu.
183
	 * Keep all vcpus with non-zero id in power-off state so that
184
	 * they can be brought up using SBI HSM extension.
185
	 */
186
	if (vcpu->vcpu_idx != 0)
187
		kvm_riscv_vcpu_power_off(vcpu);
188
}
189

190
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
191
{
192
	kvm_riscv_vcpu_sbi_deinit(vcpu);
193

194
	/* Cleanup VCPU AIA context */
195
	kvm_riscv_vcpu_aia_deinit(vcpu);
196

197
	/* Cleanup VCPU timer */
198
	kvm_riscv_vcpu_timer_deinit(vcpu);
199

200
	kvm_riscv_vcpu_pmu_deinit(vcpu);
201

202
	/* Free unused pages pre-allocated for G-stage page table mappings */
203
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
204

205
	/* Free vector context space for host and guest kernel */
206
	kvm_riscv_vcpu_free_vector_context(vcpu);
207
}
208

209
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
210
{
211
	return kvm_riscv_vcpu_timer_pending(vcpu);
212
}
213

214
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
215
{
216
	return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&
217
		!kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause);
218
}
219

220
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
221
{
222
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
223
}
224

225
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
226
{
227
	return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
228
}
229

230
#ifdef CONFIG_GUEST_PERF_EVENTS
231
unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
232
{
233
	return vcpu->arch.guest_context.sepc;
234
}
235
#endif
236

237
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
238
{
239
	return VM_FAULT_SIGBUS;
240
}
241

242
long kvm_arch_vcpu_async_ioctl(struct file *filp,
243
			       unsigned int ioctl, unsigned long arg)
244
{
245
	struct kvm_vcpu *vcpu = filp->private_data;
246
	void __user *argp = (void __user *)arg;
247

248
	if (ioctl == KVM_INTERRUPT) {
249
		struct kvm_interrupt irq;
250

251
		if (copy_from_user(&irq, argp, sizeof(irq)))
252
			return -EFAULT;
253

254
		if (irq.irq == KVM_INTERRUPT_SET)
255
			return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
256
		else
257
			return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
258
	}
259

260
	return -ENOIOCTLCMD;
261
}
262

263
long kvm_arch_vcpu_ioctl(struct file *filp,
264
			 unsigned int ioctl, unsigned long arg)
265
{
266
	struct kvm_vcpu *vcpu = filp->private_data;
267
	void __user *argp = (void __user *)arg;
268
	long r = -EINVAL;
269

270
	switch (ioctl) {
271
	case KVM_SET_ONE_REG:
272
	case KVM_GET_ONE_REG: {
273
		struct kvm_one_reg reg;
274

275
		r = -EFAULT;
276
		if (copy_from_user(&reg, argp, sizeof(reg)))
277
			break;
278

279
		if (ioctl == KVM_SET_ONE_REG)
280
			r = kvm_riscv_vcpu_set_reg(vcpu, &reg);
281
		else
282
			r = kvm_riscv_vcpu_get_reg(vcpu, &reg);
283
		break;
284
	}
285
	case KVM_GET_REG_LIST: {
286
		struct kvm_reg_list __user *user_list = argp;
287
		struct kvm_reg_list reg_list;
288
		unsigned int n;
289

290
		r = -EFAULT;
291
		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
292
			break;
293
		n = reg_list.n;
294
		reg_list.n = kvm_riscv_vcpu_num_regs(vcpu);
295
		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
296
			break;
297
		r = -E2BIG;
298
		if (n < reg_list.n)
299
			break;
300
		r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg);
301
		break;
302
	}
303
	default:
304
		break;
305
	}
306

307
	return r;
308
}
309

310
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
311
				  struct kvm_sregs *sregs)
312
{
313
	return -EINVAL;
314
}
315

316
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
317
				  struct kvm_sregs *sregs)
318
{
319
	return -EINVAL;
320
}
321

322
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
323
{
324
	return -EINVAL;
325
}
326

327
int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
328
{
329
	return -EINVAL;
330
}
331

332
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
333
				  struct kvm_translation *tr)
334
{
335
	return -EINVAL;
336
}
337

338
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
339
{
340
	return -EINVAL;
341
}
342

343
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
344
{
345
	return -EINVAL;
346
}
347

348
void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
349
{
350
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
351
	unsigned long mask, val;
352

353
	if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {
354
		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);
355
		val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;
356

357
		csr->hvip &= ~mask;
358
		csr->hvip |= val;
359
	}
360

361
	/* Flush AIA high interrupts */
362
	kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
363
}
364

365
void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
366
{
367
	unsigned long hvip;
368
	struct kvm_vcpu_arch *v = &vcpu->arch;
369
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
370

371
	/* Read current HVIP and VSIE CSRs */
372
	csr->vsie = ncsr_read(CSR_VSIE);
373

374
	/* Sync-up HVIP.VSSIP bit changes does by Guest */
375
	hvip = ncsr_read(CSR_HVIP);
376
	if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
377
		if (hvip & (1UL << IRQ_VS_SOFT)) {
378
			if (!test_and_set_bit(IRQ_VS_SOFT,
379
					      v->irqs_pending_mask))
380
				set_bit(IRQ_VS_SOFT, v->irqs_pending);
381
		} else {
382
			if (!test_and_set_bit(IRQ_VS_SOFT,
383
					      v->irqs_pending_mask))
384
				clear_bit(IRQ_VS_SOFT, v->irqs_pending);
385
		}
386
	}
387

388
	/* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */
389
	if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) {
390
		if (!(hvip & (1UL << IRQ_PMU_OVF)) &&
391
		    !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
392
			clear_bit(IRQ_PMU_OVF, v->irqs_pending);
393
	}
394

395
	/* Sync-up AIA high interrupts */
396
	kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
397

398
	/* Sync-up timer CSRs */
399
	kvm_riscv_vcpu_timer_sync(vcpu);
400
}
401

402
int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
403
{
404
	/*
405
	 * We only allow VS-mode software, timer, and external
406
	 * interrupts when irq is one of the local interrupts
407
	 * defined by RISC-V privilege specification.
408
	 */
409
	if (irq < IRQ_LOCAL_MAX &&
410
	    irq != IRQ_VS_SOFT &&
411
	    irq != IRQ_VS_TIMER &&
412
	    irq != IRQ_VS_EXT &&
413
	    irq != IRQ_PMU_OVF)
414
		return -EINVAL;
415

416
	set_bit(irq, vcpu->arch.irqs_pending);
417
	smp_mb__before_atomic();
418
	set_bit(irq, vcpu->arch.irqs_pending_mask);
419

420
	kvm_vcpu_kick(vcpu);
421

422
	return 0;
423
}
424

425
int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
426
{
427
	/*
428
	 * We only allow VS-mode software, timer, counter overflow and external
429
	 * interrupts when irq is one of the local interrupts
430
	 * defined by RISC-V privilege specification.
431
	 */
432
	if (irq < IRQ_LOCAL_MAX &&
433
	    irq != IRQ_VS_SOFT &&
434
	    irq != IRQ_VS_TIMER &&
435
	    irq != IRQ_VS_EXT &&
436
	    irq != IRQ_PMU_OVF)
437
		return -EINVAL;
438

439
	clear_bit(irq, vcpu->arch.irqs_pending);
440
	smp_mb__before_atomic();
441
	set_bit(irq, vcpu->arch.irqs_pending_mask);
442

443
	return 0;
444
}
445

446
bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
447
{
448
	unsigned long ie;
449

450
	ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
451
		<< VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
452
	ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
453
		(unsigned long)mask;
454
	if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)
455
		return true;
456

457
	/* Check AIA high interrupts */
458
	return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
459
}
460

461
void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
462
{
463
	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
464
	kvm_make_request(KVM_REQ_SLEEP, vcpu);
465
	kvm_vcpu_kick(vcpu);
466
}
467

468
void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
469
{
470
	spin_lock(&vcpu->arch.mp_state_lock);
471
	__kvm_riscv_vcpu_power_off(vcpu);
472
	spin_unlock(&vcpu->arch.mp_state_lock);
473
}
474

475
void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
476
{
477
	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
478
	kvm_vcpu_wake_up(vcpu);
479
}
480

481
void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
482
{
483
	spin_lock(&vcpu->arch.mp_state_lock);
484
	__kvm_riscv_vcpu_power_on(vcpu);
485
	spin_unlock(&vcpu->arch.mp_state_lock);
486
}
487

488
bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu)
489
{
490
	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
491
}
492

493
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
494
				    struct kvm_mp_state *mp_state)
495
{
496
	*mp_state = READ_ONCE(vcpu->arch.mp_state);
497

498
	return 0;
499
}
500

501
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
502
				    struct kvm_mp_state *mp_state)
503
{
504
	int ret = 0;
505

506
	spin_lock(&vcpu->arch.mp_state_lock);
507

508
	switch (mp_state->mp_state) {
509
	case KVM_MP_STATE_RUNNABLE:
510
		WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
511
		break;
512
	case KVM_MP_STATE_STOPPED:
513
		__kvm_riscv_vcpu_power_off(vcpu);
514
		break;
515
	case KVM_MP_STATE_INIT_RECEIVED:
516
		if (vcpu->kvm->arch.mp_state_reset)
517
			kvm_riscv_reset_vcpu(vcpu, false);
518
		else
519
			ret = -EINVAL;
520
		break;
521
	default:
522
		ret = -EINVAL;
523
	}
524

525
	spin_unlock(&vcpu->arch.mp_state_lock);
526

527
	return ret;
528
}
529

530
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
531
					struct kvm_guest_debug *dbg)
532
{
533
	if (dbg->control & KVM_GUESTDBG_ENABLE) {
534
		vcpu->guest_debug = dbg->control;
535
		vcpu->arch.cfg.hedeleg &= ~BIT(EXC_BREAKPOINT);
536
	} else {
537
		vcpu->guest_debug = 0;
538
		vcpu->arch.cfg.hedeleg |= BIT(EXC_BREAKPOINT);
539
	}
540

541
	return 0;
542
}
543

544
static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu)
545
{
546
	const unsigned long *isa = vcpu->arch.isa;
547
	struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
548

549
	if (riscv_isa_extension_available(isa, SVPBMT))
550
		cfg->henvcfg |= ENVCFG_PBMTE;
551

552
	if (riscv_isa_extension_available(isa, SSTC))
553
		cfg->henvcfg |= ENVCFG_STCE;
554

555
	if (riscv_isa_extension_available(isa, ZICBOM))
556
		cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE);
557

558
	if (riscv_isa_extension_available(isa, ZICBOZ))
559
		cfg->henvcfg |= ENVCFG_CBZE;
560

561
	if (riscv_isa_extension_available(isa, SVADU) &&
562
	    !riscv_isa_extension_available(isa, SVADE))
563
		cfg->henvcfg |= ENVCFG_ADUE;
564

565
	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
566
		cfg->hstateen0 |= SMSTATEEN0_HSENVCFG;
567
		if (riscv_isa_extension_available(isa, SSAIA))
568
			cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC |
569
					  SMSTATEEN0_AIA |
570
					  SMSTATEEN0_AIA_ISEL;
571
		if (riscv_isa_extension_available(isa, SMSTATEEN))
572
			cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0;
573
	}
574

575
	if (vcpu->guest_debug)
576
		cfg->hedeleg &= ~BIT(EXC_BREAKPOINT);
577
}
578

579
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
580
{
581
	void *nsh;
582
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
583
	struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
584

585
	if (kvm_riscv_nacl_sync_csr_available()) {
586
		nsh = nacl_shmem();
587
		nacl_csr_write(nsh, CSR_VSSTATUS, csr->vsstatus);
588
		nacl_csr_write(nsh, CSR_VSIE, csr->vsie);
589
		nacl_csr_write(nsh, CSR_VSTVEC, csr->vstvec);
590
		nacl_csr_write(nsh, CSR_VSSCRATCH, csr->vsscratch);
591
		nacl_csr_write(nsh, CSR_VSEPC, csr->vsepc);
592
		nacl_csr_write(nsh, CSR_VSCAUSE, csr->vscause);
593
		nacl_csr_write(nsh, CSR_VSTVAL, csr->vstval);
594
		nacl_csr_write(nsh, CSR_HEDELEG, cfg->hedeleg);
595
		nacl_csr_write(nsh, CSR_HVIP, csr->hvip);
596
		nacl_csr_write(nsh, CSR_VSATP, csr->vsatp);
597
		nacl_csr_write(nsh, CSR_HENVCFG, cfg->henvcfg);
598
		if (IS_ENABLED(CONFIG_32BIT))
599
			nacl_csr_write(nsh, CSR_HENVCFGH, cfg->henvcfg >> 32);
600
		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
601
			nacl_csr_write(nsh, CSR_HSTATEEN0, cfg->hstateen0);
602
			if (IS_ENABLED(CONFIG_32BIT))
603
				nacl_csr_write(nsh, CSR_HSTATEEN0H, cfg->hstateen0 >> 32);
604
		}
605
	} else {
606
		csr_write(CSR_VSSTATUS, csr->vsstatus);
607
		csr_write(CSR_VSIE, csr->vsie);
608
		csr_write(CSR_VSTVEC, csr->vstvec);
609
		csr_write(CSR_VSSCRATCH, csr->vsscratch);
610
		csr_write(CSR_VSEPC, csr->vsepc);
611
		csr_write(CSR_VSCAUSE, csr->vscause);
612
		csr_write(CSR_VSTVAL, csr->vstval);
613
		csr_write(CSR_HEDELEG, cfg->hedeleg);
614
		csr_write(CSR_HVIP, csr->hvip);
615
		csr_write(CSR_VSATP, csr->vsatp);
616
		csr_write(CSR_HENVCFG, cfg->henvcfg);
617
		if (IS_ENABLED(CONFIG_32BIT))
618
			csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32);
619
		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
620
			csr_write(CSR_HSTATEEN0, cfg->hstateen0);
621
			if (IS_ENABLED(CONFIG_32BIT))
622
				csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32);
623
		}
624
	}
625

626
	kvm_riscv_mmu_update_hgatp(vcpu);
627

628
	kvm_riscv_vcpu_timer_restore(vcpu);
629

630
	kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
631
	kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
632
					vcpu->arch.isa);
633
	kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);
634
	kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context,
635
					    vcpu->arch.isa);
636

637
	kvm_riscv_vcpu_aia_load(vcpu, cpu);
638

639
	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
640

641
	vcpu->cpu = cpu;
642
}
643

644
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
645
{
646
	void *nsh;
647
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
648

649
	vcpu->cpu = -1;
650

651
	kvm_riscv_vcpu_aia_put(vcpu);
652

653
	kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
654
				     vcpu->arch.isa);
655
	kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
656

657
	kvm_riscv_vcpu_timer_save(vcpu);
658
	kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context,
659
					 vcpu->arch.isa);
660
	kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context);
661

662
	if (kvm_riscv_nacl_available()) {
663
		nsh = nacl_shmem();
664
		csr->vsstatus = nacl_csr_read(nsh, CSR_VSSTATUS);
665
		csr->vsie = nacl_csr_read(nsh, CSR_VSIE);
666
		csr->vstvec = nacl_csr_read(nsh, CSR_VSTVEC);
667
		csr->vsscratch = nacl_csr_read(nsh, CSR_VSSCRATCH);
668
		csr->vsepc = nacl_csr_read(nsh, CSR_VSEPC);
669
		csr->vscause = nacl_csr_read(nsh, CSR_VSCAUSE);
670
		csr->vstval = nacl_csr_read(nsh, CSR_VSTVAL);
671
		csr->hvip = nacl_csr_read(nsh, CSR_HVIP);
672
		csr->vsatp = nacl_csr_read(nsh, CSR_VSATP);
673
	} else {
674
		csr->vsstatus = csr_read(CSR_VSSTATUS);
675
		csr->vsie = csr_read(CSR_VSIE);
676
		csr->vstvec = csr_read(CSR_VSTVEC);
677
		csr->vsscratch = csr_read(CSR_VSSCRATCH);
678
		csr->vsepc = csr_read(CSR_VSEPC);
679
		csr->vscause = csr_read(CSR_VSCAUSE);
680
		csr->vstval = csr_read(CSR_VSTVAL);
681
		csr->hvip = csr_read(CSR_HVIP);
682
		csr->vsatp = csr_read(CSR_VSATP);
683
	}
684
}
685

686
/**
687
 * kvm_riscv_check_vcpu_requests - check and handle pending vCPU requests
688
 * @vcpu:	the VCPU pointer
689
 *
690
 * Return: 1 if we should enter the guest
691
 *	    0 if we should exit to userspace
692
 */
693
static int kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
694
{
695
	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
696

697
	if (kvm_request_pending(vcpu)) {
698
		if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
699
			kvm_vcpu_srcu_read_unlock(vcpu);
700
			rcuwait_wait_event(wait,
701
				(!kvm_riscv_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
702
				TASK_INTERRUPTIBLE);
703
			kvm_vcpu_srcu_read_lock(vcpu);
704

705
			if (kvm_riscv_vcpu_stopped(vcpu) || vcpu->arch.pause) {
706
				/*
707
				 * Awaken to handle a signal, request to
708
				 * sleep again later.
709
				 */
710
				kvm_make_request(KVM_REQ_SLEEP, vcpu);
711
			}
712
		}
713

714
		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
715
			kvm_riscv_reset_vcpu(vcpu, true);
716

717
		if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
718
			kvm_riscv_mmu_update_hgatp(vcpu);
719

720
		if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
721
			kvm_riscv_fence_i_process(vcpu);
722

723
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
724
			kvm_riscv_tlb_flush_process(vcpu);
725

726
		if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
727
			kvm_riscv_hfence_vvma_all_process(vcpu);
728

729
		if (kvm_check_request(KVM_REQ_HFENCE, vcpu))
730
			kvm_riscv_hfence_process(vcpu);
731

732
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
733
			kvm_riscv_vcpu_record_steal_time(vcpu);
734

735
		if (kvm_dirty_ring_check_request(vcpu))
736
			return 0;
737
	}
738

739
	return 1;
740
}
741

742
static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
743
{
744
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
745

746
	ncsr_write(CSR_HVIP, csr->hvip);
747
	kvm_riscv_vcpu_aia_update_hvip(vcpu);
748
}
749

750
static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu)
751
{
752
	struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
753
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
754
	struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
755

756
	vcpu->arch.host_scounteren = csr_swap(CSR_SCOUNTEREN, csr->scounteren);
757
	vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg);
758
	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) &&
759
	    (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
760
		vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0,
761
						     smcsr->sstateen0);
762
}
763

764
static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu)
765
{
766
	struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
767
	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
768
	struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
769

770
	csr->scounteren = csr_swap(CSR_SCOUNTEREN, vcpu->arch.host_scounteren);
771
	csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg);
772
	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) &&
773
	    (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
774
		smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0,
775
					    vcpu->arch.host_sstateen0);
776
}
777

778
/*
779
 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
780
 * the vCPU is running.
781
 *
782
 * This must be noinstr as instrumentation may make use of RCU, and this is not
783
 * safe during the EQS.
784
 */
785
static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
786
					      struct kvm_cpu_trap *trap)
787
{
788
	void *nsh;
789
	struct kvm_cpu_context *gcntx = &vcpu->arch.guest_context;
790
	struct kvm_cpu_context *hcntx = &vcpu->arch.host_context;
791

792
	/*
793
	 * We save trap CSRs (such as SEPC, SCAUSE, STVAL, HTVAL, and
794
	 * HTINST) here because we do local_irq_enable() after this
795
	 * function in kvm_arch_vcpu_ioctl_run() which can result in
796
	 * an interrupt immediately after local_irq_enable() and can
797
	 * potentially change trap CSRs.
798
	 */
799

800
	kvm_riscv_vcpu_swap_in_guest_state(vcpu);
801
	guest_state_enter_irqoff();
802

803
	if (kvm_riscv_nacl_sync_sret_available()) {
804
		nsh = nacl_shmem();
805

806
		if (kvm_riscv_nacl_autoswap_csr_available()) {
807
			hcntx->hstatus =
808
				nacl_csr_read(nsh, CSR_HSTATUS);
809
			nacl_scratch_write_long(nsh,
810
						SBI_NACL_SHMEM_AUTOSWAP_OFFSET +
811
						SBI_NACL_SHMEM_AUTOSWAP_HSTATUS,
812
						gcntx->hstatus);
813
			nacl_scratch_write_long(nsh,
814
						SBI_NACL_SHMEM_AUTOSWAP_OFFSET,
815
						SBI_NACL_SHMEM_AUTOSWAP_FLAG_HSTATUS);
816
		} else if (kvm_riscv_nacl_sync_csr_available()) {
817
			hcntx->hstatus = nacl_csr_swap(nsh,
818
						       CSR_HSTATUS, gcntx->hstatus);
819
		} else {
820
			hcntx->hstatus = csr_swap(CSR_HSTATUS, gcntx->hstatus);
821
		}
822

823
		nacl_scratch_write_longs(nsh,
824
					 SBI_NACL_SHMEM_SRET_OFFSET +
825
					 SBI_NACL_SHMEM_SRET_X(1),
826
					 &gcntx->ra,
827
					 SBI_NACL_SHMEM_SRET_X_LAST);
828

829
		__kvm_riscv_nacl_switch_to(&vcpu->arch, SBI_EXT_NACL,
830
					   SBI_EXT_NACL_SYNC_SRET);
831

832
		if (kvm_riscv_nacl_autoswap_csr_available()) {
833
			nacl_scratch_write_long(nsh,
834
						SBI_NACL_SHMEM_AUTOSWAP_OFFSET,
835
						0);
836
			gcntx->hstatus = nacl_scratch_read_long(nsh,
837
								SBI_NACL_SHMEM_AUTOSWAP_OFFSET +
838
								SBI_NACL_SHMEM_AUTOSWAP_HSTATUS);
839
		} else {
840
			gcntx->hstatus = csr_swap(CSR_HSTATUS, hcntx->hstatus);
841
		}
842

843
		trap->htval = nacl_csr_read(nsh, CSR_HTVAL);
844
		trap->htinst = nacl_csr_read(nsh, CSR_HTINST);
845
	} else {
846
		hcntx->hstatus = csr_swap(CSR_HSTATUS, gcntx->hstatus);
847

848
		__kvm_riscv_switch_to(&vcpu->arch);
849

850
		gcntx->hstatus = csr_swap(CSR_HSTATUS, hcntx->hstatus);
851

852
		trap->htval = csr_read(CSR_HTVAL);
853
		trap->htinst = csr_read(CSR_HTINST);
854
	}
855

856
	trap->sepc = gcntx->sepc;
857
	trap->scause = csr_read(CSR_SCAUSE);
858
	trap->stval = csr_read(CSR_STVAL);
859

860
	vcpu->arch.last_exit_cpu = vcpu->cpu;
861
	guest_state_exit_irqoff();
862
	kvm_riscv_vcpu_swap_in_host_state(vcpu);
863
}
864

865
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
866
{
867
	int ret;
868
	struct kvm_cpu_trap trap;
869
	struct kvm_run *run = vcpu->run;
870

871
	if (!vcpu->arch.ran_atleast_once)
872
		kvm_riscv_vcpu_setup_config(vcpu);
873

874
	/* Mark this VCPU ran at least once */
875
	vcpu->arch.ran_atleast_once = true;
876

877
	kvm_vcpu_srcu_read_lock(vcpu);
878

879
	switch (run->exit_reason) {
880
	case KVM_EXIT_MMIO:
881
		/* Process MMIO value returned from user-space */
882
		ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
883
		break;
884
	case KVM_EXIT_RISCV_SBI:
885
		/* Process SBI value returned from user-space */
886
		ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
887
		break;
888
	case KVM_EXIT_RISCV_CSR:
889
		/* Process CSR value returned from user-space */
890
		ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
891
		break;
892
	default:
893
		ret = 0;
894
		break;
895
	}
896
	if (ret) {
897
		kvm_vcpu_srcu_read_unlock(vcpu);
898
		return ret;
899
	}
900

901
	if (!vcpu->wants_to_run) {
902
		kvm_vcpu_srcu_read_unlock(vcpu);
903
		return -EINTR;
904
	}
905

906
	vcpu_load(vcpu);
907

908
	kvm_sigset_activate(vcpu);
909

910
	ret = 1;
911
	run->exit_reason = KVM_EXIT_UNKNOWN;
912
	while (ret > 0) {
913
		/* Check conditions before entering the guest */
914
		ret = xfer_to_guest_mode_handle_work(vcpu);
915
		if (ret)
916
			continue;
917
		ret = 1;
918

919
		kvm_riscv_gstage_vmid_update(vcpu);
920

921
		ret = kvm_riscv_check_vcpu_requests(vcpu);
922
		if (ret <= 0)
923
			continue;
924

925
		preempt_disable();
926

927
		/* Update AIA HW state before entering guest */
928
		ret = kvm_riscv_vcpu_aia_update(vcpu);
929
		if (ret <= 0) {
930
			preempt_enable();
931
			continue;
932
		}
933

934
		local_irq_disable();
935

936
		/*
937
		 * Ensure we set mode to IN_GUEST_MODE after we disable
938
		 * interrupts and before the final VCPU requests check.
939
		 * See the comment in kvm_vcpu_exiting_guest_mode() and
940
		 * Documentation/virt/kvm/vcpu-requests.rst
941
		 */
942
		vcpu->mode = IN_GUEST_MODE;
943

944
		kvm_vcpu_srcu_read_unlock(vcpu);
945
		smp_mb__after_srcu_read_unlock();
946

947
		/*
948
		 * We might have got VCPU interrupts updated asynchronously
949
		 * so update it in HW.
950
		 */
951
		kvm_riscv_vcpu_flush_interrupts(vcpu);
952

953
		/* Update HVIP CSR for current CPU */
954
		kvm_riscv_update_hvip(vcpu);
955

956
		if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
957
		    kvm_request_pending(vcpu) ||
958
		    xfer_to_guest_mode_work_pending()) {
959
			vcpu->mode = OUTSIDE_GUEST_MODE;
960
			local_irq_enable();
961
			preempt_enable();
962
			kvm_vcpu_srcu_read_lock(vcpu);
963
			continue;
964
		}
965

966
		/*
967
		 * Sanitize VMID mappings cached (TLB) on current CPU
968
		 *
969
		 * Note: This should be done after G-stage VMID has been
970
		 * updated using kvm_riscv_gstage_vmid_ver_changed()
971
		 */
972
		kvm_riscv_gstage_vmid_sanitize(vcpu);
973

974
		trace_kvm_entry(vcpu);
975

976
		guest_timing_enter_irqoff();
977

978
		kvm_riscv_vcpu_enter_exit(vcpu, &trap);
979

980
		vcpu->mode = OUTSIDE_GUEST_MODE;
981
		vcpu->stat.exits++;
982

983
		/* Syncup interrupts state with HW */
984
		kvm_riscv_vcpu_sync_interrupts(vcpu);
985

986
		/*
987
		 * We must ensure that any pending interrupts are taken before
988
		 * we exit guest timing so that timer ticks are accounted as
989
		 * guest time. Transiently unmask interrupts so that any
990
		 * pending interrupts are taken.
991
		 *
992
		 * There's no barrier which ensures that pending interrupts are
993
		 * recognised, so we just hope that the CPU takes any pending
994
		 * interrupts between the enable and disable.
995
		 */
996
		local_irq_enable();
997
		local_irq_disable();
998

999
		guest_timing_exit_irqoff();
1000

1001
		local_irq_enable();
1002

1003
		trace_kvm_exit(&trap);
1004

1005
		preempt_enable();
1006

1007
		kvm_vcpu_srcu_read_lock(vcpu);
1008

1009
		ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
1010
	}
1011

1012
	kvm_sigset_deactivate(vcpu);
1013

1014
	vcpu_put(vcpu);
1015

1016
	kvm_vcpu_srcu_read_unlock(vcpu);
1017

1018
	return ret;
1019
}
1020

1021