1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4
 */
5

6
#include <linux/kvm_host.h>
7
#include <linux/entry-kvm.h>
8
#include <asm/fpu.h>
9
#include <asm/lbt.h>
10
#include <asm/loongarch.h>
11
#include <asm/setup.h>
12
#include <asm/time.h>
13

14
#define CREATE_TRACE_POINTS
15
#include "trace.h"
16

17
const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
18
	KVM_GENERIC_VCPU_STATS(),
19
	STATS_DESC_COUNTER(VCPU, int_exits),
20
	STATS_DESC_COUNTER(VCPU, idle_exits),
21
	STATS_DESC_COUNTER(VCPU, cpucfg_exits),
22
	STATS_DESC_COUNTER(VCPU, signal_exits),
23
	STATS_DESC_COUNTER(VCPU, hypercall_exits),
24
	STATS_DESC_COUNTER(VCPU, ipi_read_exits),
25
	STATS_DESC_COUNTER(VCPU, ipi_write_exits),
26
	STATS_DESC_COUNTER(VCPU, eiointc_read_exits),
27
	STATS_DESC_COUNTER(VCPU, eiointc_write_exits),
28
	STATS_DESC_COUNTER(VCPU, pch_pic_read_exits),
29
	STATS_DESC_COUNTER(VCPU, pch_pic_write_exits)
30
};
31

32
const struct kvm_stats_header kvm_vcpu_stats_header = {
33
	.name_size = KVM_STATS_NAME_SIZE,
34
	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
35
	.id_offset = sizeof(struct kvm_stats_header),
36
	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
37
	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
38
		       sizeof(kvm_vcpu_stats_desc),
39
};
40

41
static inline void kvm_save_host_pmu(struct kvm_vcpu *vcpu)
42
{
43
	struct kvm_context *context;
44

45
	context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
46
	context->perf_cntr[0] = read_csr_perfcntr0();
47
	context->perf_cntr[1] = read_csr_perfcntr1();
48
	context->perf_cntr[2] = read_csr_perfcntr2();
49
	context->perf_cntr[3] = read_csr_perfcntr3();
50
	context->perf_ctrl[0] = write_csr_perfctrl0(0);
51
	context->perf_ctrl[1] = write_csr_perfctrl1(0);
52
	context->perf_ctrl[2] = write_csr_perfctrl2(0);
53
	context->perf_ctrl[3] = write_csr_perfctrl3(0);
54
}
55

56
static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu)
57
{
58
	struct kvm_context *context;
59

60
	context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
61
	write_csr_perfcntr0(context->perf_cntr[0]);
62
	write_csr_perfcntr1(context->perf_cntr[1]);
63
	write_csr_perfcntr2(context->perf_cntr[2]);
64
	write_csr_perfcntr3(context->perf_cntr[3]);
65
	write_csr_perfctrl0(context->perf_ctrl[0]);
66
	write_csr_perfctrl1(context->perf_ctrl[1]);
67
	write_csr_perfctrl2(context->perf_ctrl[2]);
68
	write_csr_perfctrl3(context->perf_ctrl[3]);
69
}
70

71

72
static inline void kvm_save_guest_pmu(struct kvm_vcpu *vcpu)
73
{
74
	struct loongarch_csrs *csr = vcpu->arch.csr;
75

76
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0);
77
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1);
78
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2);
79
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3);
80
	kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
81
	kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
82
	kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
83
	kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
84
}
85

86
static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu)
87
{
88
	struct loongarch_csrs *csr = vcpu->arch.csr;
89

90
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0);
91
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1);
92
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2);
93
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3);
94
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
95
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
96
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
97
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
98
}
99

100
static int kvm_own_pmu(struct kvm_vcpu *vcpu)
101
{
102
	unsigned long val;
103

104
	if (!kvm_guest_has_pmu(&vcpu->arch))
105
		return -EINVAL;
106

107
	kvm_save_host_pmu(vcpu);
108

109
	/* Set PM0-PM(num) to guest */
110
	val = read_csr_gcfg() & ~CSR_GCFG_GPERF;
111
	val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT;
112
	write_csr_gcfg(val);
113

114
	kvm_restore_guest_pmu(vcpu);
115

116
	return 0;
117
}
118

119
static void kvm_lose_pmu(struct kvm_vcpu *vcpu)
120
{
121
	unsigned long val;
122
	struct loongarch_csrs *csr = vcpu->arch.csr;
123

124
	if (!(vcpu->arch.aux_inuse & KVM_LARCH_PMU))
125
		return;
126

127
	kvm_save_guest_pmu(vcpu);
128

129
	/* Disable pmu access from guest */
130
	write_csr_gcfg(read_csr_gcfg() & ~CSR_GCFG_GPERF);
131

132
	/*
133
	 * Clear KVM_LARCH_PMU if the guest is not using PMU CSRs when
134
	 * exiting the guest, so that the next time trap into the guest.
135
	 * We don't need to deal with PMU CSRs contexts.
136
	 */
137
	val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
138
	val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
139
	val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
140
	val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
141
	if (!(val & KVM_PMU_EVENT_ENABLED))
142
		vcpu->arch.aux_inuse &= ~KVM_LARCH_PMU;
143

144
	kvm_restore_host_pmu(vcpu);
145
}
146

147
static void kvm_restore_pmu(struct kvm_vcpu *vcpu)
148
{
149
	if ((vcpu->arch.aux_inuse & KVM_LARCH_PMU))
150
		kvm_make_request(KVM_REQ_PMU, vcpu);
151
}
152

153
static void kvm_check_pmu(struct kvm_vcpu *vcpu)
154
{
155
	if (kvm_check_request(KVM_REQ_PMU, vcpu)) {
156
		kvm_own_pmu(vcpu);
157
		vcpu->arch.aux_inuse |= KVM_LARCH_PMU;
158
	}
159
}
160

161
static void kvm_update_stolen_time(struct kvm_vcpu *vcpu)
162
{
163
	u32 version;
164
	u64 steal;
165
	gpa_t gpa;
166
	struct kvm_memslots *slots;
167
	struct kvm_steal_time __user *st;
168
	struct gfn_to_hva_cache *ghc;
169

170
	ghc = &vcpu->arch.st.cache;
171
	gpa = vcpu->arch.st.guest_addr;
172
	if (!(gpa & KVM_STEAL_PHYS_VALID))
173
		return;
174

175
	gpa &= KVM_STEAL_PHYS_MASK;
176
	slots = kvm_memslots(vcpu->kvm);
177
	if (slots->generation != ghc->generation || gpa != ghc->gpa) {
178
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) {
179
			ghc->gpa = INVALID_GPA;
180
			return;
181
		}
182
	}
183

184
	st = (struct kvm_steal_time __user *)ghc->hva;
185
	unsafe_get_user(version, &st->version, out);
186
	if (version & 1)
187
		version += 1; /* first time write, random junk */
188

189
	version += 1;
190
	unsafe_put_user(version, &st->version, out);
191
	smp_wmb();
192

193
	unsafe_get_user(steal, &st->steal, out);
194
	steal += current->sched_info.run_delay - vcpu->arch.st.last_steal;
195
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
196
	unsafe_put_user(steal, &st->steal, out);
197

198
	smp_wmb();
199
	version += 1;
200
	unsafe_put_user(version, &st->version, out);
201
out:
202
	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
203
}
204

205
/*
206
 * kvm_check_requests - check and handle pending vCPU requests
207
 *
208
 * Return: RESUME_GUEST if we should enter the guest
209
 *         RESUME_HOST  if we should exit to userspace
210
 */
211
static int kvm_check_requests(struct kvm_vcpu *vcpu)
212
{
213
	if (!kvm_request_pending(vcpu))
214
		return RESUME_GUEST;
215

216
	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
217
		vcpu->arch.vpid = 0;  /* Drop vpid for this vCPU */
218

219
	if (kvm_dirty_ring_check_request(vcpu))
220
		return RESUME_HOST;
221

222
	if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
223
		kvm_update_stolen_time(vcpu);
224

225
	return RESUME_GUEST;
226
}
227

228
static void kvm_late_check_requests(struct kvm_vcpu *vcpu)
229
{
230
	lockdep_assert_irqs_disabled();
231
	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GPA, vcpu))
232
		if (vcpu->arch.flush_gpa != INVALID_GPA) {
233
			kvm_flush_tlb_gpa(vcpu, vcpu->arch.flush_gpa);
234
			vcpu->arch.flush_gpa = INVALID_GPA;
235
		}
236
}
237

238
/*
239
 * Check and handle pending signal and vCPU requests etc
240
 * Run with irq enabled and preempt enabled
241
 *
242
 * Return: RESUME_GUEST if we should enter the guest
243
 *         RESUME_HOST  if we should exit to userspace
244
 *         < 0 if we should exit to userspace, where the return value
245
 *         indicates an error
246
 */
247
static int kvm_enter_guest_check(struct kvm_vcpu *vcpu)
248
{
249
	int idx, ret;
250

251
	/*
252
	 * Check conditions before entering the guest
253
	 */
254
	ret = xfer_to_guest_mode_handle_work(vcpu);
255
	if (ret < 0)
256
		return ret;
257

258
	idx = srcu_read_lock(&vcpu->kvm->srcu);
259
	ret = kvm_check_requests(vcpu);
260
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
261

262
	return ret;
263
}
264

265
/*
266
 * Called with irq enabled
267
 *
268
 * Return: RESUME_GUEST if we should enter the guest, and irq disabled
269
 *         Others if we should exit to userspace
270
 */
271
static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu)
272
{
273
	int ret;
274

275
	do {
276
		ret = kvm_enter_guest_check(vcpu);
277
		if (ret != RESUME_GUEST)
278
			break;
279

280
		/*
281
		 * Handle vcpu timer, interrupts, check requests and
282
		 * check vmid before vcpu enter guest
283
		 */
284
		local_irq_disable();
285
		kvm_deliver_intr(vcpu);
286
		kvm_deliver_exception(vcpu);
287
		/* Make sure the vcpu mode has been written */
288
		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
289
		kvm_check_vpid(vcpu);
290
		kvm_check_pmu(vcpu);
291

292
		/*
293
		 * Called after function kvm_check_vpid()
294
		 * Since it updates CSR.GSTAT used by kvm_flush_tlb_gpa(),
295
		 * and it may also clear KVM_REQ_TLB_FLUSH_GPA pending bit
296
		 */
297
		kvm_late_check_requests(vcpu);
298
		vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY);
299
		/* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */
300
		vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST;
301

302
		if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) {
303
			kvm_lose_pmu(vcpu);
304
			/* make sure the vcpu mode has been written */
305
			smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE);
306
			local_irq_enable();
307
			ret = -EAGAIN;
308
		}
309
	} while (ret != RESUME_GUEST);
310

311
	return ret;
312
}
313

314
/*
315
 * Return 1 for resume guest and "<= 0" for resume host.
316
 */
317
static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
318
{
319
	int ret = RESUME_GUEST;
320
	unsigned long estat = vcpu->arch.host_estat;
321
	u32 intr = estat & CSR_ESTAT_IS;
322
	u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT;
323

324
	vcpu->mode = OUTSIDE_GUEST_MODE;
325

326
	/* Set a default exit reason */
327
	run->exit_reason = KVM_EXIT_UNKNOWN;
328

329
	kvm_lose_pmu(vcpu);
330

331
	guest_timing_exit_irqoff();
332
	guest_state_exit_irqoff();
333
	local_irq_enable();
334

335
	trace_kvm_exit(vcpu, ecode);
336
	if (ecode) {
337
		ret = kvm_handle_fault(vcpu, ecode);
338
	} else {
339
		WARN(!intr, "vm exiting with suspicious irq\n");
340
		++vcpu->stat.int_exits;
341
	}
342

343
	if (ret == RESUME_GUEST)
344
		ret = kvm_pre_enter_guest(vcpu);
345

346
	if (ret != RESUME_GUEST) {
347
		local_irq_disable();
348
		return ret;
349
	}
350

351
	guest_timing_enter_irqoff();
352
	guest_state_enter_irqoff();
353
	trace_kvm_reenter(vcpu);
354

355
	return RESUME_GUEST;
356
}
357

358
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
359
{
360
	return !!(vcpu->arch.irq_pending) &&
361
		vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE;
362
}
363

364
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
365
{
366
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
367
}
368

369
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
370
{
371
	unsigned long val;
372

373
	preempt_disable();
374
	val = gcsr_read(LOONGARCH_CSR_CRMD);
375
	preempt_enable();
376

377
	return (val & CSR_PRMD_PPLV) == PLV_KERN;
378
}
379

380
#ifdef CONFIG_GUEST_PERF_EVENTS
381
unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
382
{
383
	return vcpu->arch.pc;
384
}
385

386
/*
387
 * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event,
388
 * arrived in guest context.  For LoongArch64, if PMU is not passthrough to VM,
389
 * any event that arrives while a vCPU is loaded is considered to be "in guest".
390
 */
391
bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu)
392
{
393
	return (vcpu && !(vcpu->arch.aux_inuse & KVM_LARCH_PMU));
394
}
395
#endif
396

397
bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)
398
{
399
	return false;
400
}
401

402
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
403
{
404
	return VM_FAULT_SIGBUS;
405
}
406

407
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
408
				  struct kvm_translation *tr)
409
{
410
	return -EINVAL;
411
}
412

413
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
414
{
415
	int ret;
416

417
	/* Protect from TOD sync and vcpu_load/put() */
418
	preempt_disable();
419
	ret = kvm_pending_timer(vcpu) ||
420
		kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI);
421
	preempt_enable();
422

423
	return ret;
424
}
425

426
int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu)
427
{
428
	int i;
429

430
	kvm_debug("vCPU Register Dump:\n");
431
	kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc);
432
	kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending);
433

434
	for (i = 0; i < 32; i += 4) {
435
		kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i,
436
		       vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1],
437
		       vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]);
438
	}
439

440
	kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n",
441
		  kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD),
442
		  kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT));
443

444
	kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA));
445

446
	return 0;
447
}
448

449
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
450
				struct kvm_mp_state *mp_state)
451
{
452
	*mp_state = vcpu->arch.mp_state;
453

454
	return 0;
455
}
456

457
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
458
				struct kvm_mp_state *mp_state)
459
{
460
	int ret = 0;
461

462
	switch (mp_state->mp_state) {
463
	case KVM_MP_STATE_RUNNABLE:
464
		vcpu->arch.mp_state = *mp_state;
465
		break;
466
	default:
467
		ret = -EINVAL;
468
	}
469

470
	return ret;
471
}
472

473
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
474
					struct kvm_guest_debug *dbg)
475
{
476
	if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
477
		return -EINVAL;
478

479
	if (dbg->control & KVM_GUESTDBG_ENABLE)
480
		vcpu->guest_debug = dbg->control;
481
	else
482
		vcpu->guest_debug = 0;
483

484
	return 0;
485
}
486

487
static inline int kvm_set_cpuid(struct kvm_vcpu *vcpu, u64 val)
488
{
489
	int cpuid;
490
	struct kvm_phyid_map *map;
491
	struct loongarch_csrs *csr = vcpu->arch.csr;
492

493
	if (val >= KVM_MAX_PHYID)
494
		return -EINVAL;
495

496
	map = vcpu->kvm->arch.phyid_map;
497
	cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID);
498

499
	spin_lock(&vcpu->kvm->arch.phyid_map_lock);
500
	if ((cpuid < KVM_MAX_PHYID) && map->phys_map[cpuid].enabled) {
501
		/* Discard duplicated CPUID set operation */
502
		if (cpuid == val) {
503
			spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
504
			return 0;
505
		}
506

507
		/*
508
		 * CPUID is already set before
509
		 * Forbid changing to a different CPUID at runtime
510
		 */
511
		spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
512
		return -EINVAL;
513
	}
514

515
	if (map->phys_map[val].enabled) {
516
		/* Discard duplicated CPUID set operation */
517
		if (vcpu == map->phys_map[val].vcpu) {
518
			spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
519
			return 0;
520
		}
521

522
		/*
523
		 * New CPUID is already set with other vcpu
524
		 * Forbid sharing the same CPUID between different vcpus
525
		 */
526
		spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
527
		return -EINVAL;
528
	}
529

530
	kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, val);
531
	map->phys_map[val].enabled	= true;
532
	map->phys_map[val].vcpu		= vcpu;
533
	spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
534

535
	return 0;
536
}
537

538
static inline void kvm_drop_cpuid(struct kvm_vcpu *vcpu)
539
{
540
	int cpuid;
541
	struct kvm_phyid_map *map;
542
	struct loongarch_csrs *csr = vcpu->arch.csr;
543

544
	map = vcpu->kvm->arch.phyid_map;
545
	cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID);
546

547
	if (cpuid >= KVM_MAX_PHYID)
548
		return;
549

550
	spin_lock(&vcpu->kvm->arch.phyid_map_lock);
551
	if (map->phys_map[cpuid].enabled) {
552
		map->phys_map[cpuid].vcpu = NULL;
553
		map->phys_map[cpuid].enabled = false;
554
		kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID);
555
	}
556
	spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
557
}
558

559
struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid)
560
{
561
	struct kvm_phyid_map *map;
562

563
	if (cpuid >= KVM_MAX_PHYID)
564
		return NULL;
565

566
	map = kvm->arch.phyid_map;
567
	if (!map->phys_map[cpuid].enabled)
568
		return NULL;
569

570
	return map->phys_map[cpuid].vcpu;
571
}
572

573
static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val)
574
{
575
	unsigned long gintc;
576
	struct loongarch_csrs *csr = vcpu->arch.csr;
577

578
	if (get_gcsr_flag(id) & INVALID_GCSR)
579
		return -EINVAL;
580

581
	if (id == LOONGARCH_CSR_ESTAT) {
582
		preempt_disable();
583
		vcpu_load(vcpu);
584
		/*
585
		 * Sync pending interrupts into ESTAT so that interrupt
586
		 * remains during VM migration stage
587
		 */
588
		kvm_deliver_intr(vcpu);
589
		vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST;
590
		vcpu_put(vcpu);
591
		preempt_enable();
592

593
		/* ESTAT IP0~IP7 get from GINTC */
594
		gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & 0xff;
595
		*val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) | (gintc << 2);
596
		return 0;
597
	}
598

599
	/*
600
	 * Get software CSR state since software state is consistent
601
	 * with hardware for synchronous ioctl
602
	 */
603
	*val = kvm_read_sw_gcsr(csr, id);
604

605
	return 0;
606
}
607

608
static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val)
609
{
610
	int ret = 0, gintc;
611
	struct loongarch_csrs *csr = vcpu->arch.csr;
612

613
	if (get_gcsr_flag(id) & INVALID_GCSR)
614
		return -EINVAL;
615

616
	if (id == LOONGARCH_CSR_CPUID)
617
		return kvm_set_cpuid(vcpu, val);
618

619
	if (id == LOONGARCH_CSR_ESTAT) {
620
		/* ESTAT IP0~IP7 inject through GINTC */
621
		gintc = (val >> 2) & 0xff;
622
		kvm_set_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc);
623

624
		gintc = val & ~(0xffUL << 2);
625
		kvm_set_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, gintc);
626

627
		return ret;
628
	}
629

630
	kvm_write_sw_gcsr(csr, id, val);
631

632
	/*
633
	 * After modifying the PMU CSR register value of the vcpu.
634
	 * If the PMU CSRs are used, we need to set KVM_REQ_PMU.
635
	 */
636
	if (id >= LOONGARCH_CSR_PERFCTRL0 && id <= LOONGARCH_CSR_PERFCNTR3) {
637
		unsigned long val;
638

639
		val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0) |
640
		      kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1) |
641
		      kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2) |
642
		      kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
643

644
		if (val & KVM_PMU_EVENT_ENABLED)
645
			kvm_make_request(KVM_REQ_PMU, vcpu);
646
	}
647

648
	return ret;
649
}
650

651
static int _kvm_get_cpucfg_mask(int id, u64 *v)
652
{
653
	if (id < 0 || id >= KVM_MAX_CPUCFG_REGS)
654
		return -EINVAL;
655

656
	switch (id) {
657
	case LOONGARCH_CPUCFG0:
658
		*v = GENMASK(31, 0);
659
		return 0;
660
	case LOONGARCH_CPUCFG1:
661
		/* CPUCFG1_MSGINT is not supported by KVM */
662
		*v = GENMASK(25, 0);
663
		return 0;
664
	case LOONGARCH_CPUCFG2:
665
		/* CPUCFG2 features unconditionally supported by KVM */
666
		*v = CPUCFG2_FP     | CPUCFG2_FPSP  | CPUCFG2_FPDP     |
667
		     CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV |
668
		     CPUCFG2_LSPW | CPUCFG2_LAM;
669
		/*
670
		 * For the ISA extensions listed below, if one is supported
671
		 * by the host, then it is also supported by KVM.
672
		 */
673
		if (cpu_has_lsx)
674
			*v |= CPUCFG2_LSX;
675
		if (cpu_has_lasx)
676
			*v |= CPUCFG2_LASX;
677
		if (cpu_has_lbt_x86)
678
			*v |= CPUCFG2_X86BT;
679
		if (cpu_has_lbt_arm)
680
			*v |= CPUCFG2_ARMBT;
681
		if (cpu_has_lbt_mips)
682
			*v |= CPUCFG2_MIPSBT;
683
		if (cpu_has_ptw)
684
			*v |= CPUCFG2_PTW;
685

686
		return 0;
687
	case LOONGARCH_CPUCFG3:
688
		*v = GENMASK(16, 0);
689
		return 0;
690
	case LOONGARCH_CPUCFG4:
691
	case LOONGARCH_CPUCFG5:
692
		*v = GENMASK(31, 0);
693
		return 0;
694
	case LOONGARCH_CPUCFG6:
695
		if (cpu_has_pmp)
696
			*v = GENMASK(14, 0);
697
		else
698
			*v = 0;
699
		return 0;
700
	case LOONGARCH_CPUCFG16:
701
		*v = GENMASK(16, 0);
702
		return 0;
703
	case LOONGARCH_CPUCFG17 ... LOONGARCH_CPUCFG20:
704
		*v = GENMASK(30, 0);
705
		return 0;
706
	default:
707
		/*
708
		 * CPUCFG bits should be zero if reserved by HW or not
709
		 * supported by KVM.
710
		 */
711
		*v = 0;
712
		return 0;
713
	}
714
}
715

716
static int kvm_check_cpucfg(int id, u64 val)
717
{
718
	int ret;
719
	u64 mask = 0;
720

721
	ret = _kvm_get_cpucfg_mask(id, &mask);
722
	if (ret)
723
		return ret;
724

725
	if (val & ~mask)
726
		/* Unsupported features and/or the higher 32 bits should not be set */
727
		return -EINVAL;
728

729
	switch (id) {
730
	case LOONGARCH_CPUCFG2:
731
		if (!(val & CPUCFG2_LLFTP))
732
			/* Guests must have a constant timer */
733
			return -EINVAL;
734
		if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP)))
735
			/* Single and double float point must both be set when FP is enabled */
736
			return -EINVAL;
737
		if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP))
738
			/* LSX architecturally implies FP but val does not satisfy that */
739
			return -EINVAL;
740
		if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX))
741
			/* LASX architecturally implies LSX and FP but val does not satisfy that */
742
			return -EINVAL;
743
		return 0;
744
	case LOONGARCH_CPUCFG6:
745
		if (val & CPUCFG6_PMP) {
746
			u32 host = read_cpucfg(LOONGARCH_CPUCFG6);
747
			if ((val & CPUCFG6_PMBITS) != (host & CPUCFG6_PMBITS))
748
				return -EINVAL;
749
			if ((val & CPUCFG6_PMNUM) > (host & CPUCFG6_PMNUM))
750
				return -EINVAL;
751
			if ((val & CPUCFG6_UPM) && !(host & CPUCFG6_UPM))
752
				return -EINVAL;
753
		}
754
		return 0;
755
	default:
756
		/*
757
		 * Values for the other CPUCFG IDs are not being further validated
758
		 * besides the mask check above.
759
		 */
760
		return 0;
761
	}
762
}
763

764
static int kvm_get_one_reg(struct kvm_vcpu *vcpu,
765
		const struct kvm_one_reg *reg, u64 *v)
766
{
767
	int id, ret = 0;
768
	u64 type = reg->id & KVM_REG_LOONGARCH_MASK;
769

770
	switch (type) {
771
	case KVM_REG_LOONGARCH_CSR:
772
		id = KVM_GET_IOC_CSR_IDX(reg->id);
773
		ret = _kvm_getcsr(vcpu, id, v);
774
		break;
775
	case KVM_REG_LOONGARCH_CPUCFG:
776
		id = KVM_GET_IOC_CPUCFG_IDX(reg->id);
777
		if (id >= 0 && id < KVM_MAX_CPUCFG_REGS)
778
			*v = vcpu->arch.cpucfg[id];
779
		else
780
			ret = -EINVAL;
781
		break;
782
	case KVM_REG_LOONGARCH_LBT:
783
		if (!kvm_guest_has_lbt(&vcpu->arch))
784
			return -ENXIO;
785

786
		switch (reg->id) {
787
		case KVM_REG_LOONGARCH_LBT_SCR0:
788
			*v = vcpu->arch.lbt.scr0;
789
			break;
790
		case KVM_REG_LOONGARCH_LBT_SCR1:
791
			*v = vcpu->arch.lbt.scr1;
792
			break;
793
		case KVM_REG_LOONGARCH_LBT_SCR2:
794
			*v = vcpu->arch.lbt.scr2;
795
			break;
796
		case KVM_REG_LOONGARCH_LBT_SCR3:
797
			*v = vcpu->arch.lbt.scr3;
798
			break;
799
		case KVM_REG_LOONGARCH_LBT_EFLAGS:
800
			*v = vcpu->arch.lbt.eflags;
801
			break;
802
		case KVM_REG_LOONGARCH_LBT_FTOP:
803
			*v = vcpu->arch.fpu.ftop;
804
			break;
805
		default:
806
			ret = -EINVAL;
807
			break;
808
		}
809
		break;
810
	case KVM_REG_LOONGARCH_KVM:
811
		switch (reg->id) {
812
		case KVM_REG_LOONGARCH_COUNTER:
813
			*v = drdtime() + vcpu->kvm->arch.time_offset;
814
			break;
815
		case KVM_REG_LOONGARCH_DEBUG_INST:
816
			*v = INSN_HVCL | KVM_HCALL_SWDBG;
817
			break;
818
		default:
819
			ret = -EINVAL;
820
			break;
821
		}
822
		break;
823
	default:
824
		ret = -EINVAL;
825
		break;
826
	}
827

828
	return ret;
829
}
830

831
static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
832
{
833
	int ret = 0;
834
	u64 v, size = reg->id & KVM_REG_SIZE_MASK;
835

836
	switch (size) {
837
	case KVM_REG_SIZE_U64:
838
		ret = kvm_get_one_reg(vcpu, reg, &v);
839
		if (ret)
840
			return ret;
841
		ret = put_user(v, (u64 __user *)(long)reg->addr);
842
		break;
843
	default:
844
		ret = -EINVAL;
845
		break;
846
	}
847

848
	return ret;
849
}
850

851
static int kvm_set_one_reg(struct kvm_vcpu *vcpu,
852
			const struct kvm_one_reg *reg, u64 v)
853
{
854
	int id, ret = 0;
855
	u64 type = reg->id & KVM_REG_LOONGARCH_MASK;
856

857
	switch (type) {
858
	case KVM_REG_LOONGARCH_CSR:
859
		id = KVM_GET_IOC_CSR_IDX(reg->id);
860
		ret = _kvm_setcsr(vcpu, id, v);
861
		break;
862
	case KVM_REG_LOONGARCH_CPUCFG:
863
		id = KVM_GET_IOC_CPUCFG_IDX(reg->id);
864
		ret = kvm_check_cpucfg(id, v);
865
		if (ret)
866
			break;
867
		vcpu->arch.cpucfg[id] = (u32)v;
868
		if (id == LOONGARCH_CPUCFG6)
869
			vcpu->arch.max_pmu_csrid =
870
				LOONGARCH_CSR_PERFCTRL0 + 2 * kvm_get_pmu_num(&vcpu->arch) + 1;
871
		break;
872
	case KVM_REG_LOONGARCH_LBT:
873
		if (!kvm_guest_has_lbt(&vcpu->arch))
874
			return -ENXIO;
875

876
		switch (reg->id) {
877
		case KVM_REG_LOONGARCH_LBT_SCR0:
878
			vcpu->arch.lbt.scr0 = v;
879
			break;
880
		case KVM_REG_LOONGARCH_LBT_SCR1:
881
			vcpu->arch.lbt.scr1 = v;
882
			break;
883
		case KVM_REG_LOONGARCH_LBT_SCR2:
884
			vcpu->arch.lbt.scr2 = v;
885
			break;
886
		case KVM_REG_LOONGARCH_LBT_SCR3:
887
			vcpu->arch.lbt.scr3 = v;
888
			break;
889
		case KVM_REG_LOONGARCH_LBT_EFLAGS:
890
			vcpu->arch.lbt.eflags = v;
891
			break;
892
		case KVM_REG_LOONGARCH_LBT_FTOP:
893
			vcpu->arch.fpu.ftop = v;
894
			break;
895
		default:
896
			ret = -EINVAL;
897
			break;
898
		}
899
		break;
900
	case KVM_REG_LOONGARCH_KVM:
901
		switch (reg->id) {
902
		case KVM_REG_LOONGARCH_COUNTER:
903
			/*
904
			 * gftoffset is relative with board, not vcpu
905
			 * only set for the first time for smp system
906
			 */
907
			if (vcpu->vcpu_id == 0)
908
				vcpu->kvm->arch.time_offset = (signed long)(v - drdtime());
909
			break;
910
		case KVM_REG_LOONGARCH_VCPU_RESET:
911
			vcpu->arch.st.guest_addr = 0;
912
			memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending));
913
			memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear));
914

915
			/*
916
			 * When vCPU reset, clear the ESTAT and GINTC registers
917
			 * Other CSR registers are cleared with function _kvm_setcsr().
918
			 */
919
			kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_GINTC, 0);
920
			kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_ESTAT, 0);
921
			break;
922
		default:
923
			ret = -EINVAL;
924
			break;
925
		}
926
		break;
927
	default:
928
		ret = -EINVAL;
929
		break;
930
	}
931

932
	return ret;
933
}
934

935
static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
936
{
937
	int ret = 0;
938
	u64 v, size = reg->id & KVM_REG_SIZE_MASK;
939

940
	switch (size) {
941
	case KVM_REG_SIZE_U64:
942
		ret = get_user(v, (u64 __user *)(long)reg->addr);
943
		if (ret)
944
			return ret;
945
		break;
946
	default:
947
		return -EINVAL;
948
	}
949

950
	return kvm_set_one_reg(vcpu, reg, v);
951
}
952

953
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
954
{
955
	return -ENOIOCTLCMD;
956
}
957

958
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
959
{
960
	return -ENOIOCTLCMD;
961
}
962

963
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
964
{
965
	int i;
966

967
	for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
968
		regs->gpr[i] = vcpu->arch.gprs[i];
969

970
	regs->pc = vcpu->arch.pc;
971

972
	return 0;
973
}
974

975
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
976
{
977
	int i;
978

979
	for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
980
		vcpu->arch.gprs[i] = regs->gpr[i];
981

982
	vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
983
	vcpu->arch.pc = regs->pc;
984

985
	return 0;
986
}
987

988
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
989
				     struct kvm_enable_cap *cap)
990
{
991
	/* FPU is enabled by default, will support LSX/LASX later. */
992
	return -EINVAL;
993
}
994

995
static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu,
996
					 struct kvm_device_attr *attr)
997
{
998
	switch (attr->attr) {
999
	case LOONGARCH_CPUCFG2:
1000
	case LOONGARCH_CPUCFG6:
1001
		return 0;
1002
	case CPUCFG_KVM_FEATURE:
1003
		return 0;
1004
	default:
1005
		return -ENXIO;
1006
	}
1007

1008
	return -ENXIO;
1009
}
1010

1011
static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu,
1012
					 struct kvm_device_attr *attr)
1013
{
1014
	if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
1015
			|| attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
1016
		return -ENXIO;
1017

1018
	return 0;
1019
}
1020

1021
static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu,
1022
				       struct kvm_device_attr *attr)
1023
{
1024
	int ret = -ENXIO;
1025

1026
	switch (attr->group) {
1027
	case KVM_LOONGARCH_VCPU_CPUCFG:
1028
		ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr);
1029
		break;
1030
	case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
1031
		ret = kvm_loongarch_pvtime_has_attr(vcpu, attr);
1032
		break;
1033
	default:
1034
		break;
1035
	}
1036

1037
	return ret;
1038
}
1039

1040
static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu,
1041
					 struct kvm_device_attr *attr)
1042
{
1043
	int ret = 0;
1044
	uint64_t val;
1045
	uint64_t __user *uaddr = (uint64_t __user *)attr->addr;
1046

1047
	switch (attr->attr) {
1048
	case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
1049
		ret = _kvm_get_cpucfg_mask(attr->attr, &val);
1050
		if (ret)
1051
			return ret;
1052
		break;
1053
	case CPUCFG_KVM_FEATURE:
1054
		val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
1055
		break;
1056
	default:
1057
		return -ENXIO;
1058
	}
1059

1060
	put_user(val, uaddr);
1061

1062
	return ret;
1063
}
1064

1065
static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu,
1066
					 struct kvm_device_attr *attr)
1067
{
1068
	u64 gpa;
1069
	u64 __user *user = (u64 __user *)attr->addr;
1070

1071
	if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
1072
			|| attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
1073
		return -ENXIO;
1074

1075
	gpa = vcpu->arch.st.guest_addr;
1076
	if (put_user(gpa, user))
1077
		return -EFAULT;
1078

1079
	return 0;
1080
}
1081

1082
static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu,
1083
				       struct kvm_device_attr *attr)
1084
{
1085
	int ret = -ENXIO;
1086

1087
	switch (attr->group) {
1088
	case KVM_LOONGARCH_VCPU_CPUCFG:
1089
		ret = kvm_loongarch_cpucfg_get_attr(vcpu, attr);
1090
		break;
1091
	case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
1092
		ret = kvm_loongarch_pvtime_get_attr(vcpu, attr);
1093
		break;
1094
	default:
1095
		break;
1096
	}
1097

1098
	return ret;
1099
}
1100

1101
static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu,
1102
					 struct kvm_device_attr *attr)
1103
{
1104
	u64 val, valid;
1105
	u64 __user *user = (u64 __user *)attr->addr;
1106
	struct kvm *kvm = vcpu->kvm;
1107

1108
	switch (attr->attr) {
1109
	case CPUCFG_KVM_FEATURE:
1110
		if (get_user(val, user))
1111
			return -EFAULT;
1112

1113
		valid = LOONGARCH_PV_FEAT_MASK;
1114
		if (val & ~valid)
1115
			return -EINVAL;
1116

1117
		/* All vCPUs need set the same PV features */
1118
		if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED)
1119
				&& ((kvm->arch.pv_features & valid) != val))
1120
			return -EINVAL;
1121
		kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED;
1122
		return 0;
1123
	default:
1124
		return -ENXIO;
1125
	}
1126
}
1127

1128
static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu,
1129
					 struct kvm_device_attr *attr)
1130
{
1131
	int idx, ret = 0;
1132
	u64 gpa, __user *user = (u64 __user *)attr->addr;
1133
	struct kvm *kvm = vcpu->kvm;
1134

1135
	if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
1136
			|| attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
1137
		return -ENXIO;
1138

1139
	if (get_user(gpa, user))
1140
		return -EFAULT;
1141

1142
	if (gpa & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
1143
		return -EINVAL;
1144

1145
	if (!(gpa & KVM_STEAL_PHYS_VALID)) {
1146
		vcpu->arch.st.guest_addr = gpa;
1147
		return 0;
1148
	}
1149

1150
	/* Check the address is in a valid memslot */
1151
	idx = srcu_read_lock(&kvm->srcu);
1152
	if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT)))
1153
		ret = -EINVAL;
1154
	srcu_read_unlock(&kvm->srcu, idx);
1155

1156
	if (!ret) {
1157
		vcpu->arch.st.guest_addr = gpa;
1158
		vcpu->arch.st.last_steal = current->sched_info.run_delay;
1159
		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
1160
	}
1161

1162
	return ret;
1163
}
1164

1165
static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu,
1166
				       struct kvm_device_attr *attr)
1167
{
1168
	int ret = -ENXIO;
1169

1170
	switch (attr->group) {
1171
	case KVM_LOONGARCH_VCPU_CPUCFG:
1172
		ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr);
1173
		break;
1174
	case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
1175
		ret = kvm_loongarch_pvtime_set_attr(vcpu, attr);
1176
		break;
1177
	default:
1178
		break;
1179
	}
1180

1181
	return ret;
1182
}
1183

1184
long kvm_arch_vcpu_ioctl(struct file *filp,
1185
			 unsigned int ioctl, unsigned long arg)
1186
{
1187
	long r;
1188
	struct kvm_device_attr attr;
1189
	void __user *argp = (void __user *)arg;
1190
	struct kvm_vcpu *vcpu = filp->private_data;
1191

1192
	/*
1193
	 * Only software CSR should be modified
1194
	 *
1195
	 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair
1196
	 * should be used. Since CSR registers owns by this vcpu, if switch
1197
	 * to other vcpus, other vcpus need reload CSR registers.
1198
	 *
1199
	 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should
1200
	 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check
1201
	 * aux_inuse flag and reload CSR registers form software.
1202
	 */
1203

1204
	switch (ioctl) {
1205
	case KVM_SET_ONE_REG:
1206
	case KVM_GET_ONE_REG: {
1207
		struct kvm_one_reg reg;
1208

1209
		r = -EFAULT;
1210
		if (copy_from_user(&reg, argp, sizeof(reg)))
1211
			break;
1212
		if (ioctl == KVM_SET_ONE_REG) {
1213
			r = kvm_set_reg(vcpu, &reg);
1214
			vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE;
1215
		} else
1216
			r = kvm_get_reg(vcpu, &reg);
1217
		break;
1218
	}
1219
	case KVM_ENABLE_CAP: {
1220
		struct kvm_enable_cap cap;
1221

1222
		r = -EFAULT;
1223
		if (copy_from_user(&cap, argp, sizeof(cap)))
1224
			break;
1225
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1226
		break;
1227
	}
1228
	case KVM_HAS_DEVICE_ATTR: {
1229
		r = -EFAULT;
1230
		if (copy_from_user(&attr, argp, sizeof(attr)))
1231
			break;
1232
		r = kvm_loongarch_vcpu_has_attr(vcpu, &attr);
1233
		break;
1234
	}
1235
	case KVM_GET_DEVICE_ATTR: {
1236
		r = -EFAULT;
1237
		if (copy_from_user(&attr, argp, sizeof(attr)))
1238
			break;
1239
		r = kvm_loongarch_vcpu_get_attr(vcpu, &attr);
1240
		break;
1241
	}
1242
	case KVM_SET_DEVICE_ATTR: {
1243
		r = -EFAULT;
1244
		if (copy_from_user(&attr, argp, sizeof(attr)))
1245
			break;
1246
		r = kvm_loongarch_vcpu_set_attr(vcpu, &attr);
1247
		break;
1248
	}
1249
	default:
1250
		r = -ENOIOCTLCMD;
1251
		break;
1252
	}
1253

1254
	return r;
1255
}
1256

1257
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1258
{
1259
	int i = 0;
1260

1261
	fpu->fcc = vcpu->arch.fpu.fcc;
1262
	fpu->fcsr = vcpu->arch.fpu.fcsr;
1263
	for (i = 0; i < NUM_FPU_REGS; i++)
1264
		memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64);
1265

1266
	return 0;
1267
}
1268

1269
int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1270
{
1271
	int i = 0;
1272

1273
	vcpu->arch.fpu.fcc = fpu->fcc;
1274
	vcpu->arch.fpu.fcsr = fpu->fcsr;
1275
	for (i = 0; i < NUM_FPU_REGS; i++)
1276
		memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64);
1277

1278
	return 0;
1279
}
1280

1281
#ifdef CONFIG_CPU_HAS_LBT
1282
int kvm_own_lbt(struct kvm_vcpu *vcpu)
1283
{
1284
	if (!kvm_guest_has_lbt(&vcpu->arch))
1285
		return -EINVAL;
1286

1287
	preempt_disable();
1288
	if (!(vcpu->arch.aux_inuse & KVM_LARCH_LBT)) {
1289
		set_csr_euen(CSR_EUEN_LBTEN);
1290
		_restore_lbt(&vcpu->arch.lbt);
1291
		vcpu->arch.aux_inuse |= KVM_LARCH_LBT;
1292
	}
1293
	preempt_enable();
1294

1295
	return 0;
1296
}
1297

1298
static void kvm_lose_lbt(struct kvm_vcpu *vcpu)
1299
{
1300
	preempt_disable();
1301
	if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) {
1302
		_save_lbt(&vcpu->arch.lbt);
1303
		clear_csr_euen(CSR_EUEN_LBTEN);
1304
		vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT;
1305
	}
1306
	preempt_enable();
1307
}
1308

1309
static void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr)
1310
{
1311
	/*
1312
	 * If TM is enabled, top register save/restore will
1313
	 * cause lbt exception, here enable lbt in advance
1314
	 */
1315
	if (fcsr & FPU_CSR_TM)
1316
		kvm_own_lbt(vcpu);
1317
}
1318

1319
static void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu)
1320
{
1321
	if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1322
		if (vcpu->arch.aux_inuse & KVM_LARCH_LBT)
1323
			return;
1324
		kvm_check_fcsr(vcpu, read_fcsr(LOONGARCH_FCSR0));
1325
	}
1326
}
1327
#else
1328
static inline void kvm_lose_lbt(struct kvm_vcpu *vcpu) { }
1329
static inline void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) { }
1330
static inline void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) { }
1331
#endif
1332

1333
/* Enable FPU and restore context */
1334
void kvm_own_fpu(struct kvm_vcpu *vcpu)
1335
{
1336
	preempt_disable();
1337

1338
	/*
1339
	 * Enable FPU for guest
1340
	 * Set FR and FRE according to guest context
1341
	 */
1342
	kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1343
	set_csr_euen(CSR_EUEN_FPEN);
1344

1345
	kvm_restore_fpu(&vcpu->arch.fpu);
1346
	vcpu->arch.aux_inuse |= KVM_LARCH_FPU;
1347
	trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU);
1348

1349
	preempt_enable();
1350
}
1351

1352
#ifdef CONFIG_CPU_HAS_LSX
1353
/* Enable LSX and restore context */
1354
int kvm_own_lsx(struct kvm_vcpu *vcpu)
1355
{
1356
	if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch))
1357
		return -EINVAL;
1358

1359
	preempt_disable();
1360

1361
	/* Enable LSX for guest */
1362
	kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1363
	set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN);
1364
	switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1365
	case KVM_LARCH_FPU:
1366
		/*
1367
		 * Guest FPU state already loaded,
1368
		 * only restore upper LSX state
1369
		 */
1370
		_restore_lsx_upper(&vcpu->arch.fpu);
1371
		break;
1372
	default:
1373
		/* Neither FP or LSX already active,
1374
		 * restore full LSX state
1375
		 */
1376
		kvm_restore_lsx(&vcpu->arch.fpu);
1377
		break;
1378
	}
1379

1380
	trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX);
1381
	vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU;
1382
	preempt_enable();
1383

1384
	return 0;
1385
}
1386
#endif
1387

1388
#ifdef CONFIG_CPU_HAS_LASX
1389
/* Enable LASX and restore context */
1390
int kvm_own_lasx(struct kvm_vcpu *vcpu)
1391
{
1392
	if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch))
1393
		return -EINVAL;
1394

1395
	preempt_disable();
1396

1397
	kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1398
	set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
1399
	switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) {
1400
	case KVM_LARCH_LSX:
1401
	case KVM_LARCH_LSX | KVM_LARCH_FPU:
1402
		/* Guest LSX state already loaded, only restore upper LASX state */
1403
		_restore_lasx_upper(&vcpu->arch.fpu);
1404
		break;
1405
	case KVM_LARCH_FPU:
1406
		/* Guest FP state already loaded, only restore upper LSX & LASX state */
1407
		_restore_lsx_upper(&vcpu->arch.fpu);
1408
		_restore_lasx_upper(&vcpu->arch.fpu);
1409
		break;
1410
	default:
1411
		/* Neither FP or LSX already active, restore full LASX state */
1412
		kvm_restore_lasx(&vcpu->arch.fpu);
1413
		break;
1414
	}
1415

1416
	trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX);
1417
	vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU;
1418
	preempt_enable();
1419

1420
	return 0;
1421
}
1422
#endif
1423

1424
/* Save context and disable FPU */
1425
void kvm_lose_fpu(struct kvm_vcpu *vcpu)
1426
{
1427
	preempt_disable();
1428

1429
	kvm_check_fcsr_alive(vcpu);
1430
	if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) {
1431
		kvm_save_lasx(&vcpu->arch.fpu);
1432
		vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX);
1433
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX);
1434

1435
		/* Disable LASX & LSX & FPU */
1436
		clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
1437
	} else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
1438
		kvm_save_lsx(&vcpu->arch.fpu);
1439
		vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU);
1440
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX);
1441

1442
		/* Disable LSX & FPU */
1443
		clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN);
1444
	} else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1445
		kvm_save_fpu(&vcpu->arch.fpu);
1446
		vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU;
1447
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);
1448

1449
		/* Disable FPU */
1450
		clear_csr_euen(CSR_EUEN_FPEN);
1451
	}
1452
	kvm_lose_lbt(vcpu);
1453

1454
	preempt_enable();
1455
}
1456

1457
int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1458
{
1459
	int intr = (int)irq->irq;
1460

1461
	if (intr > 0)
1462
		kvm_queue_irq(vcpu, intr);
1463
	else if (intr < 0)
1464
		kvm_dequeue_irq(vcpu, -intr);
1465
	else {
1466
		kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq);
1467
		return -EINVAL;
1468
	}
1469

1470
	kvm_vcpu_kick(vcpu);
1471

1472
	return 0;
1473
}
1474

1475
long kvm_arch_vcpu_async_ioctl(struct file *filp,
1476
			       unsigned int ioctl, unsigned long arg)
1477
{
1478
	void __user *argp = (void __user *)arg;
1479
	struct kvm_vcpu *vcpu = filp->private_data;
1480

1481
	if (ioctl == KVM_INTERRUPT) {
1482
		struct kvm_interrupt irq;
1483

1484
		if (copy_from_user(&irq, argp, sizeof(irq)))
1485
			return -EFAULT;
1486

1487
		kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq);
1488

1489
		return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1490
	}
1491

1492
	return -ENOIOCTLCMD;
1493
}
1494

1495
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
1496
{
1497
	return 0;
1498
}
1499

1500
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
1501
{
1502
	unsigned long timer_hz;
1503
	struct loongarch_csrs *csr;
1504

1505
	vcpu->arch.vpid = 0;
1506
	vcpu->arch.flush_gpa = INVALID_GPA;
1507

1508
	hrtimer_setup(&vcpu->arch.swtimer, kvm_swtimer_wakeup, CLOCK_MONOTONIC,
1509
		      HRTIMER_MODE_ABS_PINNED_HARD);
1510

1511
	/* Get GPA (=HVA) of PGD for kvm hypervisor */
1512
	vcpu->arch.kvm_pgd = __pa(vcpu->kvm->arch.pgd);
1513

1514
	/*
1515
	 * Get PGD for primary mmu, virtual address is used since there is
1516
	 * memory access after loading from CSR_PGD in tlb exception fast path.
1517
	 */
1518
	vcpu->arch.host_pgd = (unsigned long)vcpu->kvm->mm->pgd;
1519

1520
	vcpu->arch.handle_exit = kvm_handle_exit;
1521
	vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry;
1522
	vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL);
1523
	if (!vcpu->arch.csr)
1524
		return -ENOMEM;
1525

1526
	/*
1527
	 * All kvm exceptions share one exception entry, and host <-> guest
1528
	 * switch also switch ECFG.VS field, keep host ECFG.VS info here.
1529
	 */
1530
	vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS);
1531

1532
	/* Init */
1533
	vcpu->arch.last_sched_cpu = -1;
1534

1535
	/* Init ipi_state lock */
1536
	spin_lock_init(&vcpu->arch.ipi_state.lock);
1537

1538
	/*
1539
	 * Initialize guest register state to valid architectural reset state.
1540
	 */
1541
	timer_hz = calc_const_freq();
1542
	kvm_init_timer(vcpu, timer_hz);
1543

1544
	/* Set Initialize mode for guest */
1545
	csr = vcpu->arch.csr;
1546
	kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA);
1547

1548
	/* Set cpuid */
1549
	kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id);
1550
	kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID);
1551

1552
	/* Start with no pending virtual guest interrupts */
1553
	csr->csrs[LOONGARCH_CSR_GINTC] = 0;
1554

1555
	return 0;
1556
}
1557

1558
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1559
{
1560
}
1561

1562
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1563
{
1564
	int cpu;
1565
	struct kvm_context *context;
1566

1567
	hrtimer_cancel(&vcpu->arch.swtimer);
1568
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
1569
	kvm_drop_cpuid(vcpu);
1570
	kfree(vcpu->arch.csr);
1571

1572
	/*
1573
	 * If the vCPU is freed and reused as another vCPU, we don't want the
1574
	 * matching pointer wrongly hanging around in last_vcpu.
1575
	 */
1576
	for_each_possible_cpu(cpu) {
1577
		context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
1578
		if (context->last_vcpu == vcpu)
1579
			context->last_vcpu = NULL;
1580
	}
1581
}
1582

1583
static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1584
{
1585
	bool migrated;
1586
	struct kvm_context *context;
1587
	struct loongarch_csrs *csr = vcpu->arch.csr;
1588

1589
	/*
1590
	 * Have we migrated to a different CPU?
1591
	 * If so, any old guest TLB state may be stale.
1592
	 */
1593
	migrated = (vcpu->arch.last_sched_cpu != cpu);
1594

1595
	/*
1596
	 * Was this the last vCPU to run on this CPU?
1597
	 * If not, any old guest state from this vCPU will have been clobbered.
1598
	 */
1599
	context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
1600
	if (migrated || (context->last_vcpu != vcpu))
1601
		vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE;
1602
	context->last_vcpu = vcpu;
1603

1604
	/* Restore timer state regardless */
1605
	kvm_restore_timer(vcpu);
1606
	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
1607

1608
	/* Restore hardware PMU CSRs */
1609
	kvm_restore_pmu(vcpu);
1610

1611
	/* Don't bother restoring registers multiple times unless necessary */
1612
	if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE)
1613
		return 0;
1614

1615
	write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset);
1616

1617
	/* Restore guest CSR registers */
1618
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD);
1619
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD);
1620
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN);
1621
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC);
1622
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG);
1623
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA);
1624
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV);
1625
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI);
1626
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY);
1627
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX);
1628
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI);
1629
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0);
1630
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1);
1631
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID);
1632
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL);
1633
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH);
1634
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0);
1635
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1);
1636
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE);
1637
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG);
1638
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID);
1639
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0);
1640
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1);
1641
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2);
1642
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3);
1643
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4);
1644
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5);
1645
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6);
1646
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7);
1647
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID);
1648
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC);
1649
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY);
1650
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV);
1651
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA);
1652
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE);
1653
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0);
1654
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1);
1655
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI);
1656
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD);
1657
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0);
1658
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1);
1659
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
1660
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
1661
	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL);
1662

1663
	/* Restore Root.GINTC from unused Guest.GINTC register */
1664
	write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]);
1665

1666
	/*
1667
	 * We should clear linked load bit to break interrupted atomics. This
1668
	 * prevents a SC on the next vCPU from succeeding by matching a LL on
1669
	 * the previous vCPU.
1670
	 */
1671
	if (vcpu->kvm->created_vcpus > 1)
1672
		set_gcsr_llbctl(CSR_LLBCTL_WCLLB);
1673

1674
	vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE;
1675

1676
	return 0;
1677
}
1678

1679
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1680
{
1681
	unsigned long flags;
1682

1683
	local_irq_save(flags);
1684
	/* Restore guest state to registers */
1685
	_kvm_vcpu_load(vcpu, cpu);
1686
	local_irq_restore(flags);
1687
}
1688

1689
static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
1690
{
1691
	struct loongarch_csrs *csr = vcpu->arch.csr;
1692

1693
	kvm_lose_fpu(vcpu);
1694

1695
	/*
1696
	 * Update CSR state from hardware if software CSR state is stale,
1697
	 * most CSR registers are kept unchanged during process context
1698
	 * switch except CSR registers like remaining timer tick value and
1699
	 * injected interrupt state.
1700
	 */
1701
	if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST)
1702
		goto out;
1703

1704
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD);
1705
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD);
1706
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN);
1707
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC);
1708
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG);
1709
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA);
1710
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV);
1711
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI);
1712
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY);
1713
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX);
1714
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI);
1715
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0);
1716
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1);
1717
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID);
1718
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL);
1719
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH);
1720
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0);
1721
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1);
1722
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE);
1723
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG);
1724
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID);
1725
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1);
1726
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2);
1727
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3);
1728
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0);
1729
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1);
1730
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2);
1731
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3);
1732
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4);
1733
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5);
1734
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6);
1735
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7);
1736
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID);
1737
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC);
1738
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL);
1739
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY);
1740
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV);
1741
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA);
1742
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE);
1743
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0);
1744
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1);
1745
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI);
1746
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD);
1747
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0);
1748
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1);
1749
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
1750
	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
1751

1752
	vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST;
1753

1754
out:
1755
	kvm_save_timer(vcpu);
1756
	/* Save Root.GINTC into unused Guest.GINTC register */
1757
	csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc();
1758

1759
	return 0;
1760
}
1761

1762
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1763
{
1764
	int cpu;
1765
	unsigned long flags;
1766

1767
	local_irq_save(flags);
1768
	cpu = smp_processor_id();
1769
	vcpu->arch.last_sched_cpu = cpu;
1770

1771
	/* Save guest state in registers */
1772
	_kvm_vcpu_put(vcpu, cpu);
1773
	local_irq_restore(flags);
1774
}
1775

1776
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1777
{
1778
	int r = -EINTR;
1779
	struct kvm_run *run = vcpu->run;
1780

1781
	if (vcpu->mmio_needed) {
1782
		if (!vcpu->mmio_is_write)
1783
			kvm_complete_mmio_read(vcpu, run);
1784
		vcpu->mmio_needed = 0;
1785
	}
1786

1787
	switch (run->exit_reason) {
1788
	case KVM_EXIT_HYPERCALL:
1789
		kvm_complete_user_service(vcpu, run);
1790
		break;
1791
	case KVM_EXIT_LOONGARCH_IOCSR:
1792
		if (!run->iocsr_io.is_write)
1793
			kvm_complete_iocsr_read(vcpu, run);
1794
		break;
1795
	}
1796

1797
	if (!vcpu->wants_to_run)
1798
		return r;
1799

1800
	/* Clear exit_reason */
1801
	run->exit_reason = KVM_EXIT_UNKNOWN;
1802
	lose_fpu(1);
1803
	vcpu_load(vcpu);
1804
	kvm_sigset_activate(vcpu);
1805
	r = kvm_pre_enter_guest(vcpu);
1806
	if (r != RESUME_GUEST)
1807
		goto out;
1808

1809
	guest_timing_enter_irqoff();
1810
	guest_state_enter_irqoff();
1811
	trace_kvm_enter(vcpu);
1812
	r = kvm_loongarch_ops->enter_guest(run, vcpu);
1813

1814
	trace_kvm_out(vcpu);
1815
	/*
1816
	 * Guest exit is already recorded at kvm_handle_exit()
1817
	 * return value must not be RESUME_GUEST
1818
	 */
1819
	local_irq_enable();
1820
out:
1821
	kvm_sigset_deactivate(vcpu);
1822
	vcpu_put(vcpu);
1823

1824
	return r;
1825
}
1826

1827