1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4
 */
5

6
#include <linux/err.h>
7
#include <linux/errno.h>
8
#include <linux/kvm_host.h>
9
#include <linux/module.h>
10
#include <linux/preempt.h>
11
#include <linux/vmalloc.h>
12
#include <trace/events/kvm.h>
13
#include <asm/fpu.h>
14
#include <asm/inst.h>
15
#include <asm/loongarch.h>
16
#include <asm/mmzone.h>
17
#include <asm/numa.h>
18
#include <asm/time.h>
19
#include <asm/tlb.h>
20
#include <asm/kvm_csr.h>
21
#include <asm/kvm_vcpu.h>
22
#include "trace.h"
23

24
static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst)
25
{
26
	int rd, rj;
27
	unsigned int index, ret;
28

29
	if (inst.reg2_format.opcode != cpucfg_op)
30
		return EMULATE_FAIL;
31

32
	rd = inst.reg2_format.rd;
33
	rj = inst.reg2_format.rj;
34
	++vcpu->stat.cpucfg_exits;
35
	index = vcpu->arch.gprs[rj];
36

37
	/*
38
	 * By LoongArch Reference Manual 2.2.10.5
39
	 * Return value is 0 for undefined CPUCFG index
40
	 *
41
	 * Disable preemption since hw gcsr is accessed
42
	 */
43
	preempt_disable();
44
	switch (index) {
45
	case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
46
		vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index];
47
		break;
48
	case CPUCFG_KVM_SIG:
49
		/* CPUCFG emulation between 0x40000000 -- 0x400000ff */
50
		vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE;
51
		break;
52
	case CPUCFG_KVM_FEATURE:
53
		ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
54
		vcpu->arch.gprs[rd] = ret;
55
		break;
56
	default:
57
		vcpu->arch.gprs[rd] = 0;
58
		break;
59
	}
60
	preempt_enable();
61

62
	return EMULATE_DONE;
63
}
64

65
static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid)
66
{
67
	unsigned long val = 0;
68
	struct loongarch_csrs *csr = vcpu->arch.csr;
69

70
	/*
71
	 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1
72
	 * For undefined CSR id, return value is 0
73
	 */
74
	if (get_gcsr_flag(csrid) & SW_GCSR)
75
		val = kvm_read_sw_gcsr(csr, csrid);
76
	else
77
		pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
78

79
	return val;
80
}
81

82
static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val)
83
{
84
	unsigned long old = 0;
85
	struct loongarch_csrs *csr = vcpu->arch.csr;
86

87
	if (get_gcsr_flag(csrid) & SW_GCSR) {
88
		old = kvm_read_sw_gcsr(csr, csrid);
89
		kvm_write_sw_gcsr(csr, csrid, val);
90
	} else
91
		pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
92

93
	return old;
94
}
95

96
static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
97
				unsigned long csr_mask, unsigned long val)
98
{
99
	unsigned long old = 0;
100
	struct loongarch_csrs *csr = vcpu->arch.csr;
101

102
	if (get_gcsr_flag(csrid) & SW_GCSR) {
103
		old = kvm_read_sw_gcsr(csr, csrid);
104
		val = (old & ~csr_mask) | (val & csr_mask);
105
		kvm_write_sw_gcsr(csr, csrid, val);
106
		old = old & csr_mask;
107
	} else
108
		pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
109

110
	return old;
111
}
112

113
static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst)
114
{
115
	unsigned int rd, rj, csrid;
116
	unsigned long csr_mask, val = 0;
117

118
	/*
119
	 * CSR value mask imm
120
	 * rj = 0 means csrrd
121
	 * rj = 1 means csrwr
122
	 * rj != 0,1 means csrxchg
123
	 */
124
	rd = inst.reg2csr_format.rd;
125
	rj = inst.reg2csr_format.rj;
126
	csrid = inst.reg2csr_format.csr;
127

128
	if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) {
129
		if (kvm_guest_has_pmu(&vcpu->arch)) {
130
			vcpu->arch.pc -= 4;
131
			kvm_make_request(KVM_REQ_PMU, vcpu);
132
			return EMULATE_DONE;
133
		}
134
	}
135

136
	/* Process CSR ops */
137
	switch (rj) {
138
	case 0: /* process csrrd */
139
		val = kvm_emu_read_csr(vcpu, csrid);
140
		vcpu->arch.gprs[rd] = val;
141
		break;
142
	case 1: /* process csrwr */
143
		val = vcpu->arch.gprs[rd];
144
		val = kvm_emu_write_csr(vcpu, csrid, val);
145
		vcpu->arch.gprs[rd] = val;
146
		break;
147
	default: /* process csrxchg */
148
		val = vcpu->arch.gprs[rd];
149
		csr_mask = vcpu->arch.gprs[rj];
150
		val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val);
151
		vcpu->arch.gprs[rd] = val;
152
	}
153

154
	return EMULATE_DONE;
155
}
156

157
int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu)
158
{
159
	int idx, ret;
160
	unsigned long *val;
161
	u32 addr, rd, rj, opcode;
162

163
	/*
164
	 * Each IOCSR with different opcode
165
	 */
166
	rd = inst.reg2_format.rd;
167
	rj = inst.reg2_format.rj;
168
	opcode = inst.reg2_format.opcode;
169
	addr = vcpu->arch.gprs[rj];
170
	run->iocsr_io.phys_addr = addr;
171
	run->iocsr_io.is_write = 0;
172
	val = &vcpu->arch.gprs[rd];
173

174
	/* LoongArch is Little endian */
175
	switch (opcode) {
176
	case iocsrrdb_op:
177
		run->iocsr_io.len = 1;
178
		break;
179
	case iocsrrdh_op:
180
		run->iocsr_io.len = 2;
181
		break;
182
	case iocsrrdw_op:
183
		run->iocsr_io.len = 4;
184
		break;
185
	case iocsrrdd_op:
186
		run->iocsr_io.len = 8;
187
		break;
188
	case iocsrwrb_op:
189
		run->iocsr_io.len = 1;
190
		run->iocsr_io.is_write = 1;
191
		break;
192
	case iocsrwrh_op:
193
		run->iocsr_io.len = 2;
194
		run->iocsr_io.is_write = 1;
195
		break;
196
	case iocsrwrw_op:
197
		run->iocsr_io.len = 4;
198
		run->iocsr_io.is_write = 1;
199
		break;
200
	case iocsrwrd_op:
201
		run->iocsr_io.len = 8;
202
		run->iocsr_io.is_write = 1;
203
		break;
204
	default:
205
		return EMULATE_FAIL;
206
	}
207

208
	if (run->iocsr_io.is_write) {
209
		idx = srcu_read_lock(&vcpu->kvm->srcu);
210
		ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val);
211
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
212
		if (ret == 0)
213
			ret = EMULATE_DONE;
214
		else {
215
			ret = EMULATE_DO_IOCSR;
216
			/* Save data and let user space to write it */
217
			memcpy(run->iocsr_io.data, val, run->iocsr_io.len);
218
		}
219
		trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val);
220
	} else {
221
		vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */
222
		idx = srcu_read_lock(&vcpu->kvm->srcu);
223
		ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr,
224
				      run->iocsr_io.len, run->iocsr_io.data);
225
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
226
		if (ret == 0) {
227
			kvm_complete_iocsr_read(vcpu, run);
228
			ret = EMULATE_DONE;
229
		} else
230
			ret = EMULATE_DO_IOCSR;
231
		trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL);
232
	}
233

234
	return ret;
235
}
236

237
int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
238
{
239
	enum emulation_result er = EMULATE_DONE;
240
	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
241

242
	switch (run->iocsr_io.len) {
243
	case 1:
244
		*gpr = *(s8 *)run->iocsr_io.data;
245
		break;
246
	case 2:
247
		*gpr = *(s16 *)run->iocsr_io.data;
248
		break;
249
	case 4:
250
		*gpr = *(s32 *)run->iocsr_io.data;
251
		break;
252
	case 8:
253
		*gpr = *(s64 *)run->iocsr_io.data;
254
		break;
255
	default:
256
		kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n",
257
				run->iocsr_io.len, vcpu->arch.badv);
258
		er = EMULATE_FAIL;
259
		break;
260
	}
261

262
	return er;
263
}
264

265
int kvm_emu_idle(struct kvm_vcpu *vcpu)
266
{
267
	++vcpu->stat.idle_exits;
268
	trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE);
269

270
	if (!kvm_arch_vcpu_runnable(vcpu))
271
		kvm_vcpu_halt(vcpu);
272

273
	return EMULATE_DONE;
274
}
275

276
static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
277
{
278
	unsigned long curr_pc;
279
	larch_inst inst;
280
	enum emulation_result er = EMULATE_DONE;
281
	struct kvm_run *run = vcpu->run;
282

283
	/* Fetch the instruction */
284
	inst.word = vcpu->arch.badi;
285
	curr_pc = vcpu->arch.pc;
286
	update_pc(&vcpu->arch);
287

288
	trace_kvm_exit_gspr(vcpu, inst.word);
289
	er = EMULATE_FAIL;
290
	switch (((inst.word >> 24) & 0xff)) {
291
	case 0x0: /* CPUCFG GSPR */
292
		trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG);
293
		er = kvm_emu_cpucfg(vcpu, inst);
294
		break;
295
	case 0x4: /* CSR{RD,WR,XCHG} GSPR */
296
		trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR);
297
		er = kvm_handle_csr(vcpu, inst);
298
		break;
299
	case 0x6: /* Cache, Idle and IOCSR GSPR */
300
		switch (((inst.word >> 22) & 0x3ff)) {
301
		case 0x18: /* Cache GSPR */
302
			er = EMULATE_DONE;
303
			trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE);
304
			break;
305
		case 0x19: /* Idle/IOCSR GSPR */
306
			switch (((inst.word >> 15) & 0x1ffff)) {
307
			case 0xc90: /* IOCSR GSPR */
308
				er = kvm_emu_iocsr(inst, run, vcpu);
309
				break;
310
			case 0xc91: /* Idle GSPR */
311
				er = kvm_emu_idle(vcpu);
312
				break;
313
			default:
314
				er = EMULATE_FAIL;
315
				break;
316
			}
317
			break;
318
		default:
319
			er = EMULATE_FAIL;
320
			break;
321
		}
322
		break;
323
	default:
324
		er = EMULATE_FAIL;
325
		break;
326
	}
327

328
	/* Rollback PC only if emulation was unsuccessful */
329
	if (er == EMULATE_FAIL) {
330
		kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n",
331
			curr_pc, __func__, inst.word);
332

333
		kvm_arch_vcpu_dump_regs(vcpu);
334
		vcpu->arch.pc = curr_pc;
335
	}
336

337
	return er;
338
}
339

340
/*
341
 * Trigger GSPR:
342
 * 1) Execute CPUCFG instruction;
343
 * 2) Execute CACOP/IDLE instructions;
344
 * 3) Access to unimplemented CSRs/IOCSRs.
345
 */
346
static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode)
347
{
348
	int ret = RESUME_GUEST;
349
	enum emulation_result er = EMULATE_DONE;
350

351
	er = kvm_trap_handle_gspr(vcpu);
352

353
	if (er == EMULATE_DONE) {
354
		ret = RESUME_GUEST;
355
	} else if (er == EMULATE_DO_MMIO) {
356
		vcpu->run->exit_reason = KVM_EXIT_MMIO;
357
		ret = RESUME_HOST;
358
	} else if (er == EMULATE_DO_IOCSR) {
359
		vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR;
360
		ret = RESUME_HOST;
361
	} else {
362
		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
363
		ret = RESUME_GUEST;
364
	}
365

366
	return ret;
367
}
368

369
int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
370
{
371
	int idx, ret;
372
	unsigned int op8, opcode, rd;
373
	struct kvm_run *run = vcpu->run;
374

375
	run->mmio.phys_addr = vcpu->arch.badv;
376
	vcpu->mmio_needed = 2;	/* signed */
377
	op8 = (inst.word >> 24) & 0xff;
378
	ret = EMULATE_DO_MMIO;
379

380
	switch (op8) {
381
	case 0x24 ... 0x27:	/* ldptr.w/d process */
382
		rd = inst.reg2i14_format.rd;
383
		opcode = inst.reg2i14_format.opcode;
384

385
		switch (opcode) {
386
		case ldptrw_op:
387
			run->mmio.len = 4;
388
			break;
389
		case ldptrd_op:
390
			run->mmio.len = 8;
391
			break;
392
		default:
393
			break;
394
		}
395
		break;
396
	case 0x28 ... 0x2e:	/* ld.b/h/w/d, ld.bu/hu/wu process */
397
		rd = inst.reg2i12_format.rd;
398
		opcode = inst.reg2i12_format.opcode;
399

400
		switch (opcode) {
401
		case ldb_op:
402
			run->mmio.len = 1;
403
			break;
404
		case ldbu_op:
405
			vcpu->mmio_needed = 1;	/* unsigned */
406
			run->mmio.len = 1;
407
			break;
408
		case ldh_op:
409
			run->mmio.len = 2;
410
			break;
411
		case ldhu_op:
412
			vcpu->mmio_needed = 1;	/* unsigned */
413
			run->mmio.len = 2;
414
			break;
415
		case ldw_op:
416
			run->mmio.len = 4;
417
			break;
418
		case ldwu_op:
419
			vcpu->mmio_needed = 1;	/* unsigned */
420
			run->mmio.len = 4;
421
			break;
422
		case ldd_op:
423
			run->mmio.len = 8;
424
			break;
425
		default:
426
			ret = EMULATE_FAIL;
427
			break;
428
		}
429
		break;
430
	case 0x38:	/* ldx.b/h/w/d, ldx.bu/hu/wu process */
431
		rd = inst.reg3_format.rd;
432
		opcode = inst.reg3_format.opcode;
433

434
		switch (opcode) {
435
		case ldxb_op:
436
			run->mmio.len = 1;
437
			break;
438
		case ldxbu_op:
439
			run->mmio.len = 1;
440
			vcpu->mmio_needed = 1;	/* unsigned */
441
			break;
442
		case ldxh_op:
443
			run->mmio.len = 2;
444
			break;
445
		case ldxhu_op:
446
			run->mmio.len = 2;
447
			vcpu->mmio_needed = 1;	/* unsigned */
448
			break;
449
		case ldxw_op:
450
			run->mmio.len = 4;
451
			break;
452
		case ldxwu_op:
453
			run->mmio.len = 4;
454
			vcpu->mmio_needed = 1;	/* unsigned */
455
			break;
456
		case ldxd_op:
457
			run->mmio.len = 8;
458
			break;
459
		default:
460
			ret = EMULATE_FAIL;
461
			break;
462
		}
463
		break;
464
	default:
465
		ret = EMULATE_FAIL;
466
	}
467

468
	if (ret == EMULATE_DO_MMIO) {
469
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL);
470

471
		vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */
472

473
		/*
474
		 * If mmio device such as PCH-PIC is emulated in KVM,
475
		 * it need not return to user space to handle the mmio
476
		 * exception.
477
		 */
478
		idx = srcu_read_lock(&vcpu->kvm->srcu);
479
		ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv,
480
				      run->mmio.len, run->mmio.data);
481
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
482
		if (!ret) {
483
			kvm_complete_mmio_read(vcpu, run);
484
			update_pc(&vcpu->arch);
485
			vcpu->mmio_needed = 0;
486
			return EMULATE_DONE;
487
		}
488

489
		run->mmio.is_write = 0;
490
		vcpu->mmio_is_write = 0;
491
		return EMULATE_DO_MMIO;
492
	}
493

494
	kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
495
			inst.word, vcpu->arch.pc, vcpu->arch.badv);
496
	kvm_arch_vcpu_dump_regs(vcpu);
497
	vcpu->mmio_needed = 0;
498

499
	return ret;
500
}
501

502
int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
503
{
504
	enum emulation_result er = EMULATE_DONE;
505
	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
506

507
	/* Update with new PC */
508
	update_pc(&vcpu->arch);
509
	switch (run->mmio.len) {
510
	case 1:
511
		if (vcpu->mmio_needed == 2)
512
			*gpr = *(s8 *)run->mmio.data;
513
		else
514
			*gpr = *(u8 *)run->mmio.data;
515
		break;
516
	case 2:
517
		if (vcpu->mmio_needed == 2)
518
			*gpr = *(s16 *)run->mmio.data;
519
		else
520
			*gpr = *(u16 *)run->mmio.data;
521
		break;
522
	case 4:
523
		if (vcpu->mmio_needed == 2)
524
			*gpr = *(s32 *)run->mmio.data;
525
		else
526
			*gpr = *(u32 *)run->mmio.data;
527
		break;
528
	case 8:
529
		*gpr = *(s64 *)run->mmio.data;
530
		break;
531
	default:
532
		kvm_err("Bad MMIO length: %d, addr is 0x%lx\n",
533
				run->mmio.len, vcpu->arch.badv);
534
		er = EMULATE_FAIL;
535
		break;
536
	}
537

538
	trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len,
539
			run->mmio.phys_addr, run->mmio.data);
540

541
	return er;
542
}
543

544
int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst)
545
{
546
	int idx, ret;
547
	unsigned int rd, op8, opcode;
548
	unsigned long curr_pc, rd_val = 0;
549
	struct kvm_run *run = vcpu->run;
550
	void *data = run->mmio.data;
551

552
	/*
553
	 * Update PC and hold onto current PC in case there is
554
	 * an error and we want to rollback the PC
555
	 */
556
	curr_pc = vcpu->arch.pc;
557
	update_pc(&vcpu->arch);
558

559
	op8 = (inst.word >> 24) & 0xff;
560
	run->mmio.phys_addr = vcpu->arch.badv;
561
	ret = EMULATE_DO_MMIO;
562
	switch (op8) {
563
	case 0x24 ... 0x27:	/* stptr.w/d process */
564
		rd = inst.reg2i14_format.rd;
565
		opcode = inst.reg2i14_format.opcode;
566

567
		switch (opcode) {
568
		case stptrw_op:
569
			run->mmio.len = 4;
570
			*(unsigned int *)data = vcpu->arch.gprs[rd];
571
			break;
572
		case stptrd_op:
573
			run->mmio.len = 8;
574
			*(unsigned long *)data = vcpu->arch.gprs[rd];
575
			break;
576
		default:
577
			ret = EMULATE_FAIL;
578
			break;
579
		}
580
		break;
581
	case 0x28 ... 0x2e:	/* st.b/h/w/d  process */
582
		rd = inst.reg2i12_format.rd;
583
		opcode = inst.reg2i12_format.opcode;
584
		rd_val = vcpu->arch.gprs[rd];
585

586
		switch (opcode) {
587
		case stb_op:
588
			run->mmio.len = 1;
589
			*(unsigned char *)data = rd_val;
590
			break;
591
		case sth_op:
592
			run->mmio.len = 2;
593
			*(unsigned short *)data = rd_val;
594
			break;
595
		case stw_op:
596
			run->mmio.len = 4;
597
			*(unsigned int *)data = rd_val;
598
			break;
599
		case std_op:
600
			run->mmio.len = 8;
601
			*(unsigned long *)data = rd_val;
602
			break;
603
		default:
604
			ret = EMULATE_FAIL;
605
			break;
606
		}
607
		break;
608
	case 0x38:	/* stx.b/h/w/d process */
609
		rd = inst.reg3_format.rd;
610
		opcode = inst.reg3_format.opcode;
611

612
		switch (opcode) {
613
		case stxb_op:
614
			run->mmio.len = 1;
615
			*(unsigned char *)data = vcpu->arch.gprs[rd];
616
			break;
617
		case stxh_op:
618
			run->mmio.len = 2;
619
			*(unsigned short *)data = vcpu->arch.gprs[rd];
620
			break;
621
		case stxw_op:
622
			run->mmio.len = 4;
623
			*(unsigned int *)data = vcpu->arch.gprs[rd];
624
			break;
625
		case stxd_op:
626
			run->mmio.len = 8;
627
			*(unsigned long *)data = vcpu->arch.gprs[rd];
628
			break;
629
		default:
630
			ret = EMULATE_FAIL;
631
			break;
632
		}
633
		break;
634
	default:
635
		ret = EMULATE_FAIL;
636
	}
637

638
	if (ret == EMULATE_DO_MMIO) {
639
		trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data);
640

641
		/*
642
		 * If mmio device such as PCH-PIC is emulated in KVM,
643
		 * it need not return to user space to handle the mmio
644
		 * exception.
645
		 */
646
		idx = srcu_read_lock(&vcpu->kvm->srcu);
647
		ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data);
648
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
649
		if (!ret)
650
			return EMULATE_DONE;
651

652
		run->mmio.is_write = 1;
653
		vcpu->mmio_needed = 1;
654
		vcpu->mmio_is_write = 1;
655
		return EMULATE_DO_MMIO;
656
	}
657

658
	vcpu->arch.pc = curr_pc;
659
	kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
660
			inst.word, vcpu->arch.pc, vcpu->arch.badv);
661
	kvm_arch_vcpu_dump_regs(vcpu);
662
	/* Rollback PC if emulation was unsuccessful */
663

664
	return ret;
665
}
666

667
static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode)
668
{
669
	int ret;
670
	larch_inst inst;
671
	enum emulation_result er = EMULATE_DONE;
672
	struct kvm_run *run = vcpu->run;
673
	unsigned long badv = vcpu->arch.badv;
674

675
	/* Inject ADE exception if exceed max GPA size */
676
	if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) {
677
		kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
678
		return RESUME_GUEST;
679
	}
680

681
	ret = kvm_handle_mm_fault(vcpu, badv, write, ecode);
682
	if (ret) {
683
		/* Treat as MMIO */
684
		inst.word = vcpu->arch.badi;
685
		if (write) {
686
			er = kvm_emu_mmio_write(vcpu, inst);
687
		} else {
688
			/* A code fetch fault doesn't count as an MMIO */
689
			if (kvm_is_ifetch_fault(&vcpu->arch)) {
690
				kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF);
691
				return RESUME_GUEST;
692
			}
693

694
			er = kvm_emu_mmio_read(vcpu, inst);
695
		}
696
	}
697

698
	if (er == EMULATE_DONE) {
699
		ret = RESUME_GUEST;
700
	} else if (er == EMULATE_DO_MMIO) {
701
		run->exit_reason = KVM_EXIT_MMIO;
702
		ret = RESUME_HOST;
703
	} else {
704
		kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
705
		ret = RESUME_GUEST;
706
	}
707

708
	return ret;
709
}
710

711
static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode)
712
{
713
	return kvm_handle_rdwr_fault(vcpu, false, ecode);
714
}
715

716
static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode)
717
{
718
	return kvm_handle_rdwr_fault(vcpu, true, ecode);
719
}
720

721
int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run)
722
{
723
	update_pc(&vcpu->arch);
724
	kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret);
725

726
	return 0;
727
}
728

729
/**
730
 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host
731
 * @vcpu:	Virtual CPU context.
732
 * @ecode:	Exception code.
733
 *
734
 * Handle when the guest attempts to use fpu which hasn't been allowed
735
 * by the root context.
736
 */
737
static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode)
738
{
739
	struct kvm_run *run = vcpu->run;
740

741
	if (!kvm_guest_has_fpu(&vcpu->arch)) {
742
		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
743
		return RESUME_GUEST;
744
	}
745

746
	/*
747
	 * If guest FPU not present, the FPU operation should have been
748
	 * treated as a reserved instruction!
749
	 * If FPU already in use, we shouldn't get this at all.
750
	 */
751
	if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
752
		kvm_err("%s internal error\n", __func__);
753
		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
754
		return RESUME_HOST;
755
	}
756

757
	kvm_own_fpu(vcpu);
758

759
	return RESUME_GUEST;
760
}
761

762
static long kvm_save_notify(struct kvm_vcpu *vcpu)
763
{
764
	unsigned long id, data;
765

766
	id   = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
767
	data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
768
	switch (id) {
769
	case BIT(KVM_FEATURE_STEAL_TIME):
770
		if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
771
			return KVM_HCALL_INVALID_PARAMETER;
772

773
		vcpu->arch.st.guest_addr = data;
774
		if (!(data & KVM_STEAL_PHYS_VALID))
775
			return 0;
776

777
		vcpu->arch.st.last_steal = current->sched_info.run_delay;
778
		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
779
		return 0;
780
	default:
781
		return KVM_HCALL_INVALID_CODE;
782
	}
783
}
784

785
/*
786
 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
787
 * @vcpu:      Virtual CPU context.
788
 * @ecode:	Exception code.
789
 *
790
 * Handle when the guest attempts to use LSX when it is disabled in the root
791
 * context.
792
 */
793
static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode)
794
{
795
	if (kvm_own_lsx(vcpu))
796
		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
797

798
	return RESUME_GUEST;
799
}
800

801
/*
802
 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
803
 * @vcpu:	Virtual CPU context.
804
 * @ecode:	Exception code.
805
 *
806
 * Handle when the guest attempts to use LASX when it is disabled in the root
807
 * context.
808
 */
809
static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode)
810
{
811
	if (kvm_own_lasx(vcpu))
812
		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
813

814
	return RESUME_GUEST;
815
}
816

817
static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode)
818
{
819
	if (kvm_own_lbt(vcpu))
820
		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
821

822
	return RESUME_GUEST;
823
}
824

825
static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
826
{
827
	unsigned int min, cpu;
828
	struct kvm_vcpu *dest;
829
	DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = {
830
		kvm_read_reg(vcpu, LOONGARCH_GPR_A1),
831
		kvm_read_reg(vcpu, LOONGARCH_GPR_A2)
832
	};
833

834
	min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
835
	for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) {
836
		dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
837
		if (!dest)
838
			continue;
839

840
		/* Send SWI0 to dest vcpu to emulate IPI interrupt */
841
		kvm_queue_irq(dest, INT_SWI0);
842
		kvm_vcpu_kick(dest);
843
	}
844
}
845

846
/*
847
 * Hypercall emulation always return to guest, Caller should check retval.
848
 */
849
static void kvm_handle_service(struct kvm_vcpu *vcpu)
850
{
851
	long ret = KVM_HCALL_INVALID_CODE;
852
	unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
853

854
	switch (func) {
855
	case KVM_HCALL_FUNC_IPI:
856
		if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) {
857
			kvm_send_pv_ipi(vcpu);
858
			ret = KVM_HCALL_SUCCESS;
859
		}
860
		break;
861
	case KVM_HCALL_FUNC_NOTIFY:
862
		if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME))
863
			ret = kvm_save_notify(vcpu);
864
		break;
865
	default:
866
		break;
867
	}
868

869
	kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret);
870
}
871

872
static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode)
873
{
874
	int ret;
875
	larch_inst inst;
876
	unsigned int code;
877

878
	inst.word = vcpu->arch.badi;
879
	code = inst.reg0i15_format.immediate;
880
	ret = RESUME_GUEST;
881

882
	switch (code) {
883
	case KVM_HCALL_SERVICE:
884
		vcpu->stat.hypercall_exits++;
885
		kvm_handle_service(vcpu);
886
		break;
887
	case KVM_HCALL_USER_SERVICE:
888
		if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) {
889
			kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
890
			break;
891
		}
892

893
		vcpu->stat.hypercall_exits++;
894
		vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
895
		vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE;
896
		vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
897
		vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
898
		vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
899
		vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
900
		vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4);
901
		vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5);
902
		vcpu->run->hypercall.flags = 0;
903
		/*
904
		 * Set invalid return value by default, let user-mode VMM modify it.
905
		 */
906
		vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE;
907
		ret = RESUME_HOST;
908
		break;
909
	case KVM_HCALL_SWDBG:
910
		/* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */
911
		if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) {
912
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
913
			ret = RESUME_HOST;
914
			break;
915
		}
916
		fallthrough;
917
	default:
918
		/* Treat it as noop intruction, only set return value */
919
		kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
920
		break;
921
	}
922

923
	if (ret == RESUME_GUEST)
924
		update_pc(&vcpu->arch);
925

926
	return ret;
927
}
928

929
/*
930
 * LoongArch KVM callback handling for unimplemented guest exiting
931
 */
932
static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode)
933
{
934
	unsigned int inst;
935
	unsigned long badv;
936

937
	/* Fetch the instruction */
938
	inst = vcpu->arch.badi;
939
	badv = vcpu->arch.badv;
940
	kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n",
941
			ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat());
942
	kvm_arch_vcpu_dump_regs(vcpu);
943
	kvm_queue_exception(vcpu, EXCCODE_INE, 0);
944

945
	return RESUME_GUEST;
946
}
947

948
static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
949
	[0 ... EXCCODE_INT_START - 1]	= kvm_fault_ni,
950
	[EXCCODE_TLBI]			= kvm_handle_read_fault,
951
	[EXCCODE_TLBL]			= kvm_handle_read_fault,
952
	[EXCCODE_TLBS]			= kvm_handle_write_fault,
953
	[EXCCODE_TLBM]			= kvm_handle_write_fault,
954
	[EXCCODE_FPDIS]			= kvm_handle_fpu_disabled,
955
	[EXCCODE_LSXDIS]		= kvm_handle_lsx_disabled,
956
	[EXCCODE_LASXDIS]		= kvm_handle_lasx_disabled,
957
	[EXCCODE_BTDIS]			= kvm_handle_lbt_disabled,
958
	[EXCCODE_GSPR]			= kvm_handle_gspr,
959
	[EXCCODE_HVC]			= kvm_handle_hypercall,
960
};
961

962
int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault)
963
{
964
	return kvm_fault_tables[fault](vcpu, fault);
965
}
966

967