1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4
 * Author: Christoffer Dall <[email protected]>
5
 */
6

7
#include <linux/bug.h>
8
#include <linux/cpu_pm.h>
9
#include <linux/entry-kvm.h>
10
#include <linux/errno.h>
11
#include <linux/err.h>
12
#include <linux/kvm_host.h>
13
#include <linux/list.h>
14
#include <linux/module.h>
15
#include <linux/vmalloc.h>
16
#include <linux/fs.h>
17
#include <linux/mman.h>
18
#include <linux/sched.h>
19
#include <linux/kvm.h>
20
#include <linux/kvm_irqfd.h>
21
#include <linux/irqbypass.h>
22
#include <linux/sched/stat.h>
23
#include <linux/psci.h>
24
#include <trace/events/kvm.h>
25

26
#define CREATE_TRACE_POINTS
27
#include "trace_arm.h"
28

29
#include <linux/uaccess.h>
30
#include <asm/ptrace.h>
31
#include <asm/mman.h>
32
#include <asm/tlbflush.h>
33
#include <asm/cacheflush.h>
34
#include <asm/cpufeature.h>
35
#include <asm/virt.h>
36
#include <asm/kvm_arm.h>
37
#include <asm/kvm_asm.h>
38
#include <asm/kvm_emulate.h>
39
#include <asm/kvm_mmu.h>
40
#include <asm/kvm_nested.h>
41
#include <asm/kvm_pkvm.h>
42
#include <asm/kvm_ptrauth.h>
43
#include <asm/sections.h>
44

45
#include <kvm/arm_hypercalls.h>
46
#include <kvm/arm_pmu.h>
47
#include <kvm/arm_psci.h>
48

49
#include "sys_regs.h"
50

51
static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
52

53
enum kvm_wfx_trap_policy {
54
	KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
55
	KVM_WFX_NOTRAP,
56
	KVM_WFX_TRAP,
57
};
58

59
static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
60
static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
61

62
DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
63

64
DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
65
DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
66

67
DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
68

69
static bool vgic_present, kvm_arm_initialised;
70

71
static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
72

73
bool is_kvm_arm_initialised(void)
74
{
75
	return kvm_arm_initialised;
76
}
77

78
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
79
{
80
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
81
}
82

83
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
84
			    struct kvm_enable_cap *cap)
85
{
86
	int r = -EINVAL;
87

88
	if (cap->flags)
89
		return -EINVAL;
90

91
	if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
92
		return -EINVAL;
93

94
	switch (cap->cap) {
95
	case KVM_CAP_ARM_NISV_TO_USER:
96
		r = 0;
97
		set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
98
			&kvm->arch.flags);
99
		break;
100
	case KVM_CAP_ARM_MTE:
101
		mutex_lock(&kvm->lock);
102
		if (system_supports_mte() && !kvm->created_vcpus) {
103
			r = 0;
104
			set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
105
		}
106
		mutex_unlock(&kvm->lock);
107
		break;
108
	case KVM_CAP_ARM_SYSTEM_SUSPEND:
109
		r = 0;
110
		set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
111
		break;
112
	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
113
		mutex_lock(&kvm->slots_lock);
114
		/*
115
		 * To keep things simple, allow changing the chunk
116
		 * size only when no memory slots have been created.
117
		 */
118
		if (kvm_are_all_memslots_empty(kvm)) {
119
			u64 new_cap = cap->args[0];
120

121
			if (!new_cap || kvm_is_block_size_supported(new_cap)) {
122
				r = 0;
123
				kvm->arch.mmu.split_page_chunk_size = new_cap;
124
			}
125
		}
126
		mutex_unlock(&kvm->slots_lock);
127
		break;
128
	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
129
		mutex_lock(&kvm->lock);
130
		if (!kvm->created_vcpus) {
131
			r = 0;
132
			set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
133
		}
134
		mutex_unlock(&kvm->lock);
135
		break;
136
	default:
137
		break;
138
	}
139

140
	return r;
141
}
142

143
static int kvm_arm_default_max_vcpus(void)
144
{
145
	return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
146
}
147

148
/**
149
 * kvm_arch_init_vm - initializes a VM data structure
150
 * @kvm:	pointer to the KVM struct
151
 * @type:	kvm device type
152
 */
153
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
154
{
155
	int ret;
156

157
	mutex_init(&kvm->arch.config_lock);
158

159
#ifdef CONFIG_LOCKDEP
160
	/* Clue in lockdep that the config_lock must be taken inside kvm->lock */
161
	mutex_lock(&kvm->lock);
162
	mutex_lock(&kvm->arch.config_lock);
163
	mutex_unlock(&kvm->arch.config_lock);
164
	mutex_unlock(&kvm->lock);
165
#endif
166

167
	kvm_init_nested(kvm);
168

169
	ret = kvm_share_hyp(kvm, kvm + 1);
170
	if (ret)
171
		return ret;
172

173
	if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
174
		ret = -ENOMEM;
175
		goto err_unshare_kvm;
176
	}
177
	cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
178

179
	ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
180
	if (ret)
181
		goto err_free_cpumask;
182

183
	if (is_protected_kvm_enabled()) {
184
		/*
185
		 * If any failures occur after this is successful, make sure to
186
		 * call __pkvm_unreserve_vm to unreserve the VM in hyp.
187
		 */
188
		ret = pkvm_init_host_vm(kvm);
189
		if (ret)
190
			goto err_free_cpumask;
191
	}
192

193
	kvm_vgic_early_init(kvm);
194

195
	kvm_timer_init_vm(kvm);
196

197
	/* The maximum number of VCPUs is limited by the host's GIC model */
198
	kvm->max_vcpus = kvm_arm_default_max_vcpus();
199

200
	kvm_arm_init_hypercalls(kvm);
201

202
	bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
203

204
	return 0;
205

206
err_free_cpumask:
207
	free_cpumask_var(kvm->arch.supported_cpus);
208
err_unshare_kvm:
209
	kvm_unshare_hyp(kvm, kvm + 1);
210
	return ret;
211
}
212

213
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
214
{
215
	return VM_FAULT_SIGBUS;
216
}
217

218
void kvm_arch_create_vm_debugfs(struct kvm *kvm)
219
{
220
	kvm_sys_regs_create_debugfs(kvm);
221
	kvm_s2_ptdump_create_debugfs(kvm);
222
}
223

224
static void kvm_destroy_mpidr_data(struct kvm *kvm)
225
{
226
	struct kvm_mpidr_data *data;
227

228
	mutex_lock(&kvm->arch.config_lock);
229

230
	data = rcu_dereference_protected(kvm->arch.mpidr_data,
231
					 lockdep_is_held(&kvm->arch.config_lock));
232
	if (data) {
233
		rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
234
		synchronize_rcu();
235
		kfree(data);
236
	}
237

238
	mutex_unlock(&kvm->arch.config_lock);
239
}
240

241
/**
242
 * kvm_arch_destroy_vm - destroy the VM data structure
243
 * @kvm:	pointer to the KVM struct
244
 */
245
void kvm_arch_destroy_vm(struct kvm *kvm)
246
{
247
	bitmap_free(kvm->arch.pmu_filter);
248
	free_cpumask_var(kvm->arch.supported_cpus);
249

250
	kvm_vgic_destroy(kvm);
251

252
	if (is_protected_kvm_enabled())
253
		pkvm_destroy_hyp_vm(kvm);
254

255
	kvm_destroy_mpidr_data(kvm);
256

257
	kfree(kvm->arch.sysreg_masks);
258
	kvm_destroy_vcpus(kvm);
259

260
	kvm_unshare_hyp(kvm, kvm + 1);
261

262
	kvm_arm_teardown_hypercalls(kvm);
263
}
264

265
static bool kvm_has_full_ptr_auth(void)
266
{
267
	bool apa, gpa, api, gpi, apa3, gpa3;
268
	u64 isar1, isar2, val;
269

270
	/*
271
	 * Check that:
272
	 *
273
	 * - both Address and Generic auth are implemented for a given
274
         *   algorithm (Q5, IMPDEF or Q3)
275
	 * - only a single algorithm is implemented.
276
	 */
277
	if (!system_has_full_ptr_auth())
278
		return false;
279

280
	isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
281
	isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
282

283
	apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
284
	val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
285
	gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
286

287
	api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
288
	val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
289
	gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
290

291
	apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
292
	val  = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
293
	gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
294

295
	return (apa == gpa && api == gpi && apa3 == gpa3 &&
296
		(apa + api + apa3) == 1);
297
}
298

299
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
300
{
301
	int r;
302

303
	if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
304
		return 0;
305

306
	switch (ext) {
307
	case KVM_CAP_IRQCHIP:
308
		r = vgic_present;
309
		break;
310
	case KVM_CAP_IOEVENTFD:
311
	case KVM_CAP_USER_MEMORY:
312
	case KVM_CAP_SYNC_MMU:
313
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
314
	case KVM_CAP_ONE_REG:
315
	case KVM_CAP_ARM_PSCI:
316
	case KVM_CAP_ARM_PSCI_0_2:
317
	case KVM_CAP_READONLY_MEM:
318
	case KVM_CAP_MP_STATE:
319
	case KVM_CAP_IMMEDIATE_EXIT:
320
	case KVM_CAP_VCPU_EVENTS:
321
	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
322
	case KVM_CAP_ARM_NISV_TO_USER:
323
	case KVM_CAP_ARM_INJECT_EXT_DABT:
324
	case KVM_CAP_SET_GUEST_DEBUG:
325
	case KVM_CAP_VCPU_ATTRIBUTES:
326
	case KVM_CAP_PTP_KVM:
327
	case KVM_CAP_ARM_SYSTEM_SUSPEND:
328
	case KVM_CAP_IRQFD_RESAMPLE:
329
	case KVM_CAP_COUNTER_OFFSET:
330
	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
331
		r = 1;
332
		break;
333
	case KVM_CAP_SET_GUEST_DEBUG2:
334
		return KVM_GUESTDBG_VALID_MASK;
335
	case KVM_CAP_ARM_SET_DEVICE_ADDR:
336
		r = 1;
337
		break;
338
	case KVM_CAP_NR_VCPUS:
339
		/*
340
		 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
341
		 * architectures, as it does not always bound it to
342
		 * KVM_CAP_MAX_VCPUS. It should not matter much because
343
		 * this is just an advisory value.
344
		 */
345
		r = min_t(unsigned int, num_online_cpus(),
346
			  kvm_arm_default_max_vcpus());
347
		break;
348
	case KVM_CAP_MAX_VCPUS:
349
	case KVM_CAP_MAX_VCPU_ID:
350
		if (kvm)
351
			r = kvm->max_vcpus;
352
		else
353
			r = kvm_arm_default_max_vcpus();
354
		break;
355
	case KVM_CAP_MSI_DEVID:
356
		if (!kvm)
357
			r = -EINVAL;
358
		else
359
			r = kvm->arch.vgic.msis_require_devid;
360
		break;
361
	case KVM_CAP_ARM_USER_IRQ:
362
		/*
363
		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
364
		 * (bump this number if adding more devices)
365
		 */
366
		r = 1;
367
		break;
368
	case KVM_CAP_ARM_MTE:
369
		r = system_supports_mte();
370
		break;
371
	case KVM_CAP_STEAL_TIME:
372
		r = kvm_arm_pvtime_supported();
373
		break;
374
	case KVM_CAP_ARM_EL1_32BIT:
375
		r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
376
		break;
377
	case KVM_CAP_ARM_EL2:
378
		r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT);
379
		break;
380
	case KVM_CAP_ARM_EL2_E2H0:
381
		r = cpus_have_final_cap(ARM64_HAS_HCR_NV1);
382
		break;
383
	case KVM_CAP_GUEST_DEBUG_HW_BPS:
384
		r = get_num_brps();
385
		break;
386
	case KVM_CAP_GUEST_DEBUG_HW_WPS:
387
		r = get_num_wrps();
388
		break;
389
	case KVM_CAP_ARM_PMU_V3:
390
		r = kvm_supports_guest_pmuv3();
391
		break;
392
	case KVM_CAP_ARM_INJECT_SERROR_ESR:
393
		r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
394
		break;
395
	case KVM_CAP_ARM_VM_IPA_SIZE:
396
		r = get_kvm_ipa_limit();
397
		break;
398
	case KVM_CAP_ARM_SVE:
399
		r = system_supports_sve();
400
		break;
401
	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
402
	case KVM_CAP_ARM_PTRAUTH_GENERIC:
403
		r = kvm_has_full_ptr_auth();
404
		break;
405
	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
406
		if (kvm)
407
			r = kvm->arch.mmu.split_page_chunk_size;
408
		else
409
			r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
410
		break;
411
	case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
412
		r = kvm_supported_block_sizes();
413
		break;
414
	case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
415
		r = BIT(0);
416
		break;
417
	case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
418
		if (!kvm)
419
			r = -EINVAL;
420
		else
421
			r = kvm_supports_cacheable_pfnmap();
422
		break;
423

424
	default:
425
		r = 0;
426
	}
427

428
	return r;
429
}
430

431
long kvm_arch_dev_ioctl(struct file *filp,
432
			unsigned int ioctl, unsigned long arg)
433
{
434
	return -EINVAL;
435
}
436

437
struct kvm *kvm_arch_alloc_vm(void)
438
{
439
	size_t sz = sizeof(struct kvm);
440

441
	if (!has_vhe())
442
		return kzalloc(sz, GFP_KERNEL_ACCOUNT);
443

444
	return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
445
}
446

447
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
448
{
449
	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
450
		return -EBUSY;
451

452
	if (id >= kvm->max_vcpus)
453
		return -EINVAL;
454

455
	return 0;
456
}
457

458
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
459
{
460
	int err;
461

462
	spin_lock_init(&vcpu->arch.mp_state_lock);
463

464
#ifdef CONFIG_LOCKDEP
465
	/* Inform lockdep that the config_lock is acquired after vcpu->mutex */
466
	mutex_lock(&vcpu->mutex);
467
	mutex_lock(&vcpu->kvm->arch.config_lock);
468
	mutex_unlock(&vcpu->kvm->arch.config_lock);
469
	mutex_unlock(&vcpu->mutex);
470
#endif
471

472
	/* Force users to call KVM_ARM_VCPU_INIT */
473
	vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
474

475
	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
476

477
	/* Set up the timer */
478
	kvm_timer_vcpu_init(vcpu);
479

480
	kvm_pmu_vcpu_init(vcpu);
481

482
	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
483

484
	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
485

486
	/*
487
	 * This vCPU may have been created after mpidr_data was initialized.
488
	 * Throw out the pre-computed mappings if that is the case which forces
489
	 * KVM to fall back to iteratively searching the vCPUs.
490
	 */
491
	kvm_destroy_mpidr_data(vcpu->kvm);
492

493
	err = kvm_vgic_vcpu_init(vcpu);
494
	if (err)
495
		return err;
496

497
	err = kvm_share_hyp(vcpu, vcpu + 1);
498
	if (err)
499
		kvm_vgic_vcpu_destroy(vcpu);
500

501
	return err;
502
}
503

504
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
505
{
506
}
507

508
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
509
{
510
	if (!is_protected_kvm_enabled())
511
		kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
512
	else
513
		free_hyp_memcache(&vcpu->arch.pkvm_memcache);
514
	kvm_timer_vcpu_terminate(vcpu);
515
	kvm_pmu_vcpu_destroy(vcpu);
516
	kvm_vgic_vcpu_destroy(vcpu);
517
	kvm_arm_vcpu_destroy(vcpu);
518
}
519

520
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
521
{
522

523
}
524

525
void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
526
{
527

528
}
529

530
static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
531
{
532
	if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
533
		/*
534
		 * Either we're running an L2 guest, and the API/APK bits come
535
		 * from L1's HCR_EL2, or API/APK are both set.
536
		 */
537
		if (unlikely(is_nested_ctxt(vcpu))) {
538
			u64 val;
539

540
			val = __vcpu_sys_reg(vcpu, HCR_EL2);
541
			val &= (HCR_API | HCR_APK);
542
			vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
543
			vcpu->arch.hcr_el2 |= val;
544
		} else {
545
			vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
546
		}
547

548
		/*
549
		 * Save the host keys if there is any chance for the guest
550
		 * to use pauth, as the entry code will reload the guest
551
		 * keys in that case.
552
		 */
553
		if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
554
			struct kvm_cpu_context *ctxt;
555

556
			ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
557
			ptrauth_save_keys(ctxt);
558
		}
559
	}
560
}
561

562
static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
563
{
564
	if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
565
		return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
566

567
	return single_task_running() &&
568
	       (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
569
		vcpu->kvm->arch.vgic.nassgireq);
570
}
571

572
static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
573
{
574
	if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
575
		return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
576

577
	return single_task_running();
578
}
579

580
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
581
{
582
	struct kvm_s2_mmu *mmu;
583
	int *last_ran;
584

585
	if (is_protected_kvm_enabled())
586
		goto nommu;
587

588
	if (vcpu_has_nv(vcpu))
589
		kvm_vcpu_load_hw_mmu(vcpu);
590

591
	mmu = vcpu->arch.hw_mmu;
592
	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
593

594
	/*
595
	 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,
596
	 * which happens eagerly in VHE.
597
	 *
598
	 * Also, the VMID allocator only preserves VMIDs that are active at the
599
	 * time of rollover, so KVM might need to grab a new VMID for the MMU if
600
	 * this is called from kvm_sched_in().
601
	 */
602
	kvm_arm_vmid_update(&mmu->vmid);
603

604
	/*
605
	 * We guarantee that both TLBs and I-cache are private to each
606
	 * vcpu. If detecting that a vcpu from the same VM has
607
	 * previously run on the same physical CPU, call into the
608
	 * hypervisor code to nuke the relevant contexts.
609
	 *
610
	 * We might get preempted before the vCPU actually runs, but
611
	 * over-invalidation doesn't affect correctness.
612
	 */
613
	if (*last_ran != vcpu->vcpu_idx) {
614
		kvm_call_hyp(__kvm_flush_cpu_context, mmu);
615
		*last_ran = vcpu->vcpu_idx;
616
	}
617

618
nommu:
619
	vcpu->cpu = cpu;
620

621
	/*
622
	 * The timer must be loaded before the vgic to correctly set up physical
623
	 * interrupt deactivation in nested state (e.g. timer interrupt).
624
	 */
625
	kvm_timer_vcpu_load(vcpu);
626
	kvm_vgic_load(vcpu);
627
	kvm_vcpu_load_debug(vcpu);
628
	if (has_vhe())
629
		kvm_vcpu_load_vhe(vcpu);
630
	kvm_arch_vcpu_load_fp(vcpu);
631
	kvm_vcpu_pmu_restore_guest(vcpu);
632
	if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
633
		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
634

635
	if (kvm_vcpu_should_clear_twe(vcpu))
636
		vcpu->arch.hcr_el2 &= ~HCR_TWE;
637
	else
638
		vcpu->arch.hcr_el2 |= HCR_TWE;
639

640
	if (kvm_vcpu_should_clear_twi(vcpu))
641
		vcpu->arch.hcr_el2 &= ~HCR_TWI;
642
	else
643
		vcpu->arch.hcr_el2 |= HCR_TWI;
644

645
	vcpu_set_pauth_traps(vcpu);
646

647
	if (is_protected_kvm_enabled()) {
648
		kvm_call_hyp_nvhe(__pkvm_vcpu_load,
649
				  vcpu->kvm->arch.pkvm.handle,
650
				  vcpu->vcpu_idx, vcpu->arch.hcr_el2);
651
		kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
652
			     &vcpu->arch.vgic_cpu.vgic_v3);
653
	}
654

655
	if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
656
		vcpu_set_on_unsupported_cpu(vcpu);
657
}
658

659
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
660
{
661
	if (is_protected_kvm_enabled()) {
662
		kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
663
			     &vcpu->arch.vgic_cpu.vgic_v3);
664
		kvm_call_hyp_nvhe(__pkvm_vcpu_put);
665
	}
666

667
	kvm_vcpu_put_debug(vcpu);
668
	kvm_arch_vcpu_put_fp(vcpu);
669
	if (has_vhe())
670
		kvm_vcpu_put_vhe(vcpu);
671
	kvm_timer_vcpu_put(vcpu);
672
	kvm_vgic_put(vcpu);
673
	kvm_vcpu_pmu_restore_host(vcpu);
674
	if (vcpu_has_nv(vcpu))
675
		kvm_vcpu_put_hw_mmu(vcpu);
676
	kvm_arm_vmid_clear_active();
677

678
	vcpu_clear_on_unsupported_cpu(vcpu);
679
	vcpu->cpu = -1;
680
}
681

682
static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
683
{
684
	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
685
	kvm_make_request(KVM_REQ_SLEEP, vcpu);
686
	kvm_vcpu_kick(vcpu);
687
}
688

689
void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
690
{
691
	spin_lock(&vcpu->arch.mp_state_lock);
692
	__kvm_arm_vcpu_power_off(vcpu);
693
	spin_unlock(&vcpu->arch.mp_state_lock);
694
}
695

696
bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
697
{
698
	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
699
}
700

701
static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
702
{
703
	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
704
	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
705
	kvm_vcpu_kick(vcpu);
706
}
707

708
static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
709
{
710
	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
711
}
712

713
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
714
				    struct kvm_mp_state *mp_state)
715
{
716
	*mp_state = READ_ONCE(vcpu->arch.mp_state);
717

718
	return 0;
719
}
720

721
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
722
				    struct kvm_mp_state *mp_state)
723
{
724
	int ret = 0;
725

726
	spin_lock(&vcpu->arch.mp_state_lock);
727

728
	switch (mp_state->mp_state) {
729
	case KVM_MP_STATE_RUNNABLE:
730
		WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
731
		break;
732
	case KVM_MP_STATE_STOPPED:
733
		__kvm_arm_vcpu_power_off(vcpu);
734
		break;
735
	case KVM_MP_STATE_SUSPENDED:
736
		kvm_arm_vcpu_suspend(vcpu);
737
		break;
738
	default:
739
		ret = -EINVAL;
740
	}
741

742
	spin_unlock(&vcpu->arch.mp_state_lock);
743

744
	return ret;
745
}
746

747
/**
748
 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
749
 * @v:		The VCPU pointer
750
 *
751
 * If the guest CPU is not waiting for interrupts or an interrupt line is
752
 * asserted, the CPU is by definition runnable.
753
 */
754
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
755
{
756
	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
757

758
	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
759
		&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
760
}
761

762
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
763
{
764
	return vcpu_mode_priv(vcpu);
765
}
766

767
#ifdef CONFIG_GUEST_PERF_EVENTS
768
unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
769
{
770
	return *vcpu_pc(vcpu);
771
}
772
#endif
773

774
static void kvm_init_mpidr_data(struct kvm *kvm)
775
{
776
	struct kvm_mpidr_data *data = NULL;
777
	unsigned long c, mask, nr_entries;
778
	u64 aff_set = 0, aff_clr = ~0UL;
779
	struct kvm_vcpu *vcpu;
780

781
	mutex_lock(&kvm->arch.config_lock);
782

783
	if (rcu_access_pointer(kvm->arch.mpidr_data) ||
784
	    atomic_read(&kvm->online_vcpus) == 1)
785
		goto out;
786

787
	kvm_for_each_vcpu(c, vcpu, kvm) {
788
		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
789
		aff_set |= aff;
790
		aff_clr &= aff;
791
	}
792

793
	/*
794
	 * A significant bit can be either 0 or 1, and will only appear in
795
	 * aff_set. Use aff_clr to weed out the useless stuff.
796
	 */
797
	mask = aff_set ^ aff_clr;
798
	nr_entries = BIT_ULL(hweight_long(mask));
799

800
	/*
801
	 * Don't let userspace fool us. If we need more than a single page
802
	 * to describe the compressed MPIDR array, just fall back to the
803
	 * iterative method. Single vcpu VMs do not need this either.
804
	 */
805
	if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
806
		data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
807
			       GFP_KERNEL_ACCOUNT);
808

809
	if (!data)
810
		goto out;
811

812
	data->mpidr_mask = mask;
813

814
	kvm_for_each_vcpu(c, vcpu, kvm) {
815
		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
816
		u16 index = kvm_mpidr_index(data, aff);
817

818
		data->cmpidr_to_idx[index] = c;
819
	}
820

821
	rcu_assign_pointer(kvm->arch.mpidr_data, data);
822
out:
823
	mutex_unlock(&kvm->arch.config_lock);
824
}
825

826
/*
827
 * Handle both the initialisation that is being done when the vcpu is
828
 * run for the first time, as well as the updates that must be
829
 * performed each time we get a new thread dealing with this vcpu.
830
 */
831
int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
832
{
833
	struct kvm *kvm = vcpu->kvm;
834
	int ret;
835

836
	if (!kvm_vcpu_initialized(vcpu))
837
		return -ENOEXEC;
838

839
	if (!kvm_arm_vcpu_is_finalized(vcpu))
840
		return -EPERM;
841

842
	if (likely(vcpu_has_run_once(vcpu)))
843
		return 0;
844

845
	kvm_init_mpidr_data(kvm);
846

847
	if (likely(irqchip_in_kernel(kvm))) {
848
		/*
849
		 * Map the VGIC hardware resources before running a vcpu the
850
		 * first time on this VM.
851
		 */
852
		ret = kvm_vgic_map_resources(kvm);
853
		if (ret)
854
			return ret;
855
	}
856

857
	ret = kvm_finalize_sys_regs(vcpu);
858
	if (ret)
859
		return ret;
860

861
	if (vcpu_has_nv(vcpu)) {
862
		ret = kvm_vcpu_allocate_vncr_tlb(vcpu);
863
		if (ret)
864
			return ret;
865

866
		ret = kvm_vgic_vcpu_nv_init(vcpu);
867
		if (ret)
868
			return ret;
869
	}
870

871
	/*
872
	 * This needs to happen after any restriction has been applied
873
	 * to the feature set.
874
	 */
875
	kvm_calculate_traps(vcpu);
876

877
	ret = kvm_timer_enable(vcpu);
878
	if (ret)
879
		return ret;
880

881
	if (kvm_vcpu_has_pmu(vcpu)) {
882
		ret = kvm_arm_pmu_v3_enable(vcpu);
883
		if (ret)
884
			return ret;
885
	}
886

887
	if (is_protected_kvm_enabled()) {
888
		ret = pkvm_create_hyp_vm(kvm);
889
		if (ret)
890
			return ret;
891

892
		ret = pkvm_create_hyp_vcpu(vcpu);
893
		if (ret)
894
			return ret;
895
	}
896

897
	mutex_lock(&kvm->arch.config_lock);
898
	set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
899
	mutex_unlock(&kvm->arch.config_lock);
900

901
	return ret;
902
}
903

904
bool kvm_arch_intc_initialized(struct kvm *kvm)
905
{
906
	return vgic_initialized(kvm);
907
}
908

909
void kvm_arm_halt_guest(struct kvm *kvm)
910
{
911
	unsigned long i;
912
	struct kvm_vcpu *vcpu;
913

914
	kvm_for_each_vcpu(i, vcpu, kvm)
915
		vcpu->arch.pause = true;
916
	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
917
}
918

919
void kvm_arm_resume_guest(struct kvm *kvm)
920
{
921
	unsigned long i;
922
	struct kvm_vcpu *vcpu;
923

924
	kvm_for_each_vcpu(i, vcpu, kvm) {
925
		vcpu->arch.pause = false;
926
		__kvm_vcpu_wake_up(vcpu);
927
	}
928
}
929

930
static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
931
{
932
	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
933

934
	rcuwait_wait_event(wait,
935
			   (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
936
			   TASK_INTERRUPTIBLE);
937

938
	if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
939
		/* Awaken to handle a signal, request we sleep again later. */
940
		kvm_make_request(KVM_REQ_SLEEP, vcpu);
941
	}
942

943
	/*
944
	 * Make sure we will observe a potential reset request if we've
945
	 * observed a change to the power state. Pairs with the smp_wmb() in
946
	 * kvm_psci_vcpu_on().
947
	 */
948
	smp_rmb();
949
}
950

951
/**
952
 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
953
 * @vcpu:	The VCPU pointer
954
 *
955
 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
956
 * the vCPU is runnable.  The vCPU may or may not be scheduled out, depending
957
 * on when a wake event arrives, e.g. there may already be a pending wake event.
958
 */
959
void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
960
{
961
	/*
962
	 * Sync back the state of the GIC CPU interface so that we have
963
	 * the latest PMR and group enables. This ensures that
964
	 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
965
	 * we have pending interrupts, e.g. when determining if the
966
	 * vCPU should block.
967
	 *
968
	 * For the same reason, we want to tell GICv4 that we need
969
	 * doorbells to be signalled, should an interrupt become pending.
970
	 */
971
	preempt_disable();
972
	vcpu_set_flag(vcpu, IN_WFI);
973
	kvm_vgic_put(vcpu);
974
	preempt_enable();
975

976
	kvm_vcpu_halt(vcpu);
977
	vcpu_clear_flag(vcpu, IN_WFIT);
978

979
	preempt_disable();
980
	vcpu_clear_flag(vcpu, IN_WFI);
981
	kvm_vgic_load(vcpu);
982
	preempt_enable();
983
}
984

985
static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
986
{
987
	if (!kvm_arm_vcpu_suspended(vcpu))
988
		return 1;
989

990
	kvm_vcpu_wfi(vcpu);
991

992
	/*
993
	 * The suspend state is sticky; we do not leave it until userspace
994
	 * explicitly marks the vCPU as runnable. Request that we suspend again
995
	 * later.
996
	 */
997
	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
998

999
	/*
1000
	 * Check to make sure the vCPU is actually runnable. If so, exit to
1001
	 * userspace informing it of the wakeup condition.
1002
	 */
1003
	if (kvm_arch_vcpu_runnable(vcpu)) {
1004
		memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
1005
		vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
1006
		vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
1007
		return 0;
1008
	}
1009

1010
	/*
1011
	 * Otherwise, we were unblocked to process a different event, such as a
1012
	 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
1013
	 * process the event.
1014
	 */
1015
	return 1;
1016
}
1017

1018
/**
1019
 * check_vcpu_requests - check and handle pending vCPU requests
1020
 * @vcpu:	the VCPU pointer
1021
 *
1022
 * Return: 1 if we should enter the guest
1023
 *	   0 if we should exit to userspace
1024
 *	   < 0 if we should exit to userspace, where the return value indicates
1025
 *	   an error
1026
 */
1027
static int check_vcpu_requests(struct kvm_vcpu *vcpu)
1028
{
1029
	if (kvm_request_pending(vcpu)) {
1030
		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
1031
			return -EIO;
1032

1033
		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1034
			kvm_vcpu_sleep(vcpu);
1035

1036
		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1037
			kvm_reset_vcpu(vcpu);
1038

1039
		/*
1040
		 * Clear IRQ_PENDING requests that were made to guarantee
1041
		 * that a VCPU sees new virtual interrupts.
1042
		 */
1043
		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1044

1045
		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1046
			kvm_update_stolen_time(vcpu);
1047

1048
		if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1049
			/* The distributor enable bits were changed */
1050
			preempt_disable();
1051
			vgic_v4_put(vcpu);
1052
			vgic_v4_load(vcpu);
1053
			preempt_enable();
1054
		}
1055

1056
		if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1057
			kvm_vcpu_reload_pmu(vcpu);
1058

1059
		if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1060
			kvm_vcpu_pmu_restore_guest(vcpu);
1061

1062
		if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1063
			return kvm_vcpu_suspend(vcpu);
1064

1065
		if (kvm_dirty_ring_check_request(vcpu))
1066
			return 0;
1067

1068
		check_nested_vcpu_requests(vcpu);
1069
	}
1070

1071
	return 1;
1072
}
1073

1074
static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1075
{
1076
	if (likely(!vcpu_mode_is_32bit(vcpu)))
1077
		return false;
1078

1079
	if (vcpu_has_nv(vcpu))
1080
		return true;
1081

1082
	return !kvm_supports_32bit_el0();
1083
}
1084

1085
/**
1086
 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1087
 * @vcpu:	The VCPU pointer
1088
 * @ret:	Pointer to write optional return code
1089
 *
1090
 * Returns: true if the VCPU needs to return to a preemptible + interruptible
1091
 *	    and skip guest entry.
1092
 *
1093
 * This function disambiguates between two different types of exits: exits to a
1094
 * preemptible + interruptible kernel context and exits to userspace. For an
1095
 * exit to userspace, this function will write the return code to ret and return
1096
 * true. For an exit to preemptible + interruptible kernel context (i.e. check
1097
 * for pending work and re-enter), return true without writing to ret.
1098
 */
1099
static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1100
{
1101
	struct kvm_run *run = vcpu->run;
1102

1103
	/*
1104
	 * If we're using a userspace irqchip, then check if we need
1105
	 * to tell a userspace irqchip about timer or PMU level
1106
	 * changes and if so, exit to userspace (the actual level
1107
	 * state gets updated in kvm_timer_update_run and
1108
	 * kvm_pmu_update_run below).
1109
	 */
1110
	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1111
		if (kvm_timer_should_notify_user(vcpu) ||
1112
		    kvm_pmu_should_notify_user(vcpu)) {
1113
			*ret = -EINTR;
1114
			run->exit_reason = KVM_EXIT_INTR;
1115
			return true;
1116
		}
1117
	}
1118

1119
	if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1120
		run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1121
		run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1122
		run->fail_entry.cpu = smp_processor_id();
1123
		*ret = 0;
1124
		return true;
1125
	}
1126

1127
	return kvm_request_pending(vcpu) ||
1128
			xfer_to_guest_mode_work_pending();
1129
}
1130

1131
/*
1132
 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1133
 * the vCPU is running.
1134
 *
1135
 * This must be noinstr as instrumentation may make use of RCU, and this is not
1136
 * safe during the EQS.
1137
 */
1138
static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1139
{
1140
	int ret;
1141

1142
	guest_state_enter_irqoff();
1143
	ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1144
	guest_state_exit_irqoff();
1145

1146
	return ret;
1147
}
1148

1149
/**
1150
 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1151
 * @vcpu:	The VCPU pointer
1152
 *
1153
 * This function is called through the VCPU_RUN ioctl called from user space. It
1154
 * will execute VM code in a loop until the time slice for the process is used
1155
 * or some emulation is needed from user space in which case the function will
1156
 * return with return value 0 and with the kvm_run structure filled in with the
1157
 * required data for the requested emulation.
1158
 */
1159
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1160
{
1161
	struct kvm_run *run = vcpu->run;
1162
	int ret;
1163

1164
	if (run->exit_reason == KVM_EXIT_MMIO) {
1165
		ret = kvm_handle_mmio_return(vcpu);
1166
		if (ret <= 0)
1167
			return ret;
1168
	}
1169

1170
	vcpu_load(vcpu);
1171

1172
	if (!vcpu->wants_to_run) {
1173
		ret = -EINTR;
1174
		goto out;
1175
	}
1176

1177
	kvm_sigset_activate(vcpu);
1178

1179
	ret = 1;
1180
	run->exit_reason = KVM_EXIT_UNKNOWN;
1181
	run->flags = 0;
1182
	while (ret > 0) {
1183
		/*
1184
		 * Check conditions before entering the guest
1185
		 */
1186
		ret = xfer_to_guest_mode_handle_work(vcpu);
1187
		if (!ret)
1188
			ret = 1;
1189

1190
		if (ret > 0)
1191
			ret = check_vcpu_requests(vcpu);
1192

1193
		/*
1194
		 * Preparing the interrupts to be injected also
1195
		 * involves poking the GIC, which must be done in a
1196
		 * non-preemptible context.
1197
		 */
1198
		preempt_disable();
1199

1200
		kvm_nested_flush_hwstate(vcpu);
1201

1202
		if (kvm_vcpu_has_pmu(vcpu))
1203
			kvm_pmu_flush_hwstate(vcpu);
1204

1205
		local_irq_disable();
1206

1207
		kvm_vgic_flush_hwstate(vcpu);
1208

1209
		kvm_pmu_update_vcpu_events(vcpu);
1210

1211
		/*
1212
		 * Ensure we set mode to IN_GUEST_MODE after we disable
1213
		 * interrupts and before the final VCPU requests check.
1214
		 * See the comment in kvm_vcpu_exiting_guest_mode() and
1215
		 * Documentation/virt/kvm/vcpu-requests.rst
1216
		 */
1217
		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1218

1219
		if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1220
			vcpu->mode = OUTSIDE_GUEST_MODE;
1221
			isb(); /* Ensure work in x_flush_hwstate is committed */
1222
			if (kvm_vcpu_has_pmu(vcpu))
1223
				kvm_pmu_sync_hwstate(vcpu);
1224
			if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1225
				kvm_timer_sync_user(vcpu);
1226
			kvm_vgic_sync_hwstate(vcpu);
1227
			local_irq_enable();
1228
			preempt_enable();
1229
			continue;
1230
		}
1231

1232
		kvm_arch_vcpu_ctxflush_fp(vcpu);
1233

1234
		/**************************************************************
1235
		 * Enter the guest
1236
		 */
1237
		trace_kvm_entry(*vcpu_pc(vcpu));
1238
		guest_timing_enter_irqoff();
1239

1240
		ret = kvm_arm_vcpu_enter_exit(vcpu);
1241

1242
		vcpu->mode = OUTSIDE_GUEST_MODE;
1243
		vcpu->stat.exits++;
1244
		/*
1245
		 * Back from guest
1246
		 *************************************************************/
1247

1248
		/*
1249
		 * We must sync the PMU state before the vgic state so
1250
		 * that the vgic can properly sample the updated state of the
1251
		 * interrupt line.
1252
		 */
1253
		if (kvm_vcpu_has_pmu(vcpu))
1254
			kvm_pmu_sync_hwstate(vcpu);
1255

1256
		/*
1257
		 * Sync the vgic state before syncing the timer state because
1258
		 * the timer code needs to know if the virtual timer
1259
		 * interrupts are active.
1260
		 */
1261
		kvm_vgic_sync_hwstate(vcpu);
1262

1263
		/*
1264
		 * Sync the timer hardware state before enabling interrupts as
1265
		 * we don't want vtimer interrupts to race with syncing the
1266
		 * timer virtual interrupt state.
1267
		 */
1268
		if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1269
			kvm_timer_sync_user(vcpu);
1270

1271
		if (is_hyp_ctxt(vcpu))
1272
			kvm_timer_sync_nested(vcpu);
1273

1274
		kvm_arch_vcpu_ctxsync_fp(vcpu);
1275

1276
		/*
1277
		 * We must ensure that any pending interrupts are taken before
1278
		 * we exit guest timing so that timer ticks are accounted as
1279
		 * guest time. Transiently unmask interrupts so that any
1280
		 * pending interrupts are taken.
1281
		 *
1282
		 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1283
		 * context synchronization event) is necessary to ensure that
1284
		 * pending interrupts are taken.
1285
		 */
1286
		if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1287
			local_irq_enable();
1288
			isb();
1289
			local_irq_disable();
1290
		}
1291

1292
		guest_timing_exit_irqoff();
1293

1294
		local_irq_enable();
1295

1296
		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1297

1298
		/* Exit types that need handling before we can be preempted */
1299
		handle_exit_early(vcpu, ret);
1300

1301
		kvm_nested_sync_hwstate(vcpu);
1302

1303
		preempt_enable();
1304

1305
		/*
1306
		 * The ARMv8 architecture doesn't give the hypervisor
1307
		 * a mechanism to prevent a guest from dropping to AArch32 EL0
1308
		 * if implemented by the CPU. If we spot the guest in such
1309
		 * state and that we decided it wasn't supposed to do so (like
1310
		 * with the asymmetric AArch32 case), return to userspace with
1311
		 * a fatal error.
1312
		 */
1313
		if (vcpu_mode_is_bad_32bit(vcpu)) {
1314
			/*
1315
			 * As we have caught the guest red-handed, decide that
1316
			 * it isn't fit for purpose anymore by making the vcpu
1317
			 * invalid. The VMM can try and fix it by issuing  a
1318
			 * KVM_ARM_VCPU_INIT if it really wants to.
1319
			 */
1320
			vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1321
			ret = ARM_EXCEPTION_IL;
1322
		}
1323

1324
		ret = handle_exit(vcpu, ret);
1325
	}
1326

1327
	/* Tell userspace about in-kernel device output levels */
1328
	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1329
		kvm_timer_update_run(vcpu);
1330
		kvm_pmu_update_run(vcpu);
1331
	}
1332

1333
	kvm_sigset_deactivate(vcpu);
1334

1335
out:
1336
	/*
1337
	 * In the unlikely event that we are returning to userspace
1338
	 * with pending exceptions or PC adjustment, commit these
1339
	 * adjustments in order to give userspace a consistent view of
1340
	 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1341
	 * being preempt-safe on VHE.
1342
	 */
1343
	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1344
		     vcpu_get_flag(vcpu, INCREMENT_PC)))
1345
		kvm_call_hyp(__kvm_adjust_pc, vcpu);
1346

1347
	vcpu_put(vcpu);
1348
	return ret;
1349
}
1350

1351
static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1352
{
1353
	int bit_index;
1354
	bool set;
1355
	unsigned long *hcr;
1356

1357
	if (number == KVM_ARM_IRQ_CPU_IRQ)
1358
		bit_index = __ffs(HCR_VI);
1359
	else /* KVM_ARM_IRQ_CPU_FIQ */
1360
		bit_index = __ffs(HCR_VF);
1361

1362
	hcr = vcpu_hcr(vcpu);
1363
	if (level)
1364
		set = test_and_set_bit(bit_index, hcr);
1365
	else
1366
		set = test_and_clear_bit(bit_index, hcr);
1367

1368
	/*
1369
	 * If we didn't change anything, no need to wake up or kick other CPUs
1370
	 */
1371
	if (set == level)
1372
		return 0;
1373

1374
	/*
1375
	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1376
	 * trigger a world-switch round on the running physical CPU to set the
1377
	 * virtual IRQ/FIQ fields in the HCR appropriately.
1378
	 */
1379
	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1380
	kvm_vcpu_kick(vcpu);
1381

1382
	return 0;
1383
}
1384

1385
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1386
			  bool line_status)
1387
{
1388
	u32 irq = irq_level->irq;
1389
	unsigned int irq_type, vcpu_id, irq_num;
1390
	struct kvm_vcpu *vcpu = NULL;
1391
	bool level = irq_level->level;
1392

1393
	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1394
	vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1395
	vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1396
	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1397

1398
	trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1399

1400
	switch (irq_type) {
1401
	case KVM_ARM_IRQ_TYPE_CPU:
1402
		if (irqchip_in_kernel(kvm))
1403
			return -ENXIO;
1404

1405
		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1406
		if (!vcpu)
1407
			return -EINVAL;
1408

1409
		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1410
			return -EINVAL;
1411

1412
		return vcpu_interrupt_line(vcpu, irq_num, level);
1413
	case KVM_ARM_IRQ_TYPE_PPI:
1414
		if (!irqchip_in_kernel(kvm))
1415
			return -ENXIO;
1416

1417
		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1418
		if (!vcpu)
1419
			return -EINVAL;
1420

1421
		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1422
			return -EINVAL;
1423

1424
		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1425
	case KVM_ARM_IRQ_TYPE_SPI:
1426
		if (!irqchip_in_kernel(kvm))
1427
			return -ENXIO;
1428

1429
		if (irq_num < VGIC_NR_PRIVATE_IRQS)
1430
			return -EINVAL;
1431

1432
		return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1433
	}
1434

1435
	return -EINVAL;
1436
}
1437

1438
static unsigned long system_supported_vcpu_features(void)
1439
{
1440
	unsigned long features = KVM_VCPU_VALID_FEATURES;
1441

1442
	if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1443
		clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1444

1445
	if (!kvm_supports_guest_pmuv3())
1446
		clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1447

1448
	if (!system_supports_sve())
1449
		clear_bit(KVM_ARM_VCPU_SVE, &features);
1450

1451
	if (!kvm_has_full_ptr_auth()) {
1452
		clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1453
		clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1454
	}
1455

1456
	if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1457
		clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1458

1459
	return features;
1460
}
1461

1462
static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1463
					const struct kvm_vcpu_init *init)
1464
{
1465
	unsigned long features = init->features[0];
1466
	int i;
1467

1468
	if (features & ~KVM_VCPU_VALID_FEATURES)
1469
		return -ENOENT;
1470

1471
	for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1472
		if (init->features[i])
1473
			return -ENOENT;
1474
	}
1475

1476
	if (features & ~system_supported_vcpu_features())
1477
		return -EINVAL;
1478

1479
	/*
1480
	 * For now make sure that both address/generic pointer authentication
1481
	 * features are requested by the userspace together.
1482
	 */
1483
	if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1484
	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1485
		return -EINVAL;
1486

1487
	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1488
		return 0;
1489

1490
	/* MTE is incompatible with AArch32 */
1491
	if (kvm_has_mte(vcpu->kvm))
1492
		return -EINVAL;
1493

1494
	/* NV is incompatible with AArch32 */
1495
	if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1496
		return -EINVAL;
1497

1498
	return 0;
1499
}
1500

1501
static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1502
				  const struct kvm_vcpu_init *init)
1503
{
1504
	unsigned long features = init->features[0];
1505

1506
	return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1507
			     KVM_VCPU_MAX_FEATURES);
1508
}
1509

1510
static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1511
{
1512
	struct kvm *kvm = vcpu->kvm;
1513
	int ret = 0;
1514

1515
	/*
1516
	 * When the vCPU has a PMU, but no PMU is set for the guest
1517
	 * yet, set the default one.
1518
	 */
1519
	if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1520
		ret = kvm_arm_set_default_pmu(kvm);
1521

1522
	/* Prepare for nested if required */
1523
	if (!ret && vcpu_has_nv(vcpu))
1524
		ret = kvm_vcpu_init_nested(vcpu);
1525

1526
	return ret;
1527
}
1528

1529
static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1530
				 const struct kvm_vcpu_init *init)
1531
{
1532
	unsigned long features = init->features[0];
1533
	struct kvm *kvm = vcpu->kvm;
1534
	int ret = -EINVAL;
1535

1536
	mutex_lock(&kvm->arch.config_lock);
1537

1538
	if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1539
	    kvm_vcpu_init_changed(vcpu, init))
1540
		goto out_unlock;
1541

1542
	bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1543

1544
	ret = kvm_setup_vcpu(vcpu);
1545
	if (ret)
1546
		goto out_unlock;
1547

1548
	/* Now we know what it is, we can reset it. */
1549
	kvm_reset_vcpu(vcpu);
1550

1551
	set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1552
	vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1553
	ret = 0;
1554
out_unlock:
1555
	mutex_unlock(&kvm->arch.config_lock);
1556
	return ret;
1557
}
1558

1559
static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1560
			       const struct kvm_vcpu_init *init)
1561
{
1562
	int ret;
1563

1564
	if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1565
	    init->target != kvm_target_cpu())
1566
		return -EINVAL;
1567

1568
	ret = kvm_vcpu_init_check_features(vcpu, init);
1569
	if (ret)
1570
		return ret;
1571

1572
	if (!kvm_vcpu_initialized(vcpu))
1573
		return __kvm_vcpu_set_target(vcpu, init);
1574

1575
	if (kvm_vcpu_init_changed(vcpu, init))
1576
		return -EINVAL;
1577

1578
	kvm_reset_vcpu(vcpu);
1579
	return 0;
1580
}
1581

1582
static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1583
					 struct kvm_vcpu_init *init)
1584
{
1585
	bool power_off = false;
1586
	int ret;
1587

1588
	/*
1589
	 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1590
	 * reflecting it in the finalized feature set, thus limiting its scope
1591
	 * to a single KVM_ARM_VCPU_INIT call.
1592
	 */
1593
	if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1594
		init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1595
		power_off = true;
1596
	}
1597

1598
	ret = kvm_vcpu_set_target(vcpu, init);
1599
	if (ret)
1600
		return ret;
1601

1602
	/*
1603
	 * Ensure a rebooted VM will fault in RAM pages and detect if the
1604
	 * guest MMU is turned off and flush the caches as needed.
1605
	 *
1606
	 * S2FWB enforces all memory accesses to RAM being cacheable,
1607
	 * ensuring that the data side is always coherent. We still
1608
	 * need to invalidate the I-cache though, as FWB does *not*
1609
	 * imply CTR_EL0.DIC.
1610
	 */
1611
	if (vcpu_has_run_once(vcpu)) {
1612
		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1613
			stage2_unmap_vm(vcpu->kvm);
1614
		else
1615
			icache_inval_all_pou();
1616
	}
1617

1618
	vcpu_reset_hcr(vcpu);
1619

1620
	/*
1621
	 * Handle the "start in power-off" case.
1622
	 */
1623
	spin_lock(&vcpu->arch.mp_state_lock);
1624

1625
	if (power_off)
1626
		__kvm_arm_vcpu_power_off(vcpu);
1627
	else
1628
		WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1629

1630
	spin_unlock(&vcpu->arch.mp_state_lock);
1631

1632
	return 0;
1633
}
1634

1635
static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1636
				 struct kvm_device_attr *attr)
1637
{
1638
	int ret = -ENXIO;
1639

1640
	switch (attr->group) {
1641
	default:
1642
		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1643
		break;
1644
	}
1645

1646
	return ret;
1647
}
1648

1649
static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1650
				 struct kvm_device_attr *attr)
1651
{
1652
	int ret = -ENXIO;
1653

1654
	switch (attr->group) {
1655
	default:
1656
		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1657
		break;
1658
	}
1659

1660
	return ret;
1661
}
1662

1663
static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1664
				 struct kvm_device_attr *attr)
1665
{
1666
	int ret = -ENXIO;
1667

1668
	switch (attr->group) {
1669
	default:
1670
		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1671
		break;
1672
	}
1673

1674
	return ret;
1675
}
1676

1677
static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1678
				   struct kvm_vcpu_events *events)
1679
{
1680
	memset(events, 0, sizeof(*events));
1681

1682
	return __kvm_arm_vcpu_get_events(vcpu, events);
1683
}
1684

1685
static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1686
				   struct kvm_vcpu_events *events)
1687
{
1688
	int i;
1689

1690
	/* check whether the reserved field is zero */
1691
	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1692
		if (events->reserved[i])
1693
			return -EINVAL;
1694

1695
	/* check whether the pad field is zero */
1696
	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1697
		if (events->exception.pad[i])
1698
			return -EINVAL;
1699

1700
	return __kvm_arm_vcpu_set_events(vcpu, events);
1701
}
1702

1703
long kvm_arch_vcpu_ioctl(struct file *filp,
1704
			 unsigned int ioctl, unsigned long arg)
1705
{
1706
	struct kvm_vcpu *vcpu = filp->private_data;
1707
	void __user *argp = (void __user *)arg;
1708
	struct kvm_device_attr attr;
1709
	long r;
1710

1711
	switch (ioctl) {
1712
	case KVM_ARM_VCPU_INIT: {
1713
		struct kvm_vcpu_init init;
1714

1715
		r = -EFAULT;
1716
		if (copy_from_user(&init, argp, sizeof(init)))
1717
			break;
1718

1719
		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1720
		break;
1721
	}
1722
	case KVM_SET_ONE_REG:
1723
	case KVM_GET_ONE_REG: {
1724
		struct kvm_one_reg reg;
1725

1726
		r = -ENOEXEC;
1727
		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1728
			break;
1729

1730
		r = -EFAULT;
1731
		if (copy_from_user(&reg, argp, sizeof(reg)))
1732
			break;
1733

1734
		/*
1735
		 * We could owe a reset due to PSCI. Handle the pending reset
1736
		 * here to ensure userspace register accesses are ordered after
1737
		 * the reset.
1738
		 */
1739
		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1740
			kvm_reset_vcpu(vcpu);
1741

1742
		if (ioctl == KVM_SET_ONE_REG)
1743
			r = kvm_arm_set_reg(vcpu, &reg);
1744
		else
1745
			r = kvm_arm_get_reg(vcpu, &reg);
1746
		break;
1747
	}
1748
	case KVM_GET_REG_LIST: {
1749
		struct kvm_reg_list __user *user_list = argp;
1750
		struct kvm_reg_list reg_list;
1751
		unsigned n;
1752

1753
		r = -ENOEXEC;
1754
		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1755
			break;
1756

1757
		r = -EPERM;
1758
		if (!kvm_arm_vcpu_is_finalized(vcpu))
1759
			break;
1760

1761
		r = -EFAULT;
1762
		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1763
			break;
1764
		n = reg_list.n;
1765
		reg_list.n = kvm_arm_num_regs(vcpu);
1766
		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1767
			break;
1768
		r = -E2BIG;
1769
		if (n < reg_list.n)
1770
			break;
1771
		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1772
		break;
1773
	}
1774
	case KVM_SET_DEVICE_ATTR: {
1775
		r = -EFAULT;
1776
		if (copy_from_user(&attr, argp, sizeof(attr)))
1777
			break;
1778
		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1779
		break;
1780
	}
1781
	case KVM_GET_DEVICE_ATTR: {
1782
		r = -EFAULT;
1783
		if (copy_from_user(&attr, argp, sizeof(attr)))
1784
			break;
1785
		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1786
		break;
1787
	}
1788
	case KVM_HAS_DEVICE_ATTR: {
1789
		r = -EFAULT;
1790
		if (copy_from_user(&attr, argp, sizeof(attr)))
1791
			break;
1792
		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1793
		break;
1794
	}
1795
	case KVM_GET_VCPU_EVENTS: {
1796
		struct kvm_vcpu_events events;
1797

1798
		if (kvm_arm_vcpu_get_events(vcpu, &events))
1799
			return -EINVAL;
1800

1801
		if (copy_to_user(argp, &events, sizeof(events)))
1802
			return -EFAULT;
1803

1804
		return 0;
1805
	}
1806
	case KVM_SET_VCPU_EVENTS: {
1807
		struct kvm_vcpu_events events;
1808

1809
		if (copy_from_user(&events, argp, sizeof(events)))
1810
			return -EFAULT;
1811

1812
		return kvm_arm_vcpu_set_events(vcpu, &events);
1813
	}
1814
	case KVM_ARM_VCPU_FINALIZE: {
1815
		int what;
1816

1817
		if (!kvm_vcpu_initialized(vcpu))
1818
			return -ENOEXEC;
1819

1820
		if (get_user(what, (const int __user *)argp))
1821
			return -EFAULT;
1822

1823
		return kvm_arm_vcpu_finalize(vcpu, what);
1824
	}
1825
	default:
1826
		r = -EINVAL;
1827
	}
1828

1829
	return r;
1830
}
1831

1832
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1833
{
1834

1835
}
1836

1837
static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1838
					struct kvm_arm_device_addr *dev_addr)
1839
{
1840
	switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1841
	case KVM_ARM_DEVICE_VGIC_V2:
1842
		if (!vgic_present)
1843
			return -ENXIO;
1844
		return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1845
	default:
1846
		return -ENODEV;
1847
	}
1848
}
1849

1850
static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1851
{
1852
	switch (attr->group) {
1853
	case KVM_ARM_VM_SMCCC_CTRL:
1854
		return kvm_vm_smccc_has_attr(kvm, attr);
1855
	default:
1856
		return -ENXIO;
1857
	}
1858
}
1859

1860
static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1861
{
1862
	switch (attr->group) {
1863
	case KVM_ARM_VM_SMCCC_CTRL:
1864
		return kvm_vm_smccc_set_attr(kvm, attr);
1865
	default:
1866
		return -ENXIO;
1867
	}
1868
}
1869

1870
int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1871
{
1872
	struct kvm *kvm = filp->private_data;
1873
	void __user *argp = (void __user *)arg;
1874
	struct kvm_device_attr attr;
1875

1876
	switch (ioctl) {
1877
	case KVM_CREATE_IRQCHIP: {
1878
		int ret;
1879
		if (!vgic_present)
1880
			return -ENXIO;
1881
		mutex_lock(&kvm->lock);
1882
		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1883
		mutex_unlock(&kvm->lock);
1884
		return ret;
1885
	}
1886
	case KVM_ARM_SET_DEVICE_ADDR: {
1887
		struct kvm_arm_device_addr dev_addr;
1888

1889
		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1890
			return -EFAULT;
1891
		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1892
	}
1893
	case KVM_ARM_PREFERRED_TARGET: {
1894
		struct kvm_vcpu_init init = {
1895
			.target = KVM_ARM_TARGET_GENERIC_V8,
1896
		};
1897

1898
		if (copy_to_user(argp, &init, sizeof(init)))
1899
			return -EFAULT;
1900

1901
		return 0;
1902
	}
1903
	case KVM_ARM_MTE_COPY_TAGS: {
1904
		struct kvm_arm_copy_mte_tags copy_tags;
1905

1906
		if (copy_from_user(&copy_tags, argp, sizeof(copy_tags)))
1907
			return -EFAULT;
1908
		return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
1909
	}
1910
	case KVM_ARM_SET_COUNTER_OFFSET: {
1911
		struct kvm_arm_counter_offset offset;
1912

1913
		if (copy_from_user(&offset, argp, sizeof(offset)))
1914
			return -EFAULT;
1915
		return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1916
	}
1917
	case KVM_HAS_DEVICE_ATTR: {
1918
		if (copy_from_user(&attr, argp, sizeof(attr)))
1919
			return -EFAULT;
1920

1921
		return kvm_vm_has_attr(kvm, &attr);
1922
	}
1923
	case KVM_SET_DEVICE_ATTR: {
1924
		if (copy_from_user(&attr, argp, sizeof(attr)))
1925
			return -EFAULT;
1926

1927
		return kvm_vm_set_attr(kvm, &attr);
1928
	}
1929
	case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1930
		struct reg_mask_range range;
1931

1932
		if (copy_from_user(&range, argp, sizeof(range)))
1933
			return -EFAULT;
1934
		return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1935
	}
1936
	default:
1937
		return -EINVAL;
1938
	}
1939
}
1940

1941
static unsigned long nvhe_percpu_size(void)
1942
{
1943
	return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1944
		(unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1945
}
1946

1947
static unsigned long nvhe_percpu_order(void)
1948
{
1949
	unsigned long size = nvhe_percpu_size();
1950

1951
	return size ? get_order(size) : 0;
1952
}
1953

1954
static size_t pkvm_host_sve_state_order(void)
1955
{
1956
	return get_order(pkvm_host_sve_state_size());
1957
}
1958

1959
/* A lookup table holding the hypervisor VA for each vector slot */
1960
static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1961

1962
static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1963
{
1964
	hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1965
}
1966

1967
static int kvm_init_vector_slots(void)
1968
{
1969
	int err;
1970
	void *base;
1971

1972
	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1973
	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1974

1975
	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1976
	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1977

1978
	if (kvm_system_needs_idmapped_vectors() &&
1979
	    !is_protected_kvm_enabled()) {
1980
		err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1981
					       __BP_HARDEN_HYP_VECS_SZ, &base);
1982
		if (err)
1983
			return err;
1984
	}
1985

1986
	kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1987
	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
1988
	return 0;
1989
}
1990

1991
static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
1992
{
1993
	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
1994
	unsigned long tcr;
1995

1996
	/*
1997
	 * Calculate the raw per-cpu offset without a translation from the
1998
	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
1999
	 * so that we can use adr_l to access per-cpu variables in EL2.
2000
	 * Also drop the KASAN tag which gets in the way...
2001
	 */
2002
	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2003
			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2004

2005
	params->mair_el2 = read_sysreg(mair_el1);
2006

2007
	tcr = read_sysreg(tcr_el1);
2008
	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2009
		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
2010
		tcr |= TCR_EPD1_MASK;
2011
	} else {
2012
		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
2013

2014
		tcr &= TCR_EL2_MASK;
2015
		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
2016
		if (lpa2_is_enabled())
2017
			tcr |= TCR_EL2_DS;
2018
	}
2019
	tcr |= TCR_T0SZ(hyp_va_bits);
2020
	params->tcr_el2 = tcr;
2021

2022
	params->pgd_pa = kvm_mmu_get_httbr();
2023
	if (is_protected_kvm_enabled())
2024
		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2025
	else
2026
		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2027
	if (cpus_have_final_cap(ARM64_KVM_HVHE))
2028
		params->hcr_el2 |= HCR_E2H;
2029
	params->vttbr = params->vtcr = 0;
2030

2031
	/*
2032
	 * Flush the init params from the data cache because the struct will
2033
	 * be read while the MMU is off.
2034
	 */
2035
	kvm_flush_dcache_to_poc(params, sizeof(*params));
2036
}
2037

2038
static void hyp_install_host_vector(void)
2039
{
2040
	struct kvm_nvhe_init_params *params;
2041
	struct arm_smccc_res res;
2042

2043
	/* Switch from the HYP stub to our own HYP init vector */
2044
	__hyp_set_vectors(kvm_get_idmap_vector());
2045

2046
	/*
2047
	 * Call initialization code, and switch to the full blown HYP code.
2048
	 * If the cpucaps haven't been finalized yet, something has gone very
2049
	 * wrong, and hyp will crash and burn when it uses any
2050
	 * cpus_have_*_cap() wrapper.
2051
	 */
2052
	BUG_ON(!system_capabilities_finalized());
2053
	params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2054
	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2055
	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2056
}
2057

2058
static void cpu_init_hyp_mode(void)
2059
{
2060
	hyp_install_host_vector();
2061

2062
	/*
2063
	 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2064
	 * at EL2.
2065
	 */
2066
	if (this_cpu_has_cap(ARM64_SSBS) &&
2067
	    arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2068
		kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2069
	}
2070
}
2071

2072
static void cpu_hyp_reset(void)
2073
{
2074
	if (!is_kernel_in_hyp_mode())
2075
		__hyp_reset_vectors();
2076
}
2077

2078
/*
2079
 * EL2 vectors can be mapped and rerouted in a number of ways,
2080
 * depending on the kernel configuration and CPU present:
2081
 *
2082
 * - If the CPU is affected by Spectre-v2, the hardening sequence is
2083
 *   placed in one of the vector slots, which is executed before jumping
2084
 *   to the real vectors.
2085
 *
2086
 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2087
 *   containing the hardening sequence is mapped next to the idmap page,
2088
 *   and executed before jumping to the real vectors.
2089
 *
2090
 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2091
 *   empty slot is selected, mapped next to the idmap page, and
2092
 *   executed before jumping to the real vectors.
2093
 *
2094
 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2095
 * VHE, as we don't have hypervisor-specific mappings. If the system
2096
 * is VHE and yet selects this capability, it will be ignored.
2097
 */
2098
static void cpu_set_hyp_vector(void)
2099
{
2100
	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2101
	void *vector = hyp_spectre_vector_selector[data->slot];
2102

2103
	if (!is_protected_kvm_enabled())
2104
		*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2105
	else
2106
		kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2107
}
2108

2109
static void cpu_hyp_init_context(void)
2110
{
2111
	kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2112
	kvm_init_host_debug_data();
2113

2114
	if (!is_kernel_in_hyp_mode())
2115
		cpu_init_hyp_mode();
2116
}
2117

2118
static void cpu_hyp_init_features(void)
2119
{
2120
	cpu_set_hyp_vector();
2121

2122
	if (is_kernel_in_hyp_mode()) {
2123
		kvm_timer_init_vhe();
2124
		kvm_debug_init_vhe();
2125
	}
2126

2127
	if (vgic_present)
2128
		kvm_vgic_init_cpu_hardware();
2129
}
2130

2131
static void cpu_hyp_reinit(void)
2132
{
2133
	cpu_hyp_reset();
2134
	cpu_hyp_init_context();
2135
	cpu_hyp_init_features();
2136
}
2137

2138
static void cpu_hyp_init(void *discard)
2139
{
2140
	if (!__this_cpu_read(kvm_hyp_initialized)) {
2141
		cpu_hyp_reinit();
2142
		__this_cpu_write(kvm_hyp_initialized, 1);
2143
	}
2144
}
2145

2146
static void cpu_hyp_uninit(void *discard)
2147
{
2148
	if (!is_protected_kvm_enabled() && __this_cpu_read(kvm_hyp_initialized)) {
2149
		cpu_hyp_reset();
2150
		__this_cpu_write(kvm_hyp_initialized, 0);
2151
	}
2152
}
2153

2154
int kvm_arch_enable_virtualization_cpu(void)
2155
{
2156
	/*
2157
	 * Most calls to this function are made with migration
2158
	 * disabled, but not with preemption disabled. The former is
2159
	 * enough to ensure correctness, but most of the helpers
2160
	 * expect the later and will throw a tantrum otherwise.
2161
	 */
2162
	preempt_disable();
2163

2164
	cpu_hyp_init(NULL);
2165

2166
	kvm_vgic_cpu_up();
2167
	kvm_timer_cpu_up();
2168

2169
	preempt_enable();
2170

2171
	return 0;
2172
}
2173

2174
void kvm_arch_disable_virtualization_cpu(void)
2175
{
2176
	kvm_timer_cpu_down();
2177
	kvm_vgic_cpu_down();
2178

2179
	if (!is_protected_kvm_enabled())
2180
		cpu_hyp_uninit(NULL);
2181
}
2182

2183
#ifdef CONFIG_CPU_PM
2184
static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2185
				    unsigned long cmd,
2186
				    void *v)
2187
{
2188
	/*
2189
	 * kvm_hyp_initialized is left with its old value over
2190
	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2191
	 * re-enable hyp.
2192
	 */
2193
	switch (cmd) {
2194
	case CPU_PM_ENTER:
2195
		if (__this_cpu_read(kvm_hyp_initialized))
2196
			/*
2197
			 * don't update kvm_hyp_initialized here
2198
			 * so that the hyp will be re-enabled
2199
			 * when we resume. See below.
2200
			 */
2201
			cpu_hyp_reset();
2202

2203
		return NOTIFY_OK;
2204
	case CPU_PM_ENTER_FAILED:
2205
	case CPU_PM_EXIT:
2206
		if (__this_cpu_read(kvm_hyp_initialized))
2207
			/* The hyp was enabled before suspend. */
2208
			cpu_hyp_reinit();
2209

2210
		return NOTIFY_OK;
2211

2212
	default:
2213
		return NOTIFY_DONE;
2214
	}
2215
}
2216

2217
static struct notifier_block hyp_init_cpu_pm_nb = {
2218
	.notifier_call = hyp_init_cpu_pm_notifier,
2219
};
2220

2221
static void __init hyp_cpu_pm_init(void)
2222
{
2223
	if (!is_protected_kvm_enabled())
2224
		cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2225
}
2226
static void __init hyp_cpu_pm_exit(void)
2227
{
2228
	if (!is_protected_kvm_enabled())
2229
		cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2230
}
2231
#else
2232
static inline void __init hyp_cpu_pm_init(void)
2233
{
2234
}
2235
static inline void __init hyp_cpu_pm_exit(void)
2236
{
2237
}
2238
#endif
2239

2240
static void __init init_cpu_logical_map(void)
2241
{
2242
	unsigned int cpu;
2243

2244
	/*
2245
	 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2246
	 * Only copy the set of online CPUs whose features have been checked
2247
	 * against the finalized system capabilities. The hypervisor will not
2248
	 * allow any other CPUs from the `possible` set to boot.
2249
	 */
2250
	for_each_online_cpu(cpu)
2251
		hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2252
}
2253

2254
#define init_psci_0_1_impl_state(config, what)	\
2255
	config.psci_0_1_ ## what ## _implemented = psci_ops.what
2256

2257
static bool __init init_psci_relay(void)
2258
{
2259
	/*
2260
	 * If PSCI has not been initialized, protected KVM cannot install
2261
	 * itself on newly booted CPUs.
2262
	 */
2263
	if (!psci_ops.get_version) {
2264
		kvm_err("Cannot initialize protected mode without PSCI\n");
2265
		return false;
2266
	}
2267

2268
	kvm_host_psci_config.version = psci_ops.get_version();
2269
	kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2270

2271
	if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2272
		kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2273
		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2274
		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2275
		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2276
		init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2277
	}
2278
	return true;
2279
}
2280

2281
static int __init init_subsystems(void)
2282
{
2283
	int err = 0;
2284

2285
	/*
2286
	 * Enable hardware so that subsystem initialisation can access EL2.
2287
	 */
2288
	on_each_cpu(cpu_hyp_init, NULL, 1);
2289

2290
	/*
2291
	 * Register CPU lower-power notifier
2292
	 */
2293
	hyp_cpu_pm_init();
2294

2295
	/*
2296
	 * Init HYP view of VGIC
2297
	 */
2298
	err = kvm_vgic_hyp_init();
2299
	switch (err) {
2300
	case 0:
2301
		vgic_present = true;
2302
		break;
2303
	case -ENODEV:
2304
	case -ENXIO:
2305
		/*
2306
		 * No VGIC? No pKVM for you.
2307
		 *
2308
		 * Protected mode assumes that VGICv3 is present, so no point
2309
		 * in trying to hobble along if vgic initialization fails.
2310
		 */
2311
		if (is_protected_kvm_enabled())
2312
			goto out;
2313

2314
		/*
2315
		 * Otherwise, userspace could choose to implement a GIC for its
2316
		 * guest on non-cooperative hardware.
2317
		 */
2318
		vgic_present = false;
2319
		err = 0;
2320
		break;
2321
	default:
2322
		goto out;
2323
	}
2324

2325
	if (kvm_mode == KVM_MODE_NV &&
2326
		!(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 ||
2327
				   kvm_vgic_global_state.has_gcie_v3_compat))) {
2328
		kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n");
2329
		err = -EINVAL;
2330
		goto out;
2331
	}
2332

2333
	/*
2334
	 * Init HYP architected timer support
2335
	 */
2336
	err = kvm_timer_hyp_init(vgic_present);
2337
	if (err)
2338
		goto out;
2339

2340
	kvm_register_perf_callbacks(NULL);
2341

2342
out:
2343
	if (err)
2344
		hyp_cpu_pm_exit();
2345

2346
	if (err || !is_protected_kvm_enabled())
2347
		on_each_cpu(cpu_hyp_uninit, NULL, 1);
2348

2349
	return err;
2350
}
2351

2352
static void __init teardown_subsystems(void)
2353
{
2354
	kvm_unregister_perf_callbacks();
2355
	hyp_cpu_pm_exit();
2356
}
2357

2358
static void __init teardown_hyp_mode(void)
2359
{
2360
	bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2361
	int cpu;
2362

2363
	free_hyp_pgds();
2364
	for_each_possible_cpu(cpu) {
2365
		if (per_cpu(kvm_hyp_initialized, cpu))
2366
			continue;
2367

2368
		free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2369

2370
		if (!kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu])
2371
			continue;
2372

2373
		if (free_sve) {
2374
			struct cpu_sve_state *sve_state;
2375

2376
			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2377
			free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2378
		}
2379

2380
		free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2381

2382
	}
2383
}
2384

2385
static int __init do_pkvm_init(u32 hyp_va_bits)
2386
{
2387
	void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2388
	int ret;
2389

2390
	preempt_disable();
2391
	cpu_hyp_init_context();
2392
	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2393
				num_possible_cpus(), kern_hyp_va(per_cpu_base),
2394
				hyp_va_bits);
2395
	cpu_hyp_init_features();
2396

2397
	/*
2398
	 * The stub hypercalls are now disabled, so set our local flag to
2399
	 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu().
2400
	 */
2401
	__this_cpu_write(kvm_hyp_initialized, 1);
2402
	preempt_enable();
2403

2404
	return ret;
2405
}
2406

2407
static u64 get_hyp_id_aa64pfr0_el1(void)
2408
{
2409
	/*
2410
	 * Track whether the system isn't affected by spectre/meltdown in the
2411
	 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2412
	 * Although this is per-CPU, we make it global for simplicity, e.g., not
2413
	 * to have to worry about vcpu migration.
2414
	 *
2415
	 * Unlike for non-protected VMs, userspace cannot override this for
2416
	 * protected VMs.
2417
	 */
2418
	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2419

2420
	val &= ~(ID_AA64PFR0_EL1_CSV2 |
2421
		 ID_AA64PFR0_EL1_CSV3);
2422

2423
	val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV2,
2424
			  arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2425
	val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV3,
2426
			  arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2427

2428
	return val;
2429
}
2430

2431
static void kvm_hyp_init_symbols(void)
2432
{
2433
	kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2434
	kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2435
	kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2436
	kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2437
	kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2438
	kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2439
	kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2440
	kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2441
	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2442
	kvm_nvhe_sym(__icache_flags) = __icache_flags;
2443
	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2444

2445
	/* Propagate the FGT state to the the nVHE side */
2446
	kvm_nvhe_sym(hfgrtr_masks)  = hfgrtr_masks;
2447
	kvm_nvhe_sym(hfgwtr_masks)  = hfgwtr_masks;
2448
	kvm_nvhe_sym(hfgitr_masks)  = hfgitr_masks;
2449
	kvm_nvhe_sym(hdfgrtr_masks) = hdfgrtr_masks;
2450
	kvm_nvhe_sym(hdfgwtr_masks) = hdfgwtr_masks;
2451
	kvm_nvhe_sym(hafgrtr_masks) = hafgrtr_masks;
2452
	kvm_nvhe_sym(hfgrtr2_masks) = hfgrtr2_masks;
2453
	kvm_nvhe_sym(hfgwtr2_masks) = hfgwtr2_masks;
2454
	kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks;
2455
	kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks;
2456
	kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks;
2457

2458
	/*
2459
	 * Flush entire BSS since part of its data containing init symbols is read
2460
	 * while the MMU is off.
2461
	 */
2462
	kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
2463
				kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
2464
}
2465

2466
static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2467
{
2468
	void *addr = phys_to_virt(hyp_mem_base);
2469
	int ret;
2470

2471
	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2472
	if (ret)
2473
		return ret;
2474

2475
	ret = do_pkvm_init(hyp_va_bits);
2476
	if (ret)
2477
		return ret;
2478

2479
	free_hyp_pgds();
2480

2481
	return 0;
2482
}
2483

2484
static int init_pkvm_host_sve_state(void)
2485
{
2486
	int cpu;
2487

2488
	if (!system_supports_sve())
2489
		return 0;
2490

2491
	/* Allocate pages for host sve state in protected mode. */
2492
	for_each_possible_cpu(cpu) {
2493
		struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2494

2495
		if (!page)
2496
			return -ENOMEM;
2497

2498
		per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2499
	}
2500

2501
	/*
2502
	 * Don't map the pages in hyp since these are only used in protected
2503
	 * mode, which will (re)create its own mapping when initialized.
2504
	 */
2505

2506
	return 0;
2507
}
2508

2509
/*
2510
 * Finalizes the initialization of hyp mode, once everything else is initialized
2511
 * and the initialziation process cannot fail.
2512
 */
2513
static void finalize_init_hyp_mode(void)
2514
{
2515
	int cpu;
2516

2517
	if (system_supports_sve() && is_protected_kvm_enabled()) {
2518
		for_each_possible_cpu(cpu) {
2519
			struct cpu_sve_state *sve_state;
2520

2521
			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2522
			per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2523
				kern_hyp_va(sve_state);
2524
		}
2525
	}
2526
}
2527

2528
static void pkvm_hyp_init_ptrauth(void)
2529
{
2530
	struct kvm_cpu_context *hyp_ctxt;
2531
	int cpu;
2532

2533
	for_each_possible_cpu(cpu) {
2534
		hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2535
		hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2536
		hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2537
		hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2538
		hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2539
		hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2540
		hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2541
		hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2542
		hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2543
		hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2544
		hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2545
	}
2546
}
2547

2548
/* Inits Hyp-mode on all online CPUs */
2549
static int __init init_hyp_mode(void)
2550
{
2551
	u32 hyp_va_bits;
2552
	int cpu;
2553
	int err = -ENOMEM;
2554

2555
	/*
2556
	 * The protected Hyp-mode cannot be initialized if the memory pool
2557
	 * allocation has failed.
2558
	 */
2559
	if (is_protected_kvm_enabled() && !hyp_mem_base)
2560
		goto out_err;
2561

2562
	/*
2563
	 * Allocate Hyp PGD and setup Hyp identity mapping
2564
	 */
2565
	err = kvm_mmu_init(&hyp_va_bits);
2566
	if (err)
2567
		goto out_err;
2568

2569
	/*
2570
	 * Allocate stack pages for Hypervisor-mode
2571
	 */
2572
	for_each_possible_cpu(cpu) {
2573
		unsigned long stack_base;
2574

2575
		stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2576
		if (!stack_base) {
2577
			err = -ENOMEM;
2578
			goto out_err;
2579
		}
2580

2581
		per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2582
	}
2583

2584
	/*
2585
	 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2586
	 */
2587
	for_each_possible_cpu(cpu) {
2588
		struct page *page;
2589
		void *page_addr;
2590

2591
		page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2592
		if (!page) {
2593
			err = -ENOMEM;
2594
			goto out_err;
2595
		}
2596

2597
		page_addr = page_address(page);
2598
		memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2599
		kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2600
	}
2601

2602
	/*
2603
	 * Map the Hyp-code called directly from the host
2604
	 */
2605
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2606
				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2607
	if (err) {
2608
		kvm_err("Cannot map world-switch code\n");
2609
		goto out_err;
2610
	}
2611

2612
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
2613
				  kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
2614
	if (err) {
2615
		kvm_err("Cannot map .hyp.data section\n");
2616
		goto out_err;
2617
	}
2618

2619
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2620
				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2621
	if (err) {
2622
		kvm_err("Cannot map .hyp.rodata section\n");
2623
		goto out_err;
2624
	}
2625

2626
	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2627
				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2628
	if (err) {
2629
		kvm_err("Cannot map rodata section\n");
2630
		goto out_err;
2631
	}
2632

2633
	/*
2634
	 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2635
	 * section thanks to an assertion in the linker script. Map it RW and
2636
	 * the rest of .bss RO.
2637
	 */
2638
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2639
				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2640
	if (err) {
2641
		kvm_err("Cannot map hyp bss section: %d\n", err);
2642
		goto out_err;
2643
	}
2644

2645
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2646
				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2647
	if (err) {
2648
		kvm_err("Cannot map bss section\n");
2649
		goto out_err;
2650
	}
2651

2652
	/*
2653
	 * Map the Hyp stack pages
2654
	 */
2655
	for_each_possible_cpu(cpu) {
2656
		struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2657
		char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2658

2659
		err = create_hyp_stack(__pa(stack_base), &params->stack_hyp_va);
2660
		if (err) {
2661
			kvm_err("Cannot map hyp stack\n");
2662
			goto out_err;
2663
		}
2664

2665
		/*
2666
		 * Save the stack PA in nvhe_init_params. This will be needed
2667
		 * to recreate the stack mapping in protected nVHE mode.
2668
		 * __hyp_pa() won't do the right thing there, since the stack
2669
		 * has been mapped in the flexible private VA space.
2670
		 */
2671
		params->stack_pa = __pa(stack_base);
2672
	}
2673

2674
	for_each_possible_cpu(cpu) {
2675
		char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2676
		char *percpu_end = percpu_begin + nvhe_percpu_size();
2677

2678
		/* Map Hyp percpu pages */
2679
		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2680
		if (err) {
2681
			kvm_err("Cannot map hyp percpu region\n");
2682
			goto out_err;
2683
		}
2684

2685
		/* Prepare the CPU initialization parameters */
2686
		cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2687
	}
2688

2689
	kvm_hyp_init_symbols();
2690

2691
	if (is_protected_kvm_enabled()) {
2692
		if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2693
		    cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2694
			pkvm_hyp_init_ptrauth();
2695

2696
		init_cpu_logical_map();
2697

2698
		if (!init_psci_relay()) {
2699
			err = -ENODEV;
2700
			goto out_err;
2701
		}
2702

2703
		err = init_pkvm_host_sve_state();
2704
		if (err)
2705
			goto out_err;
2706

2707
		err = kvm_hyp_init_protection(hyp_va_bits);
2708
		if (err) {
2709
			kvm_err("Failed to init hyp memory protection\n");
2710
			goto out_err;
2711
		}
2712
	}
2713

2714
	return 0;
2715

2716
out_err:
2717
	teardown_hyp_mode();
2718
	kvm_err("error initializing Hyp mode: %d\n", err);
2719
	return err;
2720
}
2721

2722
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2723
{
2724
	struct kvm_vcpu *vcpu = NULL;
2725
	struct kvm_mpidr_data *data;
2726
	unsigned long i;
2727

2728
	mpidr &= MPIDR_HWID_BITMASK;
2729

2730
	rcu_read_lock();
2731
	data = rcu_dereference(kvm->arch.mpidr_data);
2732

2733
	if (data) {
2734
		u16 idx = kvm_mpidr_index(data, mpidr);
2735

2736
		vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2737
		if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2738
			vcpu = NULL;
2739
	}
2740

2741
	rcu_read_unlock();
2742

2743
	if (vcpu)
2744
		return vcpu;
2745

2746
	kvm_for_each_vcpu(i, vcpu, kvm) {
2747
		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2748
			return vcpu;
2749
	}
2750
	return NULL;
2751
}
2752

2753
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2754
{
2755
	return irqchip_in_kernel(kvm);
2756
}
2757

2758
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2759
				      struct irq_bypass_producer *prod)
2760
{
2761
	struct kvm_kernel_irqfd *irqfd =
2762
		container_of(cons, struct kvm_kernel_irqfd, consumer);
2763
	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2764

2765
	/*
2766
	 * The only thing we have a chance of directly-injecting is LPIs. Maybe
2767
	 * one day...
2768
	 */
2769
	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2770
		return 0;
2771

2772
	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2773
					  &irqfd->irq_entry);
2774
}
2775

2776
void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2777
				      struct irq_bypass_producer *prod)
2778
{
2779
	struct kvm_kernel_irqfd *irqfd =
2780
		container_of(cons, struct kvm_kernel_irqfd, consumer);
2781
	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2782

2783
	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2784
		return;
2785

2786
	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq);
2787
}
2788

2789
void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
2790
				   struct kvm_kernel_irq_routing_entry *old,
2791
				   struct kvm_kernel_irq_routing_entry *new)
2792
{
2793
	if (old->type == KVM_IRQ_ROUTING_MSI &&
2794
	    new->type == KVM_IRQ_ROUTING_MSI &&
2795
	    !memcmp(&old->msi, &new->msi, sizeof(new->msi)))
2796
		return;
2797

2798
	/*
2799
	 * Remapping the vLPI requires taking the its_lock mutex to resolve
2800
	 * the new translation. We're in spinlock land at this point, so no
2801
	 * chance of resolving the translation.
2802
	 *
2803
	 * Unmap the vLPI and fall back to software LPI injection.
2804
	 */
2805
	return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq);
2806
}
2807

2808
void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2809
{
2810
	struct kvm_kernel_irqfd *irqfd =
2811
		container_of(cons, struct kvm_kernel_irqfd, consumer);
2812

2813
	kvm_arm_halt_guest(irqfd->kvm);
2814
}
2815

2816
void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2817
{
2818
	struct kvm_kernel_irqfd *irqfd =
2819
		container_of(cons, struct kvm_kernel_irqfd, consumer);
2820

2821
	kvm_arm_resume_guest(irqfd->kvm);
2822
}
2823

2824
/* Initialize Hyp-mode and memory mappings on all CPUs */
2825
static __init int kvm_arm_init(void)
2826
{
2827
	int err;
2828
	bool in_hyp_mode;
2829

2830
	if (!is_hyp_mode_available()) {
2831
		kvm_info("HYP mode not available\n");
2832
		return -ENODEV;
2833
	}
2834

2835
	if (kvm_get_mode() == KVM_MODE_NONE) {
2836
		kvm_info("KVM disabled from command line\n");
2837
		return -ENODEV;
2838
	}
2839

2840
	err = kvm_sys_reg_table_init();
2841
	if (err) {
2842
		kvm_info("Error initializing system register tables");
2843
		return err;
2844
	}
2845

2846
	in_hyp_mode = is_kernel_in_hyp_mode();
2847

2848
	if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2849
	    cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2850
		kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2851
			 "Only trusted guests should be used on this system.\n");
2852

2853
	err = kvm_set_ipa_limit();
2854
	if (err)
2855
		return err;
2856

2857
	err = kvm_arm_init_sve();
2858
	if (err)
2859
		return err;
2860

2861
	err = kvm_arm_vmid_alloc_init();
2862
	if (err) {
2863
		kvm_err("Failed to initialize VMID allocator.\n");
2864
		return err;
2865
	}
2866

2867
	if (!in_hyp_mode) {
2868
		err = init_hyp_mode();
2869
		if (err)
2870
			goto out_err;
2871
	}
2872

2873
	err = kvm_init_vector_slots();
2874
	if (err) {
2875
		kvm_err("Cannot initialise vector slots\n");
2876
		goto out_hyp;
2877
	}
2878

2879
	err = init_subsystems();
2880
	if (err)
2881
		goto out_hyp;
2882

2883
	kvm_info("%s%sVHE%s mode initialized successfully\n",
2884
		 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2885
				     "Protected " : "Hyp "),
2886
		 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2887
				     "h" : "n"),
2888
		 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": "");
2889

2890
	/*
2891
	 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2892
	 * hypervisor protection is finalized.
2893
	 */
2894
	err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2895
	if (err)
2896
		goto out_subs;
2897

2898
	/*
2899
	 * This should be called after initialization is done and failure isn't
2900
	 * possible anymore.
2901
	 */
2902
	if (!in_hyp_mode)
2903
		finalize_init_hyp_mode();
2904

2905
	kvm_arm_initialised = true;
2906

2907
	return 0;
2908

2909
out_subs:
2910
	teardown_subsystems();
2911
out_hyp:
2912
	if (!in_hyp_mode)
2913
		teardown_hyp_mode();
2914
out_err:
2915
	kvm_arm_vmid_alloc_free();
2916
	return err;
2917
}
2918

2919
static int __init early_kvm_mode_cfg(char *arg)
2920
{
2921
	if (!arg)
2922
		return -EINVAL;
2923

2924
	if (strcmp(arg, "none") == 0) {
2925
		kvm_mode = KVM_MODE_NONE;
2926
		return 0;
2927
	}
2928

2929
	if (!is_hyp_mode_available()) {
2930
		pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2931
		return 0;
2932
	}
2933

2934
	if (strcmp(arg, "protected") == 0) {
2935
		if (!is_kernel_in_hyp_mode())
2936
			kvm_mode = KVM_MODE_PROTECTED;
2937
		else
2938
			pr_warn_once("Protected KVM not available with VHE\n");
2939

2940
		return 0;
2941
	}
2942

2943
	if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2944
		kvm_mode = KVM_MODE_DEFAULT;
2945
		return 0;
2946
	}
2947

2948
	if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2949
		kvm_mode = KVM_MODE_NV;
2950
		return 0;
2951
	}
2952

2953
	return -EINVAL;
2954
}
2955
early_param("kvm-arm.mode", early_kvm_mode_cfg);
2956

2957
static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2958
{
2959
	if (!arg)
2960
		return -EINVAL;
2961

2962
	if (strcmp(arg, "trap") == 0) {
2963
		*p = KVM_WFX_TRAP;
2964
		return 0;
2965
	}
2966

2967
	if (strcmp(arg, "notrap") == 0) {
2968
		*p = KVM_WFX_NOTRAP;
2969
		return 0;
2970
	}
2971

2972
	return -EINVAL;
2973
}
2974

2975
static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2976
{
2977
	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2978
}
2979
early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
2980

2981
static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
2982
{
2983
	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
2984
}
2985
early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
2986

2987
enum kvm_mode kvm_get_mode(void)
2988
{
2989
	return kvm_mode;
2990
}
2991

2992
module_init(kvm_arm_init);
2993

2994