1
// SPDX-License-Identifier: GPL-2.0
2
#include <linux/backing-dev.h>
3
#include <linux/falloc.h>
4
#include <linux/kvm_host.h>
5
#include <linux/pagemap.h>
6
#include <linux/anon_inodes.h>
7

8
#include "kvm_mm.h"
9

10
struct kvm_gmem {
11
	struct kvm *kvm;
12
	struct xarray bindings;
13
	struct list_head entry;
14
};
15

16
/**
17
 * folio_file_pfn - like folio_file_page, but return a pfn.
18
 * @folio: The folio which contains this index.
19
 * @index: The index we want to look up.
20
 *
21
 * Return: The pfn for this index.
22
 */
23
static inline kvm_pfn_t folio_file_pfn(struct folio *folio, pgoff_t index)
24
{
25
	return folio_pfn(folio) + (index & (folio_nr_pages(folio) - 1));
26
}
27

28
static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
29
				    pgoff_t index, struct folio *folio)
30
{
31
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
32
	kvm_pfn_t pfn = folio_file_pfn(folio, index);
33
	gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff;
34
	int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio));
35
	if (rc) {
36
		pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n",
37
				    index, gfn, pfn, rc);
38
		return rc;
39
	}
40
#endif
41

42
	return 0;
43
}
44

45
static inline void kvm_gmem_mark_prepared(struct folio *folio)
46
{
47
	folio_mark_uptodate(folio);
48
}
49

50
/*
51
 * Process @folio, which contains @gfn, so that the guest can use it.
52
 * The folio must be locked and the gfn must be contained in @slot.
53
 * On successful return the guest sees a zero page so as to avoid
54
 * leaking host data and the up-to-date flag is set.
55
 */
56
static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
57
				  gfn_t gfn, struct folio *folio)
58
{
59
	unsigned long nr_pages, i;
60
	pgoff_t index;
61
	int r;
62

63
	nr_pages = folio_nr_pages(folio);
64
	for (i = 0; i < nr_pages; i++)
65
		clear_highpage(folio_page(folio, i));
66

67
	/*
68
	 * Preparing huge folios should always be safe, since it should
69
	 * be possible to split them later if needed.
70
	 *
71
	 * Right now the folio order is always going to be zero, but the
72
	 * code is ready for huge folios.  The only assumption is that
73
	 * the base pgoff of memslots is naturally aligned with the
74
	 * requested page order, ensuring that huge folios can also use
75
	 * huge page table entries for GPA->HPA mapping.
76
	 *
77
	 * The order will be passed when creating the guest_memfd, and
78
	 * checked when creating memslots.
79
	 */
80
	WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, 1 << folio_order(folio)));
81
	index = gfn - slot->base_gfn + slot->gmem.pgoff;
82
	index = ALIGN_DOWN(index, 1 << folio_order(folio));
83
	r = __kvm_gmem_prepare_folio(kvm, slot, index, folio);
84
	if (!r)
85
		kvm_gmem_mark_prepared(folio);
86

87
	return r;
88
}
89

90
/*
91
 * Returns a locked folio on success.  The caller is responsible for
92
 * setting the up-to-date flag before the memory is mapped into the guest.
93
 * There is no backing storage for the memory, so the folio will remain
94
 * up-to-date until it's removed.
95
 *
96
 * Ignore accessed, referenced, and dirty flags.  The memory is
97
 * unevictable and there is no storage to write back to.
98
 */
99
static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
100
{
101
	/* TODO: Support huge pages. */
102
	return filemap_grab_folio(inode->i_mapping, index);
103
}
104

105
static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
106
				      pgoff_t end)
107
{
108
	bool flush = false, found_memslot = false;
109
	struct kvm_memory_slot *slot;
110
	struct kvm *kvm = gmem->kvm;
111
	unsigned long index;
112

113
	xa_for_each_range(&gmem->bindings, index, slot, start, end - 1) {
114
		pgoff_t pgoff = slot->gmem.pgoff;
115

116
		struct kvm_gfn_range gfn_range = {
117
			.start = slot->base_gfn + max(pgoff, start) - pgoff,
118
			.end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff,
119
			.slot = slot,
120
			.may_block = true,
121
			/* guest memfd is relevant to only private mappings. */
122
			.attr_filter = KVM_FILTER_PRIVATE,
123
		};
124

125
		if (!found_memslot) {
126
			found_memslot = true;
127

128
			KVM_MMU_LOCK(kvm);
129
			kvm_mmu_invalidate_begin(kvm);
130
		}
131

132
		flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
133
	}
134

135
	if (flush)
136
		kvm_flush_remote_tlbs(kvm);
137

138
	if (found_memslot)
139
		KVM_MMU_UNLOCK(kvm);
140
}
141

142
static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,
143
				    pgoff_t end)
144
{
145
	struct kvm *kvm = gmem->kvm;
146

147
	if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
148
		KVM_MMU_LOCK(kvm);
149
		kvm_mmu_invalidate_end(kvm);
150
		KVM_MMU_UNLOCK(kvm);
151
	}
152
}
153

154
static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
155
{
156
	struct list_head *gmem_list = &inode->i_mapping->i_private_list;
157
	pgoff_t start = offset >> PAGE_SHIFT;
158
	pgoff_t end = (offset + len) >> PAGE_SHIFT;
159
	struct kvm_gmem *gmem;
160

161
	/*
162
	 * Bindings must be stable across invalidation to ensure the start+end
163
	 * are balanced.
164
	 */
165
	filemap_invalidate_lock(inode->i_mapping);
166

167
	list_for_each_entry(gmem, gmem_list, entry)
168
		kvm_gmem_invalidate_begin(gmem, start, end);
169

170
	truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);
171

172
	list_for_each_entry(gmem, gmem_list, entry)
173
		kvm_gmem_invalidate_end(gmem, start, end);
174

175
	filemap_invalidate_unlock(inode->i_mapping);
176

177
	return 0;
178
}
179

180
static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
181
{
182
	struct address_space *mapping = inode->i_mapping;
183
	pgoff_t start, index, end;
184
	int r;
185

186
	/* Dedicated guest is immutable by default. */
187
	if (offset + len > i_size_read(inode))
188
		return -EINVAL;
189

190
	filemap_invalidate_lock_shared(mapping);
191

192
	start = offset >> PAGE_SHIFT;
193
	end = (offset + len) >> PAGE_SHIFT;
194

195
	r = 0;
196
	for (index = start; index < end; ) {
197
		struct folio *folio;
198

199
		if (signal_pending(current)) {
200
			r = -EINTR;
201
			break;
202
		}
203

204
		folio = kvm_gmem_get_folio(inode, index);
205
		if (IS_ERR(folio)) {
206
			r = PTR_ERR(folio);
207
			break;
208
		}
209

210
		index = folio_next_index(folio);
211

212
		folio_unlock(folio);
213
		folio_put(folio);
214

215
		/* 64-bit only, wrapping the index should be impossible. */
216
		if (WARN_ON_ONCE(!index))
217
			break;
218

219
		cond_resched();
220
	}
221

222
	filemap_invalidate_unlock_shared(mapping);
223

224
	return r;
225
}
226

227
static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
228
			       loff_t len)
229
{
230
	int ret;
231

232
	if (!(mode & FALLOC_FL_KEEP_SIZE))
233
		return -EOPNOTSUPP;
234

235
	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
236
		return -EOPNOTSUPP;
237

238
	if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
239
		return -EINVAL;
240

241
	if (mode & FALLOC_FL_PUNCH_HOLE)
242
		ret = kvm_gmem_punch_hole(file_inode(file), offset, len);
243
	else
244
		ret = kvm_gmem_allocate(file_inode(file), offset, len);
245

246
	if (!ret)
247
		file_modified(file);
248
	return ret;
249
}
250

251
static int kvm_gmem_release(struct inode *inode, struct file *file)
252
{
253
	struct kvm_gmem *gmem = file->private_data;
254
	struct kvm_memory_slot *slot;
255
	struct kvm *kvm = gmem->kvm;
256
	unsigned long index;
257

258
	/*
259
	 * Prevent concurrent attempts to *unbind* a memslot.  This is the last
260
	 * reference to the file and thus no new bindings can be created, but
261
	 * dereferencing the slot for existing bindings needs to be protected
262
	 * against memslot updates, specifically so that unbind doesn't race
263
	 * and free the memslot (kvm_gmem_get_file() will return NULL).
264
	 *
265
	 * Since .release is called only when the reference count is zero,
266
	 * after which file_ref_get() and get_file_active() fail,
267
	 * kvm_gmem_get_pfn() cannot be using the file concurrently.
268
	 * file_ref_put() provides a full barrier, and get_file_active() the
269
	 * matching acquire barrier.
270
	 */
271
	mutex_lock(&kvm->slots_lock);
272

273
	filemap_invalidate_lock(inode->i_mapping);
274

275
	xa_for_each(&gmem->bindings, index, slot)
276
		WRITE_ONCE(slot->gmem.file, NULL);
277

278
	/*
279
	 * All in-flight operations are gone and new bindings can be created.
280
	 * Zap all SPTEs pointed at by this file.  Do not free the backing
281
	 * memory, as its lifetime is associated with the inode, not the file.
282
	 */
283
	kvm_gmem_invalidate_begin(gmem, 0, -1ul);
284
	kvm_gmem_invalidate_end(gmem, 0, -1ul);
285

286
	list_del(&gmem->entry);
287

288
	filemap_invalidate_unlock(inode->i_mapping);
289

290
	mutex_unlock(&kvm->slots_lock);
291

292
	xa_destroy(&gmem->bindings);
293
	kfree(gmem);
294

295
	kvm_put_kvm(kvm);
296

297
	return 0;
298
}
299

300
static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)
301
{
302
	/*
303
	 * Do not return slot->gmem.file if it has already been closed;
304
	 * there might be some time between the last fput() and when
305
	 * kvm_gmem_release() clears slot->gmem.file.
306
	 */
307
	return get_file_active(&slot->gmem.file);
308
}
309

310
static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn)
311
{
312
	return gfn - slot->base_gfn + slot->gmem.pgoff;
313
}
314

315
static bool kvm_gmem_supports_mmap(struct inode *inode)
316
{
317
	const u64 flags = (u64)inode->i_private;
318

319
	return flags & GUEST_MEMFD_FLAG_MMAP;
320
}
321

322
static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf)
323
{
324
	struct inode *inode = file_inode(vmf->vma->vm_file);
325
	struct folio *folio;
326
	vm_fault_t ret = VM_FAULT_LOCKED;
327

328
	if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))
329
		return VM_FAULT_SIGBUS;
330

331
	folio = kvm_gmem_get_folio(inode, vmf->pgoff);
332
	if (IS_ERR(folio)) {
333
		int err = PTR_ERR(folio);
334

335
		if (err == -EAGAIN)
336
			return VM_FAULT_RETRY;
337

338
		return vmf_error(err);
339
	}
340

341
	if (WARN_ON_ONCE(folio_test_large(folio))) {
342
		ret = VM_FAULT_SIGBUS;
343
		goto out_folio;
344
	}
345

346
	if (!folio_test_uptodate(folio)) {
347
		clear_highpage(folio_page(folio, 0));
348
		kvm_gmem_mark_prepared(folio);
349
	}
350

351
	vmf->page = folio_file_page(folio, vmf->pgoff);
352

353
out_folio:
354
	if (ret != VM_FAULT_LOCKED) {
355
		folio_unlock(folio);
356
		folio_put(folio);
357
	}
358

359
	return ret;
360
}
361

362
static const struct vm_operations_struct kvm_gmem_vm_ops = {
363
	.fault = kvm_gmem_fault_user_mapping,
364
};
365

366
static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma)
367
{
368
	if (!kvm_gmem_supports_mmap(file_inode(file)))
369
		return -ENODEV;
370

371
	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) !=
372
	    (VM_SHARED | VM_MAYSHARE)) {
373
		return -EINVAL;
374
	}
375

376
	vma->vm_ops = &kvm_gmem_vm_ops;
377

378
	return 0;
379
}
380

381
static struct file_operations kvm_gmem_fops = {
382
	.mmap		= kvm_gmem_mmap,
383
	.open		= generic_file_open,
384
	.release	= kvm_gmem_release,
385
	.fallocate	= kvm_gmem_fallocate,
386
};
387

388
void kvm_gmem_init(struct module *module)
389
{
390
	kvm_gmem_fops.owner = module;
391
}
392

393
static int kvm_gmem_migrate_folio(struct address_space *mapping,
394
				  struct folio *dst, struct folio *src,
395
				  enum migrate_mode mode)
396
{
397
	WARN_ON_ONCE(1);
398
	return -EINVAL;
399
}
400

401
static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio)
402
{
403
	struct list_head *gmem_list = &mapping->i_private_list;
404
	struct kvm_gmem *gmem;
405
	pgoff_t start, end;
406

407
	filemap_invalidate_lock_shared(mapping);
408

409
	start = folio->index;
410
	end = start + folio_nr_pages(folio);
411

412
	list_for_each_entry(gmem, gmem_list, entry)
413
		kvm_gmem_invalidate_begin(gmem, start, end);
414

415
	/*
416
	 * Do not truncate the range, what action is taken in response to the
417
	 * error is userspace's decision (assuming the architecture supports
418
	 * gracefully handling memory errors).  If/when the guest attempts to
419
	 * access a poisoned page, kvm_gmem_get_pfn() will return -EHWPOISON,
420
	 * at which point KVM can either terminate the VM or propagate the
421
	 * error to userspace.
422
	 */
423

424
	list_for_each_entry(gmem, gmem_list, entry)
425
		kvm_gmem_invalidate_end(gmem, start, end);
426

427
	filemap_invalidate_unlock_shared(mapping);
428

429
	return MF_DELAYED;
430
}
431

432
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
433
static void kvm_gmem_free_folio(struct folio *folio)
434
{
435
	struct page *page = folio_page(folio, 0);
436
	kvm_pfn_t pfn = page_to_pfn(page);
437
	int order = folio_order(folio);
438

439
	kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order));
440
}
441
#endif
442

443
static const struct address_space_operations kvm_gmem_aops = {
444
	.dirty_folio = noop_dirty_folio,
445
	.migrate_folio	= kvm_gmem_migrate_folio,
446
	.error_remove_folio = kvm_gmem_error_folio,
447
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
448
	.free_folio = kvm_gmem_free_folio,
449
#endif
450
};
451

452
static int kvm_gmem_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
453
			    struct iattr *attr)
454
{
455
	return -EINVAL;
456
}
457
static const struct inode_operations kvm_gmem_iops = {
458
	.setattr	= kvm_gmem_setattr,
459
};
460

461
bool __weak kvm_arch_supports_gmem_mmap(struct kvm *kvm)
462
{
463
	return true;
464
}
465

466
static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
467
{
468
	const char *anon_name = "[kvm-gmem]";
469
	struct kvm_gmem *gmem;
470
	struct inode *inode;
471
	struct file *file;
472
	int fd, err;
473

474
	fd = get_unused_fd_flags(0);
475
	if (fd < 0)
476
		return fd;
477

478
	gmem = kzalloc(sizeof(*gmem), GFP_KERNEL);
479
	if (!gmem) {
480
		err = -ENOMEM;
481
		goto err_fd;
482
	}
483

484
	file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem,
485
					 O_RDWR, NULL);
486
	if (IS_ERR(file)) {
487
		err = PTR_ERR(file);
488
		goto err_gmem;
489
	}
490

491
	file->f_flags |= O_LARGEFILE;
492

493
	inode = file->f_inode;
494
	WARN_ON(file->f_mapping != inode->i_mapping);
495

496
	inode->i_private = (void *)(unsigned long)flags;
497
	inode->i_op = &kvm_gmem_iops;
498
	inode->i_mapping->a_ops = &kvm_gmem_aops;
499
	inode->i_mode |= S_IFREG;
500
	inode->i_size = size;
501
	mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
502
	mapping_set_inaccessible(inode->i_mapping);
503
	/* Unmovable mappings are supposed to be marked unevictable as well. */
504
	WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
505

506
	kvm_get_kvm(kvm);
507
	gmem->kvm = kvm;
508
	xa_init(&gmem->bindings);
509
	list_add(&gmem->entry, &inode->i_mapping->i_private_list);
510

511
	fd_install(fd, file);
512
	return fd;
513

514
err_gmem:
515
	kfree(gmem);
516
err_fd:
517
	put_unused_fd(fd);
518
	return err;
519
}
520

521
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
522
{
523
	loff_t size = args->size;
524
	u64 flags = args->flags;
525
	u64 valid_flags = 0;
526

527
	if (kvm_arch_supports_gmem_mmap(kvm))
528
		valid_flags |= GUEST_MEMFD_FLAG_MMAP;
529

530
	if (flags & ~valid_flags)
531
		return -EINVAL;
532

533
	if (size <= 0 || !PAGE_ALIGNED(size))
534
		return -EINVAL;
535

536
	return __kvm_gmem_create(kvm, size, flags);
537
}
538

539
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
540
		  unsigned int fd, loff_t offset)
541
{
542
	loff_t size = slot->npages << PAGE_SHIFT;
543
	unsigned long start, end;
544
	struct kvm_gmem *gmem;
545
	struct inode *inode;
546
	struct file *file;
547
	int r = -EINVAL;
548

549
	BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
550

551
	file = fget(fd);
552
	if (!file)
553
		return -EBADF;
554

555
	if (file->f_op != &kvm_gmem_fops)
556
		goto err;
557

558
	gmem = file->private_data;
559
	if (gmem->kvm != kvm)
560
		goto err;
561

562
	inode = file_inode(file);
563

564
	if (offset < 0 || !PAGE_ALIGNED(offset) ||
565
	    offset + size > i_size_read(inode))
566
		goto err;
567

568
	filemap_invalidate_lock(inode->i_mapping);
569

570
	start = offset >> PAGE_SHIFT;
571
	end = start + slot->npages;
572

573
	if (!xa_empty(&gmem->bindings) &&
574
	    xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
575
		filemap_invalidate_unlock(inode->i_mapping);
576
		goto err;
577
	}
578

579
	/*
580
	 * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so
581
	 * kvm_gmem_bind() must occur on a new memslot.  Because the memslot
582
	 * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file.
583
	 */
584
	WRITE_ONCE(slot->gmem.file, file);
585
	slot->gmem.pgoff = start;
586
	if (kvm_gmem_supports_mmap(inode))
587
		slot->flags |= KVM_MEMSLOT_GMEM_ONLY;
588

589
	xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL);
590
	filemap_invalidate_unlock(inode->i_mapping);
591

592
	/*
593
	 * Drop the reference to the file, even on success.  The file pins KVM,
594
	 * not the other way 'round.  Active bindings are invalidated if the
595
	 * file is closed before memslots are destroyed.
596
	 */
597
	r = 0;
598
err:
599
	fput(file);
600
	return r;
601
}
602

603
void kvm_gmem_unbind(struct kvm_memory_slot *slot)
604
{
605
	unsigned long start = slot->gmem.pgoff;
606
	unsigned long end = start + slot->npages;
607
	struct kvm_gmem *gmem;
608
	struct file *file;
609

610
	/*
611
	 * Nothing to do if the underlying file was already closed (or is being
612
	 * closed right now), kvm_gmem_release() invalidates all bindings.
613
	 */
614
	file = kvm_gmem_get_file(slot);
615
	if (!file)
616
		return;
617

618
	gmem = file->private_data;
619

620
	filemap_invalidate_lock(file->f_mapping);
621
	xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL);
622

623
	/*
624
	 * synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn()
625
	 * cannot see this memslot.
626
	 */
627
	WRITE_ONCE(slot->gmem.file, NULL);
628
	filemap_invalidate_unlock(file->f_mapping);
629

630
	fput(file);
631
}
632

633
/* Returns a locked folio on success.  */
634
static struct folio *__kvm_gmem_get_pfn(struct file *file,
635
					struct kvm_memory_slot *slot,
636
					pgoff_t index, kvm_pfn_t *pfn,
637
					bool *is_prepared, int *max_order)
638
{
639
	struct file *gmem_file = READ_ONCE(slot->gmem.file);
640
	struct kvm_gmem *gmem = file->private_data;
641
	struct folio *folio;
642

643
	if (file != gmem_file) {
644
		WARN_ON_ONCE(gmem_file);
645
		return ERR_PTR(-EFAULT);
646
	}
647

648
	gmem = file->private_data;
649
	if (xa_load(&gmem->bindings, index) != slot) {
650
		WARN_ON_ONCE(xa_load(&gmem->bindings, index));
651
		return ERR_PTR(-EIO);
652
	}
653

654
	folio = kvm_gmem_get_folio(file_inode(file), index);
655
	if (IS_ERR(folio))
656
		return folio;
657

658
	if (folio_test_hwpoison(folio)) {
659
		folio_unlock(folio);
660
		folio_put(folio);
661
		return ERR_PTR(-EHWPOISON);
662
	}
663

664
	*pfn = folio_file_pfn(folio, index);
665
	if (max_order)
666
		*max_order = 0;
667

668
	*is_prepared = folio_test_uptodate(folio);
669
	return folio;
670
}
671

672
int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
673
		     gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
674
		     int *max_order)
675
{
676
	pgoff_t index = kvm_gmem_get_index(slot, gfn);
677
	struct file *file = kvm_gmem_get_file(slot);
678
	struct folio *folio;
679
	bool is_prepared = false;
680
	int r = 0;
681

682
	if (!file)
683
		return -EFAULT;
684

685
	folio = __kvm_gmem_get_pfn(file, slot, index, pfn, &is_prepared, max_order);
686
	if (IS_ERR(folio)) {
687
		r = PTR_ERR(folio);
688
		goto out;
689
	}
690

691
	if (!is_prepared)
692
		r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
693

694
	folio_unlock(folio);
695

696
	if (!r)
697
		*page = folio_file_page(folio, index);
698
	else
699
		folio_put(folio);
700

701
out:
702
	fput(file);
703
	return r;
704
}
705
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);
706

707
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE
708
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages,
709
		       kvm_gmem_populate_cb post_populate, void *opaque)
710
{
711
	struct file *file;
712
	struct kvm_memory_slot *slot;
713
	void __user *p;
714

715
	int ret = 0, max_order;
716
	long i;
717

718
	lockdep_assert_held(&kvm->slots_lock);
719

720
	if (WARN_ON_ONCE(npages <= 0))
721
		return -EINVAL;
722

723
	slot = gfn_to_memslot(kvm, start_gfn);
724
	if (!kvm_slot_has_gmem(slot))
725
		return -EINVAL;
726

727
	file = kvm_gmem_get_file(slot);
728
	if (!file)
729
		return -EFAULT;
730

731
	filemap_invalidate_lock(file->f_mapping);
732

733
	npages = min_t(ulong, slot->npages - (start_gfn - slot->base_gfn), npages);
734
	for (i = 0; i < npages; i += (1 << max_order)) {
735
		struct folio *folio;
736
		gfn_t gfn = start_gfn + i;
737
		pgoff_t index = kvm_gmem_get_index(slot, gfn);
738
		bool is_prepared = false;
739
		kvm_pfn_t pfn;
740

741
		if (signal_pending(current)) {
742
			ret = -EINTR;
743
			break;
744
		}
745

746
		folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, &is_prepared, &max_order);
747
		if (IS_ERR(folio)) {
748
			ret = PTR_ERR(folio);
749
			break;
750
		}
751

752
		if (is_prepared) {
753
			folio_unlock(folio);
754
			folio_put(folio);
755
			ret = -EEXIST;
756
			break;
757
		}
758

759
		folio_unlock(folio);
760
		WARN_ON(!IS_ALIGNED(gfn, 1 << max_order) ||
761
			(npages - i) < (1 << max_order));
762

763
		ret = -EINVAL;
764
		while (!kvm_range_has_memory_attributes(kvm, gfn, gfn + (1 << max_order),
765
							KVM_MEMORY_ATTRIBUTE_PRIVATE,
766
							KVM_MEMORY_ATTRIBUTE_PRIVATE)) {
767
			if (!max_order)
768
				goto put_folio_and_exit;
769
			max_order--;
770
		}
771

772
		p = src ? src + i * PAGE_SIZE : NULL;
773
		ret = post_populate(kvm, gfn, pfn, p, max_order, opaque);
774
		if (!ret)
775
			kvm_gmem_mark_prepared(folio);
776

777
put_folio_and_exit:
778
		folio_put(folio);
779
		if (ret)
780
			break;
781
	}
782

783
	filemap_invalidate_unlock(file->f_mapping);
784

785
	fput(file);
786
	return ret && !i ? ret : i;
787
}
788
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_populate);
789
#endif
790

791