1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Copyright (C) 2024 Loongson Technology Corporation Limited
4
 */
5

6
#include <linux/kvm_host.h>
7
#include <asm/kvm_ipi.h>
8
#include <asm/kvm_vcpu.h>
9

10
static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data)
11
{
12
	uint32_t status;
13
	struct kvm_interrupt irq;
14

15
	spin_lock(&vcpu->arch.ipi_state.lock);
16
	status = vcpu->arch.ipi_state.status;
17
	vcpu->arch.ipi_state.status |= data;
18
	spin_unlock(&vcpu->arch.ipi_state.lock);
19
	if ((status == 0) && data) {
20
		irq.irq = LARCH_INT_IPI;
21
		kvm_vcpu_ioctl_interrupt(vcpu, &irq);
22
	}
23
}
24

25
static void ipi_send(struct kvm *kvm, uint64_t data)
26
{
27
	int cpu;
28
	struct kvm_vcpu *vcpu;
29

30
	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
31
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
32
	if (unlikely(vcpu == NULL)) {
33
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
34
		return;
35
	}
36

37
	ipi_set(vcpu, BIT(data & 0x1f));
38
}
39

40
static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data)
41
{
42
	uint32_t status;
43
	struct kvm_interrupt irq;
44

45
	spin_lock(&vcpu->arch.ipi_state.lock);
46
	vcpu->arch.ipi_state.status &= ~data;
47
	status = vcpu->arch.ipi_state.status;
48
	spin_unlock(&vcpu->arch.ipi_state.lock);
49
	if (status == 0) {
50
		irq.irq = -LARCH_INT_IPI;
51
		kvm_vcpu_ioctl_interrupt(vcpu, &irq);
52
	}
53
}
54

55
static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len)
56
{
57
	uint64_t data = 0;
58

59
	spin_lock(&vcpu->arch.ipi_state.lock);
60
	data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20));
61
	spin_unlock(&vcpu->arch.ipi_state.lock);
62

63
	switch (len) {
64
	case 1:
65
		return data & 0xff;
66
	case 2:
67
		return data & 0xffff;
68
	case 4:
69
		return data & 0xffffffff;
70
	case 8:
71
		return data;
72
	default:
73
		kvm_err("%s: unknown data len: %d\n", __func__, len);
74
		return 0;
75
	}
76
}
77

78
static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len)
79
{
80
	void *pbuf;
81

82
	spin_lock(&vcpu->arch.ipi_state.lock);
83
	pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20);
84

85
	switch (len) {
86
	case 1:
87
		*(unsigned char *)pbuf = (unsigned char)data;
88
		break;
89
	case 2:
90
		*(unsigned short *)pbuf = (unsigned short)data;
91
		break;
92
	case 4:
93
		*(unsigned int *)pbuf = (unsigned int)data;
94
		break;
95
	case 8:
96
		*(unsigned long *)pbuf = (unsigned long)data;
97
		break;
98
	default:
99
		kvm_err("%s: unknown data len: %d\n", __func__, len);
100
	}
101
	spin_unlock(&vcpu->arch.ipi_state.lock);
102
}
103

104
static int mail_send(struct kvm *kvm, uint64_t data)
105
{
106
	int i, cpu, mailbox, offset;
107
	uint32_t val = 0, mask = 0;
108
	struct kvm_vcpu *vcpu;
109

110
	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
111
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
112
	if (unlikely(vcpu == NULL)) {
113
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
114
		return -EINVAL;
115
	}
116
	mailbox = ((data & 0xffffffff) >> 2) & 0x7;
117
	offset = IOCSR_IPI_BUF_20 + mailbox * 4;
118
	if ((data >> 27) & 0xf) {
119
		val = read_mailbox(vcpu, offset, 4);
120
		for (i = 0; i < 4; i++)
121
			if (data & (BIT(27 + i)))
122
				mask |= (0xff << (i * 8));
123
		val &= mask;
124
	}
125

126
	val |= ((uint32_t)(data >> 32) & ~mask);
127
	write_mailbox(vcpu, offset, val, 4);
128

129
	return 0;
130
}
131

132
static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data)
133
{
134
	int i, idx, ret;
135
	uint64_t val = 0, mask = 0;
136

137
	/*
138
	 * Bit 27-30 is mask for byte writing.
139
	 * If the mask is 0, we need not to do anything.
140
	 */
141
	if ((data >> 27) & 0xf) {
142
		/* Read the old val */
143
		idx = srcu_read_lock(&vcpu->kvm->srcu);
144
		ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
145
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
146
		if (unlikely(ret)) {
147
			kvm_err("%s: : read data from addr %llx failed\n", __func__, addr);
148
			return ret;
149
		}
150
		/* Construct the mask by scanning the bit 27-30 */
151
		for (i = 0; i < 4; i++) {
152
			if (data & (BIT(27 + i)))
153
				mask |= (0xff << (i * 8));
154
		}
155
		/* Save the old part of val */
156
		val &= mask;
157
	}
158
	val |= ((uint32_t)(data >> 32) & ~mask);
159
	idx = srcu_read_lock(&vcpu->kvm->srcu);
160
	ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
161
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
162
	if (unlikely(ret))
163
		kvm_err("%s: : write data to addr %llx failed\n", __func__, addr);
164

165
	return ret;
166
}
167

168
static int any_send(struct kvm *kvm, uint64_t data)
169
{
170
	int cpu, offset;
171
	struct kvm_vcpu *vcpu;
172

173
	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
174
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
175
	if (unlikely(vcpu == NULL)) {
176
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
177
		return -EINVAL;
178
	}
179
	offset = data & 0xffff;
180

181
	return send_ipi_data(vcpu, offset, data);
182
}
183

184
static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val)
185
{
186
	int ret = 0;
187
	uint32_t offset;
188
	uint64_t res = 0;
189

190
	offset = (uint32_t)(addr & 0x1ff);
191
	WARN_ON_ONCE(offset & (len - 1));
192

193
	switch (offset) {
194
	case IOCSR_IPI_STATUS:
195
		spin_lock(&vcpu->arch.ipi_state.lock);
196
		res = vcpu->arch.ipi_state.status;
197
		spin_unlock(&vcpu->arch.ipi_state.lock);
198
		break;
199
	case IOCSR_IPI_EN:
200
		spin_lock(&vcpu->arch.ipi_state.lock);
201
		res = vcpu->arch.ipi_state.en;
202
		spin_unlock(&vcpu->arch.ipi_state.lock);
203
		break;
204
	case IOCSR_IPI_SET:
205
		res = 0;
206
		break;
207
	case IOCSR_IPI_CLEAR:
208
		res = 0;
209
		break;
210
	case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
211
		if (offset + len > IOCSR_IPI_BUF_38 + 8) {
212
			kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
213
				__func__, offset, len);
214
			ret = -EINVAL;
215
			break;
216
		}
217
		res = read_mailbox(vcpu, offset, len);
218
		break;
219
	default:
220
		kvm_err("%s: unknown addr: %llx\n", __func__, addr);
221
		ret = -EINVAL;
222
		break;
223
	}
224
	*(uint64_t *)val = res;
225

226
	return ret;
227
}
228

229
static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val)
230
{
231
	int ret = 0;
232
	uint64_t data;
233
	uint32_t offset;
234

235
	data = *(uint64_t *)val;
236

237
	offset = (uint32_t)(addr & 0x1ff);
238
	WARN_ON_ONCE(offset & (len - 1));
239

240
	switch (offset) {
241
	case IOCSR_IPI_STATUS:
242
		ret = -EINVAL;
243
		break;
244
	case IOCSR_IPI_EN:
245
		spin_lock(&vcpu->arch.ipi_state.lock);
246
		vcpu->arch.ipi_state.en = data;
247
		spin_unlock(&vcpu->arch.ipi_state.lock);
248
		break;
249
	case IOCSR_IPI_SET:
250
		ipi_set(vcpu, data);
251
		break;
252
	case IOCSR_IPI_CLEAR:
253
		/* Just clear the status of the current vcpu */
254
		ipi_clear(vcpu, data);
255
		break;
256
	case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
257
		if (offset + len > IOCSR_IPI_BUF_38 + 8) {
258
			kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
259
				__func__, offset, len);
260
			ret = -EINVAL;
261
			break;
262
		}
263
		write_mailbox(vcpu, offset, data, len);
264
		break;
265
	case IOCSR_IPI_SEND:
266
		ipi_send(vcpu->kvm, data);
267
		break;
268
	case IOCSR_MAIL_SEND:
269
		ret = mail_send(vcpu->kvm, data);
270
		break;
271
	case IOCSR_ANY_SEND:
272
		ret = any_send(vcpu->kvm, data);
273
		break;
274
	default:
275
		kvm_err("%s: unknown addr: %llx\n", __func__, addr);
276
		ret = -EINVAL;
277
		break;
278
	}
279

280
	return ret;
281
}
282

283
static int kvm_ipi_read(struct kvm_vcpu *vcpu,
284
			struct kvm_io_device *dev,
285
			gpa_t addr, int len, void *val)
286
{
287
	vcpu->stat.ipi_read_exits++;
288
	return loongarch_ipi_readl(vcpu, addr, len, val);
289
}
290

291
static int kvm_ipi_write(struct kvm_vcpu *vcpu,
292
			struct kvm_io_device *dev,
293
			gpa_t addr, int len, const void *val)
294
{
295
	vcpu->stat.ipi_write_exits++;
296
	return loongarch_ipi_writel(vcpu, addr, len, val);
297
}
298

299
static const struct kvm_io_device_ops kvm_ipi_ops = {
300
	.read	= kvm_ipi_read,
301
	.write	= kvm_ipi_write,
302
};
303

304
static int kvm_ipi_regs_access(struct kvm_device *dev,
305
				struct kvm_device_attr *attr,
306
				bool is_write)
307
{
308
	int len = 4;
309
	int cpu, addr;
310
	uint64_t val;
311
	void *p = NULL;
312
	struct kvm_vcpu *vcpu;
313

314
	cpu = (attr->attr >> 16) & 0x3ff;
315
	addr = attr->attr & 0xff;
316

317
	vcpu = kvm_get_vcpu_by_id(dev->kvm, cpu);
318
	if (unlikely(vcpu == NULL)) {
319
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
320
		return -EINVAL;
321
	}
322

323
	switch (addr) {
324
	case IOCSR_IPI_STATUS:
325
		p = &vcpu->arch.ipi_state.status;
326
		break;
327
	case IOCSR_IPI_EN:
328
		p = &vcpu->arch.ipi_state.en;
329
		break;
330
	case IOCSR_IPI_SET:
331
		p = &vcpu->arch.ipi_state.set;
332
		break;
333
	case IOCSR_IPI_CLEAR:
334
		p = &vcpu->arch.ipi_state.clear;
335
		break;
336
	case IOCSR_IPI_BUF_20:
337
		p = &vcpu->arch.ipi_state.buf[0];
338
		len = 8;
339
		break;
340
	case IOCSR_IPI_BUF_28:
341
		p = &vcpu->arch.ipi_state.buf[1];
342
		len = 8;
343
		break;
344
	case IOCSR_IPI_BUF_30:
345
		p = &vcpu->arch.ipi_state.buf[2];
346
		len = 8;
347
		break;
348
	case IOCSR_IPI_BUF_38:
349
		p = &vcpu->arch.ipi_state.buf[3];
350
		len = 8;
351
		break;
352
	default:
353
		kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr);
354
		return -EINVAL;
355
	}
356

357
	if (is_write) {
358
		if (len == 4) {
359
			if (get_user(val, (uint32_t __user *)attr->addr))
360
				return -EFAULT;
361
			*(uint32_t *)p = (uint32_t)val;
362
		} else if (len == 8) {
363
			if (get_user(val, (uint64_t __user *)attr->addr))
364
				return -EFAULT;
365
			*(uint64_t *)p = val;
366
		}
367
	} else {
368
		if (len == 4) {
369
			val = *(uint32_t *)p;
370
			return put_user(val, (uint32_t __user *)attr->addr);
371
		} else if (len == 8) {
372
			val = *(uint64_t *)p;
373
			return put_user(val, (uint64_t __user *)attr->addr);
374
		}
375
	}
376

377
	return 0;
378
}
379

380
static int kvm_ipi_get_attr(struct kvm_device *dev,
381
			struct kvm_device_attr *attr)
382
{
383
	switch (attr->group) {
384
	case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
385
		return kvm_ipi_regs_access(dev, attr, false);
386
	default:
387
		kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
388
		return -EINVAL;
389
	}
390
}
391

392
static int kvm_ipi_set_attr(struct kvm_device *dev,
393
			struct kvm_device_attr *attr)
394
{
395
	switch (attr->group) {
396
	case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
397
		return kvm_ipi_regs_access(dev, attr, true);
398
	default:
399
		kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
400
		return -EINVAL;
401
	}
402
}
403

404
static int kvm_ipi_create(struct kvm_device *dev, u32 type)
405
{
406
	int ret;
407
	struct kvm *kvm;
408
	struct kvm_io_device *device;
409
	struct loongarch_ipi *s;
410

411
	if (!dev) {
412
		kvm_err("%s: kvm_device ptr is invalid!\n", __func__);
413
		return -EINVAL;
414
	}
415

416
	kvm = dev->kvm;
417
	if (kvm->arch.ipi) {
418
		kvm_err("%s: LoongArch IPI has already been created!\n", __func__);
419
		return -EINVAL;
420
	}
421

422
	s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL);
423
	if (!s)
424
		return -ENOMEM;
425

426
	spin_lock_init(&s->lock);
427
	s->kvm = kvm;
428

429
	/*
430
	 * Initialize IOCSR device
431
	 */
432
	device = &s->device;
433
	kvm_iodevice_init(device, &kvm_ipi_ops);
434
	mutex_lock(&kvm->slots_lock);
435
	ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
436
	mutex_unlock(&kvm->slots_lock);
437
	if (ret < 0) {
438
		kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret);
439
		goto err;
440
	}
441

442
	kvm->arch.ipi = s;
443
	return 0;
444

445
err:
446
	kfree(s);
447
	return -EFAULT;
448
}
449

450
static void kvm_ipi_destroy(struct kvm_device *dev)
451
{
452
	struct kvm *kvm;
453
	struct loongarch_ipi *ipi;
454

455
	if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
456
		return;
457

458
	kvm = dev->kvm;
459
	ipi = kvm->arch.ipi;
460
	kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device);
461
	kfree(ipi);
462
}
463

464
static struct kvm_device_ops kvm_ipi_dev_ops = {
465
	.name = "kvm-loongarch-ipi",
466
	.create = kvm_ipi_create,
467
	.destroy = kvm_ipi_destroy,
468
	.set_attr = kvm_ipi_set_attr,
469
	.get_attr = kvm_ipi_get_attr,
470
};
471

472
int kvm_loongarch_register_ipi_device(void)
473
{
474
	return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI);
475
}
476

477