1
// SPDX-License-Identifier: GPL-2.0-only
2
/* Copyright(c) 2020 Intel Corporation. */
3

4
#include <linux/io-64-nonatomic-lo-hi.h>
5
#include <linux/firmware.h>
6
#include <linux/device.h>
7
#include <linux/slab.h>
8
#include <linux/idr.h>
9
#include <linux/pci.h>
10
#include <cxlmem.h>
11
#include "trace.h"
12
#include "core.h"
13

14
static DECLARE_RWSEM(cxl_memdev_rwsem);
15

16
/*
17
 * An entire PCI topology full of devices should be enough for any
18
 * config
19
 */
20
#define CXL_MEM_MAX_DEVS 65536
21

22
static int cxl_mem_major;
23
static DEFINE_IDA(cxl_memdev_ida);
24

25
static void cxl_memdev_release(struct device *dev)
26
{
27
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
28

29
	ida_free(&cxl_memdev_ida, cxlmd->id);
30
	devm_cxl_memdev_edac_release(cxlmd);
31
	kfree(cxlmd);
32
}
33

34
static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
35
				kgid_t *gid)
36
{
37
	return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
38
}
39

40
static ssize_t firmware_version_show(struct device *dev,
41
				     struct device_attribute *attr, char *buf)
42
{
43
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
44
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
45
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
46

47
	if (!mds)
48
		return sysfs_emit(buf, "\n");
49
	return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
50
}
51
static DEVICE_ATTR_RO(firmware_version);
52

53
static ssize_t payload_max_show(struct device *dev,
54
				struct device_attribute *attr, char *buf)
55
{
56
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
57
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
58
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
59

60
	if (!mds)
61
		return sysfs_emit(buf, "\n");
62
	return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
63
}
64
static DEVICE_ATTR_RO(payload_max);
65

66
static ssize_t label_storage_size_show(struct device *dev,
67
				       struct device_attribute *attr, char *buf)
68
{
69
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
70
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
71
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
72

73
	if (!mds)
74
		return sysfs_emit(buf, "\n");
75
	return sysfs_emit(buf, "%zu\n", mds->lsa_size);
76
}
77
static DEVICE_ATTR_RO(label_storage_size);
78

79
static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
80
{
81
	/* Static RAM is only expected at partition 0. */
82
	if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
83
		return 0;
84
	return resource_size(&cxlds->part[0].res);
85
}
86

87
static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
88
			     char *buf)
89
{
90
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
91
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
92
	unsigned long long len = cxl_ram_size(cxlds);
93

94
	return sysfs_emit(buf, "%#llx\n", len);
95
}
96

97
static struct device_attribute dev_attr_ram_size =
98
	__ATTR(size, 0444, ram_size_show, NULL);
99

100
static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
101
			      char *buf)
102
{
103
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
104
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
105
	unsigned long long len = cxl_pmem_size(cxlds);
106

107
	return sysfs_emit(buf, "%#llx\n", len);
108
}
109

110
static struct device_attribute dev_attr_pmem_size =
111
	__ATTR(size, 0444, pmem_size_show, NULL);
112

113
static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
114
			   char *buf)
115
{
116
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
117
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
118

119
	return sysfs_emit(buf, "%#llx\n", cxlds->serial);
120
}
121
static DEVICE_ATTR_RO(serial);
122

123
static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
124
			      char *buf)
125
{
126
	return sysfs_emit(buf, "%d\n", dev_to_node(dev));
127
}
128
static DEVICE_ATTR_RO(numa_node);
129

130
static ssize_t security_state_show(struct device *dev,
131
				   struct device_attribute *attr,
132
				   char *buf)
133
{
134
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
135
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
136
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
137
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
138
	unsigned long state = mds->security.state;
139
	int rc = 0;
140

141
	/* sync with latest submission state */
142
	mutex_lock(&cxl_mbox->mbox_mutex);
143
	if (mds->security.sanitize_active)
144
		rc = sysfs_emit(buf, "sanitize\n");
145
	mutex_unlock(&cxl_mbox->mbox_mutex);
146
	if (rc)
147
		return rc;
148

149
	if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
150
		return sysfs_emit(buf, "disabled\n");
151
	if (state & CXL_PMEM_SEC_STATE_FROZEN ||
152
	    state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
153
	    state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
154
		return sysfs_emit(buf, "frozen\n");
155
	if (state & CXL_PMEM_SEC_STATE_LOCKED)
156
		return sysfs_emit(buf, "locked\n");
157

158
	return sysfs_emit(buf, "unlocked\n");
159
}
160
static struct device_attribute dev_attr_security_state =
161
	__ATTR(state, 0444, security_state_show, NULL);
162

163
static ssize_t security_sanitize_store(struct device *dev,
164
				       struct device_attribute *attr,
165
				       const char *buf, size_t len)
166
{
167
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
168
	bool sanitize;
169
	ssize_t rc;
170

171
	if (kstrtobool(buf, &sanitize) || !sanitize)
172
		return -EINVAL;
173

174
	rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
175
	if (rc)
176
		return rc;
177

178
	return len;
179
}
180
static struct device_attribute dev_attr_security_sanitize =
181
	__ATTR(sanitize, 0200, NULL, security_sanitize_store);
182

183
static ssize_t security_erase_store(struct device *dev,
184
				    struct device_attribute *attr,
185
				    const char *buf, size_t len)
186
{
187
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
188
	ssize_t rc;
189
	bool erase;
190

191
	if (kstrtobool(buf, &erase) || !erase)
192
		return -EINVAL;
193

194
	rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
195
	if (rc)
196
		return rc;
197

198
	return len;
199
}
200
static struct device_attribute dev_attr_security_erase =
201
	__ATTR(erase, 0200, NULL, security_erase_store);
202

203
bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd,
204
			       enum poison_cmd_enabled_bits cmd)
205
{
206
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
207

208
	return test_bit(cmd, mds->poison.enabled_cmds);
209
}
210

211
static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
212
{
213
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
214
	u64 offset, length;
215
	int rc = 0;
216

217
	/* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
218
	for (int i = 0; i < cxlds->nr_partitions; i++) {
219
		const struct resource *res = &cxlds->part[i].res;
220

221
		offset = res->start;
222
		length = resource_size(res);
223
		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
224
		/*
225
		 * Invalid Physical Address is not an error for
226
		 * volatile addresses. Device support is optional.
227
		 */
228
		if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
229
			rc = 0;
230
	}
231
	return rc;
232
}
233

234
int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
235
{
236
	struct cxl_port *port;
237
	int rc;
238

239
	port = cxlmd->endpoint;
240
	if (!port || !is_cxl_endpoint(port))
241
		return -EINVAL;
242

243
	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
244
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
245
		return rc;
246

247
	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
248
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
249
		return rc;
250

251
	if (cxl_num_decoders_committed(port) == 0) {
252
		/* No regions mapped to this memdev */
253
		rc = cxl_get_poison_by_memdev(cxlmd);
254
	} else {
255
		/* Regions mapped, collect poison by endpoint */
256
		rc =  cxl_get_poison_by_endpoint(port);
257
	}
258

259
	return rc;
260
}
261
EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");
262

263
static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
264
{
265
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
266

267
	if (!IS_ENABLED(CONFIG_DEBUG_FS))
268
		return 0;
269

270
	if (!resource_size(&cxlds->dpa_res)) {
271
		dev_dbg(cxlds->dev, "device has no dpa resource\n");
272
		return -EINVAL;
273
	}
274
	if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) {
275
		dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
276
			dpa, &cxlds->dpa_res);
277
		return -EINVAL;
278
	}
279
	if (!IS_ALIGNED(dpa, 64)) {
280
		dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
281
		return -EINVAL;
282
	}
283

284
	return 0;
285
}
286

287
int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
288
{
289
	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
290
	struct cxl_mbox_inject_poison inject;
291
	struct cxl_poison_record record;
292
	struct cxl_mbox_cmd mbox_cmd;
293
	struct cxl_region *cxlr;
294
	int rc;
295

296
	if (!IS_ENABLED(CONFIG_DEBUG_FS))
297
		return 0;
298

299
	lockdep_assert_held(&cxl_rwsem.dpa);
300
	lockdep_assert_held(&cxl_rwsem.region);
301

302
	rc = cxl_validate_poison_dpa(cxlmd, dpa);
303
	if (rc)
304
		return rc;
305

306
	inject.address = cpu_to_le64(dpa);
307
	mbox_cmd = (struct cxl_mbox_cmd) {
308
		.opcode = CXL_MBOX_OP_INJECT_POISON,
309
		.size_in = sizeof(inject),
310
		.payload_in = &inject,
311
	};
312
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
313
	if (rc)
314
		return rc;
315

316
	cxlr = cxl_dpa_to_region(cxlmd, dpa);
317
	if (cxlr)
318
		dev_warn_once(cxl_mbox->host,
319
			      "poison inject dpa:%#llx region: %s\n", dpa,
320
			      dev_name(&cxlr->dev));
321

322
	record = (struct cxl_poison_record) {
323
		.address = cpu_to_le64(dpa),
324
		.length = cpu_to_le32(1),
325
	};
326
	trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
327

328
	return 0;
329
}
330

331
int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
332
{
333
	int rc;
334

335
	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
336
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
337
		return rc;
338

339
	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
340
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
341
		return rc;
342

343
	return cxl_inject_poison_locked(cxlmd, dpa);
344
}
345
EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
346

347
int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
348
{
349
	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
350
	struct cxl_mbox_clear_poison clear;
351
	struct cxl_poison_record record;
352
	struct cxl_mbox_cmd mbox_cmd;
353
	struct cxl_region *cxlr;
354
	int rc;
355

356
	if (!IS_ENABLED(CONFIG_DEBUG_FS))
357
		return 0;
358

359
	lockdep_assert_held(&cxl_rwsem.dpa);
360
	lockdep_assert_held(&cxl_rwsem.region);
361

362
	rc = cxl_validate_poison_dpa(cxlmd, dpa);
363
	if (rc)
364
		return rc;
365

366
	/*
367
	 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
368
	 * is defined to accept 64 bytes of write-data, along with the
369
	 * address to clear. This driver uses zeroes as write-data.
370
	 */
371
	clear = (struct cxl_mbox_clear_poison) {
372
		.address = cpu_to_le64(dpa)
373
	};
374

375
	mbox_cmd = (struct cxl_mbox_cmd) {
376
		.opcode = CXL_MBOX_OP_CLEAR_POISON,
377
		.size_in = sizeof(clear),
378
		.payload_in = &clear,
379
	};
380

381
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
382
	if (rc)
383
		return rc;
384

385
	cxlr = cxl_dpa_to_region(cxlmd, dpa);
386
	if (cxlr)
387
		dev_warn_once(cxl_mbox->host,
388
			      "poison clear dpa:%#llx region: %s\n", dpa,
389
			      dev_name(&cxlr->dev));
390

391
	record = (struct cxl_poison_record) {
392
		.address = cpu_to_le64(dpa),
393
		.length = cpu_to_le32(1),
394
	};
395
	trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
396

397
	return 0;
398
}
399

400
int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
401
{
402
	int rc;
403

404
	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
405
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
406
		return rc;
407

408
	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
409
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
410
		return rc;
411

412
	return cxl_clear_poison_locked(cxlmd, dpa);
413
}
414
EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");
415

416
static struct attribute *cxl_memdev_attributes[] = {
417
	&dev_attr_serial.attr,
418
	&dev_attr_firmware_version.attr,
419
	&dev_attr_payload_max.attr,
420
	&dev_attr_label_storage_size.attr,
421
	&dev_attr_numa_node.attr,
422
	NULL,
423
};
424

425
static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
426
{
427
	for (int i = 0; i < cxlds->nr_partitions; i++)
428
		if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
429
			return &cxlds->part[i].perf;
430
	return NULL;
431
}
432

433
static ssize_t pmem_qos_class_show(struct device *dev,
434
				   struct device_attribute *attr, char *buf)
435
{
436
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
437
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
438

439
	return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);
440
}
441

442
static struct device_attribute dev_attr_pmem_qos_class =
443
	__ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
444

445
static struct attribute *cxl_memdev_pmem_attributes[] = {
446
	&dev_attr_pmem_size.attr,
447
	&dev_attr_pmem_qos_class.attr,
448
	NULL,
449
};
450

451
static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
452
{
453
	if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
454
		return NULL;
455
	return &cxlds->part[0].perf;
456
}
457

458
static ssize_t ram_qos_class_show(struct device *dev,
459
				  struct device_attribute *attr, char *buf)
460
{
461
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
462
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
463

464
	return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);
465
}
466

467
static struct device_attribute dev_attr_ram_qos_class =
468
	__ATTR(qos_class, 0444, ram_qos_class_show, NULL);
469

470
static struct attribute *cxl_memdev_ram_attributes[] = {
471
	&dev_attr_ram_size.attr,
472
	&dev_attr_ram_qos_class.attr,
473
	NULL,
474
};
475

476
static struct attribute *cxl_memdev_security_attributes[] = {
477
	&dev_attr_security_state.attr,
478
	&dev_attr_security_sanitize.attr,
479
	&dev_attr_security_erase.attr,
480
	NULL,
481
};
482

483
static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
484
				  int n)
485
{
486
	if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
487
		return 0;
488
	return a->mode;
489
}
490

491
static struct attribute_group cxl_memdev_attribute_group = {
492
	.attrs = cxl_memdev_attributes,
493
	.is_visible = cxl_memdev_visible,
494
};
495

496
static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
497
{
498
	struct device *dev = kobj_to_dev(kobj);
499
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
500
	struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds);
501

502
	if (a == &dev_attr_ram_qos_class.attr &&
503
	    (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
504
		return 0;
505

506
	return a->mode;
507
}
508

509
static struct attribute_group cxl_memdev_ram_attribute_group = {
510
	.name = "ram",
511
	.attrs = cxl_memdev_ram_attributes,
512
	.is_visible = cxl_ram_visible,
513
};
514

515
static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
516
{
517
	struct device *dev = kobj_to_dev(kobj);
518
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
519
	struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds);
520

521
	if (a == &dev_attr_pmem_qos_class.attr &&
522
	    (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
523
		return 0;
524

525
	return a->mode;
526
}
527

528
static struct attribute_group cxl_memdev_pmem_attribute_group = {
529
	.name = "pmem",
530
	.attrs = cxl_memdev_pmem_attributes,
531
	.is_visible = cxl_pmem_visible,
532
};
533

534
static umode_t cxl_memdev_security_visible(struct kobject *kobj,
535
					   struct attribute *a, int n)
536
{
537
	struct device *dev = kobj_to_dev(kobj);
538
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
539
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
540

541
	if (a == &dev_attr_security_sanitize.attr &&
542
	    !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
543
		return 0;
544

545
	if (a == &dev_attr_security_erase.attr &&
546
	    !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
547
		return 0;
548

549
	return a->mode;
550
}
551

552
static struct attribute_group cxl_memdev_security_attribute_group = {
553
	.name = "security",
554
	.attrs = cxl_memdev_security_attributes,
555
	.is_visible = cxl_memdev_security_visible,
556
};
557

558
static const struct attribute_group *cxl_memdev_attribute_groups[] = {
559
	&cxl_memdev_attribute_group,
560
	&cxl_memdev_ram_attribute_group,
561
	&cxl_memdev_pmem_attribute_group,
562
	&cxl_memdev_security_attribute_group,
563
	NULL,
564
};
565

566
void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
567
{
568
	sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
569
	sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
570
}
571
EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");
572

573
static const struct device_type cxl_memdev_type = {
574
	.name = "cxl_memdev",
575
	.release = cxl_memdev_release,
576
	.devnode = cxl_memdev_devnode,
577
	.groups = cxl_memdev_attribute_groups,
578
};
579

580
bool is_cxl_memdev(const struct device *dev)
581
{
582
	return dev->type == &cxl_memdev_type;
583
}
584
EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
585

586
/**
587
 * set_exclusive_cxl_commands() - atomically disable user cxl commands
588
 * @mds: The device state to operate on
589
 * @cmds: bitmap of commands to mark exclusive
590
 *
591
 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
592
 * invocations of the ioctl path and then disable future execution of
593
 * commands with the command ids set in @cmds.
594
 */
595
void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
596
				unsigned long *cmds)
597
{
598
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
599

600
	guard(rwsem_write)(&cxl_memdev_rwsem);
601
	bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
602
		  cmds, CXL_MEM_COMMAND_ID_MAX);
603
}
604
EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");
605

606
/**
607
 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
608
 * @mds: The device state to modify
609
 * @cmds: bitmap of commands to mark available for userspace
610
 */
611
void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
612
				  unsigned long *cmds)
613
{
614
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
615

616
	guard(rwsem_write)(&cxl_memdev_rwsem);
617
	bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
618
		      cmds, CXL_MEM_COMMAND_ID_MAX);
619
}
620
EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");
621

622
static void cxl_memdev_shutdown(struct device *dev)
623
{
624
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
625

626
	guard(rwsem_write)(&cxl_memdev_rwsem);
627
	cxlmd->cxlds = NULL;
628
}
629

630
static void cxl_memdev_unregister(void *_cxlmd)
631
{
632
	struct cxl_memdev *cxlmd = _cxlmd;
633
	struct device *dev = &cxlmd->dev;
634

635
	cdev_device_del(&cxlmd->cdev, dev);
636
	cxl_memdev_shutdown(dev);
637
	put_device(dev);
638
}
639

640
static void detach_memdev(struct work_struct *work)
641
{
642
	struct cxl_memdev *cxlmd;
643

644
	cxlmd = container_of(work, typeof(*cxlmd), detach_work);
645
	device_release_driver(&cxlmd->dev);
646
	put_device(&cxlmd->dev);
647
}
648

649
static struct lock_class_key cxl_memdev_key;
650

651
static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
652
					   const struct file_operations *fops)
653
{
654
	struct cxl_memdev *cxlmd;
655
	struct device *dev;
656
	struct cdev *cdev;
657
	int rc;
658

659
	cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
660
	if (!cxlmd)
661
		return ERR_PTR(-ENOMEM);
662

663
	rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
664
	if (rc < 0)
665
		goto err;
666
	cxlmd->id = rc;
667
	cxlmd->depth = -1;
668

669
	dev = &cxlmd->dev;
670
	device_initialize(dev);
671
	lockdep_set_class(&dev->mutex, &cxl_memdev_key);
672
	dev->parent = cxlds->dev;
673
	dev->bus = &cxl_bus_type;
674
	dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
675
	dev->type = &cxl_memdev_type;
676
	device_set_pm_not_required(dev);
677
	INIT_WORK(&cxlmd->detach_work, detach_memdev);
678

679
	cdev = &cxlmd->cdev;
680
	cdev_init(cdev, fops);
681
	return cxlmd;
682

683
err:
684
	kfree(cxlmd);
685
	return ERR_PTR(rc);
686
}
687

688
static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
689
			       unsigned long arg)
690
{
691
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
692
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
693

694
	switch (cmd) {
695
	case CXL_MEM_QUERY_COMMANDS:
696
		return cxl_query_cmd(cxl_mbox, (void __user *)arg);
697
	case CXL_MEM_SEND_COMMAND:
698
		return cxl_send_cmd(cxl_mbox, (void __user *)arg);
699
	default:
700
		return -ENOTTY;
701
	}
702
}
703

704
static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
705
			     unsigned long arg)
706
{
707
	struct cxl_memdev *cxlmd = file->private_data;
708
	struct cxl_dev_state *cxlds;
709

710
	guard(rwsem_read)(&cxl_memdev_rwsem);
711
	cxlds = cxlmd->cxlds;
712
	if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
713
		return __cxl_memdev_ioctl(cxlmd, cmd, arg);
714

715
	return -ENXIO;
716
}
717

718
static int cxl_memdev_open(struct inode *inode, struct file *file)
719
{
720
	struct cxl_memdev *cxlmd =
721
		container_of(inode->i_cdev, typeof(*cxlmd), cdev);
722

723
	get_device(&cxlmd->dev);
724
	file->private_data = cxlmd;
725

726
	return 0;
727
}
728

729
static int cxl_memdev_release_file(struct inode *inode, struct file *file)
730
{
731
	struct cxl_memdev *cxlmd =
732
		container_of(inode->i_cdev, typeof(*cxlmd), cdev);
733

734
	put_device(&cxlmd->dev);
735

736
	return 0;
737
}
738

739
/**
740
 * cxl_mem_get_fw_info - Get Firmware info
741
 * @mds: The device data for the operation
742
 *
743
 * Retrieve firmware info for the device specified.
744
 *
745
 * Return: 0 if no error: or the result of the mailbox command.
746
 *
747
 * See CXL-3.0 8.2.9.3.1 Get FW Info
748
 */
749
static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
750
{
751
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
752
	struct cxl_mbox_get_fw_info info;
753
	struct cxl_mbox_cmd mbox_cmd;
754
	int rc;
755

756
	mbox_cmd = (struct cxl_mbox_cmd) {
757
		.opcode = CXL_MBOX_OP_GET_FW_INFO,
758
		.size_out = sizeof(info),
759
		.payload_out = &info,
760
	};
761

762
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
763
	if (rc < 0)
764
		return rc;
765

766
	mds->fw.num_slots = info.num_slots;
767
	mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
768
				       info.slot_info);
769

770
	return 0;
771
}
772

773
/**
774
 * cxl_mem_activate_fw - Activate Firmware
775
 * @mds: The device data for the operation
776
 * @slot: slot number to activate
777
 *
778
 * Activate firmware in a given slot for the device specified.
779
 *
780
 * Return: 0 if no error: or the result of the mailbox command.
781
 *
782
 * See CXL-3.0 8.2.9.3.3 Activate FW
783
 */
784
static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
785
{
786
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
787
	struct cxl_mbox_activate_fw activate;
788
	struct cxl_mbox_cmd mbox_cmd;
789

790
	if (slot == 0 || slot > mds->fw.num_slots)
791
		return -EINVAL;
792

793
	mbox_cmd = (struct cxl_mbox_cmd) {
794
		.opcode = CXL_MBOX_OP_ACTIVATE_FW,
795
		.size_in = sizeof(activate),
796
		.payload_in = &activate,
797
	};
798

799
	/* Only offline activation supported for now */
800
	activate.action = CXL_FW_ACTIVATE_OFFLINE;
801
	activate.slot = slot;
802

803
	return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
804
}
805

806
/**
807
 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
808
 * @mds: The device data for the operation
809
 *
810
 * Abort an in-progress firmware transfer for the device specified.
811
 *
812
 * Return: 0 if no error: or the result of the mailbox command.
813
 *
814
 * See CXL-3.0 8.2.9.3.2 Transfer FW
815
 */
816
static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
817
{
818
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
819
	struct cxl_mbox_transfer_fw *transfer;
820
	struct cxl_mbox_cmd mbox_cmd;
821
	int rc;
822

823
	transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
824
	if (!transfer)
825
		return -ENOMEM;
826

827
	/* Set a 1s poll interval and a total wait time of 30s */
828
	mbox_cmd = (struct cxl_mbox_cmd) {
829
		.opcode = CXL_MBOX_OP_TRANSFER_FW,
830
		.size_in = sizeof(*transfer),
831
		.payload_in = transfer,
832
		.poll_interval_ms = 1000,
833
		.poll_count = 30,
834
	};
835

836
	transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
837

838
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
839
	kfree(transfer);
840
	return rc;
841
}
842

843
static void cxl_fw_cleanup(struct fw_upload *fwl)
844
{
845
	struct cxl_memdev_state *mds = fwl->dd_handle;
846

847
	mds->fw.next_slot = 0;
848
}
849

850
static int cxl_fw_do_cancel(struct fw_upload *fwl)
851
{
852
	struct cxl_memdev_state *mds = fwl->dd_handle;
853
	struct cxl_dev_state *cxlds = &mds->cxlds;
854
	struct cxl_memdev *cxlmd = cxlds->cxlmd;
855
	int rc;
856

857
	rc = cxl_mem_abort_fw_xfer(mds);
858
	if (rc < 0)
859
		dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
860

861
	return FW_UPLOAD_ERR_CANCELED;
862
}
863

864
static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
865
					 u32 size)
866
{
867
	struct cxl_memdev_state *mds = fwl->dd_handle;
868
	struct cxl_mbox_transfer_fw *transfer;
869
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
870

871
	if (!size)
872
		return FW_UPLOAD_ERR_INVALID_SIZE;
873

874
	mds->fw.oneshot = struct_size(transfer, data, size) <
875
			    cxl_mbox->payload_size;
876

877
	if (cxl_mem_get_fw_info(mds))
878
		return FW_UPLOAD_ERR_HW_ERROR;
879

880
	/*
881
	 * So far no state has been changed, hence no other cleanup is
882
	 * necessary. Simply return the cancelled status.
883
	 */
884
	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
885
		return FW_UPLOAD_ERR_CANCELED;
886

887
	return FW_UPLOAD_ERR_NONE;
888
}
889

890
static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
891
				       u32 offset, u32 size, u32 *written)
892
{
893
	struct cxl_memdev_state *mds = fwl->dd_handle;
894
	struct cxl_dev_state *cxlds = &mds->cxlds;
895
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
896
	struct cxl_memdev *cxlmd = cxlds->cxlmd;
897
	struct cxl_mbox_transfer_fw *transfer;
898
	struct cxl_mbox_cmd mbox_cmd;
899
	u32 cur_size, remaining;
900
	size_t size_in;
901
	int rc;
902

903
	*written = 0;
904

905
	/* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
906
	if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
907
		dev_err(&cxlmd->dev,
908
			"misaligned offset for FW transfer slice (%u)\n",
909
			offset);
910
		return FW_UPLOAD_ERR_RW_ERROR;
911
	}
912

913
	/*
914
	 * Pick transfer size based on mds->payload_size @size must bw 128-byte
915
	 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
916
	 * sizeof(*transfer) is 128.  These constraints imply that @cur_size
917
	 * will always be 128b aligned.
918
	 */
919
	cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
920

921
	remaining = size - cur_size;
922
	size_in = struct_size(transfer, data, cur_size);
923

924
	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
925
		return cxl_fw_do_cancel(fwl);
926

927
	/*
928
	 * Slot numbers are 1-indexed
929
	 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
930
	 * Check for rollover using modulo, and 1-index it by adding 1
931
	 */
932
	mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
933

934
	/* Do the transfer via mailbox cmd */
935
	transfer = kzalloc(size_in, GFP_KERNEL);
936
	if (!transfer)
937
		return FW_UPLOAD_ERR_RW_ERROR;
938

939
	transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
940
	memcpy(transfer->data, data + offset, cur_size);
941
	if (mds->fw.oneshot) {
942
		transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
943
		transfer->slot = mds->fw.next_slot;
944
	} else {
945
		if (offset == 0) {
946
			transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
947
		} else if (remaining == 0) {
948
			transfer->action = CXL_FW_TRANSFER_ACTION_END;
949
			transfer->slot = mds->fw.next_slot;
950
		} else {
951
			transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
952
		}
953
	}
954

955
	mbox_cmd = (struct cxl_mbox_cmd) {
956
		.opcode = CXL_MBOX_OP_TRANSFER_FW,
957
		.size_in = size_in,
958
		.payload_in = transfer,
959
		.poll_interval_ms = 1000,
960
		.poll_count = 30,
961
	};
962

963
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
964
	if (rc < 0) {
965
		rc = FW_UPLOAD_ERR_RW_ERROR;
966
		goto out_free;
967
	}
968

969
	*written = cur_size;
970

971
	/* Activate FW if oneshot or if the last slice was written */
972
	if (mds->fw.oneshot || remaining == 0) {
973
		dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
974
			mds->fw.next_slot);
975
		rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
976
		if (rc < 0) {
977
			dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
978
				rc);
979
			rc = FW_UPLOAD_ERR_HW_ERROR;
980
			goto out_free;
981
		}
982
	}
983

984
	rc = FW_UPLOAD_ERR_NONE;
985

986
out_free:
987
	kfree(transfer);
988
	return rc;
989
}
990

991
static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
992
{
993
	struct cxl_memdev_state *mds = fwl->dd_handle;
994

995
	/*
996
	 * cxl_internal_send_cmd() handles background operations synchronously.
997
	 * No need to wait for completions here - any errors would've been
998
	 * reported and handled during the ->write() call(s).
999
	 * Just check if a cancel request was received, and return success.
1000
	 */
1001
	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
1002
		return cxl_fw_do_cancel(fwl);
1003

1004
	return FW_UPLOAD_ERR_NONE;
1005
}
1006

1007
static void cxl_fw_cancel(struct fw_upload *fwl)
1008
{
1009
	struct cxl_memdev_state *mds = fwl->dd_handle;
1010

1011
	set_bit(CXL_FW_CANCEL, mds->fw.state);
1012
}
1013

1014
static const struct fw_upload_ops cxl_memdev_fw_ops = {
1015
        .prepare = cxl_fw_prepare,
1016
        .write = cxl_fw_write,
1017
        .poll_complete = cxl_fw_poll_complete,
1018
        .cancel = cxl_fw_cancel,
1019
        .cleanup = cxl_fw_cleanup,
1020
};
1021

1022
static void cxl_remove_fw_upload(void *fwl)
1023
{
1024
	firmware_upload_unregister(fwl);
1025
}
1026

1027
int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
1028
{
1029
	struct cxl_dev_state *cxlds = &mds->cxlds;
1030
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
1031
	struct device *dev = &cxlds->cxlmd->dev;
1032
	struct fw_upload *fwl;
1033

1034
	if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
1035
		return 0;
1036

1037
	fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
1038
				       &cxl_memdev_fw_ops, mds);
1039
	if (IS_ERR(fwl))
1040
		return PTR_ERR(fwl);
1041
	return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1042
}
1043
EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");
1044

1045
static const struct file_operations cxl_memdev_fops = {
1046
	.owner = THIS_MODULE,
1047
	.unlocked_ioctl = cxl_memdev_ioctl,
1048
	.open = cxl_memdev_open,
1049
	.release = cxl_memdev_release_file,
1050
	.compat_ioctl = compat_ptr_ioctl,
1051
	.llseek = noop_llseek,
1052
};
1053

1054
struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
1055
				       struct cxl_dev_state *cxlds)
1056
{
1057
	struct cxl_memdev *cxlmd;
1058
	struct device *dev;
1059
	struct cdev *cdev;
1060
	int rc;
1061

1062
	cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1063
	if (IS_ERR(cxlmd))
1064
		return cxlmd;
1065

1066
	dev = &cxlmd->dev;
1067
	rc = dev_set_name(dev, "mem%d", cxlmd->id);
1068
	if (rc)
1069
		goto err;
1070

1071
	/*
1072
	 * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1073
	 * needed as this is ordered with cdev_add() publishing the device.
1074
	 */
1075
	cxlmd->cxlds = cxlds;
1076
	cxlds->cxlmd = cxlmd;
1077

1078
	cdev = &cxlmd->cdev;
1079
	rc = cdev_device_add(cdev, dev);
1080
	if (rc)
1081
		goto err;
1082

1083
	rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1084
	if (rc)
1085
		return ERR_PTR(rc);
1086
	return cxlmd;
1087

1088
err:
1089
	/*
1090
	 * The cdev was briefly live, shutdown any ioctl operations that
1091
	 * saw that state.
1092
	 */
1093
	cxl_memdev_shutdown(dev);
1094
	put_device(dev);
1095
	return ERR_PTR(rc);
1096
}
1097
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL");
1098

1099
static void sanitize_teardown_notifier(void *data)
1100
{
1101
	struct cxl_memdev_state *mds = data;
1102
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1103
	struct kernfs_node *state;
1104

1105
	/*
1106
	 * Prevent new irq triggered invocations of the workqueue and
1107
	 * flush inflight invocations.
1108
	 */
1109
	mutex_lock(&cxl_mbox->mbox_mutex);
1110
	state = mds->security.sanitize_node;
1111
	mds->security.sanitize_node = NULL;
1112
	mutex_unlock(&cxl_mbox->mbox_mutex);
1113

1114
	cancel_delayed_work_sync(&mds->security.poll_dwork);
1115
	sysfs_put(state);
1116
}
1117

1118
int devm_cxl_sanitize_setup_notifier(struct device *host,
1119
				     struct cxl_memdev *cxlmd)
1120
{
1121
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
1122
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1123
	struct kernfs_node *sec;
1124

1125
	if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1126
		return 0;
1127

1128
	/*
1129
	 * Note, the expectation is that @cxlmd would have failed to be
1130
	 * created if these sysfs_get_dirent calls fail.
1131
	 */
1132
	sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1133
	if (!sec)
1134
		return -ENOENT;
1135
	mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1136
	sysfs_put(sec);
1137
	if (!mds->security.sanitize_node)
1138
		return -ENOENT;
1139

1140
	return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1141
}
1142
EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");
1143

1144
__init int cxl_memdev_init(void)
1145
{
1146
	dev_t devt;
1147
	int rc;
1148

1149
	rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1150
	if (rc)
1151
		return rc;
1152

1153
	cxl_mem_major = MAJOR(devt);
1154

1155
	return 0;
1156
}
1157

1158
void cxl_memdev_exit(void)
1159
{
1160
	unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1161
}
1162

1163