1
// SPDX-License-Identifier: GPL-2.0-only
2
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3
#include <linux/platform_device.h>
4
#include <linux/memregion.h>
5
#include <linux/workqueue.h>
6
#include <linux/debugfs.h>
7
#include <linux/device.h>
8
#include <linux/module.h>
9
#include <linux/pci.h>
10
#include <linux/slab.h>
11
#include <linux/idr.h>
12
#include <linux/node.h>
13
#include <cxl/einj.h>
14
#include <cxlmem.h>
15
#include <cxlpci.h>
16
#include <cxl.h>
17
#include "core.h"
18

19
/**
20
 * DOC: cxl core
21
 *
22
 * The CXL core provides a set of interfaces that can be consumed by CXL aware
23
 * drivers. The interfaces allow for creation, modification, and destruction of
24
 * regions, memory devices, ports, and decoders. CXL aware drivers must register
25
 * with the CXL core via these interfaces in order to be able to participate in
26
 * cross-device interleave coordination. The CXL core also establishes and
27
 * maintains the bridge to the nvdimm subsystem.
28
 *
29
 * CXL core introduces sysfs hierarchy to control the devices that are
30
 * instantiated by the core.
31
 */
32

33
static DEFINE_IDA(cxl_port_ida);
34
static DEFINE_XARRAY(cxl_root_buses);
35

36
/*
37
 * The terminal device in PCI is NULL and @platform_bus
38
 * for platform devices (for cxl_test)
39
 */
40
static bool is_cxl_host_bridge(struct device *dev)
41
{
42
	return (!dev || dev == &platform_bus);
43
}
44

45
int cxl_num_decoders_committed(struct cxl_port *port)
46
{
47
	lockdep_assert_held(&cxl_rwsem.region);
48

49
	return port->commit_end + 1;
50
}
51

52
static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
53
			    char *buf)
54
{
55
	return sysfs_emit(buf, "%s\n", dev->type->name);
56
}
57
static DEVICE_ATTR_RO(devtype);
58

59
static int cxl_device_id(const struct device *dev)
60
{
61
	if (dev->type == &cxl_nvdimm_bridge_type)
62
		return CXL_DEVICE_NVDIMM_BRIDGE;
63
	if (dev->type == &cxl_nvdimm_type)
64
		return CXL_DEVICE_NVDIMM;
65
	if (dev->type == CXL_PMEM_REGION_TYPE())
66
		return CXL_DEVICE_PMEM_REGION;
67
	if (dev->type == CXL_DAX_REGION_TYPE())
68
		return CXL_DEVICE_DAX_REGION;
69
	if (is_cxl_port(dev)) {
70
		if (is_cxl_root(to_cxl_port(dev)))
71
			return CXL_DEVICE_ROOT;
72
		return CXL_DEVICE_PORT;
73
	}
74
	if (is_cxl_memdev(dev))
75
		return CXL_DEVICE_MEMORY_EXPANDER;
76
	if (dev->type == CXL_REGION_TYPE())
77
		return CXL_DEVICE_REGION;
78
	if (dev->type == &cxl_pmu_type)
79
		return CXL_DEVICE_PMU;
80
	return 0;
81
}
82

83
static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
84
			     char *buf)
85
{
86
	return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
87
}
88
static DEVICE_ATTR_RO(modalias);
89

90
static struct attribute *cxl_base_attributes[] = {
91
	&dev_attr_devtype.attr,
92
	&dev_attr_modalias.attr,
93
	NULL,
94
};
95

96
struct attribute_group cxl_base_attribute_group = {
97
	.attrs = cxl_base_attributes,
98
};
99

100
static ssize_t start_show(struct device *dev, struct device_attribute *attr,
101
			  char *buf)
102
{
103
	struct cxl_decoder *cxld = to_cxl_decoder(dev);
104

105
	return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
106
}
107
static DEVICE_ATTR_ADMIN_RO(start);
108

109
static ssize_t size_show(struct device *dev, struct device_attribute *attr,
110
			char *buf)
111
{
112
	struct cxl_decoder *cxld = to_cxl_decoder(dev);
113

114
	return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
115
}
116
static DEVICE_ATTR_RO(size);
117

118
#define CXL_DECODER_FLAG_ATTR(name, flag)                            \
119
static ssize_t name##_show(struct device *dev,                       \
120
			   struct device_attribute *attr, char *buf) \
121
{                                                                    \
122
	struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
123
                                                                     \
124
	return sysfs_emit(buf, "%s\n",                               \
125
			  (cxld->flags & (flag)) ? "1" : "0");       \
126
}                                                                    \
127
static DEVICE_ATTR_RO(name)
128

129
CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
130
CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
131
CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
132
CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
133
CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
134

135
static ssize_t target_type_show(struct device *dev,
136
				struct device_attribute *attr, char *buf)
137
{
138
	struct cxl_decoder *cxld = to_cxl_decoder(dev);
139

140
	switch (cxld->target_type) {
141
	case CXL_DECODER_DEVMEM:
142
		return sysfs_emit(buf, "accelerator\n");
143
	case CXL_DECODER_HOSTONLYMEM:
144
		return sysfs_emit(buf, "expander\n");
145
	}
146
	return -ENXIO;
147
}
148
static DEVICE_ATTR_RO(target_type);
149

150
static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
151
{
152
	struct cxl_decoder *cxld = &cxlsd->cxld;
153
	ssize_t offset = 0;
154
	int i, rc = 0;
155

156
	for (i = 0; i < cxld->interleave_ways; i++) {
157
		struct cxl_dport *dport = cxlsd->target[i];
158
		struct cxl_dport *next = NULL;
159

160
		if (!dport)
161
			break;
162

163
		if (i + 1 < cxld->interleave_ways)
164
			next = cxlsd->target[i + 1];
165
		rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
166
				   next ? "," : "");
167
		if (rc < 0)
168
			return rc;
169
		offset += rc;
170
	}
171

172
	return offset;
173
}
174

175
static ssize_t target_list_show(struct device *dev,
176
				struct device_attribute *attr, char *buf)
177
{
178
	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
179
	ssize_t offset;
180
	int rc;
181

182
	guard(rwsem_read)(&cxl_rwsem.region);
183
	rc = emit_target_list(cxlsd, buf);
184
	if (rc < 0)
185
		return rc;
186
	offset = rc;
187

188
	rc = sysfs_emit_at(buf, offset, "\n");
189
	if (rc < 0)
190
		return rc;
191

192
	return offset + rc;
193
}
194
static DEVICE_ATTR_RO(target_list);
195

196
static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
197
			 char *buf)
198
{
199
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
200
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
201
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
202
	/* without @cxl_rwsem.dpa, make sure @part is not reloaded */
203
	int part = READ_ONCE(cxled->part);
204
	const char *desc;
205

206
	if (part < 0)
207
		desc = "none";
208
	else
209
		desc = cxlds->part[part].res.name;
210

211
	return sysfs_emit(buf, "%s\n", desc);
212
}
213

214
static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
215
			  const char *buf, size_t len)
216
{
217
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
218
	enum cxl_partition_mode mode;
219
	ssize_t rc;
220

221
	if (sysfs_streq(buf, "pmem"))
222
		mode = CXL_PARTMODE_PMEM;
223
	else if (sysfs_streq(buf, "ram"))
224
		mode = CXL_PARTMODE_RAM;
225
	else
226
		return -EINVAL;
227

228
	rc = cxl_dpa_set_part(cxled, mode);
229
	if (rc)
230
		return rc;
231

232
	return len;
233
}
234
static DEVICE_ATTR_RW(mode);
235

236
static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
237
			    char *buf)
238
{
239
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
240

241
	guard(rwsem_read)(&cxl_rwsem.dpa);
242
	return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
243
}
244
static DEVICE_ATTR_RO(dpa_resource);
245

246
static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
247
			     char *buf)
248
{
249
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
250
	resource_size_t size = cxl_dpa_size(cxled);
251

252
	return sysfs_emit(buf, "%pa\n", &size);
253
}
254

255
static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
256
			      const char *buf, size_t len)
257
{
258
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
259
	unsigned long long size;
260
	ssize_t rc;
261

262
	rc = kstrtoull(buf, 0, &size);
263
	if (rc)
264
		return rc;
265

266
	if (!IS_ALIGNED(size, SZ_256M))
267
		return -EINVAL;
268

269
	rc = cxl_dpa_free(cxled);
270
	if (rc)
271
		return rc;
272

273
	if (size == 0)
274
		return len;
275

276
	rc = cxl_dpa_alloc(cxled, size);
277
	if (rc)
278
		return rc;
279

280
	return len;
281
}
282
static DEVICE_ATTR_RW(dpa_size);
283

284
static ssize_t interleave_granularity_show(struct device *dev,
285
					   struct device_attribute *attr,
286
					   char *buf)
287
{
288
	struct cxl_decoder *cxld = to_cxl_decoder(dev);
289

290
	return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
291
}
292

293
static DEVICE_ATTR_RO(interleave_granularity);
294

295
static ssize_t interleave_ways_show(struct device *dev,
296
				    struct device_attribute *attr, char *buf)
297
{
298
	struct cxl_decoder *cxld = to_cxl_decoder(dev);
299

300
	return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
301
}
302

303
static DEVICE_ATTR_RO(interleave_ways);
304

305
static ssize_t qos_class_show(struct device *dev,
306
			      struct device_attribute *attr, char *buf)
307
{
308
	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
309

310
	return sysfs_emit(buf, "%d\n", cxlrd->qos_class);
311
}
312
static DEVICE_ATTR_RO(qos_class);
313

314
static struct attribute *cxl_decoder_base_attrs[] = {
315
	&dev_attr_start.attr,
316
	&dev_attr_size.attr,
317
	&dev_attr_locked.attr,
318
	&dev_attr_interleave_granularity.attr,
319
	&dev_attr_interleave_ways.attr,
320
	NULL,
321
};
322

323
static struct attribute_group cxl_decoder_base_attribute_group = {
324
	.attrs = cxl_decoder_base_attrs,
325
};
326

327
static struct attribute *cxl_decoder_root_attrs[] = {
328
	&dev_attr_cap_pmem.attr,
329
	&dev_attr_cap_ram.attr,
330
	&dev_attr_cap_type2.attr,
331
	&dev_attr_cap_type3.attr,
332
	&dev_attr_target_list.attr,
333
	&dev_attr_qos_class.attr,
334
	SET_CXL_REGION_ATTR(create_pmem_region)
335
	SET_CXL_REGION_ATTR(create_ram_region)
336
	SET_CXL_REGION_ATTR(delete_region)
337
	NULL,
338
};
339

340
static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
341
{
342
	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
343

344
	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
345
}
346

347
static bool can_create_ram(struct cxl_root_decoder *cxlrd)
348
{
349
	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
350

351
	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
352
}
353

354
static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
355
{
356
	struct device *dev = kobj_to_dev(kobj);
357
	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
358

359
	if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
360
		return 0;
361

362
	if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
363
		return 0;
364

365
	if (a == CXL_REGION_ATTR(delete_region) &&
366
	    !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
367
		return 0;
368

369
	return a->mode;
370
}
371

372
static struct attribute_group cxl_decoder_root_attribute_group = {
373
	.attrs = cxl_decoder_root_attrs,
374
	.is_visible = cxl_root_decoder_visible,
375
};
376

377
static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
378
	&cxl_decoder_root_attribute_group,
379
	&cxl_decoder_base_attribute_group,
380
	&cxl_base_attribute_group,
381
	NULL,
382
};
383

384
static struct attribute *cxl_decoder_switch_attrs[] = {
385
	&dev_attr_target_type.attr,
386
	&dev_attr_target_list.attr,
387
	SET_CXL_REGION_ATTR(region)
388
	NULL,
389
};
390

391
static struct attribute_group cxl_decoder_switch_attribute_group = {
392
	.attrs = cxl_decoder_switch_attrs,
393
};
394

395
static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
396
	&cxl_decoder_switch_attribute_group,
397
	&cxl_decoder_base_attribute_group,
398
	&cxl_base_attribute_group,
399
	NULL,
400
};
401

402
static struct attribute *cxl_decoder_endpoint_attrs[] = {
403
	&dev_attr_target_type.attr,
404
	&dev_attr_mode.attr,
405
	&dev_attr_dpa_size.attr,
406
	&dev_attr_dpa_resource.attr,
407
	SET_CXL_REGION_ATTR(region)
408
	NULL,
409
};
410

411
static struct attribute_group cxl_decoder_endpoint_attribute_group = {
412
	.attrs = cxl_decoder_endpoint_attrs,
413
};
414

415
static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
416
	&cxl_decoder_base_attribute_group,
417
	&cxl_decoder_endpoint_attribute_group,
418
	&cxl_base_attribute_group,
419
	NULL,
420
};
421

422
static void __cxl_decoder_release(struct cxl_decoder *cxld)
423
{
424
	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
425

426
	ida_free(&port->decoder_ida, cxld->id);
427
	put_device(&port->dev);
428
}
429

430
static void cxl_endpoint_decoder_release(struct device *dev)
431
{
432
	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
433

434
	__cxl_decoder_release(&cxled->cxld);
435
	kfree(cxled);
436
}
437

438
static void cxl_switch_decoder_release(struct device *dev)
439
{
440
	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
441

442
	__cxl_decoder_release(&cxlsd->cxld);
443
	kfree(cxlsd);
444
}
445

446
struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
447
{
448
	if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
449
			  "not a cxl_root_decoder device\n"))
450
		return NULL;
451
	return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
452
}
453
EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL");
454

455
static void cxl_root_decoder_release(struct device *dev)
456
{
457
	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
458

459
	if (atomic_read(&cxlrd->region_id) >= 0)
460
		memregion_free(atomic_read(&cxlrd->region_id));
461
	__cxl_decoder_release(&cxlrd->cxlsd.cxld);
462
	kfree(cxlrd->ops);
463
	kfree(cxlrd);
464
}
465

466
static const struct device_type cxl_decoder_endpoint_type = {
467
	.name = "cxl_decoder_endpoint",
468
	.release = cxl_endpoint_decoder_release,
469
	.groups = cxl_decoder_endpoint_attribute_groups,
470
};
471

472
static const struct device_type cxl_decoder_switch_type = {
473
	.name = "cxl_decoder_switch",
474
	.release = cxl_switch_decoder_release,
475
	.groups = cxl_decoder_switch_attribute_groups,
476
};
477

478
static const struct device_type cxl_decoder_root_type = {
479
	.name = "cxl_decoder_root",
480
	.release = cxl_root_decoder_release,
481
	.groups = cxl_decoder_root_attribute_groups,
482
};
483

484
bool is_endpoint_decoder(struct device *dev)
485
{
486
	return dev->type == &cxl_decoder_endpoint_type;
487
}
488
EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL");
489

490
bool is_root_decoder(struct device *dev)
491
{
492
	return dev->type == &cxl_decoder_root_type;
493
}
494
EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL");
495

496
bool is_switch_decoder(struct device *dev)
497
{
498
	return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
499
}
500
EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL");
501

502
struct cxl_decoder *to_cxl_decoder(struct device *dev)
503
{
504
	if (dev_WARN_ONCE(dev,
505
			  !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
506
			  "not a cxl_decoder device\n"))
507
		return NULL;
508
	return container_of(dev, struct cxl_decoder, dev);
509
}
510
EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL");
511

512
struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
513
{
514
	if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
515
			  "not a cxl_endpoint_decoder device\n"))
516
		return NULL;
517
	return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
518
}
519
EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL");
520

521
struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
522
{
523
	if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
524
			  "not a cxl_switch_decoder device\n"))
525
		return NULL;
526
	return container_of(dev, struct cxl_switch_decoder, cxld.dev);
527
}
528
EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
529

530
static void cxl_ep_release(struct cxl_ep *ep)
531
{
532
	put_device(ep->ep);
533
	kfree(ep);
534
}
535

536
static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
537
{
538
	if (!ep)
539
		return;
540
	xa_erase(&port->endpoints, (unsigned long) ep->ep);
541
	cxl_ep_release(ep);
542
}
543

544
static void cxl_port_release(struct device *dev)
545
{
546
	struct cxl_port *port = to_cxl_port(dev);
547
	unsigned long index;
548
	struct cxl_ep *ep;
549

550
	xa_for_each(&port->endpoints, index, ep)
551
		cxl_ep_remove(port, ep);
552
	xa_destroy(&port->endpoints);
553
	xa_destroy(&port->dports);
554
	xa_destroy(&port->regions);
555
	ida_free(&cxl_port_ida, port->id);
556
	if (is_cxl_root(port))
557
		kfree(to_cxl_root(port));
558
	else
559
		kfree(port);
560
}
561

562
static ssize_t decoders_committed_show(struct device *dev,
563
				       struct device_attribute *attr, char *buf)
564
{
565
	struct cxl_port *port = to_cxl_port(dev);
566

567
	guard(rwsem_read)(&cxl_rwsem.region);
568
	return sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
569
}
570

571
static DEVICE_ATTR_RO(decoders_committed);
572

573
static struct attribute *cxl_port_attrs[] = {
574
	&dev_attr_decoders_committed.attr,
575
	NULL,
576
};
577

578
static struct attribute_group cxl_port_attribute_group = {
579
	.attrs = cxl_port_attrs,
580
};
581

582
static const struct attribute_group *cxl_port_attribute_groups[] = {
583
	&cxl_base_attribute_group,
584
	&cxl_port_attribute_group,
585
	NULL,
586
};
587

588
static const struct device_type cxl_port_type = {
589
	.name = "cxl_port",
590
	.release = cxl_port_release,
591
	.groups = cxl_port_attribute_groups,
592
};
593

594
bool is_cxl_port(const struct device *dev)
595
{
596
	return dev->type == &cxl_port_type;
597
}
598
EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL");
599

600
struct cxl_port *to_cxl_port(const struct device *dev)
601
{
602
	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
603
			  "not a cxl_port device\n"))
604
		return NULL;
605
	return container_of(dev, struct cxl_port, dev);
606
}
607
EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL");
608

609
struct cxl_port *parent_port_of(struct cxl_port *port)
610
{
611
	if (!port || !port->parent_dport)
612
		return NULL;
613
	return port->parent_dport->port;
614
}
615

616
static void unregister_port(void *_port)
617
{
618
	struct cxl_port *port = _port;
619
	struct cxl_port *parent = parent_port_of(port);
620
	struct device *lock_dev;
621

622
	/*
623
	 * CXL root port's and the first level of ports are unregistered
624
	 * under the platform firmware device lock, all other ports are
625
	 * unregistered while holding their parent port lock.
626
	 */
627
	if (!parent)
628
		lock_dev = port->uport_dev;
629
	else if (is_cxl_root(parent))
630
		lock_dev = parent->uport_dev;
631
	else
632
		lock_dev = &parent->dev;
633

634
	device_lock_assert(lock_dev);
635
	port->dead = true;
636
	device_unregister(&port->dev);
637
}
638

639
static void cxl_unlink_uport(void *_port)
640
{
641
	struct cxl_port *port = _port;
642

643
	sysfs_remove_link(&port->dev.kobj, "uport");
644
}
645

646
static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
647
{
648
	int rc;
649

650
	rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
651
			       "uport");
652
	if (rc)
653
		return rc;
654
	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
655
}
656

657
static void cxl_unlink_parent_dport(void *_port)
658
{
659
	struct cxl_port *port = _port;
660

661
	sysfs_remove_link(&port->dev.kobj, "parent_dport");
662
}
663

664
static int devm_cxl_link_parent_dport(struct device *host,
665
				      struct cxl_port *port,
666
				      struct cxl_dport *parent_dport)
667
{
668
	int rc;
669

670
	if (!parent_dport)
671
		return 0;
672

673
	rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
674
			       "parent_dport");
675
	if (rc)
676
		return rc;
677
	return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
678
}
679

680
static struct lock_class_key cxl_port_key;
681

682
static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
683
				       struct cxl_dport *parent_dport)
684
{
685
	struct cxl_root *cxl_root __free(kfree) = NULL;
686
	struct cxl_port *port, *_port __free(kfree) = NULL;
687
	struct device *dev;
688
	int rc;
689

690
	/* No parent_dport, root cxl_port */
691
	if (!parent_dport) {
692
		cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
693
		if (!cxl_root)
694
			return ERR_PTR(-ENOMEM);
695
	} else {
696
		_port = kzalloc(sizeof(*port), GFP_KERNEL);
697
		if (!_port)
698
			return ERR_PTR(-ENOMEM);
699
	}
700

701
	rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
702
	if (rc < 0)
703
		return ERR_PTR(rc);
704

705
	if (cxl_root)
706
		port = &no_free_ptr(cxl_root)->port;
707
	else
708
		port = no_free_ptr(_port);
709

710
	port->id = rc;
711
	port->uport_dev = uport_dev;
712

713
	/*
714
	 * The top-level cxl_port "cxl_root" does not have a cxl_port as
715
	 * its parent and it does not have any corresponding component
716
	 * registers as its decode is described by a fixed platform
717
	 * description.
718
	 */
719
	dev = &port->dev;
720
	if (parent_dport) {
721
		struct cxl_port *parent_port = parent_dport->port;
722
		struct cxl_port *iter;
723

724
		dev->parent = &parent_port->dev;
725
		port->depth = parent_port->depth + 1;
726
		port->parent_dport = parent_dport;
727

728
		/*
729
		 * walk to the host bridge, or the first ancestor that knows
730
		 * the host bridge
731
		 */
732
		iter = port;
733
		while (!iter->host_bridge &&
734
		       !is_cxl_root(to_cxl_port(iter->dev.parent)))
735
			iter = to_cxl_port(iter->dev.parent);
736
		if (iter->host_bridge)
737
			port->host_bridge = iter->host_bridge;
738
		else if (parent_dport->rch)
739
			port->host_bridge = parent_dport->dport_dev;
740
		else
741
			port->host_bridge = iter->uport_dev;
742
		dev_dbg(uport_dev, "host-bridge: %s\n",
743
			dev_name(port->host_bridge));
744
	} else
745
		dev->parent = uport_dev;
746

747
	ida_init(&port->decoder_ida);
748
	port->hdm_end = -1;
749
	port->commit_end = -1;
750
	xa_init(&port->dports);
751
	xa_init(&port->endpoints);
752
	xa_init(&port->regions);
753
	port->component_reg_phys = CXL_RESOURCE_NONE;
754

755
	device_initialize(dev);
756
	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
757
	device_set_pm_not_required(dev);
758
	dev->bus = &cxl_bus_type;
759
	dev->type = &cxl_port_type;
760

761
	return port;
762
}
763

764
static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
765
			       resource_size_t component_reg_phys)
766
{
767
	*map = (struct cxl_register_map) {
768
		.host = host,
769
		.reg_type = CXL_REGLOC_RBI_EMPTY,
770
		.resource = component_reg_phys,
771
	};
772

773
	if (component_reg_phys == CXL_RESOURCE_NONE)
774
		return 0;
775

776
	map->reg_type = CXL_REGLOC_RBI_COMPONENT;
777
	map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
778

779
	return cxl_setup_regs(map);
780
}
781

782
static int cxl_port_setup_regs(struct cxl_port *port,
783
			resource_size_t component_reg_phys)
784
{
785
	if (dev_is_platform(port->uport_dev))
786
		return 0;
787
	return cxl_setup_comp_regs(&port->dev, &port->reg_map,
788
				   component_reg_phys);
789
}
790

791
static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
792
				resource_size_t component_reg_phys)
793
{
794
	int rc;
795

796
	if (dev_is_platform(dport->dport_dev))
797
		return 0;
798

799
	/*
800
	 * use @dport->dport_dev for the context for error messages during
801
	 * register probing, and fixup @host after the fact, since @host may be
802
	 * NULL.
803
	 */
804
	rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
805
				 component_reg_phys);
806
	dport->reg_map.host = host;
807
	return rc;
808
}
809

810
DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
811

812
static int cxl_einj_inject(void *data, u64 type)
813
{
814
	struct cxl_dport *dport = data;
815

816
	if (dport->rch)
817
		return einj_cxl_inject_rch_error(dport->rcrb.base, type);
818

819
	return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type);
820
}
821
DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
822
			 "0x%llx\n");
823

824
static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
825
{
826
	struct dentry *dir;
827

828
	if (!einj_cxl_is_initialized())
829
		return;
830

831
	/*
832
	 * dport_dev needs to be a PCIe port for CXL 2.0+ ports because
833
	 * EINJ expects a dport SBDF to be specified for 2.0 error injection.
834
	 */
835
	if (!dport->rch && !dev_is_pci(dport->dport_dev))
836
		return;
837

838
	dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
839

840
	debugfs_create_file("einj_inject", 0200, dir, dport,
841
			    &cxl_einj_inject_fops);
842
}
843

844
static int cxl_port_add(struct cxl_port *port,
845
			resource_size_t component_reg_phys,
846
			struct cxl_dport *parent_dport)
847
{
848
	struct device *dev __free(put_device) = &port->dev;
849
	int rc;
850

851
	if (is_cxl_memdev(port->uport_dev)) {
852
		struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
853
		struct cxl_dev_state *cxlds = cxlmd->cxlds;
854

855
		rc = dev_set_name(dev, "endpoint%d", port->id);
856
		if (rc)
857
			return rc;
858

859
		/*
860
		 * The endpoint driver already enumerated the component and RAS
861
		 * registers. Reuse that enumeration while prepping them to be
862
		 * mapped by the cxl_port driver.
863
		 */
864
		port->reg_map = cxlds->reg_map;
865
		port->reg_map.host = &port->dev;
866
		cxlmd->endpoint = port;
867
	} else if (parent_dport) {
868
		rc = dev_set_name(dev, "port%d", port->id);
869
		if (rc)
870
			return rc;
871

872
		port->component_reg_phys = component_reg_phys;
873
	} else {
874
		rc = dev_set_name(dev, "root%d", port->id);
875
		if (rc)
876
			return rc;
877
	}
878

879
	rc = device_add(dev);
880
	if (rc)
881
		return rc;
882

883
	/* Inhibit the cleanup function invoked */
884
	dev = NULL;
885
	return 0;
886
}
887

888
static struct cxl_port *__devm_cxl_add_port(struct device *host,
889
					    struct device *uport_dev,
890
					    resource_size_t component_reg_phys,
891
					    struct cxl_dport *parent_dport)
892
{
893
	struct cxl_port *port;
894
	int rc;
895

896
	port = cxl_port_alloc(uport_dev, parent_dport);
897
	if (IS_ERR(port))
898
		return port;
899

900
	rc = cxl_port_add(port, component_reg_phys, parent_dport);
901
	if (rc)
902
		return ERR_PTR(rc);
903

904
	rc = devm_add_action_or_reset(host, unregister_port, port);
905
	if (rc)
906
		return ERR_PTR(rc);
907

908
	rc = devm_cxl_link_uport(host, port);
909
	if (rc)
910
		return ERR_PTR(rc);
911

912
	rc = devm_cxl_link_parent_dport(host, port, parent_dport);
913
	if (rc)
914
		return ERR_PTR(rc);
915

916
	if (parent_dport && dev_is_pci(uport_dev))
917
		port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
918

919
	return port;
920
}
921

922
/**
923
 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
924
 * @host: host device for devm operations
925
 * @uport_dev: "physical" device implementing this upstream port
926
 * @component_reg_phys: (optional) for configurable cxl_port instances
927
 * @parent_dport: next hop up in the CXL memory decode hierarchy
928
 */
929
struct cxl_port *devm_cxl_add_port(struct device *host,
930
				   struct device *uport_dev,
931
				   resource_size_t component_reg_phys,
932
				   struct cxl_dport *parent_dport)
933
{
934
	struct cxl_port *port, *parent_port;
935

936
	port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
937
				   parent_dport);
938

939
	parent_port = parent_dport ? parent_dport->port : NULL;
940
	if (IS_ERR(port)) {
941
		dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
942
			parent_port ? " port to " : "",
943
			parent_port ? dev_name(&parent_port->dev) : "",
944
			parent_port ? "" : " root port",
945
			PTR_ERR(port));
946
	} else {
947
		dev_dbg(uport_dev, "%s added%s%s%s\n",
948
			dev_name(&port->dev),
949
			parent_port ? " to " : "",
950
			parent_port ? dev_name(&parent_port->dev) : "",
951
			parent_port ? "" : " (root port)");
952
	}
953

954
	return port;
955
}
956
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL");
957

958
struct cxl_root *devm_cxl_add_root(struct device *host,
959
				   const struct cxl_root_ops *ops)
960
{
961
	struct cxl_root *cxl_root;
962
	struct cxl_port *port;
963

964
	port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
965
	if (IS_ERR(port))
966
		return ERR_CAST(port);
967

968
	cxl_root = to_cxl_root(port);
969
	cxl_root->ops = ops;
970
	return cxl_root;
971
}
972
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL");
973

974
struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
975
{
976
	/* There is no pci_bus associated with a CXL platform-root port */
977
	if (is_cxl_root(port))
978
		return NULL;
979

980
	if (dev_is_pci(port->uport_dev)) {
981
		struct pci_dev *pdev = to_pci_dev(port->uport_dev);
982

983
		return pdev->subordinate;
984
	}
985

986
	return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
987
}
988
EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL");
989

990
static void unregister_pci_bus(void *uport_dev)
991
{
992
	xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
993
}
994

995
int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
996
			      struct pci_bus *bus)
997
{
998
	int rc;
999

1000
	if (dev_is_pci(uport_dev))
1001
		return -EINVAL;
1002

1003
	rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
1004
		       GFP_KERNEL);
1005
	if (rc)
1006
		return rc;
1007
	return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
1008
}
1009
EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL");
1010

1011
static bool dev_is_cxl_root_child(struct device *dev)
1012
{
1013
	struct cxl_port *port, *parent;
1014

1015
	if (!is_cxl_port(dev))
1016
		return false;
1017

1018
	port = to_cxl_port(dev);
1019
	if (is_cxl_root(port))
1020
		return false;
1021

1022
	parent = to_cxl_port(port->dev.parent);
1023
	if (is_cxl_root(parent))
1024
		return true;
1025

1026
	return false;
1027
}
1028

1029
struct cxl_root *find_cxl_root(struct cxl_port *port)
1030
{
1031
	struct cxl_port *iter = port;
1032

1033
	while (iter && !is_cxl_root(iter))
1034
		iter = to_cxl_port(iter->dev.parent);
1035

1036
	if (!iter)
1037
		return NULL;
1038
	get_device(&iter->dev);
1039
	return to_cxl_root(iter);
1040
}
1041
EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL");
1042

1043
static struct cxl_dport *find_dport(struct cxl_port *port, int id)
1044
{
1045
	struct cxl_dport *dport;
1046
	unsigned long index;
1047

1048
	device_lock_assert(&port->dev);
1049
	xa_for_each(&port->dports, index, dport)
1050
		if (dport->port_id == id)
1051
			return dport;
1052
	return NULL;
1053
}
1054

1055
static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1056
{
1057
	struct cxl_dport *dup;
1058
	int rc;
1059

1060
	device_lock_assert(&port->dev);
1061
	dup = find_dport(port, dport->port_id);
1062
	if (dup) {
1063
		dev_err(&port->dev,
1064
			"unable to add dport%d-%s non-unique port id (%s)\n",
1065
			dport->port_id, dev_name(dport->dport_dev),
1066
			dev_name(dup->dport_dev));
1067
		return -EBUSY;
1068
	}
1069

1070
	rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
1071
		       GFP_KERNEL);
1072
	if (rc)
1073
		return rc;
1074

1075
	port->nr_dports++;
1076
	return 0;
1077
}
1078

1079
/*
1080
 * Since root-level CXL dports cannot be enumerated by PCI they are not
1081
 * enumerated by the common port driver that acquires the port lock over
1082
 * dport add/remove. Instead, root dports are manually added by a
1083
 * platform driver and cond_cxl_root_lock() is used to take the missing
1084
 * port lock in that case.
1085
 */
1086
static void cond_cxl_root_lock(struct cxl_port *port)
1087
{
1088
	if (is_cxl_root(port))
1089
		device_lock(&port->dev);
1090
}
1091

1092
static void cond_cxl_root_unlock(struct cxl_port *port)
1093
{
1094
	if (is_cxl_root(port))
1095
		device_unlock(&port->dev);
1096
}
1097

1098
static void cxl_dport_remove(void *data)
1099
{
1100
	struct cxl_dport *dport = data;
1101
	struct cxl_port *port = dport->port;
1102

1103
	xa_erase(&port->dports, (unsigned long) dport->dport_dev);
1104
	put_device(dport->dport_dev);
1105
}
1106

1107
static void cxl_dport_unlink(void *data)
1108
{
1109
	struct cxl_dport *dport = data;
1110
	struct cxl_port *port = dport->port;
1111
	char link_name[CXL_TARGET_STRLEN];
1112

1113
	sprintf(link_name, "dport%d", dport->port_id);
1114
	sysfs_remove_link(&port->dev.kobj, link_name);
1115
}
1116

1117
static struct cxl_dport *
1118
__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1119
		     int port_id, resource_size_t component_reg_phys,
1120
		     resource_size_t rcrb)
1121
{
1122
	char link_name[CXL_TARGET_STRLEN];
1123
	struct cxl_dport *dport;
1124
	struct device *host;
1125
	int rc;
1126

1127
	if (is_cxl_root(port))
1128
		host = port->uport_dev;
1129
	else
1130
		host = &port->dev;
1131

1132
	if (!host->driver) {
1133
		dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1134
			      dev_name(dport_dev));
1135
		return ERR_PTR(-ENXIO);
1136
	}
1137

1138
	if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
1139
	    CXL_TARGET_STRLEN)
1140
		return ERR_PTR(-EINVAL);
1141

1142
	dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1143
	if (!dport)
1144
		return ERR_PTR(-ENOMEM);
1145

1146
	dport->dport_dev = dport_dev;
1147
	dport->port_id = port_id;
1148
	dport->port = port;
1149

1150
	if (rcrb == CXL_RESOURCE_NONE) {
1151
		rc = cxl_dport_setup_regs(&port->dev, dport,
1152
					  component_reg_phys);
1153
		if (rc)
1154
			return ERR_PTR(rc);
1155
	} else {
1156
		dport->rcrb.base = rcrb;
1157
		component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1158
							 CXL_RCRB_DOWNSTREAM);
1159
		if (component_reg_phys == CXL_RESOURCE_NONE) {
1160
			dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1161
			return ERR_PTR(-ENXIO);
1162
		}
1163

1164
		/*
1165
		 * RCH @dport is not ready to map until associated with its
1166
		 * memdev
1167
		 */
1168
		rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1169
		if (rc)
1170
			return ERR_PTR(rc);
1171

1172
		dport->rch = true;
1173
	}
1174

1175
	if (component_reg_phys != CXL_RESOURCE_NONE)
1176
		dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1177
			&component_reg_phys);
1178

1179
	cond_cxl_root_lock(port);
1180
	rc = add_dport(port, dport);
1181
	cond_cxl_root_unlock(port);
1182
	if (rc)
1183
		return ERR_PTR(rc);
1184

1185
	get_device(dport_dev);
1186
	rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1187
	if (rc)
1188
		return ERR_PTR(rc);
1189

1190
	rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1191
	if (rc)
1192
		return ERR_PTR(rc);
1193

1194
	rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1195
	if (rc)
1196
		return ERR_PTR(rc);
1197

1198
	if (dev_is_pci(dport_dev))
1199
		dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1200

1201
	cxl_debugfs_create_dport_dir(dport);
1202

1203
	/*
1204
	 * Setup port register if this is the first dport showed up. Having
1205
	 * a dport also means that there is at least 1 active link.
1206
	 */
1207
	if (port->nr_dports == 1 &&
1208
	    port->component_reg_phys != CXL_RESOURCE_NONE) {
1209
		rc = cxl_port_setup_regs(port, port->component_reg_phys);
1210
		if (rc)
1211
			return ERR_PTR(rc);
1212
		port->component_reg_phys = CXL_RESOURCE_NONE;
1213
	}
1214

1215
	return dport;
1216
}
1217

1218
/**
1219
 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1220
 * @port: the cxl_port that references this dport
1221
 * @dport_dev: firmware or PCI device representing the dport
1222
 * @port_id: identifier for this dport in a decoder's target list
1223
 * @component_reg_phys: optional location of CXL component registers
1224
 *
1225
 * Note that dports are appended to the devm release action's of the
1226
 * either the port's host (for root ports), or the port itself (for
1227
 * switch ports)
1228
 */
1229
struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1230
				     struct device *dport_dev, int port_id,
1231
				     resource_size_t component_reg_phys)
1232
{
1233
	struct cxl_dport *dport;
1234

1235
	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1236
				     component_reg_phys, CXL_RESOURCE_NONE);
1237
	if (IS_ERR(dport)) {
1238
		dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1239
			dev_name(&port->dev), PTR_ERR(dport));
1240
	} else {
1241
		dev_dbg(dport_dev, "dport added to %s\n",
1242
			dev_name(&port->dev));
1243
	}
1244

1245
	return dport;
1246
}
1247
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL");
1248

1249
/**
1250
 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1251
 * @port: the cxl_port that references this dport
1252
 * @dport_dev: firmware or PCI device representing the dport
1253
 * @port_id: identifier for this dport in a decoder's target list
1254
 * @rcrb: mandatory location of a Root Complex Register Block
1255
 *
1256
 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1257
 */
1258
struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1259
					 struct device *dport_dev, int port_id,
1260
					 resource_size_t rcrb)
1261
{
1262
	struct cxl_dport *dport;
1263

1264
	if (rcrb == CXL_RESOURCE_NONE) {
1265
		dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1266
		return ERR_PTR(-EINVAL);
1267
	}
1268

1269
	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1270
				     CXL_RESOURCE_NONE, rcrb);
1271
	if (IS_ERR(dport)) {
1272
		dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1273
			dev_name(&port->dev), PTR_ERR(dport));
1274
	} else {
1275
		dev_dbg(dport_dev, "RCH dport added to %s\n",
1276
			dev_name(&port->dev));
1277
	}
1278

1279
	return dport;
1280
}
1281
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL");
1282

1283
static int add_ep(struct cxl_ep *new)
1284
{
1285
	struct cxl_port *port = new->dport->port;
1286

1287
	guard(device)(&port->dev);
1288
	if (port->dead)
1289
		return -ENXIO;
1290

1291
	return xa_insert(&port->endpoints, (unsigned long)new->ep,
1292
			 new, GFP_KERNEL);
1293
}
1294

1295
/**
1296
 * cxl_add_ep - register an endpoint's interest in a port
1297
 * @dport: the dport that routes to @ep_dev
1298
 * @ep_dev: device representing the endpoint
1299
 *
1300
 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1301
 * When those endpoints depart the port can be destroyed once all
1302
 * endpoints that care about that port have been removed.
1303
 */
1304
static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1305
{
1306
	struct cxl_ep *ep;
1307
	int rc;
1308

1309
	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1310
	if (!ep)
1311
		return -ENOMEM;
1312

1313
	ep->ep = get_device(ep_dev);
1314
	ep->dport = dport;
1315

1316
	rc = add_ep(ep);
1317
	if (rc)
1318
		cxl_ep_release(ep);
1319
	return rc;
1320
}
1321

1322
struct cxl_find_port_ctx {
1323
	const struct device *dport_dev;
1324
	const struct cxl_port *parent_port;
1325
	struct cxl_dport **dport;
1326
};
1327

1328
static int match_port_by_dport(struct device *dev, const void *data)
1329
{
1330
	const struct cxl_find_port_ctx *ctx = data;
1331
	struct cxl_dport *dport;
1332
	struct cxl_port *port;
1333

1334
	if (!is_cxl_port(dev))
1335
		return 0;
1336
	if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1337
		return 0;
1338

1339
	port = to_cxl_port(dev);
1340
	dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1341
	if (ctx->dport)
1342
		*ctx->dport = dport;
1343
	return dport != NULL;
1344
}
1345

1346
static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1347
{
1348
	struct device *dev;
1349

1350
	if (!ctx->dport_dev)
1351
		return NULL;
1352

1353
	dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1354
	if (dev)
1355
		return to_cxl_port(dev);
1356
	return NULL;
1357
}
1358

1359
static struct cxl_port *find_cxl_port(struct device *dport_dev,
1360
				      struct cxl_dport **dport)
1361
{
1362
	struct cxl_find_port_ctx ctx = {
1363
		.dport_dev = dport_dev,
1364
		.dport = dport,
1365
	};
1366
	struct cxl_port *port;
1367

1368
	port = __find_cxl_port(&ctx);
1369
	return port;
1370
}
1371

1372
/*
1373
 * All users of grandparent() are using it to walk PCIe-like switch port
1374
 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1375
 * upstream switch port and N bridges representing downstream switch ports. When
1376
 * bridges stack the grand-parent of a downstream switch port is another
1377
 * downstream switch port in the immediate ancestor switch.
1378
 */
1379
static struct device *grandparent(struct device *dev)
1380
{
1381
	if (dev && dev->parent)
1382
		return dev->parent->parent;
1383
	return NULL;
1384
}
1385

1386
static struct device *endpoint_host(struct cxl_port *endpoint)
1387
{
1388
	struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1389

1390
	if (is_cxl_root(port))
1391
		return port->uport_dev;
1392
	return &port->dev;
1393
}
1394

1395
static void delete_endpoint(void *data)
1396
{
1397
	struct cxl_memdev *cxlmd = data;
1398
	struct cxl_port *endpoint = cxlmd->endpoint;
1399
	struct device *host = endpoint_host(endpoint);
1400

1401
	scoped_guard(device, host) {
1402
		if (host->driver && !endpoint->dead) {
1403
			devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1404
			devm_release_action(host, cxl_unlink_uport, endpoint);
1405
			devm_release_action(host, unregister_port, endpoint);
1406
		}
1407
		cxlmd->endpoint = NULL;
1408
	}
1409
	put_device(&endpoint->dev);
1410
	put_device(host);
1411
}
1412

1413
int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1414
{
1415
	struct device *host = endpoint_host(endpoint);
1416
	struct device *dev = &cxlmd->dev;
1417

1418
	get_device(host);
1419
	get_device(&endpoint->dev);
1420
	cxlmd->depth = endpoint->depth;
1421
	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1422
}
1423
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
1424

1425
/*
1426
 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1427
 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1428
 * for a port to be unregistered is when all memdevs beneath that port have gone
1429
 * through ->remove(). This "bottom-up" removal selectively removes individual
1430
 * child ports manually. This depends on devm_cxl_add_port() to not change is
1431
 * devm action registration order, and for dports to have already been
1432
 * destroyed by del_dports().
1433
 */
1434
static void delete_switch_port(struct cxl_port *port)
1435
{
1436
	devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1437
	devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1438
	devm_release_action(port->dev.parent, unregister_port, port);
1439
}
1440

1441
static void del_dport(struct cxl_dport *dport)
1442
{
1443
	struct cxl_port *port = dport->port;
1444

1445
	devm_release_action(&port->dev, cxl_dport_unlink, dport);
1446
	devm_release_action(&port->dev, cxl_dport_remove, dport);
1447
	devm_kfree(&port->dev, dport);
1448
}
1449

1450
static void del_dports(struct cxl_port *port)
1451
{
1452
	struct cxl_dport *dport;
1453
	unsigned long index;
1454

1455
	device_lock_assert(&port->dev);
1456

1457
	xa_for_each(&port->dports, index, dport)
1458
		del_dport(dport);
1459
}
1460

1461
struct detach_ctx {
1462
	struct cxl_memdev *cxlmd;
1463
	int depth;
1464
};
1465

1466
static int port_has_memdev(struct device *dev, const void *data)
1467
{
1468
	const struct detach_ctx *ctx = data;
1469
	struct cxl_port *port;
1470

1471
	if (!is_cxl_port(dev))
1472
		return 0;
1473

1474
	port = to_cxl_port(dev);
1475
	if (port->depth != ctx->depth)
1476
		return 0;
1477

1478
	return !!cxl_ep_load(port, ctx->cxlmd);
1479
}
1480

1481
static void cxl_detach_ep(void *data)
1482
{
1483
	struct cxl_memdev *cxlmd = data;
1484

1485
	for (int i = cxlmd->depth - 1; i >= 1; i--) {
1486
		struct cxl_port *port, *parent_port;
1487
		struct detach_ctx ctx = {
1488
			.cxlmd = cxlmd,
1489
			.depth = i,
1490
		};
1491
		struct cxl_ep *ep;
1492
		bool died = false;
1493

1494
		struct device *dev __free(put_device) =
1495
			bus_find_device(&cxl_bus_type, NULL, &ctx, port_has_memdev);
1496
		if (!dev)
1497
			continue;
1498
		port = to_cxl_port(dev);
1499

1500
		parent_port = to_cxl_port(port->dev.parent);
1501
		device_lock(&parent_port->dev);
1502
		device_lock(&port->dev);
1503
		ep = cxl_ep_load(port, cxlmd);
1504
		dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1505
			ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1506
		cxl_ep_remove(port, ep);
1507
		if (ep && !port->dead && xa_empty(&port->endpoints) &&
1508
		    !is_cxl_root(parent_port) && parent_port->dev.driver) {
1509
			/*
1510
			 * This was the last ep attached to a dynamically
1511
			 * enumerated port. Block new cxl_add_ep() and garbage
1512
			 * collect the port.
1513
			 */
1514
			died = true;
1515
			port->dead = true;
1516
			del_dports(port);
1517
		}
1518
		device_unlock(&port->dev);
1519

1520
		if (died) {
1521
			dev_dbg(&cxlmd->dev, "delete %s\n",
1522
				dev_name(&port->dev));
1523
			delete_switch_port(port);
1524
		}
1525
		device_unlock(&parent_port->dev);
1526
	}
1527
}
1528

1529
static resource_size_t find_component_registers(struct device *dev)
1530
{
1531
	struct cxl_register_map map;
1532
	struct pci_dev *pdev;
1533

1534
	/*
1535
	 * Theoretically, CXL component registers can be hosted on a
1536
	 * non-PCI device, in practice, only cxl_test hits this case.
1537
	 */
1538
	if (!dev_is_pci(dev))
1539
		return CXL_RESOURCE_NONE;
1540

1541
	pdev = to_pci_dev(dev);
1542

1543
	cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1544
	return map.resource;
1545
}
1546

1547
static int match_port_by_uport(struct device *dev, const void *data)
1548
{
1549
	const struct device *uport_dev = data;
1550
	struct cxl_port *port;
1551

1552
	if (!is_cxl_port(dev))
1553
		return 0;
1554

1555
	port = to_cxl_port(dev);
1556
	return uport_dev == port->uport_dev;
1557
}
1558

1559
/*
1560
 * Function takes a device reference on the port device. Caller should do a
1561
 * put_device() when done.
1562
 */
1563
static struct cxl_port *find_cxl_port_by_uport(struct device *uport_dev)
1564
{
1565
	struct device *dev;
1566

1567
	dev = bus_find_device(&cxl_bus_type, NULL, uport_dev, match_port_by_uport);
1568
	if (dev)
1569
		return to_cxl_port(dev);
1570
	return NULL;
1571
}
1572

1573
static int update_decoder_targets(struct device *dev, void *data)
1574
{
1575
	struct cxl_dport *dport = data;
1576
	struct cxl_switch_decoder *cxlsd;
1577
	struct cxl_decoder *cxld;
1578
	int i;
1579

1580
	if (!is_switch_decoder(dev))
1581
		return 0;
1582

1583
	cxlsd = to_cxl_switch_decoder(dev);
1584
	cxld = &cxlsd->cxld;
1585
	guard(rwsem_write)(&cxl_rwsem.region);
1586

1587
	for (i = 0; i < cxld->interleave_ways; i++) {
1588
		if (cxld->target_map[i] == dport->port_id) {
1589
			cxlsd->target[i] = dport;
1590
			dev_dbg(dev, "dport%d found in target list, index %d\n",
1591
				dport->port_id, i);
1592
			return 1;
1593
		}
1594
	}
1595

1596
	return 0;
1597
}
1598

1599
DEFINE_FREE(del_cxl_dport, struct cxl_dport *, if (!IS_ERR_OR_NULL(_T)) del_dport(_T))
1600
static struct cxl_dport *cxl_port_add_dport(struct cxl_port *port,
1601
					    struct device *dport_dev)
1602
{
1603
	struct cxl_dport *dport;
1604
	int rc;
1605

1606
	device_lock_assert(&port->dev);
1607
	if (!port->dev.driver)
1608
		return ERR_PTR(-ENXIO);
1609

1610
	dport = cxl_find_dport_by_dev(port, dport_dev);
1611
	if (dport) {
1612
		dev_dbg(&port->dev, "dport%d:%s already exists\n",
1613
			dport->port_id, dev_name(dport_dev));
1614
		return ERR_PTR(-EBUSY);
1615
	}
1616

1617
	struct cxl_dport *new_dport __free(del_cxl_dport) =
1618
		devm_cxl_add_dport_by_dev(port, dport_dev);
1619
	if (IS_ERR(new_dport))
1620
		return new_dport;
1621

1622
	cxl_switch_parse_cdat(new_dport);
1623

1624
	if (ida_is_empty(&port->decoder_ida)) {
1625
		rc = devm_cxl_switch_port_decoders_setup(port);
1626
		if (rc)
1627
			return ERR_PTR(rc);
1628
		dev_dbg(&port->dev, "first dport%d:%s added with decoders\n",
1629
			new_dport->port_id, dev_name(dport_dev));
1630
		return no_free_ptr(new_dport);
1631
	}
1632

1633
	/* New dport added, update the decoder targets */
1634
	device_for_each_child(&port->dev, new_dport, update_decoder_targets);
1635

1636
	dev_dbg(&port->dev, "dport%d:%s added\n", new_dport->port_id,
1637
		dev_name(dport_dev));
1638

1639
	return no_free_ptr(new_dport);
1640
}
1641

1642
static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
1643
					      struct cxl_port *parent_port,
1644
					      struct cxl_dport *parent_dport,
1645
					      struct device *uport_dev,
1646
					      struct device *dport_dev)
1647
{
1648
	resource_size_t component_reg_phys;
1649

1650
	device_lock_assert(&parent_port->dev);
1651
	if (!parent_port->dev.driver) {
1652
		dev_warn(ep_dev,
1653
			 "port %s:%s:%s disabled, failed to enumerate CXL.mem\n",
1654
			 dev_name(&parent_port->dev), dev_name(uport_dev),
1655
			 dev_name(dport_dev));
1656
	}
1657

1658
	struct cxl_port *port __free(put_cxl_port) =
1659
		find_cxl_port_by_uport(uport_dev);
1660
	if (!port) {
1661
		component_reg_phys = find_component_registers(uport_dev);
1662
		port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1663
					 component_reg_phys, parent_dport);
1664
		if (IS_ERR(port))
1665
			return ERR_CAST(port);
1666

1667
		/*
1668
		 * retry to make sure a port is found. a port device
1669
		 * reference is taken.
1670
		 */
1671
		port = find_cxl_port_by_uport(uport_dev);
1672
		if (!port)
1673
			return ERR_PTR(-ENODEV);
1674

1675
		dev_dbg(ep_dev, "created port %s:%s\n",
1676
			dev_name(&port->dev), dev_name(port->uport_dev));
1677
	} else {
1678
		/*
1679
		 * Port was created before right before this function is
1680
		 * called. Signal the caller to deal with it.
1681
		 */
1682
		return ERR_PTR(-EAGAIN);
1683
	}
1684

1685
	guard(device)(&port->dev);
1686
	return cxl_port_add_dport(port, dport_dev);
1687
}
1688

1689
static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1690
			      struct device *uport_dev,
1691
			      struct device *dport_dev)
1692
{
1693
	struct device *dparent = grandparent(dport_dev);
1694
	struct cxl_dport *dport, *parent_dport;
1695
	int rc;
1696

1697
	if (is_cxl_host_bridge(dparent)) {
1698
		/*
1699
		 * The iteration reached the topology root without finding the
1700
		 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1701
		 * be re-probed after platform driver attaches.
1702
		 */
1703
		dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1704
			dev_name(dport_dev));
1705
		return -ENXIO;
1706
	}
1707

1708
	struct cxl_port *parent_port __free(put_cxl_port) =
1709
		find_cxl_port_by_uport(dparent->parent);
1710
	if (!parent_port) {
1711
		/* iterate to create this parent_port */
1712
		return -EAGAIN;
1713
	}
1714

1715
	scoped_guard(device, &parent_port->dev) {
1716
		parent_dport = cxl_find_dport_by_dev(parent_port, dparent);
1717
		if (!parent_dport) {
1718
			parent_dport = cxl_port_add_dport(parent_port, dparent);
1719
			if (IS_ERR(parent_dport))
1720
				return PTR_ERR(parent_dport);
1721
		}
1722

1723
		dport = devm_cxl_create_port(&cxlmd->dev, parent_port,
1724
					     parent_dport, uport_dev,
1725
					     dport_dev);
1726
		if (IS_ERR(dport)) {
1727
			/* Port already exists, restart iteration */
1728
			if (PTR_ERR(dport) == -EAGAIN)
1729
				return 0;
1730
			return PTR_ERR(dport);
1731
		}
1732
	}
1733

1734
	rc = cxl_add_ep(dport, &cxlmd->dev);
1735
	if (rc == -EBUSY) {
1736
		/*
1737
		 * "can't" happen, but this error code means
1738
		 * something to the caller, so translate it.
1739
		 */
1740
		rc = -ENXIO;
1741
	}
1742

1743
	return rc;
1744
}
1745

1746
static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
1747
					   struct device *dport_dev)
1748
{
1749
	struct cxl_dport *dport;
1750

1751
	device_lock_assert(&port->dev);
1752
	dport = cxl_find_dport_by_dev(port, dport_dev);
1753
	if (!dport) {
1754
		dport = cxl_port_add_dport(port, dport_dev);
1755
		if (IS_ERR(dport))
1756
			return dport;
1757

1758
		/* New dport added, restart iteration */
1759
		return ERR_PTR(-EAGAIN);
1760
	}
1761

1762
	return dport;
1763
}
1764

1765
int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1766
{
1767
	struct device *dev = &cxlmd->dev;
1768
	struct device *iter;
1769
	int rc;
1770

1771
	/*
1772
	 * Skip intermediate port enumeration in the RCH case, there
1773
	 * are no ports in between a host bridge and an endpoint.
1774
	 */
1775
	if (cxlmd->cxlds->rcd)
1776
		return 0;
1777

1778
	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1779
	if (rc)
1780
		return rc;
1781

1782
	/*
1783
	 * Scan for and add all cxl_ports in this device's ancestry.
1784
	 * Repeat until no more ports are added. Abort if a port add
1785
	 * attempt fails.
1786
	 */
1787
retry:
1788
	for (iter = dev; iter; iter = grandparent(iter)) {
1789
		struct device *dport_dev = grandparent(iter);
1790
		struct device *uport_dev;
1791
		struct cxl_dport *dport;
1792

1793
		if (is_cxl_host_bridge(dport_dev))
1794
			return 0;
1795

1796
		uport_dev = dport_dev->parent;
1797
		if (!uport_dev) {
1798
			dev_warn(dev, "at %s no parent for dport: %s\n",
1799
				 dev_name(iter), dev_name(dport_dev));
1800
			return -ENXIO;
1801
		}
1802

1803
		dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1804
			dev_name(iter), dev_name(dport_dev),
1805
			dev_name(uport_dev));
1806
		struct cxl_port *port __free(put_cxl_port) =
1807
			find_cxl_port_by_uport(uport_dev);
1808
		if (port) {
1809
			dev_dbg(&cxlmd->dev,
1810
				"found already registered port %s:%s\n",
1811
				dev_name(&port->dev),
1812
				dev_name(port->uport_dev));
1813

1814
			/*
1815
			 * RP port enumerated by cxl_acpi without dport will
1816
			 * have the dport added here.
1817
			 */
1818
			scoped_guard(device, &port->dev) {
1819
				dport = find_or_add_dport(port, dport_dev);
1820
				if (IS_ERR(dport)) {
1821
					if (PTR_ERR(dport) == -EAGAIN)
1822
						goto retry;
1823
					return PTR_ERR(dport);
1824
				}
1825
			}
1826

1827
			rc = cxl_add_ep(dport, &cxlmd->dev);
1828

1829
			/*
1830
			 * If the endpoint already exists in the port's list,
1831
			 * that's ok, it was added on a previous pass.
1832
			 * Otherwise, retry in add_port_attach_ep() after taking
1833
			 * the parent_port lock as the current port may be being
1834
			 * reaped.
1835
			 */
1836
			if (rc && rc != -EBUSY)
1837
				return rc;
1838

1839
			cxl_gpf_port_setup(dport);
1840

1841
			/* Any more ports to add between this one and the root? */
1842
			if (!dev_is_cxl_root_child(&port->dev))
1843
				continue;
1844

1845
			return 0;
1846
		}
1847

1848
		rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1849
		/* port missing, try to add parent */
1850
		if (rc == -EAGAIN)
1851
			continue;
1852
		/* failed to add ep or port */
1853
		if (rc)
1854
			return rc;
1855
		/* port added, new descendants possible, start over */
1856
		goto retry;
1857
	}
1858

1859
	return 0;
1860
}
1861
EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
1862

1863
struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1864
				   struct cxl_dport **dport)
1865
{
1866
	return find_cxl_port(pdev->dev.parent, dport);
1867
}
1868
EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
1869

1870
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1871
				   struct cxl_dport **dport)
1872
{
1873
	return find_cxl_port(grandparent(&cxlmd->dev), dport);
1874
}
1875
EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
1876

1877
static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1878
				    struct cxl_port *port)
1879
{
1880
	struct cxl_decoder *cxld = &cxlsd->cxld;
1881
	int i;
1882

1883
	device_lock_assert(&port->dev);
1884

1885
	if (xa_empty(&port->dports))
1886
		return 0;
1887

1888
	guard(rwsem_write)(&cxl_rwsem.region);
1889
	for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1890
		struct cxl_dport *dport = find_dport(port, cxld->target_map[i]);
1891

1892
		if (!dport) {
1893
			/* dport may be activated later */
1894
			continue;
1895
		}
1896
		cxlsd->target[i] = dport;
1897
	}
1898

1899
	return 0;
1900
}
1901

1902
static struct lock_class_key cxl_decoder_key;
1903

1904
/**
1905
 * cxl_decoder_init - Common decoder setup / initialization
1906
 * @port: owning port of this decoder
1907
 * @cxld: common decoder properties to initialize
1908
 *
1909
 * A port may contain one or more decoders. Each of those decoders
1910
 * enable some address space for CXL.mem utilization. A decoder is
1911
 * expected to be configured by the caller before registering via
1912
 * cxl_decoder_add()
1913
 */
1914
static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1915
{
1916
	struct device *dev;
1917
	int rc;
1918

1919
	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1920
	if (rc < 0)
1921
		return rc;
1922

1923
	/* need parent to stick around to release the id */
1924
	get_device(&port->dev);
1925
	cxld->id = rc;
1926

1927
	dev = &cxld->dev;
1928
	device_initialize(dev);
1929
	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1930
	device_set_pm_not_required(dev);
1931
	dev->parent = &port->dev;
1932
	dev->bus = &cxl_bus_type;
1933

1934
	/* Pre initialize an "empty" decoder */
1935
	cxld->interleave_ways = 1;
1936
	cxld->interleave_granularity = PAGE_SIZE;
1937
	cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1938
	cxld->hpa_range = (struct range) {
1939
		.start = 0,
1940
		.end = -1,
1941
	};
1942

1943
	return 0;
1944
}
1945

1946
static int cxl_switch_decoder_init(struct cxl_port *port,
1947
				   struct cxl_switch_decoder *cxlsd,
1948
				   int nr_targets)
1949
{
1950
	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1951
		return -EINVAL;
1952

1953
	cxlsd->nr_targets = nr_targets;
1954
	return cxl_decoder_init(port, &cxlsd->cxld);
1955
}
1956

1957
/**
1958
 * cxl_root_decoder_alloc - Allocate a root level decoder
1959
 * @port: owning CXL root of this decoder
1960
 * @nr_targets: static number of downstream targets
1961
 *
1962
 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1963
 * 'CXL root' decoder is one that decodes from a top-level / static platform
1964
 * firmware description of CXL resources into a CXL standard decode
1965
 * topology.
1966
 */
1967
struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1968
						unsigned int nr_targets)
1969
{
1970
	struct cxl_root_decoder *cxlrd;
1971
	struct cxl_switch_decoder *cxlsd;
1972
	struct cxl_decoder *cxld;
1973
	int rc;
1974

1975
	if (!is_cxl_root(port))
1976
		return ERR_PTR(-EINVAL);
1977

1978
	cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1979
			GFP_KERNEL);
1980
	if (!cxlrd)
1981
		return ERR_PTR(-ENOMEM);
1982

1983
	cxlsd = &cxlrd->cxlsd;
1984
	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1985
	if (rc) {
1986
		kfree(cxlrd);
1987
		return ERR_PTR(rc);
1988
	}
1989

1990
	mutex_init(&cxlrd->range_lock);
1991

1992
	cxld = &cxlsd->cxld;
1993
	cxld->dev.type = &cxl_decoder_root_type;
1994
	/*
1995
	 * cxl_root_decoder_release() special cases negative ids to
1996
	 * detect memregion_alloc() failures.
1997
	 */
1998
	atomic_set(&cxlrd->region_id, -1);
1999
	rc = memregion_alloc(GFP_KERNEL);
2000
	if (rc < 0) {
2001
		put_device(&cxld->dev);
2002
		return ERR_PTR(rc);
2003
	}
2004

2005
	atomic_set(&cxlrd->region_id, rc);
2006
	cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
2007
	return cxlrd;
2008
}
2009
EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL");
2010

2011
/**
2012
 * cxl_switch_decoder_alloc - Allocate a switch level decoder
2013
 * @port: owning CXL switch port of this decoder
2014
 * @nr_targets: max number of dynamically addressable downstream targets
2015
 *
2016
 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
2017
 * 'switch' decoder is any decoder that can be enumerated by PCIe
2018
 * topology and the HDM Decoder Capability. This includes the decoders
2019
 * that sit between Switch Upstream Ports / Switch Downstream Ports and
2020
 * Host Bridges / Root Ports.
2021
 */
2022
struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
2023
						    unsigned int nr_targets)
2024
{
2025
	struct cxl_switch_decoder *cxlsd;
2026
	struct cxl_decoder *cxld;
2027
	int rc;
2028

2029
	if (is_cxl_root(port) || is_cxl_endpoint(port))
2030
		return ERR_PTR(-EINVAL);
2031

2032
	cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
2033
	if (!cxlsd)
2034
		return ERR_PTR(-ENOMEM);
2035

2036
	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
2037
	if (rc) {
2038
		kfree(cxlsd);
2039
		return ERR_PTR(rc);
2040
	}
2041

2042
	cxld = &cxlsd->cxld;
2043
	cxld->dev.type = &cxl_decoder_switch_type;
2044
	return cxlsd;
2045
}
2046
EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL");
2047

2048
/**
2049
 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
2050
 * @port: owning port of this decoder
2051
 *
2052
 * Return: A new cxl decoder to be registered by cxl_decoder_add()
2053
 */
2054
struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
2055
{
2056
	struct cxl_endpoint_decoder *cxled;
2057
	struct cxl_decoder *cxld;
2058
	int rc;
2059

2060
	if (!is_cxl_endpoint(port))
2061
		return ERR_PTR(-EINVAL);
2062

2063
	cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
2064
	if (!cxled)
2065
		return ERR_PTR(-ENOMEM);
2066

2067
	cxled->pos = -1;
2068
	cxled->part = -1;
2069
	cxld = &cxled->cxld;
2070
	rc = cxl_decoder_init(port, cxld);
2071
	if (rc)	 {
2072
		kfree(cxled);
2073
		return ERR_PTR(rc);
2074
	}
2075

2076
	cxld->dev.type = &cxl_decoder_endpoint_type;
2077
	return cxled;
2078
}
2079
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL");
2080

2081
/**
2082
 * cxl_decoder_add_locked - Add a decoder with targets
2083
 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2084
 *
2085
 * Certain types of decoders may not have any targets. The main example of this
2086
 * is an endpoint device. A more awkward example is a hostbridge whose root
2087
 * ports get hot added (technically possible, though unlikely).
2088
 *
2089
 * This is the locked variant of cxl_decoder_add().
2090
 *
2091
 * Context: Process context. Expects the device lock of the port that owns the
2092
 *	    @cxld to be held.
2093
 *
2094
 * Return: Negative error code if the decoder wasn't properly configured; else
2095
 *	   returns 0.
2096
 */
2097
int cxl_decoder_add_locked(struct cxl_decoder *cxld)
2098
{
2099
	struct cxl_port *port;
2100
	struct device *dev;
2101
	int rc;
2102

2103
	if (WARN_ON_ONCE(!cxld))
2104
		return -EINVAL;
2105

2106
	if (WARN_ON_ONCE(IS_ERR(cxld)))
2107
		return PTR_ERR(cxld);
2108

2109
	if (cxld->interleave_ways < 1)
2110
		return -EINVAL;
2111

2112
	dev = &cxld->dev;
2113

2114
	port = to_cxl_port(cxld->dev.parent);
2115
	if (!is_endpoint_decoder(dev)) {
2116
		struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
2117

2118
		rc = decoder_populate_targets(cxlsd, port);
2119
		if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
2120
			dev_err(&port->dev,
2121
				"Failed to populate active decoder targets\n");
2122
			return rc;
2123
		}
2124
	}
2125

2126
	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
2127
	if (rc)
2128
		return rc;
2129

2130
	return device_add(dev);
2131
}
2132
EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL");
2133

2134
/**
2135
 * cxl_decoder_add - Add a decoder with targets
2136
 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2137
 *
2138
 * This is the unlocked variant of cxl_decoder_add_locked().
2139
 * See cxl_decoder_add_locked().
2140
 *
2141
 * Context: Process context. Takes and releases the device lock of the port that
2142
 *	    owns the @cxld.
2143
 */
2144
int cxl_decoder_add(struct cxl_decoder *cxld)
2145
{
2146
	struct cxl_port *port;
2147

2148
	if (WARN_ON_ONCE(!cxld))
2149
		return -EINVAL;
2150

2151
	if (WARN_ON_ONCE(IS_ERR(cxld)))
2152
		return PTR_ERR(cxld);
2153

2154
	port = to_cxl_port(cxld->dev.parent);
2155

2156
	guard(device)(&port->dev);
2157
	return cxl_decoder_add_locked(cxld);
2158
}
2159
EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
2160

2161
static void cxld_unregister(void *dev)
2162
{
2163
	if (is_endpoint_decoder(dev))
2164
		cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
2165
				   DETACH_INVALIDATE);
2166

2167
	device_unregister(dev);
2168
}
2169

2170
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
2171
{
2172
	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2173
}
2174
EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
2175

2176
/**
2177
 * __cxl_driver_register - register a driver for the cxl bus
2178
 * @cxl_drv: cxl driver structure to attach
2179
 * @owner: owning module/driver
2180
 * @modname: KBUILD_MODNAME for parent driver
2181
 */
2182
int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2183
			  const char *modname)
2184
{
2185
	if (!cxl_drv->probe) {
2186
		pr_debug("%s ->probe() must be specified\n", modname);
2187
		return -EINVAL;
2188
	}
2189

2190
	if (!cxl_drv->name) {
2191
		pr_debug("%s ->name must be specified\n", modname);
2192
		return -EINVAL;
2193
	}
2194

2195
	if (!cxl_drv->id) {
2196
		pr_debug("%s ->id must be specified\n", modname);
2197
		return -EINVAL;
2198
	}
2199

2200
	cxl_drv->drv.bus = &cxl_bus_type;
2201
	cxl_drv->drv.owner = owner;
2202
	cxl_drv->drv.mod_name = modname;
2203
	cxl_drv->drv.name = cxl_drv->name;
2204

2205
	return driver_register(&cxl_drv->drv);
2206
}
2207
EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL");
2208

2209
void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2210
{
2211
	driver_unregister(&cxl_drv->drv);
2212
}
2213
EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL");
2214

2215
static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2216
{
2217
	return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
2218
			      cxl_device_id(dev));
2219
}
2220

2221
static int cxl_bus_match(struct device *dev, const struct device_driver *drv)
2222
{
2223
	return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2224
}
2225

2226
static int cxl_bus_probe(struct device *dev)
2227
{
2228
	int rc;
2229

2230
	rc = to_cxl_drv(dev->driver)->probe(dev);
2231
	dev_dbg(dev, "probe: %d\n", rc);
2232
	return rc;
2233
}
2234

2235
static void cxl_bus_remove(struct device *dev)
2236
{
2237
	struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2238

2239
	if (cxl_drv->remove)
2240
		cxl_drv->remove(dev);
2241
}
2242

2243
static struct workqueue_struct *cxl_bus_wq;
2244

2245
static int cxl_rescan_attach(struct device *dev, void *data)
2246
{
2247
	int rc = device_attach(dev);
2248

2249
	dev_vdbg(dev, "rescan: %s\n", rc ? "attach" : "detached");
2250

2251
	return 0;
2252
}
2253

2254
static void cxl_bus_rescan_queue(struct work_struct *w)
2255
{
2256
	bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_rescan_attach);
2257
}
2258

2259
void cxl_bus_rescan(void)
2260
{
2261
	static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2262

2263
	queue_work(cxl_bus_wq, &rescan_work);
2264
}
2265
EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL");
2266

2267
void cxl_bus_drain(void)
2268
{
2269
	drain_workqueue(cxl_bus_wq);
2270
}
2271
EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
2272

2273
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2274
{
2275
	return queue_work(cxl_bus_wq, &cxlmd->detach_work);
2276
}
2277
EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
2278

2279
static void add_latency(struct access_coordinate *c, long latency)
2280
{
2281
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2282
		c[i].write_latency += latency;
2283
		c[i].read_latency += latency;
2284
	}
2285
}
2286

2287
static bool coordinates_valid(struct access_coordinate *c)
2288
{
2289
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2290
		if (c[i].read_bandwidth && c[i].write_bandwidth &&
2291
		    c[i].read_latency && c[i].write_latency)
2292
			continue;
2293
		return false;
2294
	}
2295

2296
	return true;
2297
}
2298

2299
static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw)
2300
{
2301
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2302
		c[i].write_bandwidth = min(c[i].write_bandwidth, bw);
2303
		c[i].read_bandwidth = min(c[i].read_bandwidth, bw);
2304
	}
2305
}
2306

2307
static void set_access_coordinates(struct access_coordinate *out,
2308
				   struct access_coordinate *in)
2309
{
2310
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
2311
		out[i] = in[i];
2312
}
2313

2314
static bool parent_port_is_cxl_root(struct cxl_port *port)
2315
{
2316
	return is_cxl_root(to_cxl_port(port->dev.parent));
2317
}
2318

2319
/**
2320
 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2321
 *				   of CXL path
2322
 * @port: endpoint cxl_port
2323
 * @coord: output performance data
2324
 *
2325
 * Return: errno on failure, 0 on success.
2326
 */
2327
int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2328
				      struct access_coordinate *coord)
2329
{
2330
	struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
2331
	struct access_coordinate c[] = {
2332
		{
2333
			.read_bandwidth = UINT_MAX,
2334
			.write_bandwidth = UINT_MAX,
2335
		},
2336
		{
2337
			.read_bandwidth = UINT_MAX,
2338
			.write_bandwidth = UINT_MAX,
2339
		},
2340
	};
2341
	struct cxl_port *iter = port;
2342
	struct cxl_dport *dport;
2343
	struct pci_dev *pdev;
2344
	struct device *dev;
2345
	unsigned int bw;
2346
	bool is_cxl_root;
2347

2348
	if (!is_cxl_endpoint(port))
2349
		return -EINVAL;
2350

2351
	/*
2352
	 * Skip calculation for RCD. Expectation is HMAT already covers RCD case
2353
	 * since RCH does not support hotplug.
2354
	 */
2355
	if (cxlmd->cxlds->rcd)
2356
		return 0;
2357

2358
	/*
2359
	 * Exit the loop when the parent port of the current iter port is cxl
2360
	 * root. The iterative loop starts at the endpoint and gathers the
2361
	 * latency of the CXL link from the current device/port to the connected
2362
	 * downstream port each iteration.
2363
	 */
2364
	do {
2365
		dport = iter->parent_dport;
2366
		iter = to_cxl_port(iter->dev.parent);
2367
		is_cxl_root = parent_port_is_cxl_root(iter);
2368

2369
		/*
2370
		 * There's no valid access_coordinate for a root port since RPs do not
2371
		 * have CDAT and therefore needs to be skipped.
2372
		 */
2373
		if (!is_cxl_root) {
2374
			if (!coordinates_valid(dport->coord))
2375
				return -EINVAL;
2376
			cxl_coordinates_combine(c, c, dport->coord);
2377
		}
2378
		add_latency(c, dport->link_latency);
2379
	} while (!is_cxl_root);
2380

2381
	dport = iter->parent_dport;
2382
	/* Retrieve HB coords */
2383
	if (!coordinates_valid(dport->coord))
2384
		return -EINVAL;
2385
	cxl_coordinates_combine(c, c, dport->coord);
2386

2387
	dev = port->uport_dev->parent;
2388
	if (!dev_is_pci(dev))
2389
		return -ENODEV;
2390

2391
	/* Get the calculated PCI paths bandwidth */
2392
	pdev = to_pci_dev(dev);
2393
	bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL);
2394
	if (bw == 0)
2395
		return -ENXIO;
2396
	bw /= BITS_PER_BYTE;
2397

2398
	set_min_bandwidth(c, bw);
2399
	set_access_coordinates(coord, c);
2400

2401
	return 0;
2402
}
2403
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL");
2404

2405
int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
2406
					struct access_coordinate *c)
2407
{
2408
	struct cxl_dport *dport = port->parent_dport;
2409

2410
	/* Check this port is connected to a switch DSP and not an RP */
2411
	if (parent_port_is_cxl_root(to_cxl_port(port->dev.parent)))
2412
		return -ENODEV;
2413

2414
	if (!coordinates_valid(dport->coord))
2415
		return -EINVAL;
2416

2417
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2418
		c[i].read_bandwidth = dport->coord[i].read_bandwidth;
2419
		c[i].write_bandwidth = dport->coord[i].write_bandwidth;
2420
	}
2421

2422
	return 0;
2423
}
2424

2425
/* for user tooling to ensure port disable work has completed */
2426
static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2427
{
2428
	if (sysfs_streq(buf, "1")) {
2429
		flush_workqueue(cxl_bus_wq);
2430
		return count;
2431
	}
2432

2433
	return -EINVAL;
2434
}
2435

2436
static BUS_ATTR_WO(flush);
2437

2438
static struct attribute *cxl_bus_attributes[] = {
2439
	&bus_attr_flush.attr,
2440
	NULL,
2441
};
2442

2443
static struct attribute_group cxl_bus_attribute_group = {
2444
	.attrs = cxl_bus_attributes,
2445
};
2446

2447
static const struct attribute_group *cxl_bus_attribute_groups[] = {
2448
	&cxl_bus_attribute_group,
2449
	NULL,
2450
};
2451

2452
const struct bus_type cxl_bus_type = {
2453
	.name = "cxl",
2454
	.uevent = cxl_bus_uevent,
2455
	.match = cxl_bus_match,
2456
	.probe = cxl_bus_probe,
2457
	.remove = cxl_bus_remove,
2458
	.bus_groups = cxl_bus_attribute_groups,
2459
};
2460
EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL");
2461

2462
static struct dentry *cxl_debugfs;
2463

2464
struct dentry *cxl_debugfs_create_dir(const char *dir)
2465
{
2466
	return debugfs_create_dir(dir, cxl_debugfs);
2467
}
2468
EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL");
2469

2470
static __init int cxl_core_init(void)
2471
{
2472
	int rc;
2473

2474
	cxl_debugfs = debugfs_create_dir("cxl", NULL);
2475

2476
	if (einj_cxl_is_initialized())
2477
		debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL,
2478
				    &einj_cxl_available_error_type_fops);
2479

2480
	cxl_mbox_init();
2481

2482
	rc = cxl_memdev_init();
2483
	if (rc)
2484
		return rc;
2485

2486
	cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2487
	if (!cxl_bus_wq) {
2488
		rc = -ENOMEM;
2489
		goto err_wq;
2490
	}
2491

2492
	rc = bus_register(&cxl_bus_type);
2493
	if (rc)
2494
		goto err_bus;
2495

2496
	rc = cxl_region_init();
2497
	if (rc)
2498
		goto err_region;
2499

2500
	rc = cxl_ras_init();
2501
	if (rc)
2502
		goto err_ras;
2503

2504
	return 0;
2505

2506
err_ras:
2507
	cxl_region_exit();
2508
err_region:
2509
	bus_unregister(&cxl_bus_type);
2510
err_bus:
2511
	destroy_workqueue(cxl_bus_wq);
2512
err_wq:
2513
	cxl_memdev_exit();
2514
	return rc;
2515
}
2516

2517
static void cxl_core_exit(void)
2518
{
2519
	cxl_ras_exit();
2520
	cxl_region_exit();
2521
	bus_unregister(&cxl_bus_type);
2522
	destroy_workqueue(cxl_bus_wq);
2523
	cxl_memdev_exit();
2524
	debugfs_remove_recursive(cxl_debugfs);
2525
}
2526

2527
subsys_initcall(cxl_core_init);
2528
module_exit(cxl_core_exit);
2529
MODULE_DESCRIPTION("CXL: Core Compute Express Link support");
2530
MODULE_LICENSE("GPL v2");
2531
MODULE_IMPORT_NS("CXL");
2532

2533