1
// SPDX-License-Identifier: GPL-2.0-only
2
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3
#include <linux/seq_file.h>
4
#include <linux/device.h>
5
#include <linux/delay.h>
6

7
#include "cxlmem.h"
8
#include "core.h"
9

10
/**
11
 * DOC: cxl core hdm
12
 *
13
 * Compute Express Link Host Managed Device Memory, starting with the
14
 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15
 * instances per CXL port and per CXL endpoint. Define common helpers
16
 * for enumerating these registers and capabilities.
17
 */
18

19
struct cxl_rwsem cxl_rwsem = {
20
	.region = __RWSEM_INITIALIZER(cxl_rwsem.region),
21
	.dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa),
22
};
23

24
static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld)
25
{
26
	int rc;
27

28
	rc = cxl_decoder_add_locked(cxld);
29
	if (rc) {
30
		put_device(&cxld->dev);
31
		dev_err(&port->dev, "Failed to add decoder\n");
32
		return rc;
33
	}
34

35
	rc = cxl_decoder_autoremove(&port->dev, cxld);
36
	if (rc)
37
		return rc;
38

39
	dev_dbg(port->uport_dev, "%s added to %s\n",
40
		dev_name(&cxld->dev), dev_name(&port->dev));
41

42
	return 0;
43
}
44

45
/*
46
 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
47
 * single ported host-bridges need not publish a decoder capability when a
48
 * passthrough decode can be assumed, i.e. all transactions that the uport sees
49
 * are claimed and passed to the single dport. Disable the range until the first
50
 * CXL region is enumerated / activated.
51
 */
52
static int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
53
{
54
	struct cxl_switch_decoder *cxlsd;
55
	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
56

57
	/*
58
	 * Capability checks are moot for passthrough decoders, support
59
	 * any and all possibilities.
60
	 */
61
	cxlhdm->interleave_mask = ~0U;
62
	cxlhdm->iw_cap_mask = ~0UL;
63

64
	cxlsd = cxl_switch_decoder_alloc(port, 1);
65
	if (IS_ERR(cxlsd))
66
		return PTR_ERR(cxlsd);
67

68
	device_lock_assert(&port->dev);
69

70
	return add_hdm_decoder(port, &cxlsd->cxld);
71
}
72

73
static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
74
{
75
	u32 hdm_cap;
76

77
	hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
78
	cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
79
	cxlhdm->target_count =
80
		FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
81
	if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
82
		cxlhdm->interleave_mask |= GENMASK(11, 8);
83
	if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
84
		cxlhdm->interleave_mask |= GENMASK(14, 12);
85
	cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);
86
	if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))
87
		cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);
88
	if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))
89
		cxlhdm->iw_cap_mask |= BIT(16);
90
}
91

92
static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
93
{
94
	struct cxl_hdm *cxlhdm;
95
	void __iomem *hdm;
96
	u32 ctrl;
97
	int i;
98

99
	if (!info)
100
		return false;
101

102
	cxlhdm = dev_get_drvdata(&info->port->dev);
103
	hdm = cxlhdm->regs.hdm_decoder;
104

105
	if (!hdm)
106
		return true;
107

108
	/*
109
	 * If HDM decoders are present and the driver is in control of
110
	 * Mem_Enable skip DVSEC based emulation
111
	 */
112
	if (!info->mem_enabled)
113
		return false;
114

115
	/*
116
	 * If any decoders are committed already, there should not be any
117
	 * emulated DVSEC decoders.
118
	 */
119
	for (i = 0; i < cxlhdm->decoder_count; i++) {
120
		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
121
		dev_dbg(&info->port->dev,
122
			"decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
123
			info->port->id, i,
124
			FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
125
			readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
126
			readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
127
			readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
128
			readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
129
		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
130
			return false;
131
	}
132

133
	return true;
134
}
135

136
/**
137
 * devm_cxl_setup_hdm - map HDM decoder component registers
138
 * @port: cxl_port to map
139
 * @info: cached DVSEC range register info
140
 */
141
static struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
142
					  struct cxl_endpoint_dvsec_info *info)
143
{
144
	struct cxl_register_map *reg_map = &port->reg_map;
145
	struct device *dev = &port->dev;
146
	struct cxl_hdm *cxlhdm;
147
	int rc;
148

149
	cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
150
	if (!cxlhdm)
151
		return ERR_PTR(-ENOMEM);
152
	cxlhdm->port = port;
153
	dev_set_drvdata(dev, cxlhdm);
154

155
	/* Memory devices can configure device HDM using DVSEC range regs. */
156
	if (reg_map->resource == CXL_RESOURCE_NONE) {
157
		if (!info || !info->mem_enabled) {
158
			dev_err(dev, "No component registers mapped\n");
159
			return ERR_PTR(-ENXIO);
160
		}
161

162
		cxlhdm->decoder_count = info->ranges;
163
		return cxlhdm;
164
	}
165

166
	if (!reg_map->component_map.hdm_decoder.valid) {
167
		dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
168
		/* unique error code to indicate no HDM decoder capability */
169
		return ERR_PTR(-ENODEV);
170
	}
171

172
	rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,
173
				    BIT(CXL_CM_CAP_CAP_ID_HDM));
174
	if (rc) {
175
		dev_err(dev, "Failed to map HDM capability.\n");
176
		return ERR_PTR(rc);
177
	}
178

179
	parse_hdm_decoder_caps(cxlhdm);
180
	if (cxlhdm->decoder_count == 0) {
181
		dev_err(dev, "Spec violation. Caps invalid\n");
182
		return ERR_PTR(-ENXIO);
183
	}
184

185
	/*
186
	 * Now that the hdm capability is parsed, decide if range
187
	 * register emulation is needed and fixup cxlhdm accordingly.
188
	 */
189
	if (should_emulate_decoders(info)) {
190
		dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
191
			str_plural(info->ranges));
192
		cxlhdm->decoder_count = info->ranges;
193
	}
194

195
	return cxlhdm;
196
}
197

198
static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
199
{
200
	unsigned long long start = r->start, end = r->end;
201

202
	seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
203
		   r->name);
204
}
205

206
void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
207
{
208
	struct resource *p1, *p2;
209

210
	guard(rwsem_read)(&cxl_rwsem.dpa);
211
	for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
212
		__cxl_dpa_debug(file, p1, 0);
213
		for (p2 = p1->child; p2; p2 = p2->sibling)
214
			__cxl_dpa_debug(file, p2, 1);
215
	}
216
}
217
EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL");
218

219
/* See request_skip() kernel-doc */
220
static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds,
221
				     const resource_size_t skip_base,
222
				     const resource_size_t skip_len,
223
				     const char *requester)
224
{
225
	const resource_size_t skip_end = skip_base + skip_len - 1;
226

227
	for (int i = 0; i < cxlds->nr_partitions; i++) {
228
		const struct resource *part_res = &cxlds->part[i].res;
229
		resource_size_t adjust_start, adjust_end, size;
230

231
		adjust_start = max(skip_base, part_res->start);
232
		adjust_end = min(skip_end, part_res->end);
233

234
		if (adjust_end < adjust_start)
235
			continue;
236

237
		size = adjust_end - adjust_start + 1;
238

239
		if (!requester)
240
			__release_region(&cxlds->dpa_res, adjust_start, size);
241
		else if (!__request_region(&cxlds->dpa_res, adjust_start, size,
242
					   requester, 0))
243
			return adjust_start - skip_base;
244
	}
245

246
	return skip_len;
247
}
248
#define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL)
249

250
/*
251
 * Must be called in a context that synchronizes against this decoder's
252
 * port ->remove() callback (like an endpoint decoder sysfs attribute)
253
 */
254
static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
255
{
256
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
257
	struct cxl_port *port = cxled_to_port(cxled);
258
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
259
	struct resource *res = cxled->dpa_res;
260
	resource_size_t skip_start;
261

262
	lockdep_assert_held_write(&cxl_rwsem.dpa);
263

264
	/* save @skip_start, before @res is released */
265
	skip_start = res->start - cxled->skip;
266
	__release_region(&cxlds->dpa_res, res->start, resource_size(res));
267
	if (cxled->skip)
268
		release_skip(cxlds, skip_start, cxled->skip);
269
	cxled->skip = 0;
270
	cxled->dpa_res = NULL;
271
	put_device(&cxled->cxld.dev);
272
	port->hdm_end--;
273
}
274

275
static void cxl_dpa_release(void *cxled)
276
{
277
	guard(rwsem_write)(&cxl_rwsem.dpa);
278
	__cxl_dpa_release(cxled);
279
}
280

281
/*
282
 * Must be called from context that will not race port device
283
 * unregistration, like decoder sysfs attribute methods
284
 */
285
static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
286
{
287
	struct cxl_port *port = cxled_to_port(cxled);
288

289
	lockdep_assert_held_write(&cxl_rwsem.dpa);
290
	devm_remove_action(&port->dev, cxl_dpa_release, cxled);
291
	__cxl_dpa_release(cxled);
292
}
293

294
/**
295
 * request_skip() - Track DPA 'skip' in @cxlds->dpa_res resource tree
296
 * @cxlds: CXL.mem device context that parents @cxled
297
 * @cxled: Endpoint decoder establishing new allocation that skips lower DPA
298
 * @skip_base: DPA < start of new DPA allocation (DPAnew)
299
 * @skip_len: @skip_base + @skip_len == DPAnew
300
 *
301
 * DPA 'skip' arises from out-of-sequence DPA allocation events relative
302
 * to free capacity across multiple partitions. It is a wasteful event
303
 * as usable DPA gets thrown away, but if a deployment has, for example,
304
 * a dual RAM+PMEM device, wants to use PMEM, and has unallocated RAM
305
 * DPA, the free RAM DPA must be sacrificed to start allocating PMEM.
306
 * See third "Implementation Note" in CXL 3.1 8.2.4.19.13 "Decoder
307
 * Protection" for more details.
308
 *
309
 * A 'skip' always covers the last allocated DPA in a previous partition
310
 * to the start of the current partition to allocate.  Allocations never
311
 * start in the middle of a partition, and allocations are always
312
 * de-allocated in reverse order (see cxl_dpa_free(), or natural devm
313
 * unwind order from forced in-order allocation).
314
 *
315
 * If @cxlds->nr_partitions was guaranteed to be <= 2 then the 'skip'
316
 * would always be contained to a single partition. Given
317
 * @cxlds->nr_partitions may be > 2 it results in cases where the 'skip'
318
 * might span "tail capacity of partition[0], all of partition[1], ...,
319
 * all of partition[N-1]" to support allocating from partition[N]. That
320
 * in turn interacts with the partition 'struct resource' boundaries
321
 * within @cxlds->dpa_res whereby 'skip' requests need to be divided by
322
 * partition. I.e. this is a quirk of using a 'struct resource' tree to
323
 * detect range conflicts while also tracking partition boundaries in
324
 * @cxlds->dpa_res.
325
 */
326
static int request_skip(struct cxl_dev_state *cxlds,
327
			struct cxl_endpoint_decoder *cxled,
328
			const resource_size_t skip_base,
329
			const resource_size_t skip_len)
330
{
331
	resource_size_t skipped = __adjust_skip(cxlds, skip_base, skip_len,
332
						dev_name(&cxled->cxld.dev));
333

334
	if (skipped == skip_len)
335
		return 0;
336

337
	dev_dbg(cxlds->dev,
338
		"%s: failed to reserve skipped space (%pa %pa %pa)\n",
339
		dev_name(&cxled->cxld.dev), &skip_base, &skip_len, &skipped);
340

341
	release_skip(cxlds, skip_base, skipped);
342

343
	return -EBUSY;
344
}
345

346
static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
347
			     resource_size_t base, resource_size_t len,
348
			     resource_size_t skipped)
349
{
350
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
351
	struct cxl_port *port = cxled_to_port(cxled);
352
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
353
	struct device *dev = &port->dev;
354
	struct resource *res;
355
	int rc;
356

357
	lockdep_assert_held_write(&cxl_rwsem.dpa);
358

359
	if (!len) {
360
		dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
361
			 port->id, cxled->cxld.id);
362
		return -EINVAL;
363
	}
364

365
	if (cxled->dpa_res) {
366
		dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
367
			port->id, cxled->cxld.id, cxled->dpa_res);
368
		return -EBUSY;
369
	}
370

371
	if (port->hdm_end + 1 != cxled->cxld.id) {
372
		/*
373
		 * Assumes alloc and commit order is always in hardware instance
374
		 * order per expectations from 8.2.5.12.20 Committing Decoder
375
		 * Programming that enforce decoder[m] committed before
376
		 * decoder[m+1] commit start.
377
		 */
378
		dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
379
			cxled->cxld.id, port->id, port->hdm_end + 1);
380
		return -EBUSY;
381
	}
382

383
	if (skipped) {
384
		rc = request_skip(cxlds, cxled, base - skipped, skipped);
385
		if (rc)
386
			return rc;
387
	}
388
	res = __request_region(&cxlds->dpa_res, base, len,
389
			       dev_name(&cxled->cxld.dev), 0);
390
	if (!res) {
391
		dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
392
			port->id, cxled->cxld.id);
393
		if (skipped)
394
			release_skip(cxlds, base - skipped, skipped);
395
		return -EBUSY;
396
	}
397
	cxled->dpa_res = res;
398
	cxled->skip = skipped;
399

400
	/*
401
	 * When allocating new capacity, ->part is already set, when
402
	 * discovering decoder settings at initial enumeration, ->part
403
	 * is not set.
404
	 */
405
	if (cxled->part < 0)
406
		for (int i = 0; cxlds->nr_partitions; i++)
407
			if (resource_contains(&cxlds->part[i].res, res)) {
408
				cxled->part = i;
409
				break;
410
			}
411

412
	if (cxled->part < 0)
413
		dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n",
414
			 port->id, cxled->cxld.id, res);
415

416
	port->hdm_end++;
417
	get_device(&cxled->cxld.dev);
418
	return 0;
419
}
420

421
static int add_dpa_res(struct device *dev, struct resource *parent,
422
		       struct resource *res, resource_size_t start,
423
		       resource_size_t size, const char *type)
424
{
425
	int rc;
426

427
	*res = (struct resource) {
428
		.name = type,
429
		.start = start,
430
		.end =  start + size - 1,
431
		.flags = IORESOURCE_MEM,
432
	};
433
	if (resource_size(res) == 0) {
434
		dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
435
		return 0;
436
	}
437
	rc = request_resource(parent, res);
438
	if (rc) {
439
		dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
440
			res, rc);
441
		return rc;
442
	}
443

444
	dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
445

446
	return 0;
447
}
448

449
static const char *cxl_mode_name(enum cxl_partition_mode mode)
450
{
451
	switch (mode) {
452
	case CXL_PARTMODE_RAM:
453
		return "ram";
454
	case CXL_PARTMODE_PMEM:
455
		return "pmem";
456
	default:
457
		return "";
458
	};
459
}
460

461
/* if this fails the caller must destroy @cxlds, there is no recovery */
462
int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
463
{
464
	struct device *dev = cxlds->dev;
465

466
	guard(rwsem_write)(&cxl_rwsem.dpa);
467

468
	if (cxlds->nr_partitions)
469
		return -EBUSY;
470

471
	if (!info->size || !info->nr_partitions) {
472
		cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
473
		cxlds->nr_partitions = 0;
474
		return 0;
475
	}
476

477
	cxlds->dpa_res = DEFINE_RES_MEM(0, info->size);
478

479
	for (int i = 0; i < info->nr_partitions; i++) {
480
		const struct cxl_dpa_part_info *part = &info->part[i];
481
		int rc;
482

483
		cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID;
484
		cxlds->part[i].mode = part->mode;
485

486
		/* Require ordered + contiguous partitions */
487
		if (i) {
488
			const struct cxl_dpa_part_info *prev = &info->part[i - 1];
489

490
			if (prev->range.end + 1 != part->range.start)
491
				return -EINVAL;
492
		}
493
		rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res,
494
				 part->range.start, range_len(&part->range),
495
				 cxl_mode_name(part->mode));
496
		if (rc)
497
			return rc;
498
		cxlds->nr_partitions++;
499
	}
500

501
	return 0;
502
}
503
EXPORT_SYMBOL_GPL(cxl_dpa_setup);
504

505
int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
506
				resource_size_t base, resource_size_t len,
507
				resource_size_t skipped)
508
{
509
	struct cxl_port *port = cxled_to_port(cxled);
510
	int rc;
511

512
	scoped_guard(rwsem_write, &cxl_rwsem.dpa)
513
		rc = __cxl_dpa_reserve(cxled, base, len, skipped);
514

515
	if (rc)
516
		return rc;
517

518
	return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
519
}
520
EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL");
521

522
resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
523
{
524
	guard(rwsem_read)(&cxl_rwsem.dpa);
525
	if (cxled->dpa_res)
526
		return resource_size(cxled->dpa_res);
527

528
	return 0;
529
}
530

531
resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
532
{
533
	resource_size_t base = -1;
534

535
	lockdep_assert_held(&cxl_rwsem.dpa);
536
	if (cxled->dpa_res)
537
		base = cxled->dpa_res->start;
538

539
	return base;
540
}
541

542
bool cxl_resource_contains_addr(const struct resource *res, const resource_size_t addr)
543
{
544
	struct resource _addr = DEFINE_RES_MEM(addr, 1);
545

546
	return resource_contains(res, &_addr);
547
}
548

549
int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
550
{
551
	struct cxl_port *port = cxled_to_port(cxled);
552
	struct device *dev = &cxled->cxld.dev;
553

554
	guard(rwsem_write)(&cxl_rwsem.dpa);
555
	if (!cxled->dpa_res)
556
		return 0;
557
	if (cxled->cxld.region) {
558
		dev_dbg(dev, "decoder assigned to: %s\n",
559
			dev_name(&cxled->cxld.region->dev));
560
		return -EBUSY;
561
	}
562
	if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
563
		dev_dbg(dev, "decoder enabled\n");
564
		return -EBUSY;
565
	}
566
	if (cxled->cxld.id != port->hdm_end) {
567
		dev_dbg(dev, "expected decoder%d.%d\n", port->id,
568
			port->hdm_end);
569
		return -EBUSY;
570
	}
571

572
	devm_cxl_dpa_release(cxled);
573
	return 0;
574
}
575

576
int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
577
		     enum cxl_partition_mode mode)
578
{
579
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
580
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
581
	struct device *dev = &cxled->cxld.dev;
582
	int part;
583

584
	guard(rwsem_write)(&cxl_rwsem.dpa);
585
	if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)
586
		return -EBUSY;
587

588
	for (part = 0; part < cxlds->nr_partitions; part++)
589
		if (cxlds->part[part].mode == mode)
590
			break;
591

592
	if (part >= cxlds->nr_partitions) {
593
		dev_dbg(dev, "unsupported mode: %d\n", mode);
594
		return -EINVAL;
595
	}
596

597
	if (!resource_size(&cxlds->part[part].res)) {
598
		dev_dbg(dev, "no available capacity for mode: %d\n", mode);
599
		return -ENXIO;
600
	}
601

602
	cxled->part = part;
603
	return 0;
604
}
605

606
static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
607
{
608
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
609
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
610
	struct device *dev = &cxled->cxld.dev;
611
	struct resource *res, *prev = NULL;
612
	resource_size_t start, avail, skip, skip_start;
613
	struct resource *p, *last;
614
	int part;
615

616
	guard(rwsem_write)(&cxl_rwsem.dpa);
617
	if (cxled->cxld.region) {
618
		dev_dbg(dev, "decoder attached to %s\n",
619
			dev_name(&cxled->cxld.region->dev));
620
		return -EBUSY;
621
	}
622

623
	if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
624
		dev_dbg(dev, "decoder enabled\n");
625
		return -EBUSY;
626
	}
627

628
	part = cxled->part;
629
	if (part < 0) {
630
		dev_dbg(dev, "partition not set\n");
631
		return -EBUSY;
632
	}
633

634
	res = &cxlds->part[part].res;
635
	for (p = res->child, last = NULL; p; p = p->sibling)
636
		last = p;
637
	if (last)
638
		start = last->end + 1;
639
	else
640
		start = res->start;
641

642
	/*
643
	 * To allocate at partition N, a skip needs to be calculated for all
644
	 * unallocated space at lower partitions indices.
645
	 *
646
	 * If a partition has any allocations, the search can end because a
647
	 * previous cxl_dpa_alloc() invocation is assumed to have accounted for
648
	 * all previous partitions.
649
	 */
650
	skip_start = CXL_RESOURCE_NONE;
651
	for (int i = part; i; i--) {
652
		prev = &cxlds->part[i - 1].res;
653
		for (p = prev->child, last = NULL; p; p = p->sibling)
654
			last = p;
655
		if (last) {
656
			skip_start = last->end + 1;
657
			break;
658
		}
659
		skip_start = prev->start;
660
	}
661

662
	avail = res->end - start + 1;
663
	if (skip_start == CXL_RESOURCE_NONE)
664
		skip = 0;
665
	else
666
		skip = res->start - skip_start;
667

668
	if (size > avail) {
669
		dev_dbg(dev, "%llu exceeds available %s capacity: %llu\n", size,
670
			res->name, (u64)avail);
671
		return -ENOSPC;
672
	}
673

674
	return __cxl_dpa_reserve(cxled, start, size, skip);
675
}
676

677
int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
678
{
679
	struct cxl_port *port = cxled_to_port(cxled);
680
	int rc;
681

682
	rc = __cxl_dpa_alloc(cxled, size);
683
	if (rc)
684
		return rc;
685

686
	return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
687
}
688

689
static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
690
{
691
	u16 eig;
692
	u8 eiw;
693

694
	/*
695
	 * Input validation ensures these warns never fire, but otherwise
696
	 * suppress unititalized variable usage warnings.
697
	 */
698
	if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
699
		      "invalid interleave_ways: %d\n", cxld->interleave_ways))
700
		return;
701
	if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
702
		      "invalid interleave_granularity: %d\n",
703
		      cxld->interleave_granularity))
704
		return;
705

706
	u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
707
	u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
708
	*ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
709
}
710

711
static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
712
{
713
	u32p_replace_bits(ctrl,
714
			  !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
715
			  CXL_HDM_DECODER0_CTRL_HOSTONLY);
716
}
717

718
static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
719
{
720
	struct cxl_dport **t = &cxlsd->target[0];
721
	int ways = cxlsd->cxld.interleave_ways;
722

723
	*tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
724
	if (ways > 1)
725
		*tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
726
	if (ways > 2)
727
		*tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
728
	if (ways > 3)
729
		*tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
730
	if (ways > 4)
731
		*tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
732
	if (ways > 5)
733
		*tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
734
	if (ways > 6)
735
		*tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
736
	if (ways > 7)
737
		*tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
738
}
739

740
/*
741
 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
742
 * committed or error within 10ms, but just be generous with 20ms to account for
743
 * clock skew and other marginal behavior
744
 */
745
#define COMMIT_TIMEOUT_MS 20
746
static int cxld_await_commit(void __iomem *hdm, int id)
747
{
748
	u32 ctrl;
749
	int i;
750

751
	for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
752
		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
753
		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
754
			ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
755
			writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
756
			return -EIO;
757
		}
758
		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
759
			return 0;
760
		fsleep(1000);
761
	}
762

763
	return -ETIMEDOUT;
764
}
765

766
static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)
767
{
768
	int id = cxld->id;
769
	u64 base, size;
770
	u32 ctrl;
771

772
	/* common decoder settings */
773
	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
774
	cxld_set_interleave(cxld, &ctrl);
775
	cxld_set_type(cxld, &ctrl);
776
	base = cxld->hpa_range.start;
777
	size = range_len(&cxld->hpa_range);
778

779
	writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
780
	writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
781
	writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
782
	writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
783

784
	if (is_switch_decoder(&cxld->dev)) {
785
		struct cxl_switch_decoder *cxlsd =
786
			to_cxl_switch_decoder(&cxld->dev);
787
		void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
788
		void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
789
		u64 targets;
790

791
		cxlsd_set_targets(cxlsd, &targets);
792
		writel(upper_32_bits(targets), tl_hi);
793
		writel(lower_32_bits(targets), tl_lo);
794
	} else {
795
		struct cxl_endpoint_decoder *cxled =
796
			to_cxl_endpoint_decoder(&cxld->dev);
797
		void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
798
		void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
799

800
		writel(upper_32_bits(cxled->skip), sk_hi);
801
		writel(lower_32_bits(cxled->skip), sk_lo);
802
	}
803

804
	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
805
}
806

807
static int cxl_decoder_commit(struct cxl_decoder *cxld)
808
{
809
	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
810
	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
811
	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
812
	int id = cxld->id, rc;
813

814
	if (cxld->flags & CXL_DECODER_F_ENABLE)
815
		return 0;
816

817
	if (cxl_num_decoders_committed(port) != id) {
818
		dev_dbg(&port->dev,
819
			"%s: out of order commit, expected decoder%d.%d\n",
820
			dev_name(&cxld->dev), port->id,
821
			cxl_num_decoders_committed(port));
822
		return -EBUSY;
823
	}
824

825
	/*
826
	 * For endpoint decoders hosted on CXL memory devices that
827
	 * support the sanitize operation, make sure sanitize is not in-flight.
828
	 */
829
	if (is_endpoint_decoder(&cxld->dev)) {
830
		struct cxl_endpoint_decoder *cxled =
831
			to_cxl_endpoint_decoder(&cxld->dev);
832
		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
833
		struct cxl_memdev_state *mds =
834
			to_cxl_memdev_state(cxlmd->cxlds);
835

836
		if (mds && mds->security.sanitize_active) {
837
			dev_dbg(&cxlmd->dev,
838
				"attempted to commit %s during sanitize\n",
839
				dev_name(&cxld->dev));
840
			return -EBUSY;
841
		}
842
	}
843

844
	scoped_guard(rwsem_read, &cxl_rwsem.dpa)
845
		setup_hw_decoder(cxld, hdm);
846

847
	port->commit_end++;
848
	rc = cxld_await_commit(hdm, cxld->id);
849
	if (rc) {
850
		dev_dbg(&port->dev, "%s: error %d committing decoder\n",
851
			dev_name(&cxld->dev), rc);
852
		cxld->reset(cxld);
853
		return rc;
854
	}
855
	cxld->flags |= CXL_DECODER_F_ENABLE;
856

857
	return 0;
858
}
859

860
static int commit_reap(struct device *dev, void *data)
861
{
862
	struct cxl_port *port = to_cxl_port(dev->parent);
863
	struct cxl_decoder *cxld;
864

865
	if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev))
866
		return 0;
867

868
	cxld = to_cxl_decoder(dev);
869
	if (port->commit_end == cxld->id &&
870
	    ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
871
		port->commit_end--;
872
		dev_dbg(&port->dev, "reap: %s commit_end: %d\n",
873
			dev_name(&cxld->dev), port->commit_end);
874
	}
875

876
	return 0;
877
}
878

879
void cxl_port_commit_reap(struct cxl_decoder *cxld)
880
{
881
	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
882

883
	lockdep_assert_held_write(&cxl_rwsem.region);
884

885
	/*
886
	 * Once the highest committed decoder is disabled, free any other
887
	 * decoders that were pinned allocated by out-of-order release.
888
	 */
889
	port->commit_end--;
890
	dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev),
891
		port->commit_end);
892
	device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL,
893
					   commit_reap);
894
}
895
EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL");
896

897
static void cxl_decoder_reset(struct cxl_decoder *cxld)
898
{
899
	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
900
	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
901
	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
902
	int id = cxld->id;
903
	u32 ctrl;
904

905
	if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
906
		return;
907

908
	if (port->commit_end == id)
909
		cxl_port_commit_reap(cxld);
910
	else
911
		dev_dbg(&port->dev,
912
			"%s: out of order reset, expected decoder%d.%d\n",
913
			dev_name(&cxld->dev), port->id, port->commit_end);
914

915
	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
916
	ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
917
	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
918

919
	writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
920
	writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
921
	writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
922
	writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
923

924
	cxld->flags &= ~CXL_DECODER_F_ENABLE;
925

926
	/* Userspace is now responsible for reconfiguring this decoder */
927
	if (is_endpoint_decoder(&cxld->dev)) {
928
		struct cxl_endpoint_decoder *cxled;
929

930
		cxled = to_cxl_endpoint_decoder(&cxld->dev);
931
		cxled->state = CXL_DECODER_STATE_MANUAL;
932
	}
933
}
934

935
static int cxl_setup_hdm_decoder_from_dvsec(
936
	struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
937
	int which, struct cxl_endpoint_dvsec_info *info)
938
{
939
	struct cxl_endpoint_decoder *cxled;
940
	u64 len;
941
	int rc;
942

943
	if (!is_cxl_endpoint(port))
944
		return -EOPNOTSUPP;
945

946
	cxled = to_cxl_endpoint_decoder(&cxld->dev);
947
	len = range_len(&info->dvsec_range[which]);
948
	if (!len)
949
		return -ENOENT;
950

951
	cxld->target_type = CXL_DECODER_HOSTONLYMEM;
952
	cxld->commit = NULL;
953
	cxld->reset = NULL;
954
	cxld->hpa_range = info->dvsec_range[which];
955

956
	/*
957
	 * Set the emulated decoder as locked pending additional support to
958
	 * change the range registers at run time.
959
	 */
960
	cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
961
	port->commit_end = cxld->id;
962

963
	rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
964
	if (rc) {
965
		dev_err(&port->dev,
966
			"decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
967
			port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
968
		return rc;
969
	}
970
	*dpa_base += len;
971
	cxled->state = CXL_DECODER_STATE_AUTO;
972

973
	return 0;
974
}
975

976
static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
977
			    void __iomem *hdm, int which,
978
			    u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
979
{
980
	struct cxl_endpoint_decoder *cxled = NULL;
981
	u64 size, base, skip, dpa_size, lo, hi;
982
	bool committed;
983
	u32 remainder;
984
	int i, rc;
985
	u32 ctrl;
986
	union {
987
		u64 value;
988
		unsigned char target_id[8];
989
	} target_list;
990

991
	if (should_emulate_decoders(info))
992
		return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
993
							which, info);
994

995
	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
996
	lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
997
	hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
998
	base = (hi << 32) + lo;
999
	lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
1000
	hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
1001
	size = (hi << 32) + lo;
1002
	committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
1003
	cxld->commit = cxl_decoder_commit;
1004
	cxld->reset = cxl_decoder_reset;
1005

1006
	if (!committed)
1007
		size = 0;
1008
	if (base == U64_MAX || size == U64_MAX) {
1009
		dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
1010
			 port->id, cxld->id);
1011
		return -ENXIO;
1012
	}
1013

1014
	if (info)
1015
		cxled = to_cxl_endpoint_decoder(&cxld->dev);
1016
	cxld->hpa_range = (struct range) {
1017
		.start = base,
1018
		.end = base + size - 1,
1019
	};
1020

1021
	/* decoders are enabled if committed */
1022
	if (committed) {
1023
		cxld->flags |= CXL_DECODER_F_ENABLE;
1024
		if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
1025
			cxld->flags |= CXL_DECODER_F_LOCK;
1026
		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
1027
			cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1028
		else
1029
			cxld->target_type = CXL_DECODER_DEVMEM;
1030

1031
		guard(rwsem_write)(&cxl_rwsem.region);
1032
		if (cxld->id != cxl_num_decoders_committed(port)) {
1033
			dev_warn(&port->dev,
1034
				 "decoder%d.%d: Committed out of order\n",
1035
				 port->id, cxld->id);
1036
			return -ENXIO;
1037
		}
1038

1039
		if (size == 0) {
1040
			dev_warn(&port->dev,
1041
				 "decoder%d.%d: Committed with zero size\n",
1042
				 port->id, cxld->id);
1043
			return -ENXIO;
1044
		}
1045
		port->commit_end = cxld->id;
1046
	} else {
1047
		if (cxled) {
1048
			struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1049
			struct cxl_dev_state *cxlds = cxlmd->cxlds;
1050

1051
			/*
1052
			 * Default by devtype until a device arrives that needs
1053
			 * more precision.
1054
			 */
1055
			if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
1056
				cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1057
			else
1058
				cxld->target_type = CXL_DECODER_DEVMEM;
1059
		} else {
1060
			/* To be overridden by region type at commit time */
1061
			cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1062
		}
1063

1064
		if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
1065
		    cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
1066
			ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
1067
			writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
1068
		}
1069
	}
1070
	rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
1071
			  &cxld->interleave_ways);
1072
	if (rc) {
1073
		dev_warn(&port->dev,
1074
			 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
1075
			 port->id, cxld->id, ctrl);
1076
		return rc;
1077
	}
1078
	rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
1079
				 &cxld->interleave_granularity);
1080
	if (rc) {
1081
		dev_warn(&port->dev,
1082
			 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",
1083
			 port->id, cxld->id, ctrl);
1084
		return rc;
1085
	}
1086

1087
	dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
1088
		port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
1089
		cxld->interleave_ways, cxld->interleave_granularity);
1090

1091
	if (!cxled) {
1092
		lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
1093
		hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
1094
		target_list.value = (hi << 32) + lo;
1095
		for (i = 0; i < cxld->interleave_ways; i++)
1096
			cxld->target_map[i] = target_list.target_id[i];
1097

1098
		return 0;
1099
	}
1100

1101
	if (!committed)
1102
		return 0;
1103

1104
	dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
1105
	if (remainder) {
1106
		dev_err(&port->dev,
1107
			"decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
1108
			port->id, cxld->id, size, cxld->interleave_ways);
1109
		return -ENXIO;
1110
	}
1111
	lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
1112
	hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
1113
	skip = (hi << 32) + lo;
1114
	rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
1115
	if (rc) {
1116
		dev_err(&port->dev,
1117
			"decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
1118
			port->id, cxld->id, *dpa_base,
1119
			*dpa_base + dpa_size + skip - 1, rc);
1120
		return rc;
1121
	}
1122
	*dpa_base += dpa_size + skip;
1123

1124
	cxled->state = CXL_DECODER_STATE_AUTO;
1125

1126
	return 0;
1127
}
1128

1129
static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
1130
{
1131
	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1132
	int committed, i;
1133
	u32 ctrl;
1134

1135
	if (!hdm)
1136
		return;
1137

1138
	/*
1139
	 * Since the register resource was recently claimed via request_region()
1140
	 * be careful about trusting the "not-committed" status until the commit
1141
	 * timeout has elapsed.  The commit timeout is 10ms (CXL 2.0
1142
	 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
1143
	 * host and target.
1144
	 */
1145
	for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
1146
		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
1147
		if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
1148
			committed++;
1149
	}
1150

1151
	/* ensure that future checks of committed can be trusted */
1152
	if (committed != cxlhdm->decoder_count)
1153
		msleep(20);
1154
}
1155

1156
/**
1157
 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
1158
 * @cxlhdm: Structure to populate with HDM capabilities
1159
 * @info: cached DVSEC range register info
1160
 */
1161
static int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
1162
				       struct cxl_endpoint_dvsec_info *info)
1163
{
1164
	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1165
	struct cxl_port *port = cxlhdm->port;
1166
	int i;
1167
	u64 dpa_base = 0;
1168

1169
	cxl_settle_decoders(cxlhdm);
1170

1171
	for (i = 0; i < cxlhdm->decoder_count; i++) {
1172
		int rc, target_count = cxlhdm->target_count;
1173
		struct cxl_decoder *cxld;
1174

1175
		if (is_cxl_endpoint(port)) {
1176
			struct cxl_endpoint_decoder *cxled;
1177

1178
			cxled = cxl_endpoint_decoder_alloc(port);
1179
			if (IS_ERR(cxled)) {
1180
				dev_warn(&port->dev,
1181
					 "Failed to allocate decoder%d.%d\n",
1182
					 port->id, i);
1183
				return PTR_ERR(cxled);
1184
			}
1185
			cxld = &cxled->cxld;
1186
		} else {
1187
			struct cxl_switch_decoder *cxlsd;
1188

1189
			cxlsd = cxl_switch_decoder_alloc(port, target_count);
1190
			if (IS_ERR(cxlsd)) {
1191
				dev_warn(&port->dev,
1192
					 "Failed to allocate decoder%d.%d\n",
1193
					 port->id, i);
1194
				return PTR_ERR(cxlsd);
1195
			}
1196
			cxld = &cxlsd->cxld;
1197
		}
1198

1199
		rc = init_hdm_decoder(port, cxld, hdm, i, &dpa_base, info);
1200
		if (rc) {
1201
			dev_warn(&port->dev,
1202
				 "Failed to initialize decoder%d.%d\n",
1203
				 port->id, i);
1204
			put_device(&cxld->dev);
1205
			return rc;
1206
		}
1207
		rc = add_hdm_decoder(port, cxld);
1208
		if (rc) {
1209
			dev_warn(&port->dev,
1210
				 "Failed to add decoder%d.%d\n", port->id, i);
1211
			return rc;
1212
		}
1213
	}
1214

1215
	return 0;
1216
}
1217

1218
/**
1219
 * __devm_cxl_switch_port_decoders_setup - allocate and setup switch decoders
1220
 * @port: CXL port context
1221
 *
1222
 * Return 0 or -errno on error
1223
 */
1224
int __devm_cxl_switch_port_decoders_setup(struct cxl_port *port)
1225
{
1226
	struct cxl_hdm *cxlhdm;
1227

1228
	if (is_cxl_root(port) || is_cxl_endpoint(port))
1229
		return -EOPNOTSUPP;
1230

1231
	cxlhdm = devm_cxl_setup_hdm(port, NULL);
1232
	if (!IS_ERR(cxlhdm))
1233
		return devm_cxl_enumerate_decoders(cxlhdm, NULL);
1234

1235
	if (PTR_ERR(cxlhdm) != -ENODEV) {
1236
		dev_err(&port->dev, "Failed to map HDM decoder capability\n");
1237
		return PTR_ERR(cxlhdm);
1238
	}
1239

1240
	if (cxl_port_get_possible_dports(port) == 1) {
1241
		dev_dbg(&port->dev, "Fallback to passthrough decoder\n");
1242
		return devm_cxl_add_passthrough_decoder(port);
1243
	}
1244

1245
	dev_err(&port->dev, "HDM decoder capability not found\n");
1246
	return -ENXIO;
1247
}
1248
EXPORT_SYMBOL_NS_GPL(__devm_cxl_switch_port_decoders_setup, "CXL");
1249

1250
/**
1251
 * devm_cxl_endpoint_decoders_setup - allocate and setup endpoint decoders
1252
 * @port: CXL port context
1253
 *
1254
 * Return 0 or -errno on error
1255
 */
1256
int devm_cxl_endpoint_decoders_setup(struct cxl_port *port)
1257
{
1258
	struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
1259
	struct cxl_endpoint_dvsec_info info = { .port = port };
1260
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
1261
	struct cxl_hdm *cxlhdm;
1262
	int rc;
1263

1264
	if (!is_cxl_endpoint(port))
1265
		return -EOPNOTSUPP;
1266

1267
	rc = cxl_dvsec_rr_decode(cxlds, &info);
1268
	if (rc < 0)
1269
		return rc;
1270

1271
	cxlhdm = devm_cxl_setup_hdm(port, &info);
1272
	if (IS_ERR(cxlhdm)) {
1273
		if (PTR_ERR(cxlhdm) == -ENODEV)
1274
			dev_err(&port->dev, "HDM decoder registers not found\n");
1275
		return PTR_ERR(cxlhdm);
1276
	}
1277

1278
	rc = cxl_hdm_decode_init(cxlds, cxlhdm, &info);
1279
	if (rc)
1280
		return rc;
1281

1282
	return devm_cxl_enumerate_decoders(cxlhdm, &info);
1283
}
1284
EXPORT_SYMBOL_NS_GPL(devm_cxl_endpoint_decoders_setup, "CXL");
1285

1286