1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
4
 *
5
 */
6

7
#include <linux/bitfield.h>
8
#include <linux/debugfs.h>
9
#include <linux/device.h>
10
#include <linux/dma-direction.h>
11
#include <linux/dma-mapping.h>
12
#include <linux/idr.h>
13
#include <linux/interrupt.h>
14
#include <linux/list.h>
15
#include <linux/mhi.h>
16
#include <linux/mod_devicetable.h>
17
#include <linux/module.h>
18
#include <linux/slab.h>
19
#include <linux/vmalloc.h>
20
#include <linux/wait.h>
21
#include "internal.h"
22

23
#define CREATE_TRACE_POINTS
24
#include "trace.h"
25

26
static DEFINE_IDA(mhi_controller_ida);
27

28
#undef mhi_ee
29
#undef mhi_ee_end
30

31
#define mhi_ee(a, b)		[MHI_EE_##a] = b,
32
#define mhi_ee_end(a, b)	[MHI_EE_##a] = b,
33

34
const char * const mhi_ee_str[MHI_EE_MAX] = {
35
	MHI_EE_LIST
36
};
37

38
#undef dev_st_trans
39
#undef dev_st_trans_end
40

41
#define dev_st_trans(a, b)	[DEV_ST_TRANSITION_##a] = b,
42
#define dev_st_trans_end(a, b)	[DEV_ST_TRANSITION_##a] = b,
43

44
const char * const dev_state_tran_str[DEV_ST_TRANSITION_MAX] = {
45
	DEV_ST_TRANSITION_LIST
46
};
47

48
#undef ch_state_type
49
#undef ch_state_type_end
50

51
#define ch_state_type(a, b)	[MHI_CH_STATE_TYPE_##a] = b,
52
#define ch_state_type_end(a, b)	[MHI_CH_STATE_TYPE_##a] = b,
53

54
const char * const mhi_ch_state_type_str[MHI_CH_STATE_TYPE_MAX] = {
55
	MHI_CH_STATE_TYPE_LIST
56
};
57

58
#undef mhi_pm_state
59
#undef mhi_pm_state_end
60

61
#define mhi_pm_state(a, b)	[MHI_PM_STATE_##a] = b,
62
#define mhi_pm_state_end(a, b)	[MHI_PM_STATE_##a] = b,
63

64
static const char * const mhi_pm_state_str[] = {
65
	MHI_PM_STATE_LIST
66
};
67

68
const char *to_mhi_pm_state_str(u32 state)
69
{
70
	int index;
71

72
	if (state)
73
		index = __fls(state);
74

75
	if (!state || index >= ARRAY_SIZE(mhi_pm_state_str))
76
		return "Invalid State";
77

78
	return mhi_pm_state_str[index];
79
}
80

81
static ssize_t serial_number_show(struct device *dev,
82
				  struct device_attribute *attr,
83
				  char *buf)
84
{
85
	struct mhi_device *mhi_dev = to_mhi_device(dev);
86
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
87

88
	return sysfs_emit(buf, "Serial Number: %u\n",
89
			mhi_cntrl->serial_number);
90
}
91
static DEVICE_ATTR_RO(serial_number);
92

93
static ssize_t oem_pk_hash_show(struct device *dev,
94
				struct device_attribute *attr,
95
				char *buf)
96
{
97
	struct mhi_device *mhi_dev = to_mhi_device(dev);
98
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
99
	u32 hash_segment[MHI_MAX_OEM_PK_HASH_SEGMENTS];
100
	int i, cnt = 0, ret;
101

102
	for (i = 0; i < MHI_MAX_OEM_PK_HASH_SEGMENTS; i++) {
103
		ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_OEMPKHASH(i), &hash_segment[i]);
104
		if (ret) {
105
			dev_err(dev, "Could not capture OEM PK HASH\n");
106
			return ret;
107
		}
108
	}
109

110
	for (i = 0; i < MHI_MAX_OEM_PK_HASH_SEGMENTS; i++)
111
		cnt += sysfs_emit_at(buf, cnt, "OEMPKHASH[%d]: 0x%x\n", i, hash_segment[i]);
112

113
	return cnt;
114
}
115
static DEVICE_ATTR_RO(oem_pk_hash);
116

117
static ssize_t soc_reset_store(struct device *dev,
118
			       struct device_attribute *attr,
119
			       const char *buf,
120
			       size_t count)
121
{
122
	struct mhi_device *mhi_dev = to_mhi_device(dev);
123
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
124

125
	mhi_soc_reset(mhi_cntrl);
126
	return count;
127
}
128
static DEVICE_ATTR_WO(soc_reset);
129

130
static ssize_t trigger_edl_store(struct device *dev,
131
			       struct device_attribute *attr,
132
			       const char *buf, size_t count)
133
{
134
	struct mhi_device *mhi_dev = to_mhi_device(dev);
135
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
136
	unsigned long val;
137
	int ret;
138

139
	ret = kstrtoul(buf, 10, &val);
140
	if (ret < 0)
141
		return ret;
142

143
	if (!val)
144
		return -EINVAL;
145

146
	ret = mhi_cntrl->edl_trigger(mhi_cntrl);
147
	if (ret)
148
		return ret;
149

150
	return count;
151
}
152
static DEVICE_ATTR_WO(trigger_edl);
153

154
static struct attribute *mhi_dev_attrs[] = {
155
	&dev_attr_serial_number.attr,
156
	&dev_attr_oem_pk_hash.attr,
157
	&dev_attr_soc_reset.attr,
158
	NULL,
159
};
160
ATTRIBUTE_GROUPS(mhi_dev);
161

162
/* MHI protocol requires the transfer ring to be aligned with ring length */
163
static int mhi_alloc_aligned_ring(struct mhi_controller *mhi_cntrl,
164
				  struct mhi_ring *ring,
165
				  u64 len)
166
{
167
	ring->alloc_size = len + (len - 1);
168
	ring->pre_aligned = dma_alloc_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size,
169
					       &ring->dma_handle, GFP_KERNEL);
170
	if (!ring->pre_aligned)
171
		return -ENOMEM;
172

173
	ring->iommu_base = (ring->dma_handle + (len - 1)) & ~(len - 1);
174
	ring->base = ring->pre_aligned + (ring->iommu_base - ring->dma_handle);
175

176
	return 0;
177
}
178

179
static void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl)
180
{
181
	int i;
182
	struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
183

184
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
185
		if (mhi_event->offload_ev)
186
			continue;
187

188
		free_irq(mhi_cntrl->irq[mhi_event->irq], mhi_event);
189
	}
190

191
	free_irq(mhi_cntrl->irq[0], mhi_cntrl);
192
}
193

194
static int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl)
195
{
196
	struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
197
	unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND;
198
	int i, ret;
199

200
	/* if controller driver has set irq_flags, use it */
201
	if (mhi_cntrl->irq_flags)
202
		irq_flags = mhi_cntrl->irq_flags;
203

204
	/* Setup BHI_INTVEC IRQ */
205
	ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler,
206
				   mhi_intvec_threaded_handler,
207
				   irq_flags,
208
				   "bhi", mhi_cntrl);
209
	if (ret)
210
		return ret;
211
	/*
212
	 * IRQs should be enabled during mhi_async_power_up(), so disable them explicitly here.
213
	 * Due to the use of IRQF_SHARED flag as default while requesting IRQs, we assume that
214
	 * IRQ_NOAUTOEN is not applicable.
215
	 */
216
	disable_irq(mhi_cntrl->irq[0]);
217

218
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
219
		if (mhi_event->offload_ev)
220
			continue;
221

222
		if (mhi_event->irq >= mhi_cntrl->nr_irqs) {
223
			dev_err(mhi_cntrl->cntrl_dev, "irq %d not available for event ring\n",
224
				mhi_event->irq);
225
			ret = -EINVAL;
226
			goto error_request;
227
		}
228

229
		ret = request_irq(mhi_cntrl->irq[mhi_event->irq],
230
				  mhi_irq_handler,
231
				  irq_flags,
232
				  "mhi", mhi_event);
233
		if (ret) {
234
			dev_err(mhi_cntrl->cntrl_dev, "Error requesting irq:%d for ev:%d\n",
235
				mhi_cntrl->irq[mhi_event->irq], i);
236
			goto error_request;
237
		}
238

239
		disable_irq(mhi_cntrl->irq[mhi_event->irq]);
240
	}
241

242
	return 0;
243

244
error_request:
245
	for (--i, --mhi_event; i >= 0; i--, mhi_event--) {
246
		if (mhi_event->offload_ev)
247
			continue;
248

249
		free_irq(mhi_cntrl->irq[mhi_event->irq], mhi_event);
250
	}
251
	free_irq(mhi_cntrl->irq[0], mhi_cntrl);
252

253
	return ret;
254
}
255

256
static void mhi_deinit_dev_ctxt(struct mhi_controller *mhi_cntrl)
257
{
258
	int i;
259
	struct mhi_ctxt *mhi_ctxt = mhi_cntrl->mhi_ctxt;
260
	struct mhi_cmd *mhi_cmd;
261
	struct mhi_event *mhi_event;
262
	struct mhi_ring *ring;
263

264
	mhi_cmd = mhi_cntrl->mhi_cmd;
265
	for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++) {
266
		ring = &mhi_cmd->ring;
267
		dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size,
268
				  ring->pre_aligned, ring->dma_handle);
269
		ring->base = NULL;
270
		ring->iommu_base = 0;
271
	}
272

273
	dma_free_coherent(mhi_cntrl->cntrl_dev,
274
			  sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS,
275
			  mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr);
276

277
	mhi_event = mhi_cntrl->mhi_event;
278
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
279
		if (mhi_event->offload_ev)
280
			continue;
281

282
		ring = &mhi_event->ring;
283
		dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size,
284
				  ring->pre_aligned, ring->dma_handle);
285
		ring->base = NULL;
286
		ring->iommu_base = 0;
287
	}
288

289
	dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->er_ctxt) *
290
			  mhi_cntrl->total_ev_rings, mhi_ctxt->er_ctxt,
291
			  mhi_ctxt->er_ctxt_addr);
292

293
	dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->chan_ctxt) *
294
			  mhi_cntrl->max_chan, mhi_ctxt->chan_ctxt,
295
			  mhi_ctxt->chan_ctxt_addr);
296

297
	kfree(mhi_ctxt);
298
	mhi_cntrl->mhi_ctxt = NULL;
299
}
300

301
static int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
302
{
303
	struct mhi_ctxt *mhi_ctxt;
304
	struct mhi_chan_ctxt *chan_ctxt;
305
	struct mhi_event_ctxt *er_ctxt;
306
	struct mhi_cmd_ctxt *cmd_ctxt;
307
	struct mhi_chan *mhi_chan;
308
	struct mhi_event *mhi_event;
309
	struct mhi_cmd *mhi_cmd;
310
	u32 tmp;
311
	int ret = -ENOMEM, i;
312

313
	atomic_set(&mhi_cntrl->dev_wake, 0);
314
	atomic_set(&mhi_cntrl->pending_pkts, 0);
315

316
	mhi_ctxt = kzalloc(sizeof(*mhi_ctxt), GFP_KERNEL);
317
	if (!mhi_ctxt)
318
		return -ENOMEM;
319

320
	/* Setup channel ctxt */
321
	mhi_ctxt->chan_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev,
322
						 sizeof(*mhi_ctxt->chan_ctxt) *
323
						 mhi_cntrl->max_chan,
324
						 &mhi_ctxt->chan_ctxt_addr,
325
						 GFP_KERNEL);
326
	if (!mhi_ctxt->chan_ctxt)
327
		goto error_alloc_chan_ctxt;
328

329
	mhi_chan = mhi_cntrl->mhi_chan;
330
	chan_ctxt = mhi_ctxt->chan_ctxt;
331
	for (i = 0; i < mhi_cntrl->max_chan; i++, chan_ctxt++, mhi_chan++) {
332
		/* Skip if it is an offload channel */
333
		if (mhi_chan->offload_ch)
334
			continue;
335

336
		tmp = le32_to_cpu(chan_ctxt->chcfg);
337
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
338
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_DISABLED);
339
		tmp &= ~CHAN_CTX_BRSTMODE_MASK;
340
		tmp |= FIELD_PREP(CHAN_CTX_BRSTMODE_MASK, mhi_chan->db_cfg.brstmode);
341
		tmp &= ~CHAN_CTX_POLLCFG_MASK;
342
		tmp |= FIELD_PREP(CHAN_CTX_POLLCFG_MASK, mhi_chan->db_cfg.pollcfg);
343
		chan_ctxt->chcfg = cpu_to_le32(tmp);
344

345
		chan_ctxt->chtype = cpu_to_le32(mhi_chan->type);
346
		chan_ctxt->erindex = cpu_to_le32(mhi_chan->er_index);
347

348
		mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
349
		mhi_chan->tre_ring.db_addr = (void __iomem *)&chan_ctxt->wp;
350
	}
351

352
	/* Setup event context */
353
	mhi_ctxt->er_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev,
354
					       sizeof(*mhi_ctxt->er_ctxt) *
355
					       mhi_cntrl->total_ev_rings,
356
					       &mhi_ctxt->er_ctxt_addr,
357
					       GFP_KERNEL);
358
	if (!mhi_ctxt->er_ctxt)
359
		goto error_alloc_er_ctxt;
360

361
	er_ctxt = mhi_ctxt->er_ctxt;
362
	mhi_event = mhi_cntrl->mhi_event;
363
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, er_ctxt++,
364
		     mhi_event++) {
365
		struct mhi_ring *ring = &mhi_event->ring;
366

367
		/* Skip if it is an offload event */
368
		if (mhi_event->offload_ev)
369
			continue;
370

371
		tmp = le32_to_cpu(er_ctxt->intmod);
372
		tmp &= ~EV_CTX_INTMODC_MASK;
373
		tmp &= ~EV_CTX_INTMODT_MASK;
374
		tmp |= FIELD_PREP(EV_CTX_INTMODT_MASK, mhi_event->intmod);
375
		er_ctxt->intmod = cpu_to_le32(tmp);
376

377
		er_ctxt->ertype = cpu_to_le32(MHI_ER_TYPE_VALID);
378
		er_ctxt->msivec = cpu_to_le32(mhi_event->irq);
379
		mhi_event->db_cfg.db_mode = true;
380

381
		ring->el_size = sizeof(struct mhi_ring_element);
382
		ring->len = ring->el_size * ring->elements;
383
		ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len);
384
		if (ret)
385
			goto error_alloc_er;
386

387
		/*
388
		 * If the read pointer equals to the write pointer, then the
389
		 * ring is empty
390
		 */
391
		ring->rp = ring->wp = ring->base;
392
		er_ctxt->rbase = cpu_to_le64(ring->iommu_base);
393
		er_ctxt->rp = er_ctxt->wp = er_ctxt->rbase;
394
		er_ctxt->rlen = cpu_to_le64(ring->len);
395
		ring->ctxt_wp = &er_ctxt->wp;
396
	}
397

398
	/* Setup cmd context */
399
	ret = -ENOMEM;
400
	mhi_ctxt->cmd_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev,
401
						sizeof(*mhi_ctxt->cmd_ctxt) *
402
						NR_OF_CMD_RINGS,
403
						&mhi_ctxt->cmd_ctxt_addr,
404
						GFP_KERNEL);
405
	if (!mhi_ctxt->cmd_ctxt)
406
		goto error_alloc_er;
407

408
	mhi_cmd = mhi_cntrl->mhi_cmd;
409
	cmd_ctxt = mhi_ctxt->cmd_ctxt;
410
	for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) {
411
		struct mhi_ring *ring = &mhi_cmd->ring;
412

413
		ring->el_size = sizeof(struct mhi_ring_element);
414
		ring->elements = CMD_EL_PER_RING;
415
		ring->len = ring->el_size * ring->elements;
416
		ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len);
417
		if (ret)
418
			goto error_alloc_cmd;
419

420
		ring->rp = ring->wp = ring->base;
421
		cmd_ctxt->rbase = cpu_to_le64(ring->iommu_base);
422
		cmd_ctxt->rp = cmd_ctxt->wp = cmd_ctxt->rbase;
423
		cmd_ctxt->rlen = cpu_to_le64(ring->len);
424
		ring->ctxt_wp = &cmd_ctxt->wp;
425
	}
426

427
	mhi_cntrl->mhi_ctxt = mhi_ctxt;
428

429
	return 0;
430

431
error_alloc_cmd:
432
	for (--i, --mhi_cmd; i >= 0; i--, mhi_cmd--) {
433
		struct mhi_ring *ring = &mhi_cmd->ring;
434

435
		dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size,
436
				  ring->pre_aligned, ring->dma_handle);
437
	}
438
	dma_free_coherent(mhi_cntrl->cntrl_dev,
439
			  sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS,
440
			  mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr);
441
	i = mhi_cntrl->total_ev_rings;
442
	mhi_event = mhi_cntrl->mhi_event + i;
443

444
error_alloc_er:
445
	for (--i, --mhi_event; i >= 0; i--, mhi_event--) {
446
		struct mhi_ring *ring = &mhi_event->ring;
447

448
		if (mhi_event->offload_ev)
449
			continue;
450

451
		dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size,
452
				  ring->pre_aligned, ring->dma_handle);
453
	}
454
	dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->er_ctxt) *
455
			  mhi_cntrl->total_ev_rings, mhi_ctxt->er_ctxt,
456
			  mhi_ctxt->er_ctxt_addr);
457

458
error_alloc_er_ctxt:
459
	dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->chan_ctxt) *
460
			  mhi_cntrl->max_chan, mhi_ctxt->chan_ctxt,
461
			  mhi_ctxt->chan_ctxt_addr);
462

463
error_alloc_chan_ctxt:
464
	kfree(mhi_ctxt);
465

466
	return ret;
467
}
468

469
int mhi_init_mmio(struct mhi_controller *mhi_cntrl)
470
{
471
	u32 val;
472
	int i, ret;
473
	struct mhi_chan *mhi_chan;
474
	struct mhi_event *mhi_event;
475
	void __iomem *base = mhi_cntrl->regs;
476
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
477
	struct {
478
		u32 offset;
479
		u32 val;
480
	} reg_info[] = {
481
		{
482
			CCABAP_HIGHER,
483
			upper_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr),
484
		},
485
		{
486
			CCABAP_LOWER,
487
			lower_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr),
488
		},
489
		{
490
			ECABAP_HIGHER,
491
			upper_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr),
492
		},
493
		{
494
			ECABAP_LOWER,
495
			lower_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr),
496
		},
497
		{
498
			CRCBAP_HIGHER,
499
			upper_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr),
500
		},
501
		{
502
			CRCBAP_LOWER,
503
			lower_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr),
504
		},
505
		{
506
			MHICTRLBASE_HIGHER,
507
			upper_32_bits(mhi_cntrl->iova_start),
508
		},
509
		{
510
			MHICTRLBASE_LOWER,
511
			lower_32_bits(mhi_cntrl->iova_start),
512
		},
513
		{
514
			MHIDATABASE_HIGHER,
515
			upper_32_bits(mhi_cntrl->iova_start),
516
		},
517
		{
518
			MHIDATABASE_LOWER,
519
			lower_32_bits(mhi_cntrl->iova_start),
520
		},
521
		{
522
			MHICTRLLIMIT_HIGHER,
523
			upper_32_bits(mhi_cntrl->iova_stop),
524
		},
525
		{
526
			MHICTRLLIMIT_LOWER,
527
			lower_32_bits(mhi_cntrl->iova_stop),
528
		},
529
		{
530
			MHIDATALIMIT_HIGHER,
531
			upper_32_bits(mhi_cntrl->iova_stop),
532
		},
533
		{
534
			MHIDATALIMIT_LOWER,
535
			lower_32_bits(mhi_cntrl->iova_stop),
536
		},
537
		{0, 0}
538
	};
539

540
	dev_dbg(dev, "Initializing MHI registers\n");
541

542
	/* Read channel db offset */
543
	ret = mhi_get_channel_doorbell_offset(mhi_cntrl, &val);
544
	if (ret)
545
		return ret;
546

547
	if (val >= mhi_cntrl->reg_len - (8 * MHI_DEV_WAKE_DB)) {
548
		dev_err(dev, "CHDB offset: 0x%x is out of range: 0x%zx\n",
549
			val, mhi_cntrl->reg_len - (8 * MHI_DEV_WAKE_DB));
550
		return -ERANGE;
551
	}
552

553
	/* Setup wake db */
554
	mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB);
555
	mhi_cntrl->wake_set = false;
556

557
	/* Setup channel db address for each channel in tre_ring */
558
	mhi_chan = mhi_cntrl->mhi_chan;
559
	for (i = 0; i < mhi_cntrl->max_chan; i++, val += 8, mhi_chan++)
560
		mhi_chan->tre_ring.db_addr = base + val;
561

562
	/* Read event ring db offset */
563
	ret = mhi_read_reg(mhi_cntrl, base, ERDBOFF, &val);
564
	if (ret) {
565
		dev_err(dev, "Unable to read ERDBOFF register\n");
566
		return -EIO;
567
	}
568

569
	if (val >= mhi_cntrl->reg_len - (8 * mhi_cntrl->total_ev_rings)) {
570
		dev_err(dev, "ERDB offset: 0x%x is out of range: 0x%zx\n",
571
			val, mhi_cntrl->reg_len - (8 * mhi_cntrl->total_ev_rings));
572
		return -ERANGE;
573
	}
574

575
	/* Setup event db address for each ev_ring */
576
	mhi_event = mhi_cntrl->mhi_event;
577
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, val += 8, mhi_event++) {
578
		if (mhi_event->offload_ev)
579
			continue;
580

581
		mhi_event->ring.db_addr = base + val;
582
	}
583

584
	/* Setup DB register for primary CMD rings */
585
	mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING].ring.db_addr = base + CRDB_LOWER;
586

587
	/* Write to MMIO registers */
588
	for (i = 0; reg_info[i].offset; i++)
589
		mhi_write_reg(mhi_cntrl, base, reg_info[i].offset,
590
			      reg_info[i].val);
591

592
	ret = mhi_write_reg_field(mhi_cntrl, base, MHICFG, MHICFG_NER_MASK,
593
				  mhi_cntrl->total_ev_rings);
594
	if (ret) {
595
		dev_err(dev, "Unable to write MHICFG register\n");
596
		return ret;
597
	}
598

599
	ret = mhi_write_reg_field(mhi_cntrl, base, MHICFG, MHICFG_NHWER_MASK,
600
				  mhi_cntrl->hw_ev_rings);
601
	if (ret) {
602
		dev_err(dev, "Unable to write MHICFG register\n");
603
		return ret;
604
	}
605

606
	return 0;
607
}
608

609
void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl,
610
			  struct mhi_chan *mhi_chan)
611
{
612
	struct mhi_ring *buf_ring;
613
	struct mhi_ring *tre_ring;
614
	struct mhi_chan_ctxt *chan_ctxt;
615
	u32 tmp;
616

617
	buf_ring = &mhi_chan->buf_ring;
618
	tre_ring = &mhi_chan->tre_ring;
619
	chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan];
620

621
	if (!chan_ctxt->rbase) /* Already uninitialized */
622
		return;
623

624
	dma_free_coherent(mhi_cntrl->cntrl_dev, tre_ring->alloc_size,
625
			  tre_ring->pre_aligned, tre_ring->dma_handle);
626
	vfree(buf_ring->base);
627

628
	buf_ring->base = tre_ring->base = NULL;
629
	tre_ring->ctxt_wp = NULL;
630
	chan_ctxt->rbase = 0;
631
	chan_ctxt->rlen = 0;
632
	chan_ctxt->rp = 0;
633
	chan_ctxt->wp = 0;
634

635
	tmp = le32_to_cpu(chan_ctxt->chcfg);
636
	tmp &= ~CHAN_CTX_CHSTATE_MASK;
637
	tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_DISABLED);
638
	chan_ctxt->chcfg = cpu_to_le32(tmp);
639

640
	/* Update to all cores */
641
	smp_wmb();
642
}
643

644
int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
645
		       struct mhi_chan *mhi_chan)
646
{
647
	struct mhi_ring *buf_ring;
648
	struct mhi_ring *tre_ring;
649
	struct mhi_chan_ctxt *chan_ctxt;
650
	u32 tmp;
651
	int ret;
652

653
	buf_ring = &mhi_chan->buf_ring;
654
	tre_ring = &mhi_chan->tre_ring;
655
	tre_ring->el_size = sizeof(struct mhi_ring_element);
656
	tre_ring->len = tre_ring->el_size * tre_ring->elements;
657
	chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan];
658
	ret = mhi_alloc_aligned_ring(mhi_cntrl, tre_ring, tre_ring->len);
659
	if (ret)
660
		return -ENOMEM;
661

662
	buf_ring->el_size = sizeof(struct mhi_buf_info);
663
	buf_ring->len = buf_ring->el_size * buf_ring->elements;
664
	buf_ring->base = vzalloc(buf_ring->len);
665

666
	if (!buf_ring->base) {
667
		dma_free_coherent(mhi_cntrl->cntrl_dev, tre_ring->alloc_size,
668
				  tre_ring->pre_aligned, tre_ring->dma_handle);
669
		return -ENOMEM;
670
	}
671

672
	tmp = le32_to_cpu(chan_ctxt->chcfg);
673
	tmp &= ~CHAN_CTX_CHSTATE_MASK;
674
	tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_ENABLED);
675
	chan_ctxt->chcfg = cpu_to_le32(tmp);
676

677
	chan_ctxt->rbase = cpu_to_le64(tre_ring->iommu_base);
678
	chan_ctxt->rp = chan_ctxt->wp = chan_ctxt->rbase;
679
	chan_ctxt->rlen = cpu_to_le64(tre_ring->len);
680
	tre_ring->ctxt_wp = &chan_ctxt->wp;
681

682
	tre_ring->rp = tre_ring->wp = tre_ring->base;
683
	buf_ring->rp = buf_ring->wp = buf_ring->base;
684
	mhi_chan->db_cfg.db_mode = 1;
685

686
	/* Update to all cores */
687
	smp_wmb();
688

689
	return 0;
690
}
691

692
static int parse_ev_cfg(struct mhi_controller *mhi_cntrl,
693
			const struct mhi_controller_config *config)
694
{
695
	struct mhi_event *mhi_event;
696
	const struct mhi_event_config *event_cfg;
697
	struct device *dev = mhi_cntrl->cntrl_dev;
698
	int i, num;
699

700
	num = config->num_events;
701
	mhi_cntrl->total_ev_rings = num;
702
	mhi_cntrl->mhi_event = kcalloc(num, sizeof(*mhi_cntrl->mhi_event),
703
				       GFP_KERNEL);
704
	if (!mhi_cntrl->mhi_event)
705
		return -ENOMEM;
706

707
	/* Populate event ring */
708
	mhi_event = mhi_cntrl->mhi_event;
709
	for (i = 0; i < num; i++) {
710
		event_cfg = &config->event_cfg[i];
711

712
		mhi_event->er_index = i;
713
		mhi_event->ring.elements = event_cfg->num_elements;
714
		mhi_event->intmod = event_cfg->irq_moderation_ms;
715
		mhi_event->irq = event_cfg->irq;
716

717
		if (event_cfg->channel != U32_MAX) {
718
			/* This event ring has a dedicated channel */
719
			mhi_event->chan = event_cfg->channel;
720
			if (mhi_event->chan >= mhi_cntrl->max_chan) {
721
				dev_err(dev,
722
					"Event Ring channel not available\n");
723
				goto error_ev_cfg;
724
			}
725

726
			mhi_event->mhi_chan =
727
				&mhi_cntrl->mhi_chan[mhi_event->chan];
728
		}
729

730
		/* Priority is fixed to 1 for now */
731
		mhi_event->priority = 1;
732

733
		mhi_event->db_cfg.brstmode = event_cfg->mode;
734
		if (MHI_INVALID_BRSTMODE(mhi_event->db_cfg.brstmode))
735
			goto error_ev_cfg;
736

737
		if (mhi_event->db_cfg.brstmode == MHI_DB_BRST_ENABLE)
738
			mhi_event->db_cfg.process_db = mhi_db_brstmode;
739
		else
740
			mhi_event->db_cfg.process_db = mhi_db_brstmode_disable;
741

742
		mhi_event->data_type = event_cfg->data_type;
743

744
		switch (mhi_event->data_type) {
745
		case MHI_ER_DATA:
746
			mhi_event->process_event = mhi_process_data_event_ring;
747
			break;
748
		case MHI_ER_CTRL:
749
			mhi_event->process_event = mhi_process_ctrl_ev_ring;
750
			break;
751
		default:
752
			dev_err(dev, "Event Ring type not supported\n");
753
			goto error_ev_cfg;
754
		}
755

756
		mhi_event->hw_ring = event_cfg->hardware_event;
757
		if (mhi_event->hw_ring)
758
			mhi_cntrl->hw_ev_rings++;
759
		else
760
			mhi_cntrl->sw_ev_rings++;
761

762
		mhi_event->cl_manage = event_cfg->client_managed;
763
		mhi_event->offload_ev = event_cfg->offload_channel;
764
		mhi_event++;
765
	}
766

767
	return 0;
768

769
error_ev_cfg:
770

771
	kfree(mhi_cntrl->mhi_event);
772
	return -EINVAL;
773
}
774

775
static int parse_ch_cfg(struct mhi_controller *mhi_cntrl,
776
			const struct mhi_controller_config *config)
777
{
778
	const struct mhi_channel_config *ch_cfg;
779
	struct device *dev = mhi_cntrl->cntrl_dev;
780
	int i;
781
	u32 chan;
782

783
	mhi_cntrl->max_chan = config->max_channels;
784

785
	/*
786
	 * The allocation of MHI channels can exceed 32KB in some scenarios,
787
	 * so to avoid any memory possible allocation failures, vzalloc is
788
	 * used here
789
	 */
790
	mhi_cntrl->mhi_chan = vcalloc(mhi_cntrl->max_chan,
791
				      sizeof(*mhi_cntrl->mhi_chan));
792
	if (!mhi_cntrl->mhi_chan)
793
		return -ENOMEM;
794

795
	INIT_LIST_HEAD(&mhi_cntrl->lpm_chans);
796

797
	/* Populate channel configurations */
798
	for (i = 0; i < config->num_channels; i++) {
799
		struct mhi_chan *mhi_chan;
800

801
		ch_cfg = &config->ch_cfg[i];
802

803
		chan = ch_cfg->num;
804
		if (chan >= mhi_cntrl->max_chan) {
805
			dev_err(dev, "Channel %d not available\n", chan);
806
			goto error_chan_cfg;
807
		}
808

809
		mhi_chan = &mhi_cntrl->mhi_chan[chan];
810
		mhi_chan->name = ch_cfg->name;
811
		mhi_chan->chan = chan;
812

813
		mhi_chan->tre_ring.elements = ch_cfg->num_elements;
814
		if (!mhi_chan->tre_ring.elements)
815
			goto error_chan_cfg;
816

817
		/*
818
		 * For some channels, local ring length should be bigger than
819
		 * the transfer ring length due to internal logical channels
820
		 * in device. So host can queue much more buffers than transfer
821
		 * ring length. Example, RSC channels should have a larger local
822
		 * channel length than transfer ring length.
823
		 */
824
		mhi_chan->buf_ring.elements = ch_cfg->local_elements;
825
		if (!mhi_chan->buf_ring.elements)
826
			mhi_chan->buf_ring.elements = mhi_chan->tre_ring.elements;
827
		mhi_chan->er_index = ch_cfg->event_ring;
828
		mhi_chan->dir = ch_cfg->dir;
829

830
		/*
831
		 * For most channels, chtype is identical to channel directions.
832
		 * So, if it is not defined then assign channel direction to
833
		 * chtype
834
		 */
835
		mhi_chan->type = ch_cfg->type;
836
		if (!mhi_chan->type)
837
			mhi_chan->type = (enum mhi_ch_type)mhi_chan->dir;
838

839
		mhi_chan->ee_mask = ch_cfg->ee_mask;
840
		mhi_chan->db_cfg.pollcfg = ch_cfg->pollcfg;
841
		mhi_chan->lpm_notify = ch_cfg->lpm_notify;
842
		mhi_chan->offload_ch = ch_cfg->offload_channel;
843
		mhi_chan->db_cfg.reset_req = ch_cfg->doorbell_mode_switch;
844
		mhi_chan->pre_alloc = ch_cfg->auto_queue;
845
		mhi_chan->wake_capable = ch_cfg->wake_capable;
846

847
		/*
848
		 * If MHI host allocates buffers, then the channel direction
849
		 * should be DMA_FROM_DEVICE
850
		 */
851
		if (mhi_chan->pre_alloc && mhi_chan->dir != DMA_FROM_DEVICE) {
852
			dev_err(dev, "Invalid channel configuration\n");
853
			goto error_chan_cfg;
854
		}
855

856
		/*
857
		 * Bi-directional and direction less channel must be an
858
		 * offload channel
859
		 */
860
		if ((mhi_chan->dir == DMA_BIDIRECTIONAL ||
861
		     mhi_chan->dir == DMA_NONE) && !mhi_chan->offload_ch) {
862
			dev_err(dev, "Invalid channel configuration\n");
863
			goto error_chan_cfg;
864
		}
865

866
		if (!mhi_chan->offload_ch) {
867
			mhi_chan->db_cfg.brstmode = ch_cfg->doorbell;
868
			if (MHI_INVALID_BRSTMODE(mhi_chan->db_cfg.brstmode)) {
869
				dev_err(dev, "Invalid Door bell mode\n");
870
				goto error_chan_cfg;
871
			}
872
		}
873

874
		if (mhi_chan->db_cfg.brstmode == MHI_DB_BRST_ENABLE)
875
			mhi_chan->db_cfg.process_db = mhi_db_brstmode;
876
		else
877
			mhi_chan->db_cfg.process_db = mhi_db_brstmode_disable;
878

879
		mhi_chan->configured = true;
880

881
		if (mhi_chan->lpm_notify)
882
			list_add_tail(&mhi_chan->node, &mhi_cntrl->lpm_chans);
883
	}
884

885
	return 0;
886

887
error_chan_cfg:
888
	vfree(mhi_cntrl->mhi_chan);
889

890
	return -EINVAL;
891
}
892

893
static int parse_config(struct mhi_controller *mhi_cntrl,
894
			const struct mhi_controller_config *config)
895
{
896
	int ret;
897

898
	/* Parse MHI channel configuration */
899
	ret = parse_ch_cfg(mhi_cntrl, config);
900
	if (ret)
901
		return ret;
902

903
	/* Parse MHI event configuration */
904
	ret = parse_ev_cfg(mhi_cntrl, config);
905
	if (ret)
906
		goto error_ev_cfg;
907

908
	mhi_cntrl->timeout_ms = config->timeout_ms;
909
	if (!mhi_cntrl->timeout_ms)
910
		mhi_cntrl->timeout_ms = MHI_TIMEOUT_MS;
911

912
	mhi_cntrl->ready_timeout_ms = config->ready_timeout_ms;
913
	mhi_cntrl->bounce_buf = config->use_bounce_buf;
914
	mhi_cntrl->buffer_len = config->buf_len;
915
	if (!mhi_cntrl->buffer_len)
916
		mhi_cntrl->buffer_len = MHI_MAX_MTU;
917

918
	/* By default, host is allowed to ring DB in both M0 and M2 states */
919
	mhi_cntrl->db_access = MHI_PM_M0 | MHI_PM_M2;
920
	if (config->m2_no_db)
921
		mhi_cntrl->db_access &= ~MHI_PM_M2;
922

923
	return 0;
924

925
error_ev_cfg:
926
	vfree(mhi_cntrl->mhi_chan);
927

928
	return ret;
929
}
930

931
int mhi_register_controller(struct mhi_controller *mhi_cntrl,
932
			    const struct mhi_controller_config *config)
933
{
934
	struct mhi_event *mhi_event;
935
	struct mhi_chan *mhi_chan;
936
	struct mhi_cmd *mhi_cmd;
937
	struct mhi_device *mhi_dev;
938
	int ret, i;
939

940
	if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->regs ||
941
	    !mhi_cntrl->runtime_get || !mhi_cntrl->runtime_put ||
942
	    !mhi_cntrl->status_cb || !mhi_cntrl->read_reg ||
943
	    !mhi_cntrl->write_reg || !mhi_cntrl->nr_irqs ||
944
	    !mhi_cntrl->irq || !mhi_cntrl->reg_len)
945
		return -EINVAL;
946

947
	ret = parse_config(mhi_cntrl, config);
948
	if (ret)
949
		return -EINVAL;
950

951
	mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS,
952
				     sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL);
953
	if (!mhi_cntrl->mhi_cmd) {
954
		ret = -ENOMEM;
955
		goto err_free_event;
956
	}
957

958
	INIT_LIST_HEAD(&mhi_cntrl->transition_list);
959
	mutex_init(&mhi_cntrl->pm_mutex);
960
	rwlock_init(&mhi_cntrl->pm_lock);
961
	spin_lock_init(&mhi_cntrl->transition_lock);
962
	spin_lock_init(&mhi_cntrl->wlock);
963
	INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker);
964
	init_waitqueue_head(&mhi_cntrl->state_event);
965

966
	mhi_cntrl->hiprio_wq = alloc_ordered_workqueue("mhi_hiprio_wq", WQ_HIGHPRI);
967
	if (!mhi_cntrl->hiprio_wq) {
968
		dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate workqueue\n");
969
		ret = -ENOMEM;
970
		goto err_free_cmd;
971
	}
972

973
	mhi_cmd = mhi_cntrl->mhi_cmd;
974
	for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++)
975
		spin_lock_init(&mhi_cmd->lock);
976

977
	mhi_event = mhi_cntrl->mhi_event;
978
	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
979
		/* Skip for offload events */
980
		if (mhi_event->offload_ev)
981
			continue;
982

983
		mhi_event->mhi_cntrl = mhi_cntrl;
984
		spin_lock_init(&mhi_event->lock);
985
		if (mhi_event->data_type == MHI_ER_CTRL)
986
			tasklet_init(&mhi_event->task, mhi_ctrl_ev_task,
987
				     (ulong)mhi_event);
988
		else
989
			tasklet_init(&mhi_event->task, mhi_ev_task,
990
				     (ulong)mhi_event);
991
	}
992

993
	mhi_chan = mhi_cntrl->mhi_chan;
994
	for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
995
		mutex_init(&mhi_chan->mutex);
996
		init_completion(&mhi_chan->completion);
997
		rwlock_init(&mhi_chan->lock);
998

999
		/* used in setting bei field of TRE */
1000
		mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
1001
		mhi_chan->intmod = mhi_event->intmod;
1002
	}
1003

1004
	if (mhi_cntrl->bounce_buf) {
1005
		mhi_cntrl->map_single = mhi_map_single_use_bb;
1006
		mhi_cntrl->unmap_single = mhi_unmap_single_use_bb;
1007
	} else {
1008
		mhi_cntrl->map_single = mhi_map_single_no_bb;
1009
		mhi_cntrl->unmap_single = mhi_unmap_single_no_bb;
1010
	}
1011

1012
	mhi_cntrl->index = ida_alloc(&mhi_controller_ida, GFP_KERNEL);
1013
	if (mhi_cntrl->index < 0) {
1014
		ret = mhi_cntrl->index;
1015
		goto err_destroy_wq;
1016
	}
1017

1018
	ret = mhi_init_irq_setup(mhi_cntrl);
1019
	if (ret)
1020
		goto err_ida_free;
1021

1022
	/* Register controller with MHI bus */
1023
	mhi_dev = mhi_alloc_device(mhi_cntrl);
1024
	if (IS_ERR(mhi_dev)) {
1025
		dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate MHI device\n");
1026
		ret = PTR_ERR(mhi_dev);
1027
		goto error_setup_irq;
1028
	}
1029

1030
	mhi_dev->dev_type = MHI_DEVICE_CONTROLLER;
1031
	mhi_dev->mhi_cntrl = mhi_cntrl;
1032
	dev_set_name(&mhi_dev->dev, "mhi%d", mhi_cntrl->index);
1033
	mhi_dev->name = dev_name(&mhi_dev->dev);
1034

1035
	/* Init wakeup source */
1036
	device_init_wakeup(&mhi_dev->dev, true);
1037

1038
	ret = device_add(&mhi_dev->dev);
1039
	if (ret)
1040
		goto err_release_dev;
1041

1042
	if (mhi_cntrl->edl_trigger) {
1043
		ret = sysfs_create_file(&mhi_dev->dev.kobj, &dev_attr_trigger_edl.attr);
1044
		if (ret)
1045
			goto err_release_dev;
1046
	}
1047

1048
	mhi_cntrl->mhi_dev = mhi_dev;
1049

1050
	mhi_create_debugfs(mhi_cntrl);
1051

1052
	return 0;
1053

1054
err_release_dev:
1055
	put_device(&mhi_dev->dev);
1056
error_setup_irq:
1057
	mhi_deinit_free_irq(mhi_cntrl);
1058
err_ida_free:
1059
	ida_free(&mhi_controller_ida, mhi_cntrl->index);
1060
err_destroy_wq:
1061
	destroy_workqueue(mhi_cntrl->hiprio_wq);
1062
err_free_cmd:
1063
	kfree(mhi_cntrl->mhi_cmd);
1064
err_free_event:
1065
	kfree(mhi_cntrl->mhi_event);
1066
	vfree(mhi_cntrl->mhi_chan);
1067

1068
	return ret;
1069
}
1070
EXPORT_SYMBOL_GPL(mhi_register_controller);
1071

1072
void mhi_unregister_controller(struct mhi_controller *mhi_cntrl)
1073
{
1074
	struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev;
1075
	struct mhi_chan *mhi_chan = mhi_cntrl->mhi_chan;
1076
	unsigned int i;
1077

1078
	mhi_deinit_free_irq(mhi_cntrl);
1079
	mhi_destroy_debugfs(mhi_cntrl);
1080

1081
	if (mhi_cntrl->edl_trigger)
1082
		sysfs_remove_file(&mhi_dev->dev.kobj, &dev_attr_trigger_edl.attr);
1083

1084
	destroy_workqueue(mhi_cntrl->hiprio_wq);
1085
	kfree(mhi_cntrl->mhi_cmd);
1086
	kfree(mhi_cntrl->mhi_event);
1087

1088
	/* Drop the references to MHI devices created for channels */
1089
	for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
1090
		if (!mhi_chan->mhi_dev)
1091
			continue;
1092

1093
		put_device(&mhi_chan->mhi_dev->dev);
1094
	}
1095
	vfree(mhi_cntrl->mhi_chan);
1096

1097
	device_del(&mhi_dev->dev);
1098
	put_device(&mhi_dev->dev);
1099

1100
	ida_free(&mhi_controller_ida, mhi_cntrl->index);
1101
}
1102
EXPORT_SYMBOL_GPL(mhi_unregister_controller);
1103

1104
struct mhi_controller *mhi_alloc_controller(void)
1105
{
1106
	struct mhi_controller *mhi_cntrl;
1107

1108
	mhi_cntrl = kzalloc(sizeof(*mhi_cntrl), GFP_KERNEL);
1109

1110
	return mhi_cntrl;
1111
}
1112
EXPORT_SYMBOL_GPL(mhi_alloc_controller);
1113

1114
void mhi_free_controller(struct mhi_controller *mhi_cntrl)
1115
{
1116
	kfree(mhi_cntrl);
1117
}
1118
EXPORT_SYMBOL_GPL(mhi_free_controller);
1119

1120
int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl)
1121
{
1122
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1123
	u32 bhi_off, bhie_off;
1124
	int ret;
1125

1126
	mutex_lock(&mhi_cntrl->pm_mutex);
1127

1128
	ret = mhi_init_dev_ctxt(mhi_cntrl);
1129
	if (ret)
1130
		goto error_dev_ctxt;
1131

1132
	ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIOFF, &bhi_off);
1133
	if (ret) {
1134
		dev_err(dev, "Error getting BHI offset\n");
1135
		goto error_reg_offset;
1136
	}
1137

1138
	if (bhi_off >= mhi_cntrl->reg_len) {
1139
		dev_err(dev, "BHI offset: 0x%x is out of range: 0x%zx\n",
1140
			bhi_off, mhi_cntrl->reg_len);
1141
		ret = -ERANGE;
1142
		goto error_reg_offset;
1143
	}
1144
	mhi_cntrl->bhi = mhi_cntrl->regs + bhi_off;
1145

1146
	if (mhi_cntrl->fbc_download || mhi_cntrl->rddm_size || mhi_cntrl->seg_len) {
1147
		ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIEOFF,
1148
				   &bhie_off);
1149
		if (ret) {
1150
			dev_err(dev, "Error getting BHIE offset\n");
1151
			goto error_reg_offset;
1152
		}
1153

1154
		if (bhie_off >= mhi_cntrl->reg_len) {
1155
			dev_err(dev,
1156
				"BHIe offset: 0x%x is out of range: 0x%zx\n",
1157
				bhie_off, mhi_cntrl->reg_len);
1158
			ret = -ERANGE;
1159
			goto error_reg_offset;
1160
		}
1161
		mhi_cntrl->bhie = mhi_cntrl->regs + bhie_off;
1162
	}
1163

1164
	if (mhi_cntrl->rddm_size) {
1165
		/*
1166
		 * This controller supports RDDM, so we need to manually clear
1167
		 * BHIE RX registers since POR values are undefined.
1168
		 */
1169
		memset_io(mhi_cntrl->bhie + BHIE_RXVECADDR_LOW_OFFS,
1170
			  0, BHIE_RXVECSTATUS_OFFS - BHIE_RXVECADDR_LOW_OFFS +
1171
			  4);
1172
		/*
1173
		 * Allocate RDDM table for debugging purpose if specified
1174
		 */
1175
		mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->rddm_image,
1176
				     mhi_cntrl->rddm_size);
1177
		if (mhi_cntrl->rddm_image) {
1178
			ret = mhi_rddm_prepare(mhi_cntrl,
1179
					       mhi_cntrl->rddm_image);
1180
			if (ret) {
1181
				mhi_free_bhie_table(mhi_cntrl,
1182
						    mhi_cntrl->rddm_image);
1183
				goto error_reg_offset;
1184
			}
1185
		}
1186
	}
1187

1188
	mutex_unlock(&mhi_cntrl->pm_mutex);
1189

1190
	return 0;
1191

1192
error_reg_offset:
1193
	mhi_deinit_dev_ctxt(mhi_cntrl);
1194

1195
error_dev_ctxt:
1196
	mutex_unlock(&mhi_cntrl->pm_mutex);
1197

1198
	return ret;
1199
}
1200
EXPORT_SYMBOL_GPL(mhi_prepare_for_power_up);
1201

1202
void mhi_unprepare_after_power_down(struct mhi_controller *mhi_cntrl)
1203
{
1204
	if (mhi_cntrl->fbc_image) {
1205
		mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image);
1206
		mhi_cntrl->fbc_image = NULL;
1207
	}
1208

1209
	if (mhi_cntrl->rddm_image) {
1210
		mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->rddm_image);
1211
		mhi_cntrl->rddm_image = NULL;
1212
	}
1213

1214
	mhi_cntrl->bhi = NULL;
1215
	mhi_cntrl->bhie = NULL;
1216

1217
	mhi_deinit_dev_ctxt(mhi_cntrl);
1218
}
1219
EXPORT_SYMBOL_GPL(mhi_unprepare_after_power_down);
1220

1221
static void mhi_release_device(struct device *dev)
1222
{
1223
	struct mhi_device *mhi_dev = to_mhi_device(dev);
1224

1225
	/*
1226
	 * We need to set the mhi_chan->mhi_dev to NULL here since the MHI
1227
	 * devices for the channels will only get created if the mhi_dev
1228
	 * associated with it is NULL. This scenario will happen during the
1229
	 * controller suspend and resume.
1230
	 */
1231
	if (mhi_dev->ul_chan)
1232
		mhi_dev->ul_chan->mhi_dev = NULL;
1233

1234
	if (mhi_dev->dl_chan)
1235
		mhi_dev->dl_chan->mhi_dev = NULL;
1236

1237
	kfree(mhi_dev);
1238
}
1239

1240
struct mhi_device *mhi_alloc_device(struct mhi_controller *mhi_cntrl)
1241
{
1242
	struct mhi_device *mhi_dev;
1243
	struct device *dev;
1244

1245
	mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL);
1246
	if (!mhi_dev)
1247
		return ERR_PTR(-ENOMEM);
1248

1249
	dev = &mhi_dev->dev;
1250
	device_initialize(dev);
1251
	dev->bus = &mhi_bus_type;
1252
	dev->release = mhi_release_device;
1253

1254
	if (mhi_cntrl->mhi_dev) {
1255
		/* for MHI client devices, parent is the MHI controller device */
1256
		dev->parent = &mhi_cntrl->mhi_dev->dev;
1257
	} else {
1258
		/* for MHI controller device, parent is the bus device (e.g. pci device) */
1259
		dev->parent = mhi_cntrl->cntrl_dev;
1260
	}
1261

1262
	mhi_dev->mhi_cntrl = mhi_cntrl;
1263
	mhi_dev->dev_wake = 0;
1264

1265
	return mhi_dev;
1266
}
1267

1268
static int mhi_driver_probe(struct device *dev)
1269
{
1270
	struct mhi_device *mhi_dev = to_mhi_device(dev);
1271
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1272
	struct device_driver *drv = dev->driver;
1273
	struct mhi_driver *mhi_drv = to_mhi_driver(drv);
1274
	struct mhi_event *mhi_event;
1275
	struct mhi_chan *ul_chan = mhi_dev->ul_chan;
1276
	struct mhi_chan *dl_chan = mhi_dev->dl_chan;
1277
	int ret;
1278

1279
	/* Bring device out of LPM */
1280
	ret = mhi_device_get_sync(mhi_dev);
1281
	if (ret)
1282
		return ret;
1283

1284
	ret = -EINVAL;
1285

1286
	if (ul_chan) {
1287
		/*
1288
		 * If channel supports LPM notifications then status_cb should
1289
		 * be provided
1290
		 */
1291
		if (ul_chan->lpm_notify && !mhi_drv->status_cb)
1292
			goto exit_probe;
1293

1294
		/* For non-offload channels then xfer_cb should be provided */
1295
		if (!ul_chan->offload_ch && !mhi_drv->ul_xfer_cb)
1296
			goto exit_probe;
1297

1298
		ul_chan->xfer_cb = mhi_drv->ul_xfer_cb;
1299
	}
1300

1301
	ret = -EINVAL;
1302
	if (dl_chan) {
1303
		/*
1304
		 * If channel supports LPM notifications then status_cb should
1305
		 * be provided
1306
		 */
1307
		if (dl_chan->lpm_notify && !mhi_drv->status_cb)
1308
			goto exit_probe;
1309

1310
		/* For non-offload channels then xfer_cb should be provided */
1311
		if (!dl_chan->offload_ch && !mhi_drv->dl_xfer_cb)
1312
			goto exit_probe;
1313

1314
		mhi_event = &mhi_cntrl->mhi_event[dl_chan->er_index];
1315

1316
		/*
1317
		 * If the channel event ring is managed by client, then
1318
		 * status_cb must be provided so that the framework can
1319
		 * notify pending data
1320
		 */
1321
		if (mhi_event->cl_manage && !mhi_drv->status_cb)
1322
			goto exit_probe;
1323

1324
		dl_chan->xfer_cb = mhi_drv->dl_xfer_cb;
1325
	}
1326

1327
	/* Call the user provided probe function */
1328
	ret = mhi_drv->probe(mhi_dev, mhi_dev->id);
1329
	if (ret)
1330
		goto exit_probe;
1331

1332
	mhi_device_put(mhi_dev);
1333

1334
	return ret;
1335

1336
exit_probe:
1337
	mhi_unprepare_from_transfer(mhi_dev);
1338

1339
	mhi_device_put(mhi_dev);
1340

1341
	return ret;
1342
}
1343

1344
static int mhi_driver_remove(struct device *dev)
1345
{
1346
	struct mhi_device *mhi_dev = to_mhi_device(dev);
1347
	struct mhi_driver *mhi_drv = to_mhi_driver(dev->driver);
1348
	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1349
	struct mhi_chan *mhi_chan;
1350
	enum mhi_ch_state ch_state[] = {
1351
		MHI_CH_STATE_DISABLED,
1352
		MHI_CH_STATE_DISABLED
1353
	};
1354
	int dir;
1355

1356
	/* Skip if it is a controller device */
1357
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1358
		return 0;
1359

1360
	/* Reset both channels */
1361
	for (dir = 0; dir < 2; dir++) {
1362
		mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1363

1364
		if (!mhi_chan)
1365
			continue;
1366

1367
		/* Wake all threads waiting for completion */
1368
		write_lock_irq(&mhi_chan->lock);
1369
		mhi_chan->ccs = MHI_EV_CC_INVALID;
1370
		complete_all(&mhi_chan->completion);
1371
		write_unlock_irq(&mhi_chan->lock);
1372

1373
		/* Set the channel state to disabled */
1374
		mutex_lock(&mhi_chan->mutex);
1375
		write_lock_irq(&mhi_chan->lock);
1376
		ch_state[dir] = mhi_chan->ch_state;
1377
		mhi_chan->ch_state = MHI_CH_STATE_SUSPENDED;
1378
		write_unlock_irq(&mhi_chan->lock);
1379

1380
		/* Reset the non-offload channel */
1381
		if (!mhi_chan->offload_ch)
1382
			mhi_reset_chan(mhi_cntrl, mhi_chan);
1383

1384
		mutex_unlock(&mhi_chan->mutex);
1385
	}
1386

1387
	mhi_drv->remove(mhi_dev);
1388

1389
	/* De-init channel if it was enabled */
1390
	for (dir = 0; dir < 2; dir++) {
1391
		mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1392

1393
		if (!mhi_chan)
1394
			continue;
1395

1396
		mutex_lock(&mhi_chan->mutex);
1397

1398
		if ((ch_state[dir] == MHI_CH_STATE_ENABLED ||
1399
		     ch_state[dir] == MHI_CH_STATE_STOP) &&
1400
		    !mhi_chan->offload_ch)
1401
			mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
1402

1403
		mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
1404

1405
		mutex_unlock(&mhi_chan->mutex);
1406
	}
1407

1408
	while (mhi_dev->dev_wake)
1409
		mhi_device_put(mhi_dev);
1410

1411
	return 0;
1412
}
1413

1414
int __mhi_driver_register(struct mhi_driver *mhi_drv, struct module *owner)
1415
{
1416
	struct device_driver *driver = &mhi_drv->driver;
1417

1418
	if (!mhi_drv->probe || !mhi_drv->remove)
1419
		return -EINVAL;
1420

1421
	driver->bus = &mhi_bus_type;
1422
	driver->owner = owner;
1423
	driver->probe = mhi_driver_probe;
1424
	driver->remove = mhi_driver_remove;
1425

1426
	return driver_register(driver);
1427
}
1428
EXPORT_SYMBOL_GPL(__mhi_driver_register);
1429

1430
void mhi_driver_unregister(struct mhi_driver *mhi_drv)
1431
{
1432
	driver_unregister(&mhi_drv->driver);
1433
}
1434
EXPORT_SYMBOL_GPL(mhi_driver_unregister);
1435

1436
static int mhi_uevent(const struct device *dev, struct kobj_uevent_env *env)
1437
{
1438
	const struct mhi_device *mhi_dev = to_mhi_device(dev);
1439

1440
	return add_uevent_var(env, "MODALIAS=" MHI_DEVICE_MODALIAS_FMT,
1441
					mhi_dev->name);
1442
}
1443

1444
static int mhi_match(struct device *dev, const struct device_driver *drv)
1445
{
1446
	struct mhi_device *mhi_dev = to_mhi_device(dev);
1447
	const struct mhi_driver *mhi_drv = to_mhi_driver(drv);
1448
	const struct mhi_device_id *id;
1449

1450
	/*
1451
	 * If the device is a controller type then there is no client driver
1452
	 * associated with it
1453
	 */
1454
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1455
		return 0;
1456

1457
	for (id = mhi_drv->id_table; id->chan[0]; id++)
1458
		if (!strcmp(mhi_dev->name, id->chan)) {
1459
			mhi_dev->id = id;
1460
			return 1;
1461
		}
1462

1463
	return 0;
1464
};
1465

1466
const struct bus_type mhi_bus_type = {
1467
	.name = "mhi",
1468
	.dev_name = "mhi",
1469
	.match = mhi_match,
1470
	.uevent = mhi_uevent,
1471
	.dev_groups = mhi_dev_groups,
1472
};
1473

1474
static int __init mhi_init(void)
1475
{
1476
	mhi_debugfs_init();
1477
	return bus_register(&mhi_bus_type);
1478
}
1479

1480
static void __exit mhi_exit(void)
1481
{
1482
	mhi_debugfs_exit();
1483
	bus_unregister(&mhi_bus_type);
1484
}
1485

1486
postcore_initcall(mhi_init);
1487
module_exit(mhi_exit);
1488

1489
MODULE_LICENSE("GPL v2");
1490
MODULE_DESCRIPTION("Modem Host Interface");
1491

1492