1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * MHI Endpoint bus stack
4
 *
5
 * Copyright (C) 2022 Linaro Ltd.
6
 * Author: Manivannan Sadhasivam <[email protected]>
7
 */
8

9
#include <linux/bitfield.h>
10
#include <linux/delay.h>
11
#include <linux/dma-direction.h>
12
#include <linux/interrupt.h>
13
#include <linux/io.h>
14
#include <linux/irq.h>
15
#include <linux/mhi_ep.h>
16
#include <linux/mod_devicetable.h>
17
#include <linux/module.h>
18
#include "internal.h"
19

20
#define M0_WAIT_DELAY_MS	100
21
#define M0_WAIT_COUNT		100
22

23
static DEFINE_IDA(mhi_ep_cntrl_ida);
24

25
static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id);
26
static int mhi_ep_destroy_device(struct device *dev, void *data);
27

28
static int mhi_ep_send_event(struct mhi_ep_cntrl *mhi_cntrl, u32 ring_idx,
29
			     struct mhi_ring_element *el, bool bei)
30
{
31
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
32
	union mhi_ep_ring_ctx *ctx;
33
	struct mhi_ep_ring *ring;
34
	int ret;
35

36
	mutex_lock(&mhi_cntrl->event_lock);
37
	ring = &mhi_cntrl->mhi_event[ring_idx].ring;
38
	ctx = (union mhi_ep_ring_ctx *)&mhi_cntrl->ev_ctx_cache[ring_idx];
39
	if (!ring->started) {
40
		ret = mhi_ep_ring_start(mhi_cntrl, ring, ctx);
41
		if (ret) {
42
			dev_err(dev, "Error starting event ring (%u)\n", ring_idx);
43
			goto err_unlock;
44
		}
45
	}
46

47
	/* Add element to the event ring */
48
	ret = mhi_ep_ring_add_element(ring, el);
49
	if (ret) {
50
		dev_err(dev, "Error adding element to event ring (%u)\n", ring_idx);
51
		goto err_unlock;
52
	}
53

54
	mutex_unlock(&mhi_cntrl->event_lock);
55

56
	/*
57
	 * As per the MHI specification, section 4.3, Interrupt moderation:
58
	 *
59
	 * 1. If BEI flag is not set, cancel any pending intmodt work if started
60
	 * for the event ring and raise IRQ immediately.
61
	 *
62
	 * 2. If both BEI and intmodt are set, and if no IRQ is pending for the
63
	 * same event ring, start the IRQ delayed work as per the value of
64
	 * intmodt. If previous IRQ is pending, then do nothing as the pending
65
	 * IRQ is enough for the host to process the current event ring element.
66
	 *
67
	 * 3. If BEI is set and intmodt is not set, no need to raise IRQ.
68
	 */
69
	if (!bei) {
70
		if (READ_ONCE(ring->irq_pending))
71
			cancel_delayed_work(&ring->intmodt_work);
72

73
		mhi_cntrl->raise_irq(mhi_cntrl, ring->irq_vector);
74
	} else if (ring->intmodt && !READ_ONCE(ring->irq_pending)) {
75
		WRITE_ONCE(ring->irq_pending, true);
76
		schedule_delayed_work(&ring->intmodt_work, msecs_to_jiffies(ring->intmodt));
77
	}
78

79
	return 0;
80

81
err_unlock:
82
	mutex_unlock(&mhi_cntrl->event_lock);
83

84
	return ret;
85
}
86

87
static int mhi_ep_send_completion_event(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_ring *ring,
88
					struct mhi_ring_element *tre, u32 len, enum mhi_ev_ccs code)
89
{
90
	struct mhi_ring_element *event;
91
	int ret;
92

93
	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
94
	if (!event)
95
		return -ENOMEM;
96

97
	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(*tre));
98
	event->dword[0] = MHI_TRE_EV_DWORD0(code, len);
99
	event->dword[1] = MHI_TRE_EV_DWORD1(ring->ch_id, MHI_PKT_TYPE_TX_EVENT);
100

101
	ret = mhi_ep_send_event(mhi_cntrl, ring->er_index, event, MHI_TRE_DATA_GET_BEI(tre));
102
	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
103

104
	return ret;
105
}
106

107
int mhi_ep_send_state_change_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_state state)
108
{
109
	struct mhi_ring_element *event;
110
	int ret;
111

112
	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
113
	if (!event)
114
		return -ENOMEM;
115

116
	event->dword[0] = MHI_SC_EV_DWORD0(state);
117
	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_STATE_CHANGE_EVENT);
118

119
	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
120
	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
121

122
	return ret;
123
}
124

125
int mhi_ep_send_ee_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ee_type exec_env)
126
{
127
	struct mhi_ring_element *event;
128
	int ret;
129

130
	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
131
	if (!event)
132
		return -ENOMEM;
133

134
	event->dword[0] = MHI_EE_EV_DWORD0(exec_env);
135
	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_EE_EVENT);
136

137
	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
138
	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
139

140
	return ret;
141
}
142

143
static int mhi_ep_send_cmd_comp_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ev_ccs code)
144
{
145
	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
146
	struct mhi_ring_element *event;
147
	int ret;
148

149
	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
150
	if (!event)
151
		return -ENOMEM;
152

153
	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(struct mhi_ring_element));
154
	event->dword[0] = MHI_CC_EV_DWORD0(code);
155
	event->dword[1] = MHI_CC_EV_DWORD1(MHI_PKT_TYPE_CMD_COMPLETION_EVENT);
156

157
	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
158
	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
159

160
	return ret;
161
}
162

163
static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_element *el)
164
{
165
	struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
166
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
167
	struct mhi_result result = {};
168
	struct mhi_ep_chan *mhi_chan;
169
	struct mhi_ep_ring *ch_ring;
170
	u32 tmp, ch_id;
171
	int ret;
172

173
	ch_id = MHI_TRE_GET_CMD_CHID(el);
174

175
	/* Check if the channel is supported by the controller */
176
	if ((ch_id >= mhi_cntrl->max_chan) || !mhi_cntrl->mhi_chan[ch_id].name) {
177
		dev_dbg(dev, "Channel (%u) not supported!\n", ch_id);
178
		return -ENODEV;
179
	}
180

181
	mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
182
	ch_ring = &mhi_cntrl->mhi_chan[ch_id].ring;
183

184
	switch (MHI_TRE_GET_CMD_TYPE(el)) {
185
	case MHI_PKT_TYPE_START_CHAN_CMD:
186
		dev_dbg(dev, "Received START command for channel (%u)\n", ch_id);
187

188
		mutex_lock(&mhi_chan->lock);
189
		/* Initialize and configure the corresponding channel ring */
190
		if (!ch_ring->started) {
191
			ret = mhi_ep_ring_start(mhi_cntrl, ch_ring,
192
				(union mhi_ep_ring_ctx *)&mhi_cntrl->ch_ctx_cache[ch_id]);
193
			if (ret) {
194
				dev_err(dev, "Failed to start ring for channel (%u)\n", ch_id);
195
				ret = mhi_ep_send_cmd_comp_event(mhi_cntrl,
196
							MHI_EV_CC_UNDEFINED_ERR);
197
				if (ret)
198
					dev_err(dev, "Error sending completion event: %d\n", ret);
199

200
				goto err_unlock;
201
			}
202

203
			mhi_chan->rd_offset = ch_ring->rd_offset;
204
		}
205

206
		/* Set channel state to RUNNING */
207
		mhi_chan->state = MHI_CH_STATE_RUNNING;
208
		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
209
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
210
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
211
		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
212

213
		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
214
		if (ret) {
215
			dev_err(dev, "Error sending command completion event (%u)\n",
216
				MHI_EV_CC_SUCCESS);
217
			goto err_unlock;
218
		}
219

220
		mutex_unlock(&mhi_chan->lock);
221

222
		/*
223
		 * Create MHI device only during UL channel start. Since the MHI
224
		 * channels operate in a pair, we'll associate both UL and DL
225
		 * channels to the same device.
226
		 *
227
		 * We also need to check for mhi_dev != NULL because, the host
228
		 * will issue START_CHAN command during resume and we don't
229
		 * destroy the device during suspend.
230
		 */
231
		if (!(ch_id % 2) && !mhi_chan->mhi_dev) {
232
			ret = mhi_ep_create_device(mhi_cntrl, ch_id);
233
			if (ret) {
234
				dev_err(dev, "Error creating device for channel (%u)\n", ch_id);
235
				mhi_ep_handle_syserr(mhi_cntrl);
236
				return ret;
237
			}
238
		}
239

240
		/* Finally, enable DB for the channel */
241
		mhi_ep_mmio_enable_chdb(mhi_cntrl, ch_id);
242

243
		break;
244
	case MHI_PKT_TYPE_STOP_CHAN_CMD:
245
		dev_dbg(dev, "Received STOP command for channel (%u)\n", ch_id);
246
		if (!ch_ring->started) {
247
			dev_err(dev, "Channel (%u) not opened\n", ch_id);
248
			return -ENODEV;
249
		}
250

251
		mutex_lock(&mhi_chan->lock);
252
		/* Disable DB for the channel */
253
		mhi_ep_mmio_disable_chdb(mhi_cntrl, ch_id);
254

255
		/* Send channel disconnect status to client drivers */
256
		if (mhi_chan->xfer_cb) {
257
			result.transaction_status = -ENOTCONN;
258
			result.bytes_xferd = 0;
259
			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
260
		}
261

262
		/* Set channel state to STOP */
263
		mhi_chan->state = MHI_CH_STATE_STOP;
264
		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
265
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
266
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_STOP);
267
		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
268

269
		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
270
		if (ret) {
271
			dev_err(dev, "Error sending command completion event (%u)\n",
272
				MHI_EV_CC_SUCCESS);
273
			goto err_unlock;
274
		}
275

276
		mutex_unlock(&mhi_chan->lock);
277
		break;
278
	case MHI_PKT_TYPE_RESET_CHAN_CMD:
279
		dev_dbg(dev, "Received RESET command for channel (%u)\n", ch_id);
280
		if (!ch_ring->started) {
281
			dev_err(dev, "Channel (%u) not opened\n", ch_id);
282
			return -ENODEV;
283
		}
284

285
		mutex_lock(&mhi_chan->lock);
286
		/* Stop and reset the transfer ring */
287
		mhi_ep_ring_reset(mhi_cntrl, ch_ring);
288

289
		/* Send channel disconnect status to client driver */
290
		if (mhi_chan->xfer_cb) {
291
			result.transaction_status = -ENOTCONN;
292
			result.bytes_xferd = 0;
293
			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
294
		}
295

296
		/* Set channel state to DISABLED */
297
		mhi_chan->state = MHI_CH_STATE_DISABLED;
298
		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
299
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
300
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_DISABLED);
301
		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
302

303
		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
304
		if (ret) {
305
			dev_err(dev, "Error sending command completion event (%u)\n",
306
				MHI_EV_CC_SUCCESS);
307
			goto err_unlock;
308
		}
309

310
		mutex_unlock(&mhi_chan->lock);
311
		break;
312
	default:
313
		dev_err(dev, "Invalid command received: %lu for channel (%u)\n",
314
			MHI_TRE_GET_CMD_TYPE(el), ch_id);
315
		return -EINVAL;
316
	}
317

318
	return 0;
319

320
err_unlock:
321
	mutex_unlock(&mhi_chan->lock);
322

323
	return ret;
324
}
325

326
bool mhi_ep_queue_is_empty(struct mhi_ep_device *mhi_dev, enum dma_data_direction dir)
327
{
328
	struct mhi_ep_chan *mhi_chan = (dir == DMA_FROM_DEVICE) ? mhi_dev->dl_chan :
329
								mhi_dev->ul_chan;
330
	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
331
	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
332

333
	return !!(mhi_chan->rd_offset == ring->wr_offset);
334
}
335
EXPORT_SYMBOL_GPL(mhi_ep_queue_is_empty);
336

337
static void mhi_ep_read_completion(struct mhi_ep_buf_info *buf_info)
338
{
339
	struct mhi_ep_device *mhi_dev = buf_info->mhi_dev;
340
	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
341
	struct mhi_ep_chan *mhi_chan = mhi_dev->ul_chan;
342
	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
343
	struct mhi_ring_element *el = &ring->ring_cache[ring->rd_offset];
344
	struct mhi_result result = {};
345
	int ret;
346

347
	if (mhi_chan->xfer_cb) {
348
		result.buf_addr = buf_info->cb_buf;
349
		result.dir = mhi_chan->dir;
350
		result.bytes_xferd = buf_info->size;
351

352
		mhi_chan->xfer_cb(mhi_dev, &result);
353
	}
354

355
	/*
356
	 * The host will split the data packet into multiple TREs if it can't fit
357
	 * the packet in a single TRE. In that case, CHAIN flag will be set by the
358
	 * host for all TREs except the last one.
359
	 */
360
	if (buf_info->code != MHI_EV_CC_OVERFLOW) {
361
		if (MHI_TRE_DATA_GET_CHAIN(el)) {
362
			/*
363
			 * IEOB (Interrupt on End of Block) flag will be set by the host if
364
			 * it expects the completion event for all TREs of a TD.
365
			 */
366
			if (MHI_TRE_DATA_GET_IEOB(el)) {
367
				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
368
							     MHI_TRE_DATA_GET_LEN(el),
369
							     MHI_EV_CC_EOB);
370
				if (ret < 0) {
371
					dev_err(&mhi_chan->mhi_dev->dev,
372
						"Error sending transfer compl. event\n");
373
					goto err_free_tre_buf;
374
				}
375
			}
376
		} else {
377
			/*
378
			 * IEOT (Interrupt on End of Transfer) flag will be set by the host
379
			 * for the last TRE of the TD and expects the completion event for
380
			 * the same.
381
			 */
382
			if (MHI_TRE_DATA_GET_IEOT(el)) {
383
				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
384
							     MHI_TRE_DATA_GET_LEN(el),
385
							     MHI_EV_CC_EOT);
386
				if (ret < 0) {
387
					dev_err(&mhi_chan->mhi_dev->dev,
388
						"Error sending transfer compl. event\n");
389
					goto err_free_tre_buf;
390
				}
391
			}
392
		}
393
	}
394

395
	mhi_ep_ring_inc_index(ring);
396

397
err_free_tre_buf:
398
	kmem_cache_free(mhi_cntrl->tre_buf_cache, buf_info->cb_buf);
399
}
400

401
static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl,
402
			       struct mhi_ep_ring *ring)
403
{
404
	struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
405
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
406
	size_t tr_len, read_offset;
407
	struct mhi_ep_buf_info buf_info = {};
408
	u32 len = MHI_EP_DEFAULT_MTU;
409
	struct mhi_ring_element *el;
410
	void *buf_addr;
411
	int ret;
412

413
	do {
414
		/* Don't process the transfer ring if the channel is not in RUNNING state */
415
		if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
416
			dev_err(dev, "Channel not available\n");
417
			return -ENODEV;
418
		}
419

420
		el = &ring->ring_cache[mhi_chan->rd_offset];
421

422
		/* Check if there is data pending to be read from previous read operation */
423
		if (mhi_chan->tre_bytes_left) {
424
			dev_dbg(dev, "TRE bytes remaining: %u\n", mhi_chan->tre_bytes_left);
425
			tr_len = min(len, mhi_chan->tre_bytes_left);
426
		} else {
427
			mhi_chan->tre_loc = MHI_TRE_DATA_GET_PTR(el);
428
			mhi_chan->tre_size = MHI_TRE_DATA_GET_LEN(el);
429
			mhi_chan->tre_bytes_left = mhi_chan->tre_size;
430

431
			tr_len = min(len, mhi_chan->tre_size);
432
		}
433

434
		read_offset = mhi_chan->tre_size - mhi_chan->tre_bytes_left;
435

436
		buf_addr = kmem_cache_zalloc(mhi_cntrl->tre_buf_cache, GFP_KERNEL);
437
		if (!buf_addr)
438
			return -ENOMEM;
439

440
		buf_info.host_addr = mhi_chan->tre_loc + read_offset;
441
		buf_info.dev_addr = buf_addr;
442
		buf_info.size = tr_len;
443
		buf_info.cb = mhi_ep_read_completion;
444
		buf_info.cb_buf = buf_addr;
445
		buf_info.mhi_dev = mhi_chan->mhi_dev;
446

447
		if (mhi_chan->tre_bytes_left - tr_len)
448
			buf_info.code = MHI_EV_CC_OVERFLOW;
449

450
		dev_dbg(dev, "Reading %zd bytes from channel (%u)\n", tr_len, ring->ch_id);
451
		ret = mhi_cntrl->read_async(mhi_cntrl, &buf_info);
452
		if (ret < 0) {
453
			dev_err(&mhi_chan->mhi_dev->dev, "Error reading from channel\n");
454
			goto err_free_buf_addr;
455
		}
456

457
		mhi_chan->tre_bytes_left -= tr_len;
458

459
		if (!mhi_chan->tre_bytes_left)
460
			mhi_chan->rd_offset = (mhi_chan->rd_offset + 1) % ring->ring_size;
461
	/* Read until the some buffer is left or the ring becomes not empty */
462
	} while (!mhi_ep_queue_is_empty(mhi_chan->mhi_dev, DMA_TO_DEVICE));
463

464
	return 0;
465

466
err_free_buf_addr:
467
	kmem_cache_free(mhi_cntrl->tre_buf_cache, buf_addr);
468

469
	return ret;
470
}
471

472
static int mhi_ep_process_ch_ring(struct mhi_ep_ring *ring)
473
{
474
	struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
475
	struct mhi_result result = {};
476
	struct mhi_ep_chan *mhi_chan;
477
	int ret;
478

479
	mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
480

481
	/*
482
	 * Bail out if transfer callback is not registered for the channel.
483
	 * This is most likely due to the client driver not loaded at this point.
484
	 */
485
	if (!mhi_chan->xfer_cb) {
486
		dev_err(&mhi_chan->mhi_dev->dev, "Client driver not available\n");
487
		return -ENODEV;
488
	}
489

490
	if (ring->ch_id % 2) {
491
		/* DL channel */
492
		result.dir = mhi_chan->dir;
493
		mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
494
	} else {
495
		/* UL channel */
496
		ret = mhi_ep_read_channel(mhi_cntrl, ring);
497
		if (ret < 0) {
498
			dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n");
499
			return ret;
500
		}
501
	}
502

503
	return 0;
504
}
505

506
static void mhi_ep_skb_completion(struct mhi_ep_buf_info *buf_info)
507
{
508
	struct mhi_ep_device *mhi_dev = buf_info->mhi_dev;
509
	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
510
	struct mhi_ep_chan *mhi_chan = mhi_dev->dl_chan;
511
	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
512
	struct mhi_ring_element *el = &ring->ring_cache[ring->rd_offset];
513
	struct device *dev = &mhi_dev->dev;
514
	struct mhi_result result = {};
515
	int ret;
516

517
	if (mhi_chan->xfer_cb) {
518
		result.buf_addr = buf_info->cb_buf;
519
		result.dir = mhi_chan->dir;
520
		result.bytes_xferd = buf_info->size;
521

522
		mhi_chan->xfer_cb(mhi_dev, &result);
523
	}
524

525
	ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el, buf_info->size,
526
					   buf_info->code);
527
	if (ret) {
528
		dev_err(dev, "Error sending transfer completion event\n");
529
		return;
530
	}
531

532
	mhi_ep_ring_inc_index(ring);
533
}
534

535
/* TODO: Handle partially formed TDs */
536
int mhi_ep_queue_skb(struct mhi_ep_device *mhi_dev, struct sk_buff *skb)
537
{
538
	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
539
	struct mhi_ep_chan *mhi_chan = mhi_dev->dl_chan;
540
	struct device *dev = &mhi_chan->mhi_dev->dev;
541
	struct mhi_ep_buf_info buf_info = {};
542
	struct mhi_ring_element *el;
543
	u32 buf_left, read_offset;
544
	struct mhi_ep_ring *ring;
545
	size_t tr_len;
546
	u32 tre_len;
547
	int ret;
548

549
	buf_left = skb->len;
550
	ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
551

552
	mutex_lock(&mhi_chan->lock);
553

554
	do {
555
		/* Don't process the transfer ring if the channel is not in RUNNING state */
556
		if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
557
			dev_err(dev, "Channel not available\n");
558
			ret = -ENODEV;
559
			goto err_exit;
560
		}
561

562
		if (mhi_ep_queue_is_empty(mhi_dev, DMA_FROM_DEVICE)) {
563
			dev_err(dev, "TRE not available!\n");
564
			ret = -ENOSPC;
565
			goto err_exit;
566
		}
567

568
		el = &ring->ring_cache[mhi_chan->rd_offset];
569
		tre_len = MHI_TRE_DATA_GET_LEN(el);
570

571
		tr_len = min(buf_left, tre_len);
572
		read_offset = skb->len - buf_left;
573

574
		buf_info.dev_addr = skb->data + read_offset;
575
		buf_info.host_addr = MHI_TRE_DATA_GET_PTR(el);
576
		buf_info.size = tr_len;
577
		buf_info.cb = mhi_ep_skb_completion;
578
		buf_info.cb_buf = skb;
579
		buf_info.mhi_dev = mhi_dev;
580

581
		/*
582
		 * For all TREs queued by the host for DL channel, only the EOT flag will be set.
583
		 * If the packet doesn't fit into a single TRE, send the OVERFLOW event to
584
		 * the host so that the host can adjust the packet boundary to next TREs. Else send
585
		 * the EOT event to the host indicating the packet boundary.
586
		 */
587
		if (buf_left - tr_len)
588
			buf_info.code = MHI_EV_CC_OVERFLOW;
589
		else
590
			buf_info.code = MHI_EV_CC_EOT;
591

592
		dev_dbg(dev, "Writing %zd bytes to channel (%u)\n", tr_len, ring->ch_id);
593
		ret = mhi_cntrl->write_async(mhi_cntrl, &buf_info);
594
		if (ret < 0) {
595
			dev_err(dev, "Error writing to the channel\n");
596
			goto err_exit;
597
		}
598

599
		buf_left -= tr_len;
600

601
		/*
602
		 * Update the read offset cached in mhi_chan. Actual read offset
603
		 * will be updated by the completion handler.
604
		 */
605
		mhi_chan->rd_offset = (mhi_chan->rd_offset + 1) % ring->ring_size;
606
	} while (buf_left);
607

608
	mutex_unlock(&mhi_chan->lock);
609

610
	return 0;
611

612
err_exit:
613
	mutex_unlock(&mhi_chan->lock);
614

615
	return ret;
616
}
617
EXPORT_SYMBOL_GPL(mhi_ep_queue_skb);
618

619
static int mhi_ep_cache_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
620
{
621
	size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
622
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
623
	int ret;
624

625
	/* Update the number of event rings (NER) programmed by the host */
626
	mhi_ep_mmio_update_ner(mhi_cntrl);
627

628
	dev_dbg(dev, "Number of Event rings: %u, HW Event rings: %u\n",
629
		 mhi_cntrl->event_rings, mhi_cntrl->hw_event_rings);
630

631
	ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
632
	ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
633
	cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
634

635
	/* Get the channel context base pointer from host */
636
	mhi_ep_mmio_get_chc_base(mhi_cntrl);
637

638
	/* Allocate and map memory for caching host channel context */
639
	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa,
640
				   &mhi_cntrl->ch_ctx_cache_phys,
641
				   (void __iomem **) &mhi_cntrl->ch_ctx_cache,
642
				   ch_ctx_host_size);
643
	if (ret) {
644
		dev_err(dev, "Failed to allocate and map ch_ctx_cache\n");
645
		return ret;
646
	}
647

648
	/* Get the event context base pointer from host */
649
	mhi_ep_mmio_get_erc_base(mhi_cntrl);
650

651
	/* Allocate and map memory for caching host event context */
652
	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa,
653
				   &mhi_cntrl->ev_ctx_cache_phys,
654
				   (void __iomem **) &mhi_cntrl->ev_ctx_cache,
655
				   ev_ctx_host_size);
656
	if (ret) {
657
		dev_err(dev, "Failed to allocate and map ev_ctx_cache\n");
658
		goto err_ch_ctx;
659
	}
660

661
	/* Get the command context base pointer from host */
662
	mhi_ep_mmio_get_crc_base(mhi_cntrl);
663

664
	/* Allocate and map memory for caching host command context */
665
	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa,
666
				   &mhi_cntrl->cmd_ctx_cache_phys,
667
				   (void __iomem **) &mhi_cntrl->cmd_ctx_cache,
668
				   cmd_ctx_host_size);
669
	if (ret) {
670
		dev_err(dev, "Failed to allocate and map cmd_ctx_cache\n");
671
		goto err_ev_ctx;
672
	}
673

674
	/* Initialize command ring */
675
	ret = mhi_ep_ring_start(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring,
676
				(union mhi_ep_ring_ctx *)mhi_cntrl->cmd_ctx_cache);
677
	if (ret) {
678
		dev_err(dev, "Failed to start the command ring\n");
679
		goto err_cmd_ctx;
680
	}
681

682
	return ret;
683

684
err_cmd_ctx:
685
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
686
			      (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
687

688
err_ev_ctx:
689
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
690
			      (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
691

692
err_ch_ctx:
693
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
694
			      (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
695

696
	return ret;
697
}
698

699
static void mhi_ep_free_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
700
{
701
	size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
702

703
	ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
704
	ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
705
	cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
706

707
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
708
			      (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
709

710
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
711
			      (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
712

713
	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
714
			      (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
715
}
716

717
static void mhi_ep_enable_int(struct mhi_ep_cntrl *mhi_cntrl)
718
{
719
	/*
720
	 * Doorbell interrupts are enabled when the corresponding channel gets started.
721
	 * Enabling all interrupts here triggers spurious irqs as some of the interrupts
722
	 * associated with hw channels always get triggered.
723
	 */
724
	mhi_ep_mmio_enable_ctrl_interrupt(mhi_cntrl);
725
	mhi_ep_mmio_enable_cmdb_interrupt(mhi_cntrl);
726
}
727

728
static int mhi_ep_enable(struct mhi_ep_cntrl *mhi_cntrl)
729
{
730
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
731
	enum mhi_state state;
732
	bool mhi_reset;
733
	u32 count = 0;
734
	int ret;
735

736
	/* Wait for Host to set the M0 state */
737
	do {
738
		msleep(M0_WAIT_DELAY_MS);
739
		mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
740
		if (mhi_reset) {
741
			/* Clear the MHI reset if host is in reset state */
742
			mhi_ep_mmio_clear_reset(mhi_cntrl);
743
			dev_info(dev, "Detected Host reset while waiting for M0\n");
744
		}
745
		count++;
746
	} while (state != MHI_STATE_M0 && count < M0_WAIT_COUNT);
747

748
	if (state != MHI_STATE_M0) {
749
		dev_err(dev, "Host failed to enter M0\n");
750
		return -ETIMEDOUT;
751
	}
752

753
	ret = mhi_ep_cache_host_cfg(mhi_cntrl);
754
	if (ret) {
755
		dev_err(dev, "Failed to cache host config\n");
756
		return ret;
757
	}
758

759
	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
760

761
	/* Enable all interrupts now */
762
	mhi_ep_enable_int(mhi_cntrl);
763

764
	return 0;
765
}
766

767
static void mhi_ep_cmd_ring_worker(struct work_struct *work)
768
{
769
	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, cmd_ring_work);
770
	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
771
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
772
	struct mhi_ring_element *el;
773
	int ret;
774

775
	/* Update the write offset for the ring */
776
	ret = mhi_ep_update_wr_offset(ring);
777
	if (ret) {
778
		dev_err(dev, "Error updating write offset for ring\n");
779
		return;
780
	}
781

782
	/* Sanity check to make sure there are elements in the ring */
783
	if (ring->rd_offset == ring->wr_offset)
784
		return;
785

786
	/*
787
	 * Process command ring element till write offset. In case of an error, just try to
788
	 * process next element.
789
	 */
790
	while (ring->rd_offset != ring->wr_offset) {
791
		el = &ring->ring_cache[ring->rd_offset];
792

793
		ret = mhi_ep_process_cmd_ring(ring, el);
794
		if (ret && ret != -ENODEV)
795
			dev_err(dev, "Error processing cmd ring element: %zu\n", ring->rd_offset);
796

797
		mhi_ep_ring_inc_index(ring);
798
	}
799
}
800

801
static void mhi_ep_ch_ring_worker(struct work_struct *work)
802
{
803
	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, ch_ring_work);
804
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
805
	struct mhi_ep_ring_item *itr, *tmp;
806
	struct mhi_ep_ring *ring;
807
	struct mhi_ep_chan *chan;
808
	unsigned long flags;
809
	LIST_HEAD(head);
810
	int ret;
811

812
	spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
813
	list_splice_tail_init(&mhi_cntrl->ch_db_list, &head);
814
	spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
815

816
	/* Process each queued channel ring. In case of an error, just process next element. */
817
	list_for_each_entry_safe(itr, tmp, &head, node) {
818
		list_del(&itr->node);
819
		ring = itr->ring;
820

821
		chan = &mhi_cntrl->mhi_chan[ring->ch_id];
822
		mutex_lock(&chan->lock);
823

824
		/*
825
		 * The ring could've stopped while we waited to grab the (chan->lock), so do
826
		 * a sanity check before going further.
827
		 */
828
		if (!ring->started) {
829
			mutex_unlock(&chan->lock);
830
			kfree(itr);
831
			continue;
832
		}
833

834
		/* Update the write offset for the ring */
835
		ret = mhi_ep_update_wr_offset(ring);
836
		if (ret) {
837
			dev_err(dev, "Error updating write offset for ring\n");
838
			mutex_unlock(&chan->lock);
839
			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
840
			continue;
841
		}
842

843
		/* Sanity check to make sure there are elements in the ring */
844
		if (chan->rd_offset == ring->wr_offset) {
845
			mutex_unlock(&chan->lock);
846
			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
847
			continue;
848
		}
849

850
		dev_dbg(dev, "Processing the ring for channel (%u)\n", ring->ch_id);
851
		ret = mhi_ep_process_ch_ring(ring);
852
		if (ret) {
853
			dev_err(dev, "Error processing ring for channel (%u): %d\n",
854
				ring->ch_id, ret);
855
			mutex_unlock(&chan->lock);
856
			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
857
			continue;
858
		}
859

860
		mutex_unlock(&chan->lock);
861
		kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
862
	}
863
}
864

865
static void mhi_ep_state_worker(struct work_struct *work)
866
{
867
	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, state_work);
868
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
869
	struct mhi_ep_state_transition *itr, *tmp;
870
	unsigned long flags;
871
	LIST_HEAD(head);
872
	int ret;
873

874
	spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
875
	list_splice_tail_init(&mhi_cntrl->st_transition_list, &head);
876
	spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
877

878
	list_for_each_entry_safe(itr, tmp, &head, node) {
879
		list_del(&itr->node);
880
		dev_dbg(dev, "Handling MHI state transition to %s\n",
881
			 mhi_state_str(itr->state));
882

883
		switch (itr->state) {
884
		case MHI_STATE_M0:
885
			ret = mhi_ep_set_m0_state(mhi_cntrl);
886
			if (ret)
887
				dev_err(dev, "Failed to transition to M0 state\n");
888
			break;
889
		case MHI_STATE_M3:
890
			ret = mhi_ep_set_m3_state(mhi_cntrl);
891
			if (ret)
892
				dev_err(dev, "Failed to transition to M3 state\n");
893
			break;
894
		default:
895
			dev_err(dev, "Invalid MHI state transition: %d\n", itr->state);
896
			break;
897
		}
898
		kfree(itr);
899
	}
900
}
901

902
static void mhi_ep_queue_channel_db(struct mhi_ep_cntrl *mhi_cntrl, unsigned long ch_int,
903
				    u32 ch_idx)
904
{
905
	struct mhi_ep_ring_item *item;
906
	struct mhi_ep_ring *ring;
907
	bool work = !!ch_int;
908
	LIST_HEAD(head);
909
	u32 i;
910

911
	/* First add the ring items to a local list */
912
	for_each_set_bit(i, &ch_int, 32) {
913
		/* Channel index varies for each register: 0, 32, 64, 96 */
914
		u32 ch_id = ch_idx + i;
915

916
		ring = &mhi_cntrl->mhi_chan[ch_id].ring;
917
		item = kmem_cache_zalloc(mhi_cntrl->ring_item_cache, GFP_ATOMIC);
918
		if (!item)
919
			return;
920

921
		item->ring = ring;
922
		list_add_tail(&item->node, &head);
923
	}
924

925
	/* Now, splice the local list into ch_db_list and queue the work item */
926
	if (work) {
927
		spin_lock(&mhi_cntrl->list_lock);
928
		list_splice_tail_init(&head, &mhi_cntrl->ch_db_list);
929
		spin_unlock(&mhi_cntrl->list_lock);
930

931
		queue_work(mhi_cntrl->wq, &mhi_cntrl->ch_ring_work);
932
	}
933
}
934

935
/*
936
 * Channel interrupt statuses are contained in 4 registers each of 32bit length.
937
 * For checking all interrupts, we need to loop through each registers and then
938
 * check for bits set.
939
 */
940
static void mhi_ep_check_channel_interrupt(struct mhi_ep_cntrl *mhi_cntrl)
941
{
942
	u32 ch_int, ch_idx, i;
943

944
	/* Bail out if there is no channel doorbell interrupt */
945
	if (!mhi_ep_mmio_read_chdb_status_interrupts(mhi_cntrl))
946
		return;
947

948
	for (i = 0; i < MHI_MASK_ROWS_CH_DB; i++) {
949
		ch_idx = i * MHI_MASK_CH_LEN;
950

951
		/* Only process channel interrupt if the mask is enabled */
952
		ch_int = mhi_cntrl->chdb[i].status & mhi_cntrl->chdb[i].mask;
953
		if (ch_int) {
954
			mhi_ep_queue_channel_db(mhi_cntrl, ch_int, ch_idx);
955
			mhi_ep_mmio_write(mhi_cntrl, MHI_CHDB_INT_CLEAR_n(i),
956
							mhi_cntrl->chdb[i].status);
957
		}
958
	}
959
}
960

961
static void mhi_ep_process_ctrl_interrupt(struct mhi_ep_cntrl *mhi_cntrl,
962
					 enum mhi_state state)
963
{
964
	struct mhi_ep_state_transition *item;
965

966
	item = kzalloc(sizeof(*item), GFP_ATOMIC);
967
	if (!item)
968
		return;
969

970
	item->state = state;
971
	spin_lock(&mhi_cntrl->list_lock);
972
	list_add_tail(&item->node, &mhi_cntrl->st_transition_list);
973
	spin_unlock(&mhi_cntrl->list_lock);
974

975
	queue_work(mhi_cntrl->wq, &mhi_cntrl->state_work);
976
}
977

978
/*
979
 * Interrupt handler that services interrupts raised by the host writing to
980
 * MHICTRL and Command ring doorbell (CRDB) registers for state change and
981
 * channel interrupts.
982
 */
983
static irqreturn_t mhi_ep_irq(int irq, void *data)
984
{
985
	struct mhi_ep_cntrl *mhi_cntrl = data;
986
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
987
	enum mhi_state state;
988
	u32 int_value;
989
	bool mhi_reset;
990

991
	/* Acknowledge the ctrl interrupt */
992
	int_value = mhi_ep_mmio_read(mhi_cntrl, MHI_CTRL_INT_STATUS);
993
	mhi_ep_mmio_write(mhi_cntrl, MHI_CTRL_INT_CLEAR, int_value);
994

995
	/* Check for ctrl interrupt */
996
	if (FIELD_GET(MHI_CTRL_INT_STATUS_MSK, int_value)) {
997
		dev_dbg(dev, "Processing ctrl interrupt\n");
998
		mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
999
		if (mhi_reset) {
1000
			dev_info(dev, "Host triggered MHI reset!\n");
1001
			disable_irq_nosync(mhi_cntrl->irq);
1002
			schedule_work(&mhi_cntrl->reset_work);
1003
			return IRQ_HANDLED;
1004
		}
1005

1006
		mhi_ep_process_ctrl_interrupt(mhi_cntrl, state);
1007
	}
1008

1009
	/* Check for command doorbell interrupt */
1010
	if (FIELD_GET(MHI_CTRL_INT_STATUS_CRDB_MSK, int_value)) {
1011
		dev_dbg(dev, "Processing command doorbell interrupt\n");
1012
		queue_work(mhi_cntrl->wq, &mhi_cntrl->cmd_ring_work);
1013
	}
1014

1015
	/* Check for channel interrupts */
1016
	mhi_ep_check_channel_interrupt(mhi_cntrl);
1017

1018
	return IRQ_HANDLED;
1019
}
1020

1021
static void mhi_ep_abort_transfer(struct mhi_ep_cntrl *mhi_cntrl)
1022
{
1023
	struct mhi_ep_ring *ch_ring, *ev_ring;
1024
	struct mhi_result result = {};
1025
	struct mhi_ep_chan *mhi_chan;
1026
	int i;
1027

1028
	/* Stop all the channels */
1029
	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1030
		mhi_chan = &mhi_cntrl->mhi_chan[i];
1031
		if (!mhi_chan->ring.started)
1032
			continue;
1033

1034
		mutex_lock(&mhi_chan->lock);
1035
		/* Send channel disconnect status to client drivers */
1036
		if (mhi_chan->xfer_cb) {
1037
			result.transaction_status = -ENOTCONN;
1038
			result.bytes_xferd = 0;
1039
			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1040
		}
1041

1042
		mhi_chan->state = MHI_CH_STATE_DISABLED;
1043
		mutex_unlock(&mhi_chan->lock);
1044
	}
1045

1046
	flush_workqueue(mhi_cntrl->wq);
1047

1048
	/* Destroy devices associated with all channels */
1049
	device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, mhi_ep_destroy_device);
1050

1051
	/* Stop and reset the transfer rings */
1052
	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1053
		mhi_chan = &mhi_cntrl->mhi_chan[i];
1054
		if (!mhi_chan->ring.started)
1055
			continue;
1056

1057
		ch_ring = &mhi_cntrl->mhi_chan[i].ring;
1058
		mutex_lock(&mhi_chan->lock);
1059
		mhi_ep_ring_reset(mhi_cntrl, ch_ring);
1060
		mutex_unlock(&mhi_chan->lock);
1061
	}
1062

1063
	/* Stop and reset the event rings */
1064
	for (i = 0; i < mhi_cntrl->event_rings; i++) {
1065
		ev_ring = &mhi_cntrl->mhi_event[i].ring;
1066
		if (!ev_ring->started)
1067
			continue;
1068

1069
		mutex_lock(&mhi_cntrl->event_lock);
1070
		mhi_ep_ring_reset(mhi_cntrl, ev_ring);
1071
		mutex_unlock(&mhi_cntrl->event_lock);
1072
	}
1073

1074
	/* Stop and reset the command ring */
1075
	mhi_ep_ring_reset(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring);
1076

1077
	mhi_ep_free_host_cfg(mhi_cntrl);
1078
	mhi_ep_mmio_mask_interrupts(mhi_cntrl);
1079

1080
	mhi_cntrl->enabled = false;
1081
}
1082

1083
static void mhi_ep_reset_worker(struct work_struct *work)
1084
{
1085
	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, reset_work);
1086
	enum mhi_state cur_state;
1087

1088
	mhi_ep_power_down(mhi_cntrl);
1089

1090
	mutex_lock(&mhi_cntrl->state_lock);
1091

1092
	/* Reset MMIO to signal host that the MHI_RESET is completed in endpoint */
1093
	mhi_ep_mmio_reset(mhi_cntrl);
1094
	cur_state = mhi_cntrl->mhi_state;
1095

1096
	/*
1097
	 * Only proceed further if the reset is due to SYS_ERR. The host will
1098
	 * issue reset during shutdown also and we don't need to do re-init in
1099
	 * that case.
1100
	 */
1101
	if (cur_state == MHI_STATE_SYS_ERR)
1102
		mhi_ep_power_up(mhi_cntrl);
1103

1104
	mutex_unlock(&mhi_cntrl->state_lock);
1105
}
1106

1107
/*
1108
 * We don't need to do anything special other than setting the MHI SYS_ERR
1109
 * state. The host will reset all contexts and issue MHI RESET so that we
1110
 * could also recover from error state.
1111
 */
1112
void mhi_ep_handle_syserr(struct mhi_ep_cntrl *mhi_cntrl)
1113
{
1114
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1115
	int ret;
1116

1117
	ret = mhi_ep_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
1118
	if (ret)
1119
		return;
1120

1121
	/* Signal host that the device went to SYS_ERR state */
1122
	ret = mhi_ep_send_state_change_event(mhi_cntrl, MHI_STATE_SYS_ERR);
1123
	if (ret)
1124
		dev_err(dev, "Failed sending SYS_ERR state change event: %d\n", ret);
1125
}
1126

1127
int mhi_ep_power_up(struct mhi_ep_cntrl *mhi_cntrl)
1128
{
1129
	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1130
	int ret, i;
1131

1132
	/*
1133
	 * Mask all interrupts until the state machine is ready. Interrupts will
1134
	 * be enabled later with mhi_ep_enable().
1135
	 */
1136
	mhi_ep_mmio_mask_interrupts(mhi_cntrl);
1137
	mhi_ep_mmio_init(mhi_cntrl);
1138

1139
	mhi_cntrl->mhi_event = kcalloc(mhi_cntrl->event_rings,
1140
				       sizeof(*mhi_cntrl->mhi_event),
1141
				       GFP_KERNEL);
1142
	if (!mhi_cntrl->mhi_event)
1143
		return -ENOMEM;
1144

1145
	/* Initialize command, channel and event rings */
1146
	mhi_ep_ring_init(&mhi_cntrl->mhi_cmd->ring, RING_TYPE_CMD, 0);
1147
	for (i = 0; i < mhi_cntrl->max_chan; i++)
1148
		mhi_ep_ring_init(&mhi_cntrl->mhi_chan[i].ring, RING_TYPE_CH, i);
1149
	for (i = 0; i < mhi_cntrl->event_rings; i++)
1150
		mhi_ep_ring_init(&mhi_cntrl->mhi_event[i].ring, RING_TYPE_ER, i);
1151

1152
	mhi_cntrl->mhi_state = MHI_STATE_RESET;
1153

1154
	/* Set AMSS EE before signaling ready state */
1155
	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1156

1157
	/* All set, notify the host that we are ready */
1158
	ret = mhi_ep_set_ready_state(mhi_cntrl);
1159
	if (ret)
1160
		goto err_free_event;
1161

1162
	dev_dbg(dev, "READY state notification sent to the host\n");
1163

1164
	ret = mhi_ep_enable(mhi_cntrl);
1165
	if (ret) {
1166
		dev_err(dev, "Failed to enable MHI endpoint\n");
1167
		goto err_free_event;
1168
	}
1169

1170
	enable_irq(mhi_cntrl->irq);
1171
	mhi_cntrl->enabled = true;
1172

1173
	return 0;
1174

1175
err_free_event:
1176
	kfree(mhi_cntrl->mhi_event);
1177

1178
	return ret;
1179
}
1180
EXPORT_SYMBOL_GPL(mhi_ep_power_up);
1181

1182
void mhi_ep_power_down(struct mhi_ep_cntrl *mhi_cntrl)
1183
{
1184
	if (mhi_cntrl->enabled) {
1185
		mhi_ep_abort_transfer(mhi_cntrl);
1186
		kfree(mhi_cntrl->mhi_event);
1187
		disable_irq(mhi_cntrl->irq);
1188
	}
1189
}
1190
EXPORT_SYMBOL_GPL(mhi_ep_power_down);
1191

1192
void mhi_ep_suspend_channels(struct mhi_ep_cntrl *mhi_cntrl)
1193
{
1194
	struct mhi_ep_chan *mhi_chan;
1195
	u32 tmp;
1196
	int i;
1197

1198
	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1199
		mhi_chan = &mhi_cntrl->mhi_chan[i];
1200

1201
		if (!mhi_chan->mhi_dev)
1202
			continue;
1203

1204
		mutex_lock(&mhi_chan->lock);
1205
		/* Skip if the channel is not currently running */
1206
		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1207
		if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_RUNNING) {
1208
			mutex_unlock(&mhi_chan->lock);
1209
			continue;
1210
		}
1211

1212
		dev_dbg(&mhi_chan->mhi_dev->dev, "Suspending channel\n");
1213
		/* Set channel state to SUSPENDED */
1214
		mhi_chan->state = MHI_CH_STATE_SUSPENDED;
1215
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
1216
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_SUSPENDED);
1217
		mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1218
		mutex_unlock(&mhi_chan->lock);
1219
	}
1220
}
1221

1222
void mhi_ep_resume_channels(struct mhi_ep_cntrl *mhi_cntrl)
1223
{
1224
	struct mhi_ep_chan *mhi_chan;
1225
	u32 tmp;
1226
	int i;
1227

1228
	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1229
		mhi_chan = &mhi_cntrl->mhi_chan[i];
1230

1231
		if (!mhi_chan->mhi_dev)
1232
			continue;
1233

1234
		mutex_lock(&mhi_chan->lock);
1235
		/* Skip if the channel is not currently suspended */
1236
		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1237
		if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_SUSPENDED) {
1238
			mutex_unlock(&mhi_chan->lock);
1239
			continue;
1240
		}
1241

1242
		dev_dbg(&mhi_chan->mhi_dev->dev, "Resuming channel\n");
1243
		/* Set channel state to RUNNING */
1244
		mhi_chan->state = MHI_CH_STATE_RUNNING;
1245
		tmp &= ~CHAN_CTX_CHSTATE_MASK;
1246
		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
1247
		mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1248
		mutex_unlock(&mhi_chan->lock);
1249
	}
1250
}
1251

1252
static void mhi_ep_release_device(struct device *dev)
1253
{
1254
	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1255

1256
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1257
		mhi_dev->mhi_cntrl->mhi_dev = NULL;
1258

1259
	/*
1260
	 * We need to set the mhi_chan->mhi_dev to NULL here since the MHI
1261
	 * devices for the channels will only get created in mhi_ep_create_device()
1262
	 * if the mhi_dev associated with it is NULL.
1263
	 */
1264
	if (mhi_dev->ul_chan)
1265
		mhi_dev->ul_chan->mhi_dev = NULL;
1266

1267
	if (mhi_dev->dl_chan)
1268
		mhi_dev->dl_chan->mhi_dev = NULL;
1269

1270
	kfree(mhi_dev);
1271
}
1272

1273
static struct mhi_ep_device *mhi_ep_alloc_device(struct mhi_ep_cntrl *mhi_cntrl,
1274
						 enum mhi_device_type dev_type)
1275
{
1276
	struct mhi_ep_device *mhi_dev;
1277
	struct device *dev;
1278

1279
	mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL);
1280
	if (!mhi_dev)
1281
		return ERR_PTR(-ENOMEM);
1282

1283
	dev = &mhi_dev->dev;
1284
	device_initialize(dev);
1285
	dev->bus = &mhi_ep_bus_type;
1286
	dev->release = mhi_ep_release_device;
1287

1288
	/* Controller device is always allocated first */
1289
	if (dev_type == MHI_DEVICE_CONTROLLER)
1290
		/* for MHI controller device, parent is the bus device (e.g. PCI EPF) */
1291
		dev->parent = mhi_cntrl->cntrl_dev;
1292
	else
1293
		/* for MHI client devices, parent is the MHI controller device */
1294
		dev->parent = &mhi_cntrl->mhi_dev->dev;
1295

1296
	mhi_dev->mhi_cntrl = mhi_cntrl;
1297
	mhi_dev->dev_type = dev_type;
1298

1299
	return mhi_dev;
1300
}
1301

1302
/*
1303
 * MHI channels are always defined in pairs with UL as the even numbered
1304
 * channel and DL as odd numbered one. This function gets UL channel (primary)
1305
 * as the ch_id and always looks after the next entry in channel list for
1306
 * the corresponding DL channel (secondary).
1307
 */
1308
static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id)
1309
{
1310
	struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
1311
	struct device *dev = mhi_cntrl->cntrl_dev;
1312
	struct mhi_ep_device *mhi_dev;
1313
	int ret;
1314

1315
	/* Check if the channel name is same for both UL and DL */
1316
	if (strcmp(mhi_chan->name, mhi_chan[1].name)) {
1317
		dev_err(dev, "UL and DL channel names are not same: (%s) != (%s)\n",
1318
			mhi_chan->name, mhi_chan[1].name);
1319
		return -EINVAL;
1320
	}
1321

1322
	mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_XFER);
1323
	if (IS_ERR(mhi_dev))
1324
		return PTR_ERR(mhi_dev);
1325

1326
	/* Configure primary channel */
1327
	mhi_dev->ul_chan = mhi_chan;
1328
	get_device(&mhi_dev->dev);
1329
	mhi_chan->mhi_dev = mhi_dev;
1330

1331
	/* Configure secondary channel as well */
1332
	mhi_chan++;
1333
	mhi_dev->dl_chan = mhi_chan;
1334
	get_device(&mhi_dev->dev);
1335
	mhi_chan->mhi_dev = mhi_dev;
1336

1337
	/* Channel name is same for both UL and DL */
1338
	mhi_dev->name = mhi_chan->name;
1339
	ret = dev_set_name(&mhi_dev->dev, "%s_%s",
1340
		     dev_name(&mhi_cntrl->mhi_dev->dev),
1341
		     mhi_dev->name);
1342
	if (ret) {
1343
		put_device(&mhi_dev->dev);
1344
		return ret;
1345
	}
1346

1347
	ret = device_add(&mhi_dev->dev);
1348
	if (ret)
1349
		put_device(&mhi_dev->dev);
1350

1351
	return ret;
1352
}
1353

1354
static int mhi_ep_destroy_device(struct device *dev, void *data)
1355
{
1356
	struct mhi_ep_device *mhi_dev;
1357
	struct mhi_ep_cntrl *mhi_cntrl;
1358
	struct mhi_ep_chan *ul_chan, *dl_chan;
1359

1360
	if (dev->bus != &mhi_ep_bus_type)
1361
		return 0;
1362

1363
	mhi_dev = to_mhi_ep_device(dev);
1364
	mhi_cntrl = mhi_dev->mhi_cntrl;
1365

1366
	/* Only destroy devices created for channels */
1367
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1368
		return 0;
1369

1370
	ul_chan = mhi_dev->ul_chan;
1371
	dl_chan = mhi_dev->dl_chan;
1372

1373
	if (ul_chan)
1374
		put_device(&ul_chan->mhi_dev->dev);
1375

1376
	if (dl_chan)
1377
		put_device(&dl_chan->mhi_dev->dev);
1378

1379
	dev_dbg(&mhi_cntrl->mhi_dev->dev, "Destroying device for chan:%s\n",
1380
		 mhi_dev->name);
1381

1382
	/* Notify the client and remove the device from MHI bus */
1383
	device_del(dev);
1384
	put_device(dev);
1385

1386
	return 0;
1387
}
1388

1389
static int mhi_ep_chan_init(struct mhi_ep_cntrl *mhi_cntrl,
1390
			    const struct mhi_ep_cntrl_config *config)
1391
{
1392
	const struct mhi_ep_channel_config *ch_cfg;
1393
	struct device *dev = mhi_cntrl->cntrl_dev;
1394
	u32 chan, i;
1395
	int ret = -EINVAL;
1396

1397
	mhi_cntrl->max_chan = config->max_channels;
1398

1399
	/*
1400
	 * Allocate max_channels supported by the MHI endpoint and populate
1401
	 * only the defined channels
1402
	 */
1403
	mhi_cntrl->mhi_chan = kcalloc(mhi_cntrl->max_chan, sizeof(*mhi_cntrl->mhi_chan),
1404
				      GFP_KERNEL);
1405
	if (!mhi_cntrl->mhi_chan)
1406
		return -ENOMEM;
1407

1408
	for (i = 0; i < config->num_channels; i++) {
1409
		struct mhi_ep_chan *mhi_chan;
1410

1411
		ch_cfg = &config->ch_cfg[i];
1412

1413
		chan = ch_cfg->num;
1414
		if (chan >= mhi_cntrl->max_chan) {
1415
			dev_err(dev, "Channel (%u) exceeds maximum available channels (%u)\n",
1416
				chan, mhi_cntrl->max_chan);
1417
			goto error_chan_cfg;
1418
		}
1419

1420
		/* Bi-directional and direction less channels are not supported */
1421
		if (ch_cfg->dir == DMA_BIDIRECTIONAL || ch_cfg->dir == DMA_NONE) {
1422
			dev_err(dev, "Invalid direction (%u) for channel (%u)\n",
1423
				ch_cfg->dir, chan);
1424
			goto error_chan_cfg;
1425
		}
1426

1427
		mhi_chan = &mhi_cntrl->mhi_chan[chan];
1428
		mhi_chan->name = ch_cfg->name;
1429
		mhi_chan->chan = chan;
1430
		mhi_chan->dir = ch_cfg->dir;
1431
		mutex_init(&mhi_chan->lock);
1432
	}
1433

1434
	return 0;
1435

1436
error_chan_cfg:
1437
	kfree(mhi_cntrl->mhi_chan);
1438

1439
	return ret;
1440
}
1441

1442
/*
1443
 * Allocate channel and command rings here. Event rings will be allocated
1444
 * in mhi_ep_power_up() as the config comes from the host.
1445
 */
1446
int mhi_ep_register_controller(struct mhi_ep_cntrl *mhi_cntrl,
1447
				const struct mhi_ep_cntrl_config *config)
1448
{
1449
	struct mhi_ep_device *mhi_dev;
1450
	int ret;
1451

1452
	if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->mmio || !mhi_cntrl->irq)
1453
		return -EINVAL;
1454

1455
	if (!mhi_cntrl->read_sync || !mhi_cntrl->write_sync ||
1456
	    !mhi_cntrl->read_async || !mhi_cntrl->write_async)
1457
		return -EINVAL;
1458

1459
	ret = mhi_ep_chan_init(mhi_cntrl, config);
1460
	if (ret)
1461
		return ret;
1462

1463
	mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS, sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL);
1464
	if (!mhi_cntrl->mhi_cmd) {
1465
		ret = -ENOMEM;
1466
		goto err_free_ch;
1467
	}
1468

1469
	mhi_cntrl->ev_ring_el_cache = kmem_cache_create("mhi_ep_event_ring_el",
1470
							sizeof(struct mhi_ring_element), 0,
1471
							0, NULL);
1472
	if (!mhi_cntrl->ev_ring_el_cache) {
1473
		ret = -ENOMEM;
1474
		goto err_free_cmd;
1475
	}
1476

1477
	mhi_cntrl->tre_buf_cache = kmem_cache_create("mhi_ep_tre_buf", MHI_EP_DEFAULT_MTU, 0,
1478
						      0, NULL);
1479
	if (!mhi_cntrl->tre_buf_cache) {
1480
		ret = -ENOMEM;
1481
		goto err_destroy_ev_ring_el_cache;
1482
	}
1483

1484
	mhi_cntrl->ring_item_cache = kmem_cache_create("mhi_ep_ring_item",
1485
							sizeof(struct mhi_ep_ring_item), 0,
1486
							0, NULL);
1487
	if (!mhi_cntrl->ring_item_cache) {
1488
		ret = -ENOMEM;
1489
		goto err_destroy_tre_buf_cache;
1490
	}
1491

1492
	INIT_WORK(&mhi_cntrl->state_work, mhi_ep_state_worker);
1493
	INIT_WORK(&mhi_cntrl->reset_work, mhi_ep_reset_worker);
1494
	INIT_WORK(&mhi_cntrl->cmd_ring_work, mhi_ep_cmd_ring_worker);
1495
	INIT_WORK(&mhi_cntrl->ch_ring_work, mhi_ep_ch_ring_worker);
1496

1497
	mhi_cntrl->wq = alloc_workqueue("mhi_ep_wq", 0, 0);
1498
	if (!mhi_cntrl->wq) {
1499
		ret = -ENOMEM;
1500
		goto err_destroy_ring_item_cache;
1501
	}
1502

1503
	INIT_LIST_HEAD(&mhi_cntrl->st_transition_list);
1504
	INIT_LIST_HEAD(&mhi_cntrl->ch_db_list);
1505
	spin_lock_init(&mhi_cntrl->list_lock);
1506
	mutex_init(&mhi_cntrl->state_lock);
1507
	mutex_init(&mhi_cntrl->event_lock);
1508

1509
	/* Set MHI version and AMSS EE before enumeration */
1510
	mhi_ep_mmio_write(mhi_cntrl, EP_MHIVER, config->mhi_version);
1511
	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1512

1513
	/* Set controller index */
1514
	ret = ida_alloc(&mhi_ep_cntrl_ida, GFP_KERNEL);
1515
	if (ret < 0)
1516
		goto err_destroy_wq;
1517

1518
	mhi_cntrl->index = ret;
1519

1520
	irq_set_status_flags(mhi_cntrl->irq, IRQ_NOAUTOEN);
1521
	ret = request_irq(mhi_cntrl->irq, mhi_ep_irq, IRQF_TRIGGER_HIGH,
1522
			  "doorbell_irq", mhi_cntrl);
1523
	if (ret) {
1524
		dev_err(mhi_cntrl->cntrl_dev, "Failed to request Doorbell IRQ\n");
1525
		goto err_ida_free;
1526
	}
1527

1528
	/* Allocate the controller device */
1529
	mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_CONTROLLER);
1530
	if (IS_ERR(mhi_dev)) {
1531
		dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate controller device\n");
1532
		ret = PTR_ERR(mhi_dev);
1533
		goto err_free_irq;
1534
	}
1535

1536
	ret = dev_set_name(&mhi_dev->dev, "mhi_ep%u", mhi_cntrl->index);
1537
	if (ret)
1538
		goto err_put_dev;
1539

1540
	mhi_dev->name = dev_name(&mhi_dev->dev);
1541
	mhi_cntrl->mhi_dev = mhi_dev;
1542

1543
	ret = device_add(&mhi_dev->dev);
1544
	if (ret)
1545
		goto err_put_dev;
1546

1547
	dev_dbg(&mhi_dev->dev, "MHI EP Controller registered\n");
1548

1549
	return 0;
1550

1551
err_put_dev:
1552
	put_device(&mhi_dev->dev);
1553
err_free_irq:
1554
	free_irq(mhi_cntrl->irq, mhi_cntrl);
1555
err_ida_free:
1556
	ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1557
err_destroy_wq:
1558
	destroy_workqueue(mhi_cntrl->wq);
1559
err_destroy_ring_item_cache:
1560
	kmem_cache_destroy(mhi_cntrl->ring_item_cache);
1561
err_destroy_ev_ring_el_cache:
1562
	kmem_cache_destroy(mhi_cntrl->ev_ring_el_cache);
1563
err_destroy_tre_buf_cache:
1564
	kmem_cache_destroy(mhi_cntrl->tre_buf_cache);
1565
err_free_cmd:
1566
	kfree(mhi_cntrl->mhi_cmd);
1567
err_free_ch:
1568
	kfree(mhi_cntrl->mhi_chan);
1569

1570
	return ret;
1571
}
1572
EXPORT_SYMBOL_GPL(mhi_ep_register_controller);
1573

1574
/*
1575
 * It is expected that the controller drivers will power down the MHI EP stack
1576
 * using "mhi_ep_power_down()" before calling this function to unregister themselves.
1577
 */
1578
void mhi_ep_unregister_controller(struct mhi_ep_cntrl *mhi_cntrl)
1579
{
1580
	struct mhi_ep_device *mhi_dev = mhi_cntrl->mhi_dev;
1581

1582
	destroy_workqueue(mhi_cntrl->wq);
1583

1584
	free_irq(mhi_cntrl->irq, mhi_cntrl);
1585

1586
	kmem_cache_destroy(mhi_cntrl->tre_buf_cache);
1587
	kmem_cache_destroy(mhi_cntrl->ev_ring_el_cache);
1588
	kmem_cache_destroy(mhi_cntrl->ring_item_cache);
1589
	kfree(mhi_cntrl->mhi_cmd);
1590
	kfree(mhi_cntrl->mhi_chan);
1591

1592
	device_del(&mhi_dev->dev);
1593
	put_device(&mhi_dev->dev);
1594

1595
	ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1596
}
1597
EXPORT_SYMBOL_GPL(mhi_ep_unregister_controller);
1598

1599
static int mhi_ep_driver_probe(struct device *dev)
1600
{
1601
	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1602
	struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1603
	struct mhi_ep_chan *ul_chan = mhi_dev->ul_chan;
1604
	struct mhi_ep_chan *dl_chan = mhi_dev->dl_chan;
1605

1606
	ul_chan->xfer_cb = mhi_drv->ul_xfer_cb;
1607
	dl_chan->xfer_cb = mhi_drv->dl_xfer_cb;
1608

1609
	return mhi_drv->probe(mhi_dev, mhi_dev->id);
1610
}
1611

1612
static int mhi_ep_driver_remove(struct device *dev)
1613
{
1614
	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1615
	struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1616
	struct mhi_result result = {};
1617
	struct mhi_ep_chan *mhi_chan;
1618
	int dir;
1619

1620
	/* Skip if it is a controller device */
1621
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1622
		return 0;
1623

1624
	/* Disconnect the channels associated with the driver */
1625
	for (dir = 0; dir < 2; dir++) {
1626
		mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1627

1628
		if (!mhi_chan)
1629
			continue;
1630

1631
		mutex_lock(&mhi_chan->lock);
1632
		/* Send channel disconnect status to the client driver */
1633
		if (mhi_chan->xfer_cb) {
1634
			result.transaction_status = -ENOTCONN;
1635
			result.bytes_xferd = 0;
1636
			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1637
		}
1638

1639
		mhi_chan->state = MHI_CH_STATE_DISABLED;
1640
		mhi_chan->xfer_cb = NULL;
1641
		mutex_unlock(&mhi_chan->lock);
1642
	}
1643

1644
	/* Remove the client driver now */
1645
	mhi_drv->remove(mhi_dev);
1646

1647
	return 0;
1648
}
1649

1650
int __mhi_ep_driver_register(struct mhi_ep_driver *mhi_drv, struct module *owner)
1651
{
1652
	struct device_driver *driver = &mhi_drv->driver;
1653

1654
	if (!mhi_drv->probe || !mhi_drv->remove)
1655
		return -EINVAL;
1656

1657
	/* Client drivers should have callbacks defined for both channels */
1658
	if (!mhi_drv->ul_xfer_cb || !mhi_drv->dl_xfer_cb)
1659
		return -EINVAL;
1660

1661
	driver->bus = &mhi_ep_bus_type;
1662
	driver->owner = owner;
1663
	driver->probe = mhi_ep_driver_probe;
1664
	driver->remove = mhi_ep_driver_remove;
1665

1666
	return driver_register(driver);
1667
}
1668
EXPORT_SYMBOL_GPL(__mhi_ep_driver_register);
1669

1670
void mhi_ep_driver_unregister(struct mhi_ep_driver *mhi_drv)
1671
{
1672
	driver_unregister(&mhi_drv->driver);
1673
}
1674
EXPORT_SYMBOL_GPL(mhi_ep_driver_unregister);
1675

1676
static int mhi_ep_uevent(const struct device *dev, struct kobj_uevent_env *env)
1677
{
1678
	const struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1679

1680
	return add_uevent_var(env, "MODALIAS=" MHI_EP_DEVICE_MODALIAS_FMT,
1681
					mhi_dev->name);
1682
}
1683

1684
static int mhi_ep_match(struct device *dev, const struct device_driver *drv)
1685
{
1686
	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1687
	const struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(drv);
1688
	const struct mhi_device_id *id;
1689

1690
	/*
1691
	 * If the device is a controller type then there is no client driver
1692
	 * associated with it
1693
	 */
1694
	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1695
		return 0;
1696

1697
	for (id = mhi_drv->id_table; id->chan[0]; id++)
1698
		if (!strcmp(mhi_dev->name, id->chan)) {
1699
			mhi_dev->id = id;
1700
			return 1;
1701
		}
1702

1703
	return 0;
1704
};
1705

1706
struct bus_type mhi_ep_bus_type = {
1707
	.name = "mhi_ep",
1708
	.dev_name = "mhi_ep",
1709
	.match = mhi_ep_match,
1710
	.uevent = mhi_ep_uevent,
1711
};
1712

1713
static int __init mhi_ep_init(void)
1714
{
1715
	return bus_register(&mhi_ep_bus_type);
1716
}
1717

1718
static void __exit mhi_ep_exit(void)
1719
{
1720
	bus_unregister(&mhi_ep_bus_type);
1721
}
1722

1723
postcore_initcall(mhi_ep_init);
1724
module_exit(mhi_ep_exit);
1725

1726
MODULE_LICENSE("GPL v2");
1727
MODULE_DESCRIPTION("MHI Bus Endpoint stack");
1728
MODULE_AUTHOR("Manivannan Sadhasivam <[email protected]>");
1729

1730