1
/*
2
 * Copyright (c) 2018 naehrwert
3
 * Copyright (c) 2018-2025 CTCaer
4
 *
5
 * This program is free software; you can redistribute it and/or modify it
6
 * under the terms and conditions of the GNU General Public License,
7
 * version 2, as published by the Free Software Foundation.
8
 *
9
 * This program is distributed in the hope it will be useful, but WITHOUT
10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12
 * more details.
13
 *
14
 * You should have received a copy of the GNU General Public License
15
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16
 */
17

18
#include <string.h>
19

20
#include <storage/mmc.h>
21
#include <storage/sdmmc.h>
22
#include <gfx_utils.h>
23
#include <power/max7762x.h>
24
#include <soc/bpmp.h>
25
#include <soc/clock.h>
26
#include <soc/gpio.h>
27
#include <soc/hw_init.h>
28
#include <soc/pinmux.h>
29
#include <soc/pmc.h>
30
#include <soc/timer.h>
31
#include <soc/t210.h>
32

33
//#define DPRINTF(...) gfx_printf(__VA_ARGS__)
34
//#define ERROR_EXTRA_PRINTING
35
#define DPRINTF(...)
36

37
#ifdef BDK_SDMMC_EXTRA_PRINT
38
#define ERROR_EXTRA_PRINTING
39
#endif
40

41
/*! SCMMC controller base addresses. */
42
static const u16 _sdmmc_base_offsets[4] = { 0x0, 0x200, 0x400, 0x600 };
43

44
int sdmmc_get_io_power(sdmmc_t *sdmmc)
45
{
46
	u32 p = sdmmc->regs->pwrcon;
47
	if (!(p & SDHCI_POWER_ON))
48
		return SDMMC_POWER_OFF;
49
	if (p & SDHCI_POWER_180)
50
		return SDMMC_POWER_1_8;
51
	if (p & SDHCI_POWER_330)
52
		return SDMMC_POWER_3_3;
53
	return -1;
54
}
55

56
static int _sdmmc_set_io_power(sdmmc_t *sdmmc, u32 power)
57
{
58
	switch (power)
59
	{
60
	case SDMMC_POWER_OFF:
61
		sdmmc->regs->pwrcon &= ~SDHCI_POWER_ON;
62
		break;
63

64
	case SDMMC_POWER_1_8:
65
		sdmmc->regs->pwrcon = SDHCI_POWER_180;
66
		break;
67

68
	case SDMMC_POWER_3_3:
69
		sdmmc->regs->pwrcon = SDHCI_POWER_330;
70
		break;
71

72
	default:
73
		return 0;
74
	}
75

76
	if (power != SDMMC_POWER_OFF)
77
		sdmmc->regs->pwrcon |= SDHCI_POWER_ON;
78

79
	return 1;
80
}
81

82
u32 sdmmc_get_bus_width(sdmmc_t *sdmmc)
83
{
84
	u32 h = sdmmc->regs->hostctl;
85
	if (h & SDHCI_CTRL_8BITBUS) // eMMC only (or UHS-II).
86
		return SDMMC_BUS_WIDTH_8;
87
	if (h & SDHCI_CTRL_4BITBUS) // SD only.
88
		return SDMMC_BUS_WIDTH_4;
89
	return SDMMC_BUS_WIDTH_1;
90
}
91

92
void sdmmc_set_bus_width(sdmmc_t *sdmmc, u32 bus_width)
93
{
94
	u32 host_control = sdmmc->regs->hostctl & ~(SDHCI_CTRL_4BITBUS | SDHCI_CTRL_8BITBUS);
95

96
	if (bus_width == SDMMC_BUS_WIDTH_1)
97
		sdmmc->regs->hostctl = host_control;
98
	else if (bus_width == SDMMC_BUS_WIDTH_4)
99
		sdmmc->regs->hostctl = host_control | SDHCI_CTRL_4BITBUS; // SD only.
100
	else if (bus_width == SDMMC_BUS_WIDTH_8)
101
		sdmmc->regs->hostctl = host_control | SDHCI_CTRL_8BITBUS; // eMMC only (or UHS-II).
102
}
103

104
void sdmmc_save_tap_value(sdmmc_t *sdmmc)
105
{
106
	sdmmc->venclkctl_tap = (sdmmc->regs->venclkctl & 0xFF0000) >> 16;
107
	sdmmc->venclkctl_set = 1;
108
}
109

110
static int _sdmmc_config_tap_val(sdmmc_t *sdmmc, u32 type)
111
{
112
	static const u32 dqs_trim_val = 40; // 24 if HS533/HS667.
113
	static const u8  tap_values_t210[4] = { 4, 0, 3, 0 };
114

115
	u32 tap_val = 0;
116

117
	if (type == SDHCI_TIMING_MMC_HS400)
118
		sdmmc->regs->vencapover = (sdmmc->regs->vencapover & 0xFFFFC0FF) | (dqs_trim_val << 8);
119

120
	sdmmc->regs->ventunctl0 &= ~SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
121

122
	if (type == SDHCI_TIMING_MMC_HS400)
123
	{
124
		if (!sdmmc->venclkctl_set)
125
			return 0;
126

127
		tap_val = sdmmc->venclkctl_tap;
128
	}
129
	else
130
		tap_val = sdmmc->t210b01 ? 11 : tap_values_t210[sdmmc->id];
131

132
	sdmmc->regs->venclkctl = (sdmmc->regs->venclkctl & 0xFF00FFFF) | (tap_val << 16);
133

134
	return 1;
135
}
136

137
static void _sdmmc_commit_changes(sdmmc_t *sdmmc)
138
{
139
	(void)sdmmc->regs->clkcon;
140
}
141

142
static void _sdmmc_pad_config_fallback(sdmmc_t *sdmmc, u32 power)
143
{
144
	_sdmmc_commit_changes(sdmmc);
145
	switch (sdmmc->id)
146
	{
147
	case SDMMC_1: // 33 Ohm 2X Driver.
148
		if (power == SDMMC_POWER_OFF)
149
			break;
150
		u32 sdmmc1_pad_cfg = APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL) & 0xF8080FFF;
151
		if (sdmmc->t210b01)
152
			sdmmc1_pad_cfg |= (0x808 << 12); // Up:  8, Dn:  8. For 33 ohm.
153
		else if (power == SDMMC_POWER_1_8)
154
			sdmmc1_pad_cfg |= (0xB0F << 12); // Up: 11, Dn: 15. For 33 ohm.
155
		else if (power == SDMMC_POWER_3_3)
156
			sdmmc1_pad_cfg |= (0xC0C << 12); // Up: 12, Dn: 12. For 33 ohm.
157
		APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL) = sdmmc1_pad_cfg;
158
		(void)APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL); // Commit write.
159
		break;
160

161
	case SDMMC_2:
162
		if (sdmmc->t210b01)
163
			APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL) = (APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL) & 0xF8080FFF) | 0xA0A000;
164
		else
165
			APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL) = (APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL) & 0xFFFFC003) | 0x1040; // PU:16, PD:16.
166
		(void)APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL);
167
		break;
168

169
	case SDMMC_4: // 50 Ohm 2X Driver. PU:16, PD:16, B01: PU:10, PD:10.
170
		APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL) = (APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL) & 0xFFFFC003) |
171
													 (sdmmc->t210b01 ? 0xA28 : 0x1040);
172
		(void)APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL); // Commit write.
173
		break;
174
	}
175
}
176

177
static void _sdmmc_autocal_execute(sdmmc_t *sdmmc, u32 power)
178
{
179
	bool should_enable_sd_clock = false;
180
	if (sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN)
181
	{
182
		should_enable_sd_clock = true;
183
		sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
184
	}
185

186
	// Enable E_INPUT (SD) or Disable E_PWRD (eMMC) power.
187
	if (!(sdmmc->regs->sdmemcmppadctl & SDHCI_TEGRA_PADCTRL_E_INPUT_PWRD))
188
	{
189
		sdmmc->regs->sdmemcmppadctl |= SDHCI_TEGRA_PADCTRL_E_INPUT_PWRD;
190
		_sdmmc_commit_changes(sdmmc);
191
		usleep(1);
192
	}
193

194
	// Enable auto calibration and start auto configuration.
195
	sdmmc->regs->autocalcfg |= SDHCI_TEGRA_AUTOCAL_ENABLE | SDHCI_TEGRA_AUTOCAL_START;
196
	_sdmmc_commit_changes(sdmmc);
197
	usleep(2);
198

199
	u32 timeout = get_tmr_ms() + 10;
200
	while (sdmmc->regs->autocalsts & SDHCI_TEGRA_AUTOCAL_ACTIVE)
201
	{
202
		if (get_tmr_ms() > timeout)
203
		{
204
			timeout = 0; // Set timeout to 0 if we timed out.
205
			break;
206
		}
207
	}
208

209
#ifdef ERROR_EXTRA_PRINTING
210
	// Check if Comp pad is open or short to ground.
211
	// SDMMC1: CZ pads - T210/T210B01: 7-bit/5-bit. SDMMC2/4: LV_CZ pads - 5-bit.
212
	// Use 0x1F mask for all.
213
	u8 autocal_pu_status = sdmmc->regs->autocalsts & 0x1F;
214
	if (!autocal_pu_status)
215
		EPRINTFARGS("SDMMC%d: Comp Pad open!", sdmmc->id + 1); // Or resistance is extreme.
216
	else if (autocal_pu_status == 0x1F)
217
		EPRINTFARGS("SDMMC%d: Comp Pad short to gnd!", sdmmc->id + 1);
218
#endif
219

220
	// In case auto calibration fails, we load suggested standard values.
221
	if (!timeout)
222
	{
223
		sdmmc->regs->autocalcfg &= ~SDHCI_TEGRA_AUTOCAL_ENABLE;
224
		_sdmmc_pad_config_fallback(sdmmc, power);
225
#ifdef ERROR_EXTRA_PRINTING
226
		EPRINTFARGS("SDMMC%d: Comp Pad cal timeout!", sdmmc->id + 1);
227
#endif
228
	}
229

230
	// Disable E_INPUT (SD) or enable E_PWRD (eMMC) to conserve power.
231
	sdmmc->regs->sdmemcmppadctl &= ~SDHCI_TEGRA_PADCTRL_E_INPUT_PWRD;
232

233
	if (should_enable_sd_clock)
234
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
235
}
236

237
static int _sdmmc_dll_cal_execute(sdmmc_t *sdmmc)
238
{
239
	int result = 1, should_disable_sd_clock = 0;
240

241
	if (!(sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN))
242
	{
243
		should_disable_sd_clock = 1;
244
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
245
	}
246

247
	 // Add -4 TX_DLY_CODE_OFFSET if HS533/HS667.
248
	 // if (sdmmc->id == SDMMC_4 && sdmmc->card_clock > 208000)
249
	 //	sdmmc->regs->vendllctl0 = sdmmc->regs->vendllctl0 &= 0xFFFFC07F | (0x7C << 7);
250

251
	sdmmc->regs->vendllcalcfg |= SDHCI_TEGRA_DLLCAL_CALIBRATE;
252
	_sdmmc_commit_changes(sdmmc);
253

254
	u32 timeout = get_tmr_ms() + 5;
255
	while (sdmmc->regs->vendllcalcfg & SDHCI_TEGRA_DLLCAL_CALIBRATE)
256
	{
257
		if (get_tmr_ms() > timeout)
258
		{
259
			result = 0;
260
			goto out;
261
		}
262
	}
263

264
	timeout = get_tmr_ms() + 10;
265
	while (sdmmc->regs->vendllcalcfgsts & SDHCI_TEGRA_DLLCAL_ACTIVE)
266
	{
267
		if (get_tmr_ms() > timeout)
268
		{
269
			result = 0;
270
			goto out;
271
		}
272
	}
273

274
out:;
275
	if (should_disable_sd_clock)
276
		sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
277
	return result;
278
}
279

280
static void _sdmmc_reset_cmd_data(sdmmc_t *sdmmc)
281
{
282
	sdmmc->regs->swrst |= SDHCI_RESET_CMD | SDHCI_RESET_DATA;
283
	_sdmmc_commit_changes(sdmmc);
284
	u32 timeout = get_tmr_ms() + 2000;
285
	while ((sdmmc->regs->swrst & (SDHCI_RESET_CMD | SDHCI_RESET_DATA)) && get_tmr_ms() < timeout)
286
		;
287
}
288

289
static void _sdmmc_reset_all(sdmmc_t *sdmmc)
290
{
291
	sdmmc->regs->swrst |= SDHCI_RESET_ALL;
292
	_sdmmc_commit_changes(sdmmc);
293
	u32 timeout = get_tmr_ms() + 2000;//100ms
294
	while ((sdmmc->regs->swrst & SDHCI_RESET_ALL) && get_tmr_ms() < timeout)
295
		;
296
}
297

298
void sdmmc_setup_drv_type(sdmmc_t *sdmmc, u32 type)
299
{
300
	sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_DRV_TYPE_MASK)) | SDHCI_CTRL_DRV_TYPE(type);
301

302
	_sdmmc_commit_changes(sdmmc);
303
}
304

305
int sdmmc_setup_clock(sdmmc_t *sdmmc, u32 type)
306
{
307
	// Disable the SD clock if it was enabled, and reenable it later.
308
	bool should_enable_sd_clock = false;
309
	if (sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN)
310
	{
311
		should_enable_sd_clock = true;
312
		sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
313
	}
314

315
	_sdmmc_config_tap_val(sdmmc, type);
316

317
	_sdmmc_reset_cmd_data(sdmmc);
318

319
	switch (type)
320
	{
321
	case SDHCI_TIMING_MMC_ID:
322
	case SDHCI_TIMING_MMC_LS26:
323
	case SDHCI_TIMING_SD_ID:
324
	case SDHCI_TIMING_SD_DS12:
325
		sdmmc->regs->hostctl  &= ~SDHCI_CTRL_HISPD;
326
		sdmmc->regs->hostctl2 &= ~SDHCI_CTRL_VDD_180;
327
		break;
328

329
	case SDHCI_TIMING_MMC_HS52:
330
	case SDHCI_TIMING_SD_HS25:
331
		sdmmc->regs->hostctl  |= SDHCI_CTRL_HISPD;
332
		sdmmc->regs->hostctl2 &= ~SDHCI_CTRL_VDD_180;
333
		break;
334

335
	case SDHCI_TIMING_MMC_HS200:
336
	case SDHCI_TIMING_UHS_SDR50: // T210 Errata: the host must be set to SDR104 to WAR a CRC issue.
337
	case SDHCI_TIMING_UHS_SDR104:
338
	case SDHCI_TIMING_UHS_SDR82:
339
	case SDHCI_TIMING_MMC_HS100:
340
		sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_UHS_MASK)) | UHS_SDR104_BUS_SPEED;
341
		sdmmc->regs->hostctl2 |= SDHCI_CTRL_VDD_180;
342
		break;
343

344
	case SDHCI_TIMING_MMC_HS400:
345
		// Non standard.
346
		sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_UHS_MASK)) | HS400_BUS_SPEED;
347
		sdmmc->regs->hostctl2 |= SDHCI_CTRL_VDD_180;
348
		break;
349

350
	case SDHCI_TIMING_UHS_SDR25:
351
		sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_UHS_MASK)) | UHS_SDR25_BUS_SPEED;
352
		sdmmc->regs->hostctl2 |= SDHCI_CTRL_VDD_180;
353
		break;
354

355
	case SDHCI_TIMING_UHS_SDR12:
356
		sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_UHS_MASK)) | UHS_SDR12_BUS_SPEED;
357
		sdmmc->regs->hostctl2 |= SDHCI_CTRL_VDD_180;
358
		break;
359

360
	case SDHCI_TIMING_UHS_DDR50:
361
#ifdef BDK_SDMMC_UHS_DDR200_SUPPORT
362
	case SDHCI_TIMING_UHS_DDR200:
363
#endif
364
		sdmmc->regs->hostctl2  = (sdmmc->regs->hostctl2 & (~SDHCI_CTRL_UHS_MASK)) | UHS_DDR50_BUS_SPEED;
365
		sdmmc->regs->hostctl2 |= SDHCI_CTRL_VDD_180;
366
		break;
367
	}
368

369
	_sdmmc_commit_changes(sdmmc);
370

371
	u32 clock;
372
	u16 divisor;
373
	clock_sdmmc_get_card_clock_div(&clock, &divisor, type);
374
	clock_sdmmc_config_clock_source(&clock, sdmmc->id, clock);
375
	sdmmc->card_clock = (clock + divisor - 1) / divisor;
376

377
	// (divisor != 1) && (divisor & 1) -> error
378

379
	u16 div_lo = divisor >> 1;
380
	u16 div_hi = div_lo  >> 8;
381

382
	sdmmc->regs->clkcon = (sdmmc->regs->clkcon & ~(SDHCI_DIV_MASK | SDHCI_DIV_HI_MASK)) |
383
						  (div_lo << SDHCI_DIV_LO_SHIFT) | (div_hi << SDHCI_DIV_HI_SHIFT);
384

385
	// Enable the SD clock again.
386
	if (should_enable_sd_clock)
387
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
388

389
	if (type == SDHCI_TIMING_MMC_HS400)
390
		return _sdmmc_dll_cal_execute(sdmmc);
391

392
	return 1;
393
}
394

395
static void _sdmmc_card_clock_enable(sdmmc_t *sdmmc)
396
{
397
	// Recalibrate periodically if needed.
398
	if (sdmmc->periodic_calibration && !sdmmc->powersave_enabled)
399
		_sdmmc_autocal_execute(sdmmc, sdmmc_get_io_power(sdmmc));
400

401
	if (!sdmmc->powersave_enabled)
402
	{
403
		if (!(sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN))
404
			sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
405
	}
406
	sdmmc->card_clock_enabled = 1;
407
}
408

409
static void _sdmmc_card_clock_disable(sdmmc_t *sdmmc)
410
{
411
	sdmmc->card_clock_enabled = 0;
412
	sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
413
}
414

415
void sdmmc_card_clock_powersave(sdmmc_t *sdmmc, int powersave_enable)
416
{
417
	// Recalibrate periodically if needed.
418
	if (sdmmc->periodic_calibration && !powersave_enable && sdmmc->card_clock_enabled)
419
		_sdmmc_autocal_execute(sdmmc, sdmmc_get_io_power(sdmmc));
420

421
	sdmmc->powersave_enabled = powersave_enable;
422
	if (powersave_enable)
423
	{
424
		if (sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN)
425
			sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
426
		return;
427
	}
428

429
	if (sdmmc->card_clock_enabled)
430
		if (!(sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN))
431
			sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
432
}
433

434
static int _sdmmc_cache_rsp(sdmmc_t *sdmmc, u32 *rsp, u32 type)
435
{
436
	switch (type)
437
	{
438
	case SDMMC_RSP_TYPE_1:
439
	case SDMMC_RSP_TYPE_3:
440
	case SDMMC_RSP_TYPE_4:
441
	case SDMMC_RSP_TYPE_5:
442
		rsp[0] = sdmmc->regs->rspreg[0];
443
		break;
444

445
	case SDMMC_RSP_TYPE_2:
446
		// CRC is stripped, so shifting is needed.
447
		for (u32 i = 0; i < 4; i++)
448
		{
449
			u32 tempreg = sdmmc->regs->rspreg[3 - i];
450
			rsp[i] = tempreg << 8;
451

452
			if (i != 0)
453
				rsp[i - 1] |= (tempreg >> 24) & 0xFF;
454
		}
455
		break;
456

457
	default:
458
		return 0;
459
	}
460

461
	return 1;
462
}
463

464
int sdmmc_get_cached_rsp(sdmmc_t *sdmmc, u32 *rsp, u32 type)
465
{
466
	if (!rsp || sdmmc->expected_rsp_type != type)
467
		return 0;
468

469
	switch (type)
470
	{
471
	case SDMMC_RSP_TYPE_1:
472
	case SDMMC_RSP_TYPE_3:
473
	case SDMMC_RSP_TYPE_4:
474
	case SDMMC_RSP_TYPE_5:
475
		rsp[0] = sdmmc->rsp[0];
476
		break;
477

478
	case SDMMC_RSP_TYPE_2:
479
		for (u32 i = 0; i < 4; i++)
480
			rsp[i] = sdmmc->rsp[i];
481
		break;
482

483
	default:
484
		return 0;
485
	}
486

487
	return 1;
488
}
489

490
static int _sdmmc_wait_cmd_data_inhibit(sdmmc_t *sdmmc, bool wait_dat)
491
{
492
	_sdmmc_commit_changes(sdmmc);
493

494
	u32 timeout = get_tmr_ms() + 2000;
495
	while (sdmmc->regs->prnsts & SDHCI_CMD_INHIBIT)
496
		if (get_tmr_ms() > timeout)
497
		{
498
			_sdmmc_reset_cmd_data(sdmmc);
499
			return 0;
500
		}
501

502
	if (wait_dat)
503
	{
504
		timeout = get_tmr_ms() + 2000;
505
		while (sdmmc->regs->prnsts & SDHCI_DATA_INHIBIT)
506
			if (get_tmr_ms() > timeout)
507
			{
508
				_sdmmc_reset_cmd_data(sdmmc);
509
				return 0;
510
			}
511
	}
512

513
	return 1;
514
}
515

516
static int _sdmmc_wait_card_busy(sdmmc_t *sdmmc)
517
{
518
	_sdmmc_commit_changes(sdmmc);
519

520
	u32 timeout = get_tmr_ms() + 2000;
521
	while (!(sdmmc->regs->prnsts & SDHCI_DATA_0_LVL))
522
		if (get_tmr_ms() > timeout)
523
		{
524
			_sdmmc_reset_cmd_data(sdmmc);
525
			return 0;
526
		}
527

528
	return 1;
529
}
530

531
static int _sdmmc_setup_read_small_block(sdmmc_t *sdmmc)
532
{
533
	switch (sdmmc_get_bus_width(sdmmc))
534
	{
535
	case SDMMC_BUS_WIDTH_1:
536
		return 0;
537

538
	case SDMMC_BUS_WIDTH_4:
539
		sdmmc->regs->blksize = 64;
540
		break;
541

542
	case SDMMC_BUS_WIDTH_8:
543
		sdmmc->regs->blksize = 128;
544
		break;
545
	}
546

547
	sdmmc->regs->blkcnt = 1;
548
	sdmmc->regs->trnmod = SDHCI_TRNS_READ;
549

550
	return 1;
551
}
552

553
static int _sdmmc_send_cmd(sdmmc_t *sdmmc, const sdmmc_cmd_t *cmd, bool is_data_present)
554
{
555
	u16 cmdflags = 0;
556

557
	switch (cmd->rsp_type)
558
	{
559
	case SDMMC_RSP_TYPE_0:
560
		break;
561

562
	case SDMMC_RSP_TYPE_1:
563
	case SDMMC_RSP_TYPE_4:
564
	case SDMMC_RSP_TYPE_5:
565
		if (cmd->check_busy)
566
			cmdflags = SDHCI_CMD_RESP_LEN48_BUSY | SDHCI_CMD_INDEX | SDHCI_CMD_CRC;
567
		else
568
			cmdflags = SDHCI_CMD_RESP_LEN48      | SDHCI_CMD_INDEX | SDHCI_CMD_CRC;
569
		break;
570

571
	case SDMMC_RSP_TYPE_2:
572
		cmdflags = SDHCI_CMD_RESP_LEN136 | SDHCI_CMD_CRC;
573
		break;
574

575
	case SDMMC_RSP_TYPE_3:
576
		cmdflags = SDHCI_CMD_RESP_LEN48;
577
		break;
578

579
	default:
580
		return 0;
581
	}
582

583
	if (is_data_present)
584
		cmdflags |= SDHCI_CMD_DATA;
585

586
	sdmmc->regs->argument = cmd->arg;
587
	sdmmc->regs->cmdreg   = SDHCI_CMD_IDX(cmd->cmd) | cmdflags;
588

589
	return 1;
590
}
591

592
static void _sdmmc_send_tuning_cmd(sdmmc_t *sdmmc, u32 cmd)
593
{
594
	sdmmc_cmd_t cmdbuf;
595
	cmdbuf.cmd = cmd;
596
	cmdbuf.arg = 0;
597
	cmdbuf.rsp_type = SDMMC_RSP_TYPE_1;
598
	cmdbuf.check_busy = 0;
599
	_sdmmc_send_cmd(sdmmc, &cmdbuf, true);
600
}
601

602
static int _sdmmc_tuning_execute_once(sdmmc_t *sdmmc, u32 cmd, u32 tap)
603
{
604
	if (!_sdmmc_wait_cmd_data_inhibit(sdmmc, true))
605
		return 0;
606

607
	_sdmmc_setup_read_small_block(sdmmc);
608

609
	sdmmc->regs->norintstsen |= SDHCI_INT_DATA_AVAIL;
610
	sdmmc->regs->norintsts = sdmmc->regs->norintsts;
611
	sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
612

613
#ifdef BDK_SDMMC_UHS_DDR200_SUPPORT
614
	// Set tap if manual tuning.
615
	if (tap != HW_TAP_TUNING)
616
	{
617
		sdmmc->regs->ventunctl0 &= ~SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
618
		sdmmc->regs->venclkctl   = (sdmmc->regs->venclkctl & 0xFF00FFFF) | (tap << 16);
619
		sdmmc->regs->ventunctl0 |=  SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
620
	}
621
#endif
622

623
	_sdmmc_send_tuning_cmd(sdmmc, cmd);
624
	_sdmmc_commit_changes(sdmmc);
625
	usleep(1);
626

627
	_sdmmc_reset_cmd_data(sdmmc);
628

629
	sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
630
	_sdmmc_commit_changes(sdmmc);
631

632
	u32 timeout = get_tmr_us() + 5000;
633
	while (get_tmr_us() < timeout)
634
	{
635
		if (sdmmc->regs->norintsts & SDHCI_INT_DATA_AVAIL)
636
		{
637
			sdmmc->regs->norintsts = SDHCI_INT_DATA_AVAIL;
638
			sdmmc->regs->norintstsen &= ~SDHCI_INT_DATA_AVAIL;
639
			_sdmmc_commit_changes(sdmmc);
640
			usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
641
			return 1;
642
		}
643
	}
644

645
	_sdmmc_reset_cmd_data(sdmmc);
646

647
	sdmmc->regs->norintstsen &= ~SDHCI_INT_DATA_AVAIL;
648
	_sdmmc_commit_changes(sdmmc);
649
	usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
650

651
	return 0;
652
}
653

654
#ifdef BDK_SDMMC_UHS_DDR200_SUPPORT
655
typedef struct _sdmmc_manual_tuning_t
656
{
657
	u32 result[8];
658
	u32 num_iter;
659
	u32 tap_start;
660
	u32 tap_end;
661
} sdmmc_manual_tuning_t;
662

663
static int _sdmmc_manual_tuning_set_tap(sdmmc_t *sdmmc, sdmmc_manual_tuning_t *tuning)
664
{
665
	u32 tap_start = INVALID_TAP;
666
	u32 win_size  = 0;
667
	u32 best_tap  = 0;
668
	u32 best_size = 0;
669

670
	for (u32 i = 0; i < tuning->num_iter; i++)
671
	{
672
		u32 iter_end = i == (tuning->num_iter - 1) ? 1 : 0;
673
		u32 stable = tuning->result[i / 32] & BIT(i % 32);
674
		if (stable && !iter_end)
675
		{
676
			if (tap_start == INVALID_TAP)
677
				tap_start = i;
678

679
			win_size++;
680
		}
681
		else
682
		{
683
			if (tap_start != INVALID_TAP)
684
			{
685
				u32 tap_end = !iter_end ? (i - 1) : i;
686

687
				// Check if window is wider.
688
				if (win_size > best_size)
689
				{
690
					best_tap  = (tap_start + tap_end) / 2;
691
					best_size = win_size + iter_end;
692
				}
693

694
				tap_start = INVALID_TAP;
695
				win_size  = 0;
696
			}
697
		}
698
	}
699

700

701
	// Check if failed or window too small.
702
	if (!best_tap || best_size < SAMPLING_WINDOW_SIZE_MIN)
703
		return 0;
704

705
	sdmmc->regs->clkcon     &= ~SDHCI_CLOCK_CARD_EN;
706
	sdmmc->regs->ventunctl0 &= ~SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
707

708
	// Set tap.
709
	sdmmc->regs->venclkctl   = (sdmmc->regs->venclkctl & 0xFF00FFFF) | (best_tap << 16);
710

711
	sdmmc->regs->ventunctl0 |=  SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
712
	sdmmc->regs->clkcon     |= SDHCI_CLOCK_CARD_EN;
713

714
	return 1;
715
}
716

717
/*
718
 * SD Card DDR200 (DDR208) support
719
 *
720
 * On Tegra X1, that can be done with DDR50 host mode.
721
 * That's because HS400 4-bit or HS400 generally, is not supported on SDMMC1/3.
722
 * And also, tuning can't be done automatically on any DDR mode.
723
 * So it needs to be done manually and selected tap will be applied from the biggest
724
 * sampling window.
725
 * That allows DDR200 support on every DDR200 sd card, other than the original maker
726
 * of DDR200, Sandisk. Since Sandisk cards mandate DLL syncing.
727
 */
728
static int sdmmc_tuning_execute_ddr200(sdmmc_t *sdmmc)
729
{
730
	sdmmc_manual_tuning_t manual_tuning = { 0 };
731
	manual_tuning.num_iter = 128;
732

733
	sdmmc->regs->ventunctl1  = 0; // step_size 1.
734
	sdmmc->regs->ventunctl0  = (sdmmc->regs->ventunctl0 & 0xFFFF1FFF) | (2 << 13); // 128 Tries.
735
	sdmmc->regs->ventunctl0  = (sdmmc->regs->ventunctl0 & 0xFFFFE03F) | (1 << 6);  // 1x Multiplier.
736
	sdmmc->regs->ventunctl0 |= SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
737

738
	sdmmc->regs->hostctl2   |= SDHCI_CTRL_EXEC_TUNING;
739

740
	for (u32 i = 0; i < manual_tuning.num_iter; i++)
741
	{
742
		_sdmmc_tuning_execute_once(sdmmc, MMC_SEND_TUNING_BLOCK, i);
743

744
		// Save result for manual tuning.
745
		int sampled = (sdmmc->regs->hostctl2 >> SDHCI_CTRL_TUNED_CLK_SHIFT) & 1;
746
		manual_tuning.result[i / 32] |= sampled << (i % 32);
747

748
		if (!(sdmmc->regs->hostctl2 & SDHCI_CTRL_EXEC_TUNING))
749
			break;
750
	}
751

752
	return _sdmmc_manual_tuning_set_tap(sdmmc, &manual_tuning);
753
}
754
#endif
755

756
int sdmmc_tuning_execute(sdmmc_t *sdmmc, u32 type, u32 cmd)
757
{
758
	u32 num_iter, flag;
759

760
	if (sdmmc->powersave_enabled)
761
		return 0;
762

763
	switch (type)
764
	{
765
	case SDHCI_TIMING_MMC_HS200:
766
	case SDHCI_TIMING_UHS_SDR104:
767
	case SDHCI_TIMING_UHS_SDR82:
768
		num_iter = 128;
769
		flag = (2 << 13); // 128 iterations.
770
		break;
771

772
	case SDHCI_TIMING_UHS_SDR50:
773
	case SDHCI_TIMING_UHS_DDR50: // HW tuning is not supported on DDR modes. But it sets tap to 0 which is proper.
774
	case SDHCI_TIMING_MMC_HS100:
775
		num_iter = 256;
776
		flag = (4 << 13); // 256 iterations.
777
		break;
778

779
	case SDHCI_TIMING_MMC_HS400:
780
	case SDHCI_TIMING_UHS_SDR12:
781
	case SDHCI_TIMING_UHS_SDR25:
782
		return 1;
783

784
#ifdef BDK_SDMMC_UHS_DDR200_SUPPORT
785
	case SDHCI_TIMING_UHS_DDR200:
786
		return sdmmc_tuning_execute_ddr200(sdmmc);
787
#endif
788

789
	default:
790
		return 0;
791
	}
792

793
	sdmmc->regs->ventunctl1  = 0; // step_size 1.
794
	sdmmc->regs->ventunctl0  = (sdmmc->regs->ventunctl0 & 0xFFFF1FFF) | flag;     // Tries.
795
	sdmmc->regs->ventunctl0  = (sdmmc->regs->ventunctl0 & 0xFFFFE03F) | (1 << 6); // 1x Multiplier.
796
	sdmmc->regs->ventunctl0 |= SDHCI_TEGRA_TUNING_TAP_HW_UPDATED;
797

798
	sdmmc->regs->hostctl2   |= SDHCI_CTRL_EXEC_TUNING;
799

800
	for (u32 i = 0; i < num_iter; i++)
801
	{
802
		_sdmmc_tuning_execute_once(sdmmc, cmd, HW_TAP_TUNING);
803

804
		if (!(sdmmc->regs->hostctl2 & SDHCI_CTRL_EXEC_TUNING))
805
			break;
806
	}
807

808
	if (sdmmc->regs->hostctl2 & SDHCI_CTRL_TUNED_CLK)
809
		return 1;
810

811
	return 0;
812
}
813

814
static int _sdmmc_enable_internal_clock(sdmmc_t *sdmmc)
815
{
816
	//Enable internal clock and wait till it is stable.
817
	sdmmc->regs->clkcon |= SDHCI_CLOCK_INT_EN;
818
	_sdmmc_commit_changes(sdmmc);
819
	u32 timeout = get_tmr_ms() + 2000;
820
	while (!(sdmmc->regs->clkcon & SDHCI_CLOCK_INT_STABLE))
821
	{
822
		if (get_tmr_ms() > timeout)
823
			return 0;
824
	}
825

826
	sdmmc->regs->hostctl2 &= ~SDHCI_CTRL_PRESET_VAL_EN;
827
	sdmmc->regs->clkcon   &= ~SDHCI_PROG_CLOCK_MODE;
828
	// Enable 32/64bit addressing if used (sysad. if blkcnt it fallbacks to 16bit).
829
	sdmmc->regs->hostctl2 |= SDHCI_HOST_VERSION_4_EN;
830

831
	if (!(sdmmc->regs->capareg & SDHCI_CAP_64BIT))
832
		return 0;
833

834
	sdmmc->regs->hostctl2  |= SDHCI_ADDRESSING_64BIT_EN;
835
	sdmmc->regs->hostctl   &= ~SDHCI_CTRL_DMA_MASK; // Use SDMA. Host V4 enabled so adma address regs in use.
836
	sdmmc->regs->timeoutcon = (sdmmc->regs->timeoutcon & 0xF0) | 14; // TMCLK * 2^27.
837

838
	return 1;
839
}
840

841
static int _sdmmc_autocal_config_offset(sdmmc_t *sdmmc, u32 power)
842
{
843
	u32 off_pd = 0;
844
	u32 off_pu = 0;
845

846
	switch (sdmmc->id)
847
	{
848
	case SDMMC_2:
849
	case SDMMC_4:
850
		if (power != SDMMC_POWER_1_8)
851
			return 0;
852
		off_pd = 5;
853
		off_pu = 5;
854
		break;
855

856
	case SDMMC_1:
857
		if (power == SDMMC_POWER_1_8)
858
		{
859
			if (!sdmmc->t210b01)
860
			{
861
				off_pd = 0x7B; // -5.
862
				off_pu = 0x7B; // -5.
863
			}
864
			else
865
			{
866
				off_pd = 6;
867
				off_pu = 6;
868
			}
869
		}
870
		else if (power == SDMMC_POWER_3_3)
871
		{
872
			if (!sdmmc->t210b01)
873
			{
874
				off_pd = 0x7D; // -3.
875
				off_pu = 0;
876
			}
877
		}
878
		else
879
			return 0;
880
		break;
881
	}
882

883
	sdmmc->regs->autocalcfg = (sdmmc->regs->autocalcfg & 0xFFFF8080) | (off_pd << 8) | off_pu;
884
	return 1;
885
}
886

887
static void _sdmmc_enable_interrupts(sdmmc_t *sdmmc)
888
{
889
	sdmmc->regs->norintstsen |= SDHCI_INT_DMA_END | SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE;
890
	sdmmc->regs->errintstsen |= SDHCI_ERR_INT_ALL_EXCEPT_ADMA_BUSPWR;
891
	sdmmc->regs->norintsts = sdmmc->regs->norintsts;
892
	sdmmc->regs->errintsts = sdmmc->regs->errintsts;
893
	sdmmc->error_sts = 0;
894
}
895

896
static void _sdmmc_mask_interrupts(sdmmc_t *sdmmc)
897
{
898
	sdmmc->regs->errintstsen &= ~SDHCI_ERR_INT_ALL_EXCEPT_ADMA_BUSPWR;
899
	sdmmc->regs->norintstsen &= ~(SDHCI_INT_DMA_END | SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE);
900
}
901

902
static u32 _sdmmc_check_mask_interrupt(sdmmc_t *sdmmc, u16 *pout, u16 mask)
903
{
904
	u16 norintsts = sdmmc->regs->norintsts;
905
	u16 errintsts = sdmmc->regs->errintsts;
906

907
	DPRINTF("norintsts %08X, errintsts %08X\n", norintsts, errintsts);
908

909
	if (pout)
910
		*pout = norintsts;
911

912
	// Check for error interrupt.
913
	if (norintsts & SDHCI_INT_ERROR)
914
	{
915
#ifdef ERROR_EXTRA_PRINTING
916
		EPRINTFARGS("SDMMC%d: intsts %08X, errintsts %08X", sdmmc->id + 1, norintsts, errintsts);
917
#endif
918
		sdmmc->error_sts = errintsts;
919
		sdmmc->regs->errintsts = errintsts;
920
		return SDMMC_MASKINT_ERROR;
921
	}
922
	else if (norintsts & mask)
923
	{
924
		sdmmc->regs->norintsts = norintsts & mask;
925
		return SDMMC_MASKINT_MASKED;
926
	}
927

928
	return SDMMC_MASKINT_NOERROR;
929
}
930

931
static int _sdmmc_wait_response(sdmmc_t *sdmmc)
932
{
933
	_sdmmc_commit_changes(sdmmc);
934

935
	u32 timeout = get_tmr_ms() + 2000;
936
	while (true)
937
	{
938
		u32 result = _sdmmc_check_mask_interrupt(sdmmc, NULL, SDHCI_INT_RESPONSE);
939
		if (result == SDMMC_MASKINT_MASKED)
940
			break;
941
		if (result != SDMMC_MASKINT_NOERROR || get_tmr_ms() > timeout)
942
		{
943
			_sdmmc_reset_cmd_data(sdmmc);
944
			return 0;
945
		}
946
	}
947

948
	return 1;
949
}
950

951
static int _sdmmc_stop_transmission_inner(sdmmc_t *sdmmc, u32 *rsp)
952
{
953
	sdmmc_cmd_t cmd;
954

955
	if (!_sdmmc_wait_cmd_data_inhibit(sdmmc, false))
956
		return 0;
957

958
	_sdmmc_enable_interrupts(sdmmc);
959

960
	cmd.cmd = MMC_STOP_TRANSMISSION;
961
	cmd.arg = 0;
962
	cmd.rsp_type = SDMMC_RSP_TYPE_1;
963
	cmd.check_busy = 1;
964

965
	_sdmmc_send_cmd(sdmmc, &cmd, false);
966

967
	int result = _sdmmc_wait_response(sdmmc);
968
	_sdmmc_mask_interrupts(sdmmc);
969

970
	if (!result)
971
		return 0;
972

973
	_sdmmc_cache_rsp(sdmmc, rsp, SDMMC_RSP_TYPE_1);
974

975
	return _sdmmc_wait_card_busy(sdmmc);
976
}
977

978
int sdmmc_stop_transmission(sdmmc_t *sdmmc, u32 *rsp)
979
{
980
	if (!sdmmc->card_clock_enabled)
981
		return 0;
982

983
	// Recalibrate periodically if needed.
984
	if (sdmmc->periodic_calibration && sdmmc->powersave_enabled)
985
		_sdmmc_autocal_execute(sdmmc, sdmmc_get_io_power(sdmmc));
986

987
	bool should_disable_sd_clock = false;
988
	if (!(sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN))
989
	{
990
		should_disable_sd_clock = true;
991
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
992
		_sdmmc_commit_changes(sdmmc);
993
		usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
994
	}
995

996
	int result = _sdmmc_stop_transmission_inner(sdmmc, rsp);
997
	usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
998

999
	if (should_disable_sd_clock)
1000
		sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
1001

1002
	return result;
1003
}
1004

1005
static int _sdmmc_config_sdma(sdmmc_t *sdmmc, u32 *blkcnt_out, const sdmmc_req_t *req)
1006
{
1007
	if (!req->blksize || !req->num_sectors)
1008
		return 0;
1009

1010
	u32 blkcnt = req->num_sectors;
1011
	if (blkcnt >= 0xFFFF)
1012
		blkcnt = 0xFFFF;
1013
	u32 admaaddr = (u32)req->buf;
1014

1015
	// Check alignment.
1016
	if (admaaddr & 7)
1017
		return 0;
1018

1019
	sdmmc->regs->admaaddr = admaaddr;
1020
	sdmmc->regs->admaaddr_hi = 0;
1021

1022
	sdmmc->dma_addr_next = ALIGN_DOWN((admaaddr + SZ_512K), SZ_512K);
1023

1024
	sdmmc->regs->blksize = req->blksize | (7u << 12); // SDMA DMA 512KB Boundary (Detects A18 carry out).
1025
	sdmmc->regs->blkcnt  = blkcnt;
1026

1027
	if (blkcnt_out)
1028
		*blkcnt_out = blkcnt;
1029

1030
	u32 trnmode = SDHCI_TRNS_DMA | SDHCI_TRNS_RTYPE_R1;
1031

1032
	// Set multiblock request.
1033
	if (req->is_multi_block)
1034
		trnmode |= SDHCI_TRNS_MULTI | SDHCI_TRNS_BLK_CNT_EN;
1035

1036
	// Set request direction.
1037
	if (!req->is_write)
1038
		trnmode |= SDHCI_TRNS_READ;
1039

1040
	// Automatic send of stop transmission or set block count cmd.
1041
	if (req->is_auto_stop_trn)
1042
		trnmode |= SDHCI_TRNS_AUTO_CMD12;
1043
	//else if (req->is_auto_set_blkcnt)
1044
	//	trnmode |= SDHCI_TRNS_AUTO_CMD23;
1045

1046
	sdmmc->regs->trnmod = trnmode;
1047

1048
	return 1;
1049
}
1050

1051
static int _sdmmc_update_sdma(sdmmc_t *sdmmc)
1052
{
1053
	u16 blkcnt = 0;
1054
	do
1055
	{
1056
		blkcnt = sdmmc->regs->blkcnt;
1057
		u32 timeout = get_tmr_ms() + 1500;
1058
		do
1059
		{
1060
			u32 result = SDMMC_MASKINT_MASKED;
1061
			while (true)
1062
			{
1063
				u16 intr = 0;
1064
				result = _sdmmc_check_mask_interrupt(sdmmc, &intr,
1065
					SDHCI_INT_DATA_END | SDHCI_INT_DMA_END);
1066
				if (result != SDMMC_MASKINT_MASKED)
1067
					break;
1068

1069
				if (intr & SDHCI_INT_DATA_END)
1070
					return 1; // Transfer complete.
1071

1072
				if (intr & SDHCI_INT_DMA_END)
1073
				{
1074
					// Update DMA.
1075
					sdmmc->regs->admaaddr = sdmmc->dma_addr_next;
1076
					sdmmc->regs->admaaddr_hi = 0;
1077
					sdmmc->dma_addr_next += SZ_512K;
1078
				}
1079
			}
1080

1081
			if (result != SDMMC_MASKINT_NOERROR)
1082
			{
1083
#ifdef ERROR_EXTRA_PRINTING
1084
				EPRINTFARGS("SDMMC%d: int error!", sdmmc->id + 1);
1085
#endif
1086
				_sdmmc_reset_cmd_data(sdmmc);
1087

1088
				return 0;
1089
			}
1090
		} while (get_tmr_ms() < timeout);
1091
	} while (sdmmc->regs->blkcnt != blkcnt);
1092

1093
	_sdmmc_reset_cmd_data(sdmmc);
1094

1095
	return 0;
1096
}
1097

1098
static int _sdmmc_execute_cmd_inner(sdmmc_t *sdmmc, sdmmc_cmd_t *cmd, sdmmc_req_t *req, u32 *blkcnt_out)
1099
{
1100
	bool has_req_or_check_busy = req || cmd->check_busy;
1101
	if (!_sdmmc_wait_cmd_data_inhibit(sdmmc, has_req_or_check_busy))
1102
		return 0;
1103

1104
	u32 blkcnt = 0;
1105
	bool is_data_present = false;
1106
	if (req)
1107
	{
1108
		if (!_sdmmc_config_sdma(sdmmc, &blkcnt, req))
1109
		{
1110
#ifdef ERROR_EXTRA_PRINTING
1111
			EPRINTFARGS("SDMMC%d: DMA Wrong cfg!", sdmmc->id + 1);
1112
#endif
1113
			return 0;
1114
		}
1115

1116
		// Flush cache before starting the transfer.
1117
		bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLEAN_WAY, false);
1118

1119
		is_data_present = true;
1120
	}
1121

1122
	_sdmmc_enable_interrupts(sdmmc);
1123

1124
	if (!_sdmmc_send_cmd(sdmmc, cmd, is_data_present))
1125
	{
1126
#ifdef ERROR_EXTRA_PRINTING
1127
		EPRINTFARGS("SDMMC%d: Wrong Response type %08X!", sdmmc->id + 1, cmd->rsp_type);
1128
#endif
1129
		return 0;
1130
	}
1131

1132
	int result = _sdmmc_wait_response(sdmmc);
1133
#ifdef ERROR_EXTRA_PRINTING
1134
	if (!result)
1135
		EPRINTFARGS("SDMMC%d: Transfer error!", sdmmc->id + 1);
1136
#endif
1137
	DPRINTF("rsp(%d): %08X, %08X, %08X, %08X\n", result,
1138
		sdmmc->regs->rspreg[0], sdmmc->regs->rspreg[1], sdmmc->regs->rspreg[2], sdmmc->regs->rspreg[3]);
1139
	if (result)
1140
	{
1141
		if (cmd->rsp_type)
1142
		{
1143
			sdmmc->expected_rsp_type = cmd->rsp_type;
1144
			result = _sdmmc_cache_rsp(sdmmc, sdmmc->rsp, cmd->rsp_type);
1145
#ifdef ERROR_EXTRA_PRINTING
1146
			if (!result)
1147
				EPRINTFARGS("SDMMC%d: Unknown response type!", sdmmc->id + 1);
1148
#endif
1149
		}
1150
		if (req && result)
1151
		{
1152
			result = _sdmmc_update_sdma(sdmmc);
1153
#ifdef ERROR_EXTRA_PRINTING
1154
			if (!result)
1155
				EPRINTFARGS("SDMMC%d: DMA Update failed!", sdmmc->id + 1);
1156
#endif
1157
		}
1158
	}
1159

1160
	_sdmmc_mask_interrupts(sdmmc);
1161

1162
	if (result)
1163
	{
1164
		if (req)
1165
		{
1166
			// Invalidate cache after transfer.
1167
			bpmp_mmu_maintenance(BPMP_MMU_MAINT_INVALID_WAY, false);
1168

1169
			if (blkcnt_out)
1170
				*blkcnt_out = blkcnt;
1171

1172
			if (req->is_auto_stop_trn)
1173
				sdmmc->stop_trn_rsp = sdmmc->regs->rspreg[3];
1174
		}
1175

1176
		if (has_req_or_check_busy)
1177
		{
1178
			result = _sdmmc_wait_card_busy(sdmmc);
1179
#ifdef ERROR_EXTRA_PRINTING
1180
			if (!result)
1181
				EPRINTFARGS("SDMMC%d: Busy timeout!", sdmmc->id + 1);
1182
#endif
1183
			return result;
1184
		}
1185
	}
1186

1187
	return result;
1188
}
1189

1190
bool sdmmc_get_sd_inserted()
1191
{
1192
	return (!gpio_read(GPIO_PORT_Z, GPIO_PIN_1));
1193
}
1194

1195
static void _sdmmc_config_sdmmc1_schmitt()
1196
{
1197
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK)  |= PINMUX_SCHMT;
1198
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CMD)  |= PINMUX_SCHMT;
1199
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT3) |= PINMUX_SCHMT;
1200
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT2) |= PINMUX_SCHMT;
1201
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT1) |= PINMUX_SCHMT;
1202
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT0) |= PINMUX_SCHMT;
1203
}
1204

1205
static void _sdmmc_config_sdmmc2_schmitt()
1206
{
1207
	PINMUX_AUX(PINMUX_AUX_SDMMC2_CLK)  |= PINMUX_SCHMT;
1208
	PINMUX_AUX(PINMUX_AUX_SDMMC2_CMD)  |= PINMUX_SCHMT;
1209
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT7) |= PINMUX_SCHMT;
1210
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT6) |= PINMUX_SCHMT;
1211
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT5) |= PINMUX_SCHMT;
1212
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT4) |= PINMUX_SCHMT;
1213
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT3) |= PINMUX_SCHMT;
1214
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT2) |= PINMUX_SCHMT;
1215
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT1) |= PINMUX_SCHMT;
1216
	PINMUX_AUX(PINMUX_AUX_SDMMC2_DAT0) |= PINMUX_SCHMT;
1217
}
1218

1219
static void _sdmmc_config_sdmmc1_pads(bool discharge)
1220
{
1221
	u32 sdmmc1_pin_mask = GPIO_PIN_0 | GPIO_PIN_1 | GPIO_PIN_2 | GPIO_PIN_3 | GPIO_PIN_4 | GPIO_PIN_5;
1222

1223
	// Set values for Reset state.
1224
	u32 function = GPIO_MODE_SPIO;
1225
	u32 level    = GPIO_LOW;
1226
	u32 output   = GPIO_OUTPUT_DISABLE;
1227

1228
	// Set values for discharging.
1229
	if (discharge)
1230
	{
1231
		function = GPIO_MODE_GPIO;
1232
		level    = GPIO_HIGH;
1233
		output   = GPIO_OUTPUT_ENABLE;
1234
	}
1235

1236
	// Set all pads function.
1237
	gpio_config(GPIO_PORT_M, sdmmc1_pin_mask, function);
1238
	// Set all pads output level.
1239
	gpio_write(GPIO_PORT_M, sdmmc1_pin_mask, level);
1240
	// Set all pads output.
1241
	gpio_output_enable(GPIO_PORT_M, sdmmc1_pin_mask, output);
1242
}
1243

1244
static int _sdmmc_config_sdmmc1(bool t210b01)
1245
{
1246
	// Configure SD card detect.
1247
	PINMUX_AUX(PINMUX_AUX_GPIO_PZ1) = PINMUX_INPUT_ENABLE | PINMUX_PULL_UP | 2; // GPIO control, pull up.
1248
	APB_MISC(APB_MISC_GP_VGPIO_GPIO_MUX_SEL) = 0;
1249
	gpio_direction_input(GPIO_PORT_Z, GPIO_PIN_1);
1250
	usleep(100);
1251

1252
	// Check if SD card is inserted.
1253
	if (!sdmmc_get_sd_inserted())
1254
		return 0;
1255

1256
	// Enable deep loopback for SDMMC1 CLK pad so reads work.
1257
	APB_MISC(APB_MISC_GP_SDMMC1_CLK_LPBK_CONTROL) = 1;
1258

1259
	// Configure SDMMC1 CLK pinmux, based on state and SoC type.
1260
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK) &= ~PINMUX_SCHMT;
1261
	if (PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK) != (PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_DOWN)) // Check if CLK pad is already configured.
1262
		PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK) = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | (t210b01 ? PINMUX_PULL_NONE : PINMUX_PULL_DOWN);
1263

1264
	// Configure reset state of SDMMC1 pins pinmux.
1265
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CMD)  = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_UP;
1266
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT3) = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_UP;
1267
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT2) = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_UP;
1268
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT1) = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_UP;
1269
	PINMUX_AUX(PINMUX_AUX_SDMMC1_DAT0) = PINMUX_DRIVE_2X | PINMUX_INPUT_ENABLE | PINMUX_PULL_UP;
1270

1271
	// Force schmitt trigger for T210B01.
1272
	if (t210b01)
1273
		_sdmmc_config_sdmmc1_schmitt();
1274

1275
	// Make sure the SDMMC1 controller is powered.
1276
	PMC(APBDEV_PMC_NO_IOPOWER) |=  PMC_NO_IOPOWER_SDMMC1;
1277
	usleep(1000);
1278
	PMC(APBDEV_PMC_NO_IOPOWER) &= ~PMC_NO_IOPOWER_SDMMC1;
1279
	(void)PMC(APBDEV_PMC_NO_IOPOWER); // Commit write.
1280

1281
	// Enable SD card power. Powers LDO2 also.
1282
	PINMUX_AUX(PINMUX_AUX_DMIC3_CLK) = PINMUX_PULL_DOWN | 2;
1283
	gpio_direction_output(GPIO_PORT_E, GPIO_PIN_4, GPIO_HIGH);
1284
	usleep(10000); // Minimum 3 to 10 ms.
1285

1286
	// Inform IO pads that voltage is gonna be 3.3V.
1287
	PMC(APBDEV_PMC_PWR_DET_VAL) |= PMC_PWR_DET_33V_SDMMC1;
1288
	(void)PMC(APBDEV_PMC_PWR_DET_VAL); // Commit write.
1289

1290
	// Enable SD card IO power.
1291
	max7762x_regulator_set_voltage(REGULATOR_LDO2, 3300000);
1292
	max7762x_regulator_enable(REGULATOR_LDO2, true);
1293
	usleep(1000);
1294

1295
	// Set pad slew codes to get good quality clock.
1296
	if (!t210b01)
1297
	{
1298
		APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL) = (APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL) & 0xFFFFFFF) | 0x50000000;
1299
		(void)APB_MISC(APB_MISC_GP_SDMMC1_PAD_CFGPADCTRL); // Commit write.
1300
		usleep(1000);
1301
	}
1302

1303
	return 1;
1304
}
1305

1306
static void _sdmmc_config_emmc(u32 id, bool t210b01)
1307
{
1308
	switch (id)
1309
	{
1310
	case SDMMC_2:
1311
		if (!t210b01)
1312
		{
1313
			// Unset park for pads.
1314
			APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL) &= 0xF8003FFF;
1315
			(void)APB_MISC(APB_MISC_GP_EMMC2_PAD_CFGPADCTRL); // Commit write.
1316
		}
1317
		else // Enable schmitt trigger for T210B01.
1318
			_sdmmc_config_sdmmc2_schmitt();
1319
		break;
1320

1321
	case SDMMC_4:
1322
		// Unset park for pads.
1323
		APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL) &= 0xF8003FFF;
1324
		// Set default pad cfg.
1325
		if (t210b01)
1326
			APB_MISC(APB_MISC_GP_EMMC4_PAD_PUPD_CFGPADCTRL) &= 0xFFBFFFF9; // Unset CMD/CLK/DQS weak pull up/down.
1327
		// Enable schmitt trigger.
1328
		APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL) |= 1;
1329
		(void)APB_MISC(APB_MISC_GP_EMMC4_PAD_CFGPADCTRL); // Commit write.
1330
		break;
1331
	}
1332
}
1333

1334
int sdmmc_init(sdmmc_t *sdmmc, u32 id, u32 power, u32 bus_width, u32 type)
1335
{
1336
	u32 clock;
1337
	u16 divisor;
1338
	u8 vref_sel = 7;
1339

1340
	static const u8 trim_values_t210[4]    = {  2,  8,  3,  8 };
1341
	static const u8 trim_values_t210b01[4] = { 14, 13, 15, 13 };
1342
	const u8 *trim_values;
1343

1344
	if (id > SDMMC_4 || id == SDMMC_3)
1345
		return 0;
1346

1347
	memset(sdmmc, 0, sizeof(sdmmc_t));
1348

1349
	sdmmc->regs = (t210_sdmmc_t *)(SDMMC_BASE + (u32)_sdmmc_base_offsets[id]);
1350
	sdmmc->id = id;
1351
	sdmmc->clock_stopped = 1;
1352
	sdmmc->t210b01 = hw_get_chip_id() == GP_HIDREV_MAJOR_T210B01;
1353

1354
	trim_values = sdmmc->t210b01 ? trim_values_t210b01 : trim_values_t210;
1355

1356
	// Do specific SDMMC HW configuration.
1357
	switch (id)
1358
	{
1359
	case SDMMC_1:
1360
		if (!_sdmmc_config_sdmmc1(sdmmc->t210b01))
1361
			return 0;
1362
		if (sdmmc->t210b01)
1363
			vref_sel = 0;
1364
		else
1365
			sdmmc->periodic_calibration = 1;
1366
		break;
1367

1368
	case SDMMC_2:
1369
	case SDMMC_4:
1370
		_sdmmc_config_emmc(id, sdmmc->t210b01);
1371
		break;
1372
	}
1373

1374
	// Disable clock if enabled.
1375
	if (clock_sdmmc_is_not_reset_and_enabled(id))
1376
	{
1377
		_sdmmc_card_clock_disable(sdmmc);
1378
		_sdmmc_commit_changes(sdmmc);
1379
	}
1380

1381
	// Configure and enable selected clock.
1382
	clock_sdmmc_get_card_clock_div(&clock, &divisor, type);
1383
	clock_sdmmc_enable(id, clock);
1384
	sdmmc->clock_stopped = 0;
1385

1386
	// Make sure all sdmmc registers are reset.
1387
	_sdmmc_reset_all(sdmmc);
1388

1389
	// Set default pad IO trimming configuration.
1390
	sdmmc->regs->iospare |= BIT(19);      // Enable 1 cycle delayed cmd_oen.
1391
	sdmmc->regs->veniotrimctl &= ~BIT(2); // Set Band Gap VREG to supply DLL.
1392
	sdmmc->regs->venclkctl = (sdmmc->regs->venclkctl & 0xE0FFFFFB) | ((u32)trim_values[sdmmc->id] << 24);
1393
	sdmmc->regs->sdmemcmppadctl = (sdmmc->regs->sdmemcmppadctl & ~SDHCI_TEGRA_PADCTRL_VREF_SEL_MASK) | vref_sel;
1394

1395
	// Configure auto calibration values.
1396
	if (!_sdmmc_autocal_config_offset(sdmmc, power))
1397
		return 0;
1398

1399
	_sdmmc_commit_changes(sdmmc);
1400

1401
	// Calibrate pads.
1402
	_sdmmc_autocal_execute(sdmmc, power);
1403

1404
	// Enable internal clock and power.
1405
	if (_sdmmc_enable_internal_clock(sdmmc))
1406
	{
1407
		sdmmc_set_bus_width(sdmmc, bus_width);
1408
		_sdmmc_set_io_power(sdmmc, power);
1409

1410
		if (sdmmc_setup_clock(sdmmc, type))
1411
		{
1412
			sdmmc_card_clock_powersave(sdmmc, SDMMC_POWER_SAVE_DISABLE);
1413
			_sdmmc_card_clock_enable(sdmmc);
1414
			_sdmmc_commit_changes(sdmmc);
1415

1416
			return 1;
1417
		}
1418
	}
1419

1420
	return 0;
1421
}
1422

1423
void sdmmc1_disable_power()
1424
{
1425
	// T210B01 WAR: Clear pull down from CLK pad.
1426
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK) &= ~PINMUX_PULL_MASK;
1427

1428
	// T210B01 WAR: Set pads to discharge state.
1429
	_sdmmc_config_sdmmc1_pads(true);
1430

1431
	// Disable SD card IO power.
1432
	max7762x_regulator_enable(REGULATOR_LDO2, false);
1433
	usleep(4000);
1434

1435
	// Disable SD card power.
1436
	gpio_write(GPIO_PORT_E, GPIO_PIN_4, GPIO_LOW);
1437

1438
	// T210/T210B01 WAR: Set start timer for IO and Controller power discharge.
1439
	sd_power_cycle_time_start = get_tmr_ms();
1440
	usleep(10000); // To power cycle, min 1ms without power is needed.
1441

1442
	// Disable SDMMC1 controller power.
1443
	PMC(APBDEV_PMC_NO_IOPOWER) |= PMC_NO_IOPOWER_SDMMC1;
1444
	(void)PMC(APBDEV_PMC_NO_IOPOWER); // Commit write.
1445

1446
	// Inform IO pads that next voltage might be 3.3V.
1447
	PMC(APBDEV_PMC_PWR_DET_VAL) |= PMC_PWR_DET_33V_SDMMC1;
1448
	(void)PMC(APBDEV_PMC_PWR_DET_VAL); // Commit write.
1449

1450
	// T210B01 WAR: Restore pads to reset state.
1451
	_sdmmc_config_sdmmc1_pads(false);
1452

1453
	// T210B01 WAR: Restore pull down to CLK pad.
1454
	PINMUX_AUX(PINMUX_AUX_SDMMC1_CLK) |= PINMUX_PULL_DOWN;
1455
}
1456

1457
void sdmmc_end(sdmmc_t *sdmmc)
1458
{
1459
	if (!sdmmc->clock_stopped)
1460
	{
1461
		_sdmmc_card_clock_disable(sdmmc);
1462
		// Disable SDMMC power.
1463
		_sdmmc_set_io_power(sdmmc, SDMMC_POWER_OFF);
1464
		_sdmmc_commit_changes(sdmmc);
1465

1466
		// Disable SD card power.
1467
		if (sdmmc->id == SDMMC_1)
1468
			sdmmc1_disable_power();
1469

1470
		clock_sdmmc_disable(sdmmc->id);
1471
		sdmmc->clock_stopped = 1;
1472
	}
1473
}
1474

1475
void sdmmc_init_cmd(sdmmc_cmd_t *cmdbuf, u16 cmd, u32 arg, u32 rsp_type, u32 check_busy)
1476
{
1477
	cmdbuf->cmd = cmd;
1478
	cmdbuf->arg = arg;
1479
	cmdbuf->rsp_type = rsp_type;
1480
	cmdbuf->check_busy = check_busy;
1481
}
1482

1483
int sdmmc_execute_cmd(sdmmc_t *sdmmc, sdmmc_cmd_t *cmd, sdmmc_req_t *req, u32 *blkcnt_out)
1484
{
1485
	if (!sdmmc->card_clock_enabled)
1486
		return 0;
1487

1488
	// Recalibrate periodically if needed.
1489
	if (sdmmc->periodic_calibration && sdmmc->powersave_enabled)
1490
		_sdmmc_autocal_execute(sdmmc, sdmmc_get_io_power(sdmmc));
1491

1492
	int should_disable_sd_clock = 0;
1493
	if (!(sdmmc->regs->clkcon & SDHCI_CLOCK_CARD_EN))
1494
	{
1495
		should_disable_sd_clock = 1;
1496
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
1497
		_sdmmc_commit_changes(sdmmc);
1498
		usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
1499
	}
1500

1501
	int result = _sdmmc_execute_cmd_inner(sdmmc, cmd, req, blkcnt_out);
1502
	usleep((8 * 1000 + sdmmc->card_clock - 1) / sdmmc->card_clock); // Wait 8 cycles.
1503

1504
	if (should_disable_sd_clock)
1505
		sdmmc->regs->clkcon &= ~SDHCI_CLOCK_CARD_EN;
1506

1507
	return result;
1508
}
1509

1510
int sdmmc_enable_low_voltage(sdmmc_t *sdmmc)
1511
{
1512
	if (sdmmc->id != SDMMC_1)
1513
		return 0;
1514

1515
	_sdmmc_commit_changes(sdmmc);
1516

1517
	// Switch to 1.8V and wait for regulator to stabilize. Assume max possible wait needed.
1518
	max7762x_regulator_set_voltage(REGULATOR_LDO2, 1800000);
1519
	usleep(150);
1520

1521
	// Inform IO pads that we switched to 1.8V.
1522
	PMC(APBDEV_PMC_PWR_DET_VAL) &= ~PMC_PWR_DET_33V_SDMMC1;
1523
	(void)PMC(APBDEV_PMC_PWR_DET_VAL); // Commit write.
1524

1525
	// Enable schmitt trigger for better duty cycle and low jitter clock.
1526
	_sdmmc_config_sdmmc1_schmitt();
1527

1528
	_sdmmc_autocal_config_offset(sdmmc, SDMMC_POWER_1_8);
1529
	_sdmmc_autocal_execute(sdmmc, SDMMC_POWER_1_8);
1530
	_sdmmc_set_io_power(sdmmc, SDMMC_POWER_1_8);
1531
	_sdmmc_commit_changes(sdmmc);
1532
	msleep(5); // Wait minimum 5ms before turning on the card clock.
1533

1534
	// Turn on SDCLK.
1535
	if (sdmmc->regs->hostctl2 & SDHCI_CTRL_VDD_180)
1536
	{
1537
		sdmmc->regs->clkcon |= SDHCI_CLOCK_CARD_EN;
1538
		_sdmmc_commit_changes(sdmmc);
1539
		usleep(1000);
1540
		if ((sdmmc->regs->prnsts & SDHCI_DATA_LVL_MASK) == SDHCI_DATA_LVL_MASK)
1541
			return 1;
1542
	}
1543

1544
	return 0;
1545
}
1546

1547