1
/*
2
 * BPMP-Lite Cache/MMU and Frequency driver for Tegra X1
3
 *
4
 * Copyright (c) 2019-2024 CTCaer
5
 *
6
 * This program is free software; you can redistribute it and/or modify it
7
 * under the terms and conditions of the GNU General Public License,
8
 * version 2, as published by the Free Software Foundation.
9
 *
10
 * This program is distributed in the hope it will be useful, but WITHOUT
11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13
 * more details.
14
 *
15
 * You should have received a copy of the GNU General Public License
16
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17
 */
18

19
#include <soc/bpmp.h>
20
#include <soc/clock.h>
21
#include <soc/timer.h>
22
#include <soc/t210.h>
23
#include <memory_map.h>
24

25
#define BPMP_MMU_CACHE_LINE_SIZE        0x20
26

27
#define BPMP_CACHE_CONFIG               0x0
28
#define  CFG_ENABLE_CACHE               BIT(0)
29
#define  CFG_ENABLE_SKEW_ASSOC          BIT(1)
30
#define  CFG_DISABLE_RANDOM_ALLOC       BIT(2)
31
#define  CFG_FORCE_WRITE_THROUGH        BIT(3)
32
#define  CFG_NEVER_ALLOCATE             BIT(6)
33
#define  CFG_ENABLE_INTERRUPT           BIT(7)
34
#define  CFG_MMU_TAG_MODE(x)            ((x) << 8)
35
#define   TAG_MODE_PARALLEL             0
36
#define   TAG_MODE_TAG_FIRST            1
37
#define   TAG_MODE_MMU_FIRST            2
38
#define  CFG_DISABLE_WRITE_BUFFER       BIT(10)
39
#define  CFG_DISABLE_READ_BUFFER        BIT(11)
40
#define  CFG_ENABLE_HANG_DETECT         BIT(12)
41
#define  CFG_FULL_LINE_DIRTY            BIT(13)
42
#define  CFG_TAG_CHK_ABRT_ON_ERR        BIT(14)
43
#define  CFG_TAG_CHK_CLR_ERR            BIT(15)
44
#define  CFG_DISABLE_SAMELINE           BIT(16)
45
#define  CFG_OBS_BUS_EN                 BIT(31)
46

47
#define BPMP_CACHE_LOCK                 0x4
48
#define  LOCK_LINE(x)                   BIT((x))
49

50
#define BPMP_CACHE_SIZE                 0xC
51
#define BPMP_CACHE_LFSR                 0x10
52

53
#define BPMP_CACHE_TAG_STATUS           0x14
54
#define  TAG_STATUS_TAG_CHECK_ERROR     BIT(0)
55
#define  TAG_STATUS_CONFLICT_ADDR_MASK  0xFFFFFFE0
56

57
#define BPMP_CACHE_CLKEN_OVERRIDE       0x18
58
#define  CLKEN_OVERRIDE_WR_MCCIF_CLKEN  BIT(0)
59
#define  CLKEN_OVERRIDE_RD_MCCIF_CLKEN  BIT(1)
60

61
#define BPMP_CACHE_MAINT_ADDR           0x20
62
#define BPMP_CACHE_MAINT_DATA           0x24
63

64
#define BPMP_CACHE_MAINT_REQ            0x28
65
#define  MAINT_REQ_WAY_BITMAP(x)        ((x) << 8)
66

67
#define BPMP_CACHE_INT_MASK             0x40
68
#define BPMP_CACHE_INT_CLEAR            0x44
69
#define BPMP_CACHE_INT_RAW_EVENT        0x48
70
#define BPMP_CACHE_INT_STATUS           0x4C
71
#define  INT_MAINT_DONE                 BIT(0)
72
#define  INT_MAINT_ERROR                BIT(1)
73

74
#define BPMP_CACHE_RB_CFG               0x80
75
#define BPMP_CACHE_WB_CFG               0x84
76

77
#define BPMP_CACHE_MMU_FALLBACK_ENTRY   0xA0
78
#define BPMP_CACHE_MMU_SHADOW_COPY_MASK 0xA4
79

80
#define BPMP_CACHE_MMU_CFG              0xAC
81
#define  MMU_CFG_BLOCK_MAIN_ENTRY_WR    BIT(0)
82
#define  MMU_CFG_SEQ_EN                 BIT(1)
83
#define  MMU_CFG_TLB_EN                 BIT(2)
84
#define  MMU_CFG_SEG_CHECK_ALL_ENTRIES  BIT(3)
85
#define  MMU_CFG_ABORT_STORE_LAST       BIT(4)
86
#define  MMU_CFG_CLR_ABORT              BIT(5)
87

88
#define BPMP_CACHE_MMU_CMD              0xB0
89
#define  MMU_CMD_NOP                    0
90
#define  MMU_CMD_INIT                   1
91
#define  MMU_CMD_COPY_SHADOW            2
92

93
#define BPMP_CACHE_MMU_ABORT_STAT       0xB4
94
#define  ABORT_STAT_UNIT_MASK           0x7
95
#define  ABORT_STAT_UNIT_NONE           0
96
#define  ABORT_STAT_UNIT_CACHE          1
97
#define  ABORT_STAT_UNIT_SEQ            2
98
#define  ABORT_STAT_UNIT_TLB            3
99
#define  ABORT_STAT_UNIT_SEG            4
100
#define  ABORT_STAT_UNIT_FALLBACK       5
101
#define  ABORT_STAT_OVERLAP             BIT(3)
102
#define  ABORT_STAT_ENTRY               (0x1F << 4)
103
#define  ABORT_STAT_TYPE_MASK           (3 << 16)
104
#define  ABORT_STAT_TYPE_EXE            (0 << 16)
105
#define  ABORT_STAT_TYPE_RD             (1 << 16)
106
#define  ABORT_STAT_TYPE_WR             (2 << 16)
107
#define  ABORT_STAT_SIZE                (3 << 18)
108
#define  ABORT_STAT_SEQ                 BIT(20)
109
#define  ABORT_STAT_PROT                BIT(21)
110

111
#define BPMP_CACHE_MMU_ABORT_ADDR       0xB8
112
#define BPMP_CACHE_MMU_ACTIVE_ENTRIES   0xBC
113

114
#define BPMP_MMU_SHADOW_ENTRY_BASE      (BPMP_CACHE_BASE + 0x400)
115
#define BPMP_MMU_MAIN_ENTRY_BASE        (BPMP_CACHE_BASE + 0x800)
116
#define  MMU_EN_CACHED                  BIT(0)
117
#define  MMU_EN_EXEC                    BIT(1)
118
#define  MMU_EN_READ                    BIT(2)
119
#define  MMU_EN_WRITE                   BIT(3)
120

121
static const bpmp_mmu_entry_t mmu_entries[] =
122
{
123
	{ DRAM_START, 0xFFFFFFFF, MMU_EN_READ | MMU_EN_WRITE | MMU_EN_EXEC | MMU_EN_CACHED, true },
124
	{ IRAM_BASE,  0x4003FFFF, MMU_EN_READ | MMU_EN_WRITE | MMU_EN_EXEC | MMU_EN_CACHED, true }
125
};
126

127
void bpmp_mmu_maintenance(u32 op, bool force)
128
{
129
	if (!force && !(BPMP_CACHE_CTRL(BPMP_CACHE_CONFIG) & CFG_ENABLE_CACHE))
130
		return;
131

132
	BPMP_CACHE_CTRL(BPMP_CACHE_INT_CLEAR) = INT_MAINT_DONE;
133

134
	// This is a blocking operation.
135
	BPMP_CACHE_CTRL(BPMP_CACHE_MAINT_REQ) = MAINT_REQ_WAY_BITMAP(0xF) | op;
136

137
	while (!(BPMP_CACHE_CTRL(BPMP_CACHE_INT_RAW_EVENT) & INT_MAINT_DONE))
138
		;
139

140
	BPMP_CACHE_CTRL(BPMP_CACHE_INT_CLEAR) = BPMP_CACHE_CTRL(BPMP_CACHE_INT_RAW_EVENT);
141
}
142

143
void bpmp_mmu_set_entry(int idx, const bpmp_mmu_entry_t *entry, bool apply)
144
{
145
	if (idx > 31)
146
		return;
147

148
	volatile bpmp_mmu_entry_t *mmu_entry = (bpmp_mmu_entry_t *)(BPMP_MMU_SHADOW_ENTRY_BASE + sizeof(bpmp_mmu_entry_t) * idx);
149

150
	if (entry->enable)
151
	{
152
		mmu_entry->start_addr = ALIGN(entry->start_addr, BPMP_MMU_CACHE_LINE_SIZE);
153
		mmu_entry->end_addr   = ALIGN_DOWN(entry->end_addr, BPMP_MMU_CACHE_LINE_SIZE);
154
		mmu_entry->attr       = entry->attr;
155

156
		BPMP_CACHE_CTRL(BPMP_CACHE_MMU_SHADOW_COPY_MASK) |= BIT(idx);
157

158
		if (apply)
159
			BPMP_CACHE_CTRL(BPMP_CACHE_MMU_CMD) = MMU_CMD_COPY_SHADOW;
160
	}
161
}
162

163
void bpmp_mmu_enable()
164
{
165
	if (BPMP_CACHE_CTRL(BPMP_CACHE_CONFIG) & CFG_ENABLE_CACHE)
166
		return;
167

168
	// Init BPMP MMU.
169
	BPMP_CACHE_CTRL(BPMP_CACHE_MMU_CMD)            = MMU_CMD_INIT;
170
	BPMP_CACHE_CTRL(BPMP_CACHE_MMU_FALLBACK_ENTRY) = MMU_EN_READ | MMU_EN_WRITE | MMU_EN_EXEC; // RWX for non-defined regions.
171
	BPMP_CACHE_CTRL(BPMP_CACHE_MMU_CFG)            = MMU_CFG_SEQ_EN | MMU_CFG_TLB_EN | MMU_CFG_ABORT_STORE_LAST;
172

173
	// Init BPMP MMU entries.
174
	BPMP_CACHE_CTRL(BPMP_CACHE_MMU_SHADOW_COPY_MASK) = 0;
175
	for (u32 idx = 0; idx < ARRAY_SIZE(mmu_entries); idx++)
176
		bpmp_mmu_set_entry(idx, &mmu_entries[idx], false);
177

178
	BPMP_CACHE_CTRL(BPMP_CACHE_MMU_CMD) = MMU_CMD_COPY_SHADOW;
179

180
	// Invalidate cache.
181
	bpmp_mmu_maintenance(BPMP_MMU_MAINT_INVALID_WAY, true);
182

183
	// Enable cache.
184
	BPMP_CACHE_CTRL(BPMP_CACHE_CONFIG) = CFG_ENABLE_CACHE | CFG_FORCE_WRITE_THROUGH |
185
		CFG_MMU_TAG_MODE(TAG_MODE_PARALLEL) | CFG_TAG_CHK_ABRT_ON_ERR;
186

187
	// HW bug. Invalidate cache again.
188
	bpmp_mmu_maintenance(BPMP_MMU_MAINT_INVALID_WAY, false);
189
}
190

191
void bpmp_mmu_disable()
192
{
193
	if (!(BPMP_CACHE_CTRL(BPMP_CACHE_CONFIG) & CFG_ENABLE_CACHE))
194
		return;
195

196
	// Clean and invalidate cache.
197
	bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
198

199
	// Disable cache.
200
	BPMP_CACHE_CTRL(BPMP_CACHE_CONFIG) = 0;
201
}
202

203

204
/*
205
 * CLK_RST_CONTROLLER_SCLK_BURST_POLICY:
206
 * 0 = CLKM
207
 * 1 = PLLC_OUT1
208
 * 2 = PLLC4_OUT3
209
 * 3 = PLLP_OUT0
210
 * 4 = PLLP_OUT2
211
 * 5 = PLLC4_OUT1
212
 * 6 = CLK_S
213
 * 7 = PLLC4_OUT2
214
 */
215

216
bpmp_freq_t bpmp_fid_current = BPMP_CLK_NORMAL;
217

218
void bpmp_clk_rate_relaxed(bool enable)
219
{
220
	// This is a glitch-free way to reduce the SCLK timings.
221
	if (enable)
222
	{
223
		// Restore to PLLP source during PLLC configuration.
224
		CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = 0x20003330; // PLLP_OUT.
225
		usleep(100); // Wait a bit for clock source change.
226
		CLOCK(CLK_RST_CONTROLLER_CLK_SYSTEM_RATE) = 2; // PCLK = HCLK / (2 + 1). HCLK == SCLK.
227
	}
228
	else if (bpmp_fid_current)
229
	{
230
		// Restore to PLLC_OUT1.
231
		CLOCK(CLK_RST_CONTROLLER_CLK_SYSTEM_RATE) = 3; // PCLK = HCLK / (3 + 1). HCLK == SCLK.
232
		CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = 0x20003310; // PLLC_OUT1 and CLKM for idle.
233
		usleep(100); // Wait a bit for clock source change.
234
	}
235
}
236

237
// APB clock affects RTC, PWM, MEMFETCH, APE, USB, SOR PWM,
238
// I2C host, DC/DSI/DISP. UART gives extra stress.
239
// 92: 100% success ratio. 93-94: 595-602MHz has 99% success ratio. 95: 608MHz less.
240
// APB clock max is supposed to be 204 MHz though.
241
static const u8 pll_divn[] = {
242
	0,   // BPMP_CLK_NORMAL:      408MHz  0% - 136MHz APB.
243
	85,  // BPMP_CLK_HIGH_BOOST:  544MHz 33% - 136MHz APB.
244
	88,  // BPMP_CLK_HIGH2_BOOST: 563MHz 38% - 141MHz APB.
245
	90,  // BPMP_CLK_SUPER_BOOST: 576MHz 41% - 144MHz APB.
246
	92   // BPMP_CLK_HYPER_BOOST: 589MHz 44% - 147MHz APB.
247
	// Do not use for public releases!
248
	//95   // BPMP_CLK_DEV_BOOST: 608MHz 49% - 152MHz APB.
249
};
250

251
void bpmp_clk_rate_get()
252
{
253
	bool clk_src_is_pllp = ((CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) >> 4) & 7) == 3;
254

255
	if (clk_src_is_pllp)
256
		bpmp_fid_current = BPMP_CLK_NORMAL;
257
	else
258
	{
259
		bpmp_fid_current = BPMP_CLK_HIGH_BOOST;
260

261
		u8 pll_divn_curr = (CLOCK(CLK_RST_CONTROLLER_PLLC_BASE) >> 10) & 0xFF;
262
		for (u32 i = 1; i < sizeof(pll_divn); i++)
263
		{
264
			if (pll_divn[i] == pll_divn_curr)
265
			{
266
				bpmp_fid_current = i;
267
				break;
268
			}
269
		}
270
	}
271
}
272

273
void bpmp_clk_rate_set(bpmp_freq_t fid)
274
{
275
	if (fid > (BPMP_CLK_MAX - 1))
276
		fid = BPMP_CLK_MAX - 1;
277

278
	if (bpmp_fid_current == fid)
279
		return;
280

281
	bpmp_fid_current = fid;
282

283
	if (fid)
284
	{
285
		// Use default SCLK / HCLK / PCLK clocks.
286
		bpmp_clk_rate_relaxed(true);
287

288
		// Configure and enable PLLC.
289
		clock_enable_pllc(pll_divn[fid]);
290

291
		// Set new source and SCLK / HCLK / PCLK dividers.
292
		bpmp_clk_rate_relaxed(false);
293
	}
294
	else
295
	{
296
		// Use default SCLK / HCLK / PCLK clocks.
297
		bpmp_clk_rate_relaxed(true);
298

299
		// Disable PLLC to save power.
300
		clock_disable_pllc();
301
	}
302
}
303

304
// State is reset to RUN on any clock or source set via SW.
305
void bpmp_state_set(bpmp_state_t state)
306
{
307
	u32 cfg = CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) & ~0xF0000000u;
308
	CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = cfg | (state << 28u);
309
}
310

311
// The following functions halt BPMP to reduce power while sleeping.
312
// They are not as accurate as RTC at big values but they guarantee time+ delay.
313
void bpmp_usleep(u32 us)
314
{
315
	u32 delay;
316

317
	// Each iteration takes 1us.
318
	while (us)
319
	{
320
		delay = (us > HALT_MAX_CNT) ? HALT_MAX_CNT : us;
321
		us -= delay;
322

323
		FLOW_CTLR(FLOW_CTLR_HALT_COP_EVENTS) = HALT_MODE_WAITEVENT | HALT_USEC | delay;
324
	}
325
}
326

327
void bpmp_msleep(u32 ms)
328
{
329
	u32 delay;
330

331
	// Iteration time is variable. ~200 - 1000us.
332
	while (ms)
333
	{
334
		delay = (ms > HALT_MAX_CNT) ? HALT_MAX_CNT : ms;
335
		ms -= delay;
336

337
		FLOW_CTLR(FLOW_CTLR_HALT_COP_EVENTS) = HALT_MODE_WAITEVENT | HALT_MSEC | delay;
338
	}
339
}
340

341
void bpmp_halt()
342
{
343
	FLOW_CTLR(FLOW_CTLR_HALT_COP_EVENTS) = HALT_MODE_WAITEVENT | HALT_JTAG;
344
}
345

346