1
/*
2
 * Copyright (c) 2018 naehrwert
3
 * Copyright (c) 2018 balika011
4
 * Copyright (c) 2018-2024 CTCaer
5
 *
6
 * This program is free software; you can redistribute it and/or modify it
7
 * under the terms and conditions of the GNU General Public License,
8
 * version 2, as published by the Free Software Foundation.
9
 *
10
 * This program is distributed in the hope it will be useful, but WITHOUT
11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13
 * more details.
14
 *
15
 * You should have received a copy of the GNU General Public License
16
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17
 */
18

19
#include <string.h>
20

21
#include <soc/bpmp.h>
22
#include <soc/ccplex.h>
23
#include <soc/timer.h>
24
#include <soc/t210.h>
25
#include <mem/mc_t210.h>
26
#include <mem/smmu.h>
27
#include <memory_map.h>
28

29
/*! SMMU register defines */
30
#define SMMU_ASID(asid)          (((asid) << 24u) | ((asid) << 16u) | ((asid) << 8u) | (asid))
31
#define SMMU_ENABLE              BIT(31)
32
#define SMMU_TLB_ACTIVE_LINES(l) ((l) << 0u)
33
#define SMMU_TLB_RR_ARBITRATION  BIT(28)
34
#define SMMU_TLB_HIT_UNDER_MISS  BIT(29)
35
#define SMMU_TLB_STATS_ENABLE    BIT(31)
36
#define SMUU_PTC_INDEX_MAP(m)    ((m) << 0u)
37
#define SMUU_PTC_LINE_MASK(m)    ((m) << 8u)
38
#define SMUU_PTC_REQ_LIMIT(l)    ((l) << 24u)
39
#define SMUU_PTC_CACHE_ENABLE    BIT(29)
40
#define SMUU_PTC_STATS_ENABLE    BIT(31)
41

42
/*! Page table defines */
43
#define SMMU_4MB_REGION 0
44
#define SMMU_PAGE_TABLE 1
45
#define SMMU_PDIR_COUNT 1024
46
#define SMMU_PTBL_COUNT 1024
47
#define SMMU_PAGE_SHIFT 12u
48
#define SMMU_PTN_SHIFT  SMMU_PAGE_SHIFT
49
#define SMMU_PDN_SHIFT  22u
50
#define SMMU_ADDR_TO_PFN(addr) ((addr) >> SMMU_PAGE_SHIFT)
51
#define SMMU_ADDR_TO_PTN(addr) ((addr) >> SMMU_PTN_SHIFT)
52
#define SMMU_ADDR_TO_PDN(addr) ((addr) >> SMMU_PDN_SHIFT)
53
#define SMMU_PTN_TO_ADDR(ptn)  ((ptn)  << SMMU_PTN_SHIFT)
54
#define SMMU_PDN_TO_ADDR(pdn)  ((pdn)  << SMMU_PDN_SHIFT)
55
#define SMMU_PTB(page, attr) (((attr) << 29u) | ((page) >> SMMU_PAGE_SHIFT))
56

57
static void *smmu_heap = (void *)SMMU_HEAP_ADDR;
58

59
// Enabling SMMU requires a TZ (EL3) secure write. MC(MC_SMMU_CONFIG) = 1;
60
static const u8 smmu_enable_payload[] = {
61
	0xC1, 0x00, 0x00, 0x18, // 0x00: LDR  W1, =0x70019010
62
	0x20, 0x00, 0x80, 0xD2, // 0x04: MOV  X0, #0x1
63
	0x20, 0x00, 0x00, 0xB9, // 0x08: STR  W0, [X1]
64
	0x1F, 0x71, 0x08, 0xD5, // 0x0C: IC   IALLUIS
65
	0x9F, 0x3B, 0x03, 0xD5, // 0x10: DSB  ISH
66
	0xFE, 0xFF, 0xFF, 0x17, // 0x14: B    loop
67
	0x10, 0x90, 0x01, 0x70, // 0x18: MC_SMMU_CONFIG
68
};
69

70
void *smmu_page_zalloc(u32 num)
71
{
72
	void *page = smmu_heap;
73
	memset(page, 0, SZ_PAGE * num);
74

75
	smmu_heap += SZ_PAGE * num;
76

77
	return page;
78
}
79

80
static pde_t *_smmu_pdir_alloc()
81
{
82
	pde_t *pdir = (pde_t *)smmu_page_zalloc(1);
83

84
	// Initialize pdes with no permissions.
85
	for (u32 pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
86
		pdir[pdn].huge.page = pdn;
87

88
	return pdir;
89
}
90

91
static void _smmu_flush_regs()
92
{
93
	(void)MC(MC_SMMU_PTB_DATA);
94
}
95

96
void smmu_flush_all()
97
{
98

99
	// Flush the entire page table cache.
100
	MC(MC_SMMU_PTC_FLUSH) = 0;
101
	_smmu_flush_regs();
102

103
	// Flush the entire table.
104
	MC(MC_SMMU_TLB_FLUSH) = 0;
105
	_smmu_flush_regs();
106
}
107

108
void smmu_init()
109
{
110
	MC(MC_SMMU_PTB_ASID)   = 0;
111
	MC(MC_SMMU_PTB_DATA)   = 0;
112
	MC(MC_SMMU_TLB_CONFIG) = SMMU_TLB_HIT_UNDER_MISS | SMMU_TLB_RR_ARBITRATION | SMMU_TLB_ACTIVE_LINES(48);
113
	MC(MC_SMMU_PTC_CONFIG) = SMUU_PTC_CACHE_ENABLE | SMUU_PTC_REQ_LIMIT(8) | SMUU_PTC_LINE_MASK(0xF) | SMUU_PTC_INDEX_MAP(0x3F);
114
	MC(MC_SMMU_PTC_FLUSH)  = 0;
115
	MC(MC_SMMU_TLB_FLUSH)  = 0;
116
}
117

118
void smmu_enable()
119
{
120
	static bool enabled = false;
121

122
	if (enabled)
123
		return;
124

125
	// Launch payload on CCPLEX in order to set SMMU enable bit.
126
	ccplex_boot_cpu0((u32)smmu_enable_payload, false);
127
	msleep(100);
128
	ccplex_powergate_cpu0();
129

130
	smmu_flush_all();
131

132
	enabled = true;
133
}
134

135
void smmu_reset_heap()
136
{
137
	smmu_heap = (void *)SMMU_HEAP_ADDR;
138
}
139

140
void *smmu_init_domain(u32 dev_base, u32 asid)
141
{
142
	void *ptb = _smmu_pdir_alloc();
143

144
	MC(MC_SMMU_PTB_ASID) = asid;
145
	MC(MC_SMMU_PTB_DATA) = SMMU_PTB((u32)ptb, SMMU_ATTR_ALL);
146
	_smmu_flush_regs();
147

148
	// Use the same macro for both quad and single domains. Reserved bits are not set anyway.
149
	MC(dev_base) = SMMU_ENABLE | SMMU_ASID(asid);
150
	_smmu_flush_regs();
151

152
	return ptb;
153
}
154

155
void smmu_deinit_domain(u32 dev_base, u32 asid)
156
{
157
	MC(MC_SMMU_PTB_ASID) = asid;
158
	MC(MC_SMMU_PTB_DATA) = 0;
159
	MC(dev_base)         = 0;
160
	_smmu_flush_regs();
161
}
162

163
void smmu_domain_bypass(u32 dev_base, bool bypass)
164
{
165
	if (bypass)
166
	{
167
		smmu_flush_all();
168
		bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
169
		MC(dev_base) &= ~SMMU_ENABLE;
170
	}
171
	else
172
	{
173
		bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
174
		MC(dev_base) |=  SMMU_ENABLE;
175
		smmu_flush_all();
176
	}
177
	_smmu_flush_regs();
178
}
179

180
static pte_t *_smmu_get_pte(pde_t *pdir, u32 iova)
181
{
182
	u32 pdn = SMMU_ADDR_TO_PDN(iova);
183
	pte_t *ptbl;
184

185
	// Get 4MB page table or initialize one.
186
	if (pdir[pdn].tbl.attr)
187
		ptbl = (pte_t *)(SMMU_PTN_TO_ADDR(pdir[pdn].tbl.table));
188
	else
189
	{
190
		// Allocate page table.
191
		ptbl = (pte_t *)smmu_page_zalloc(1);
192

193
		// Get address.
194
		u32 addr = SMMU_PDN_TO_ADDR(pdn);
195

196
		// Initialize page table with no permissions.
197
		for (u32 pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SZ_PAGE)
198
			ptbl[pn].page = SMMU_ADDR_TO_PFN(addr);
199

200
		// Set page table to the page directory.
201
		pdir[pdn].tbl.table = SMMU_ADDR_TO_PTN((u32)ptbl);
202
		pdir[pdn].tbl.next  = SMMU_PAGE_TABLE;
203
		pdir[pdn].tbl.attr  = SMMU_ATTR_ALL;
204

205
		smmu_flush_all();
206
	}
207

208
	return &ptbl[SMMU_ADDR_TO_PTN(iova) % SMMU_PTBL_COUNT];
209
}
210

211
void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr)
212
{
213
	// Map pages to page table entries. VA/PA should be aligned to 4KB.
214
	for (u32 i = 0; i < pages; i++)
215
	{
216
		pte_t *pte = _smmu_get_pte((pde_t *)ptb, iova);
217

218
		pte->page = SMMU_ADDR_TO_PFN(iopa);
219
		pte->attr = attr;
220

221
		iova += SZ_PAGE;
222
		iopa += SZ_PAGE;
223
	}
224

225
	smmu_flush_all();
226
}
227

228
void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr)
229
{
230
	pde_t *pdir = (pde_t *)ptb;
231

232
	// Map 4MB regions to page directory entries. VA/PA should be aligned to 4MB.
233
	for (u32 i = 0; i < regions; i++)
234
	{
235
		u32 pdn = SMMU_ADDR_TO_PDN(iova);
236
		pdir[pdn].huge.page = SMMU_ADDR_TO_PDN(iopa);
237
		pdir[pdn].huge.next = SMMU_4MB_REGION;
238
		pdir[pdn].huge.attr = attr;
239

240
		iova += SZ_4M;
241
		iopa += SZ_4M;
242
	}
243

244
	smmu_flush_all();
245
}
246

247