1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Processor capabilities determination functions.
4
 *
5
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6
 */
7
#include <linux/init.h>
8
#include <linux/kernel.h>
9
#include <linux/ptrace.h>
10
#include <linux/smp.h>
11
#include <linux/stddef.h>
12
#include <linux/export.h>
13
#include <linux/printk.h>
14
#include <linux/uaccess.h>
15

16
#include <asm/cpu-features.h>
17
#include <asm/elf.h>
18
#include <asm/fpu.h>
19
#include <asm/loongarch.h>
20
#include <asm/pgtable-bits.h>
21
#include <asm/setup.h>
22

23
/* Hardware capabilities */
24
unsigned int elf_hwcap __read_mostly;
25
EXPORT_SYMBOL_GPL(elf_hwcap);
26

27
/*
28
 * Determine the FCSR mask for FPU hardware.
29
 */
30
static inline void cpu_set_fpu_fcsr_mask(struct cpuinfo_loongarch *c)
31
{
32
	unsigned long sr, mask, fcsr, fcsr0, fcsr1;
33

34
	fcsr = c->fpu_csr0;
35
	mask = FPU_CSR_ALL_X | FPU_CSR_ALL_E | FPU_CSR_ALL_S | FPU_CSR_RM;
36

37
	sr = read_csr_euen();
38
	enable_fpu();
39

40
	fcsr0 = fcsr & mask;
41
	write_fcsr(LOONGARCH_FCSR0, fcsr0);
42
	fcsr0 = read_fcsr(LOONGARCH_FCSR0);
43

44
	fcsr1 = fcsr | ~mask;
45
	write_fcsr(LOONGARCH_FCSR0, fcsr1);
46
	fcsr1 = read_fcsr(LOONGARCH_FCSR0);
47

48
	write_fcsr(LOONGARCH_FCSR0, fcsr);
49

50
	write_csr_euen(sr);
51

52
	c->fpu_mask = ~(fcsr0 ^ fcsr1) & ~mask;
53
}
54

55
/* simd = -1/0/128/256 */
56
static unsigned int simd = -1U;
57

58
static int __init cpu_setup_simd(char *str)
59
{
60
	get_option(&str, &simd);
61
	pr_info("Set SIMD width = %u\n", simd);
62

63
	return 0;
64
}
65

66
early_param("simd", cpu_setup_simd);
67

68
static int __init cpu_final_simd(void)
69
{
70
	struct cpuinfo_loongarch *c = &cpu_data[0];
71

72
	if (simd < 128) {
73
		c->options &= ~LOONGARCH_CPU_LSX;
74
		elf_hwcap &= ~HWCAP_LOONGARCH_LSX;
75
	}
76

77
	if (simd < 256) {
78
		c->options &= ~LOONGARCH_CPU_LASX;
79
		elf_hwcap &= ~HWCAP_LOONGARCH_LASX;
80
	}
81

82
	simd = 0;
83

84
	if (c->options & LOONGARCH_CPU_LSX)
85
		simd = 128;
86

87
	if (c->options & LOONGARCH_CPU_LASX)
88
		simd = 256;
89

90
	pr_info("Final SIMD width = %u\n", simd);
91

92
	return 0;
93
}
94

95
arch_initcall(cpu_final_simd);
96

97
static inline void set_elf_platform(int cpu, const char *plat)
98
{
99
	if (cpu == 0)
100
		__elf_platform = plat;
101
}
102

103
/* MAP BASE */
104
unsigned long vm_map_base;
105
EXPORT_SYMBOL(vm_map_base);
106

107
static void cpu_probe_addrbits(struct cpuinfo_loongarch *c)
108
{
109
#ifdef __NEED_ADDRBITS_PROBE
110
	c->pabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_PABITS) >> 4;
111
	c->vabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_VABITS) >> 12;
112
	vm_map_base = 0UL - (1UL << c->vabits);
113
#endif
114
}
115

116
static void set_isa(struct cpuinfo_loongarch *c, unsigned int isa)
117
{
118
	switch (isa) {
119
	case LOONGARCH_CPU_ISA_LA64:
120
		c->isa_level |= LOONGARCH_CPU_ISA_LA64;
121
		fallthrough;
122
	case LOONGARCH_CPU_ISA_LA32S:
123
		c->isa_level |= LOONGARCH_CPU_ISA_LA32S;
124
		fallthrough;
125
	case LOONGARCH_CPU_ISA_LA32R:
126
		c->isa_level |= LOONGARCH_CPU_ISA_LA32R;
127
		break;
128
	}
129
}
130

131
static void cpu_probe_common(struct cpuinfo_loongarch *c)
132
{
133
	unsigned int config;
134
	unsigned long asid_mask;
135

136
	c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | LOONGARCH_CPU_VINT;
137

138
	elf_hwcap = HWCAP_LOONGARCH_CPUCFG;
139

140
	config = read_cpucfg(LOONGARCH_CPUCFG1);
141

142
	switch (config & CPUCFG1_ISA) {
143
	case 0:
144
		set_isa(c, LOONGARCH_CPU_ISA_LA32R);
145
		break;
146
	case 1:
147
		set_isa(c, LOONGARCH_CPU_ISA_LA32S);
148
		break;
149
	case 2:
150
		set_isa(c, LOONGARCH_CPU_ISA_LA64);
151
		break;
152
	default:
153
		pr_warn("Warning: unknown ISA level\n");
154
	}
155

156
	if (config & CPUCFG1_PAGING)
157
		c->options |= LOONGARCH_CPU_TLB;
158
	if (config & CPUCFG1_IOCSR)
159
		c->options |= LOONGARCH_CPU_IOCSR;
160
	if (config & CPUCFG1_UAL) {
161
		c->options |= LOONGARCH_CPU_UAL;
162
		elf_hwcap |= HWCAP_LOONGARCH_UAL;
163
	}
164
	if (config & CPUCFG1_CRC32) {
165
		c->options |= LOONGARCH_CPU_CRC32;
166
		elf_hwcap |= HWCAP_LOONGARCH_CRC32;
167
	}
168

169
	config = read_cpucfg(LOONGARCH_CPUCFG2);
170
	if (config & CPUCFG2_LAM) {
171
		c->options |= LOONGARCH_CPU_LAM;
172
		elf_hwcap |= HWCAP_LOONGARCH_LAM;
173
	}
174
	if (config & CPUCFG2_FP) {
175
		c->options |= LOONGARCH_CPU_FPU;
176
		elf_hwcap |= HWCAP_LOONGARCH_FPU;
177
	}
178
#ifdef CONFIG_CPU_HAS_LSX
179
	if ((config & CPUCFG2_LSX) && (simd >= 128)) {
180
		c->options |= LOONGARCH_CPU_LSX;
181
		elf_hwcap |= HWCAP_LOONGARCH_LSX;
182
	}
183
#endif
184
#ifdef CONFIG_CPU_HAS_LASX
185
	if ((config & CPUCFG2_LASX) && (simd >= 256)) {
186
		c->options |= LOONGARCH_CPU_LASX;
187
		elf_hwcap |= HWCAP_LOONGARCH_LASX;
188
	}
189
#endif
190
	if (config & CPUCFG2_COMPLEX) {
191
		c->options |= LOONGARCH_CPU_COMPLEX;
192
		elf_hwcap |= HWCAP_LOONGARCH_COMPLEX;
193
	}
194
	if (config & CPUCFG2_CRYPTO) {
195
		c->options |= LOONGARCH_CPU_CRYPTO;
196
		elf_hwcap |= HWCAP_LOONGARCH_CRYPTO;
197
	}
198
	if (config & CPUCFG2_PTW) {
199
		c->options |= LOONGARCH_CPU_PTW;
200
		elf_hwcap |= HWCAP_LOONGARCH_PTW;
201
	}
202
	if (config & CPUCFG2_LSPW) {
203
		c->options |= LOONGARCH_CPU_LSPW;
204
		elf_hwcap |= HWCAP_LOONGARCH_LSPW;
205
	}
206
	if (config & CPUCFG2_LVZP) {
207
		c->options |= LOONGARCH_CPU_LVZ;
208
		elf_hwcap |= HWCAP_LOONGARCH_LVZ;
209
	}
210
#ifdef CONFIG_CPU_HAS_LBT
211
	if (config & CPUCFG2_X86BT) {
212
		c->options |= LOONGARCH_CPU_LBT_X86;
213
		elf_hwcap |= HWCAP_LOONGARCH_LBT_X86;
214
	}
215
	if (config & CPUCFG2_ARMBT) {
216
		c->options |= LOONGARCH_CPU_LBT_ARM;
217
		elf_hwcap |= HWCAP_LOONGARCH_LBT_ARM;
218
	}
219
	if (config & CPUCFG2_MIPSBT) {
220
		c->options |= LOONGARCH_CPU_LBT_MIPS;
221
		elf_hwcap |= HWCAP_LOONGARCH_LBT_MIPS;
222
	}
223
#endif
224

225
	config = read_cpucfg(LOONGARCH_CPUCFG6);
226
	if (config & CPUCFG6_PMP)
227
		c->options |= LOONGARCH_CPU_PMP;
228

229
	config = csr_read32(LOONGARCH_CSR_ASID);
230
	config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT;
231
	asid_mask = GENMASK(config - 1, 0);
232
	set_cpu_asid_mask(c, asid_mask);
233

234
	config = read_csr_prcfg1();
235
	c->timerbits = (config & CSR_CONF1_TMRBITS) >> CSR_CONF1_TMRBITS_SHIFT;
236
	c->ksave_mask = GENMASK((config & CSR_CONF1_KSNUM) - 1, 0);
237
	c->ksave_mask &= ~(EXC_KSAVE_MASK | PERCPU_KSAVE_MASK | KVM_KSAVE_MASK);
238

239
	config = read_csr_prcfg3();
240
	switch (config & CSR_CONF3_TLBTYPE) {
241
	case 0:
242
		c->tlbsizemtlb = 0;
243
		c->tlbsizestlbsets = 0;
244
		c->tlbsizestlbways = 0;
245
		c->tlbsize = 0;
246
		break;
247
	case 1:
248
		c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1;
249
		c->tlbsizestlbsets = 0;
250
		c->tlbsizestlbways = 0;
251
		c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways;
252
		break;
253
	case 2:
254
		c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1;
255
		c->tlbsizestlbsets = 1 << ((config & CSR_CONF3_STLBIDX) >> CSR_CONF3_STLBIDX_SHIFT);
256
		c->tlbsizestlbways = ((config & CSR_CONF3_STLBWAYS) >> CSR_CONF3_STLBWAYS_SHIFT) + 1;
257
		c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways;
258
		break;
259
	default:
260
		pr_warn("Warning: unknown TLB type\n");
261
	}
262

263
	if (get_num_brps() + get_num_wrps())
264
		c->options |= LOONGARCH_CPU_WATCH;
265
}
266

267
#define MAX_NAME_LEN	32
268
#define VENDOR_OFFSET	0
269
#define CPUNAME_OFFSET	9
270

271
static char cpu_full_name[MAX_NAME_LEN] = "        -        ";
272

273
static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int cpu)
274
{
275
	uint32_t config;
276
	uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]);
277
	uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]);
278
	const char *core_name = "Unknown";
279

280
	switch (BIT(fls(c->isa_level) - 1)) {
281
	case LOONGARCH_CPU_ISA_LA32R:
282
	case LOONGARCH_CPU_ISA_LA32S:
283
		c->cputype = CPU_LOONGSON32;
284
		__cpu_family[cpu] = "Loongson-32bit";
285
		break;
286
	case LOONGARCH_CPU_ISA_LA64:
287
		c->cputype = CPU_LOONGSON64;
288
		__cpu_family[cpu] = "Loongson-64bit";
289
		break;
290
	}
291

292
	switch (c->processor_id & PRID_SERIES_MASK) {
293
	case PRID_SERIES_LA132:
294
		core_name = "LA132";
295
		break;
296
	case PRID_SERIES_LA264:
297
		core_name = "LA264";
298
		break;
299
	case PRID_SERIES_LA364:
300
		core_name = "LA364";
301
		break;
302
	case PRID_SERIES_LA464:
303
		core_name = "LA464";
304
		break;
305
	case PRID_SERIES_LA664:
306
		core_name = "LA664";
307
		break;
308
	}
309

310
	pr_info("%s Processor probed (%s Core)\n", __cpu_family[cpu], core_name);
311

312
	if (!cpu_has_iocsr)
313
		return;
314

315
	if (!__cpu_full_name[cpu])
316
		__cpu_full_name[cpu] = cpu_full_name;
317

318
	*vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR);
319
	*cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME);
320

321
	config = iocsr_read32(LOONGARCH_IOCSR_FEATURES);
322
	if (config & IOCSRF_CSRIPI)
323
		c->options |= LOONGARCH_CPU_CSRIPI;
324
	if (config & IOCSRF_EXTIOI)
325
		c->options |= LOONGARCH_CPU_EXTIOI;
326
	if (config & IOCSRF_FREQSCALE)
327
		c->options |= LOONGARCH_CPU_SCALEFREQ;
328
	if (config & IOCSRF_FLATMODE)
329
		c->options |= LOONGARCH_CPU_FLATMODE;
330
	if (config & IOCSRF_EIODECODE)
331
		c->options |= LOONGARCH_CPU_EIODECODE;
332
	if (config & IOCSRF_AVEC)
333
		c->options |= LOONGARCH_CPU_AVECINT;
334
	if (config & IOCSRF_VM)
335
		c->options |= LOONGARCH_CPU_HYPERVISOR;
336
}
337

338
#ifdef CONFIG_64BIT
339
/* For use by uaccess.h */
340
u64 __ua_limit;
341
EXPORT_SYMBOL(__ua_limit);
342
#endif
343

344
const char *__cpu_family[NR_CPUS];
345
const char *__cpu_full_name[NR_CPUS];
346
const char *__elf_platform;
347

348
static void cpu_report(void)
349
{
350
	struct cpuinfo_loongarch *c = &current_cpu_data;
351

352
	pr_info("CPU%d revision is: %08x (%s)\n",
353
		smp_processor_id(), c->processor_id, cpu_family_string());
354
	if (c->options & LOONGARCH_CPU_FPU)
355
		pr_info("FPU%d revision is: %08x\n", smp_processor_id(), c->fpu_vers);
356
}
357

358
void cpu_probe(void)
359
{
360
	unsigned int cpu = smp_processor_id();
361
	struct cpuinfo_loongarch *c = &current_cpu_data;
362

363
	/*
364
	 * Set a default ELF platform, cpu probe may later
365
	 * overwrite it with a more precise value
366
	 */
367
	set_elf_platform(cpu, "loongarch");
368

369
	c->cputype	= CPU_UNKNOWN;
370
	c->processor_id = read_cpucfg(LOONGARCH_CPUCFG0);
371
	c->fpu_vers     = (read_cpucfg(LOONGARCH_CPUCFG2) & CPUCFG2_FPVERS) >> 3;
372

373
	c->fpu_csr0	= FPU_CSR_RN;
374
	c->fpu_mask	= FPU_CSR_RSVD;
375

376
	cpu_probe_common(c);
377

378
	per_cpu_trap_init(cpu);
379

380
	switch (c->processor_id & PRID_COMP_MASK) {
381
	case PRID_COMP_LOONGSON:
382
		cpu_probe_loongson(c, cpu);
383
		break;
384
	}
385

386
	BUG_ON(!__cpu_family[cpu]);
387
	BUG_ON(c->cputype == CPU_UNKNOWN);
388

389
	cpu_probe_addrbits(c);
390

391
#ifdef CONFIG_64BIT
392
	if (cpu == 0)
393
		__ua_limit = ~((1ull << cpu_vabits) - 1);
394
#endif
395

396
	cpu_report();
397
}
398

399