1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Support for Kernel relocation at boot time
4
 *
5
 * Copyright (C) 2023 Loongson Technology Corporation Limited
6
 */
7

8
#include <linux/elf.h>
9
#include <linux/kernel.h>
10
#include <linux/printk.h>
11
#include <linux/panic_notifier.h>
12
#include <linux/start_kernel.h>
13
#include <asm/bootinfo.h>
14
#include <asm/early_ioremap.h>
15
#include <asm/inst.h>
16
#include <asm/io.h>
17
#include <asm/sections.h>
18
#include <asm/setup.h>
19

20
#define RELOCATED(x) ((void *)((long)x + reloc_offset))
21
#define RELOCATED_KASLR(x) ((void *)((long)x + random_offset))
22

23
static unsigned long reloc_offset;
24

25
static inline void __init relocate_relative(void)
26
{
27
	Elf64_Rela *rela, *rela_end;
28
	rela = (Elf64_Rela *)&__rela_dyn_begin;
29
	rela_end = (Elf64_Rela *)&__rela_dyn_end;
30

31
	for ( ; rela < rela_end; rela++) {
32
		Elf64_Addr addr = rela->r_offset;
33
		Elf64_Addr relocated_addr = rela->r_addend;
34

35
		if (rela->r_info != R_LARCH_RELATIVE)
36
			continue;
37

38
		relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr);
39
		*(Elf64_Addr *)RELOCATED(addr) = relocated_addr;
40
	}
41

42
#ifdef CONFIG_RELR
43
	u64 *addr = NULL;
44
	u64 *relr = (u64 *)&__relr_dyn_begin;
45
	u64 *relr_end = (u64 *)&__relr_dyn_end;
46

47
	for ( ; relr < relr_end; relr++) {
48
		if ((*relr & 1) == 0) {
49
			addr = (u64 *)(*relr + reloc_offset);
50
			*addr++ += reloc_offset;
51
		} else {
52
			for (u64 *p = addr, r = *relr >> 1; r; p++, r >>= 1)
53
				if (r & 1)
54
					*p += reloc_offset;
55
			addr += 63;
56
		}
57
	}
58
#endif
59
}
60

61
static inline void __init relocate_absolute(long random_offset)
62
{
63
	void *begin, *end;
64
	struct rela_la_abs *p;
65

66
	begin = RELOCATED_KASLR(&__la_abs_begin);
67
	end   = RELOCATED_KASLR(&__la_abs_end);
68

69
	for (p = begin; (void *)p < end; p++) {
70
		long v = p->symvalue;
71
		uint32_t lu12iw, ori, lu32id, lu52id;
72
		union loongarch_instruction *insn = (void *)p->pc;
73

74
		lu12iw = (v >> 12) & 0xfffff;
75
		ori    = v & 0xfff;
76
		lu32id = (v >> 32) & 0xfffff;
77
		lu52id = v >> 52;
78

79
		insn[0].reg1i20_format.immediate = lu12iw;
80
		insn[1].reg2i12_format.immediate = ori;
81
		insn[2].reg1i20_format.immediate = lu32id;
82
		insn[3].reg2i12_format.immediate = lu52id;
83
	}
84
}
85

86
#ifdef CONFIG_RANDOMIZE_BASE
87
static inline __init unsigned long rotate_xor(unsigned long hash,
88
					      const void *area, size_t size)
89
{
90
	size_t i, diff;
91
	const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash));
92

93
	diff = (void *)ptr - area;
94
	if (size < diff + sizeof(hash))
95
		return hash;
96

97
	size = ALIGN_DOWN(size - diff, sizeof(hash));
98

99
	for (i = 0; i < size / sizeof(hash); i++) {
100
		/* Rotate by odd number of bits and XOR. */
101
		hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
102
		hash ^= ptr[i];
103
	}
104

105
	return hash;
106
}
107

108
static inline __init unsigned long get_random_boot(void)
109
{
110
	unsigned long hash = 0;
111
	unsigned long entropy = random_get_entropy();
112

113
	/* Attempt to create a simple but unpredictable starting entropy. */
114
	hash = rotate_xor(hash, linux_banner, strlen(linux_banner));
115

116
	/* Add in any runtime entropy we can get */
117
	hash = rotate_xor(hash, &entropy, sizeof(entropy));
118

119
	return hash;
120
}
121

122
static int __init nokaslr(char *p)
123
{
124
	pr_info("KASLR is disabled.\n");
125

126
	return 0; /* Print a notice and silence the boot warning */
127
}
128
early_param("nokaslr", nokaslr);
129

130
static inline __init bool kaslr_disabled(void)
131
{
132
	char *str;
133
	const char *builtin_cmdline = CONFIG_CMDLINE;
134

135
	str = strstr(builtin_cmdline, "nokaslr");
136
	if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
137
		return true;
138

139
	str = strstr(boot_command_line, "nokaslr");
140
	if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
141
		return true;
142

143
#ifdef CONFIG_HIBERNATION
144
	str = strstr(builtin_cmdline, "nohibernate");
145
	if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
146
		return false;
147

148
	str = strstr(boot_command_line, "nohibernate");
149
	if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
150
		return false;
151

152
	str = strstr(builtin_cmdline, "noresume");
153
	if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
154
		return false;
155

156
	str = strstr(boot_command_line, "noresume");
157
	if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
158
		return false;
159

160
	str = strstr(builtin_cmdline, "resume=");
161
	if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
162
		return true;
163

164
	str = strstr(boot_command_line, "resume=");
165
	if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
166
		return true;
167
#endif
168

169
	str = strstr(boot_command_line, "kexec_file");
170
	if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
171
		return true;
172

173
	return false;
174
}
175

176
/* Choose a new address for the kernel */
177
static inline void __init *determine_relocation_address(void)
178
{
179
	unsigned long kernel_length;
180
	unsigned long random_offset;
181
	void *destination = _text;
182

183
	if (kaslr_disabled())
184
		return destination;
185

186
	kernel_length = (long)_end - (long)_text;
187

188
	random_offset = get_random_boot() << 16;
189
	random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1);
190
	if (random_offset < kernel_length)
191
		random_offset += ALIGN(kernel_length, 0xffff);
192

193
	return RELOCATED_KASLR(destination);
194
}
195

196
static inline int __init relocation_addr_valid(void *location_new)
197
{
198
	if ((unsigned long)location_new & 0x00000ffff)
199
		return 0; /* Inappropriately aligned new location */
200

201
	if ((unsigned long)location_new < (unsigned long)_end)
202
		return 0; /* New location overlaps original kernel */
203

204
	return 1;
205
}
206
#endif
207

208
static inline void __init update_reloc_offset(unsigned long *addr, long random_offset)
209
{
210
	unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr);
211

212
	*new_addr = (unsigned long)reloc_offset;
213
}
214

215
unsigned long __init relocate_kernel(void)
216
{
217
	unsigned long kernel_length;
218
	unsigned long random_offset = 0;
219
	void *location_new = _text; /* Default to original kernel start */
220
	char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */
221

222
	strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE);
223

224
#ifdef CONFIG_RANDOMIZE_BASE
225
	location_new = determine_relocation_address();
226

227
	/* Sanity check relocation address */
228
	if (relocation_addr_valid(location_new))
229
		random_offset = (unsigned long)location_new - (unsigned long)(_text);
230
#endif
231
	reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS;
232
	early_memunmap(cmdline, COMMAND_LINE_SIZE);
233

234
	if (random_offset) {
235
		kernel_length = (long)(_end) - (long)(_text);
236

237
		/* Copy the kernel to it's new location */
238
		memcpy(location_new, _text, kernel_length);
239

240
		/* Sync the caches ready for execution of new kernel */
241
		__asm__ __volatile__ (
242
			"ibar 0 \t\n"
243
			"dbar 0 \t\n"
244
			::: "memory");
245

246
		reloc_offset += random_offset;
247

248
		/* The current thread is now within the relocated kernel */
249
		__current_thread_info = RELOCATED_KASLR(__current_thread_info);
250

251
		update_reloc_offset(&reloc_offset, random_offset);
252
	}
253

254
	if (reloc_offset)
255
		relocate_relative();
256

257
	relocate_absolute(random_offset);
258

259
	return random_offset;
260
}
261

262
/*
263
 * Show relocation information on panic.
264
 */
265
static void show_kernel_relocation(const char *level)
266
{
267
	if (reloc_offset > 0) {
268
		printk(level);
269
		pr_cont("Kernel relocated by 0x%lx\n", reloc_offset);
270
		pr_cont(" .text @ 0x%px\n", _text);
271
		pr_cont(" .data @ 0x%px\n", _sdata);
272
		pr_cont(" .bss  @ 0x%px\n", __bss_start);
273
	}
274
}
275

276
static int kernel_location_notifier_fn(struct notifier_block *self,
277
				       unsigned long v, void *p)
278
{
279
	show_kernel_relocation(KERN_EMERG);
280
	return NOTIFY_DONE;
281
}
282

283
static struct notifier_block kernel_location_notifier = {
284
	.notifier_call = kernel_location_notifier_fn
285
};
286

287
static int __init register_kernel_offset_dumper(void)
288
{
289
	atomic_notifier_chain_register(&panic_notifier_list,
290
				       &kernel_location_notifier);
291
	return 0;
292
}
293

294
arch_initcall(register_kernel_offset_dumper);
295

296