1
// SPDX-License-Identifier: GPL-2.0-only
2
/*
3
 * BPF JIT compiler for LoongArch
4
 *
5
 * Copyright (C) 2022 Loongson Technology Corporation Limited
6
 */
7
#include <linux/memory.h>
8
#include "bpf_jit.h"
9

10
#define LOONGARCH_MAX_REG_ARGS 8
11

12
#define LOONGARCH_LONG_JUMP_NINSNS 5
13
#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
14

15
#define LOONGARCH_FENTRY_NINSNS 2
16
#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
17
#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
18

19
#define REG_TCC		LOONGARCH_GPR_A6
20
#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
21

22
static const int regmap[] = {
23
	/* return value from in-kernel function, and exit value for eBPF program */
24
	[BPF_REG_0] = LOONGARCH_GPR_A5,
25
	/* arguments from eBPF program to in-kernel function */
26
	[BPF_REG_1] = LOONGARCH_GPR_A0,
27
	[BPF_REG_2] = LOONGARCH_GPR_A1,
28
	[BPF_REG_3] = LOONGARCH_GPR_A2,
29
	[BPF_REG_4] = LOONGARCH_GPR_A3,
30
	[BPF_REG_5] = LOONGARCH_GPR_A4,
31
	/* callee saved registers that in-kernel function will preserve */
32
	[BPF_REG_6] = LOONGARCH_GPR_S0,
33
	[BPF_REG_7] = LOONGARCH_GPR_S1,
34
	[BPF_REG_8] = LOONGARCH_GPR_S2,
35
	[BPF_REG_9] = LOONGARCH_GPR_S3,
36
	/* read-only frame pointer to access stack */
37
	[BPF_REG_FP] = LOONGARCH_GPR_S4,
38
	/* temporary register for blinding constants */
39
	[BPF_REG_AX] = LOONGARCH_GPR_T0,
40
};
41

42
static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
43
{
44
	const struct bpf_prog *prog = ctx->prog;
45
	const bool is_main_prog = !bpf_is_subprog(prog);
46

47
	if (is_main_prog) {
48
		/*
49
		 * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
50
		 * if (REG_TCC > T3 )
51
		 *	std REG_TCC -> LOONGARCH_GPR_SP + store_offset
52
		 * else
53
		 *	std REG_TCC -> LOONGARCH_GPR_SP + store_offset
54
		 *	REG_TCC = LOONGARCH_GPR_SP + store_offset
55
		 *
56
		 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset
57
		 *
58
		 * The purpose of this code is to first push the TCC into stack,
59
		 * and then push the address of TCC into stack.
60
		 * In cases where bpf2bpf and tailcall are used in combination,
61
		 * the value in REG_TCC may be a count or an address,
62
		 * these two cases need to be judged and handled separately.
63
		 */
64
		emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
65
		*store_offset -= sizeof(long);
66

67
		emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
68

69
		/*
70
		 * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
71
		 * push tcc into stack
72
		 */
73
		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
74

75
		/* Push the address of TCC into the REG_TCC */
76
		emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
77

78
		emit_uncond_jmp(ctx, 2);
79

80
		/*
81
		 * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
82
		 * push tcc_ptr into stack
83
		 */
84
		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
85
	} else {
86
		*store_offset -= sizeof(long);
87
		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
88
	}
89

90
	/* Push tcc_ptr into stack */
91
	*store_offset -= sizeof(long);
92
	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
93
}
94

95
/*
96
 * eBPF prog stack layout:
97
 *
98
 *                                        high
99
 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
100
 *                            |           $ra           |
101
 *                            +-------------------------+
102
 *                            |           $fp           |
103
 *                            +-------------------------+
104
 *                            |           $s0           |
105
 *                            +-------------------------+
106
 *                            |           $s1           |
107
 *                            +-------------------------+
108
 *                            |           $s2           |
109
 *                            +-------------------------+
110
 *                            |           $s3           |
111
 *                            +-------------------------+
112
 *                            |           $s4           |
113
 *                            +-------------------------+
114
 *                            |           $s5           |
115
 *                            +-------------------------+
116
 *                            |           tcc           |
117
 *                            +-------------------------+
118
 *                            |           tcc_ptr       |
119
 *                            +-------------------------+ <--BPF_REG_FP
120
 *                            |  prog->aux->stack_depth |
121
 *                            |        (optional)       |
122
 * current $sp -------------> +-------------------------+
123
 *                                        low
124
 */
125
static void build_prologue(struct jit_ctx *ctx)
126
{
127
	int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
128
	const struct bpf_prog *prog = ctx->prog;
129
	const bool is_main_prog = !bpf_is_subprog(prog);
130

131
	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
132

133
	/* To store ra, fp, s0, s1, s2, s3, s4, s5 */
134
	stack_adjust += sizeof(long) * 8;
135

136
	/* To store tcc and tcc_ptr */
137
	stack_adjust += sizeof(long) * 2;
138

139
	stack_adjust = round_up(stack_adjust, 16);
140
	stack_adjust += bpf_stack_adjust;
141

142
	/* Reserve space for the move_imm + jirl instruction */
143
	for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
144
		emit_insn(ctx, nop);
145

146
	/*
147
	 * First instruction initializes the tail call count (TCC)
148
	 * register to zero. On tail call we skip this instruction,
149
	 * and the TCC is passed in REG_TCC from the caller.
150
	 */
151
	if (is_main_prog)
152
		emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
153

154
	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
155

156
	store_offset = stack_adjust - sizeof(long);
157
	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
158

159
	store_offset -= sizeof(long);
160
	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
161

162
	store_offset -= sizeof(long);
163
	emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
164

165
	store_offset -= sizeof(long);
166
	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
167

168
	store_offset -= sizeof(long);
169
	emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
170

171
	store_offset -= sizeof(long);
172
	emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
173

174
	store_offset -= sizeof(long);
175
	emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
176

177
	store_offset -= sizeof(long);
178
	emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
179

180
	prepare_bpf_tail_call_cnt(ctx, &store_offset);
181

182
	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
183

184
	if (bpf_stack_adjust)
185
		emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
186

187
	ctx->stack_size = stack_adjust;
188
}
189

190
static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
191
{
192
	int stack_adjust = ctx->stack_size;
193
	int load_offset;
194

195
	load_offset = stack_adjust - sizeof(long);
196
	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
197

198
	load_offset -= sizeof(long);
199
	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
200

201
	load_offset -= sizeof(long);
202
	emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
203

204
	load_offset -= sizeof(long);
205
	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
206

207
	load_offset -= sizeof(long);
208
	emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
209

210
	load_offset -= sizeof(long);
211
	emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
212

213
	load_offset -= sizeof(long);
214
	emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
215

216
	load_offset -= sizeof(long);
217
	emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
218

219
	/*
220
	 * When push into the stack, follow the order of tcc then tcc_ptr.
221
	 * When pop from the stack, first pop tcc_ptr then followed by tcc.
222
	 */
223
	load_offset -= 2 * sizeof(long);
224
	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
225

226
	load_offset += sizeof(long);
227
	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
228

229
	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
230

231
	if (!is_tail_call) {
232
		/* Set return value */
233
		emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
234
		/* Return to the caller */
235
		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
236
	} else {
237
		/*
238
		 * Call the next bpf prog and skip the first instruction
239
		 * of TCC initialization.
240
		 */
241
		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6);
242
	}
243
}
244

245
static void build_epilogue(struct jit_ctx *ctx)
246
{
247
	__build_epilogue(ctx, false);
248
}
249

250
bool bpf_jit_supports_kfunc_call(void)
251
{
252
	return true;
253
}
254

255
bool bpf_jit_supports_far_kfunc_call(void)
256
{
257
	return true;
258
}
259

260
static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
261
{
262
	int off, tc_ninsn = 0;
263
	int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
264
	u8 a1 = LOONGARCH_GPR_A1;
265
	u8 a2 = LOONGARCH_GPR_A2;
266
	u8 t1 = LOONGARCH_GPR_T1;
267
	u8 t2 = LOONGARCH_GPR_T2;
268
	u8 t3 = LOONGARCH_GPR_T3;
269
	const int idx0 = ctx->idx;
270

271
#define cur_offset (ctx->idx - idx0)
272
#define jmp_offset (tc_ninsn - (cur_offset))
273

274
	/*
275
	 * a0: &ctx
276
	 * a1: &array
277
	 * a2: index
278
	 *
279
	 * if (index >= array->map.max_entries)
280
	 *	 goto out;
281
	 */
282
	tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
283
	off = offsetof(struct bpf_array, map.max_entries);
284
	emit_insn(ctx, ldwu, t1, a1, off);
285
	/* bgeu $a2, $t1, jmp_offset */
286
	if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
287
		goto toofar;
288

289
	/*
290
	 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
291
	 *      goto out;
292
	 */
293
	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
294
	emit_insn(ctx, ldd, t3, REG_TCC, 0);
295
	emit_insn(ctx, addid, t3, t3, 1);
296
	emit_insn(ctx, std, t3, REG_TCC, 0);
297
	emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
298
	if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0)
299
		goto toofar;
300

301
	/*
302
	 * prog = array->ptrs[index];
303
	 * if (!prog)
304
	 *	 goto out;
305
	 */
306
	emit_insn(ctx, alsld, t2, a2, a1, 2);
307
	off = offsetof(struct bpf_array, ptrs);
308
	emit_insn(ctx, ldd, t2, t2, off);
309
	/* beq $t2, $zero, jmp_offset */
310
	if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
311
		goto toofar;
312

313
	/* goto *(prog->bpf_func + 4); */
314
	off = offsetof(struct bpf_prog, bpf_func);
315
	emit_insn(ctx, ldd, t3, t2, off);
316
	__build_epilogue(ctx, true);
317

318
	return 0;
319

320
toofar:
321
	pr_info_once("tail_call: jump too far\n");
322
	return -1;
323
#undef cur_offset
324
#undef jmp_offset
325
}
326

327
static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
328
{
329
	const u8 t1 = LOONGARCH_GPR_T1;
330
	const u8 t2 = LOONGARCH_GPR_T2;
331
	const u8 t3 = LOONGARCH_GPR_T3;
332
	const u8 r0 = regmap[BPF_REG_0];
333
	const u8 src = regmap[insn->src_reg];
334
	const u8 dst = regmap[insn->dst_reg];
335
	const s16 off = insn->off;
336
	const s32 imm = insn->imm;
337
	const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
338

339
	move_imm(ctx, t1, off, false);
340
	emit_insn(ctx, addd, t1, dst, t1);
341
	move_reg(ctx, t3, src);
342

343
	switch (imm) {
344
	/* lock *(size *)(dst + off) <op>= src */
345
	case BPF_ADD:
346
		if (isdw)
347
			emit_insn(ctx, amaddd, t2, t1, src);
348
		else
349
			emit_insn(ctx, amaddw, t2, t1, src);
350
		break;
351
	case BPF_AND:
352
		if (isdw)
353
			emit_insn(ctx, amandd, t2, t1, src);
354
		else
355
			emit_insn(ctx, amandw, t2, t1, src);
356
		break;
357
	case BPF_OR:
358
		if (isdw)
359
			emit_insn(ctx, amord, t2, t1, src);
360
		else
361
			emit_insn(ctx, amorw, t2, t1, src);
362
		break;
363
	case BPF_XOR:
364
		if (isdw)
365
			emit_insn(ctx, amxord, t2, t1, src);
366
		else
367
			emit_insn(ctx, amxorw, t2, t1, src);
368
		break;
369
	/* src = atomic_fetch_<op>(dst + off, src) */
370
	case BPF_ADD | BPF_FETCH:
371
		if (isdw) {
372
			emit_insn(ctx, amaddd, src, t1, t3);
373
		} else {
374
			emit_insn(ctx, amaddw, src, t1, t3);
375
			emit_zext_32(ctx, src, true);
376
		}
377
		break;
378
	case BPF_AND | BPF_FETCH:
379
		if (isdw) {
380
			emit_insn(ctx, amandd, src, t1, t3);
381
		} else {
382
			emit_insn(ctx, amandw, src, t1, t3);
383
			emit_zext_32(ctx, src, true);
384
		}
385
		break;
386
	case BPF_OR | BPF_FETCH:
387
		if (isdw) {
388
			emit_insn(ctx, amord, src, t1, t3);
389
		} else {
390
			emit_insn(ctx, amorw, src, t1, t3);
391
			emit_zext_32(ctx, src, true);
392
		}
393
		break;
394
	case BPF_XOR | BPF_FETCH:
395
		if (isdw) {
396
			emit_insn(ctx, amxord, src, t1, t3);
397
		} else {
398
			emit_insn(ctx, amxorw, src, t1, t3);
399
			emit_zext_32(ctx, src, true);
400
		}
401
		break;
402
	/* src = atomic_xchg(dst + off, src); */
403
	case BPF_XCHG:
404
		if (isdw) {
405
			emit_insn(ctx, amswapd, src, t1, t3);
406
		} else {
407
			emit_insn(ctx, amswapw, src, t1, t3);
408
			emit_zext_32(ctx, src, true);
409
		}
410
		break;
411
	/* r0 = atomic_cmpxchg(dst + off, r0, src); */
412
	case BPF_CMPXCHG:
413
		move_reg(ctx, t2, r0);
414
		if (isdw) {
415
			emit_insn(ctx, lld, r0, t1, 0);
416
			emit_insn(ctx, bne, t2, r0, 4);
417
			move_reg(ctx, t3, src);
418
			emit_insn(ctx, scd, t3, t1, 0);
419
			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
420
		} else {
421
			emit_insn(ctx, llw, r0, t1, 0);
422
			emit_zext_32(ctx, t2, true);
423
			emit_zext_32(ctx, r0, true);
424
			emit_insn(ctx, bne, t2, r0, 4);
425
			move_reg(ctx, t3, src);
426
			emit_insn(ctx, scw, t3, t1, 0);
427
			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
428
			emit_zext_32(ctx, r0, true);
429
		}
430
		break;
431
	}
432
}
433

434
static bool is_signed_bpf_cond(u8 cond)
435
{
436
	return cond == BPF_JSGT || cond == BPF_JSLT ||
437
	       cond == BPF_JSGE || cond == BPF_JSLE;
438
}
439

440
#define BPF_FIXUP_REG_MASK	GENMASK(31, 27)
441
#define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)
442

443
bool ex_handler_bpf(const struct exception_table_entry *ex,
444
		    struct pt_regs *regs)
445
{
446
	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
447
	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
448

449
	regs->regs[dst_reg] = 0;
450
	regs->csr_era = (unsigned long)&ex->fixup - offset;
451

452
	return true;
453
}
454

455
/* For accesses to BTF pointers, add an entry to the exception table */
456
static int add_exception_handler(const struct bpf_insn *insn,
457
				 struct jit_ctx *ctx,
458
				 int dst_reg)
459
{
460
	unsigned long pc;
461
	off_t offset;
462
	struct exception_table_entry *ex;
463

464
	if (!ctx->image || !ctx->prog->aux->extable)
465
		return 0;
466

467
	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
468
	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
469
		return 0;
470

471
	if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
472
		return -EINVAL;
473

474
	ex = &ctx->prog->aux->extable[ctx->num_exentries];
475
	pc = (unsigned long)&ctx->image[ctx->idx - 1];
476

477
	offset = pc - (long)&ex->insn;
478
	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
479
		return -ERANGE;
480

481
	ex->insn = offset;
482

483
	/*
484
	 * Since the extable follows the program, the fixup offset is always
485
	 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
486
	 * to keep things simple, and put the destination register in the upper
487
	 * bits. We don't need to worry about buildtime or runtime sort
488
	 * modifying the upper bits because the table is already sorted, and
489
	 * isn't part of the main exception table.
490
	 */
491
	offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
492
	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
493
		return -ERANGE;
494

495
	ex->type = EX_TYPE_BPF;
496
	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
497

498
	ctx->num_exentries++;
499

500
	return 0;
501
}
502

503
static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
504
{
505
	u8 tm = -1;
506
	u64 func_addr;
507
	bool func_addr_fixed, sign_extend;
508
	int i = insn - ctx->prog->insnsi;
509
	int ret, jmp_offset, tcc_ptr_off;
510
	const u8 code = insn->code;
511
	const u8 cond = BPF_OP(code);
512
	const u8 t1 = LOONGARCH_GPR_T1;
513
	const u8 t2 = LOONGARCH_GPR_T2;
514
	const u8 src = regmap[insn->src_reg];
515
	const u8 dst = regmap[insn->dst_reg];
516
	const s16 off = insn->off;
517
	const s32 imm = insn->imm;
518
	const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
519

520
	switch (code) {
521
	/* dst = src */
522
	case BPF_ALU | BPF_MOV | BPF_X:
523
	case BPF_ALU64 | BPF_MOV | BPF_X:
524
		switch (off) {
525
		case 0:
526
			move_reg(ctx, dst, src);
527
			emit_zext_32(ctx, dst, is32);
528
			break;
529
		case 8:
530
			emit_insn(ctx, extwb, dst, src);
531
			emit_zext_32(ctx, dst, is32);
532
			break;
533
		case 16:
534
			emit_insn(ctx, extwh, dst, src);
535
			emit_zext_32(ctx, dst, is32);
536
			break;
537
		case 32:
538
			emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
539
			break;
540
		}
541
		break;
542

543
	/* dst = imm */
544
	case BPF_ALU | BPF_MOV | BPF_K:
545
	case BPF_ALU64 | BPF_MOV | BPF_K:
546
		move_imm(ctx, dst, imm, is32);
547
		break;
548

549
	/* dst = dst + src */
550
	case BPF_ALU | BPF_ADD | BPF_X:
551
	case BPF_ALU64 | BPF_ADD | BPF_X:
552
		emit_insn(ctx, addd, dst, dst, src);
553
		emit_zext_32(ctx, dst, is32);
554
		break;
555

556
	/* dst = dst + imm */
557
	case BPF_ALU | BPF_ADD | BPF_K:
558
	case BPF_ALU64 | BPF_ADD | BPF_K:
559
		if (is_signed_imm12(imm)) {
560
			emit_insn(ctx, addid, dst, dst, imm);
561
		} else {
562
			move_imm(ctx, t1, imm, is32);
563
			emit_insn(ctx, addd, dst, dst, t1);
564
		}
565
		emit_zext_32(ctx, dst, is32);
566
		break;
567

568
	/* dst = dst - src */
569
	case BPF_ALU | BPF_SUB | BPF_X:
570
	case BPF_ALU64 | BPF_SUB | BPF_X:
571
		emit_insn(ctx, subd, dst, dst, src);
572
		emit_zext_32(ctx, dst, is32);
573
		break;
574

575
	/* dst = dst - imm */
576
	case BPF_ALU | BPF_SUB | BPF_K:
577
	case BPF_ALU64 | BPF_SUB | BPF_K:
578
		if (is_signed_imm12(-imm)) {
579
			emit_insn(ctx, addid, dst, dst, -imm);
580
		} else {
581
			move_imm(ctx, t1, imm, is32);
582
			emit_insn(ctx, subd, dst, dst, t1);
583
		}
584
		emit_zext_32(ctx, dst, is32);
585
		break;
586

587
	/* dst = dst * src */
588
	case BPF_ALU | BPF_MUL | BPF_X:
589
	case BPF_ALU64 | BPF_MUL | BPF_X:
590
		emit_insn(ctx, muld, dst, dst, src);
591
		emit_zext_32(ctx, dst, is32);
592
		break;
593

594
	/* dst = dst * imm */
595
	case BPF_ALU | BPF_MUL | BPF_K:
596
	case BPF_ALU64 | BPF_MUL | BPF_K:
597
		move_imm(ctx, t1, imm, is32);
598
		emit_insn(ctx, muld, dst, dst, t1);
599
		emit_zext_32(ctx, dst, is32);
600
		break;
601

602
	/* dst = dst / src */
603
	case BPF_ALU | BPF_DIV | BPF_X:
604
	case BPF_ALU64 | BPF_DIV | BPF_X:
605
		if (!off) {
606
			emit_zext_32(ctx, dst, is32);
607
			move_reg(ctx, t1, src);
608
			emit_zext_32(ctx, t1, is32);
609
			emit_insn(ctx, divdu, dst, dst, t1);
610
			emit_zext_32(ctx, dst, is32);
611
		} else {
612
			emit_sext_32(ctx, dst, is32);
613
			move_reg(ctx, t1, src);
614
			emit_sext_32(ctx, t1, is32);
615
			emit_insn(ctx, divd, dst, dst, t1);
616
			emit_sext_32(ctx, dst, is32);
617
		}
618
		break;
619

620
	/* dst = dst / imm */
621
	case BPF_ALU | BPF_DIV | BPF_K:
622
	case BPF_ALU64 | BPF_DIV | BPF_K:
623
		if (!off) {
624
			move_imm(ctx, t1, imm, is32);
625
			emit_zext_32(ctx, dst, is32);
626
			emit_insn(ctx, divdu, dst, dst, t1);
627
			emit_zext_32(ctx, dst, is32);
628
		} else {
629
			move_imm(ctx, t1, imm, false);
630
			emit_sext_32(ctx, t1, is32);
631
			emit_sext_32(ctx, dst, is32);
632
			emit_insn(ctx, divd, dst, dst, t1);
633
			emit_sext_32(ctx, dst, is32);
634
		}
635
		break;
636

637
	/* dst = dst % src */
638
	case BPF_ALU | BPF_MOD | BPF_X:
639
	case BPF_ALU64 | BPF_MOD | BPF_X:
640
		if (!off) {
641
			emit_zext_32(ctx, dst, is32);
642
			move_reg(ctx, t1, src);
643
			emit_zext_32(ctx, t1, is32);
644
			emit_insn(ctx, moddu, dst, dst, t1);
645
			emit_zext_32(ctx, dst, is32);
646
		} else {
647
			emit_sext_32(ctx, dst, is32);
648
			move_reg(ctx, t1, src);
649
			emit_sext_32(ctx, t1, is32);
650
			emit_insn(ctx, modd, dst, dst, t1);
651
			emit_sext_32(ctx, dst, is32);
652
		}
653
		break;
654

655
	/* dst = dst % imm */
656
	case BPF_ALU | BPF_MOD | BPF_K:
657
	case BPF_ALU64 | BPF_MOD | BPF_K:
658
		if (!off) {
659
			move_imm(ctx, t1, imm, is32);
660
			emit_zext_32(ctx, dst, is32);
661
			emit_insn(ctx, moddu, dst, dst, t1);
662
			emit_zext_32(ctx, dst, is32);
663
		} else {
664
			move_imm(ctx, t1, imm, false);
665
			emit_sext_32(ctx, t1, is32);
666
			emit_sext_32(ctx, dst, is32);
667
			emit_insn(ctx, modd, dst, dst, t1);
668
			emit_sext_32(ctx, dst, is32);
669
		}
670
		break;
671

672
	/* dst = -dst */
673
	case BPF_ALU | BPF_NEG:
674
	case BPF_ALU64 | BPF_NEG:
675
		move_imm(ctx, t1, imm, is32);
676
		emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
677
		emit_zext_32(ctx, dst, is32);
678
		break;
679

680
	/* dst = dst & src */
681
	case BPF_ALU | BPF_AND | BPF_X:
682
	case BPF_ALU64 | BPF_AND | BPF_X:
683
		emit_insn(ctx, and, dst, dst, src);
684
		emit_zext_32(ctx, dst, is32);
685
		break;
686

687
	/* dst = dst & imm */
688
	case BPF_ALU | BPF_AND | BPF_K:
689
	case BPF_ALU64 | BPF_AND | BPF_K:
690
		if (is_unsigned_imm12(imm)) {
691
			emit_insn(ctx, andi, dst, dst, imm);
692
		} else {
693
			move_imm(ctx, t1, imm, is32);
694
			emit_insn(ctx, and, dst, dst, t1);
695
		}
696
		emit_zext_32(ctx, dst, is32);
697
		break;
698

699
	/* dst = dst | src */
700
	case BPF_ALU | BPF_OR | BPF_X:
701
	case BPF_ALU64 | BPF_OR | BPF_X:
702
		emit_insn(ctx, or, dst, dst, src);
703
		emit_zext_32(ctx, dst, is32);
704
		break;
705

706
	/* dst = dst | imm */
707
	case BPF_ALU | BPF_OR | BPF_K:
708
	case BPF_ALU64 | BPF_OR | BPF_K:
709
		if (is_unsigned_imm12(imm)) {
710
			emit_insn(ctx, ori, dst, dst, imm);
711
		} else {
712
			move_imm(ctx, t1, imm, is32);
713
			emit_insn(ctx, or, dst, dst, t1);
714
		}
715
		emit_zext_32(ctx, dst, is32);
716
		break;
717

718
	/* dst = dst ^ src */
719
	case BPF_ALU | BPF_XOR | BPF_X:
720
	case BPF_ALU64 | BPF_XOR | BPF_X:
721
		emit_insn(ctx, xor, dst, dst, src);
722
		emit_zext_32(ctx, dst, is32);
723
		break;
724

725
	/* dst = dst ^ imm */
726
	case BPF_ALU | BPF_XOR | BPF_K:
727
	case BPF_ALU64 | BPF_XOR | BPF_K:
728
		if (is_unsigned_imm12(imm)) {
729
			emit_insn(ctx, xori, dst, dst, imm);
730
		} else {
731
			move_imm(ctx, t1, imm, is32);
732
			emit_insn(ctx, xor, dst, dst, t1);
733
		}
734
		emit_zext_32(ctx, dst, is32);
735
		break;
736

737
	/* dst = dst << src (logical) */
738
	case BPF_ALU | BPF_LSH | BPF_X:
739
		emit_insn(ctx, sllw, dst, dst, src);
740
		emit_zext_32(ctx, dst, is32);
741
		break;
742

743
	case BPF_ALU64 | BPF_LSH | BPF_X:
744
		emit_insn(ctx, slld, dst, dst, src);
745
		break;
746

747
	/* dst = dst << imm (logical) */
748
	case BPF_ALU | BPF_LSH | BPF_K:
749
		emit_insn(ctx, slliw, dst, dst, imm);
750
		emit_zext_32(ctx, dst, is32);
751
		break;
752

753
	case BPF_ALU64 | BPF_LSH | BPF_K:
754
		emit_insn(ctx, sllid, dst, dst, imm);
755
		break;
756

757
	/* dst = dst >> src (logical) */
758
	case BPF_ALU | BPF_RSH | BPF_X:
759
		emit_insn(ctx, srlw, dst, dst, src);
760
		emit_zext_32(ctx, dst, is32);
761
		break;
762

763
	case BPF_ALU64 | BPF_RSH | BPF_X:
764
		emit_insn(ctx, srld, dst, dst, src);
765
		break;
766

767
	/* dst = dst >> imm (logical) */
768
	case BPF_ALU | BPF_RSH | BPF_K:
769
		emit_insn(ctx, srliw, dst, dst, imm);
770
		emit_zext_32(ctx, dst, is32);
771
		break;
772

773
	case BPF_ALU64 | BPF_RSH | BPF_K:
774
		emit_insn(ctx, srlid, dst, dst, imm);
775
		break;
776

777
	/* dst = dst >> src (arithmetic) */
778
	case BPF_ALU | BPF_ARSH | BPF_X:
779
		emit_insn(ctx, sraw, dst, dst, src);
780
		emit_zext_32(ctx, dst, is32);
781
		break;
782

783
	case BPF_ALU64 | BPF_ARSH | BPF_X:
784
		emit_insn(ctx, srad, dst, dst, src);
785
		break;
786

787
	/* dst = dst >> imm (arithmetic) */
788
	case BPF_ALU | BPF_ARSH | BPF_K:
789
		emit_insn(ctx, sraiw, dst, dst, imm);
790
		emit_zext_32(ctx, dst, is32);
791
		break;
792

793
	case BPF_ALU64 | BPF_ARSH | BPF_K:
794
		emit_insn(ctx, sraid, dst, dst, imm);
795
		break;
796

797
	/* dst = BSWAP##imm(dst) */
798
	case BPF_ALU | BPF_END | BPF_FROM_LE:
799
		switch (imm) {
800
		case 16:
801
			/* zero-extend 16 bits into 64 bits */
802
			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
803
			break;
804
		case 32:
805
			/* zero-extend 32 bits into 64 bits */
806
			emit_zext_32(ctx, dst, is32);
807
			break;
808
		case 64:
809
			/* do nothing */
810
			break;
811
		}
812
		break;
813

814
	case BPF_ALU | BPF_END | BPF_FROM_BE:
815
	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
816
		switch (imm) {
817
		case 16:
818
			emit_insn(ctx, revb2h, dst, dst);
819
			/* zero-extend 16 bits into 64 bits */
820
			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
821
			break;
822
		case 32:
823
			emit_insn(ctx, revb2w, dst, dst);
824
			/* clear the upper 32 bits */
825
			emit_zext_32(ctx, dst, true);
826
			break;
827
		case 64:
828
			emit_insn(ctx, revbd, dst, dst);
829
			break;
830
		}
831
		break;
832

833
	/* PC += off if dst cond src */
834
	case BPF_JMP | BPF_JEQ | BPF_X:
835
	case BPF_JMP | BPF_JNE | BPF_X:
836
	case BPF_JMP | BPF_JGT | BPF_X:
837
	case BPF_JMP | BPF_JGE | BPF_X:
838
	case BPF_JMP | BPF_JLT | BPF_X:
839
	case BPF_JMP | BPF_JLE | BPF_X:
840
	case BPF_JMP | BPF_JSGT | BPF_X:
841
	case BPF_JMP | BPF_JSGE | BPF_X:
842
	case BPF_JMP | BPF_JSLT | BPF_X:
843
	case BPF_JMP | BPF_JSLE | BPF_X:
844
	case BPF_JMP32 | BPF_JEQ | BPF_X:
845
	case BPF_JMP32 | BPF_JNE | BPF_X:
846
	case BPF_JMP32 | BPF_JGT | BPF_X:
847
	case BPF_JMP32 | BPF_JGE | BPF_X:
848
	case BPF_JMP32 | BPF_JLT | BPF_X:
849
	case BPF_JMP32 | BPF_JLE | BPF_X:
850
	case BPF_JMP32 | BPF_JSGT | BPF_X:
851
	case BPF_JMP32 | BPF_JSGE | BPF_X:
852
	case BPF_JMP32 | BPF_JSLT | BPF_X:
853
	case BPF_JMP32 | BPF_JSLE | BPF_X:
854
		jmp_offset = bpf2la_offset(i, off, ctx);
855
		move_reg(ctx, t1, dst);
856
		move_reg(ctx, t2, src);
857
		if (is_signed_bpf_cond(BPF_OP(code))) {
858
			emit_sext_32(ctx, t1, is32);
859
			emit_sext_32(ctx, t2, is32);
860
		} else {
861
			emit_zext_32(ctx, t1, is32);
862
			emit_zext_32(ctx, t2, is32);
863
		}
864
		if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
865
			goto toofar;
866
		break;
867

868
	/* PC += off if dst cond imm */
869
	case BPF_JMP | BPF_JEQ | BPF_K:
870
	case BPF_JMP | BPF_JNE | BPF_K:
871
	case BPF_JMP | BPF_JGT | BPF_K:
872
	case BPF_JMP | BPF_JGE | BPF_K:
873
	case BPF_JMP | BPF_JLT | BPF_K:
874
	case BPF_JMP | BPF_JLE | BPF_K:
875
	case BPF_JMP | BPF_JSGT | BPF_K:
876
	case BPF_JMP | BPF_JSGE | BPF_K:
877
	case BPF_JMP | BPF_JSLT | BPF_K:
878
	case BPF_JMP | BPF_JSLE | BPF_K:
879
	case BPF_JMP32 | BPF_JEQ | BPF_K:
880
	case BPF_JMP32 | BPF_JNE | BPF_K:
881
	case BPF_JMP32 | BPF_JGT | BPF_K:
882
	case BPF_JMP32 | BPF_JGE | BPF_K:
883
	case BPF_JMP32 | BPF_JLT | BPF_K:
884
	case BPF_JMP32 | BPF_JLE | BPF_K:
885
	case BPF_JMP32 | BPF_JSGT | BPF_K:
886
	case BPF_JMP32 | BPF_JSGE | BPF_K:
887
	case BPF_JMP32 | BPF_JSLT | BPF_K:
888
	case BPF_JMP32 | BPF_JSLE | BPF_K:
889
		jmp_offset = bpf2la_offset(i, off, ctx);
890
		if (imm) {
891
			move_imm(ctx, t1, imm, false);
892
			tm = t1;
893
		} else {
894
			/* If imm is 0, simply use zero register. */
895
			tm = LOONGARCH_GPR_ZERO;
896
		}
897
		move_reg(ctx, t2, dst);
898
		if (is_signed_bpf_cond(BPF_OP(code))) {
899
			emit_sext_32(ctx, tm, is32);
900
			emit_sext_32(ctx, t2, is32);
901
		} else {
902
			emit_zext_32(ctx, tm, is32);
903
			emit_zext_32(ctx, t2, is32);
904
		}
905
		if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
906
			goto toofar;
907
		break;
908

909
	/* PC += off if dst & src */
910
	case BPF_JMP | BPF_JSET | BPF_X:
911
	case BPF_JMP32 | BPF_JSET | BPF_X:
912
		jmp_offset = bpf2la_offset(i, off, ctx);
913
		emit_insn(ctx, and, t1, dst, src);
914
		emit_zext_32(ctx, t1, is32);
915
		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
916
			goto toofar;
917
		break;
918

919
	/* PC += off if dst & imm */
920
	case BPF_JMP | BPF_JSET | BPF_K:
921
	case BPF_JMP32 | BPF_JSET | BPF_K:
922
		jmp_offset = bpf2la_offset(i, off, ctx);
923
		move_imm(ctx, t1, imm, is32);
924
		emit_insn(ctx, and, t1, dst, t1);
925
		emit_zext_32(ctx, t1, is32);
926
		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
927
			goto toofar;
928
		break;
929

930
	/* PC += off */
931
	case BPF_JMP | BPF_JA:
932
	case BPF_JMP32 | BPF_JA:
933
		if (BPF_CLASS(code) == BPF_JMP)
934
			jmp_offset = bpf2la_offset(i, off, ctx);
935
		else
936
			jmp_offset = bpf2la_offset(i, imm, ctx);
937
		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
938
			goto toofar;
939
		break;
940

941
	/* function call */
942
	case BPF_JMP | BPF_CALL:
943
		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
944
					    &func_addr, &func_addr_fixed);
945
		if (ret < 0)
946
			return ret;
947

948
		if (insn->src_reg == BPF_PSEUDO_CALL) {
949
			tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
950
			emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
951
		}
952

953
		move_addr(ctx, t1, func_addr);
954
		emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
955

956
		if (insn->src_reg != BPF_PSEUDO_CALL)
957
			move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
958

959
		break;
960

961
	/* tail call */
962
	case BPF_JMP | BPF_TAIL_CALL:
963
		if (emit_bpf_tail_call(ctx, i) < 0)
964
			return -EINVAL;
965
		break;
966

967
	/* function return */
968
	case BPF_JMP | BPF_EXIT:
969
		if (i == ctx->prog->len - 1)
970
			break;
971

972
		jmp_offset = epilogue_offset(ctx);
973
		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
974
			goto toofar;
975
		break;
976

977
	/* dst = imm64 */
978
	case BPF_LD | BPF_IMM | BPF_DW:
979
	{
980
		const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
981

982
		if (bpf_pseudo_func(insn))
983
			move_addr(ctx, dst, imm64);
984
		else
985
			move_imm(ctx, dst, imm64, is32);
986
		return 1;
987
	}
988

989
	/* dst = *(size *)(src + off) */
990
	case BPF_LDX | BPF_MEM | BPF_B:
991
	case BPF_LDX | BPF_MEM | BPF_H:
992
	case BPF_LDX | BPF_MEM | BPF_W:
993
	case BPF_LDX | BPF_MEM | BPF_DW:
994
	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
995
	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
996
	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
997
	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
998
	/* dst_reg = (s64)*(signed size *)(src_reg + off) */
999
	case BPF_LDX | BPF_MEMSX | BPF_B:
1000
	case BPF_LDX | BPF_MEMSX | BPF_H:
1001
	case BPF_LDX | BPF_MEMSX | BPF_W:
1002
	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1003
	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1004
	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1005
		sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
1006
			      BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
1007
		switch (BPF_SIZE(code)) {
1008
		case BPF_B:
1009
			if (is_signed_imm12(off)) {
1010
				if (sign_extend)
1011
					emit_insn(ctx, ldb, dst, src, off);
1012
				else
1013
					emit_insn(ctx, ldbu, dst, src, off);
1014
			} else {
1015
				move_imm(ctx, t1, off, is32);
1016
				if (sign_extend)
1017
					emit_insn(ctx, ldxb, dst, src, t1);
1018
				else
1019
					emit_insn(ctx, ldxbu, dst, src, t1);
1020
			}
1021
			break;
1022
		case BPF_H:
1023
			if (is_signed_imm12(off)) {
1024
				if (sign_extend)
1025
					emit_insn(ctx, ldh, dst, src, off);
1026
				else
1027
					emit_insn(ctx, ldhu, dst, src, off);
1028
			} else {
1029
				move_imm(ctx, t1, off, is32);
1030
				if (sign_extend)
1031
					emit_insn(ctx, ldxh, dst, src, t1);
1032
				else
1033
					emit_insn(ctx, ldxhu, dst, src, t1);
1034
			}
1035
			break;
1036
		case BPF_W:
1037
			if (is_signed_imm12(off)) {
1038
				if (sign_extend)
1039
					emit_insn(ctx, ldw, dst, src, off);
1040
				else
1041
					emit_insn(ctx, ldwu, dst, src, off);
1042
			} else {
1043
				move_imm(ctx, t1, off, is32);
1044
				if (sign_extend)
1045
					emit_insn(ctx, ldxw, dst, src, t1);
1046
				else
1047
					emit_insn(ctx, ldxwu, dst, src, t1);
1048
			}
1049
			break;
1050
		case BPF_DW:
1051
			move_imm(ctx, t1, off, is32);
1052
			emit_insn(ctx, ldxd, dst, src, t1);
1053
			break;
1054
		}
1055

1056
		ret = add_exception_handler(insn, ctx, dst);
1057
		if (ret)
1058
			return ret;
1059
		break;
1060

1061
	/* *(size *)(dst + off) = imm */
1062
	case BPF_ST | BPF_MEM | BPF_B:
1063
	case BPF_ST | BPF_MEM | BPF_H:
1064
	case BPF_ST | BPF_MEM | BPF_W:
1065
	case BPF_ST | BPF_MEM | BPF_DW:
1066
		switch (BPF_SIZE(code)) {
1067
		case BPF_B:
1068
			move_imm(ctx, t1, imm, is32);
1069
			if (is_signed_imm12(off)) {
1070
				emit_insn(ctx, stb, t1, dst, off);
1071
			} else {
1072
				move_imm(ctx, t2, off, is32);
1073
				emit_insn(ctx, stxb, t1, dst, t2);
1074
			}
1075
			break;
1076
		case BPF_H:
1077
			move_imm(ctx, t1, imm, is32);
1078
			if (is_signed_imm12(off)) {
1079
				emit_insn(ctx, sth, t1, dst, off);
1080
			} else {
1081
				move_imm(ctx, t2, off, is32);
1082
				emit_insn(ctx, stxh, t1, dst, t2);
1083
			}
1084
			break;
1085
		case BPF_W:
1086
			move_imm(ctx, t1, imm, is32);
1087
			if (is_signed_imm12(off)) {
1088
				emit_insn(ctx, stw, t1, dst, off);
1089
			} else if (is_signed_imm14(off)) {
1090
				emit_insn(ctx, stptrw, t1, dst, off);
1091
			} else {
1092
				move_imm(ctx, t2, off, is32);
1093
				emit_insn(ctx, stxw, t1, dst, t2);
1094
			}
1095
			break;
1096
		case BPF_DW:
1097
			move_imm(ctx, t1, imm, is32);
1098
			if (is_signed_imm12(off)) {
1099
				emit_insn(ctx, std, t1, dst, off);
1100
			} else if (is_signed_imm14(off)) {
1101
				emit_insn(ctx, stptrd, t1, dst, off);
1102
			} else {
1103
				move_imm(ctx, t2, off, is32);
1104
				emit_insn(ctx, stxd, t1, dst, t2);
1105
			}
1106
			break;
1107
		}
1108
		break;
1109

1110
	/* *(size *)(dst + off) = src */
1111
	case BPF_STX | BPF_MEM | BPF_B:
1112
	case BPF_STX | BPF_MEM | BPF_H:
1113
	case BPF_STX | BPF_MEM | BPF_W:
1114
	case BPF_STX | BPF_MEM | BPF_DW:
1115
		switch (BPF_SIZE(code)) {
1116
		case BPF_B:
1117
			if (is_signed_imm12(off)) {
1118
				emit_insn(ctx, stb, src, dst, off);
1119
			} else {
1120
				move_imm(ctx, t1, off, is32);
1121
				emit_insn(ctx, stxb, src, dst, t1);
1122
			}
1123
			break;
1124
		case BPF_H:
1125
			if (is_signed_imm12(off)) {
1126
				emit_insn(ctx, sth, src, dst, off);
1127
			} else {
1128
				move_imm(ctx, t1, off, is32);
1129
				emit_insn(ctx, stxh, src, dst, t1);
1130
			}
1131
			break;
1132
		case BPF_W:
1133
			if (is_signed_imm12(off)) {
1134
				emit_insn(ctx, stw, src, dst, off);
1135
			} else if (is_signed_imm14(off)) {
1136
				emit_insn(ctx, stptrw, src, dst, off);
1137
			} else {
1138
				move_imm(ctx, t1, off, is32);
1139
				emit_insn(ctx, stxw, src, dst, t1);
1140
			}
1141
			break;
1142
		case BPF_DW:
1143
			if (is_signed_imm12(off)) {
1144
				emit_insn(ctx, std, src, dst, off);
1145
			} else if (is_signed_imm14(off)) {
1146
				emit_insn(ctx, stptrd, src, dst, off);
1147
			} else {
1148
				move_imm(ctx, t1, off, is32);
1149
				emit_insn(ctx, stxd, src, dst, t1);
1150
			}
1151
			break;
1152
		}
1153
		break;
1154

1155
	case BPF_STX | BPF_ATOMIC | BPF_W:
1156
	case BPF_STX | BPF_ATOMIC | BPF_DW:
1157
		emit_atomic(insn, ctx);
1158
		break;
1159

1160
	/* Speculation barrier */
1161
	case BPF_ST | BPF_NOSPEC:
1162
		break;
1163

1164
	default:
1165
		pr_err("bpf_jit: unknown opcode %02x\n", code);
1166
		return -EINVAL;
1167
	}
1168

1169
	return 0;
1170

1171
toofar:
1172
	pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1173
	return -E2BIG;
1174
}
1175

1176
static int build_body(struct jit_ctx *ctx, bool extra_pass)
1177
{
1178
	int i;
1179
	const struct bpf_prog *prog = ctx->prog;
1180

1181
	for (i = 0; i < prog->len; i++) {
1182
		const struct bpf_insn *insn = &prog->insnsi[i];
1183
		int ret;
1184

1185
		if (ctx->image == NULL)
1186
			ctx->offset[i] = ctx->idx;
1187

1188
		ret = build_insn(insn, ctx, extra_pass);
1189
		if (ret > 0) {
1190
			i++;
1191
			if (ctx->image == NULL)
1192
				ctx->offset[i] = ctx->idx;
1193
			continue;
1194
		}
1195
		if (ret)
1196
			return ret;
1197
	}
1198

1199
	if (ctx->image == NULL)
1200
		ctx->offset[i] = ctx->idx;
1201

1202
	return 0;
1203
}
1204

1205
/* Fill space with break instructions */
1206
static void jit_fill_hole(void *area, unsigned int size)
1207
{
1208
	u32 *ptr;
1209

1210
	/* We are guaranteed to have aligned memory */
1211
	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1212
		*ptr++ = INSN_BREAK;
1213
}
1214

1215
static int validate_code(struct jit_ctx *ctx)
1216
{
1217
	int i;
1218
	union loongarch_instruction insn;
1219

1220
	for (i = 0; i < ctx->idx; i++) {
1221
		insn = ctx->image[i];
1222
		/* Check INSN_BREAK */
1223
		if (insn.word == INSN_BREAK)
1224
			return -1;
1225
	}
1226

1227
	return 0;
1228
}
1229

1230
static int validate_ctx(struct jit_ctx *ctx)
1231
{
1232
	if (validate_code(ctx))
1233
		return -1;
1234

1235
	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1236
		return -1;
1237

1238
	return 0;
1239
}
1240

1241
static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
1242
{
1243
	if (!target) {
1244
		pr_err("bpf_jit: jump target address is error\n");
1245
		return -EFAULT;
1246
	}
1247

1248
	move_imm(ctx, LOONGARCH_GPR_T1, target, false);
1249
	emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
1250

1251
	return 0;
1252
}
1253

1254
static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
1255
{
1256
	int i;
1257
	struct jit_ctx ctx;
1258

1259
	ctx.idx = 0;
1260
	ctx.image = (union loongarch_instruction *)insns;
1261

1262
	if (!target) {
1263
		for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1264
			emit_insn((&ctx), nop);
1265
		return 0;
1266
	}
1267

1268
	return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target);
1269
}
1270

1271
static int emit_call(struct jit_ctx *ctx, u64 addr)
1272
{
1273
	return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
1274
}
1275

1276
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
1277
{
1278
	int ret;
1279

1280
	mutex_lock(&text_mutex);
1281
	ret = larch_insn_text_copy(dst, src, len);
1282
	mutex_unlock(&text_mutex);
1283

1284
	return ret ? ERR_PTR(-EINVAL) : dst;
1285
}
1286

1287
int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
1288
		       void *old_addr, void *new_addr)
1289
{
1290
	int ret;
1291
	bool is_call = (poke_type == BPF_MOD_CALL);
1292
	u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1293
	u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1294

1295
	/* Only poking bpf text is supported. Since kernel function entry
1296
	 * is set up by ftrace, we rely on ftrace to poke kernel functions.
1297
	 */
1298
	if (!is_bpf_text_address((unsigned long)ip))
1299
		return -ENOTSUPP;
1300

1301
	ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call);
1302
	if (ret)
1303
		return ret;
1304

1305
	if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES))
1306
		return -EFAULT;
1307

1308
	ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call);
1309
	if (ret)
1310
		return ret;
1311

1312
	mutex_lock(&text_mutex);
1313
	if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
1314
		ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
1315
	mutex_unlock(&text_mutex);
1316

1317
	return ret;
1318
}
1319

1320
int bpf_arch_text_invalidate(void *dst, size_t len)
1321
{
1322
	int i;
1323
	int ret = 0;
1324
	u32 *inst;
1325

1326
	inst = kvmalloc(len, GFP_KERNEL);
1327
	if (!inst)
1328
		return -ENOMEM;
1329

1330
	for (i = 0; i < (len / sizeof(u32)); i++)
1331
		inst[i] = INSN_BREAK;
1332

1333
	mutex_lock(&text_mutex);
1334
	if (larch_insn_text_copy(dst, inst, len))
1335
		ret = -EINVAL;
1336
	mutex_unlock(&text_mutex);
1337

1338
	kvfree(inst);
1339

1340
	return ret;
1341
}
1342

1343
static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
1344
{
1345
	int i;
1346

1347
	for (i = 0; i < nargs; i++) {
1348
		emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1349
		args_off -= 8;
1350
	}
1351
}
1352

1353
static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
1354
{
1355
	int i;
1356

1357
	for (i = 0; i < nargs; i++) {
1358
		emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1359
		args_off -= 8;
1360
	}
1361
}
1362

1363
static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
1364
			   int args_off, int retval_off, int run_ctx_off, bool save_ret)
1365
{
1366
	int ret;
1367
	u32 *branch;
1368
	struct bpf_prog *p = l->link.prog;
1369
	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
1370

1371
	if (l->cookie) {
1372
		move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
1373
		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1374
	} else {
1375
		emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1376
	}
1377

1378
	/* arg1: prog */
1379
	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1380
	/* arg2: &run_ctx */
1381
	emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
1382
	ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
1383
	if (ret)
1384
		return ret;
1385

1386
	/* store prog start time */
1387
	move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
1388

1389
	/*
1390
	 * if (__bpf_prog_enter(prog) == 0)
1391
	 *      goto skip_exec_of_prog;
1392
	 */
1393
	branch = (u32 *)ctx->image + ctx->idx;
1394
	/* nop reserved for conditional jump */
1395
	emit_insn(ctx, nop);
1396

1397
	/* arg1: &args_off */
1398
	emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
1399
	if (!p->jited)
1400
		move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
1401
	ret = emit_call(ctx, (const u64)p->bpf_func);
1402
	if (ret)
1403
		return ret;
1404

1405
	if (save_ret) {
1406
		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1407
		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1408
	}
1409

1410
	/* update branch with beqz */
1411
	if (ctx->image) {
1412
		int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
1413
		*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
1414
	}
1415

1416
	/* arg1: prog */
1417
	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1418
	/* arg2: prog start time */
1419
	move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
1420
	/* arg3: &run_ctx */
1421
	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
1422
	ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
1423

1424
	return ret;
1425
}
1426

1427
static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
1428
			       int args_off, int retval_off, int run_ctx_off, u32 **branches)
1429
{
1430
	int i;
1431

1432
	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
1433
	for (i = 0; i < tl->nr_links; i++) {
1434
		invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true);
1435
		emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
1436
		branches[i] = (u32 *)ctx->image + ctx->idx;
1437
		emit_insn(ctx, nop);
1438
	}
1439
}
1440

1441
void *arch_alloc_bpf_trampoline(unsigned int size)
1442
{
1443
	return bpf_prog_pack_alloc(size, jit_fill_hole);
1444
}
1445

1446
void arch_free_bpf_trampoline(void *image, unsigned int size)
1447
{
1448
	bpf_prog_pack_free(image, size);
1449
}
1450

1451
/*
1452
 * Sign-extend the register if necessary
1453
 */
1454
static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign)
1455
{
1456
	/* ABI requires unsigned char/short to be zero-extended */
1457
	if (!sign && (size == 1 || size == 2)) {
1458
		if (rd != rj)
1459
			move_reg(ctx, rd, rj);
1460
		return;
1461
	}
1462

1463
	switch (size) {
1464
	case 1:
1465
		emit_insn(ctx, extwb, rd, rj);
1466
		break;
1467
	case 2:
1468
		emit_insn(ctx, extwh, rd, rj);
1469
		break;
1470
	case 4:
1471
		emit_insn(ctx, addiw, rd, rj, 0);
1472
		break;
1473
	case 8:
1474
		if (rd != rj)
1475
			move_reg(ctx, rd, rj);
1476
		break;
1477
	default:
1478
		pr_warn("bpf_jit: invalid size %d for sign_extend\n", size);
1479
	}
1480
}
1481

1482
static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
1483
					 const struct btf_func_model *m, struct bpf_tramp_links *tlinks,
1484
					 void *func_addr, u32 flags)
1485
{
1486
	int i, ret, save_ret;
1487
	int stack_size, nargs;
1488
	int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
1489
	bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
1490
	void *orig_call = func_addr;
1491
	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
1492
	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
1493
	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1494
	u32 **branches = NULL;
1495

1496
	/*
1497
	 * FP + 8       [ RA to parent func ] return address to parent
1498
	 *                    function
1499
	 * FP + 0       [ FP of parent func ] frame pointer of parent
1500
	 *                    function
1501
	 * FP - 8       [ T0 to traced func ] return address of traced
1502
	 *                    function
1503
	 * FP - 16      [ FP of traced func ] frame pointer of traced
1504
	 *                    function
1505
	 *
1506
	 * FP - retval_off  [ return value      ] BPF_TRAMP_F_CALL_ORIG or
1507
	 *                    BPF_TRAMP_F_RET_FENTRY_RET
1508
	 *                  [ argN              ]
1509
	 *                  [ ...               ]
1510
	 * FP - args_off    [ arg1              ]
1511
	 *
1512
	 * FP - nargs_off   [ regs count        ]
1513
	 *
1514
	 * FP - ip_off      [ traced func   ] BPF_TRAMP_F_IP_ARG
1515
	 *
1516
	 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
1517
	 *
1518
	 * FP - sreg_off    [ callee saved reg  ]
1519
	 *
1520
	 * FP - tcc_ptr_off [ tail_call_cnt_ptr ]
1521
	 */
1522

1523
	if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
1524
		return -ENOTSUPP;
1525

1526
	/* FIXME: No support of struct argument */
1527
	for (i = 0; i < m->nr_args; i++) {
1528
		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
1529
			return -ENOTSUPP;
1530
	}
1531

1532
	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
1533
		return -ENOTSUPP;
1534

1535
	/* Room of trampoline frame to store return address and frame pointer */
1536
	stack_size = 16;
1537

1538
	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1539
	if (save_ret)
1540
		stack_size += 16; /* Save BPF R0 and A0 */
1541

1542
	retval_off = stack_size;
1543

1544
	/* Room of trampoline frame to store args */
1545
	nargs = m->nr_args;
1546
	stack_size += nargs * 8;
1547
	args_off = stack_size;
1548

1549
	/* Room of trampoline frame to store args number */
1550
	stack_size += 8;
1551
	nargs_off = stack_size;
1552

1553
	/* Room of trampoline frame to store ip address */
1554
	if (flags & BPF_TRAMP_F_IP_ARG) {
1555
		stack_size += 8;
1556
		ip_off = stack_size;
1557
	}
1558

1559
	/* Room of trampoline frame to store struct bpf_tramp_run_ctx */
1560
	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
1561
	run_ctx_off = stack_size;
1562

1563
	stack_size += 8;
1564
	sreg_off = stack_size;
1565

1566
	/* Room of trampoline frame to store tail_call_cnt_ptr */
1567
	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
1568
		stack_size += 8;
1569
		tcc_ptr_off = stack_size;
1570
	}
1571

1572
	stack_size = round_up(stack_size, 16);
1573

1574
	if (is_struct_ops) {
1575
		/*
1576
		 * For the trampoline called directly, just handle
1577
		 * the frame of trampoline.
1578
		 */
1579
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1580
		emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1581
		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1582
		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1583
	} else {
1584
		/*
1585
		 * For the trampoline called from function entry,
1586
		 * the frame of traced function and the frame of
1587
		 * trampoline need to be considered.
1588
		 */
1589
		/* RA and FP for parent function */
1590
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
1591
		emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1592
		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1593
		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
1594

1595
		/* RA and FP for traced function */
1596
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1597
		emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1598
		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1599
		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1600
	}
1601

1602
	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1603
		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1604

1605
	/* callee saved register S1 to pass start time */
1606
	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1607

1608
	/* store ip address of the traced function */
1609
	if (flags & BPF_TRAMP_F_IP_ARG) {
1610
		move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
1611
		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
1612
	}
1613

1614
	/* store nargs number */
1615
	move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
1616
	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
1617

1618
	store_args(ctx, nargs, args_off);
1619

1620
	/* To traced function */
1621
	/* Ftrace jump skips 2 NOP instructions */
1622
	if (is_kernel_text((unsigned long)orig_call))
1623
		orig_call += LOONGARCH_FENTRY_NBYTES;
1624
	/* Direct jump skips 5 NOP instructions */
1625
	else if (is_bpf_text_address((unsigned long)orig_call))
1626
		orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
1627

1628
	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1629
		move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1630
		ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
1631
		if (ret)
1632
			return ret;
1633
	}
1634

1635
	for (i = 0; i < fentry->nr_links; i++) {
1636
		ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off,
1637
				      run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET);
1638
		if (ret)
1639
			return ret;
1640
	}
1641
	if (fmod_ret->nr_links) {
1642
		branches  = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
1643
		if (!branches)
1644
			return -ENOMEM;
1645

1646
		invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches);
1647
	}
1648

1649
	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1650
		restore_args(ctx, m->nr_args, args_off);
1651

1652
		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1653
			emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1654

1655
		ret = emit_call(ctx, (const u64)orig_call);
1656
		if (ret)
1657
			goto out;
1658
		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1659
		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1660
		im->ip_after_call = ctx->ro_image + ctx->idx;
1661
		/* Reserve space for the move_imm + jirl instruction */
1662
		for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1663
			emit_insn(ctx, nop);
1664
	}
1665

1666
	for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
1667
		int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
1668
		*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
1669
	}
1670

1671
	for (i = 0; i < fexit->nr_links; i++) {
1672
		ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false);
1673
		if (ret)
1674
			goto out;
1675
	}
1676

1677
	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1678
		im->ip_epilogue = ctx->ro_image + ctx->idx;
1679
		move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1680
		ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
1681
		if (ret)
1682
			goto out;
1683
	}
1684

1685
	if (flags & BPF_TRAMP_F_RESTORE_REGS)
1686
		restore_args(ctx, m->nr_args, args_off);
1687

1688
	if (save_ret) {
1689
		emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1690
		if (is_struct_ops)
1691
			sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0],
1692
				    m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG);
1693
		else
1694
			emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1695
	}
1696

1697
	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1698

1699
	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1700
		emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1701

1702
	if (is_struct_ops) {
1703
		/* trampoline called directly */
1704
		emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1705
		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1706
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1707

1708
		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
1709
	} else {
1710
		/* trampoline called from function entry */
1711
		emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1712
		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1713
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1714

1715
		emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1716
		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1717
		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
1718

1719
		if (flags & BPF_TRAMP_F_SKIP_FRAME)
1720
			/* return to parent function */
1721
			emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
1722
		else
1723
			/* return to traced function */
1724
			emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
1725
	}
1726

1727
	ret = ctx->idx;
1728
out:
1729
	kfree(branches);
1730

1731
	return ret;
1732
}
1733

1734
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
1735
				void *ro_image_end, const struct btf_func_model *m,
1736
				u32 flags, struct bpf_tramp_links *tlinks, void *func_addr)
1737
{
1738
	int ret, size;
1739
	void *image, *tmp;
1740
	struct jit_ctx ctx;
1741

1742
	size = ro_image_end - ro_image;
1743
	image = kvmalloc(size, GFP_KERNEL);
1744
	if (!image)
1745
		return -ENOMEM;
1746

1747
	ctx.image = (union loongarch_instruction *)image;
1748
	ctx.ro_image = (union loongarch_instruction *)ro_image;
1749
	ctx.idx = 0;
1750

1751
	jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
1752
	ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
1753
	if (ret < 0)
1754
		goto out;
1755

1756
	if (validate_code(&ctx) < 0) {
1757
		ret = -EINVAL;
1758
		goto out;
1759
	}
1760

1761
	tmp = bpf_arch_text_copy(ro_image, image, size);
1762
	if (IS_ERR(tmp)) {
1763
		ret = PTR_ERR(tmp);
1764
		goto out;
1765
	}
1766

1767
out:
1768
	kvfree(image);
1769
	return ret < 0 ? ret : size;
1770
}
1771

1772
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1773
			     struct bpf_tramp_links *tlinks, void *func_addr)
1774
{
1775
	int ret;
1776
	struct jit_ctx ctx;
1777
	struct bpf_tramp_image im;
1778

1779
	ctx.image = NULL;
1780
	ctx.idx = 0;
1781

1782
	ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
1783

1784
	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
1785
}
1786

1787
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1788
{
1789
	bool tmp_blinded = false, extra_pass = false;
1790
	u8 *image_ptr;
1791
	int image_size, prog_size, extable_size;
1792
	struct jit_ctx ctx;
1793
	struct jit_data *jit_data;
1794
	struct bpf_binary_header *header;
1795
	struct bpf_prog *tmp, *orig_prog = prog;
1796

1797
	/*
1798
	 * If BPF JIT was not enabled then we must fall back to
1799
	 * the interpreter.
1800
	 */
1801
	if (!prog->jit_requested)
1802
		return orig_prog;
1803

1804
	tmp = bpf_jit_blind_constants(prog);
1805
	/*
1806
	 * If blinding was requested and we failed during blinding,
1807
	 * we must fall back to the interpreter. Otherwise, we save
1808
	 * the new JITed code.
1809
	 */
1810
	if (IS_ERR(tmp))
1811
		return orig_prog;
1812

1813
	if (tmp != prog) {
1814
		tmp_blinded = true;
1815
		prog = tmp;
1816
	}
1817

1818
	jit_data = prog->aux->jit_data;
1819
	if (!jit_data) {
1820
		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1821
		if (!jit_data) {
1822
			prog = orig_prog;
1823
			goto out;
1824
		}
1825
		prog->aux->jit_data = jit_data;
1826
	}
1827
	if (jit_data->ctx.offset) {
1828
		ctx = jit_data->ctx;
1829
		image_ptr = jit_data->image;
1830
		header = jit_data->header;
1831
		extra_pass = true;
1832
		prog_size = sizeof(u32) * ctx.idx;
1833
		goto skip_init_ctx;
1834
	}
1835

1836
	memset(&ctx, 0, sizeof(ctx));
1837
	ctx.prog = prog;
1838

1839
	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1840
	if (ctx.offset == NULL) {
1841
		prog = orig_prog;
1842
		goto out_offset;
1843
	}
1844

1845
	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1846
	build_prologue(&ctx);
1847
	if (build_body(&ctx, extra_pass)) {
1848
		prog = orig_prog;
1849
		goto out_offset;
1850
	}
1851
	ctx.epilogue_offset = ctx.idx;
1852
	build_epilogue(&ctx);
1853

1854
	extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1855

1856
	/* Now we know the actual image size.
1857
	 * As each LoongArch instruction is of length 32bit,
1858
	 * we are translating number of JITed intructions into
1859
	 * the size required to store these JITed code.
1860
	 */
1861
	prog_size = sizeof(u32) * ctx.idx;
1862
	image_size = prog_size + extable_size;
1863
	/* Now we know the size of the structure to make */
1864
	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1865
				      sizeof(u32), jit_fill_hole);
1866
	if (header == NULL) {
1867
		prog = orig_prog;
1868
		goto out_offset;
1869
	}
1870

1871
	/* 2. Now, the actual pass to generate final JIT code */
1872
	ctx.image = (union loongarch_instruction *)image_ptr;
1873
	if (extable_size)
1874
		prog->aux->extable = (void *)image_ptr + prog_size;
1875

1876
skip_init_ctx:
1877
	ctx.idx = 0;
1878
	ctx.num_exentries = 0;
1879

1880
	build_prologue(&ctx);
1881
	if (build_body(&ctx, extra_pass)) {
1882
		bpf_jit_binary_free(header);
1883
		prog = orig_prog;
1884
		goto out_offset;
1885
	}
1886
	build_epilogue(&ctx);
1887

1888
	/* 3. Extra pass to validate JITed code */
1889
	if (validate_ctx(&ctx)) {
1890
		bpf_jit_binary_free(header);
1891
		prog = orig_prog;
1892
		goto out_offset;
1893
	}
1894

1895
	/* And we're done */
1896
	if (bpf_jit_enable > 1)
1897
		bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1898

1899
	/* Update the icache */
1900
	flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1901

1902
	if (!prog->is_func || extra_pass) {
1903
		int err;
1904

1905
		if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1906
			pr_err_once("multi-func JIT bug %d != %d\n",
1907
				    ctx.idx, jit_data->ctx.idx);
1908
			goto out_free;
1909
		}
1910
		err = bpf_jit_binary_lock_ro(header);
1911
		if (err) {
1912
			pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
1913
				    err);
1914
			goto out_free;
1915
		}
1916
	} else {
1917
		jit_data->ctx = ctx;
1918
		jit_data->image = image_ptr;
1919
		jit_data->header = header;
1920
	}
1921
	prog->jited = 1;
1922
	prog->jited_len = prog_size;
1923
	prog->bpf_func = (void *)ctx.image;
1924

1925
	if (!prog->is_func || extra_pass) {
1926
		int i;
1927

1928
		/* offset[prog->len] is the size of program */
1929
		for (i = 0; i <= prog->len; i++)
1930
			ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1931
		bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1932

1933
out_offset:
1934
		kvfree(ctx.offset);
1935
		kfree(jit_data);
1936
		prog->aux->jit_data = NULL;
1937
	}
1938

1939
out:
1940
	if (tmp_blinded)
1941
		bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
1942

1943

1944
	return prog;
1945

1946
out_free:
1947
	bpf_jit_binary_free(header);
1948
	prog->bpf_func = NULL;
1949
	prog->jited = 0;
1950
	prog->jited_len = 0;
1951
	goto out_offset;
1952
}
1953

1954
bool bpf_jit_bypass_spec_v1(void)
1955
{
1956
	return true;
1957
}
1958

1959
bool bpf_jit_bypass_spec_v4(void)
1960
{
1961
	return true;
1962
}
1963

1964
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
1965
bool bpf_jit_supports_subprog_tailcalls(void)
1966
{
1967
	return true;
1968
}
1969

1970