path: root/arch/mips/net/bpf_jit_comp64.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Just-In-Time compiler for eBPF bytecode on MIPS.
 * Implementation of JIT functions for 64-bit CPUs.
 *
 * Copyright (c) 2021 Anyfi Networks AB.
 * Author: Johan Almbladh <johan.almbladh@gmail.com>
 *
 * Based on code and ideas from
 * Copyright (c) 2017 Cavium, Inc.
 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
 */

#include <linux/errno.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <asm/cpu-features.h>
#include <asm/isa-rev.h>
#include <asm/uasm.h>

#include "bpf_jit_comp.h"

/* MIPS t0-t3 are not available in the n64 ABI */
#undef MIPS_R_T0
#undef MIPS_R_T1
#undef MIPS_R_T2
#undef MIPS_R_T3

/* Stack is 16-byte aligned in n64 ABI */
#define MIPS_STACK_ALIGNMENT 16

/* Extra 64-bit eBPF registers used by JIT */
#define JIT_REG_TC (MAX_BPF_JIT_REG + 0)
#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1)

/* Number of prologue bytes to skip when doing a tail call */
#define JIT_TCALL_SKIP 4

/* Callee-saved CPU registers that the JIT must preserve */
#define JIT_CALLEE_REGS   \
	(BIT(MIPS_R_S0) | \
	 BIT(MIPS_R_S1) | \
	 BIT(MIPS_R_S2) | \
	 BIT(MIPS_R_S3) | \
	 BIT(MIPS_R_S4) | \
	 BIT(MIPS_R_S5) | \
	 BIT(MIPS_R_S6) | \
	 BIT(MIPS_R_S7) | \
	 BIT(MIPS_R_GP) | \
	 BIT(MIPS_R_FP) | \
	 BIT(MIPS_R_RA))

/* Caller-saved CPU registers available for JIT use */
#define JIT_CALLER_REGS	  \
	(BIT(MIPS_R_A5) | \
	 BIT(MIPS_R_A6) | \
	 BIT(MIPS_R_A7))
/*
 * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers.
 * MIPS registers t4 - t7 may be used by the JIT as temporary registers.
 * MIPS registers t8 - t9 are reserved for single-register common functions.
 */
static const u8 bpf2mips64[] = {
	/* Return value from in-kernel function, and exit value from eBPF */
	[BPF_REG_0] = MIPS_R_V0,
	/* Arguments from eBPF program to in-kernel function */
	[BPF_REG_1] = MIPS_R_A0,
	[BPF_REG_2] = MIPS_R_A1,
	[BPF_REG_3] = MIPS_R_A2,
	[BPF_REG_4] = MIPS_R_A3,
	[BPF_REG_5] = MIPS_R_A4,
	/* Callee-saved registers that in-kernel function will preserve */
	[BPF_REG_6] = MIPS_R_S0,
	[BPF_REG_7] = MIPS_R_S1,
	[BPF_REG_8] = MIPS_R_S2,
	[BPF_REG_9] = MIPS_R_S3,
	/* Read-only frame pointer to access the eBPF stack */
	[BPF_REG_FP] = MIPS_R_FP,
	/* Temporary register for blinding constants */
	[BPF_REG_AX] = MIPS_R_AT,
	/* Tail call count register, caller-saved */
	[JIT_REG_TC] = MIPS_R_A5,
	/* Constant for register zero-extension */
	[JIT_REG_ZX] = MIPS_R_V1,
};

/*
 * MIPS 32-bit operations on 64-bit registers generate a sign-extended
 * result. However, the eBPF ISA mandates zero-extension, so we rely on the
 * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic
 * operations, right shift and byte swap require properly sign-extended
 * operands or the result is unpredictable. We emit explicit sign-extensions
 * in those cases.
 */

/* Sign extension */
static void emit_sext(struct jit_context *ctx, u8 dst, u8 src)
{
	emit(ctx, sll, dst, src, 0);
	clobber_reg(ctx, dst);
}

/* Zero extension */
static void emit_zext(struct jit_context *ctx, u8 dst)
{
	if (cpu_has_mips64r2 || cpu_has_mips64r6) {
		emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
	} else {
		emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]);
		access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */
	}
	clobber_reg(ctx, dst);
}

/* Zero extension, if verifier does not do it for us  */
static void emit_zext_ver(struct jit_context *ctx, u8 dst)
{
	if (!ctx->program->aux->verifier_zext)
		emit_zext(ctx, dst);
}

/* dst = imm (64-bit) */
static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64)
{
	if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) {
		emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64);
	} else if (imm64 >= 0xffffffff80000000ULL ||
		   (imm64 < 0x80000000 && imm64 > 0xffff)) {
		emit(ctx, lui, dst, (s16)(imm64 >> 16));
		emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff);
	} else {
		u8 acc = MIPS_R_ZERO;
		int shift = 0;
		int k;

		for (k = 0; k < 4; k++) {
			u16 half = imm64 >> (48 - 16 * k);

			if (acc == dst)
				shift += 16;

			if (half) {
				if (shift)
					emit(ctx, dsll_safe, dst, dst, shift);
				emit(ctx, ori, dst, acc, half);
				acc = dst;
				shift = 0;
			}
		}
		if (shift)
			emit(ctx, dsll_safe, dst, dst, shift);
	}
	clobber_reg(ctx, dst);
}

/* ALU immediate operation (64-bit) */
static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
{
	switch (BPF_OP(op)) {
	/* dst = dst | imm */
	case BPF_OR:
		emit(ctx, ori, dst, dst, (u16)imm);
		break;
	/* dst = dst ^ imm */
	case BPF_XOR:
		emit(ctx, xori, dst, dst, (u16)imm);
		break;
	/* dst = -dst */
	case BPF_NEG:
		emit(ctx, dsubu, dst, MIPS_R_ZERO, dst);
		break;
	/* dst = dst << imm */
	case BPF_LSH:
		emit(ctx, dsll_safe, dst, dst, imm);
		break;
	/* dst = dst >> imm */
	case BPF_RSH:
		emit(ctx, dsrl_safe, dst, dst, imm);
		break;
	/* dst = dst >> imm (arithmetic) */
	case BPF