1 // SPDX-License-Identifier: GPL-2.0
3 * Common functionality for RV32 and RV64 BPF JIT compilers
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
10 #include <linux/filter.h>
13 /* Number of iterations to try until offsets converge. */
14 #define NR_JIT_ITERATIONS 32
16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
18 const struct bpf_prog *prog = ctx->prog;
21 for (i = 0; i < prog->len; i++) {
22 const struct bpf_insn *insn = &prog->insnsi[i];
25 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
26 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
30 offset[i] = ctx->ninsns;
37 bool bpf_jit_needs_zext(void)
42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
44 unsigned int prog_size = 0, extable_size = 0;
45 bool tmp_blinded = false, extra_pass = false;
46 struct bpf_prog *tmp, *orig_prog = prog;
47 int pass = 0, prev_ninsns = 0, i;
48 struct rv_jit_data *jit_data;
49 struct rv_jit_context *ctx;
51 if (!prog->jit_requested)
54 tmp = bpf_jit_blind_constants(prog);
62 jit_data = prog->aux->jit_data;
64 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
69 prog->aux->jit_data = jit_data;
76 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
81 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
87 if (build_body(ctx, extra_pass, NULL)) {
92 for (i = 0; i < prog->len; i++) {
94 ctx->offset[i] = prev_ninsns;
97 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
101 bpf_jit_build_prologue(ctx);
102 ctx->prologue_len = ctx->ninsns;
104 if (build_body(ctx, extra_pass, ctx->offset)) {
109 ctx->epilogue_offset = ctx->ninsns;
110 bpf_jit_build_epilogue(ctx);
112 if (ctx->ninsns == prev_ninsns) {
113 if (jit_data->header)
115 /* obtain the actual image size */
116 extable_size = prog->aux->num_exentries *
117 sizeof(struct exception_table_entry);
118 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
121 bpf_jit_binary_alloc(prog_size + extable_size,
125 if (!jit_data->header) {
130 ctx->insns = (u16 *)jit_data->image;
132 * Now, when the image is allocated, the image can
133 * potentially shrink more (auipc/jalr -> jal).
136 prev_ninsns = ctx->ninsns;
139 if (i == NR_JIT_ITERATIONS) {
140 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
141 if (jit_data->header)
142 bpf_jit_binary_free(jit_data->header);
148 prog->aux->extable = (void *)ctx->insns + prog_size;
155 bpf_jit_build_prologue(ctx);
156 if (build_body(ctx, extra_pass, NULL)) {
157 bpf_jit_binary_free(jit_data->header);
161 bpf_jit_build_epilogue(ctx);
163 if (bpf_jit_enable > 1)
164 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
166 prog->bpf_func = (void *)ctx->insns;
168 prog->jited_len = prog_size;
170 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
172 if (!prog->is_func || extra_pass) {
173 bpf_jit_binary_lock_ro(jit_data->header);
174 for (i = 0; i < prog->len; i++)
175 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
176 bpf_prog_fill_jited_linfo(prog, ctx->offset);
180 prog->aux->jit_data = NULL;
185 bpf_jit_prog_release_other(prog, prog == orig_prog ?
190 u64 bpf_jit_alloc_exec_limit(void)
192 return BPF_JIT_REGION_SIZE;
195 void *bpf_jit_alloc_exec(unsigned long size)
197 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
198 BPF_JIT_REGION_END, GFP_KERNEL,
199 PAGE_KERNEL, 0, NUMA_NO_NODE,
200 __builtin_return_address(0));
203 void bpf_jit_free_exec(void *addr)