arch/riscv/net/bpf_jit_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   6  *
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <linux/filter.h>
  11 #include "bpf_jit.h"
  12
  13 /* Number of iterations to try until offsets converge. */
  14 #define NR_JIT_ITERATIONS       32
  15
  16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  17 {
  18         const struct bpf_prog *prog = ctx->prog;
  19         int i;
  20
  21         for (i = 0; i < prog->len; i++) {
  22                 const struct bpf_insn *insn = &prog->insnsi[i];
  23                 int ret;
  24
  25                 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  26                 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  27                 if (ret > 0)
  28                         i++;
  29                 if (offset)
  30                         offset[i] = ctx->ninsns;
  31                 if (ret < 0)
  32                         return ret;
  33         }
  34         return 0;
  35 }
  36
  37 bool bpf_jit_needs_zext(void)
  38 {
  39         return true;
  40 }
  41
  42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  43 {
  44         unsigned int prog_size = 0, extable_size = 0;
  45         bool tmp_blinded = false, extra_pass = false;
  46         struct bpf_prog *tmp, *orig_prog = prog;
  47         int pass = 0, prev_ninsns = 0, i;
  48         struct rv_jit_data *jit_data;
  49         struct rv_jit_context *ctx;
  50
  51         if (!prog->jit_requested)
  52                 return orig_prog;
  53
  54         tmp = bpf_jit_blind_constants(prog);
  55         if (IS_ERR(tmp))
  56                 return orig_prog;
  57         if (tmp != prog) {
  58                 tmp_blinded = true;
  59                 prog = tmp;
  60         }
  61
  62         jit_data = prog->aux->jit_data;
  63         if (!jit_data) {
  64                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  65                 if (!jit_data) {
  66                         prog = orig_prog;
  67                         goto out;
  68                 }
  69                 prog->aux->jit_data = jit_data;
  70         }
  71
  72         ctx = &jit_data->ctx;
  73
  74         if (ctx->offset) {
  75                 extra_pass = true;
  76                 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
  77                 goto skip_init_ctx;
  78         }
  79
  80         ctx->prog = prog;
  81         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  82         if (!ctx->offset) {
  83                 prog = orig_prog;
  84                 goto out_offset;
  85         }
  86
  87         if (build_body(ctx, extra_pass, NULL)) {
  88                 prog = orig_prog;
  89                 goto out_offset;
  90         }
  91
  92         for (i = 0; i < prog->len; i++) {
  93                 prev_ninsns += 32;
  94                 ctx->offset[i] = prev_ninsns;
  95         }
  96
  97         for (i = 0; i < NR_JIT_ITERATIONS; i++) {
  98                 pass++;
  99                 ctx->ninsns = 0;
 100
 101                 bpf_jit_build_prologue(ctx);
 102                 ctx->prologue_len = ctx->ninsns;
 103
 104                 if (build_body(ctx, extra_pass, ctx->offset)) {
 105                         prog = orig_prog;
 106                         goto out_offset;
 107                 }
 108
 109                 ctx->epilogue_offset = ctx->ninsns;
 110                 bpf_jit_build_epilogue(ctx);
 111
 112                 if (ctx->ninsns == prev_ninsns) {
 113                         if (jit_data->header)
 114                                 break;
 115                         /* obtain the actual image size */
 116                         extable_size = prog->aux->num_exentries *
 117                                 sizeof(struct exception_table_entry);
 118                         prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 119
 120                         jit_data->header =
 121                                 bpf_jit_binary_alloc(prog_size + extable_size,
 122                                                      &jit_data->image,
 123                                                      sizeof(u32),
 124                                                      bpf_fill_ill_insns);
 125                         if (!jit_data->header) {
 126                                 prog = orig_prog;
 127                                 goto out_offset;
 128                         }
 129
 130                         ctx->insns = (u16 *)jit_data->image;
 131                         /*
 132                          * Now, when the image is allocated, the image can
 133                          * potentially shrink more (auipc/jalr -> jal).
 134                          */
 135                 }
 136                 prev_ninsns = ctx->ninsns;
 137         }
 138
 139         if (i == NR_JIT_ITERATIONS) {
 140                 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 141                 if (jit_data->header)
 142                         bpf_jit_binary_free(jit_data->header);
 143                 prog = orig_prog;
 144                 goto out_offset;
 145         }
 146
 147         if (extable_size)
 148                 prog->aux->extable = (void *)ctx->insns + prog_size;
 149
 150 skip_init_ctx:
 151         pass++;
 152         ctx->ninsns = 0;
 153         ctx->nexentries = 0;
 154
 155         bpf_jit_build_prologue(ctx);
 156         if (build_body(ctx, extra_pass, NULL)) {
 157                 bpf_jit_binary_free(jit_data->header);
 158                 prog = orig_prog;
 159                 goto out_offset;
 160         }
 161         bpf_jit_build_epilogue(ctx);
 162
 163         if (bpf_jit_enable > 1)
 164                 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
 165
 166         prog->bpf_func = (void *)ctx->insns;
 167         prog->jited = 1;
 168         prog->jited_len = prog_size;
 169
 170         bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
 171
 172         if (!prog->is_func || extra_pass) {
 173                 bpf_jit_binary_lock_ro(jit_data->header);
 174                 for (i = 0; i < prog->len; i++)
 175                         ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
 176                 bpf_prog_fill_jited_linfo(prog, ctx->offset);
 177 out_offset:
 178                 kfree(ctx->offset);
 179                 kfree(jit_data);
 180                 prog->aux->jit_data = NULL;
 181         }
 182 out:
 183
 184         if (tmp_blinded)
 185                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
 186                                            tmp : orig_prog);
 187         return prog;
 188 }
 189
 190 u64 bpf_jit_alloc_exec_limit(void)
 191 {
 192         return BPF_JIT_REGION_SIZE;
 193 }
 194
 195 void *bpf_jit_alloc_exec(unsigned long size)
 196 {
 197         return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
 198                                     BPF_JIT_REGION_END, GFP_KERNEL,
 199                                     PAGE_KERNEL, 0, NUMA_NO_NODE,
 200                                     __builtin_return_address(0));
 201 }
 202
 203 void bpf_jit_free_exec(void *addr)
 204 {
 205         return vfree(addr);
 206 }