Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

[sfrench/cifs-2.6.git] / arch / s390 / net / bpf_jit_comp.c

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 955eb355c2fdea049412ecbcf81b84427e4cf406..ce88211b9c6cdda55164f82ef00327a98ba11b6d 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -502,7 +502,8 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
  * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
  * stack space for the large switch statement.
  */
-static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)
+static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
+                                int i, bool extra_pass)
 {
        struct bpf_insn *insn = &fp->insnsi[i];
        int jmp_off, last, insn_count = 1;
@@ -1011,10 +1012,14 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
         */
        case BPF_JMP | BPF_CALL:
        {
-               /*
-                * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
-                */
-               const u64 func = (u64)__bpf_call_base + imm;
+               u64 func;
+               bool func_addr_fixed;
+               int ret;
+
+               ret = bpf_jit_get_func_addr(fp, insn, extra_pass,
+                                           &func, &func_addr_fixed);
+               if (ret < 0)
+                       return -1;
 
                REG_SET_SEEN(BPF_REG_5);
                jit->seen |= SEEN_FUNC;
@@ -1283,7 +1288,8 @@ branch_oc:
 /*
  * Compile eBPF program into s390x code
  */
-static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)
+static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
+                       bool extra_pass)
 {
        int i, insn_count;
 
@@ -1292,7 +1298,7 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)
 
        bpf_jit_prologue(jit, fp->aux->stack_depth);
        for (i = 0; i < fp->len; i += insn_count) {
-               insn_count = bpf_jit_insn(jit, fp, i);
+               insn_count = bpf_jit_insn(jit, fp, i, extra_pass);
                if (insn_count < 0)
                        return -1;
                /* Next instruction address */
@@ -1311,6 +1317,12 @@ bool bpf_jit_needs_zext(void)
        return true;
 }
 
+struct s390_jit_data {
+       struct bpf_binary_header *header;
+       struct bpf_jit ctx;
+       int pass;
+};
+
 /*
  * Compile eBPF program "fp"
  */
@@ -1318,7 +1330,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 {
        struct bpf_prog *tmp, *orig_fp = fp;
        struct bpf_binary_header *header;
+       struct s390_jit_data *jit_data;
        bool tmp_blinded = false;
+       bool extra_pass = false;
        struct bpf_jit jit;
        int pass;
 
@@ -1337,6 +1351,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
                fp = tmp;
        }
 
+       jit_data = fp->aux->jit_data;
+       if (!jit_data) {
+               jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+               if (!jit_data) {
+                       fp = orig_fp;
+                       goto out;
+               }
+               fp->aux->jit_data = jit_data;
+       }
+       if (jit_data->ctx.addrs) {
+               jit = jit_data->ctx;
+               header = jit_data->header;
+               extra_pass = true;
+               pass = jit_data->pass + 1;
+               goto skip_init_ctx;
+       }
+
        memset(&jit, 0, sizeof(jit));
        jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
        if (jit.addrs == NULL) {
@@ -1349,7 +1380,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
         *   - 3:   Calculate program size and addrs arrray
         */
        for (pass = 1; pass <= 3; pass++) {
-               if (bpf_jit_prog(&jit, fp)) {
+               if (bpf_jit_prog(&jit, fp, extra_pass)) {
                        fp = orig_fp;
                        goto free_addrs;
                }
@@ -1361,12 +1392,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
                fp = orig_fp;
                goto free_addrs;
        }
+
        header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
        if (!header) {
                fp = orig_fp;
                goto free_addrs;
        }
-       if (bpf_jit_prog(&jit, fp)) {
+skip_init_ctx:
+       if (bpf_jit_prog(&jit, fp, extra_pass)) {
                bpf_jit_binary_free(header);
                fp = orig_fp;
                goto free_addrs;
@@ -1375,12 +1408,24 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
                bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
                print_fn_code(jit.prg_buf, jit.size_prg);
        }
-       bpf_jit_binary_lock_ro(header);
+       if (!fp->is_func || extra_pass) {
+               bpf_jit_binary_lock_ro(header);
+       } else {
+               jit_data->header = header;
+               jit_data->ctx = jit;
+               jit_data->pass = pass;
+       }
        fp->bpf_func = (void *) jit.prg_buf;
        fp->jited = 1;
        fp->jited_len = jit.size;
+
+       if (!fp->is_func || extra_pass) {
+               bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);
 free_addrs:
-       kfree(jit.addrs);
+               kfree(jit.addrs);
+               kfree(jit_data);
+               fp->aux->jit_data = NULL;
+       }
 out:
        if (tmp_blinded)
                bpf_jit_prog_release_other(fp, fp == orig_fp ?