Merge tag '6.6-rc-smb3-client-fixes-part2' of git://git.samba.org/sfrench/cifs-2.6

[sfrench/cifs-2.6.git] / arch / riscv / net / bpf_jit_core.c

diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 7a26a3e1c73cfe8c75c0090c546f83d185572e2c..7b70ccb7fec345847b2dc2ab4664472f0d9ca574 100644 (file)
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -8,6 +8,8 @@
 
 #include <linux/bpf.h>
 #include <linux/filter.h>
+#include <linux/memory.h>
+#include <asm/patch.h>
 #include "bpf_jit.h"
 
 /* Number of iterations to try until offsets converge. */
@@ -117,16 +119,24 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
                                sizeof(struct exception_table_entry);
                        prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 
-                       jit_data->header =
-                               bpf_jit_binary_alloc(prog_size + extable_size,
-                                                    &jit_data->image,
-                                                    sizeof(u32),
-                                                    bpf_fill_ill_insns);
-                       if (!jit_data->header) {
+                       jit_data->ro_header =
+                               bpf_jit_binary_pack_alloc(prog_size + extable_size,
+                                                         &jit_data->ro_image, sizeof(u32),
+                                                         &jit_data->header, &jit_data->image,
+                                                         bpf_fill_ill_insns);
+                       if (!jit_data->ro_header) {
                                prog = orig_prog;
                                goto out_offset;
                        }
 
+                       /*
+                        * Use the image(RW) for writing the JITed instructions. But also save
+                        * the ro_image(RX) for calculating the offsets in the image. The RW
+                        * image will be later copied to the RX image from where the program
+                        * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
+                        * final step.
+                        */
+                       ctx->ro_insns = (u16 *)jit_data->ro_image;
                        ctx->insns = (u16 *)jit_data->image;
                        /*
                         * Now, when the image is allocated, the image can
@@ -138,14 +148,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
        if (i == NR_JIT_ITERATIONS) {
                pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
-               if (jit_data->header)
-                       bpf_jit_binary_free(jit_data->header);
                prog = orig_prog;
-               goto out_offset;
+               goto out_free_hdr;
        }
 
        if (extable_size)
-               prog->aux->extable = (void *)ctx->insns + prog_size;
+               prog->aux->extable = (void *)ctx->ro_insns + prog_size;
 
 skip_init_ctx:
        pass++;
@@ -154,23 +162,33 @@ skip_init_ctx:
 
        bpf_jit_build_prologue(ctx);
        if (build_body(ctx, extra_pass, NULL)) {
-               bpf_jit_binary_free(jit_data->header);
                prog = orig_prog;
-               goto out_offset;
+               goto out_free_hdr;
        }
        bpf_jit_build_epilogue(ctx);
 
        if (bpf_jit_enable > 1)
                bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
 
-       prog->bpf_func = (void *)ctx->insns;
+       prog->bpf_func = (void *)ctx->ro_insns;
        prog->jited = 1;
        prog->jited_len = prog_size;
 
-       bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
-
        if (!prog->is_func || extra_pass) {
-               bpf_jit_binary_lock_ro(jit_data->header);
+               if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
+                                                        jit_data->header))) {
+                       /* ro_header has been freed */
+                       jit_data->ro_header = NULL;
+                       prog = orig_prog;
+                       goto out_offset;
+               }
+               /*
+                * The instructions have now been copied to the ROX region from
+                * where they will execute.
+                * Write any modified data cache blocks out to memory and
+                * invalidate the corresponding blocks in the instruction cache.
+                */
+               bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
                for (i = 0; i < prog->len; i++)
                        ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
                bpf_prog_fill_jited_linfo(prog, ctx->offset);
@@ -185,6 +203,14 @@ out:
                bpf_jit_prog_release_other(prog, prog == orig_prog ?
                                           tmp : orig_prog);
        return prog;
+
+out_free_hdr:
+       if (jit_data->header) {
+               bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
+                                  sizeof(jit_data->header->size));
+               bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
+       }
+       goto out_offset;
 }
 
 u64 bpf_jit_alloc_exec_limit(void)
@@ -204,3 +230,51 @@ void bpf_jit_free_exec(void *addr)
 {
        return vfree(addr);
 }
+
+void *bpf_arch_text_copy(void *dst, void *src, size_t len)
+{
+       int ret;
+
+       mutex_lock(&text_mutex);
+       ret = patch_text_nosync(dst, src, len);
+       mutex_unlock(&text_mutex);
+
+       if (ret)
+               return ERR_PTR(-EINVAL);
+
+       return dst;
+}
+
+int bpf_arch_text_invalidate(void *dst, size_t len)
+{
+       int ret;
+
+       mutex_lock(&text_mutex);
+       ret = patch_text_set_nosync(dst, 0, len);
+       mutex_unlock(&text_mutex);
+
+       return ret;
+}
+
+void bpf_jit_free(struct bpf_prog *prog)
+{
+       if (prog->jited) {
+               struct rv_jit_data *jit_data = prog->aux->jit_data;
+               struct bpf_binary_header *hdr;
+
+               /*
+                * If we fail the final pass of JIT (from jit_subprogs),
+                * the program may not be finalized yet. Call finalize here
+                * before freeing it.
+                */
+               if (jit_data) {
+                       bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
+                       kfree(jit_data);
+               }
+               hdr = bpf_jit_binary_pack_hdr(prog);
+               bpf_jit_binary_pack_free(hdr, NULL);
+               WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
+       }
+
+       bpf_prog_unlock_free(prog);
+}