nfp: bpf: support u32 divide using reciprocal_div.h

NFP doesn't have integer divide instruction, this patch use reciprocal
algorithm (the basic one, reciprocal_div) to emulate it.

For each u32 divide, we would need 11 instructions to finish the operation.

  7 (for multiplication) + 4 (various ALUs) = 11

Given NFP only supports multiplication no bigger than u32, we'd require
divisor and dividend no bigger than that as well.

Also eBPF doesn't support signed divide and has enforced this on C language
level by failing compilation. However LLVM assembler hasn't enforced this,
so it is possible for negative constant to leak in as a BPF_K operand
through assembly code, we reject such cases as well.

Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Acked-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index f1b27c3a43471e4f8e205a7fbe156a1cc5d8a6da..7c9ee3d85907713546ff423c4c96c4400d58e69c 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -34,10 +34,11 @@
 #define pr_fmt(fmt)    "NFP net bpf: " fmt
 
 #include <linux/bug.h>
-#include <linux/kernel.h>
 #include <linux/bpf.h>
 #include <linux/filter.h>
+#include <linux/kernel.h>
 #include <linux/pkt_cls.h>
+#include <linux/reciprocal_div.h>
 #include <linux/unistd.h>
 
 #include "main.h"
@@ -1493,6 +1494,32 @@ wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
        return 0;
 }
 
+static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm)
+{
+       swreg dst_both = reg_both(dst), dst_a = reg_a(dst), dst_b = reg_a(dst);
+       struct reciprocal_value rvalue;
+       swreg tmp_b = imm_b(nfp_prog);
+       swreg magic;
+
+       if (imm > U32_MAX) {
+               wrp_immed(nfp_prog, dst_both, 0);
+               return 0;
+       }
+
+       rvalue = reciprocal_value(imm);
+       magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog));
+       wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a, magic,
+                   true);
+       emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB, tmp_b);
+       emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
+                SHF_SC_R_SHF, rvalue.sh1);
+       emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD, tmp_b);
+       emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
+                SHF_SC_R_SHF, rvalue.sh2);
+
+       return 0;
+}
+
 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
@@ -1807,6 +1834,21 @@ static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        return wrp_mul(nfp_prog, meta, true, false);
 }
 
+static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       const struct bpf_insn *insn = &meta->insn;
+
+       return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm);
+}
+
+static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       /* NOTE: verifier hook has rejected cases for which verifier doesn't
+        * know whether the source operand is constant or not.
+        */
+       return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src);
+}
+
 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        const struct bpf_insn *insn = &meta->insn;
@@ -2230,6 +2272,16 @@ static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        return wrp_mul(nfp_prog, meta, false, false);
 }
 
+static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       return div_reg64(nfp_prog, meta);
+}
+
+static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+       return div_imm64(nfp_prog, meta);
+}
+
 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
        u8 dst = meta->insn.dst_reg * 2;
@@ -2983,6 +3035,8 @@ static const instr_cb_t instr_cb[256] = {
        [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64,
        [BPF_ALU64 | BPF_MUL | BPF_X] = mul_reg64,
        [BPF_ALU64 | BPF_MUL | BPF_K] = mul_imm64,
+       [BPF_ALU64 | BPF_DIV | BPF_X] = div_reg64,
+       [BPF_ALU64 | BPF_DIV | BPF_K] = div_imm64,
        [BPF_ALU64 | BPF_NEG] =         neg_reg64,
        [BPF_ALU64 | BPF_LSH | BPF_X] = shl_reg64,
        [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64,
@@ -3004,6 +3058,8 @@ static const instr_cb_t instr_cb[256] = {
        [BPF_ALU | BPF_SUB | BPF_K] =   sub_imm,
        [BPF_ALU | BPF_MUL | BPF_X] =   mul_reg,
        [BPF_ALU | BPF_MUL | BPF_K] =   mul_imm,
+       [BPF_ALU | BPF_DIV | BPF_X] =   div_reg,
+       [BPF_ALU | BPF_DIV | BPF_K] =   div_imm,
        [BPF_ALU | BPF_NEG] =           neg_reg,
        [BPF_ALU | BPF_LSH | BPF_K] =   shl_imm,
        [BPF_ALU | BPF_END | BPF_X] =   end_reg32,

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h
index c10079b1a31221038eb463ef13250f9800790596..9845c1a2d4c26c08ca42baa17163edce9274a101 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h
@@ -399,6 +399,11 @@ static inline bool is_mbpf_mul(const struct nfp_insn_meta *meta)
        return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_MUL;
 }
 
+static inline bool is_mbpf_div(const struct nfp_insn_meta *meta)
+{
+       return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_DIV;
+}
+
 /**
  * struct nfp_prog - nfp BPF program
  * @bpf: backpointer to the bpf app priv structure

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 30d4f1580693c93a72a10a4496cc316ea859799f..49ba0d645d36df879d6885309603426c383a17e6 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -558,6 +558,37 @@ nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
                }
        }
 
+       /* NFP doesn't have divide instructions, we support divide by constant
+        * through reciprocal multiplication. Given NFP support multiplication
+        * no bigger than u32, we'd require divisor and dividend no bigger than
+        * that as well.
+        *
+        * Also eBPF doesn't support signed divide and has enforced this on C
+        * language level by failing compilation. However LLVM assembler hasn't
+        * enforced this, so it is possible for negative constant to leak in as
+        * a BPF_K operand through assembly code, we reject such cases as well.
+        */
+       if (is_mbpf_div(meta)) {
+               if (meta->umax_dst > U32_MAX) {
+                       pr_vlog(env, "dividend is not within u32 value range\n");
+                       return -EINVAL;
+               }
+               if (mbpf_src(meta) == BPF_X) {
+                       if (meta->umin_src != meta->umax_src) {
+                               pr_vlog(env, "divisor is not constant\n");
+                               return -EINVAL;
+                       }
+                       if (meta->umax_src > U32_MAX) {
+                               pr_vlog(env, "divisor is not within u32 value range\n");
+                               return -EINVAL;
+                       }
+               }
+               if (mbpf_src(meta) == BPF_K && meta->insn.imm < 0) {
+                       pr_vlog(env, "divide by negative constant is not supported\n");
+                       return -EINVAL;
+               }
+       }
+
        return 0;
 }