bpf: fix pointer offsets in context for 32 bit

Currently, pointer offsets in three BPF context structures are
broken in two scenarios: i) 32 bit compiled applications running
on 64 bit kernels, and ii) LLVM compiled BPF programs running
on 32 bit kernels. The latter is due to BPF target machine being
strictly 64 bit. So in each of the cases the offsets will mismatch
in verifier when checking / rewriting context access. Fix this by
providing a helper macro __bpf_md_ptr() that will enforce padding
up to 64 bit and proper alignment, and for context access a macro
bpf_ctx_range_ptr() which will cover full 64 bit member range on
32 bit archs. For flow_keys, we additionally need to force the
size check to sizeof(__u64) as with other pointer types.

Fixes: d58e468b1112 ("flow_dissector: implements flow dissector BPF hook")
Fixes: 4f738adba30a ("bpf: create tcp_bpf_ulp allowing BPF to monitor socket TX/RX data")
Fixes: 2dbb9b9e6df6 ("bpf: Introduce BPF_PROG_TYPE_SK_REUSEPORT")
Reported-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: David S. Miller <davem@davemloft.net>
Tested-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 448dcc448f1fe59cb14de60969f71f8ba6c35278..795ff0b869bbf6403c0e89b87fdc7855e3bbb5cd 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -449,6 +449,13 @@ struct sock_reuseport;
        offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
 #define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2)                             \
        offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1
+#if BITS_PER_LONG == 64
+# define bpf_ctx_range_ptr(TYPE, MEMBER)                                       \
+       offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
+#else
+# define bpf_ctx_range_ptr(TYPE, MEMBER)                                       \
+       offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1
+#endif /* BITS_PER_LONG == 64 */
 
 #define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE)                           \
        ({                                                                      \

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 852dc17ab47a07f2580ade5f9e4a1130ee779c26..426b5c8a245ba4b07527cabb7bbbe638602a58b2 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2422,6 +2422,12 @@ enum bpf_lwt_encap_mode {
        BPF_LWT_ENCAP_SEG6_INLINE
 };
 
+#define __bpf_md_ptr(type, name)       \
+union {                                        \
+       type name;                      \
+       __u64 :64;                      \
+} __attribute__((aligned(8)))
+
 /* user accessible mirror of in-kernel sk_buff.
  * new fields can only be added to the end of this structure
  */
@@ -2456,7 +2462,7 @@ struct __sk_buff {
        /* ... here. */
 
        __u32 data_meta;
-       struct bpf_flow_keys *flow_keys;
+       __bpf_md_ptr(struct bpf_flow_keys *, flow_keys);
 };
 
 struct bpf_tunnel_key {
@@ -2572,8 +2578,8 @@ enum sk_action {
  * be added to the end of this structure
  */
 struct sk_msg_md {
-       void *data;
-       void *data_end;
+       __bpf_md_ptr(void *, data);
+       __bpf_md_ptr(void *, data_end);
 
        __u32 family;
        __u32 remote_ip4;       /* Stored in network byte order */
@@ -2589,8 +2595,9 @@ struct sk_reuseport_md {
         * Start of directly accessible data. It begins from
         * the tcp/udp header.
         */
-       void *data;
-       void *data_end;         /* End of directly accessible data */
+       __bpf_md_ptr(void *, data);
+       /* End of directly accessible data */
+       __bpf_md_ptr(void *, data_end);
        /*
         * Total length of packet (starting from the tcp/udp header).
         * Note that the directly accessible bytes (data_end - data)

diff --git a/net/core/filter.c b/net/core/filter.c
index 9a1327eb25faf2c73a37d253f88b19c6b92414d6..6ee605da990fcc15852da2162469df7e55bbce5c 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -5435,8 +5435,8 @@ static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type
                if (size != size_default)
                        return false;
                break;
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
-               if (size != sizeof(struct bpf_flow_keys *))
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
+               if (size != sizeof(__u64))
                        return false;
                break;
        default:
@@ -5464,7 +5464,7 @@ static bool sk_filter_is_valid_access(int off, int size,
        case bpf_ctx_range(struct __sk_buff, data):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, data_end):
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
                return false;
        }
@@ -5489,7 +5489,7 @@ static bool cg_skb_is_valid_access(int off, int size,
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range(struct __sk_buff, data_meta):
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                return false;
        case bpf_ctx_range(struct __sk_buff, data):
        case bpf_ctx_range(struct __sk_buff, data_end):
@@ -5530,7 +5530,7 @@ static bool lwt_is_valid_access(int off, int size,
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
        case bpf_ctx_range(struct __sk_buff, data_meta):
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                return false;
        }
 
@@ -5756,7 +5756,7 @@ static bool tc_cls_act_is_valid_access(int off, int size,
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
                return false;
        }
@@ -5958,7 +5958,7 @@ static bool sk_skb_is_valid_access(int off, int size,
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range(struct __sk_buff, data_meta):
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                return false;
        }
 
@@ -6039,7 +6039,7 @@ static bool flow_dissector_is_valid_access(int off, int size,
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
-       case bpf_ctx_range(struct __sk_buff, flow_keys):
+       case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                info->reg_type = PTR_TO_FLOW_KEYS;
                break;
        case bpf_ctx_range(struct __sk_buff, tc_classid):

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 852dc17ab47a07f2580ade5f9e4a1130ee779c26..426b5c8a245ba4b07527cabb7bbbe638602a58b2 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2422,6 +2422,12 @@ enum bpf_lwt_encap_mode {
        BPF_LWT_ENCAP_SEG6_INLINE
 };
 
+#define __bpf_md_ptr(type, name)       \
+union {                                        \
+       type name;                      \
+       __u64 :64;                      \
+} __attribute__((aligned(8)))
+
 /* user accessible mirror of in-kernel sk_buff.
  * new fields can only be added to the end of this structure
  */
@@ -2456,7 +2462,7 @@ struct __sk_buff {
        /* ... here. */
 
        __u32 data_meta;
-       struct bpf_flow_keys *flow_keys;
+       __bpf_md_ptr(struct bpf_flow_keys *, flow_keys);
 };
 
 struct bpf_tunnel_key {
@@ -2572,8 +2578,8 @@ enum sk_action {
  * be added to the end of this structure
  */
 struct sk_msg_md {
-       void *data;
-       void *data_end;
+       __bpf_md_ptr(void *, data);
+       __bpf_md_ptr(void *, data_end);
 
        __u32 family;
        __u32 remote_ip4;       /* Stored in network byte order */
@@ -2589,8 +2595,9 @@ struct sk_reuseport_md {
         * Start of directly accessible data. It begins from
         * the tcp/udp header.
         */
-       void *data;
-       void *data_end;         /* End of directly accessible data */
+       __bpf_md_ptr(void *, data);
+       /* End of directly accessible data */
+       __bpf_md_ptr(void *, data_end);
        /*
         * Total length of packet (starting from the tcp/udp header).
         * Note that the directly accessible bytes (data_end - data)