arch/powerpc/net/bpf_jit_asm.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /* bpf_jit.S: Packet/header access helper functions
   3  * for PPC64 BPF compiler.
   4  *
   5  * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
   6  */
   7
   8 #include <asm/ppc_asm.h>
   9 #include <asm/asm-compat.h>
  10 #include "bpf_jit32.h"
  11
  12 /*
  13  * All of these routines are called directly from generated code,
  14  * whose register usage is:
  15  *
  16  * r3           skb
  17  * r4,r5        A,X
  18  * r6           *** address parameter to helper ***
  19  * r7-r10       scratch
  20  * r14          skb->data
  21  * r15          skb headlen
  22  * r16-31       M[]
  23  */
  24
  25 /*
  26  * To consider: These helpers are so small it could be better to just
  27  * generate them inline.  Inline code can do the simple headlen check
  28  * then branch directly to slow_path_XXX if required.  (In fact, could
  29  * load a spare GPR with the address of slow_path_generic and pass size
  30  * as an argument, making the call site a mtlr, li and bllr.)
  31  */
  32         .globl  sk_load_word
  33 sk_load_word:
  34         PPC_LCMPI       r_addr, 0
  35         blt     bpf_slow_path_word_neg
  36         .globl  sk_load_word_positive_offset
  37 sk_load_word_positive_offset:
  38         /* Are we accessing past headlen? */
  39         subi    r_scratch1, r_HL, 4
  40         PPC_LCMP        r_scratch1, r_addr
  41         blt     bpf_slow_path_word
  42         /* Nope, just hitting the header.  cr0 here is eq or gt! */
  43 #ifdef __LITTLE_ENDIAN__
  44         lwbrx   r_A, r_D, r_addr
  45 #else
  46         lwzx    r_A, r_D, r_addr
  47 #endif
  48         blr     /* Return success, cr0 != LT */
  49
  50         .globl  sk_load_half
  51 sk_load_half:
  52         PPC_LCMPI       r_addr, 0
  53         blt     bpf_slow_path_half_neg
  54         .globl  sk_load_half_positive_offset
  55 sk_load_half_positive_offset:
  56         subi    r_scratch1, r_HL, 2
  57         PPC_LCMP        r_scratch1, r_addr
  58         blt     bpf_slow_path_half
  59 #ifdef __LITTLE_ENDIAN__
  60         lhbrx   r_A, r_D, r_addr
  61 #else
  62         lhzx    r_A, r_D, r_addr
  63 #endif
  64         blr
  65
  66         .globl  sk_load_byte
  67 sk_load_byte:
  68         PPC_LCMPI       r_addr, 0
  69         blt     bpf_slow_path_byte_neg
  70         .globl  sk_load_byte_positive_offset
  71 sk_load_byte_positive_offset:
  72         PPC_LCMP        r_HL, r_addr
  73         ble     bpf_slow_path_byte
  74         lbzx    r_A, r_D, r_addr
  75         blr
  76
  77 /*
  78  * BPF_LDX | BPF_B | BPF_MSH: ldxb  4*([offset]&0xf)
  79  * r_addr is the offset value
  80  */
  81         .globl sk_load_byte_msh
  82 sk_load_byte_msh:
  83         PPC_LCMPI       r_addr, 0
  84         blt     bpf_slow_path_byte_msh_neg
  85         .globl sk_load_byte_msh_positive_offset
  86 sk_load_byte_msh_positive_offset:
  87         PPC_LCMP        r_HL, r_addr
  88         ble     bpf_slow_path_byte_msh
  89         lbzx    r_X, r_D, r_addr
  90         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
  91         blr
  92
  93 /* Call out to skb_copy_bits:
  94  * We'll need to back up our volatile regs first; we have
  95  * local variable space at r1+(BPF_PPC_STACK_BASIC).
  96  * Allocate a new stack frame here to remain ABI-compliant in
  97  * stashing LR.
  98  */
  99 #define bpf_slow_path_common(SIZE)                              \
 100         mflr    r0;                                             \
 101         PPC_STL r0, PPC_LR_STKOFF(r1);                                  \
 102         /* R3 goes in parameter space of caller's frame */      \
 103         PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);           \
 104         PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);              \
 105         PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);              \
 106         addi    r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ);         \
 107         PPC_STLU        r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 108         /* R3 = r_skb, as passed */                             \
 109         mr      r4, r_addr;                                     \
 110         li      r6, SIZE;                                       \
 111         bl      skb_copy_bits;                                  \
 112         nop;                                                    \
 113         /* R3 = 0 on success */                                 \
 114         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 115         PPC_LL  r0, PPC_LR_STKOFF(r1);                                  \
 116         PPC_LL  r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);              \
 117         PPC_LL  r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);              \
 118         mtlr    r0;                                             \
 119         PPC_LCMPI       r3, 0;                                          \
 120         blt     bpf_error;      /* cr0 = LT */                  \
 121         PPC_LL  r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);           \
 122         /* Great success! */
 123
 124 bpf_slow_path_word:
 125         bpf_slow_path_common(4)
 126         /* Data value is on stack, and cr0 != LT */
 127         lwz     r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
 128         blr
 129
 130 bpf_slow_path_half:
 131         bpf_slow_path_common(2)
 132         lhz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 133         blr
 134
 135 bpf_slow_path_byte:
 136         bpf_slow_path_common(1)
 137         lbz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 138         blr
 139
 140 bpf_slow_path_byte_msh:
 141         bpf_slow_path_common(1)
 142         lbz     r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
 143         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 144         blr
 145
 146 /* Call out to bpf_internal_load_pointer_neg_helper:
 147  * We'll need to back up our volatile regs first; we have
 148  * local variable space at r1+(BPF_PPC_STACK_BASIC).
 149  * Allocate a new stack frame here to remain ABI-compliant in
 150  * stashing LR.
 151  */
 152 #define sk_negative_common(SIZE)                                \
 153         mflr    r0;                                             \
 154         PPC_STL r0, PPC_LR_STKOFF(r1);                                  \
 155         /* R3 goes in parameter space of caller's frame */      \
 156         PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);           \
 157         PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);              \
 158         PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);              \
 159         PPC_STLU        r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 160         /* R3 = r_skb, as passed */                             \
 161         mr      r4, r_addr;                                     \
 162         li      r5, SIZE;                                       \
 163         bl      bpf_internal_load_pointer_neg_helper;           \
 164         nop;                                                    \
 165         /* R3 != 0 on success */                                \
 166         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 167         PPC_LL  r0, PPC_LR_STKOFF(r1);                                  \
 168         PPC_LL  r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);              \
 169         PPC_LL  r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);              \
 170         mtlr    r0;                                             \
 171         PPC_LCMPLI      r3, 0;                                          \
 172         beq     bpf_error_slow; /* cr0 = EQ */                  \
 173         mr      r_addr, r3;                                     \
 174         PPC_LL  r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);           \
 175         /* Great success! */
 176
 177 bpf_slow_path_word_neg:
 178         lis     r_scratch1,-32  /* SKF_LL_OFF */
 179         PPC_LCMP        r_addr, r_scratch1      /* addr < SKF_* */
 180         blt     bpf_error       /* cr0 = LT */
 181         .globl  sk_load_word_negative_offset
 182 sk_load_word_negative_offset:
 183         sk_negative_common(4)
 184         lwz     r_A, 0(r_addr)
 185         blr
 186
 187 bpf_slow_path_half_neg:
 188         lis     r_scratch1,-32  /* SKF_LL_OFF */
 189         PPC_LCMP        r_addr, r_scratch1      /* addr < SKF_* */
 190         blt     bpf_error       /* cr0 = LT */
 191         .globl  sk_load_half_negative_offset
 192 sk_load_half_negative_offset:
 193         sk_negative_common(2)
 194         lhz     r_A, 0(r_addr)
 195         blr
 196
 197 bpf_slow_path_byte_neg:
 198         lis     r_scratch1,-32  /* SKF_LL_OFF */
 199         PPC_LCMP        r_addr, r_scratch1      /* addr < SKF_* */
 200         blt     bpf_error       /* cr0 = LT */
 201         .globl  sk_load_byte_negative_offset
 202 sk_load_byte_negative_offset:
 203         sk_negative_common(1)
 204         lbz     r_A, 0(r_addr)
 205         blr
 206
 207 bpf_slow_path_byte_msh_neg:
 208         lis     r_scratch1,-32  /* SKF_LL_OFF */
 209         PPC_LCMP        r_addr, r_scratch1      /* addr < SKF_* */
 210         blt     bpf_error       /* cr0 = LT */
 211         .globl  sk_load_byte_msh_negative_offset
 212 sk_load_byte_msh_negative_offset:
 213         sk_negative_common(1)
 214         lbz     r_X, 0(r_addr)
 215         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 216         blr
 217
 218 bpf_error_slow:
 219         /* fabricate a cr0 = lt */
 220         li      r_scratch1, -1
 221         PPC_LCMPI       r_scratch1, 0
 222 bpf_error:
 223         /* Entered with cr0 = lt */
 224         li      r3, 0
 225         /* Generated code will 'blt epilogue', returning 0. */
 226         blr