arch/arm64/include/asm/percpu.h

   1 /*
   2  * Copyright (C) 2013 ARM Ltd.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16 #ifndef __ASM_PERCPU_H
  17 #define __ASM_PERCPU_H
  18
  19 #include <linux/preempt.h>
  20
  21 #include <asm/alternative.h>
  22 #include <asm/cmpxchg.h>
  23 #include <asm/stack_pointer.h>
  24
  25 static inline void set_my_cpu_offset(unsigned long off)
  26 {
  27         asm volatile(ALTERNATIVE("msr tpidr_el1, %0",
  28                                  "msr tpidr_el2, %0",
  29                                  ARM64_HAS_VIRT_HOST_EXTN)
  30                         :: "r" (off) : "memory");
  31 }
  32
  33 static inline unsigned long __my_cpu_offset(void)
  34 {
  35         unsigned long off;
  36
  37         /*
  38          * We want to allow caching the value, so avoid using volatile and
  39          * instead use a fake stack read to hazard against barrier().
  40          */
  41         asm(ALTERNATIVE("mrs %0, tpidr_el1",
  42                         "mrs %0, tpidr_el2",
  43                         ARM64_HAS_VIRT_HOST_EXTN)
  44                 : "=r" (off) :
  45                 "Q" (*(const unsigned long *)current_stack_pointer));
  46
  47         return off;
  48 }
  49 #define __my_cpu_offset __my_cpu_offset()
  50
  51 #define PERCPU_OP(op, asm_op)                                           \
  52 static inline unsigned long __percpu_##op(void *ptr,                    \
  53                         unsigned long val, int size)                    \
  54 {                                                                       \
  55         unsigned long loop, ret;                                        \
  56                                                                         \
  57         switch (size) {                                                 \
  58         case 1:                                                         \
  59                 asm ("//__per_cpu_" #op "_1\n"                          \
  60                 "1:     ldxrb     %w[ret], %[ptr]\n"                    \
  61                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  62                 "       stxrb     %w[loop], %w[ret], %[ptr]\n"          \
  63                 "       cbnz      %w[loop], 1b"                         \
  64                 : [loop] "=&r" (loop), [ret] "=&r" (ret),               \
  65                   [ptr] "+Q"(*(u8 *)ptr)                                \
  66                 : [val] "Ir" (val));                                    \
  67                 break;                                                  \
  68         case 2:                                                         \
  69                 asm ("//__per_cpu_" #op "_2\n"                          \
  70                 "1:     ldxrh     %w[ret], %[ptr]\n"                    \
  71                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  72                 "       stxrh     %w[loop], %w[ret], %[ptr]\n"          \
  73                 "       cbnz      %w[loop], 1b"                         \
  74                 : [loop] "=&r" (loop), [ret] "=&r" (ret),               \
  75                   [ptr]  "+Q"(*(u16 *)ptr)                              \
  76                 : [val] "Ir" (val));                                    \
  77                 break;                                                  \
  78         case 4:                                                         \
  79                 asm ("//__per_cpu_" #op "_4\n"                          \
  80                 "1:     ldxr      %w[ret], %[ptr]\n"                    \
  81                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  82                 "       stxr      %w[loop], %w[ret], %[ptr]\n"          \
  83                 "       cbnz      %w[loop], 1b"                         \
  84                 : [loop] "=&r" (loop), [ret] "=&r" (ret),               \
  85                   [ptr] "+Q"(*(u32 *)ptr)                               \
  86                 : [val] "Ir" (val));                                    \
  87                 break;                                                  \
  88         case 8:                                                         \
  89                 asm ("//__per_cpu_" #op "_8\n"                          \
  90                 "1:     ldxr      %[ret], %[ptr]\n"                     \
  91                         #asm_op " %[ret], %[ret], %[val]\n"             \
  92                 "       stxr      %w[loop], %[ret], %[ptr]\n"           \
  93                 "       cbnz      %w[loop], 1b"                         \
  94                 : [loop] "=&r" (loop), [ret] "=&r" (ret),               \
  95                   [ptr] "+Q"(*(u64 *)ptr)                               \
  96                 : [val] "Ir" (val));                                    \
  97                 break;                                                  \
  98         default:                                                        \
  99                 BUILD_BUG();                                            \
 100         }                                                               \
 101                                                                         \
 102         return ret;                                                     \
 103 }
 104
 105 PERCPU_OP(add, add)
 106 PERCPU_OP(and, and)
 107 PERCPU_OP(or, orr)
 108 #undef PERCPU_OP
 109
 110 static inline unsigned long __percpu_read(void *ptr, int size)
 111 {
 112         unsigned long ret;
 113
 114         switch (size) {
 115         case 1:
 116                 ret = READ_ONCE(*(u8 *)ptr);
 117                 break;
 118         case 2:
 119                 ret = READ_ONCE(*(u16 *)ptr);
 120                 break;
 121         case 4:
 122                 ret = READ_ONCE(*(u32 *)ptr);
 123                 break;
 124         case 8:
 125                 ret = READ_ONCE(*(u64 *)ptr);
 126                 break;
 127         default:
 128                 BUILD_BUG();
 129         }
 130
 131         return ret;
 132 }
 133
 134 static inline void __percpu_write(void *ptr, unsigned long val, int size)
 135 {
 136         switch (size) {
 137         case 1:
 138                 WRITE_ONCE(*(u8 *)ptr, (u8)val);
 139                 break;
 140         case 2:
 141                 WRITE_ONCE(*(u16 *)ptr, (u16)val);
 142                 break;
 143         case 4:
 144                 WRITE_ONCE(*(u32 *)ptr, (u32)val);
 145                 break;
 146         case 8:
 147                 WRITE_ONCE(*(u64 *)ptr, (u64)val);
 148                 break;
 149         default:
 150                 BUILD_BUG();
 151         }
 152 }
 153
 154 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
 155                                                 int size)
 156 {
 157         unsigned long ret, loop;
 158
 159         switch (size) {
 160         case 1:
 161                 asm ("//__percpu_xchg_1\n"
 162                 "1:     ldxrb   %w[ret], %[ptr]\n"
 163                 "       stxrb   %w[loop], %w[val], %[ptr]\n"
 164                 "       cbnz    %w[loop], 1b"
 165                 : [loop] "=&r"(loop), [ret] "=&r"(ret),
 166                   [ptr] "+Q"(*(u8 *)ptr)
 167                 : [val] "r" (val));
 168                 break;
 169         case 2:
 170                 asm ("//__percpu_xchg_2\n"
 171                 "1:     ldxrh   %w[ret], %[ptr]\n"
 172                 "       stxrh   %w[loop], %w[val], %[ptr]\n"
 173                 "       cbnz    %w[loop], 1b"
 174                 : [loop] "=&r"(loop), [ret] "=&r"(ret),
 175                   [ptr] "+Q"(*(u16 *)ptr)
 176                 : [val] "r" (val));
 177                 break;
 178         case 4:
 179                 asm ("//__percpu_xchg_4\n"
 180                 "1:     ldxr    %w[ret], %[ptr]\n"
 181                 "       stxr    %w[loop], %w[val], %[ptr]\n"
 182                 "       cbnz    %w[loop], 1b"
 183                 : [loop] "=&r"(loop), [ret] "=&r"(ret),
 184                   [ptr] "+Q"(*(u32 *)ptr)
 185                 : [val] "r" (val));
 186                 break;
 187         case 8:
 188                 asm ("//__percpu_xchg_8\n"
 189                 "1:     ldxr    %[ret], %[ptr]\n"
 190                 "       stxr    %w[loop], %[val], %[ptr]\n"
 191                 "       cbnz    %w[loop], 1b"
 192                 : [loop] "=&r"(loop), [ret] "=&r"(ret),
 193                   [ptr] "+Q"(*(u64 *)ptr)
 194                 : [val] "r" (val));
 195                 break;
 196         default:
 197                 BUILD_BUG();
 198         }
 199
 200         return ret;
 201 }
 202
 203 /* this_cpu_cmpxchg */
 204 #define _protect_cmpxchg_local(pcp, o, n)                       \
 205 ({                                                              \
 206         typeof(*raw_cpu_ptr(&(pcp))) __ret;                     \
 207         preempt_disable();                                      \
 208         __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);       \
 209         preempt_enable();                                       \
 210         __ret;                                                  \
 211 })
 212
 213 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 214 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 215 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 216 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 217
 218 #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2)           \
 219 ({                                                                      \
 220         int __ret;                                                      \
 221         preempt_disable();                                              \
 222         __ret = cmpxchg_double_local(   raw_cpu_ptr(&(ptr1)),           \
 223                                         raw_cpu_ptr(&(ptr2)),           \
 224                                         o1, o2, n1, n2);                \
 225         preempt_enable();                                               \
 226         __ret;                                                          \
 227 })
 228
 229 #define _percpu_read(pcp)                                               \
 230 ({                                                                      \
 231         typeof(pcp) __retval;                                           \
 232         preempt_disable_notrace();                                      \
 233         __retval = (typeof(pcp))__percpu_read(raw_cpu_ptr(&(pcp)),      \
 234                                               sizeof(pcp));             \
 235         preempt_enable_notrace();                                       \
 236         __retval;                                                       \
 237 })
 238
 239 #define _percpu_write(pcp, val)                                         \
 240 do {                                                                    \
 241         preempt_disable_notrace();                                      \
 242         __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val),       \
 243                                 sizeof(pcp));                           \
 244         preempt_enable_notrace();                                       \
 245 } while(0)                                                              \
 246
 247 #define _pcp_protect(operation, pcp, val)                       \
 248 ({                                                              \
 249         typeof(pcp) __retval;                                   \
 250         preempt_disable();                                      \
 251         __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),  \
 252                                           (val), sizeof(pcp));  \
 253         preempt_enable();                                       \
 254         __retval;                                               \
 255 })
 256
 257 #define _percpu_add(pcp, val) \
 258         _pcp_protect(__percpu_add, pcp, val)
 259
 260 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
 261
 262 #define _percpu_and(pcp, val) \
 263         _pcp_protect(__percpu_and, pcp, val)
 264
 265 #define _percpu_or(pcp, val) \
 266         _pcp_protect(__percpu_or, pcp, val)
 267
 268 #define _percpu_xchg(pcp, val) (typeof(pcp)) \
 269         _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))
 270
 271 #define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
 272 #define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
 273 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
 274 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
 275
 276 #define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
 277 #define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
 278 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
 279 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
 280
 281 #define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
 282 #define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
 283 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
 284 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
 285
 286 #define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
 287 #define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
 288 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
 289 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
 290
 291 #define this_cpu_read_1(pcp) _percpu_read(pcp)
 292 #define this_cpu_read_2(pcp) _percpu_read(pcp)
 293 #define this_cpu_read_4(pcp) _percpu_read(pcp)
 294 #define this_cpu_read_8(pcp) _percpu_read(pcp)
 295
 296 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
 297 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
 298 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
 299 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
 300
 301 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
 302 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
 303 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
 304 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
 305
 306 #include <asm-generic/percpu.h>
 307
 308 #endif /* __ASM_PERCPU_H */