include/asm-generic/percpu.h

   1 #ifndef _ASM_GENERIC_PERCPU_H_
   2 #define _ASM_GENERIC_PERCPU_H_
   3
   4 #include <linux/compiler.h>
   5 #include <linux/threads.h>
   6 #include <linux/percpu-defs.h>
   7
   8 #ifdef CONFIG_SMP
   9
  10 /*
  11  * per_cpu_offset() is the offset that has to be added to a
  12  * percpu variable to get to the instance for a certain processor.
  13  *
  14  * Most arches use the __per_cpu_offset array for those offsets but
  15  * some arches have their own ways of determining the offset (x86_64, s390).
  16  */
  17 #ifndef __per_cpu_offset
  18 extern unsigned long __per_cpu_offset[NR_CPUS];
  19
  20 #define per_cpu_offset(x) (__per_cpu_offset[x])
  21 #endif
  22
  23 /*
  24  * Determine the offset for the currently active processor.
  25  * An arch may define __my_cpu_offset to provide a more effective
  26  * means of obtaining the offset to the per cpu variables of the
  27  * current processor.
  28  */
  29 #ifndef __my_cpu_offset
  30 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  31 #endif
  32 #ifdef CONFIG_DEBUG_PREEMPT
  33 #define my_cpu_offset per_cpu_offset(smp_processor_id())
  34 #else
  35 #define my_cpu_offset __my_cpu_offset
  36 #endif
  37
  38 /*
  39  * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  40  * translations for raw_cpu_ptr().
  41  */
  42 #ifndef arch_raw_cpu_ptr
  43 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  44 #endif
  45
  46 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  47 extern void setup_per_cpu_areas(void);
  48 #endif
  49
  50 #endif  /* SMP */
  51
  52 #ifndef PER_CPU_BASE_SECTION
  53 #ifdef CONFIG_SMP
  54 #define PER_CPU_BASE_SECTION ".data..percpu"
  55 #else
  56 #define PER_CPU_BASE_SECTION ".data"
  57 #endif
  58 #endif
  59
  60 #ifndef PER_CPU_ATTRIBUTES
  61 #define PER_CPU_ATTRIBUTES
  62 #endif
  63
  64 #ifndef PER_CPU_DEF_ATTRIBUTES
  65 #define PER_CPU_DEF_ATTRIBUTES
  66 #endif
  67
  68 #define raw_cpu_generic_read(pcp)                                       \
  69 ({                                                                      \
  70         *raw_cpu_ptr(&(pcp));                                           \
  71 })
  72
  73 #define raw_cpu_generic_to_op(pcp, val, op)                             \
  74 do {                                                                    \
  75         *raw_cpu_ptr(&(pcp)) op val;                                    \
  76 } while (0)
  77
  78 #define raw_cpu_generic_add_return(pcp, val)                            \
  79 ({                                                                      \
  80         typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  81                                                                         \
  82         *__p += val;                                                    \
  83         *__p;                                                           \
  84 })
  85
  86 #define raw_cpu_generic_xchg(pcp, nval)                                 \
  87 ({                                                                      \
  88         typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  89         typeof(pcp) __ret;                                              \
  90         __ret = *__p;                                                   \
  91         *__p = nval;                                                    \
  92         __ret;                                                          \
  93 })
  94
  95 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  96 ({                                                                      \
  97         typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  98         typeof(pcp) __ret;                                              \
  99         __ret = *__p;                                                   \
 100         if (__ret == (oval))                                            \
 101                 *__p = nval;                                            \
 102         __ret;                                                          \
 103 })
 104
 105 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 106 ({                                                                      \
 107         typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1));                    \
 108         typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2));                    \
 109         int __ret = 0;                                                  \
 110         if (*__p1 == (oval1) && *__p2  == (oval2)) {                    \
 111                 *__p1 = nval1;                                          \
 112                 *__p2 = nval2;                                          \
 113                 __ret = 1;                                              \
 114         }                                                               \
 115         (__ret);                                                        \
 116 })
 117
 118 #define __this_cpu_generic_read_nopreempt(pcp)                          \
 119 ({                                                                      \
 120         typeof(pcp) __ret;                                              \
 121         preempt_disable_notrace();                                      \
 122         __ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));                        \
 123         preempt_enable_notrace();                                       \
 124         __ret;                                                          \
 125 })
 126
 127 #define __this_cpu_generic_read_noirq(pcp)                              \
 128 ({                                                                      \
 129         typeof(pcp) __ret;                                              \
 130         unsigned long __flags;                                          \
 131         raw_local_irq_save(__flags);                                    \
 132         __ret = raw_cpu_generic_read(pcp);                              \
 133         raw_local_irq_restore(__flags);                                 \
 134         __ret;                                                          \
 135 })
 136
 137 #define this_cpu_generic_read(pcp)                                      \
 138 ({                                                                      \
 139         typeof(pcp) __ret;                                              \
 140         if (__native_word(pcp))                                         \
 141                 __ret = __this_cpu_generic_read_nopreempt(pcp);         \
 142         else                                                            \
 143                 __ret = __this_cpu_generic_read_noirq(pcp);             \
 144         __ret;                                                          \
 145 })
 146
 147 #define this_cpu_generic_to_op(pcp, val, op)                            \
 148 do {                                                                    \
 149         unsigned long __flags;                                          \
 150         raw_local_irq_save(__flags);                                    \
 151         raw_cpu_generic_to_op(pcp, val, op);                            \
 152         raw_local_irq_restore(__flags);                                 \
 153 } while (0)
 154
 155
 156 #define this_cpu_generic_add_return(pcp, val)                           \
 157 ({                                                                      \
 158         typeof(pcp) __ret;                                              \
 159         unsigned long __flags;                                          \
 160         raw_local_irq_save(__flags);                                    \
 161         __ret = raw_cpu_generic_add_return(pcp, val);                   \
 162         raw_local_irq_restore(__flags);                                 \
 163         __ret;                                                          \
 164 })
 165
 166 #define this_cpu_generic_xchg(pcp, nval)                                \
 167 ({                                                                      \
 168         typeof(pcp) __ret;                                              \
 169         unsigned long __flags;                                          \
 170         raw_local_irq_save(__flags);                                    \
 171         __ret = raw_cpu_generic_xchg(pcp, nval);                        \
 172         raw_local_irq_restore(__flags);                                 \
 173         __ret;                                                          \
 174 })
 175
 176 #define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 177 ({                                                                      \
 178         typeof(pcp) __ret;                                              \
 179         unsigned long __flags;                                          \
 180         raw_local_irq_save(__flags);                                    \
 181         __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);               \
 182         raw_local_irq_restore(__flags);                                 \
 183         __ret;                                                          \
 184 })
 185
 186 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 187 ({                                                                      \
 188         int __ret;                                                      \
 189         unsigned long __flags;                                          \
 190         raw_local_irq_save(__flags);                                    \
 191         __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 192                         oval1, oval2, nval1, nval2);                    \
 193         raw_local_irq_restore(__flags);                                 \
 194         __ret;                                                          \
 195 })
 196
 197 #ifndef raw_cpu_read_1
 198 #define raw_cpu_read_1(pcp)             raw_cpu_generic_read(pcp)
 199 #endif
 200 #ifndef raw_cpu_read_2
 201 #define raw_cpu_read_2(pcp)             raw_cpu_generic_read(pcp)
 202 #endif
 203 #ifndef raw_cpu_read_4
 204 #define raw_cpu_read_4(pcp)             raw_cpu_generic_read(pcp)
 205 #endif
 206 #ifndef raw_cpu_read_8
 207 #define raw_cpu_read_8(pcp)             raw_cpu_generic_read(pcp)
 208 #endif
 209
 210 #ifndef raw_cpu_write_1
 211 #define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 212 #endif
 213 #ifndef raw_cpu_write_2
 214 #define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 215 #endif
 216 #ifndef raw_cpu_write_4
 217 #define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 218 #endif
 219 #ifndef raw_cpu_write_8
 220 #define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 221 #endif
 222
 223 #ifndef raw_cpu_add_1
 224 #define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 225 #endif
 226 #ifndef raw_cpu_add_2
 227 #define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 228 #endif
 229 #ifndef raw_cpu_add_4
 230 #define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 231 #endif
 232 #ifndef raw_cpu_add_8
 233 #define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 234 #endif
 235
 236 #ifndef raw_cpu_and_1
 237 #define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 238 #endif
 239 #ifndef raw_cpu_and_2
 240 #define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 241 #endif
 242 #ifndef raw_cpu_and_4
 243 #define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 244 #endif
 245 #ifndef raw_cpu_and_8
 246 #define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 247 #endif
 248
 249 #ifndef raw_cpu_or_1
 250 #define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 251 #endif
 252 #ifndef raw_cpu_or_2
 253 #define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 254 #endif
 255 #ifndef raw_cpu_or_4
 256 #define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 257 #endif
 258 #ifndef raw_cpu_or_8
 259 #define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 260 #endif
 261
 262 #ifndef raw_cpu_add_return_1
 263 #define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 264 #endif
 265 #ifndef raw_cpu_add_return_2
 266 #define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 267 #endif
 268 #ifndef raw_cpu_add_return_4
 269 #define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 270 #endif
 271 #ifndef raw_cpu_add_return_8
 272 #define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 273 #endif
 274
 275 #ifndef raw_cpu_xchg_1
 276 #define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 277 #endif
 278 #ifndef raw_cpu_xchg_2
 279 #define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 280 #endif
 281 #ifndef raw_cpu_xchg_4
 282 #define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 283 #endif
 284 #ifndef raw_cpu_xchg_8
 285 #define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 286 #endif
 287
 288 #ifndef raw_cpu_cmpxchg_1
 289 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 290         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 291 #endif
 292 #ifndef raw_cpu_cmpxchg_2
 293 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 294         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 295 #endif
 296 #ifndef raw_cpu_cmpxchg_4
 297 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 298         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 299 #endif
 300 #ifndef raw_cpu_cmpxchg_8
 301 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 302         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 303 #endif
 304
 305 #ifndef raw_cpu_cmpxchg_double_1
 306 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 307         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 308 #endif
 309 #ifndef raw_cpu_cmpxchg_double_2
 310 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 311         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 312 #endif
 313 #ifndef raw_cpu_cmpxchg_double_4
 314 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 315         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 316 #endif
 317 #ifndef raw_cpu_cmpxchg_double_8
 318 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 319         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 320 #endif
 321
 322 #ifndef this_cpu_read_1
 323 #define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 324 #endif
 325 #ifndef this_cpu_read_2
 326 #define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 327 #endif
 328 #ifndef this_cpu_read_4
 329 #define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 330 #endif
 331 #ifndef this_cpu_read_8
 332 #define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 333 #endif
 334
 335 #ifndef this_cpu_write_1
 336 #define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 337 #endif
 338 #ifndef this_cpu_write_2
 339 #define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 340 #endif
 341 #ifndef this_cpu_write_4
 342 #define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 343 #endif
 344 #ifndef this_cpu_write_8
 345 #define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 346 #endif
 347
 348 #ifndef this_cpu_add_1
 349 #define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 350 #endif
 351 #ifndef this_cpu_add_2
 352 #define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 353 #endif
 354 #ifndef this_cpu_add_4
 355 #define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 356 #endif
 357 #ifndef this_cpu_add_8
 358 #define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 359 #endif
 360
 361 #ifndef this_cpu_and_1
 362 #define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 363 #endif
 364 #ifndef this_cpu_and_2
 365 #define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 366 #endif
 367 #ifndef this_cpu_and_4
 368 #define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 369 #endif
 370 #ifndef this_cpu_and_8
 371 #define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 372 #endif
 373
 374 #ifndef this_cpu_or_1
 375 #define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 376 #endif
 377 #ifndef this_cpu_or_2
 378 #define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 379 #endif
 380 #ifndef this_cpu_or_4
 381 #define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 382 #endif
 383 #ifndef this_cpu_or_8
 384 #define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 385 #endif
 386
 387 #ifndef this_cpu_add_return_1
 388 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 389 #endif
 390 #ifndef this_cpu_add_return_2
 391 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 392 #endif
 393 #ifndef this_cpu_add_return_4
 394 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 395 #endif
 396 #ifndef this_cpu_add_return_8
 397 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 398 #endif
 399
 400 #ifndef this_cpu_xchg_1
 401 #define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 402 #endif
 403 #ifndef this_cpu_xchg_2
 404 #define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 405 #endif
 406 #ifndef this_cpu_xchg_4
 407 #define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 408 #endif
 409 #ifndef this_cpu_xchg_8
 410 #define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 411 #endif
 412
 413 #ifndef this_cpu_cmpxchg_1
 414 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
 415         this_cpu_generic_cmpxchg(pcp, oval, nval)
 416 #endif
 417 #ifndef this_cpu_cmpxchg_2
 418 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
 419         this_cpu_generic_cmpxchg(pcp, oval, nval)
 420 #endif
 421 #ifndef this_cpu_cmpxchg_4
 422 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
 423         this_cpu_generic_cmpxchg(pcp, oval, nval)
 424 #endif
 425 #ifndef this_cpu_cmpxchg_8
 426 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
 427         this_cpu_generic_cmpxchg(pcp, oval, nval)
 428 #endif
 429
 430 #ifndef this_cpu_cmpxchg_double_1
 431 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 432         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 433 #endif
 434 #ifndef this_cpu_cmpxchg_double_2
 435 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 436         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 437 #endif
 438 #ifndef this_cpu_cmpxchg_double_4
 439 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 440         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 441 #endif
 442 #ifndef this_cpu_cmpxchg_double_8
 443 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 444         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 445 #endif
 446
 447 #endif /* _ASM_GENERIC_PERCPU_H_ */