include/asm-generic/atomic-instrumented.h

   1 /*
   2  * This file provides wrappers with KASAN instrumentation for atomic operations.
   3  * To use this functionality an arch's atomic.h file needs to define all
   4  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
   5  * this file at the end. This file provides atomic_read() that forwards to
   6  * arch_atomic_read() for actual atomic operation.
   7  * Note: if an arch atomic operation is implemented by means of other atomic
   8  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
   9  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
  10  * double instrumentation.
  11  */
  12
  13 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
  14 #define _LINUX_ATOMIC_INSTRUMENTED_H
  15
  16 #include <linux/build_bug.h>
  17 #include <linux/kasan-checks.h>
  18
  19 static __always_inline int atomic_read(const atomic_t *v)
  20 {
  21         kasan_check_read(v, sizeof(*v));
  22         return arch_atomic_read(v);
  23 }
  24
  25 static __always_inline s64 atomic64_read(const atomic64_t *v)
  26 {
  27         kasan_check_read(v, sizeof(*v));
  28         return arch_atomic64_read(v);
  29 }
  30
  31 static __always_inline void atomic_set(atomic_t *v, int i)
  32 {
  33         kasan_check_write(v, sizeof(*v));
  34         arch_atomic_set(v, i);
  35 }
  36
  37 static __always_inline void atomic64_set(atomic64_t *v, s64 i)
  38 {
  39         kasan_check_write(v, sizeof(*v));
  40         arch_atomic64_set(v, i);
  41 }
  42
  43 static __always_inline int atomic_xchg(atomic_t *v, int i)
  44 {
  45         kasan_check_write(v, sizeof(*v));
  46         return arch_atomic_xchg(v, i);
  47 }
  48
  49 static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i)
  50 {
  51         kasan_check_write(v, sizeof(*v));
  52         return arch_atomic64_xchg(v, i);
  53 }
  54
  55 static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
  56 {
  57         kasan_check_write(v, sizeof(*v));
  58         return arch_atomic_cmpxchg(v, old, new);
  59 }
  60
  61 static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
  62 {
  63         kasan_check_write(v, sizeof(*v));
  64         return arch_atomic64_cmpxchg(v, old, new);
  65 }
  66
  67 #ifdef arch_atomic_try_cmpxchg
  68 #define atomic_try_cmpxchg atomic_try_cmpxchg
  69 static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
  70 {
  71         kasan_check_write(v, sizeof(*v));
  72         kasan_check_read(old, sizeof(*old));
  73         return arch_atomic_try_cmpxchg(v, old, new);
  74 }
  75 #endif
  76
  77 #ifdef arch_atomic64_try_cmpxchg
  78 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
  79 static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
  80 {
  81         kasan_check_write(v, sizeof(*v));
  82         kasan_check_read(old, sizeof(*old));
  83         return arch_atomic64_try_cmpxchg(v, old, new);
  84 }
  85 #endif
  86
  87 static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
  88 {
  89         kasan_check_write(v, sizeof(*v));
  90         return __arch_atomic_add_unless(v, a, u);
  91 }
  92
  93
  94 static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
  95 {
  96         kasan_check_write(v, sizeof(*v));
  97         return arch_atomic64_add_unless(v, a, u);
  98 }
  99
 100 static __always_inline void atomic_inc(atomic_t *v)
 101 {
 102         kasan_check_write(v, sizeof(*v));
 103         arch_atomic_inc(v);
 104 }
 105
 106 static __always_inline void atomic64_inc(atomic64_t *v)
 107 {
 108         kasan_check_write(v, sizeof(*v));
 109         arch_atomic64_inc(v);
 110 }
 111
 112 static __always_inline void atomic_dec(atomic_t *v)
 113 {
 114         kasan_check_write(v, sizeof(*v));
 115         arch_atomic_dec(v);
 116 }
 117
 118 static __always_inline void atomic64_dec(atomic64_t *v)
 119 {
 120         kasan_check_write(v, sizeof(*v));
 121         arch_atomic64_dec(v);
 122 }
 123
 124 static __always_inline void atomic_add(int i, atomic_t *v)
 125 {
 126         kasan_check_write(v, sizeof(*v));
 127         arch_atomic_add(i, v);
 128 }
 129
 130 static __always_inline void atomic64_add(s64 i, atomic64_t *v)
 131 {
 132         kasan_check_write(v, sizeof(*v));
 133         arch_atomic64_add(i, v);
 134 }
 135
 136 static __always_inline void atomic_sub(int i, atomic_t *v)
 137 {
 138         kasan_check_write(v, sizeof(*v));
 139         arch_atomic_sub(i, v);
 140 }
 141
 142 static __always_inline void atomic64_sub(s64 i, atomic64_t *v)
 143 {
 144         kasan_check_write(v, sizeof(*v));
 145         arch_atomic64_sub(i, v);
 146 }
 147
 148 static __always_inline void atomic_and(int i, atomic_t *v)
 149 {
 150         kasan_check_write(v, sizeof(*v));
 151         arch_atomic_and(i, v);
 152 }
 153
 154 static __always_inline void atomic64_and(s64 i, atomic64_t *v)
 155 {
 156         kasan_check_write(v, sizeof(*v));
 157         arch_atomic64_and(i, v);
 158 }
 159
 160 static __always_inline void atomic_or(int i, atomic_t *v)
 161 {
 162         kasan_check_write(v, sizeof(*v));
 163         arch_atomic_or(i, v);
 164 }
 165
 166 static __always_inline void atomic64_or(s64 i, atomic64_t *v)
 167 {
 168         kasan_check_write(v, sizeof(*v));
 169         arch_atomic64_or(i, v);
 170 }
 171
 172 static __always_inline void atomic_xor(int i, atomic_t *v)
 173 {
 174         kasan_check_write(v, sizeof(*v));
 175         arch_atomic_xor(i, v);
 176 }
 177
 178 static __always_inline void atomic64_xor(s64 i, atomic64_t *v)
 179 {
 180         kasan_check_write(v, sizeof(*v));
 181         arch_atomic64_xor(i, v);
 182 }
 183
 184 static __always_inline int atomic_inc_return(atomic_t *v)
 185 {
 186         kasan_check_write(v, sizeof(*v));
 187         return arch_atomic_inc_return(v);
 188 }
 189
 190 static __always_inline s64 atomic64_inc_return(atomic64_t *v)
 191 {
 192         kasan_check_write(v, sizeof(*v));
 193         return arch_atomic64_inc_return(v);
 194 }
 195
 196 static __always_inline int atomic_dec_return(atomic_t *v)
 197 {
 198         kasan_check_write(v, sizeof(*v));
 199         return arch_atomic_dec_return(v);
 200 }
 201
 202 static __always_inline s64 atomic64_dec_return(atomic64_t *v)
 203 {
 204         kasan_check_write(v, sizeof(*v));
 205         return arch_atomic64_dec_return(v);
 206 }
 207
 208 static __always_inline s64 atomic64_inc_not_zero(atomic64_t *v)
 209 {
 210         kasan_check_write(v, sizeof(*v));
 211         return arch_atomic64_inc_not_zero(v);
 212 }
 213
 214 static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v)
 215 {
 216         kasan_check_write(v, sizeof(*v));
 217         return arch_atomic64_dec_if_positive(v);
 218 }
 219
 220 static __always_inline bool atomic_dec_and_test(atomic_t *v)
 221 {
 222         kasan_check_write(v, sizeof(*v));
 223         return arch_atomic_dec_and_test(v);
 224 }
 225
 226 static __always_inline bool atomic64_dec_and_test(atomic64_t *v)
 227 {
 228         kasan_check_write(v, sizeof(*v));
 229         return arch_atomic64_dec_and_test(v);
 230 }
 231
 232 static __always_inline bool atomic_inc_and_test(atomic_t *v)
 233 {
 234         kasan_check_write(v, sizeof(*v));
 235         return arch_atomic_inc_and_test(v);
 236 }
 237
 238 static __always_inline bool atomic64_inc_and_test(atomic64_t *v)
 239 {
 240         kasan_check_write(v, sizeof(*v));
 241         return arch_atomic64_inc_and_test(v);
 242 }
 243
 244 static __always_inline int atomic_add_return(int i, atomic_t *v)
 245 {
 246         kasan_check_write(v, sizeof(*v));
 247         return arch_atomic_add_return(i, v);
 248 }
 249
 250 static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v)
 251 {
 252         kasan_check_write(v, sizeof(*v));
 253         return arch_atomic64_add_return(i, v);
 254 }
 255
 256 static __always_inline int atomic_sub_return(int i, atomic_t *v)
 257 {
 258         kasan_check_write(v, sizeof(*v));
 259         return arch_atomic_sub_return(i, v);
 260 }
 261
 262 static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v)
 263 {
 264         kasan_check_write(v, sizeof(*v));
 265         return arch_atomic64_sub_return(i, v);
 266 }
 267
 268 static __always_inline int atomic_fetch_add(int i, atomic_t *v)
 269 {
 270         kasan_check_write(v, sizeof(*v));
 271         return arch_atomic_fetch_add(i, v);
 272 }
 273
 274 static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v)
 275 {
 276         kasan_check_write(v, sizeof(*v));
 277         return arch_atomic64_fetch_add(i, v);
 278 }
 279
 280 static __always_inline int atomic_fetch_sub(int i, atomic_t *v)
 281 {
 282         kasan_check_write(v, sizeof(*v));
 283         return arch_atomic_fetch_sub(i, v);
 284 }
 285
 286 static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v)
 287 {
 288         kasan_check_write(v, sizeof(*v));
 289         return arch_atomic64_fetch_sub(i, v);
 290 }
 291
 292 static __always_inline int atomic_fetch_and(int i, atomic_t *v)
 293 {
 294         kasan_check_write(v, sizeof(*v));
 295         return arch_atomic_fetch_and(i, v);
 296 }
 297
 298 static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v)
 299 {
 300         kasan_check_write(v, sizeof(*v));
 301         return arch_atomic64_fetch_and(i, v);
 302 }
 303
 304 static __always_inline int atomic_fetch_or(int i, atomic_t *v)
 305 {
 306         kasan_check_write(v, sizeof(*v));
 307         return arch_atomic_fetch_or(i, v);
 308 }
 309
 310 static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v)
 311 {
 312         kasan_check_write(v, sizeof(*v));
 313         return arch_atomic64_fetch_or(i, v);
 314 }
 315
 316 static __always_inline int atomic_fetch_xor(int i, atomic_t *v)
 317 {
 318         kasan_check_write(v, sizeof(*v));
 319         return arch_atomic_fetch_xor(i, v);
 320 }
 321
 322 static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v)
 323 {
 324         kasan_check_write(v, sizeof(*v));
 325         return arch_atomic64_fetch_xor(i, v);
 326 }
 327
 328 static __always_inline bool atomic_sub_and_test(int i, atomic_t *v)
 329 {
 330         kasan_check_write(v, sizeof(*v));
 331         return arch_atomic_sub_and_test(i, v);
 332 }
 333
 334 static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v)
 335 {
 336         kasan_check_write(v, sizeof(*v));
 337         return arch_atomic64_sub_and_test(i, v);
 338 }
 339
 340 static __always_inline bool atomic_add_negative(int i, atomic_t *v)
 341 {
 342         kasan_check_write(v, sizeof(*v));
 343         return arch_atomic_add_negative(i, v);
 344 }
 345
 346 static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v)
 347 {
 348         kasan_check_write(v, sizeof(*v));
 349         return arch_atomic64_add_negative(i, v);
 350 }
 351
 352 static __always_inline unsigned long
 353 cmpxchg_size(volatile void *ptr, unsigned long old, unsigned long new, int size)
 354 {
 355         kasan_check_write(ptr, size);
 356         switch (size) {
 357         case 1:
 358                 return arch_cmpxchg((u8 *)ptr, (u8)old, (u8)new);
 359         case 2:
 360                 return arch_cmpxchg((u16 *)ptr, (u16)old, (u16)new);
 361         case 4:
 362                 return arch_cmpxchg((u32 *)ptr, (u32)old, (u32)new);
 363         case 8:
 364                 BUILD_BUG_ON(sizeof(unsigned long) != 8);
 365                 return arch_cmpxchg((u64 *)ptr, (u64)old, (u64)new);
 366         }
 367         BUILD_BUG();
 368         return 0;
 369 }
 370
 371 #define cmpxchg(ptr, old, new)                                          \
 372 ({                                                                      \
 373         ((__typeof__(*(ptr)))cmpxchg_size((ptr), (unsigned long)(old),  \
 374                 (unsigned long)(new), sizeof(*(ptr))));                 \
 375 })
 376
 377 static __always_inline unsigned long
 378 sync_cmpxchg_size(volatile void *ptr, unsigned long old, unsigned long new,
 379                   int size)
 380 {
 381         kasan_check_write(ptr, size);
 382         switch (size) {
 383         case 1:
 384                 return arch_sync_cmpxchg((u8 *)ptr, (u8)old, (u8)new);
 385         case 2:
 386                 return arch_sync_cmpxchg((u16 *)ptr, (u16)old, (u16)new);
 387         case 4:
 388                 return arch_sync_cmpxchg((u32 *)ptr, (u32)old, (u32)new);
 389         case 8:
 390                 BUILD_BUG_ON(sizeof(unsigned long) != 8);
 391                 return arch_sync_cmpxchg((u64 *)ptr, (u64)old, (u64)new);
 392         }
 393         BUILD_BUG();
 394         return 0;
 395 }
 396
 397 #define sync_cmpxchg(ptr, old, new)                                     \
 398 ({                                                                      \
 399         ((__typeof__(*(ptr)))sync_cmpxchg_size((ptr),                   \
 400                 (unsigned long)(old), (unsigned long)(new),             \
 401                 sizeof(*(ptr))));                                       \
 402 })
 403
 404 static __always_inline unsigned long
 405 cmpxchg_local_size(volatile void *ptr, unsigned long old, unsigned long new,
 406                    int size)
 407 {
 408         kasan_check_write(ptr, size);
 409         switch (size) {
 410         case 1:
 411                 return arch_cmpxchg_local((u8 *)ptr, (u8)old, (u8)new);
 412         case 2:
 413                 return arch_cmpxchg_local((u16 *)ptr, (u16)old, (u16)new);
 414         case 4:
 415                 return arch_cmpxchg_local((u32 *)ptr, (u32)old, (u32)new);
 416         case 8:
 417                 BUILD_BUG_ON(sizeof(unsigned long) != 8);
 418                 return arch_cmpxchg_local((u64 *)ptr, (u64)old, (u64)new);
 419         }
 420         BUILD_BUG();
 421         return 0;
 422 }
 423
 424 #define cmpxchg_local(ptr, old, new)                                    \
 425 ({                                                                      \
 426         ((__typeof__(*(ptr)))cmpxchg_local_size((ptr),                  \
 427                 (unsigned long)(old), (unsigned long)(new),             \
 428                 sizeof(*(ptr))));                                       \
 429 })
 430
 431 static __always_inline u64
 432 cmpxchg64_size(volatile u64 *ptr, u64 old, u64 new)
 433 {
 434         kasan_check_write(ptr, sizeof(*ptr));
 435         return arch_cmpxchg64(ptr, old, new);
 436 }
 437
 438 #define cmpxchg64(ptr, old, new)                                        \
 439 ({                                                                      \
 440         ((__typeof__(*(ptr)))cmpxchg64_size((ptr), (u64)(old),          \
 441                 (u64)(new)));                                           \
 442 })
 443
 444 static __always_inline u64
 445 cmpxchg64_local_size(volatile u64 *ptr, u64 old, u64 new)
 446 {
 447         kasan_check_write(ptr, sizeof(*ptr));
 448         return arch_cmpxchg64_local(ptr, old, new);
 449 }
 450
 451 #define cmpxchg64_local(ptr, old, new)                                  \
 452 ({                                                                      \
 453         ((__typeof__(*(ptr)))cmpxchg64_local_size((ptr), (u64)(old),    \
 454                 (u64)(new)));                                           \
 455 })
 456
 457 /*
 458  * Originally we had the following code here:
 459  *     __typeof__(p1) ____p1 = (p1);
 460  *     kasan_check_write(____p1, 2 * sizeof(*____p1));
 461  *     arch_cmpxchg_double(____p1, (p2), (o1), (o2), (n1), (n2));
 462  * But it leads to compilation failures (see gcc issue 72873).
 463  * So for now it's left non-instrumented.
 464  * There are few callers of cmpxchg_double(), so it's not critical.
 465  */
 466 #define cmpxchg_double(p1, p2, o1, o2, n1, n2)                          \
 467 ({                                                                      \
 468         arch_cmpxchg_double((p1), (p2), (o1), (o2), (n1), (n2));        \
 469 })
 470
 471 #define cmpxchg_double_local(p1, p2, o1, o2, n1, n2)                    \
 472 ({                                                                      \
 473         arch_cmpxchg_double_local((p1), (p2), (o1), (o2), (n1), (n2));  \
 474 })
 475
 476 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */