include/asm-generic/atomic-instrumented.h

   1 /*
   2  * This file provides wrappers with KASAN instrumentation for atomic operations.
   3  * To use this functionality an arch's atomic.h file needs to define all
   4  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
   5  * this file at the end. This file provides atomic_read() that forwards to
   6  * arch_atomic_read() for actual atomic operation.
   7  * Note: if an arch atomic operation is implemented by means of other atomic
   8  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
   9  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
  10  * double instrumentation.
  11  */
  12
  13 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
  14 #define _LINUX_ATOMIC_INSTRUMENTED_H
  15
  16 #include <linux/build_bug.h>
  17 #include <linux/kasan-checks.h>
  18
  19 static __always_inline int atomic_read(const atomic_t *v)
  20 {
  21         kasan_check_read(v, sizeof(*v));
  22         return arch_atomic_read(v);
  23 }
  24
  25 static __always_inline s64 atomic64_read(const atomic64_t *v)
  26 {
  27         kasan_check_read(v, sizeof(*v));
  28         return arch_atomic64_read(v);
  29 }
  30
  31 static __always_inline void atomic_set(atomic_t *v, int i)
  32 {
  33         kasan_check_write(v, sizeof(*v));
  34         arch_atomic_set(v, i);
  35 }
  36
  37 static __always_inline void atomic64_set(atomic64_t *v, s64 i)
  38 {
  39         kasan_check_write(v, sizeof(*v));
  40         arch_atomic64_set(v, i);
  41 }
  42
  43 static __always_inline int atomic_xchg(atomic_t *v, int i)
  44 {
  45         kasan_check_write(v, sizeof(*v));
  46         return arch_atomic_xchg(v, i);
  47 }
  48
  49 static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i)
  50 {
  51         kasan_check_write(v, sizeof(*v));
  52         return arch_atomic64_xchg(v, i);
  53 }
  54
  55 static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
  56 {
  57         kasan_check_write(v, sizeof(*v));
  58         return arch_atomic_cmpxchg(v, old, new);
  59 }
  60
  61 static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
  62 {
  63         kasan_check_write(v, sizeof(*v));
  64         return arch_atomic64_cmpxchg(v, old, new);
  65 }
  66
  67 #ifdef arch_atomic_try_cmpxchg
  68 #define atomic_try_cmpxchg atomic_try_cmpxchg
  69 static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
  70 {
  71         kasan_check_write(v, sizeof(*v));
  72         kasan_check_read(old, sizeof(*old));
  73         return arch_atomic_try_cmpxchg(v, old, new);
  74 }
  75 #endif
  76
  77 #ifdef arch_atomic64_try_cmpxchg
  78 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
  79 static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
  80 {
  81         kasan_check_write(v, sizeof(*v));
  82         kasan_check_read(old, sizeof(*old));
  83         return arch_atomic64_try_cmpxchg(v, old, new);
  84 }
  85 #endif
  86
  87 #ifdef arch_atomic_fetch_add_unless
  88 #define atomic_fetch_add_unless atomic_fetch_add_unless
  89 static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
  90 {
  91         kasan_check_write(v, sizeof(*v));
  92         return arch_atomic_fetch_add_unless(v, a, u);
  93 }
  94 #endif
  95
  96 #ifdef arch_atomic64_fetch_add_unless
  97 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
  98 static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
  99 {
 100         kasan_check_write(v, sizeof(*v));
 101         return arch_atomic64_fetch_add_unless(v, a, u);
 102 }
 103 #endif
 104
 105 #ifdef arch_atomic_inc
 106 #define atomic_inc atomic_inc
 107 static __always_inline void atomic_inc(atomic_t *v)
 108 {
 109         kasan_check_write(v, sizeof(*v));
 110         arch_atomic_inc(v);
 111 }
 112 #endif
 113
 114 #ifdef arch_atomic64_inc
 115 #define atomic64_inc atomic64_inc
 116 static __always_inline void atomic64_inc(atomic64_t *v)
 117 {
 118         kasan_check_write(v, sizeof(*v));
 119         arch_atomic64_inc(v);
 120 }
 121 #endif
 122
 123 #ifdef arch_atomic_dec
 124 #define atomic_dec atomic_dec
 125 static __always_inline void atomic_dec(atomic_t *v)
 126 {
 127         kasan_check_write(v, sizeof(*v));
 128         arch_atomic_dec(v);
 129 }
 130 #endif
 131
 132 #ifdef atch_atomic64_dec
 133 #define atomic64_dec
 134 static __always_inline void atomic64_dec(atomic64_t *v)
 135 {
 136         kasan_check_write(v, sizeof(*v));
 137         arch_atomic64_dec(v);
 138 }
 139 #endif
 140
 141 static __always_inline void atomic_add(int i, atomic_t *v)
 142 {
 143         kasan_check_write(v, sizeof(*v));
 144         arch_atomic_add(i, v);
 145 }
 146
 147 static __always_inline void atomic64_add(s64 i, atomic64_t *v)
 148 {
 149         kasan_check_write(v, sizeof(*v));
 150         arch_atomic64_add(i, v);
 151 }
 152
 153 static __always_inline void atomic_sub(int i, atomic_t *v)
 154 {
 155         kasan_check_write(v, sizeof(*v));
 156         arch_atomic_sub(i, v);
 157 }
 158
 159 static __always_inline void atomic64_sub(s64 i, atomic64_t *v)
 160 {
 161         kasan_check_write(v, sizeof(*v));
 162         arch_atomic64_sub(i, v);
 163 }
 164
 165 static __always_inline void atomic_and(int i, atomic_t *v)
 166 {
 167         kasan_check_write(v, sizeof(*v));
 168         arch_atomic_and(i, v);
 169 }
 170
 171 static __always_inline void atomic64_and(s64 i, atomic64_t *v)
 172 {
 173         kasan_check_write(v, sizeof(*v));
 174         arch_atomic64_and(i, v);
 175 }
 176
 177 static __always_inline void atomic_or(int i, atomic_t *v)
 178 {
 179         kasan_check_write(v, sizeof(*v));
 180         arch_atomic_or(i, v);
 181 }
 182
 183 static __always_inline void atomic64_or(s64 i, atomic64_t *v)
 184 {
 185         kasan_check_write(v, sizeof(*v));
 186         arch_atomic64_or(i, v);
 187 }
 188
 189 static __always_inline void atomic_xor(int i, atomic_t *v)
 190 {
 191         kasan_check_write(v, sizeof(*v));
 192         arch_atomic_xor(i, v);
 193 }
 194
 195 static __always_inline void atomic64_xor(s64 i, atomic64_t *v)
 196 {
 197         kasan_check_write(v, sizeof(*v));
 198         arch_atomic64_xor(i, v);
 199 }
 200
 201 #ifdef arch_atomic_inc_return
 202 #define atomic_inc_return atomic_inc_return
 203 static __always_inline int atomic_inc_return(atomic_t *v)
 204 {
 205         kasan_check_write(v, sizeof(*v));
 206         return arch_atomic_inc_return(v);
 207 }
 208 #endif
 209
 210 #ifdef arch_atomic64_in_return
 211 #define atomic64_inc_return atomic64_inc_return
 212 static __always_inline s64 atomic64_inc_return(atomic64_t *v)
 213 {
 214         kasan_check_write(v, sizeof(*v));
 215         return arch_atomic64_inc_return(v);
 216 }
 217 #endif
 218
 219 #ifdef arch_atomic_dec_return
 220 #define atomic_dec_return atomic_dec_return
 221 static __always_inline int atomic_dec_return(atomic_t *v)
 222 {
 223         kasan_check_write(v, sizeof(*v));
 224         return arch_atomic_dec_return(v);
 225 }
 226 #endif
 227
 228 #ifdef arch_atomic64_dec_return
 229 #define atomic64_dec_return atomic64_dec_return
 230 static __always_inline s64 atomic64_dec_return(atomic64_t *v)
 231 {
 232         kasan_check_write(v, sizeof(*v));
 233         return arch_atomic64_dec_return(v);
 234 }
 235 #endif
 236
 237 #ifdef arch_atomic64_inc_not_zero
 238 #define atomic64_inc_not_zero atomic64_inc_not_zero
 239 static __always_inline bool atomic64_inc_not_zero(atomic64_t *v)
 240 {
 241         kasan_check_write(v, sizeof(*v));
 242         return arch_atomic64_inc_not_zero(v);
 243 }
 244 #endif
 245
 246 #ifdef arch_atomic64_dec_if_positive
 247 #define atomic64_dec_if_positive atomic64_dec_if_positive
 248 static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v)
 249 {
 250         kasan_check_write(v, sizeof(*v));
 251         return arch_atomic64_dec_if_positive(v);
 252 }
 253 #endif
 254
 255 #ifdef arch_atomic_dec_and_test
 256 #define atomic_dec_and_test atomic_dec_and_test
 257 static __always_inline bool atomic_dec_and_test(atomic_t *v)
 258 {
 259         kasan_check_write(v, sizeof(*v));
 260         return arch_atomic_dec_and_test(v);
 261 }
 262 #endif
 263
 264 #ifdef arch_atomic64_dec_and_test
 265 #define atomic64_dec_and_test atomic64_dec_and_test
 266 static __always_inline bool atomic64_dec_and_test(atomic64_t *v)
 267 {
 268         kasan_check_write(v, sizeof(*v));
 269         return arch_atomic64_dec_and_test(v);
 270 }
 271 #endif
 272
 273 #ifdef arch_atomic_inc_and_test
 274 #define atomic_inc_and_test atomic_inc_and_test
 275 static __always_inline bool atomic_inc_and_test(atomic_t *v)
 276 {
 277         kasan_check_write(v, sizeof(*v));
 278         return arch_atomic_inc_and_test(v);
 279 }
 280 #endif
 281
 282 #ifdef arch_atomic64_inc_and_test
 283 #define atomic64_inc_and_test atomic64_inc_and_test
 284 static __always_inline bool atomic64_inc_and_test(atomic64_t *v)
 285 {
 286         kasan_check_write(v, sizeof(*v));
 287         return arch_atomic64_inc_and_test(v);
 288 }
 289 #endif
 290
 291 static __always_inline int atomic_add_return(int i, atomic_t *v)
 292 {
 293         kasan_check_write(v, sizeof(*v));
 294         return arch_atomic_add_return(i, v);
 295 }
 296
 297 static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v)
 298 {
 299         kasan_check_write(v, sizeof(*v));
 300         return arch_atomic64_add_return(i, v);
 301 }
 302
 303 static __always_inline int atomic_sub_return(int i, atomic_t *v)
 304 {
 305         kasan_check_write(v, sizeof(*v));
 306         return arch_atomic_sub_return(i, v);
 307 }
 308
 309 static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v)
 310 {
 311         kasan_check_write(v, sizeof(*v));
 312         return arch_atomic64_sub_return(i, v);
 313 }
 314
 315 static __always_inline int atomic_fetch_add(int i, atomic_t *v)
 316 {
 317         kasan_check_write(v, sizeof(*v));
 318         return arch_atomic_fetch_add(i, v);
 319 }
 320
 321 static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v)
 322 {
 323         kasan_check_write(v, sizeof(*v));
 324         return arch_atomic64_fetch_add(i, v);
 325 }
 326
 327 static __always_inline int atomic_fetch_sub(int i, atomic_t *v)
 328 {
 329         kasan_check_write(v, sizeof(*v));
 330         return arch_atomic_fetch_sub(i, v);
 331 }
 332
 333 static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v)
 334 {
 335         kasan_check_write(v, sizeof(*v));
 336         return arch_atomic64_fetch_sub(i, v);
 337 }
 338
 339 static __always_inline int atomic_fetch_and(int i, atomic_t *v)
 340 {
 341         kasan_check_write(v, sizeof(*v));
 342         return arch_atomic_fetch_and(i, v);
 343 }
 344
 345 static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v)
 346 {
 347         kasan_check_write(v, sizeof(*v));
 348         return arch_atomic64_fetch_and(i, v);
 349 }
 350
 351 static __always_inline int atomic_fetch_or(int i, atomic_t *v)
 352 {
 353         kasan_check_write(v, sizeof(*v));
 354         return arch_atomic_fetch_or(i, v);
 355 }
 356
 357 static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v)
 358 {
 359         kasan_check_write(v, sizeof(*v));
 360         return arch_atomic64_fetch_or(i, v);
 361 }
 362
 363 static __always_inline int atomic_fetch_xor(int i, atomic_t *v)
 364 {
 365         kasan_check_write(v, sizeof(*v));
 366         return arch_atomic_fetch_xor(i, v);
 367 }
 368
 369 static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v)
 370 {
 371         kasan_check_write(v, sizeof(*v));
 372         return arch_atomic64_fetch_xor(i, v);
 373 }
 374
 375 #ifdef arch_atomic_sub_and_test
 376 #define atomic_sub_and_test atomic_sub_and_test
 377 static __always_inline bool atomic_sub_and_test(int i, atomic_t *v)
 378 {
 379         kasan_check_write(v, sizeof(*v));
 380         return arch_atomic_sub_and_test(i, v);
 381 }
 382 #endif
 383
 384 #ifdef arch_atomic64_sub_and_test
 385 #define atomic64_sub_and_test atomic64_sub_and_test
 386 static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v)
 387 {
 388         kasan_check_write(v, sizeof(*v));
 389         return arch_atomic64_sub_and_test(i, v);
 390 }
 391 #endif
 392
 393 #ifdef arch_atomic_add_negative
 394 #define atomic_add_negative atomic_add_negative
 395 static __always_inline bool atomic_add_negative(int i, atomic_t *v)
 396 {
 397         kasan_check_write(v, sizeof(*v));
 398         return arch_atomic_add_negative(i, v);
 399 }
 400 #endif
 401
 402 #ifdef arch_atomic64_add_negative
 403 #define atomic64_add_negative atomic64_add_negative
 404 static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v)
 405 {
 406         kasan_check_write(v, sizeof(*v));
 407         return arch_atomic64_add_negative(i, v);
 408 }
 409 #endif
 410
 411 #define xchg(ptr, new)                                                  \
 412 ({                                                                      \
 413         typeof(ptr) __ai_ptr = (ptr);                                   \
 414         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 415         arch_xchg(__ai_ptr, (new));                                     \
 416 })
 417
 418 #define cmpxchg(ptr, old, new)                                          \
 419 ({                                                                      \
 420         typeof(ptr) __ai_ptr = (ptr);                                   \
 421         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 422         arch_cmpxchg(__ai_ptr, (old), (new));                           \
 423 })
 424
 425 #define sync_cmpxchg(ptr, old, new)                                     \
 426 ({                                                                      \
 427         typeof(ptr) __ai_ptr = (ptr);                                   \
 428         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 429         arch_sync_cmpxchg(__ai_ptr, (old), (new));                      \
 430 })
 431
 432 #define cmpxchg_local(ptr, old, new)                                    \
 433 ({                                                                      \
 434         typeof(ptr) __ai_ptr = (ptr);                                   \
 435         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 436         arch_cmpxchg_local(__ai_ptr, (old), (new));                     \
 437 })
 438
 439 #define cmpxchg64(ptr, old, new)                                        \
 440 ({                                                                      \
 441         typeof(ptr) __ai_ptr = (ptr);                                   \
 442         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 443         arch_cmpxchg64(__ai_ptr, (old), (new));                         \
 444 })
 445
 446 #define cmpxchg64_local(ptr, old, new)                                  \
 447 ({                                                                      \
 448         typeof(ptr) __ai_ptr = (ptr);                                   \
 449         kasan_check_write(__ai_ptr, sizeof(*__ai_ptr));                 \
 450         arch_cmpxchg64_local(__ai_ptr, (old), (new));                   \
 451 })
 452
 453 #define cmpxchg_double(p1, p2, o1, o2, n1, n2)                          \
 454 ({                                                                      \
 455         typeof(p1) __ai_p1 = (p1);                                      \
 456         kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1));               \
 457         arch_cmpxchg_double(__ai_p1, (p2), (o1), (o2), (n1), (n2));     \
 458 })
 459
 460 #define cmpxchg_double_local(p1, p2, o1, o2, n1, n2)                            \
 461 ({                                                                              \
 462         typeof(p1) __ai_p1 = (p1);                                              \
 463         kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1));                       \
 464         arch_cmpxchg_double_local(__ai_p1, (p2), (o1), (o2), (n1), (n2));       \
 465 })
 466
 467 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */