tools/testing/selftests/rseq/rseq-arm64.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-arm64.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
   7  */
   8
   9 #define RSEQ_SIG        0xd428bc00      /* BRK #0x45E0 */
  10
  11 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb ish" ::: "memory")
  12 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb ishld" ::: "memory")
  13 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb ishst" ::: "memory")
  14
  15 #define rseq_smp_load_acquire(p)                                                \
  16 __extension__ ({                                                                \
  17         __typeof(*p) ____p1;                                                    \
  18         switch (sizeof(*p)) {                                                   \
  19         case 1:                                                                 \
  20                 asm volatile ("ldarb %w0, %1"                                   \
  21                         : "=r" (*(__u8 *)p)                                     \
  22                         : "Q" (*p) : "memory");                                 \
  23                 break;                                                          \
  24         case 2:                                                                 \
  25                 asm volatile ("ldarh %w0, %1"                                   \
  26                         : "=r" (*(__u16 *)p)                                    \
  27                         : "Q" (*p) : "memory");                                 \
  28                 break;                                                          \
  29         case 4:                                                                 \
  30                 asm volatile ("ldar %w0, %1"                                    \
  31                         : "=r" (*(__u32 *)p)                                    \
  32                         : "Q" (*p) : "memory");                                 \
  33                 break;                                                          \
  34         case 8:                                                                 \
  35                 asm volatile ("ldar %0, %1"                                     \
  36                         : "=r" (*(__u64 *)p)                                    \
  37                         : "Q" (*p) : "memory");                                 \
  38                 break;                                                          \
  39         }                                                                       \
  40         ____p1;                                                                 \
  41 })
  42
  43 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  44
  45 #define rseq_smp_store_release(p, v)                                            \
  46 do {                                                                            \
  47         switch (sizeof(*p)) {                                                   \
  48         case 1:                                                                 \
  49                 asm volatile ("stlrb %w1, %0"                                   \
  50                                 : "=Q" (*p)                                     \
  51                                 : "r" ((__u8)v)                                 \
  52                                 : "memory");                                    \
  53                 break;                                                          \
  54         case 2:                                                                 \
  55                 asm volatile ("stlrh %w1, %0"                                   \
  56                                 : "=Q" (*p)                                     \
  57                                 : "r" ((__u16)v)                                \
  58                                 : "memory");                                    \
  59                 break;                                                          \
  60         case 4:                                                                 \
  61                 asm volatile ("stlr %w1, %0"                                    \
  62                                 : "=Q" (*p)                                     \
  63                                 : "r" ((__u32)v)                                \
  64                                 : "memory");                                    \
  65                 break;                                                          \
  66         case 8:                                                                 \
  67                 asm volatile ("stlr %1, %0"                                     \
  68                                 : "=Q" (*p)                                     \
  69                                 : "r" ((__u64)v)                                \
  70                                 : "memory");                                    \
  71                 break;                                                          \
  72         }                                                                       \
  73 } while (0)
  74
  75 #ifdef RSEQ_SKIP_FASTPATH
  76 #include "rseq-skip.h"
  77 #else /* !RSEQ_SKIP_FASTPATH */
  78
  79 #define RSEQ_ASM_TMP_REG32      "w15"
  80 #define RSEQ_ASM_TMP_REG        "x15"
  81 #define RSEQ_ASM_TMP_REG_2      "x14"
  82
  83 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,                \
  84                                 post_commit_offset, abort_ip)                   \
  85         "       .pushsection    __rseq_table, \"aw\"\n"                         \
  86         "       .balign 32\n"                                                   \
  87         __rseq_str(label) ":\n"                                                 \
  88         "       .long   " __rseq_str(version) ", " __rseq_str(flags) "\n"       \
  89         "       .quad   " __rseq_str(start_ip) ", "                             \
  90                           __rseq_str(post_commit_offset) ", "                   \
  91                           __rseq_str(abort_ip) "\n"                             \
  92         "       .popsection\n"
  93
  94 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
  95         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,                      \
  96                                 (post_commit_ip - start_ip), abort_ip)
  97
  98 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  99         RSEQ_INJECT_ASM(1)                                                      \
 100         "       adrp    " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"       \
 101         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 102                         ", :lo12:" __rseq_str(cs_label) "\n"                    \
 103         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"     \
 104         __rseq_str(label) ":\n"
 105
 106 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 107         "       b       222f\n"                                                 \
 108         "       .inst   "       __rseq_str(RSEQ_SIG) "\n"                       \
 109         __rseq_str(label) ":\n"                                                 \
 110         "       b       %l[" __rseq_str(abort_label) "]\n"                      \
 111         "222:\n"
 112
 113 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 114         "       str     %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 115
 116 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 117         "       stlr    %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 118
 119 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 120         RSEQ_ASM_OP_STORE(value, var)                                           \
 121         __rseq_str(post_commit_label) ":\n"
 122
 123 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)          \
 124         RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 125         __rseq_str(post_commit_label) ":\n"
 126
 127 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 128         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 129         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 130                         ", %[" __rseq_str(expect) "]\n"                         \
 131         "       cbnz    " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 132
 133 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)                                 \
 134         "       ldr     " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"       \
 135         "       sub     " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32            \
 136                         ", %w[" __rseq_str(expect) "]\n"                        \
 137         "       cbnz    " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
 138
 139 #define RSEQ_ASM_OP_CMPNE(var, expect, label)                                   \
 140         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 141         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 142                         ", %[" __rseq_str(expect) "]\n"                         \
 143         "       cbz     " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 144
 145 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 146         RSEQ_INJECT_ASM(2)                                                      \
 147         RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
 148
 149 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 150         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 151
 152 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 153         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 154
 155 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                          \
 156         "       ldr     " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG               \
 157                         ", %[" __rseq_str(offset) "]]\n"
 158
 159 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 160         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 161                         ", %[" __rseq_str(count) "]\n"
 162
 163 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 164         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 165         __rseq_str(post_commit_label) ":\n"
 166
 167 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)                                 \
 168         "       cbz     %[" __rseq_str(len) "], 333f\n"                         \
 169         "       mov     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"       \
 170         "222:   sub     " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"   \
 171         "       ldrb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"        \
 172                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 173         "       strb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"        \
 174                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 175         "       cbnz    " RSEQ_ASM_TMP_REG_2 ", 222b\n"                         \
 176         "333:\n"
 177
 178 static inline __attribute__((always_inline))
 179 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 180 {
 181         RSEQ_INJECT_C(9)
 182
 183         __asm__ __volatile__ goto (
 184                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 185                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 186                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 187                 RSEQ_INJECT_ASM(3)
 188                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 189                 RSEQ_INJECT_ASM(4)
 190 #ifdef RSEQ_COMPARE_TWICE
 191                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 192                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 193 #endif
 194                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 195                 RSEQ_INJECT_ASM(5)
 196                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 197                 : /* gcc asm goto does not allow outputs */
 198                 : [cpu_id]              "r" (cpu),
 199                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 200                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 201                   [v]                   "Qo" (*v),
 202                   [expect]              "r" (expect),
 203                   [newv]                "r" (newv)
 204                   RSEQ_INJECT_INPUT
 205                 : "memory", RSEQ_ASM_TMP_REG
 206                 : abort, cmpfail
 207 #ifdef RSEQ_COMPARE_TWICE
 208                   , error1, error2
 209 #endif
 210         );
 211
 212         return 0;
 213 abort:
 214         RSEQ_INJECT_FAILED
 215         return -1;
 216 cmpfail:
 217         return 1;
 218 #ifdef RSEQ_COMPARE_TWICE
 219 error1:
 220         rseq_bug("cpu_id comparison failed");
 221 error2:
 222         rseq_bug("expected value comparison failed");
 223 #endif
 224 }
 225
 226 static inline __attribute__((always_inline))
 227 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 228                                off_t voffp, intptr_t *load, int cpu)
 229 {
 230         RSEQ_INJECT_C(9)
 231
 232         __asm__ __volatile__ goto (
 233                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 234                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 235                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 236                 RSEQ_INJECT_ASM(3)
 237                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 238                 RSEQ_INJECT_ASM(4)
 239 #ifdef RSEQ_COMPARE_TWICE
 240                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 241                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 242 #endif
 243                 RSEQ_ASM_OP_R_LOAD(v)
 244                 RSEQ_ASM_OP_R_STORE(load)
 245                 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
 246                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 247                 RSEQ_INJECT_ASM(5)
 248                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 249                 : /* gcc asm goto does not allow outputs */
 250                 : [cpu_id]              "r" (cpu),
 251                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 252                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 253                   [v]                   "Qo" (*v),
 254                   [expectnot]           "r" (expectnot),
 255                   [load]                "Qo" (*load),
 256                   [voffp]               "r" (voffp)
 257                   RSEQ_INJECT_INPUT
 258                 : "memory", RSEQ_ASM_TMP_REG
 259                 : abort, cmpfail
 260 #ifdef RSEQ_COMPARE_TWICE
 261                   , error1, error2
 262 #endif
 263         );
 264         return 0;
 265 abort:
 266         RSEQ_INJECT_FAILED
 267         return -1;
 268 cmpfail:
 269         return 1;
 270 #ifdef RSEQ_COMPARE_TWICE
 271 error1:
 272         rseq_bug("cpu_id comparison failed");
 273 error2:
 274         rseq_bug("expected value comparison failed");
 275 #endif
 276 }
 277
 278 static inline __attribute__((always_inline))
 279 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 280 {
 281         RSEQ_INJECT_C(9)
 282
 283         __asm__ __volatile__ goto (
 284                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 285                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 286                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 287                 RSEQ_INJECT_ASM(3)
 288 #ifdef RSEQ_COMPARE_TWICE
 289                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 290 #endif
 291                 RSEQ_ASM_OP_R_LOAD(v)
 292                 RSEQ_ASM_OP_R_ADD(count)
 293                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 294                 RSEQ_INJECT_ASM(4)
 295                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 296                 : /* gcc asm goto does not allow outputs */
 297                 : [cpu_id]              "r" (cpu),
 298                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 299                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 300                   [v]                   "Qo" (*v),
 301                   [count]               "r" (count)
 302                   RSEQ_INJECT_INPUT
 303                 : "memory", RSEQ_ASM_TMP_REG
 304                 : abort
 305 #ifdef RSEQ_COMPARE_TWICE
 306                   , error1
 307 #endif
 308         );
 309         return 0;
 310 abort:
 311         RSEQ_INJECT_FAILED
 312         return -1;
 313 #ifdef RSEQ_COMPARE_TWICE
 314 error1:
 315         rseq_bug("cpu_id comparison failed");
 316 #endif
 317 }
 318
 319 static inline __attribute__((always_inline))
 320 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 321                                  intptr_t *v2, intptr_t newv2,
 322                                  intptr_t newv, int cpu)
 323 {
 324         RSEQ_INJECT_C(9)
 325
 326         __asm__ __volatile__ goto (
 327                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 328                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 329                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 330                 RSEQ_INJECT_ASM(3)
 331                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 332                 RSEQ_INJECT_ASM(4)
 333 #ifdef RSEQ_COMPARE_TWICE
 334                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 335                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 336 #endif
 337                 RSEQ_ASM_OP_STORE(newv2, v2)
 338                 RSEQ_INJECT_ASM(5)
 339                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 340                 RSEQ_INJECT_ASM(6)
 341                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 342                 : /* gcc asm goto does not allow outputs */
 343                 : [cpu_id]              "r" (cpu),
 344                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 345                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 346                   [expect]              "r" (expect),
 347                   [v]                   "Qo" (*v),
 348                   [newv]                "r" (newv),
 349                   [v2]                  "Qo" (*v2),
 350                   [newv2]               "r" (newv2)
 351                   RSEQ_INJECT_INPUT
 352                 : "memory", RSEQ_ASM_TMP_REG
 353                 : abort, cmpfail
 354 #ifdef RSEQ_COMPARE_TWICE
 355                   , error1, error2
 356 #endif
 357         );
 358
 359         return 0;
 360 abort:
 361         RSEQ_INJECT_FAILED
 362         return -1;
 363 cmpfail:
 364         return 1;
 365 #ifdef RSEQ_COMPARE_TWICE
 366 error1:
 367         rseq_bug("cpu_id comparison failed");
 368 error2:
 369         rseq_bug("expected value comparison failed");
 370 #endif
 371 }
 372
 373 static inline __attribute__((always_inline))
 374 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 375                                          intptr_t *v2, intptr_t newv2,
 376                                          intptr_t newv, int cpu)
 377 {
 378         RSEQ_INJECT_C(9)
 379
 380         __asm__ __volatile__ goto (
 381                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 382                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 383                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 384                 RSEQ_INJECT_ASM(3)
 385                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 386                 RSEQ_INJECT_ASM(4)
 387 #ifdef RSEQ_COMPARE_TWICE
 388                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 389                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 390 #endif
 391                 RSEQ_ASM_OP_STORE(newv2, v2)
 392                 RSEQ_INJECT_ASM(5)
 393                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 394                 RSEQ_INJECT_ASM(6)
 395                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 396                 : /* gcc asm goto does not allow outputs */
 397                 : [cpu_id]              "r" (cpu),
 398                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 399                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 400                   [expect]              "r" (expect),
 401                   [v]                   "Qo" (*v),
 402                   [newv]                "r" (newv),
 403                   [v2]                  "Qo" (*v2),
 404                   [newv2]               "r" (newv2)
 405                   RSEQ_INJECT_INPUT
 406                 : "memory", RSEQ_ASM_TMP_REG
 407                 : abort, cmpfail
 408 #ifdef RSEQ_COMPARE_TWICE
 409                   , error1, error2
 410 #endif
 411         );
 412
 413         return 0;
 414 abort:
 415         RSEQ_INJECT_FAILED
 416         return -1;
 417 cmpfail:
 418         return 1;
 419 #ifdef RSEQ_COMPARE_TWICE
 420 error1:
 421         rseq_bug("cpu_id comparison failed");
 422 error2:
 423         rseq_bug("expected value comparison failed");
 424 #endif
 425 }
 426
 427 static inline __attribute__((always_inline))
 428 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 429                               intptr_t *v2, intptr_t expect2,
 430                               intptr_t newv, int cpu)
 431 {
 432         RSEQ_INJECT_C(9)
 433
 434         __asm__ __volatile__ goto (
 435                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 436                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 437                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 438                 RSEQ_INJECT_ASM(3)
 439                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 440                 RSEQ_INJECT_ASM(4)
 441                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 442                 RSEQ_INJECT_ASM(5)
 443 #ifdef RSEQ_COMPARE_TWICE
 444                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 445                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 446                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 447 #endif
 448                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 449                 RSEQ_INJECT_ASM(6)
 450                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 451                 : /* gcc asm goto does not allow outputs */
 452                 : [cpu_id]              "r" (cpu),
 453                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 454                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 455                   [v]                   "Qo" (*v),
 456                   [expect]              "r" (expect),
 457                   [v2]                  "Qo" (*v2),
 458                   [expect2]             "r" (expect2),
 459                   [newv]                "r" (newv)
 460                   RSEQ_INJECT_INPUT
 461                 : "memory", RSEQ_ASM_TMP_REG
 462                 : abort, cmpfail
 463 #ifdef RSEQ_COMPARE_TWICE
 464                   , error1, error2, error3
 465 #endif
 466         );
 467
 468         return 0;
 469 abort:
 470         RSEQ_INJECT_FAILED
 471         return -1;
 472 cmpfail:
 473         return 1;
 474 #ifdef RSEQ_COMPARE_TWICE
 475 error1:
 476         rseq_bug("cpu_id comparison failed");
 477 error2:
 478         rseq_bug("expected value comparison failed");
 479 error3:
 480         rseq_bug("2nd expected value comparison failed");
 481 #endif
 482 }
 483
 484 static inline __attribute__((always_inline))
 485 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 486                                  void *dst, void *src, size_t len,
 487                                  intptr_t newv, int cpu)
 488 {
 489         RSEQ_INJECT_C(9)
 490
 491         __asm__ __volatile__ goto (
 492                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 493                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 494                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 495                 RSEQ_INJECT_ASM(3)
 496                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 497                 RSEQ_INJECT_ASM(4)
 498 #ifdef RSEQ_COMPARE_TWICE
 499                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 500                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 501 #endif
 502                 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 503                 RSEQ_INJECT_ASM(5)
 504                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 505                 RSEQ_INJECT_ASM(6)
 506                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 507                 : /* gcc asm goto does not allow outputs */
 508                 : [cpu_id]              "r" (cpu),
 509                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 510                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 511                   [expect]              "r" (expect),
 512                   [v]                   "Qo" (*v),
 513                   [newv]                "r" (newv),
 514                   [dst]                 "r" (dst),
 515                   [src]                 "r" (src),
 516                   [len]                 "r" (len)
 517                   RSEQ_INJECT_INPUT
 518                 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
 519                 : abort, cmpfail
 520 #ifdef RSEQ_COMPARE_TWICE
 521                   , error1, error2
 522 #endif
 523         );
 524
 525         return 0;
 526 abort:
 527         RSEQ_INJECT_FAILED
 528         return -1;
 529 cmpfail:
 530         return 1;
 531 #ifdef RSEQ_COMPARE_TWICE
 532 error1:
 533         rseq_bug("cpu_id comparison failed");
 534 error2:
 535         rseq_bug("expected value comparison failed");
 536 #endif
 537 }
 538
 539 static inline __attribute__((always_inline))
 540 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 541                                          void *dst, void *src, size_t len,
 542                                          intptr_t newv, int cpu)
 543 {
 544         RSEQ_INJECT_C(9)
 545
 546         __asm__ __volatile__ goto (
 547                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 548                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 549                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 550                 RSEQ_INJECT_ASM(3)
 551                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 552                 RSEQ_INJECT_ASM(4)
 553 #ifdef RSEQ_COMPARE_TWICE
 554                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 555                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 556 #endif
 557                 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 558                 RSEQ_INJECT_ASM(5)
 559                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 560                 RSEQ_INJECT_ASM(6)
 561                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 562                 : /* gcc asm goto does not allow outputs */
 563                 : [cpu_id]              "r" (cpu),
 564                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 565                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 566                   [expect]              "r" (expect),
 567                   [v]                   "Qo" (*v),
 568                   [newv]                "r" (newv),
 569                   [dst]                 "r" (dst),
 570                   [src]                 "r" (src),
 571                   [len]                 "r" (len)
 572                   RSEQ_INJECT_INPUT
 573                 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
 574                 : abort, cmpfail
 575 #ifdef RSEQ_COMPARE_TWICE
 576                   , error1, error2
 577 #endif
 578         );
 579
 580         return 0;
 581 abort:
 582         RSEQ_INJECT_FAILED
 583         return -1;
 584 cmpfail:
 585         return 1;
 586 #ifdef RSEQ_COMPARE_TWICE
 587 error1:
 588         rseq_bug("cpu_id comparison failed");
 589 error2:
 590         rseq_bug("expected value comparison failed");
 591 #endif
 592 }
 593
 594 #endif /* !RSEQ_SKIP_FASTPATH */