tools/testing/selftests/rseq/rseq-mips.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * Author: Paul Burton <paul.burton@mips.com>
   4  * (C) Copyright 2018 MIPS Tech LLC
   5  *
   6  * Based on rseq-arm.h:
   7  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  */
   9
  10 #define RSEQ_SIG        0x53053053
  11
  12 #define rseq_smp_mb()   __asm__ __volatile__ ("sync" ::: "memory")
  13 #define rseq_smp_rmb()  rseq_smp_mb()
  14 #define rseq_smp_wmb()  rseq_smp_mb()
  15
  16 #define rseq_smp_load_acquire(p)                                        \
  17 __extension__ ({                                                        \
  18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19         rseq_smp_mb();                                                  \
  20         ____p1;                                                         \
  21 })
  22
  23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  24
  25 #define rseq_smp_store_release(p, v)                                    \
  26 do {                                                                    \
  27         rseq_smp_mb();                                                  \
  28         RSEQ_WRITE_ONCE(*p, v);                                         \
  29 } while (0)
  30
  31 #ifdef RSEQ_SKIP_FASTPATH
  32 #include "rseq-skip.h"
  33 #else /* !RSEQ_SKIP_FASTPATH */
  34
  35 #if _MIPS_SZLONG == 64
  36 # define LONG                   ".dword"
  37 # define LONG_LA                "dla"
  38 # define LONG_L                 "ld"
  39 # define LONG_S                 "sd"
  40 # define LONG_ADDI              "daddiu"
  41 # define U32_U64_PAD(x)         x
  42 #elif _MIPS_SZLONG == 32
  43 # define LONG                   ".word"
  44 # define LONG_LA                "la"
  45 # define LONG_L                 "lw"
  46 # define LONG_S                 "sw"
  47 # define LONG_ADDI              "addiu"
  48 # ifdef __BIG_ENDIAN
  49 #  define U32_U64_PAD(x)        "0x0, " x
  50 # else
  51 #  define U32_U64_PAD(x)        x ", 0x0"
  52 # endif
  53 #else
  54 # error unsupported _MIPS_SZLONG
  55 #endif
  56
  57 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
  58                                 post_commit_offset, abort_ip) \
  59                 ".pushsection __rseq_table, \"aw\"\n\t" \
  60                 ".balign 32\n\t" \
  61                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  62                 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
  63                 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
  64                 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
  65                 ".popsection\n\t"
  66
  67 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
  68         __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
  69                                 (post_commit_ip - start_ip), abort_ip)
  70
  71 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
  72                 RSEQ_INJECT_ASM(1) \
  73                 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
  74                 LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
  75                 __rseq_str(label) ":\n\t"
  76
  77 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
  78                 RSEQ_INJECT_ASM(2) \
  79                 "lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
  80                 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
  81
  82 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
  83                                 abort_label, version, flags, \
  84                                 start_ip, post_commit_offset, abort_ip) \
  85                 ".balign 32\n\t" \
  86                 __rseq_str(table_label) ":\n\t" \
  87                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  88                 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
  89                 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
  90                 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
  91                 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
  92                 __rseq_str(label) ":\n\t" \
  93                 teardown \
  94                 "b %l[" __rseq_str(abort_label) "]\n\t"
  95
  96 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
  97                               start_ip, post_commit_ip, abort_ip) \
  98         __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
  99                                 abort_label, 0x0, 0x0, start_ip, \
 100                                 (post_commit_ip - start_ip), abort_ip)
 101
 102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
 103                 __rseq_str(label) ":\n\t" \
 104                 teardown \
 105                 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
 106
 107 #define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
 108
 109 static inline __attribute__((always_inline))
 110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 111 {
 112         RSEQ_INJECT_C(9)
 113
 114         rseq_workaround_gcc_asm_size_guess();
 115         __asm__ __volatile__ goto (
 116                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 117                 /* Start rseq by storing table entry pointer into rseq_cs. */
 118                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 119                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 120                 RSEQ_INJECT_ASM(3)
 121                 LONG_L " $4, %[v]\n\t"
 122                 "bne $4, %[expect], %l[cmpfail]\n\t"
 123                 RSEQ_INJECT_ASM(4)
 124 #ifdef RSEQ_COMPARE_TWICE
 125                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 126                 LONG_L " $4, %[v]\n\t"
 127                 "bne $4, %[expect], %l[error2]\n\t"
 128 #endif
 129                 /* final store */
 130                 LONG_S " %[newv], %[v]\n\t"
 131                 "2:\n\t"
 132                 RSEQ_INJECT_ASM(5)
 133                 "b 5f\n\t"
 134                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 135                 "5:\n\t"
 136                 : /* gcc asm goto does not allow outputs */
 137                 : [cpu_id]              "r" (cpu),
 138                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 139                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 140                   [v]                   "m" (*v),
 141                   [expect]              "r" (expect),
 142                   [newv]                "r" (newv)
 143                   RSEQ_INJECT_INPUT
 144                 : "$4", "memory"
 145                   RSEQ_INJECT_CLOBBER
 146                 : abort, cmpfail
 147 #ifdef RSEQ_COMPARE_TWICE
 148                   , error1, error2
 149 #endif
 150         );
 151         rseq_workaround_gcc_asm_size_guess();
 152         return 0;
 153 abort:
 154         rseq_workaround_gcc_asm_size_guess();
 155         RSEQ_INJECT_FAILED
 156         return -1;
 157 cmpfail:
 158         rseq_workaround_gcc_asm_size_guess();
 159         return 1;
 160 #ifdef RSEQ_COMPARE_TWICE
 161 error1:
 162         rseq_bug("cpu_id comparison failed");
 163 error2:
 164         rseq_bug("expected value comparison failed");
 165 #endif
 166 }
 167
 168 static inline __attribute__((always_inline))
 169 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 170                                off_t voffp, intptr_t *load, int cpu)
 171 {
 172         RSEQ_INJECT_C(9)
 173
 174         rseq_workaround_gcc_asm_size_guess();
 175         __asm__ __volatile__ goto (
 176                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 177                 /* Start rseq by storing table entry pointer into rseq_cs. */
 178                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 179                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 180                 RSEQ_INJECT_ASM(3)
 181                 LONG_L " $4, %[v]\n\t"
 182                 "beq $4, %[expectnot], %l[cmpfail]\n\t"
 183                 RSEQ_INJECT_ASM(4)
 184 #ifdef RSEQ_COMPARE_TWICE
 185                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 186                 LONG_L " $4, %[v]\n\t"
 187                 "beq $4, %[expectnot], %l[error2]\n\t"
 188 #endif
 189                 LONG_S " $4, %[load]\n\t"
 190                 LONG_ADDI " $4, %[voffp]\n\t"
 191                 LONG_L " $4, 0($4)\n\t"
 192                 /* final store */
 193                 LONG_S " $4, %[v]\n\t"
 194                 "2:\n\t"
 195                 RSEQ_INJECT_ASM(5)
 196                 "b 5f\n\t"
 197                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 198                 "5:\n\t"
 199                 : /* gcc asm goto does not allow outputs */
 200                 : [cpu_id]              "r" (cpu),
 201                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 202                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 203                   /* final store input */
 204                   [v]                   "m" (*v),
 205                   [expectnot]           "r" (expectnot),
 206                   [voffp]               "Ir" (voffp),
 207                   [load]                "m" (*load)
 208                   RSEQ_INJECT_INPUT
 209                 : "$4", "memory"
 210                   RSEQ_INJECT_CLOBBER
 211                 : abort, cmpfail
 212 #ifdef RSEQ_COMPARE_TWICE
 213                   , error1, error2
 214 #endif
 215         );
 216         rseq_workaround_gcc_asm_size_guess();
 217         return 0;
 218 abort:
 219         rseq_workaround_gcc_asm_size_guess();
 220         RSEQ_INJECT_FAILED
 221         return -1;
 222 cmpfail:
 223         rseq_workaround_gcc_asm_size_guess();
 224         return 1;
 225 #ifdef RSEQ_COMPARE_TWICE
 226 error1:
 227         rseq_bug("cpu_id comparison failed");
 228 error2:
 229         rseq_bug("expected value comparison failed");
 230 #endif
 231 }
 232
 233 static inline __attribute__((always_inline))
 234 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 235 {
 236         RSEQ_INJECT_C(9)
 237
 238         rseq_workaround_gcc_asm_size_guess();
 239         __asm__ __volatile__ goto (
 240                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 241                 /* Start rseq by storing table entry pointer into rseq_cs. */
 242                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 243                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 244                 RSEQ_INJECT_ASM(3)
 245 #ifdef RSEQ_COMPARE_TWICE
 246                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 247 #endif
 248                 LONG_L " $4, %[v]\n\t"
 249                 LONG_ADDI " $4, %[count]\n\t"
 250                 /* final store */
 251                 LONG_S " $4, %[v]\n\t"
 252                 "2:\n\t"
 253                 RSEQ_INJECT_ASM(4)
 254                 "b 5f\n\t"
 255                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 256                 "5:\n\t"
 257                 : /* gcc asm goto does not allow outputs */
 258                 : [cpu_id]              "r" (cpu),
 259                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 260                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 261                   [v]                   "m" (*v),
 262                   [count]               "Ir" (count)
 263                   RSEQ_INJECT_INPUT
 264                 : "$4", "memory"
 265                   RSEQ_INJECT_CLOBBER
 266                 : abort
 267 #ifdef RSEQ_COMPARE_TWICE
 268                   , error1
 269 #endif
 270         );
 271         rseq_workaround_gcc_asm_size_guess();
 272         return 0;
 273 abort:
 274         rseq_workaround_gcc_asm_size_guess();
 275         RSEQ_INJECT_FAILED
 276         return -1;
 277 #ifdef RSEQ_COMPARE_TWICE
 278 error1:
 279         rseq_bug("cpu_id comparison failed");
 280 #endif
 281 }
 282
 283 static inline __attribute__((always_inline))
 284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 285                                  intptr_t *v2, intptr_t newv2,
 286                                  intptr_t newv, int cpu)
 287 {
 288         RSEQ_INJECT_C(9)
 289
 290         rseq_workaround_gcc_asm_size_guess();
 291         __asm__ __volatile__ goto (
 292                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 293                 /* Start rseq by storing table entry pointer into rseq_cs. */
 294                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 295                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 296                 RSEQ_INJECT_ASM(3)
 297                 LONG_L " $4, %[v]\n\t"
 298                 "bne $4, %[expect], %l[cmpfail]\n\t"
 299                 RSEQ_INJECT_ASM(4)
 300 #ifdef RSEQ_COMPARE_TWICE
 301                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 302                 LONG_L " $4, %[v]\n\t"
 303                 "bne $4, %[expect], %l[error2]\n\t"
 304 #endif
 305                 /* try store */
 306                 LONG_S " %[newv2], %[v2]\n\t"
 307                 RSEQ_INJECT_ASM(5)
 308                 /* final store */
 309                 LONG_S " %[newv], %[v]\n\t"
 310                 "2:\n\t"
 311                 RSEQ_INJECT_ASM(6)
 312                 "b 5f\n\t"
 313                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 314                 "5:\n\t"
 315                 : /* gcc asm goto does not allow outputs */
 316                 : [cpu_id]              "r" (cpu),
 317                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 318                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 319                   /* try store input */
 320                   [v2]                  "m" (*v2),
 321                   [newv2]               "r" (newv2),
 322                   /* final store input */
 323                   [v]                   "m" (*v),
 324                   [expect]              "r" (expect),
 325                   [newv]                "r" (newv)
 326                   RSEQ_INJECT_INPUT
 327                 : "$4", "memory"
 328                   RSEQ_INJECT_CLOBBER
 329                 : abort, cmpfail
 330 #ifdef RSEQ_COMPARE_TWICE
 331                   , error1, error2
 332 #endif
 333         );
 334         rseq_workaround_gcc_asm_size_guess();
 335         return 0;
 336 abort:
 337         rseq_workaround_gcc_asm_size_guess();
 338         RSEQ_INJECT_FAILED
 339         return -1;
 340 cmpfail:
 341         rseq_workaround_gcc_asm_size_guess();
 342         return 1;
 343 #ifdef RSEQ_COMPARE_TWICE
 344 error1:
 345         rseq_bug("cpu_id comparison failed");
 346 error2:
 347         rseq_bug("expected value comparison failed");
 348 #endif
 349 }
 350
 351 static inline __attribute__((always_inline))
 352 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 353                                          intptr_t *v2, intptr_t newv2,
 354                                          intptr_t newv, int cpu)
 355 {
 356         RSEQ_INJECT_C(9)
 357
 358         rseq_workaround_gcc_asm_size_guess();
 359         __asm__ __volatile__ goto (
 360                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 361                 /* Start rseq by storing table entry pointer into rseq_cs. */
 362                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 363                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 364                 RSEQ_INJECT_ASM(3)
 365                 LONG_L " $4, %[v]\n\t"
 366                 "bne $4, %[expect], %l[cmpfail]\n\t"
 367                 RSEQ_INJECT_ASM(4)
 368 #ifdef RSEQ_COMPARE_TWICE
 369                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 370                 LONG_L " $4, %[v]\n\t"
 371                 "bne $4, %[expect], %l[error2]\n\t"
 372 #endif
 373                 /* try store */
 374                 LONG_S " %[newv2], %[v2]\n\t"
 375                 RSEQ_INJECT_ASM(5)
 376                 "sync\n\t"      /* full sync provides store-release */
 377                 /* final store */
 378                 LONG_S " %[newv], %[v]\n\t"
 379                 "2:\n\t"
 380                 RSEQ_INJECT_ASM(6)
 381                 "b 5f\n\t"
 382                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 383                 "5:\n\t"
 384                 : /* gcc asm goto does not allow outputs */
 385                 : [cpu_id]              "r" (cpu),
 386                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 387                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 388                   /* try store input */
 389                   [v2]                  "m" (*v2),
 390                   [newv2]               "r" (newv2),
 391                   /* final store input */
 392                   [v]                   "m" (*v),
 393                   [expect]              "r" (expect),
 394                   [newv]                "r" (newv)
 395                   RSEQ_INJECT_INPUT
 396                 : "$4", "memory"
 397                   RSEQ_INJECT_CLOBBER
 398                 : abort, cmpfail
 399 #ifdef RSEQ_COMPARE_TWICE
 400                   , error1, error2
 401 #endif
 402         );
 403         rseq_workaround_gcc_asm_size_guess();
 404         return 0;
 405 abort:
 406         rseq_workaround_gcc_asm_size_guess();
 407         RSEQ_INJECT_FAILED
 408         return -1;
 409 cmpfail:
 410         rseq_workaround_gcc_asm_size_guess();
 411         return 1;
 412 #ifdef RSEQ_COMPARE_TWICE
 413 error1:
 414         rseq_bug("cpu_id comparison failed");
 415 error2:
 416         rseq_bug("expected value comparison failed");
 417 #endif
 418 }
 419
 420 static inline __attribute__((always_inline))
 421 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 422                               intptr_t *v2, intptr_t expect2,
 423                               intptr_t newv, int cpu)
 424 {
 425         RSEQ_INJECT_C(9)
 426
 427         rseq_workaround_gcc_asm_size_guess();
 428         __asm__ __volatile__ goto (
 429                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 430                 /* Start rseq by storing table entry pointer into rseq_cs. */
 431                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 432                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 433                 RSEQ_INJECT_ASM(3)
 434                 LONG_L " $4, %[v]\n\t"
 435                 "bne $4, %[expect], %l[cmpfail]\n\t"
 436                 RSEQ_INJECT_ASM(4)
 437                 LONG_L " $4, %[v2]\n\t"
 438                 "bne $4, %[expect2], %l[cmpfail]\n\t"
 439                 RSEQ_INJECT_ASM(5)
 440 #ifdef RSEQ_COMPARE_TWICE
 441                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 442                 LONG_L " $4, %[v]\n\t"
 443                 "bne $4, %[expect], %l[error2]\n\t"
 444                 LONG_L " $4, %[v2]\n\t"
 445                 "bne $4, %[expect2], %l[error3]\n\t"
 446 #endif
 447                 /* final store */
 448                 LONG_S " %[newv], %[v]\n\t"
 449                 "2:\n\t"
 450                 RSEQ_INJECT_ASM(6)
 451                 "b 5f\n\t"
 452                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 453                 "5:\n\t"
 454                 : /* gcc asm goto does not allow outputs */
 455                 : [cpu_id]              "r" (cpu),
 456                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 457                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 458                   /* cmp2 input */
 459                   [v2]                  "m" (*v2),
 460                   [expect2]             "r" (expect2),
 461                   /* final store input */
 462                   [v]                   "m" (*v),
 463                   [expect]              "r" (expect),
 464                   [newv]                "r" (newv)
 465                   RSEQ_INJECT_INPUT
 466                 : "$4", "memory"
 467                   RSEQ_INJECT_CLOBBER
 468                 : abort, cmpfail
 469 #ifdef RSEQ_COMPARE_TWICE
 470                   , error1, error2, error3
 471 #endif
 472         );
 473         rseq_workaround_gcc_asm_size_guess();
 474         return 0;
 475 abort:
 476         rseq_workaround_gcc_asm_size_guess();
 477         RSEQ_INJECT_FAILED
 478         return -1;
 479 cmpfail:
 480         rseq_workaround_gcc_asm_size_guess();
 481         return 1;
 482 #ifdef RSEQ_COMPARE_TWICE
 483 error1:
 484         rseq_bug("cpu_id comparison failed");
 485 error2:
 486         rseq_bug("1st expected value comparison failed");
 487 error3:
 488         rseq_bug("2nd expected value comparison failed");
 489 #endif
 490 }
 491
 492 static inline __attribute__((always_inline))
 493 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 494                                  void *dst, void *src, size_t len,
 495                                  intptr_t newv, int cpu)
 496 {
 497         uintptr_t rseq_scratch[3];
 498
 499         RSEQ_INJECT_C(9)
 500
 501         rseq_workaround_gcc_asm_size_guess();
 502         __asm__ __volatile__ goto (
 503                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 504                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 505                 LONG_S "  %[dst], %[rseq_scratch1]\n\t"
 506                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 507                 /* Start rseq by storing table entry pointer into rseq_cs. */
 508                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 509                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 510                 RSEQ_INJECT_ASM(3)
 511                 LONG_L " $4, %[v]\n\t"
 512                 "bne $4, %[expect], 5f\n\t"
 513                 RSEQ_INJECT_ASM(4)
 514 #ifdef RSEQ_COMPARE_TWICE
 515                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 516                 LONG_L " $4, %[v]\n\t"
 517                 "bne $4, %[expect], 7f\n\t"
 518 #endif
 519                 /* try memcpy */
 520                 "beqz %[len], 333f\n\t" \
 521                 "222:\n\t" \
 522                 "lb   $4, 0(%[src])\n\t" \
 523                 "sb   $4, 0(%[dst])\n\t" \
 524                 LONG_ADDI " %[src], 1\n\t" \
 525                 LONG_ADDI " %[dst], 1\n\t" \
 526                 LONG_ADDI " %[len], -1\n\t" \
 527                 "bnez %[len], 222b\n\t" \
 528                 "333:\n\t" \
 529                 RSEQ_INJECT_ASM(5)
 530                 /* final store */
 531                 LONG_S " %[newv], %[v]\n\t"
 532                 "2:\n\t"
 533                 RSEQ_INJECT_ASM(6)
 534                 /* teardown */
 535                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 536                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 537                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 538                 "b 8f\n\t"
 539                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 540                                       /* teardown */
 541                                       LONG_L " %[len], %[rseq_scratch2]\n\t"
 542                                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
 543                                       LONG_L " %[src], %[rseq_scratch0]\n\t",
 544                                       abort, 1b, 2b, 4f)
 545                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 546                                         /* teardown */
 547                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 548                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 549                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 550                                         cmpfail)
 551 #ifdef RSEQ_COMPARE_TWICE
 552                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 553                                         /* teardown */
 554                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 555                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 556                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 557                                         error1)
 558                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 559                                         /* teardown */
 560                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 561                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 562                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 563                                         error2)
 564 #endif
 565                 "8:\n\t"
 566                 : /* gcc asm goto does not allow outputs */
 567                 : [cpu_id]              "r" (cpu),
 568                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 569                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 570                   /* final store input */
 571                   [v]                   "m" (*v),
 572                   [expect]              "r" (expect),
 573                   [newv]                "r" (newv),
 574                   /* try memcpy input */
 575                   [dst]                 "r" (dst),
 576                   [src]                 "r" (src),
 577                   [len]                 "r" (len),
 578                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 579                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 580                   [rseq_scratch2]       "m" (rseq_scratch[2])
 581                   RSEQ_INJECT_INPUT
 582                 : "$4", "memory"
 583                   RSEQ_INJECT_CLOBBER
 584                 : abort, cmpfail
 585 #ifdef RSEQ_COMPARE_TWICE
 586                   , error1, error2
 587 #endif
 588         );
 589         rseq_workaround_gcc_asm_size_guess();
 590         return 0;
 591 abort:
 592         rseq_workaround_gcc_asm_size_guess();
 593         RSEQ_INJECT_FAILED
 594         return -1;
 595 cmpfail:
 596         rseq_workaround_gcc_asm_size_guess();
 597         return 1;
 598 #ifdef RSEQ_COMPARE_TWICE
 599 error1:
 600         rseq_workaround_gcc_asm_size_guess();
 601         rseq_bug("cpu_id comparison failed");
 602 error2:
 603         rseq_workaround_gcc_asm_size_guess();
 604         rseq_bug("expected value comparison failed");
 605 #endif
 606 }
 607
 608 static inline __attribute__((always_inline))
 609 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 610                                          void *dst, void *src, size_t len,
 611                                          intptr_t newv, int cpu)
 612 {
 613         uintptr_t rseq_scratch[3];
 614
 615         RSEQ_INJECT_C(9)
 616
 617         rseq_workaround_gcc_asm_size_guess();
 618         __asm__ __volatile__ goto (
 619                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 620                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 621                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 622                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 623                 /* Start rseq by storing table entry pointer into rseq_cs. */
 624                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 625                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 626                 RSEQ_INJECT_ASM(3)
 627                 LONG_L " $4, %[v]\n\t"
 628                 "bne $4, %[expect], 5f\n\t"
 629                 RSEQ_INJECT_ASM(4)
 630 #ifdef RSEQ_COMPARE_TWICE
 631                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 632                 LONG_L " $4, %[v]\n\t"
 633                 "bne $4, %[expect], 7f\n\t"
 634 #endif
 635                 /* try memcpy */
 636                 "beqz %[len], 333f\n\t" \
 637                 "222:\n\t" \
 638                 "lb   $4, 0(%[src])\n\t" \
 639                 "sb   $4, 0(%[dst])\n\t" \
 640                 LONG_ADDI " %[src], 1\n\t" \
 641                 LONG_ADDI " %[dst], 1\n\t" \
 642                 LONG_ADDI " %[len], -1\n\t" \
 643                 "bnez %[len], 222b\n\t" \
 644                 "333:\n\t" \
 645                 RSEQ_INJECT_ASM(5)
 646                 "sync\n\t"      /* full sync provides store-release */
 647                 /* final store */
 648                 LONG_S " %[newv], %[v]\n\t"
 649                 "2:\n\t"
 650                 RSEQ_INJECT_ASM(6)
 651                 /* teardown */
 652                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 653                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 654                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 655                 "b 8f\n\t"
 656                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 657                                       /* teardown */
 658                                       LONG_L " %[len], %[rseq_scratch2]\n\t"
 659                                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
 660                                       LONG_L " %[src], %[rseq_scratch0]\n\t",
 661                                       abort, 1b, 2b, 4f)
 662                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 663                                         /* teardown */
 664                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 665                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 666                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 667                                         cmpfail)
 668 #ifdef RSEQ_COMPARE_TWICE
 669                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 670                                         /* teardown */
 671                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 672                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 673                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 674                                         error1)
 675                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 676                                         /* teardown */
 677                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 678                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 679                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 680                                         error2)
 681 #endif
 682                 "8:\n\t"
 683                 : /* gcc asm goto does not allow outputs */
 684                 : [cpu_id]              "r" (cpu),
 685                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 686                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 687                   /* final store input */
 688                   [v]                   "m" (*v),
 689                   [expect]              "r" (expect),
 690                   [newv]                "r" (newv),
 691                   /* try memcpy input */
 692                   [dst]                 "r" (dst),
 693                   [src]                 "r" (src),
 694                   [len]                 "r" (len),
 695                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 696                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 697                   [rseq_scratch2]       "m" (rseq_scratch[2])
 698                   RSEQ_INJECT_INPUT
 699                 : "$4", "memory"
 700                   RSEQ_INJECT_CLOBBER
 701                 : abort, cmpfail
 702 #ifdef RSEQ_COMPARE_TWICE
 703                   , error1, error2
 704 #endif
 705         );
 706         rseq_workaround_gcc_asm_size_guess();
 707         return 0;
 708 abort:
 709         rseq_workaround_gcc_asm_size_guess();
 710         RSEQ_INJECT_FAILED
 711         return -1;
 712 cmpfail:
 713         rseq_workaround_gcc_asm_size_guess();
 714         return 1;
 715 #ifdef RSEQ_COMPARE_TWICE
 716 error1:
 717         rseq_workaround_gcc_asm_size_guess();
 718         rseq_bug("cpu_id comparison failed");
 719 error2:
 720         rseq_workaround_gcc_asm_size_guess();
 721         rseq_bug("expected value comparison failed");
 722 #endif
 723 }
 724
 725 #endif /* !RSEQ_SKIP_FASTPATH */