Merge tag 'selinux-pr-20210629' of git://git.kernel.org/pub/scm/linux/kernel/git...

[sfrench/cifs-2.6.git] / arch / arm64 / include / asm / assembler.h

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 8418c1bd8f044e0bb27bd21f3ae6fd01bd51bff4..89faca0e740d03a306a70b49d19a91c4a3023886 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -130,15 +130,27 @@ alternative_endif
        .endm
 
 /*
- * Emit an entry into the exception table
+ * Create an exception table entry for `insn`, which will branch to `fixup`
+ * when an unhandled fault is taken.
  */
-       .macro          _asm_extable, from, to
+       .macro          _asm_extable, insn, fixup
        .pushsection    __ex_table, "a"
        .align          3
-       .long           (\from - .), (\to - .)
+       .long           (\insn - .), (\fixup - .)
        .popsection
        .endm
 
+/*
+ * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
+ * do nothing.
+ */
+       .macro          _cond_extable, insn, fixup
+       .ifnc           \fixup,
+       _asm_extable    \insn, \fixup
+       .endif
+       .endm
+
+
 #define USER(l, x...)                          \
 9999:  x;                                      \
        _asm_extable    9999b, l
@@ -232,15 +244,23 @@ lr        .req    x30             // link register
         * @dst: destination register
         */
 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
-       .macro  this_cpu_offset, dst
+       .macro  get_this_cpu_offset, dst
        mrs     \dst, tpidr_el2
        .endm
 #else
-       .macro  this_cpu_offset, dst
+       .macro  get_this_cpu_offset, dst
 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
        mrs     \dst, tpidr_el1
 alternative_else
        mrs     \dst, tpidr_el2
+alternative_endif
+       .endm
+
+       .macro  set_this_cpu_offset, src
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
+       msr     tpidr_el1, \src
+alternative_else
+       msr     tpidr_el2, \src
 alternative_endif
        .endm
 #endif
@@ -253,7 +273,7 @@ alternative_endif
        .macro adr_this_cpu, dst, sym, tmp
        adrp    \tmp, \sym
        add     \dst, \tmp, #:lo12:\sym
-       this_cpu_offset \tmp
+       get_this_cpu_offset \tmp
        add     \dst, \dst, \tmp
        .endm
 
@@ -264,7 +284,7 @@ alternative_endif
         */
        .macro ldr_this_cpu dst, sym, tmp
        adr_l   \dst, \sym
-       this_cpu_offset \tmp
+       get_this_cpu_offset \tmp
        ldr     \dst, [\dst, \tmp]
        .endm
 
@@ -375,51 +395,53 @@ alternative_cb_end
        bfi     \tcr, \tmp0, \pos, #3
        .endm
 
+       .macro __dcache_op_workaround_clean_cache, op, addr
+alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
+       dc      \op, \addr
+alternative_else
+       dc      civac, \addr
+alternative_endif
+       .endm
+
 /*
  * Macro to perform a data cache maintenance for the interval
- * [kaddr, kaddr + size)
+ * [start, end)
  *
  *     op:             operation passed to dc instruction
  *     domain:         domain used in dsb instruciton
- *     kaddr:          starting virtual address of the region
- *     size:           size of the region
- *     Corrupts:       kaddr, size, tmp1, tmp2
+ *     start:          starting virtual address of the region
+ *     end:            end virtual address of the region
+ *     fixup:          optional label to branch to on user fault
+ *     Corrupts:       start, end, tmp1, tmp2
  */
-       .macro __dcache_op_workaround_clean_cache, op, kaddr
-alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
-       dc      \op, \kaddr
-alternative_else
-       dc      civac, \kaddr
-alternative_endif
-       .endm
-
-       .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
+       .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
        dcache_line_size \tmp1, \tmp2
-       add     \size, \kaddr, \size
        sub     \tmp2, \tmp1, #1
-       bic     \kaddr, \kaddr, \tmp2
-9998:
+       bic     \start, \start, \tmp2
+.Ldcache_op\@:
        .ifc    \op, cvau
-       __dcache_op_workaround_clean_cache \op, \kaddr
+       __dcache_op_workaround_clean_cache \op, \start
        .else
        .ifc    \op, cvac
-       __dcache_op_workaround_clean_cache \op, \kaddr
+       __dcache_op_workaround_clean_cache \op, \start
        .else
        .ifc    \op, cvap
-       sys     3, c7, c12, 1, \kaddr   // dc cvap
+       sys     3, c7, c12, 1, \start   // dc cvap
        .else
        .ifc    \op, cvadp
-       sys     3, c7, c13, 1, \kaddr   // dc cvadp
+       sys     3, c7, c13, 1, \start   // dc cvadp
        .else
-       dc      \op, \kaddr
+       dc      \op, \start
        .endif
        .endif
        .endif
        .endif
-       add     \kaddr, \kaddr, \tmp1
-       cmp     \kaddr, \size
-       b.lo    9998b
+       add     \start, \start, \tmp1
+       cmp     \start, \end
+       b.lo    .Ldcache_op\@
        dsb     \domain
+
+       _cond_extable .Ldcache_op\@, \fixup
        .endm
 
 /*
@@ -427,20 +449,22 @@ alternative_endif
  * [start, end)
  *
  *     start, end:     virtual addresses describing the region
- *     label:          A label to branch to on user fault.
+ *     fixup:          optional label to branch to on user fault
  *     Corrupts:       tmp1, tmp2
  */
-       .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
+       .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup
        icache_line_size \tmp1, \tmp2
        sub     \tmp2, \tmp1, #1
        bic     \tmp2, \start, \tmp2
-9997:
-USER(\label, ic        ivau, \tmp2)                    // invalidate I line PoU
+.Licache_op\@:
+       ic      ivau, \tmp2                     // invalidate I line PoU
        add     \tmp2, \tmp2, \tmp1
        cmp     \tmp2, \end
-       b.lo    9997b
+       b.lo    .Licache_op\@
        dsb     ish
        isb
+
+       _cond_extable .Licache_op\@, \fixup
        .endm
 
 /*
@@ -745,7 +769,7 @@ USER(\label, ic     ivau, \tmp2)                    // invalidate I line PoU
        cbz             \tmp, \lbl
 #endif
        adr_l           \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING
-       this_cpu_offset \tmp2
+       get_this_cpu_offset     \tmp2
        ldr             w\tmp, [\tmp, \tmp2]
        cbnz            w\tmp, \lbl     // yield on pending softirq in task context
 .Lnoyield_\@: