Merge tag 'kvm-s390-next-4.14-2' of git://git.kernel.org/pub/scm/linux/kernel/git...

[sfrench/cifs-2.6.git] / arch / s390 / mm / pgtable.c

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 4a1f7366b17aeffacb6c766ab891227e2609f1b9..ae677f814bc07a406f7f996a81ed2db65718f5ff 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -25,8 +25,49 @@
 #include <asm/mmu_context.h>
 #include <asm/page-states.h>
 
+static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
+                                  pte_t *ptep, int nodat)
+{
+       unsigned long opt, asce;
+
+       if (MACHINE_HAS_TLB_GUEST) {
+               opt = 0;
+               asce = READ_ONCE(mm->context.gmap_asce);
+               if (asce == 0UL || nodat)
+                       opt |= IPTE_NODAT;
+               if (asce != -1UL) {
+                       asce = asce ? : mm->context.asce;
+                       opt |= IPTE_GUEST_ASCE;
+               }
+               __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
+       } else {
+               __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
+       }
+}
+
+static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
+                                   pte_t *ptep, int nodat)
+{
+       unsigned long opt, asce;
+
+       if (MACHINE_HAS_TLB_GUEST) {
+               opt = 0;
+               asce = READ_ONCE(mm->context.gmap_asce);
+               if (asce == 0UL || nodat)
+                       opt |= IPTE_NODAT;
+               if (asce != -1UL) {
+                       asce = asce ? : mm->context.asce;
+                       opt |= IPTE_GUEST_ASCE;
+               }
+               __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
+       } else {
+               __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
+       }
+}
+
 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
-                                     unsigned long addr, pte_t *ptep)
+                                     unsigned long addr, pte_t *ptep,
+                                     int nodat)
 {
        pte_t old;
 
@@ -36,15 +77,16 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm,
        atomic_inc(&mm->context.flush_count);
        if (MACHINE_HAS_TLB_LC &&
            cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
-               __ptep_ipte(addr, ptep, IPTE_LOCAL);
+               ptep_ipte_local(mm, addr, ptep, nodat);
        else
-               __ptep_ipte(addr, ptep, IPTE_GLOBAL);
+               ptep_ipte_global(mm, addr, ptep, nodat);
        atomic_dec(&mm->context.flush_count);
        return old;
 }
 
 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
-                                   unsigned long addr, pte_t *ptep)
+                                   unsigned long addr, pte_t *ptep,
+                                   int nodat)
 {
        pte_t old;
 
@@ -57,7 +99,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
                pte_val(*ptep) |= _PAGE_INVALID;
                mm->context.flush_mm = 1;
        } else
-               __ptep_ipte(addr, ptep, IPTE_GLOBAL);
+               ptep_ipte_global(mm, addr, ptep, nodat);
        atomic_dec(&mm->context.flush_count);
        return old;
 }
@@ -229,10 +271,12 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
 {
        pgste_t pgste;
        pte_t old;
+       int nodat;
 
        preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
-       old = ptep_flush_direct(mm, addr, ptep);
+       nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+       old = ptep_flush_direct(mm, addr, ptep, nodat);
        old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
        preempt_enable();
        return old;
@@ -244,10 +288,12 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 {
        pgste_t pgste;
        pte_t old;
+       int nodat;
 
        preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
-       old = ptep_flush_lazy(mm, addr, ptep);
+       nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+       old = ptep_flush_lazy(mm, addr, ptep, nodat);
        old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
        preempt_enable();
        return old;
@@ -259,10 +305,12 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 {
        pgste_t pgste;
        pte_t old;
+       int nodat;
 
        preempt_disable();
        pgste = ptep_xchg_start(mm, addr, ptep);
-       old = ptep_flush_lazy(mm, addr, ptep);
+       nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+       old = ptep_flush_lazy(mm, addr, ptep, nodat);
        if (mm_has_pgste(mm)) {
                pgste = pgste_update_all(old, pgste, mm);
                pgste_set(ptep, pgste);
@@ -290,6 +338,28 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 }
 EXPORT_SYMBOL(ptep_modify_prot_commit);
 
+static inline void pmdp_idte_local(struct mm_struct *mm,
+                                  unsigned long addr, pmd_t *pmdp)
+{
+       if (MACHINE_HAS_TLB_GUEST)
+               __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_LOCAL);
+       else
+               __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
+}
+
+static inline void pmdp_idte_global(struct mm_struct *mm,
+                                   unsigned long addr, pmd_t *pmdp)
+{
+       if (MACHINE_HAS_TLB_GUEST)
+               __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_GLOBAL);
+       else if (MACHINE_HAS_IDTE)
+               __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
+       else
+               __pmdp_csp(pmdp);
+}
+
 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
                                      unsigned long addr, pmd_t *pmdp)
 {
@@ -298,16 +368,12 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
        old = *pmdp;
        if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
                return old;
-       if (!MACHINE_HAS_IDTE) {
-               __pmdp_csp(pmdp);
-               return old;
-       }
        atomic_inc(&mm->context.flush_count);
        if (MACHINE_HAS_TLB_LC &&
            cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
-               __pmdp_idte(addr, pmdp, IDTE_LOCAL);
+               pmdp_idte_local(mm, addr, pmdp);
        else
-               __pmdp_idte(addr, pmdp, IDTE_GLOBAL);
+               pmdp_idte_global(mm, addr, pmdp);
        atomic_dec(&mm->context.flush_count);
        return old;
 }
@@ -325,10 +391,9 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
                          cpumask_of(smp_processor_id()))) {
                pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
                mm->context.flush_mm = 1;
-       } else if (MACHINE_HAS_IDTE)
-               __pmdp_idte(addr, pmdp, IDTE_GLOBAL);
-       else
-               __pmdp_csp(pmdp);
+       } else {
+               pmdp_idte_global(mm, addr, pmdp);
+       }
        atomic_dec(&mm->context.flush_count);
        return old;
 }
@@ -359,28 +424,46 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 }
 EXPORT_SYMBOL(pmdp_xchg_lazy);
 
-static inline pud_t pudp_flush_direct(struct mm_struct *mm,
-                                     unsigned long addr, pud_t *pudp)
+static inline void pudp_idte_local(struct mm_struct *mm,
+                                  unsigned long addr, pud_t *pudp)
 {
-       pud_t old;
+       if (MACHINE_HAS_TLB_GUEST)
+               __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_LOCAL);
+       else
+               __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
+}
 
-       old = *pudp;
-       if (pud_val(old) & _REGION_ENTRY_INVALID)
-               return old;
-       if (!MACHINE_HAS_IDTE) {
+static inline void pudp_idte_global(struct mm_struct *mm,
+                                   unsigned long addr, pud_t *pudp)
+{
+       if (MACHINE_HAS_TLB_GUEST)
+               __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_GLOBAL);
+       else if (MACHINE_HAS_IDTE)
+               __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
+       else
                /*
                 * Invalid bit position is the same for pmd and pud, so we can
                 * re-use _pmd_csp() here
                 */
                __pmdp_csp((pmd_t *) pudp);
+}
+
+static inline pud_t pudp_flush_direct(struct mm_struct *mm,
+                                     unsigned long addr, pud_t *pudp)
+{
+       pud_t old;
+
+       old = *pudp;
+       if (pud_val(old) & _REGION_ENTRY_INVALID)
                return old;
-       }
        atomic_inc(&mm->context.flush_count);
        if (MACHINE_HAS_TLB_LC &&
            cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
-               __pudp_idte(addr, pudp, IDTE_LOCAL);
+               pudp_idte_local(mm, addr, pudp);
        else
-               __pudp_idte(addr, pudp, IDTE_GLOBAL);
+               pudp_idte_global(mm, addr, pudp);
        atomic_dec(&mm->context.flush_count);
        return old;
 }
@@ -482,7 +565,7 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
 {
        pte_t entry;
        pgste_t pgste;
-       int pte_i, pte_p;
+       int pte_i, pte_p, nodat;
 
        pgste = pgste_get_lock(ptep);
        entry = *ptep;
@@ -495,13 +578,14 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
                return -EAGAIN;
        }
        /* Change access rights and set pgste bit */
+       nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
        if (prot == PROT_NONE && !pte_i) {
-               ptep_flush_direct(mm, addr, ptep);
+               ptep_flush_direct(mm, addr, ptep, nodat);
                pgste = pgste_update_all(entry, pgste, mm);
                pte_val(entry) |= _PAGE_INVALID;
        }
        if (prot == PROT_READ && !pte_p) {
-               ptep_flush_direct(mm, addr, ptep);
+               ptep_flush_direct(mm, addr, ptep, nodat);
                pte_val(entry) &= ~_PAGE_INVALID;
                pte_val(entry) |= _PAGE_PROTECT;
        }
@@ -541,10 +625,12 @@ int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
 {
        pgste_t pgste;
+       int nodat;
 
        pgste = pgste_get_lock(ptep);
        /* notifier is called by the caller */
-       ptep_flush_direct(mm, saddr, ptep);
+       nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+       ptep_flush_direct(mm, saddr, ptep, nodat);
        /* don't touch the storage key - it belongs to parent pgste */
        pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
        pgste_set_unlock(ptep, pgste);
@@ -617,6 +703,7 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
        pte_t *ptep;
        pte_t pte;
        bool dirty;
+       int nodat;
 
        pgd = pgd_offset(mm, addr);
        p4d = p4d_alloc(mm, pgd, addr);
@@ -645,7 +732,8 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
        pte = *ptep;
        if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
                pgste = pgste_pte_notify(mm, addr, ptep, pgste);
-               __ptep_ipte(addr, ptep, IPTE_GLOBAL);
+               nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+               ptep_ipte_global(mm, addr, ptep, nodat);
                if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
                        pte_val(pte) |= _PAGE_PROTECT;
                else
@@ -831,7 +919,7 @@ int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
        case ESSA_GET_STATE:
                break;
        case ESSA_SET_STABLE:
-               pgstev &= ~_PGSTE_GPS_USAGE_MASK;
+               pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
                pgstev |= _PGSTE_GPS_USAGE_STABLE;
                break;
        case ESSA_SET_UNUSED:
@@ -877,6 +965,10 @@ int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
                        pgstev |= _PGSTE_GPS_USAGE_STABLE;
                }
                break;
+       case ESSA_SET_STABLE_NODAT:
+               pgstev &= ~_PGSTE_GPS_USAGE_MASK;
+               pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT;
+               break;
        default:
                /* we should never get here! */
                break;