drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2016-2018 Etnaviv Project
   4  */
   5
   6 #include <linux/platform_device.h>
   7 #include <linux/sizes.h>
   8 #include <linux/slab.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/bitops.h>
  11
  12 #include "etnaviv_cmdbuf.h"
  13 #include "etnaviv_gpu.h"
  14 #include "etnaviv_mmu.h"
  15 #include "etnaviv_iommu.h"
  16 #include "state.xml.h"
  17 #include "state_hi.xml.h"
  18
  19 #define MMUv2_PTE_PRESENT               BIT(0)
  20 #define MMUv2_PTE_EXCEPTION             BIT(1)
  21 #define MMUv2_PTE_WRITEABLE             BIT(2)
  22
  23 #define MMUv2_MTLB_MASK                 0xffc00000
  24 #define MMUv2_MTLB_SHIFT                22
  25 #define MMUv2_STLB_MASK                 0x003ff000
  26 #define MMUv2_STLB_SHIFT                12
  27
  28 #define MMUv2_MAX_STLB_ENTRIES          1024
  29
  30 struct etnaviv_iommuv2_domain {
  31         struct etnaviv_iommu_domain base;
  32         /* P(age) T(able) A(rray) */
  33         u64 *pta_cpu;
  34         dma_addr_t pta_dma;
  35         /* M(aster) TLB aka first level pagetable */
  36         u32 *mtlb_cpu;
  37         dma_addr_t mtlb_dma;
  38         /* S(lave) TLB aka second level pagetable */
  39         u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];
  40         dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];
  41 };
  42
  43 static struct etnaviv_iommuv2_domain *
  44 to_etnaviv_domain(struct etnaviv_iommu_domain *domain)
  45 {
  46         return container_of(domain, struct etnaviv_iommuv2_domain, base);
  47 }
  48
  49 static int
  50 etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_domain *etnaviv_domain,
  51                             int stlb)
  52 {
  53         if (etnaviv_domain->stlb_cpu[stlb])
  54                 return 0;
  55
  56         etnaviv_domain->stlb_cpu[stlb] =
  57                         dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
  58                                      &etnaviv_domain->stlb_dma[stlb],
  59                                      GFP_KERNEL);
  60
  61         if (!etnaviv_domain->stlb_cpu[stlb])
  62                 return -ENOMEM;
  63
  64         memset32(etnaviv_domain->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,
  65                  SZ_4K / sizeof(u32));
  66
  67         etnaviv_domain->mtlb_cpu[stlb] = etnaviv_domain->stlb_dma[stlb] |
  68                                                       MMUv2_PTE_PRESENT;
  69         return 0;
  70 }
  71
  72 static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain,
  73                                unsigned long iova, phys_addr_t paddr,
  74                                size_t size, int prot)
  75 {
  76         struct etnaviv_iommuv2_domain *etnaviv_domain =
  77                         to_etnaviv_domain(domain);
  78         int mtlb_entry, stlb_entry, ret;
  79         u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;
  80
  81         if (size != SZ_4K)
  82                 return -EINVAL;
  83
  84         if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
  85                 entry |= (upper_32_bits(paddr) & 0xff) << 4;
  86
  87         if (prot & ETNAVIV_PROT_WRITE)
  88                 entry |= MMUv2_PTE_WRITEABLE;
  89
  90         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
  91         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
  92
  93         ret = etnaviv_iommuv2_ensure_stlb(etnaviv_domain, mtlb_entry);
  94         if (ret)
  95                 return ret;
  96
  97         etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
  98
  99         return 0;
 100 }
 101
 102 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain,
 103                                     unsigned long iova, size_t size)
 104 {
 105         struct etnaviv_iommuv2_domain *etnaviv_domain =
 106                         to_etnaviv_domain(domain);
 107         int mtlb_entry, stlb_entry;
 108
 109         if (size != SZ_4K)
 110                 return -EINVAL;
 111
 112         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
 113         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
 114
 115         etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
 116
 117         return SZ_4K;
 118 }
 119
 120 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 121 {
 122         u32 *p;
 123         int ret, i;
 124
 125         /* allocate scratch page */
 126         etnaviv_domain->base.bad_page_cpu =
 127                         dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
 128                                      &etnaviv_domain->base.bad_page_dma,
 129                                      GFP_KERNEL);
 130         if (!etnaviv_domain->base.bad_page_cpu) {
 131                 ret = -ENOMEM;
 132                 goto fail_mem;
 133         }
 134         p = etnaviv_domain->base.bad_page_cpu;
 135         for (i = 0; i < SZ_4K / 4; i++)
 136                 *p++ = 0xdead55aa;
 137
 138         etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
 139                                                SZ_4K, &etnaviv_domain->pta_dma,
 140                                                GFP_KERNEL);
 141         if (!etnaviv_domain->pta_cpu) {
 142                 ret = -ENOMEM;
 143                 goto fail_mem;
 144         }
 145
 146         etnaviv_domain->mtlb_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
 147                                                 SZ_4K, &etnaviv_domain->mtlb_dma,
 148                                                 GFP_KERNEL);
 149         if (!etnaviv_domain->mtlb_cpu) {
 150                 ret = -ENOMEM;
 151                 goto fail_mem;
 152         }
 153
 154         memset32(etnaviv_domain->mtlb_cpu, MMUv2_PTE_EXCEPTION,
 155                  MMUv2_MAX_STLB_ENTRIES);
 156
 157         return 0;
 158
 159 fail_mem:
 160         if (etnaviv_domain->base.bad_page_cpu)
 161                 dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 162                             etnaviv_domain->base.bad_page_cpu,
 163                             etnaviv_domain->base.bad_page_dma);
 164
 165         if (etnaviv_domain->pta_cpu)
 166                 dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 167                             etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
 168
 169         if (etnaviv_domain->mtlb_cpu)
 170                 dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 171                             etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
 172
 173         return ret;
 174 }
 175
 176 static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain)
 177 {
 178         struct etnaviv_iommuv2_domain *etnaviv_domain =
 179                         to_etnaviv_domain(domain);
 180         int i;
 181
 182         dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 183                     etnaviv_domain->base.bad_page_cpu,
 184                     etnaviv_domain->base.bad_page_dma);
 185
 186         dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 187                     etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
 188
 189         dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 190                     etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
 191
 192         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
 193                 if (etnaviv_domain->stlb_cpu[i])
 194                         dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
 195                                     etnaviv_domain->stlb_cpu[i],
 196                                     etnaviv_domain->stlb_dma[i]);
 197         }
 198
 199         vfree(etnaviv_domain);
 200 }
 201
 202 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_domain *domain)
 203 {
 204         struct etnaviv_iommuv2_domain *etnaviv_domain =
 205                         to_etnaviv_domain(domain);
 206         size_t dump_size = SZ_4K;
 207         int i;
 208
 209         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
 210                 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
 211                         dump_size += SZ_4K;
 212
 213         return dump_size;
 214 }
 215
 216 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf)
 217 {
 218         struct etnaviv_iommuv2_domain *etnaviv_domain =
 219                         to_etnaviv_domain(domain);
 220         int i;
 221
 222         memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
 223         buf += SZ_4K;
 224         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
 225                 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
 226                         memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
 227 }
 228
 229 static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu)
 230 {
 231         struct etnaviv_iommuv2_domain *etnaviv_domain =
 232                         to_etnaviv_domain(gpu->mmu->domain);
 233         u16 prefetch;
 234
 235         /* If the MMU is already enabled the state is still there. */
 236         if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
 237                 return;
 238
 239         prefetch = etnaviv_buffer_config_mmuv2(gpu,
 240                                 (u32)etnaviv_domain->mtlb_dma,
 241                                 (u32)etnaviv_domain->base.bad_page_dma);
 242         etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
 243                              prefetch);
 244         etnaviv_gpu_wait_idle(gpu, 100);
 245
 246         gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
 247 }
 248
 249 static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu)
 250 {
 251         struct etnaviv_iommuv2_domain *etnaviv_domain =
 252                                 to_etnaviv_domain(gpu->mmu->domain);
 253         u16 prefetch;
 254
 255         /* If the MMU is already enabled the state is still there. */
 256         if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE)
 257                 return;
 258
 259         gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW,
 260                   lower_32_bits(etnaviv_domain->pta_dma));
 261         gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH,
 262                   upper_32_bits(etnaviv_domain->pta_dma));
 263         gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE);
 264
 265         gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW,
 266                   lower_32_bits(etnaviv_domain->base.bad_page_dma));
 267         gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW,
 268                   lower_32_bits(etnaviv_domain->base.bad_page_dma));
 269         gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG,
 270                   VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(
 271                   upper_32_bits(etnaviv_domain->base.bad_page_dma)) |
 272                   VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(
 273                   upper_32_bits(etnaviv_domain->base.bad_page_dma)));
 274
 275         etnaviv_domain->pta_cpu[0] = etnaviv_domain->mtlb_dma |
 276                                      VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K;
 277
 278         /* trigger a PTA load through the FE */
 279         prefetch = etnaviv_buffer_config_pta(gpu);
 280         etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
 281                              prefetch);
 282         etnaviv_gpu_wait_idle(gpu, 100);
 283
 284         gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE);
 285 }
 286
 287 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
 288 {
 289         switch (gpu->sec_mode) {
 290         case ETNA_SEC_NONE:
 291                 etnaviv_iommuv2_restore_nonsec(gpu);
 292                 break;
 293         case ETNA_SEC_KERNEL:
 294                 etnaviv_iommuv2_restore_sec(gpu);
 295                 break;
 296         default:
 297                 WARN(1, "unhandled GPU security mode\n");
 298                 break;
 299         }
 300 }
 301
 302 static const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = {
 303         .free = etnaviv_iommuv2_domain_free,
 304         .map = etnaviv_iommuv2_map,
 305         .unmap = etnaviv_iommuv2_unmap,
 306         .dump_size = etnaviv_iommuv2_dump_size,
 307         .dump = etnaviv_iommuv2_dump,
 308 };
 309
 310 struct etnaviv_iommu_domain *
 311 etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
 312 {
 313         struct etnaviv_iommuv2_domain *etnaviv_domain;
 314         struct etnaviv_iommu_domain *domain;
 315         int ret;
 316
 317         etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
 318         if (!etnaviv_domain)
 319                 return NULL;
 320
 321         domain = &etnaviv_domain->base;
 322
 323         domain->dev = gpu->dev;
 324         domain->base = 0;
 325         domain->size = (u64)SZ_1G * 4;
 326         domain->ops = &etnaviv_iommuv2_ops;
 327
 328         ret = etnaviv_iommuv2_init(etnaviv_domain);
 329         if (ret)
 330                 goto out_free;
 331
 332         return &etnaviv_domain->base;
 333
 334 out_free:
 335         vfree(etnaviv_domain);
 336         return NULL;
 337 }