drivers/iommu/arm/arm-smmu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/err.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/io.h>
  28 #include <linux/iopoll.h>
  29 #include <linux/module.h>
  30 #include <linux/of.h>
  31 #include <linux/of_address.h>
  32 #include <linux/pci.h>
  33 #include <linux/platform_device.h>
  34 #include <linux/pm_runtime.h>
  35 #include <linux/ratelimit.h>
  36 #include <linux/slab.h>
  37
  38 #include <linux/fsl/mc.h>
  39
  40 #include "arm-smmu.h"
  41 #include "../../dma-iommu.h"
  42
  43 /*
  44  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  45  * global register space are still, in fact, using a hypervisor to mediate it
  46  * by trapping and emulating register accesses. Sadly, some deployed versions
  47  * of said trapping code have bugs wherein they go horribly wrong for stores
  48  * using r31 (i.e. XZR/WZR) as the source register.
  49  */
  50 #define QCOM_DUMMY_VAL -1
  51
  52 #define MSI_IOVA_BASE                   0x8000000
  53 #define MSI_IOVA_LENGTH                 0x100000
  54
  55 static int force_stage;
  56 module_param(force_stage, int, S_IRUGO);
  57 MODULE_PARM_DESC(force_stage,
  58         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  59 static bool disable_bypass =
  60         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  61 module_param(disable_bypass, bool, S_IRUGO);
  62 MODULE_PARM_DESC(disable_bypass,
  63         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  64
  65 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  66         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  67 }
  68
  69 static bool using_legacy_binding, using_generic_binding;
  70
  71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
  72 {
  73         if (pm_runtime_enabled(smmu->dev))
  74                 return pm_runtime_resume_and_get(smmu->dev);
  75
  76         return 0;
  77 }
  78
  79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
  80 {
  81         if (pm_runtime_enabled(smmu->dev))
  82                 pm_runtime_put_autosuspend(smmu->dev);
  83 }
  84
  85 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
  86 {
  87         return container_of(dom, struct arm_smmu_domain, domain);
  88 }
  89
  90 static struct platform_driver arm_smmu_driver;
  91 static struct iommu_ops arm_smmu_ops;
  92
  93 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
  94 static struct device_node *dev_get_dev_node(struct device *dev)
  95 {
  96         if (dev_is_pci(dev)) {
  97                 struct pci_bus *bus = to_pci_dev(dev)->bus;
  98
  99                 while (!pci_is_root_bus(bus))
 100                         bus = bus->parent;
 101                 return of_node_get(bus->bridge->parent->of_node);
 102         }
 103
 104         return of_node_get(dev->of_node);
 105 }
 106
 107 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 108 {
 109         *((__be32 *)data) = cpu_to_be32(alias);
 110         return 0; /* Continue walking */
 111 }
 112
 113 static int __find_legacy_master_phandle(struct device *dev, void *data)
 114 {
 115         struct of_phandle_iterator *it = *(void **)data;
 116         struct device_node *np = it->node;
 117         int err;
 118
 119         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 120                             "#stream-id-cells", -1)
 121                 if (it->node == np) {
 122                         *(void **)data = dev;
 123                         return 1;
 124                 }
 125         it->node = np;
 126         return err == -ENOENT ? 0 : err;
 127 }
 128
 129 static int arm_smmu_register_legacy_master(struct device *dev,
 130                                            struct arm_smmu_device **smmu)
 131 {
 132         struct device *smmu_dev;
 133         struct device_node *np;
 134         struct of_phandle_iterator it;
 135         void *data = &it;
 136         u32 *sids;
 137         __be32 pci_sid;
 138         int err;
 139
 140         np = dev_get_dev_node(dev);
 141         if (!np || !of_property_present(np, "#stream-id-cells")) {
 142                 of_node_put(np);
 143                 return -ENODEV;
 144         }
 145
 146         it.node = np;
 147         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 148                                      __find_legacy_master_phandle);
 149         smmu_dev = data;
 150         of_node_put(np);
 151         if (err == 0)
 152                 return -ENODEV;
 153         if (err < 0)
 154                 return err;
 155
 156         if (dev_is_pci(dev)) {
 157                 /* "mmu-masters" assumes Stream ID == Requester ID */
 158                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 159                                        &pci_sid);
 160                 it.cur = &pci_sid;
 161                 it.cur_count = 1;
 162         }
 163
 164         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 165                                 &arm_smmu_ops);
 166         if (err)
 167                 return err;
 168
 169         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 170         if (!sids)
 171                 return -ENOMEM;
 172
 173         *smmu = dev_get_drvdata(smmu_dev);
 174         of_phandle_iterator_args(&it, sids, it.cur_count);
 175         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 176         kfree(sids);
 177         return err;
 178 }
 179 #else
 180 static int arm_smmu_register_legacy_master(struct device *dev,
 181                                            struct arm_smmu_device **smmu)
 182 {
 183         return -ENODEV;
 184 }
 185 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
 186
 187 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 188 {
 189         clear_bit(idx, map);
 190 }
 191
 192 /* Wait for any pending TLB invalidations to complete */
 193 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 194                                 int sync, int status)
 195 {
 196         unsigned int spin_cnt, delay;
 197         u32 reg;
 198
 199         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 200                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 201
 202         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 203         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 204                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 205                         reg = arm_smmu_readl(smmu, page, status);
 206                         if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
 207                                 return;
 208                         cpu_relax();
 209                 }
 210                 udelay(delay);
 211         }
 212         dev_err_ratelimited(smmu->dev,
 213                             "TLB sync timed out -- SMMU may be deadlocked\n");
 214 }
 215
 216 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 217 {
 218         unsigned long flags;
 219
 220         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 221         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 222                             ARM_SMMU_GR0_sTLBGSTATUS);
 223         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 224 }
 225
 226 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 227 {
 228         struct arm_smmu_device *smmu = smmu_domain->smmu;
 229         unsigned long flags;
 230
 231         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 232         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 233                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 234         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 235 }
 236
 237 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 238 {
 239         struct arm_smmu_domain *smmu_domain = cookie;
 240         /*
 241          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 242          * current CPU are visible beforehand.
 243          */
 244         wmb();
 245         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 246                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 247         arm_smmu_tlb_sync_context(smmu_domain);
 248 }
 249
 250 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 251 {
 252         struct arm_smmu_domain *smmu_domain = cookie;
 253         struct arm_smmu_device *smmu = smmu_domain->smmu;
 254
 255         /* See above */
 256         wmb();
 257         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 258         arm_smmu_tlb_sync_global(smmu);
 259 }
 260
 261 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 262                                       size_t granule, void *cookie, int reg)
 263 {
 264         struct arm_smmu_domain *smmu_domain = cookie;
 265         struct arm_smmu_device *smmu = smmu_domain->smmu;
 266         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 267         int idx = cfg->cbndx;
 268
 269         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 270                 wmb();
 271
 272         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 273                 iova = (iova >> 12) << 12;
 274                 iova |= cfg->asid;
 275                 do {
 276                         arm_smmu_cb_write(smmu, idx, reg, iova);
 277                         iova += granule;
 278                 } while (size -= granule);
 279         } else {
 280                 iova >>= 12;
 281                 iova |= (u64)cfg->asid << 48;
 282                 do {
 283                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 284                         iova += granule >> 12;
 285                 } while (size -= granule);
 286         }
 287 }
 288
 289 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 290                                       size_t granule, void *cookie, int reg)
 291 {
 292         struct arm_smmu_domain *smmu_domain = cookie;
 293         struct arm_smmu_device *smmu = smmu_domain->smmu;
 294         int idx = smmu_domain->cfg.cbndx;
 295
 296         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 297                 wmb();
 298
 299         iova >>= 12;
 300         do {
 301                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 302                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 303                 else
 304                         arm_smmu_cb_write(smmu, idx, reg, iova);
 305                 iova += granule >> 12;
 306         } while (size -= granule);
 307 }
 308
 309 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 310                                      size_t granule, void *cookie)
 311 {
 312         struct arm_smmu_domain *smmu_domain = cookie;
 313         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 314
 315         if (cfg->flush_walk_prefer_tlbiasid) {
 316                 arm_smmu_tlb_inv_context_s1(cookie);
 317         } else {
 318                 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 319                                           ARM_SMMU_CB_S1_TLBIVA);
 320                 arm_smmu_tlb_sync_context(cookie);
 321         }
 322 }
 323
 324 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 325                                      unsigned long iova, size_t granule,
 326                                      void *cookie)
 327 {
 328         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 329                                   ARM_SMMU_CB_S1_TLBIVAL);
 330 }
 331
 332 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 333                                      size_t granule, void *cookie)
 334 {
 335         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 336                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 337         arm_smmu_tlb_sync_context(cookie);
 338 }
 339
 340 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 341                                      unsigned long iova, size_t granule,
 342                                      void *cookie)
 343 {
 344         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 345                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 346 }
 347
 348 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
 349                                         size_t granule, void *cookie)
 350 {
 351         arm_smmu_tlb_inv_context_s2(cookie);
 352 }
 353 /*
 354  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 355  * almost negligible, but the benefit of getting the first one in as far ahead
 356  * of the sync as possible is significant, hence we don't just make this a
 357  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 358  * think.
 359  */
 360 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 361                                         unsigned long iova, size_t granule,
 362                                         void *cookie)
 363 {
 364         struct arm_smmu_domain *smmu_domain = cookie;
 365         struct arm_smmu_device *smmu = smmu_domain->smmu;
 366
 367         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 368                 wmb();
 369
 370         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 371 }
 372
 373 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 374         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 375         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 376         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 377 };
 378
 379 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 380         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 381         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 382         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 383 };
 384
 385 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 386         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 387         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
 388         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 389 };
 390
 391 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 392 {
 393         u32 fsr, fsynr, cbfrsynra;
 394         unsigned long iova;
 395         struct iommu_domain *domain = dev;
 396         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 397         struct arm_smmu_device *smmu = smmu_domain->smmu;
 398         int idx = smmu_domain->cfg.cbndx;
 399         int ret;
 400
 401         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 402         if (!(fsr & ARM_SMMU_FSR_FAULT))
 403                 return IRQ_NONE;
 404
 405         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 406         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 407         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 408
 409         ret = report_iommu_fault(domain, NULL, iova,
 410                 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
 411
 412         if (ret == -ENOSYS)
 413                 dev_err_ratelimited(smmu->dev,
 414                 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 415                             fsr, iova, fsynr, cbfrsynra, idx);
 416
 417         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 418         return IRQ_HANDLED;
 419 }
 420
 421 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 422 {
 423         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 424         struct arm_smmu_device *smmu = dev;
 425         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 426                                       DEFAULT_RATELIMIT_BURST);
 427
 428         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 429         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 430         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 431         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 432
 433         if (!gfsr)
 434                 return IRQ_NONE;
 435
 436         if (__ratelimit(&rs)) {
 437                 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 438                     (gfsr & ARM_SMMU_sGFSR_USF))
 439                         dev_err(smmu->dev,
 440                                 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 441                                 (u16)gfsynr1);
 442                 else
 443                         dev_err(smmu->dev,
 444                                 "Unexpected global fault, this could be serious\n");
 445                 dev_err(smmu->dev,
 446                         "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 447                         gfsr, gfsynr0, gfsynr1, gfsynr2);
 448         }
 449
 450         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 451         return IRQ_HANDLED;
 452 }
 453
 454 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 455                                        struct io_pgtable_cfg *pgtbl_cfg)
 456 {
 457         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 458         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 459         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 460
 461         cb->cfg = cfg;
 462
 463         /* TCR */
 464         if (stage1) {
 465                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 466                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 467                 } else {
 468                         cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
 469                         cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
 470                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 471                                 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
 472                         else
 473                                 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
 474                 }
 475         } else {
 476                 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
 477         }
 478
 479         /* TTBRs */
 480         if (stage1) {
 481                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 482                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
 483                         cb->ttbr[1] = 0;
 484                 } else {
 485                         cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 486                                                  cfg->asid);
 487                         cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 488                                                  cfg->asid);
 489
 490                         if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 491                                 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 492                         else
 493                                 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 494                 }
 495         } else {
 496                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 497         }
 498
 499         /* MAIRs (stage-1 only) */
 500         if (stage1) {
 501                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 502                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 503                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 504                 } else {
 505                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 506                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
 507                 }
 508         }
 509 }
 510
 511 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 512 {
 513         u32 reg;
 514         bool stage1;
 515         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 516         struct arm_smmu_cfg *cfg = cb->cfg;
 517
 518         /* Unassigned context banks only need disabling */
 519         if (!cfg) {
 520                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 521                 return;
 522         }
 523
 524         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 525
 526         /* CBA2R */
 527         if (smmu->version > ARM_SMMU_V1) {
 528                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 529                         reg = ARM_SMMU_CBA2R_VA64;
 530                 else
 531                         reg = 0;
 532                 /* 16-bit VMIDs live in CBA2R */
 533                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 534                         reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
 535
 536                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 537         }
 538
 539         /* CBAR */
 540         reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
 541         if (smmu->version < ARM_SMMU_V2)
 542                 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
 543
 544         /*
 545          * Use the weakest shareability/memory types, so they are
 546          * overridden by the ttbcr/pte.
 547          */
 548         if (stage1) {
 549                 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
 550                                   ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
 551                        FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
 552                                   ARM_SMMU_CBAR_S1_MEMATTR_WB);
 553         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 554                 /* 8-bit VMIDs live in CBAR */
 555                 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
 556         }
 557         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 558
 559         /*
 560          * TCR
 561          * We must write this before the TTBRs, since it determines the
 562          * access behaviour of some fields (in particular, ASID[15:8]).
 563          */
 564         if (stage1 && smmu->version > ARM_SMMU_V1)
 565                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 566         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 567
 568         /* TTBRs */
 569         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 570                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 571                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 572                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 573         } else {
 574                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 575                 if (stage1)
 576                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 577                                            cb->ttbr[1]);
 578         }
 579
 580         /* MAIRs (stage-1 only) */
 581         if (stage1) {
 582                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 583                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 584         }
 585
 586         /* SCTLR */
 587         reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
 588               ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
 589         if (stage1)
 590                 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
 591         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 592                 reg |= ARM_SMMU_SCTLR_E;
 593
 594         if (smmu->impl && smmu->impl->write_sctlr)
 595                 smmu->impl->write_sctlr(smmu, idx, reg);
 596         else
 597                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 598 }
 599
 600 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
 601                                        struct arm_smmu_device *smmu,
 602                                        struct device *dev, unsigned int start)
 603 {
 604         if (smmu->impl && smmu->impl->alloc_context_bank)
 605                 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
 606
 607         return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
 608 }
 609
 610 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 611                                         struct arm_smmu_device *smmu,
 612                                         struct device *dev)
 613 {
 614         int irq, start, ret = 0;
 615         unsigned long ias, oas;
 616         struct io_pgtable_ops *pgtbl_ops;
 617         struct io_pgtable_cfg pgtbl_cfg;
 618         enum io_pgtable_fmt fmt;
 619         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 620         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 621         irqreturn_t (*context_fault)(int irq, void *dev);
 622
 623         mutex_lock(&smmu_domain->init_mutex);
 624         if (smmu_domain->smmu)
 625                 goto out_unlock;
 626
 627         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 628                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 629                 smmu_domain->smmu = smmu;
 630                 goto out_unlock;
 631         }
 632
 633         /*
 634          * Mapping the requested stage onto what we support is surprisingly
 635          * complicated, mainly because the spec allows S1+S2 SMMUs without
 636          * support for nested translation. That means we end up with the
 637          * following table:
 638          *
 639          * Requested        Supported        Actual
 640          *     S1               N              S1
 641          *     S1             S1+S2            S1
 642          *     S1               S2             S2
 643          *     S1               S1             S1
 644          *     N                N              N
 645          *     N              S1+S2            S2
 646          *     N                S2             S2
 647          *     N                S1             S1
 648          *
 649          * Note that you can't actually request stage-2 mappings.
 650          */
 651         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 652                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 653         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 654                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 655
 656         /*
 657          * Choosing a suitable context format is even more fiddly. Until we
 658          * grow some way for the caller to express a preference, and/or move
 659          * the decision into the io-pgtable code where it arguably belongs,
 660          * just aim for the closest thing to the rest of the system, and hope
 661          * that the hardware isn't esoteric enough that we can't assume AArch64
 662          * support to be a superset of AArch32 support...
 663          */
 664         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 665                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 666         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 667             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 668             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 669             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 670                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 671         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 672             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 673                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 674                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 675                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 676
 677         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 678                 ret = -EINVAL;
 679                 goto out_unlock;
 680         }
 681
 682         switch (smmu_domain->stage) {
 683         case ARM_SMMU_DOMAIN_S1:
 684                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 685                 start = smmu->num_s2_context_banks;
 686                 ias = smmu->va_size;
 687                 oas = smmu->ipa_size;
 688                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 689                         fmt = ARM_64_LPAE_S1;
 690                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 691                         fmt = ARM_32_LPAE_S1;
 692                         ias = min(ias, 32UL);
 693                         oas = min(oas, 40UL);
 694                 } else {
 695                         fmt = ARM_V7S;
 696                         ias = min(ias, 32UL);
 697                         oas = min(oas, 32UL);
 698                 }
 699                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 700                 break;
 701         case ARM_SMMU_DOMAIN_NESTED:
 702                 /*
 703                  * We will likely want to change this if/when KVM gets
 704                  * involved.
 705                  */
 706         case ARM_SMMU_DOMAIN_S2:
 707                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 708                 start = 0;
 709                 ias = smmu->ipa_size;
 710                 oas = smmu->pa_size;
 711                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 712                         fmt = ARM_64_LPAE_S2;
 713                 } else {
 714                         fmt = ARM_32_LPAE_S2;
 715                         ias = min(ias, 40UL);
 716                         oas = min(oas, 40UL);
 717                 }
 718                 if (smmu->version == ARM_SMMU_V2)
 719                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 720                 else
 721                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 722                 break;
 723         default:
 724                 ret = -EINVAL;
 725                 goto out_unlock;
 726         }
 727
 728         ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
 729         if (ret < 0) {
 730                 goto out_unlock;
 731         }
 732
 733         smmu_domain->smmu = smmu;
 734
 735         cfg->cbndx = ret;
 736         if (smmu->version < ARM_SMMU_V2) {
 737                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 738                 cfg->irptndx %= smmu->num_context_irqs;
 739         } else {
 740                 cfg->irptndx = cfg->cbndx;
 741         }
 742
 743         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 744                 cfg->vmid = cfg->cbndx + 1;
 745         else
 746                 cfg->asid = cfg->cbndx;
 747
 748         pgtbl_cfg = (struct io_pgtable_cfg) {
 749                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 750                 .ias            = ias,
 751                 .oas            = oas,
 752                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 753                 .tlb            = smmu_domain->flush_ops,
 754                 .iommu_dev      = smmu->dev,
 755         };
 756
 757         if (smmu->impl && smmu->impl->init_context) {
 758                 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
 759                 if (ret)
 760                         goto out_clear_smmu;
 761         }
 762
 763         if (smmu_domain->pgtbl_quirks)
 764                 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
 765
 766         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 767         if (!pgtbl_ops) {
 768                 ret = -ENOMEM;
 769                 goto out_clear_smmu;
 770         }
 771
 772         /* Update the domain's page sizes to reflect the page table format */
 773         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 774
 775         if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
 776                 domain->geometry.aperture_start = ~0UL << ias;
 777                 domain->geometry.aperture_end = ~0UL;
 778         } else {
 779                 domain->geometry.aperture_end = (1UL << ias) - 1;
 780         }
 781
 782         domain->geometry.force_aperture = true;
 783
 784         /* Initialise the context bank with our page table cfg */
 785         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 786         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 787
 788         /*
 789          * Request context fault interrupt. Do this last to avoid the
 790          * handler seeing a half-initialised domain state.
 791          */
 792         irq = smmu->irqs[cfg->irptndx];
 793
 794         if (smmu->impl && smmu->impl->context_fault)
 795                 context_fault = smmu->impl->context_fault;
 796         else
 797                 context_fault = arm_smmu_context_fault;
 798
 799         ret = devm_request_irq(smmu->dev, irq, context_fault,
 800                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 801         if (ret < 0) {
 802                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 803                         cfg->irptndx, irq);
 804                 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
 805         }
 806
 807         mutex_unlock(&smmu_domain->init_mutex);
 808
 809         /* Publish page table ops for map/unmap */
 810         smmu_domain->pgtbl_ops = pgtbl_ops;
 811         return 0;
 812
 813 out_clear_smmu:
 814         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 815         smmu_domain->smmu = NULL;
 816 out_unlock:
 817         mutex_unlock(&smmu_domain->init_mutex);
 818         return ret;
 819 }
 820
 821 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 822 {
 823         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 824         struct arm_smmu_device *smmu = smmu_domain->smmu;
 825         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 826         int ret, irq;
 827
 828         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 829                 return;
 830
 831         ret = arm_smmu_rpm_get(smmu);
 832         if (ret < 0)
 833                 return;
 834
 835         /*
 836          * Disable the context bank and free the page tables before freeing
 837          * it.
 838          */
 839         smmu->cbs[cfg->cbndx].cfg = NULL;
 840         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 841
 842         if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
 843                 irq = smmu->irqs[cfg->irptndx];
 844                 devm_free_irq(smmu->dev, irq, domain);
 845         }
 846
 847         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 848         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 849
 850         arm_smmu_rpm_put(smmu);
 851 }
 852
 853 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 854 {
 855         struct arm_smmu_domain *smmu_domain;
 856
 857         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
 858                 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
 859                         return NULL;
 860         }
 861         /*
 862          * Allocate the domain and initialise some of its data structures.
 863          * We can't really do anything meaningful until we've added a
 864          * master.
 865          */
 866         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 867         if (!smmu_domain)
 868                 return NULL;
 869
 870         mutex_init(&smmu_domain->init_mutex);
 871         spin_lock_init(&smmu_domain->cb_lock);
 872
 873         return &smmu_domain->domain;
 874 }
 875
 876 static void arm_smmu_domain_free(struct iommu_domain *domain)
 877 {
 878         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 879
 880         /*
 881          * Free the domain resources. We assume that all devices have
 882          * already been detached.
 883          */
 884         arm_smmu_destroy_domain_context(domain);
 885         kfree(smmu_domain);
 886 }
 887
 888 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 889 {
 890         struct arm_smmu_smr *smr = smmu->smrs + idx;
 891         u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
 892                   FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
 893
 894         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 895                 reg |= ARM_SMMU_SMR_VALID;
 896         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 897 }
 898
 899 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 900 {
 901         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 902         u32 reg;
 903
 904         if (smmu->impl && smmu->impl->write_s2cr) {
 905                 smmu->impl->write_s2cr(smmu, idx);
 906                 return;
 907         }
 908
 909         reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
 910               FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
 911               FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 912
 913         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 914             smmu->smrs[idx].valid)
 915                 reg |= ARM_SMMU_S2CR_EXIDVALID;
 916         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 917 }
 918
 919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 920 {
 921         arm_smmu_write_s2cr(smmu, idx);
 922         if (smmu->smrs)
 923                 arm_smmu_write_smr(smmu, idx);
 924 }
 925
 926 /*
 927  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 928  * should be called after sCR0 is written.
 929  */
 930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 931 {
 932         u32 smr;
 933         int i;
 934
 935         if (!smmu->smrs)
 936                 return;
 937         /*
 938          * If we've had to accommodate firmware memory regions, we may
 939          * have live SMRs by now; tread carefully...
 940          *
 941          * Somewhat perversely, not having a free SMR for this test implies we
 942          * can get away without it anyway, as we'll only be able to 'allocate'
 943          * these SMRs for the ID/mask values we're already trusting to be OK.
 944          */
 945         for (i = 0; i < smmu->num_mapping_groups; i++)
 946                 if (!smmu->smrs[i].valid)
 947                         goto smr_ok;
 948         return;
 949 smr_ok:
 950         /*
 951          * SMR.ID bits may not be preserved if the corresponding MASK
 952          * bits are set, so check each one separately. We can reject
 953          * masters later if they try to claim IDs outside these masks.
 954          */
 955         smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
 956         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 957         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 958         smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
 959
 960         smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
 961         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 962         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 963         smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
 964 }
 965
 966 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 967 {
 968         struct arm_smmu_smr *smrs = smmu->smrs;
 969         int i, free_idx = -ENOSPC;
 970
 971         /* Stream indexing is blissfully easy */
 972         if (!smrs)
 973                 return id;
 974
 975         /* Validating SMRs is... less so */
 976         for (i = 0; i < smmu->num_mapping_groups; ++i) {
 977                 if (!smrs[i].valid) {
 978                         /*
 979                          * Note the first free entry we come across, which
 980                          * we'll claim in the end if nothing else matches.
 981                          */
 982                         if (free_idx < 0)
 983                                 free_idx = i;
 984                         continue;
 985                 }
 986                 /*
 987                  * If the new entry is _entirely_ matched by an existing entry,
 988                  * then reuse that, with the guarantee that there also cannot
 989                  * be any subsequent conflicting entries. In normal use we'd
 990                  * expect simply identical entries for this case, but there's
 991                  * no harm in accommodating the generalisation.
 992                  */
 993                 if ((mask & smrs[i].mask) == mask &&
 994                     !((id ^ smrs[i].id) & ~smrs[i].mask))
 995                         return i;
 996                 /*
 997                  * If the new entry has any other overlap with an existing one,
 998                  * though, then there always exists at least one stream ID
 999                  * which would cause a conflict, and we can't allow that risk.
1000                  */
1001                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1002                         return -EINVAL;
1003         }
1004
1005         return free_idx;
1006 }
1007
1008 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1009 {
1010         if (--smmu->s2crs[idx].count)
1011                 return false;
1012
1013         smmu->s2crs[idx] = s2cr_init_val;
1014         if (smmu->smrs)
1015                 smmu->smrs[idx].valid = false;
1016
1017         return true;
1018 }
1019
1020 static int arm_smmu_master_alloc_smes(struct device *dev)
1021 {
1022         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024         struct arm_smmu_device *smmu = cfg->smmu;
1025         struct arm_smmu_smr *smrs = smmu->smrs;
1026         int i, idx, ret;
1027
1028         mutex_lock(&smmu->stream_map_mutex);
1029         /* Figure out a viable stream map entry allocation */
1030         for_each_cfg_sme(cfg, fwspec, i, idx) {
1031                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1033
1034                 if (idx != INVALID_SMENDX) {
1035                         ret = -EEXIST;
1036                         goto out_err;
1037                 }
1038
1039                 ret = arm_smmu_find_sme(smmu, sid, mask);
1040                 if (ret < 0)
1041                         goto out_err;
1042
1043                 idx = ret;
1044                 if (smrs && smmu->s2crs[idx].count == 0) {
1045                         smrs[idx].id = sid;
1046                         smrs[idx].mask = mask;
1047                         smrs[idx].valid = true;
1048                 }
1049                 smmu->s2crs[idx].count++;
1050                 cfg->smendx[i] = (s16)idx;
1051         }
1052
1053         /* It worked! Now, poke the actual hardware */
1054         for_each_cfg_sme(cfg, fwspec, i, idx)
1055                 arm_smmu_write_sme(smmu, idx);
1056
1057         mutex_unlock(&smmu->stream_map_mutex);
1058         return 0;
1059
1060 out_err:
1061         while (i--) {
1062                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063                 cfg->smendx[i] = INVALID_SMENDX;
1064         }
1065         mutex_unlock(&smmu->stream_map_mutex);
1066         return ret;
1067 }
1068
1069 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070                                       struct iommu_fwspec *fwspec)
1071 {
1072         struct arm_smmu_device *smmu = cfg->smmu;
1073         int i, idx;
1074
1075         mutex_lock(&smmu->stream_map_mutex);
1076         for_each_cfg_sme(cfg, fwspec, i, idx) {
1077                 if (arm_smmu_free_sme(smmu, idx))
1078                         arm_smmu_write_sme(smmu, idx);
1079                 cfg->smendx[i] = INVALID_SMENDX;
1080         }
1081         mutex_unlock(&smmu->stream_map_mutex);
1082 }
1083
1084 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085                                       struct arm_smmu_master_cfg *cfg,
1086                                       struct iommu_fwspec *fwspec)
1087 {
1088         struct arm_smmu_device *smmu = smmu_domain->smmu;
1089         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090         u8 cbndx = smmu_domain->cfg.cbndx;
1091         enum arm_smmu_s2cr_type type;
1092         int i, idx;
1093
1094         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095                 type = S2CR_TYPE_BYPASS;
1096         else
1097                 type = S2CR_TYPE_TRANS;
1098
1099         for_each_cfg_sme(cfg, fwspec, i, idx) {
1100                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101                         continue;
1102
1103                 s2cr[idx].type = type;
1104                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105                 s2cr[idx].cbndx = cbndx;
1106                 arm_smmu_write_s2cr(smmu, idx);
1107         }
1108         return 0;
1109 }
1110
1111 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112 {
1113         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115         struct arm_smmu_master_cfg *cfg;
1116         struct arm_smmu_device *smmu;
1117         int ret;
1118
1119         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1121                 return -ENXIO;
1122         }
1123
1124         /*
1125          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126          * domains between of_xlate() and probe_device() - we have no way to cope
1127          * with that, so until ARM gets converted to rely on groups and default
1128          * domains, just say no (but more politely than by dereferencing NULL).
1129          * This should be at least a WARN_ON once that's sorted.
1130          */
1131         cfg = dev_iommu_priv_get(dev);
1132         if (!cfg)
1133                 return -ENODEV;
1134
1135         smmu = cfg->smmu;
1136
1137         ret = arm_smmu_rpm_get(smmu);
1138         if (ret < 0)
1139                 return ret;
1140
1141         /* Ensure that the domain is finalised */
1142         ret = arm_smmu_init_domain_context(domain, smmu, dev);
1143         if (ret < 0)
1144                 goto rpm_put;
1145
1146         /*
1147          * Sanity check the domain. We don't support domains across
1148          * different SMMUs.
1149          */
1150         if (smmu_domain->smmu != smmu) {
1151                 ret = -EINVAL;
1152                 goto rpm_put;
1153         }
1154
1155         /* Looks ok, so add the device to the domain */
1156         ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1157
1158         /*
1159          * Setup an autosuspend delay to avoid bouncing runpm state.
1160          * Otherwise, if a driver for a suspended consumer device
1161          * unmaps buffers, it will runpm resume/suspend for each one.
1162          *
1163          * For example, when used by a GPU device, when an application
1164          * or game exits, it can trigger unmapping 100s or 1000s of
1165          * buffers.  With a runpm cycle for each buffer, that adds up
1166          * to 5-10sec worth of reprogramming the context bank, while
1167          * the system appears to be locked up to the user.
1168          */
1169         pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170         pm_runtime_use_autosuspend(smmu->dev);
1171
1172 rpm_put:
1173         arm_smmu_rpm_put(smmu);
1174         return ret;
1175 }
1176
1177 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1178                               phys_addr_t paddr, size_t pgsize, size_t pgcount,
1179                               int prot, gfp_t gfp, size_t *mapped)
1180 {
1181         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1183         int ret;
1184
1185         if (!ops)
1186                 return -ENODEV;
1187
1188         arm_smmu_rpm_get(smmu);
1189         ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190         arm_smmu_rpm_put(smmu);
1191
1192         return ret;
1193 }
1194
1195 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1196                                    size_t pgsize, size_t pgcount,
1197                                    struct iommu_iotlb_gather *iotlb_gather)
1198 {
1199         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1201         size_t ret;
1202
1203         if (!ops)
1204                 return 0;
1205
1206         arm_smmu_rpm_get(smmu);
1207         ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208         arm_smmu_rpm_put(smmu);
1209
1210         return ret;
1211 }
1212
1213 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1214 {
1215         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216         struct arm_smmu_device *smmu = smmu_domain->smmu;
1217
1218         if (smmu_domain->flush_ops) {
1219                 arm_smmu_rpm_get(smmu);
1220                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221                 arm_smmu_rpm_put(smmu);
1222         }
1223 }
1224
1225 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226                                 struct iommu_iotlb_gather *gather)
1227 {
1228         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229         struct arm_smmu_device *smmu = smmu_domain->smmu;
1230
1231         if (!smmu)
1232                 return;
1233
1234         arm_smmu_rpm_get(smmu);
1235         if (smmu->version == ARM_SMMU_V2 ||
1236             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237                 arm_smmu_tlb_sync_context(smmu_domain);
1238         else
1239                 arm_smmu_tlb_sync_global(smmu);
1240         arm_smmu_rpm_put(smmu);
1241 }
1242
1243 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1244                                               dma_addr_t iova)
1245 {
1246         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247         struct arm_smmu_device *smmu = smmu_domain->smmu;
1248         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250         struct device *dev = smmu->dev;
1251         void __iomem *reg;
1252         u32 tmp;
1253         u64 phys;
1254         unsigned long va, flags;
1255         int ret, idx = cfg->cbndx;
1256         phys_addr_t addr = 0;
1257
1258         ret = arm_smmu_rpm_get(smmu);
1259         if (ret < 0)
1260                 return 0;
1261
1262         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263         va = iova & ~0xfffUL;
1264         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1266         else
1267                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1268
1269         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1271                                       5, 50)) {
1272                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1273                 dev_err(dev,
1274                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1275                         &iova);
1276                 arm_smmu_rpm_put(smmu);
1277                 return ops->iova_to_phys(ops, iova);
1278         }
1279
1280         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282         if (phys & ARM_SMMU_CB_PAR_F) {
1283                 dev_err(dev, "translation fault!\n");
1284                 dev_err(dev, "PAR = 0x%llx\n", phys);
1285                 goto out;
1286         }
1287
1288         addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1289 out:
1290         arm_smmu_rpm_put(smmu);
1291
1292         return addr;
1293 }
1294
1295 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1296                                         dma_addr_t iova)
1297 {
1298         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1300
1301         if (!ops)
1302                 return 0;
1303
1304         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306                 return arm_smmu_iova_to_phys_hard(domain, iova);
1307
1308         return ops->iova_to_phys(ops, iova);
1309 }
1310
1311 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1312 {
1313         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1314
1315         switch (cap) {
1316         case IOMMU_CAP_CACHE_COHERENCY:
1317                 /*
1318                  * It's overwhelmingly the case in practice that when the pagetable
1319                  * walk interface is connected to a coherent interconnect, all the
1320                  * translation interfaces are too. Furthermore if the device is
1321                  * natively coherent, then its translation interface must also be.
1322                  */
1323                 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324                         device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325         case IOMMU_CAP_NOEXEC:
1326         case IOMMU_CAP_DEFERRED_FLUSH:
1327                 return true;
1328         default:
1329                 return false;
1330         }
1331 }
1332
1333 static
1334 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1335 {
1336         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1337                                                           fwnode);
1338         put_device(dev);
1339         return dev ? dev_get_drvdata(dev) : NULL;
1340 }
1341
1342 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1343 {
1344         struct arm_smmu_device *smmu = NULL;
1345         struct arm_smmu_master_cfg *cfg;
1346         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1347         int i, ret;
1348
1349         if (using_legacy_binding) {
1350                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1351
1352                 /*
1353                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354                  * will allocate/initialise a new one. Thus we need to update fwspec for
1355                  * later use.
1356                  */
1357                 fwspec = dev_iommu_fwspec_get(dev);
1358                 if (ret)
1359                         goto out_free;
1360         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1362         } else {
1363                 return ERR_PTR(-ENODEV);
1364         }
1365
1366         ret = -EINVAL;
1367         for (i = 0; i < fwspec->num_ids; i++) {
1368                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1369                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1370
1371                 if (sid & ~smmu->streamid_mask) {
1372                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1373                                 sid, smmu->streamid_mask);
1374                         goto out_free;
1375                 }
1376                 if (mask & ~smmu->smr_mask_mask) {
1377                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1378                                 mask, smmu->smr_mask_mask);
1379                         goto out_free;
1380                 }
1381         }
1382
1383         ret = -ENOMEM;
1384         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1385                       GFP_KERNEL);
1386         if (!cfg)
1387                 goto out_free;
1388
1389         cfg->smmu = smmu;
1390         dev_iommu_priv_set(dev, cfg);
1391         while (i--)
1392                 cfg->smendx[i] = INVALID_SMENDX;
1393
1394         ret = arm_smmu_rpm_get(smmu);
1395         if (ret < 0)
1396                 goto out_cfg_free;
1397
1398         ret = arm_smmu_master_alloc_smes(dev);
1399         arm_smmu_rpm_put(smmu);
1400
1401         if (ret)
1402                 goto out_cfg_free;
1403
1404         device_link_add(dev, smmu->dev,
1405                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1406
1407         return &smmu->iommu;
1408
1409 out_cfg_free:
1410         kfree(cfg);
1411 out_free:
1412         iommu_fwspec_free(dev);
1413         return ERR_PTR(ret);
1414 }
1415
1416 static void arm_smmu_release_device(struct device *dev)
1417 {
1418         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1419         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1420         int ret;
1421
1422         ret = arm_smmu_rpm_get(cfg->smmu);
1423         if (ret < 0)
1424                 return;
1425
1426         arm_smmu_master_free_smes(cfg, fwspec);
1427
1428         arm_smmu_rpm_put(cfg->smmu);
1429
1430         dev_iommu_priv_set(dev, NULL);
1431         kfree(cfg);
1432 }
1433
1434 static void arm_smmu_probe_finalize(struct device *dev)
1435 {
1436         struct arm_smmu_master_cfg *cfg;
1437         struct arm_smmu_device *smmu;
1438
1439         cfg = dev_iommu_priv_get(dev);
1440         smmu = cfg->smmu;
1441
1442         if (smmu->impl && smmu->impl->probe_finalize)
1443                 smmu->impl->probe_finalize(smmu, dev);
1444 }
1445
1446 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1447 {
1448         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1449         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1450         struct arm_smmu_device *smmu = cfg->smmu;
1451         struct iommu_group *group = NULL;
1452         int i, idx;
1453
1454         mutex_lock(&smmu->stream_map_mutex);
1455         for_each_cfg_sme(cfg, fwspec, i, idx) {
1456                 if (group && smmu->s2crs[idx].group &&
1457                     group != smmu->s2crs[idx].group) {
1458                         mutex_unlock(&smmu->stream_map_mutex);
1459                         return ERR_PTR(-EINVAL);
1460                 }
1461
1462                 group = smmu->s2crs[idx].group;
1463         }
1464
1465         if (group) {
1466                 mutex_unlock(&smmu->stream_map_mutex);
1467                 return iommu_group_ref_get(group);
1468         }
1469
1470         if (dev_is_pci(dev))
1471                 group = pci_device_group(dev);
1472         else if (dev_is_fsl_mc(dev))
1473                 group = fsl_mc_device_group(dev);
1474         else
1475                 group = generic_device_group(dev);
1476
1477         /* Remember group for faster lookups */
1478         if (!IS_ERR(group))
1479                 for_each_cfg_sme(cfg, fwspec, i, idx)
1480                         smmu->s2crs[idx].group = group;
1481
1482         mutex_unlock(&smmu->stream_map_mutex);
1483         return group;
1484 }
1485
1486 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1487 {
1488         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1489         int ret = 0;
1490
1491         mutex_lock(&smmu_domain->init_mutex);
1492         if (smmu_domain->smmu)
1493                 ret = -EPERM;
1494         else
1495                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1496         mutex_unlock(&smmu_domain->init_mutex);
1497
1498         return ret;
1499 }
1500
1501 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1502                 unsigned long quirks)
1503 {
1504         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1505         int ret = 0;
1506
1507         mutex_lock(&smmu_domain->init_mutex);
1508         if (smmu_domain->smmu)
1509                 ret = -EPERM;
1510         else
1511                 smmu_domain->pgtbl_quirks = quirks;
1512         mutex_unlock(&smmu_domain->init_mutex);
1513
1514         return ret;
1515 }
1516
1517 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1518 {
1519         u32 mask, fwid = 0;
1520
1521         if (args->args_count > 0)
1522                 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1523
1524         if (args->args_count > 1)
1525                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1526         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1527                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1528
1529         return iommu_fwspec_add_ids(dev, &fwid, 1);
1530 }
1531
1532 static void arm_smmu_get_resv_regions(struct device *dev,
1533                                       struct list_head *head)
1534 {
1535         struct iommu_resv_region *region;
1536         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1537
1538         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1539                                          prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1540         if (!region)
1541                 return;
1542
1543         list_add_tail(&region->list, head);
1544
1545         iommu_dma_get_resv_regions(dev, head);
1546 }
1547
1548 static int arm_smmu_def_domain_type(struct device *dev)
1549 {
1550         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1551         const struct arm_smmu_impl *impl = cfg->smmu->impl;
1552
1553         if (using_legacy_binding)
1554                 return IOMMU_DOMAIN_IDENTITY;
1555
1556         if (impl && impl->def_domain_type)
1557                 return impl->def_domain_type(dev);
1558
1559         return 0;
1560 }
1561
1562 static struct iommu_ops arm_smmu_ops = {
1563         .capable                = arm_smmu_capable,
1564         .domain_alloc           = arm_smmu_domain_alloc,
1565         .probe_device           = arm_smmu_probe_device,
1566         .release_device         = arm_smmu_release_device,
1567         .probe_finalize         = arm_smmu_probe_finalize,
1568         .device_group           = arm_smmu_device_group,
1569         .of_xlate               = arm_smmu_of_xlate,
1570         .get_resv_regions       = arm_smmu_get_resv_regions,
1571         .def_domain_type        = arm_smmu_def_domain_type,
1572         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1573         .owner                  = THIS_MODULE,
1574         .default_domain_ops = &(const struct iommu_domain_ops) {
1575                 .attach_dev             = arm_smmu_attach_dev,
1576                 .map_pages              = arm_smmu_map_pages,
1577                 .unmap_pages            = arm_smmu_unmap_pages,
1578                 .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1579                 .iotlb_sync             = arm_smmu_iotlb_sync,
1580                 .iova_to_phys           = arm_smmu_iova_to_phys,
1581                 .enable_nesting         = arm_smmu_enable_nesting,
1582                 .set_pgtable_quirks     = arm_smmu_set_pgtable_quirks,
1583                 .free                   = arm_smmu_domain_free,
1584         }
1585 };
1586
1587 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1588 {
1589         int i;
1590         u32 reg;
1591
1592         /* clear global FSR */
1593         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1594         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1595
1596         /*
1597          * Reset stream mapping groups: Initial values mark all SMRn as
1598          * invalid and all S2CRn as bypass unless overridden.
1599          */
1600         for (i = 0; i < smmu->num_mapping_groups; ++i)
1601                 arm_smmu_write_sme(smmu, i);
1602
1603         /* Make sure all context banks are disabled and clear CB_FSR  */
1604         for (i = 0; i < smmu->num_context_banks; ++i) {
1605                 arm_smmu_write_context_bank(smmu, i);
1606                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1607         }
1608
1609         /* Invalidate the TLB, just in case */
1610         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1611         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1612
1613         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1614
1615         /* Enable fault reporting */
1616         reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1617                 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1618
1619         /* Disable TLB broadcasting. */
1620         reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1621
1622         /* Enable client access, handling unmatched streams as appropriate */
1623         reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1624         if (disable_bypass)
1625                 reg |= ARM_SMMU_sCR0_USFCFG;
1626         else
1627                 reg &= ~ARM_SMMU_sCR0_USFCFG;
1628
1629         /* Disable forced broadcasting */
1630         reg &= ~ARM_SMMU_sCR0_FB;
1631
1632         /* Don't upgrade barriers */
1633         reg &= ~(ARM_SMMU_sCR0_BSU);
1634
1635         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1636                 reg |= ARM_SMMU_sCR0_VMID16EN;
1637
1638         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1639                 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1640
1641         if (smmu->impl && smmu->impl->reset)
1642                 smmu->impl->reset(smmu);
1643
1644         /* Push the button */
1645         arm_smmu_tlb_sync_global(smmu);
1646         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1647 }
1648
1649 static int arm_smmu_id_size_to_bits(int size)
1650 {
1651         switch (size) {
1652         case 0:
1653                 return 32;
1654         case 1:
1655                 return 36;
1656         case 2:
1657                 return 40;
1658         case 3:
1659                 return 42;
1660         case 4:
1661                 return 44;
1662         case 5:
1663         default:
1664                 return 48;
1665         }
1666 }
1667
1668 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1669 {
1670         unsigned int size;
1671         u32 id;
1672         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1673         int i, ret;
1674
1675         dev_notice(smmu->dev, "probing hardware configuration...\n");
1676         dev_notice(smmu->dev, "SMMUv%d with:\n",
1677                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1678
1679         /* ID0 */
1680         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1681
1682         /* Restrict available stages based on module parameter */
1683         if (force_stage == 1)
1684                 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1685         else if (force_stage == 2)
1686                 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1687
1688         if (id & ARM_SMMU_ID0_S1TS) {
1689                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1690                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1691         }
1692
1693         if (id & ARM_SMMU_ID0_S2TS) {
1694                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1695                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1696         }
1697
1698         if (id & ARM_SMMU_ID0_NTS) {
1699                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1700                 dev_notice(smmu->dev, "\tnested translation\n");
1701         }
1702
1703         if (!(smmu->features &
1704                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1705                 dev_err(smmu->dev, "\tno translation support!\n");
1706                 return -ENODEV;
1707         }
1708
1709         if ((id & ARM_SMMU_ID0_S1TS) &&
1710             ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1711                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1712                 dev_notice(smmu->dev, "\taddress translation ops\n");
1713         }
1714
1715         /*
1716          * In order for DMA API calls to work properly, we must defer to what
1717          * the FW says about coherency, regardless of what the hardware claims.
1718          * Fortunately, this also opens up a workaround for systems where the
1719          * ID register value has ended up configured incorrectly.
1720          */
1721         cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1722         if (cttw_fw || cttw_reg)
1723                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1724                            cttw_fw ? "" : "non-");
1725         if (cttw_fw != cttw_reg)
1726                 dev_notice(smmu->dev,
1727                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1728
1729         /* Max. number of entries we have for stream matching/indexing */
1730         if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1731                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1732                 size = 1 << 16;
1733         } else {
1734                 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1735         }
1736         smmu->streamid_mask = size - 1;
1737         if (id & ARM_SMMU_ID0_SMS) {
1738                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1739                 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1740                 if (size == 0) {
1741                         dev_err(smmu->dev,
1742                                 "stream-matching supported, but no SMRs present!\n");
1743                         return -ENODEV;
1744                 }
1745
1746                 /* Zero-initialised to mark as invalid */
1747                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1748                                           GFP_KERNEL);
1749                 if (!smmu->smrs)
1750                         return -ENOMEM;
1751
1752                 dev_notice(smmu->dev,
1753                            "\tstream matching with %u register groups", size);
1754         }
1755         /* s2cr->type == 0 means translation, so initialise explicitly */
1756         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1757                                          GFP_KERNEL);
1758         if (!smmu->s2crs)
1759                 return -ENOMEM;
1760         for (i = 0; i < size; i++)
1761                 smmu->s2crs[i] = s2cr_init_val;
1762
1763         smmu->num_mapping_groups = size;
1764         mutex_init(&smmu->stream_map_mutex);
1765         spin_lock_init(&smmu->global_sync_lock);
1766
1767         if (smmu->version < ARM_SMMU_V2 ||
1768             !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1769                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770                 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1771                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1772         }
1773
1774         /* ID1 */
1775         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1776         smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1777
1778         /* Check for size mismatch of SMMU address space from mapped region */
1779         size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1780         if (smmu->numpage != 2 * size << smmu->pgshift)
1781                 dev_warn(smmu->dev,
1782                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1783                         2 * size << smmu->pgshift, smmu->numpage);
1784         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1785         smmu->numpage = size;
1786
1787         smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1788         smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1789         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1790                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1791                 return -ENODEV;
1792         }
1793         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1794                    smmu->num_context_banks, smmu->num_s2_context_banks);
1795         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1796                                  sizeof(*smmu->cbs), GFP_KERNEL);
1797         if (!smmu->cbs)
1798                 return -ENOMEM;
1799
1800         /* ID2 */
1801         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1802         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1803         smmu->ipa_size = size;
1804
1805         /* The output mask is also applied for bypass */
1806         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1807         smmu->pa_size = size;
1808
1809         if (id & ARM_SMMU_ID2_VMID16)
1810                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1811
1812         /*
1813          * What the page table walker can address actually depends on which
1814          * descriptor format is in use, but since a) we don't know that yet,
1815          * and b) it can vary per context bank, this will have to do...
1816          */
1817         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1818                 dev_warn(smmu->dev,
1819                          "failed to set DMA mask for table walker\n");
1820
1821         if (smmu->version < ARM_SMMU_V2) {
1822                 smmu->va_size = smmu->ipa_size;
1823                 if (smmu->version == ARM_SMMU_V1_64K)
1824                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1825         } else {
1826                 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1827                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1828                 if (id & ARM_SMMU_ID2_PTFS_4K)
1829                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1830                 if (id & ARM_SMMU_ID2_PTFS_16K)
1831                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1832                 if (id & ARM_SMMU_ID2_PTFS_64K)
1833                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1834         }
1835
1836         if (smmu->impl && smmu->impl->cfg_probe) {
1837                 ret = smmu->impl->cfg_probe(smmu);
1838                 if (ret)
1839                         return ret;
1840         }
1841
1842         /* Now we've corralled the various formats, what'll it do? */
1843         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845         if (smmu->features &
1846             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1852
1853         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1855         else
1856                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858                    smmu->pgsize_bitmap);
1859
1860
1861         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863                            smmu->va_size, smmu->ipa_size);
1864
1865         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867                            smmu->ipa_size, smmu->pa_size);
1868
1869         return 0;
1870 }
1871
1872 struct arm_smmu_match_data {
1873         enum arm_smmu_arch_version version;
1874         enum arm_smmu_implementation model;
1875 };
1876
1877 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1878 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1879
1880 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1886
1887 static const struct of_device_id arm_smmu_of_match[] = {
1888         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1889         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1890         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1891         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1892         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1893         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1894         { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1895         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1896         { },
1897 };
1898 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1899
1900 #ifdef CONFIG_ACPI
1901 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1902 {
1903         int ret = 0;
1904
1905         switch (model) {
1906         case ACPI_IORT_SMMU_V1:
1907         case ACPI_IORT_SMMU_CORELINK_MMU400:
1908                 smmu->version = ARM_SMMU_V1;
1909                 smmu->model = GENERIC_SMMU;
1910                 break;
1911         case ACPI_IORT_SMMU_CORELINK_MMU401:
1912                 smmu->version = ARM_SMMU_V1_64K;
1913                 smmu->model = GENERIC_SMMU;
1914                 break;
1915         case ACPI_IORT_SMMU_V2:
1916                 smmu->version = ARM_SMMU_V2;
1917                 smmu->model = GENERIC_SMMU;
1918                 break;
1919         case ACPI_IORT_SMMU_CORELINK_MMU500:
1920                 smmu->version = ARM_SMMU_V2;
1921                 smmu->model = ARM_MMU500;
1922                 break;
1923         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1924                 smmu->version = ARM_SMMU_V2;
1925                 smmu->model = CAVIUM_SMMUV2;
1926                 break;
1927         default:
1928                 ret = -ENODEV;
1929         }
1930
1931         return ret;
1932 }
1933
1934 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1935                                       u32 *global_irqs, u32 *pmu_irqs)
1936 {
1937         struct device *dev = smmu->dev;
1938         struct acpi_iort_node *node =
1939                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1940         struct acpi_iort_smmu *iort_smmu;
1941         int ret;
1942
1943         /* Retrieve SMMU1/2 specific data */
1944         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1945
1946         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1947         if (ret < 0)
1948                 return ret;
1949
1950         /* Ignore the configuration access interrupt */
1951         *global_irqs = 1;
1952         *pmu_irqs = 0;
1953
1954         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1955                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1956
1957         return 0;
1958 }
1959 #else
1960 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1961                                              u32 *global_irqs, u32 *pmu_irqs)
1962 {
1963         return -ENODEV;
1964 }
1965 #endif
1966
1967 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1968                                     u32 *global_irqs, u32 *pmu_irqs)
1969 {
1970         const struct arm_smmu_match_data *data;
1971         struct device *dev = smmu->dev;
1972         bool legacy_binding;
1973
1974         if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1975                 return dev_err_probe(dev, -ENODEV,
1976                                      "missing #global-interrupts property\n");
1977         *pmu_irqs = 0;
1978
1979         data = of_device_get_match_data(dev);
1980         smmu->version = data->version;
1981         smmu->model = data->model;
1982
1983         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1984         if (legacy_binding && !using_generic_binding) {
1985                 if (!using_legacy_binding) {
1986                         pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1987                                   IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1988                 }
1989                 using_legacy_binding = true;
1990         } else if (!legacy_binding && !using_legacy_binding) {
1991                 using_generic_binding = true;
1992         } else {
1993                 dev_err(dev, "not probing due to mismatched DT properties\n");
1994                 return -ENODEV;
1995         }
1996
1997         if (of_dma_is_coherent(dev->of_node))
1998                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999
2000         return 0;
2001 }
2002
2003 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2004 {
2005         struct list_head rmr_list;
2006         struct iommu_resv_region *e;
2007         int idx, cnt = 0;
2008         u32 reg;
2009
2010         INIT_LIST_HEAD(&rmr_list);
2011         iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2012
2013         /*
2014          * Rather than trying to look at existing mappings that
2015          * are setup by the firmware and then invalidate the ones
2016          * that do no have matching RMR entries, just disable the
2017          * SMMU until it gets enabled again in the reset routine.
2018          */
2019         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2020         reg |= ARM_SMMU_sCR0_CLIENTPD;
2021         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2022
2023         list_for_each_entry(e, &rmr_list, list) {
2024                 struct iommu_iort_rmr_data *rmr;
2025                 int i;
2026
2027                 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2028                 for (i = 0; i < rmr->num_sids; i++) {
2029                         idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2030                         if (idx < 0)
2031                                 continue;
2032
2033                         if (smmu->s2crs[idx].count == 0) {
2034                                 smmu->smrs[idx].id = rmr->sids[i];
2035                                 smmu->smrs[idx].mask = 0;
2036                                 smmu->smrs[idx].valid = true;
2037                         }
2038                         smmu->s2crs[idx].count++;
2039                         smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2040                         smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2041
2042                         cnt++;
2043                 }
2044         }
2045
2046         dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2047                    cnt == 1 ? "" : "s");
2048         iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2049 }
2050
2051 static int arm_smmu_device_probe(struct platform_device *pdev)
2052 {
2053         struct resource *res;
2054         struct arm_smmu_device *smmu;
2055         struct device *dev = &pdev->dev;
2056         int num_irqs, i, err;
2057         u32 global_irqs, pmu_irqs;
2058         irqreturn_t (*global_fault)(int irq, void *dev);
2059
2060         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2061         if (!smmu) {
2062                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2063                 return -ENOMEM;
2064         }
2065         smmu->dev = dev;
2066
2067         if (dev->of_node)
2068                 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2069         else
2070                 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2071         if (err)
2072                 return err;
2073
2074         smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2075         if (IS_ERR(smmu->base))
2076                 return PTR_ERR(smmu->base);
2077         smmu->ioaddr = res->start;
2078
2079         /*
2080          * The resource size should effectively match the value of SMMU_TOP;
2081          * stash that temporarily until we know PAGESIZE to validate it with.
2082          */
2083         smmu->numpage = resource_size(res);
2084
2085         smmu = arm_smmu_impl_init(smmu);
2086         if (IS_ERR(smmu))
2087                 return PTR_ERR(smmu);
2088
2089         num_irqs = platform_irq_count(pdev);
2090
2091         smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2092         if (smmu->num_context_irqs <= 0)
2093                 return dev_err_probe(dev, -ENODEV,
2094                                 "found %d interrupts but expected at least %d\n",
2095                                 num_irqs, global_irqs + pmu_irqs + 1);
2096
2097         smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2098                                   sizeof(*smmu->irqs), GFP_KERNEL);
2099         if (!smmu->irqs)
2100                 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2101                                      smmu->num_context_irqs);
2102
2103         for (i = 0; i < smmu->num_context_irqs; i++) {
2104                 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2105
2106                 if (irq < 0)
2107                         return irq;
2108                 smmu->irqs[i] = irq;
2109         }
2110
2111         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2112         if (err < 0) {
2113                 dev_err(dev, "failed to get clocks %d\n", err);
2114                 return err;
2115         }
2116         smmu->num_clks = err;
2117
2118         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2119         if (err)
2120                 return err;
2121
2122         err = arm_smmu_device_cfg_probe(smmu);
2123         if (err)
2124                 return err;
2125
2126         if (smmu->version == ARM_SMMU_V2) {
2127                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2128                         dev_err(dev,
2129                               "found only %d context irq(s) but %d required\n",
2130                               smmu->num_context_irqs, smmu->num_context_banks);
2131                         return -ENODEV;
2132                 }
2133
2134                 /* Ignore superfluous interrupts */
2135                 smmu->num_context_irqs = smmu->num_context_banks;
2136         }
2137
2138         if (smmu->impl && smmu->impl->global_fault)
2139                 global_fault = smmu->impl->global_fault;
2140         else
2141                 global_fault = arm_smmu_global_fault;
2142
2143         for (i = 0; i < global_irqs; i++) {
2144                 int irq = platform_get_irq(pdev, i);
2145
2146                 if (irq < 0)
2147                         return irq;
2148
2149                 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2150                                        "arm-smmu global fault", smmu);
2151                 if (err)
2152                         return dev_err_probe(dev, err,
2153                                         "failed to request global IRQ %d (%u)\n",
2154                                         i, irq);
2155         }
2156
2157         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2158                                      "smmu.%pa", &smmu->ioaddr);
2159         if (err) {
2160                 dev_err(dev, "Failed to register iommu in sysfs\n");
2161                 return err;
2162         }
2163
2164         err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2165         if (err) {
2166                 dev_err(dev, "Failed to register iommu\n");
2167                 iommu_device_sysfs_remove(&smmu->iommu);
2168                 return err;
2169         }
2170
2171         platform_set_drvdata(pdev, smmu);
2172
2173         /* Check for RMRs and install bypass SMRs if any */
2174         arm_smmu_rmr_install_bypass_smr(smmu);
2175
2176         arm_smmu_device_reset(smmu);
2177         arm_smmu_test_smr_masks(smmu);
2178
2179         /*
2180          * We want to avoid touching dev->power.lock in fastpaths unless
2181          * it's really going to do something useful - pm_runtime_enabled()
2182          * can serve as an ideal proxy for that decision. So, conditionally
2183          * enable pm_runtime.
2184          */
2185         if (dev->pm_domain) {
2186                 pm_runtime_set_active(dev);
2187                 pm_runtime_enable(dev);
2188         }
2189
2190         return 0;
2191 }
2192
2193 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2194 {
2195         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2196
2197         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2198                 dev_notice(&pdev->dev, "disabling translation\n");
2199
2200         arm_smmu_rpm_get(smmu);
2201         /* Turn the thing off */
2202         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2203         arm_smmu_rpm_put(smmu);
2204
2205         if (pm_runtime_enabled(smmu->dev))
2206                 pm_runtime_force_suspend(smmu->dev);
2207         else
2208                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2209
2210         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2211 }
2212
2213 static void arm_smmu_device_remove(struct platform_device *pdev)
2214 {
2215         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2216
2217         iommu_device_unregister(&smmu->iommu);
2218         iommu_device_sysfs_remove(&smmu->iommu);
2219
2220         arm_smmu_device_shutdown(pdev);
2221 }
2222
2223 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2224 {
2225         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2226         int ret;
2227
2228         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2229         if (ret)
2230                 return ret;
2231
2232         arm_smmu_device_reset(smmu);
2233
2234         return 0;
2235 }
2236
2237 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2238 {
2239         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2240
2241         clk_bulk_disable(smmu->num_clks, smmu->clks);
2242
2243         return 0;
2244 }
2245
2246 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2247 {
2248         int ret;
2249         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2250
2251         ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2252         if (ret)
2253                 return ret;
2254
2255         if (pm_runtime_suspended(dev))
2256                 return 0;
2257
2258         ret = arm_smmu_runtime_resume(dev);
2259         if (ret)
2260                 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2261
2262         return ret;
2263 }
2264
2265 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2266 {
2267         int ret = 0;
2268         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2269
2270         if (pm_runtime_suspended(dev))
2271                 goto clk_unprepare;
2272
2273         ret = arm_smmu_runtime_suspend(dev);
2274         if (ret)
2275                 return ret;
2276
2277 clk_unprepare:
2278         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2279         return ret;
2280 }
2281
2282 static const struct dev_pm_ops arm_smmu_pm_ops = {
2283         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2284         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2285                            arm_smmu_runtime_resume, NULL)
2286 };
2287
2288 static struct platform_driver arm_smmu_driver = {
2289         .driver = {
2290                 .name                   = "arm-smmu",
2291                 .of_match_table         = arm_smmu_of_match,
2292                 .pm                     = &arm_smmu_pm_ops,
2293                 .suppress_bind_attrs    = true,
2294         },
2295         .probe  = arm_smmu_device_probe,
2296         .remove_new = arm_smmu_device_remove,
2297         .shutdown = arm_smmu_device_shutdown,
2298 };
2299 module_platform_driver(arm_smmu_driver);
2300
2301 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2302 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2303 MODULE_ALIAS("platform:arm-smmu");
2304 MODULE_LICENSE("GPL v2");