drivers/iommu/intel/iommu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright © 2006-2014 Intel Corporation.
   4  *
   5  * Authors: David Woodhouse <dwmw2@infradead.org>,
   6  *          Ashok Raj <ashok.raj@intel.com>,
   7  *          Shaohua Li <shaohua.li@intel.com>,
   8  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
   9  *          Fenghua Yu <fenghua.yu@intel.com>
  10  *          Joerg Roedel <jroedel@suse.de>
  11  */
  12
  13 #define pr_fmt(fmt)     "DMAR: " fmt
  14 #define dev_fmt(fmt)    pr_fmt(fmt)
  15
  16 #include <linux/crash_dump.h>
  17 #include <linux/dma-direct.h>
  18 #include <linux/dmi.h>
  19 #include <linux/memory.h>
  20 #include <linux/pci.h>
  21 #include <linux/pci-ats.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/syscore_ops.h>
  24 #include <linux/tboot.h>
  25 #include <uapi/linux/iommufd.h>
  26
  27 #include "iommu.h"
  28 #include "../dma-iommu.h"
  29 #include "../irq_remapping.h"
  30 #include "pasid.h"
  31 #include "cap_audit.h"
  32 #include "perfmon.h"
  33
  34 #define ROOT_SIZE               VTD_PAGE_SIZE
  35 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  36
  37 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  38 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  39 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  40 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  41
  42 #define IOAPIC_RANGE_START      (0xfee00000)
  43 #define IOAPIC_RANGE_END        (0xfeefffff)
  44 #define IOVA_START_ADDR         (0x1000)
  45
  46 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  47
  48 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
  49 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
  50
  51 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  52    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  53 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  54                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  55 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  56
  57 /* IO virtual address start page frame number */
  58 #define IOVA_START_PFN          (1)
  59
  60 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  61
  62 static void __init check_tylersburg_isoch(void);
  63 static int rwbf_quirk;
  64
  65 /*
  66  * set to 1 to panic kernel if can't successfully enable VT-d
  67  * (used when kernel is launched w/ TXT)
  68  */
  69 static int force_on = 0;
  70 static int intel_iommu_tboot_noforce;
  71 static int no_platform_optin;
  72
  73 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
  74
  75 /*
  76  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
  77  * if marked present.
  78  */
  79 static phys_addr_t root_entry_lctp(struct root_entry *re)
  80 {
  81         if (!(re->lo & 1))
  82                 return 0;
  83
  84         return re->lo & VTD_PAGE_MASK;
  85 }
  86
  87 /*
  88  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
  89  * if marked present.
  90  */
  91 static phys_addr_t root_entry_uctp(struct root_entry *re)
  92 {
  93         if (!(re->hi & 1))
  94                 return 0;
  95
  96         return re->hi & VTD_PAGE_MASK;
  97 }
  98
  99 static int device_rid_cmp_key(const void *key, const struct rb_node *node)
 100 {
 101         struct device_domain_info *info =
 102                 rb_entry(node, struct device_domain_info, node);
 103         const u16 *rid_lhs = key;
 104
 105         if (*rid_lhs < PCI_DEVID(info->bus, info->devfn))
 106                 return -1;
 107
 108         if (*rid_lhs > PCI_DEVID(info->bus, info->devfn))
 109                 return 1;
 110
 111         return 0;
 112 }
 113
 114 static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs)
 115 {
 116         struct device_domain_info *info =
 117                 rb_entry(lhs, struct device_domain_info, node);
 118         u16 key = PCI_DEVID(info->bus, info->devfn);
 119
 120         return device_rid_cmp_key(&key, rhs);
 121 }
 122
 123 /*
 124  * Looks up an IOMMU-probed device using its source ID.
 125  *
 126  * Returns the pointer to the device if there is a match. Otherwise,
 127  * returns NULL.
 128  *
 129  * Note that this helper doesn't guarantee that the device won't be
 130  * released by the iommu subsystem after being returned. The caller
 131  * should use its own synchronization mechanism to avoid the device
 132  * being released during its use if its possibly the case.
 133  */
 134 struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)
 135 {
 136         struct device_domain_info *info = NULL;
 137         struct rb_node *node;
 138         unsigned long flags;
 139
 140         spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
 141         node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);
 142         if (node)
 143                 info = rb_entry(node, struct device_domain_info, node);
 144         spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
 145
 146         return info ? info->dev : NULL;
 147 }
 148
 149 static int device_rbtree_insert(struct intel_iommu *iommu,
 150                                 struct device_domain_info *info)
 151 {
 152         struct rb_node *curr;
 153         unsigned long flags;
 154
 155         spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
 156         curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);
 157         spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
 158         if (WARN_ON(curr))
 159                 return -EEXIST;
 160
 161         return 0;
 162 }
 163
 164 static void device_rbtree_remove(struct device_domain_info *info)
 165 {
 166         struct intel_iommu *iommu = info->iommu;
 167         unsigned long flags;
 168
 169         spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
 170         rb_erase(&info->node, &iommu->device_rbtree);
 171         spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
 172 }
 173
 174 /*
 175  * This domain is a statically identity mapping domain.
 176  *      1. This domain creats a static 1:1 mapping to all usable memory.
 177  *      2. It maps to each iommu if successful.
 178  *      3. Each iommu mapps to this domain if successful.
 179  */
 180 static struct dmar_domain *si_domain;
 181 static int hw_pass_through = 1;
 182
 183 struct dmar_rmrr_unit {
 184         struct list_head list;          /* list of rmrr units   */
 185         struct acpi_dmar_header *hdr;   /* ACPI header          */
 186         u64     base_address;           /* reserved base address*/
 187         u64     end_address;            /* reserved end address */
 188         struct dmar_dev_scope *devices; /* target devices */
 189         int     devices_cnt;            /* target device count */
 190 };
 191
 192 struct dmar_atsr_unit {
 193         struct list_head list;          /* list of ATSR units */
 194         struct acpi_dmar_header *hdr;   /* ACPI header */
 195         struct dmar_dev_scope *devices; /* target devices */
 196         int devices_cnt;                /* target device count */
 197         u8 include_all:1;               /* include all ports */
 198 };
 199
 200 struct dmar_satc_unit {
 201         struct list_head list;          /* list of SATC units */
 202         struct acpi_dmar_header *hdr;   /* ACPI header */
 203         struct dmar_dev_scope *devices; /* target devices */
 204         struct intel_iommu *iommu;      /* the corresponding iommu */
 205         int devices_cnt;                /* target device count */
 206         u8 atc_required:1;              /* ATS is required */
 207 };
 208
 209 static LIST_HEAD(dmar_atsr_units);
 210 static LIST_HEAD(dmar_rmrr_units);
 211 static LIST_HEAD(dmar_satc_units);
 212
 213 #define for_each_rmrr_units(rmrr) \
 214         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 215
 216 static void intel_iommu_domain_free(struct iommu_domain *domain);
 217
 218 int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
 219 int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
 220
 221 int intel_iommu_enabled = 0;
 222 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 223
 224 static int dmar_map_gfx = 1;
 225 static int intel_iommu_superpage = 1;
 226 static int iommu_identity_mapping;
 227 static int iommu_skip_te_disable;
 228
 229 #define IDENTMAP_GFX            2
 230 #define IDENTMAP_AZALIA         4
 231
 232 const struct iommu_ops intel_iommu_ops;
 233 static const struct iommu_dirty_ops intel_dirty_ops;
 234
 235 static bool translation_pre_enabled(struct intel_iommu *iommu)
 236 {
 237         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 238 }
 239
 240 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 241 {
 242         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 243 }
 244
 245 static void init_translation_status(struct intel_iommu *iommu)
 246 {
 247         u32 gsts;
 248
 249         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 250         if (gsts & DMA_GSTS_TES)
 251                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 252 }
 253
 254 static int __init intel_iommu_setup(char *str)
 255 {
 256         if (!str)
 257                 return -EINVAL;
 258
 259         while (*str) {
 260                 if (!strncmp(str, "on", 2)) {
 261                         dmar_disabled = 0;
 262                         pr_info("IOMMU enabled\n");
 263                 } else if (!strncmp(str, "off", 3)) {
 264                         dmar_disabled = 1;
 265                         no_platform_optin = 1;
 266                         pr_info("IOMMU disabled\n");
 267                 } else if (!strncmp(str, "igfx_off", 8)) {
 268                         dmar_map_gfx = 0;
 269                         pr_info("Disable GFX device mapping\n");
 270                 } else if (!strncmp(str, "forcedac", 8)) {
 271                         pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
 272                         iommu_dma_forcedac = true;
 273                 } else if (!strncmp(str, "strict", 6)) {
 274                         pr_warn("intel_iommu=strict deprecated; use iommu.strict=1 instead\n");
 275                         iommu_set_dma_strict();
 276                 } else if (!strncmp(str, "sp_off", 6)) {
 277                         pr_info("Disable supported super page\n");
 278                         intel_iommu_superpage = 0;
 279                 } else if (!strncmp(str, "sm_on", 5)) {
 280                         pr_info("Enable scalable mode if hardware supports\n");
 281                         intel_iommu_sm = 1;
 282                 } else if (!strncmp(str, "sm_off", 6)) {
 283                         pr_info("Scalable mode is disallowed\n");
 284                         intel_iommu_sm = 0;
 285                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 286                         pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 287                         intel_iommu_tboot_noforce = 1;
 288                 } else {
 289                         pr_notice("Unknown option - '%s'\n", str);
 290                 }
 291
 292                 str += strcspn(str, ",");
 293                 while (*str == ',')
 294                         str++;
 295         }
 296
 297         return 1;
 298 }
 299 __setup("intel_iommu=", intel_iommu_setup);
 300
 301 void *alloc_pgtable_page(int node, gfp_t gfp)
 302 {
 303         struct page *page;
 304         void *vaddr = NULL;
 305
 306         page = alloc_pages_node(node, gfp | __GFP_ZERO, 0);
 307         if (page)
 308                 vaddr = page_address(page);
 309         return vaddr;
 310 }
 311
 312 void free_pgtable_page(void *vaddr)
 313 {
 314         free_page((unsigned long)vaddr);
 315 }
 316
 317 static int domain_type_is_si(struct dmar_domain *domain)
 318 {
 319         return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
 320 }
 321
 322 static int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn)
 323 {
 324         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 325
 326         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 327 }
 328
 329 /*
 330  * Calculate the Supported Adjusted Guest Address Widths of an IOMMU.
 331  * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
 332  * the returned SAGAW.
 333  */
 334 static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)
 335 {
 336         unsigned long fl_sagaw, sl_sagaw;
 337
 338         fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
 339         sl_sagaw = cap_sagaw(iommu->cap);
 340
 341         /* Second level only. */
 342         if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
 343                 return sl_sagaw;
 344
 345         /* First level only. */
 346         if (!ecap_slts(iommu->ecap))
 347                 return fl_sagaw;
 348
 349         return fl_sagaw & sl_sagaw;
 350 }
 351
 352 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 353 {
 354         unsigned long sagaw;
 355         int agaw;
 356
 357         sagaw = __iommu_calculate_sagaw(iommu);
 358         for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
 359                 if (test_bit(agaw, &sagaw))
 360                         break;
 361         }
 362
 363         return agaw;
 364 }
 365
 366 /*
 367  * Calculate max SAGAW for each iommu.
 368  */
 369 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 370 {
 371         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 372 }
 373
 374 /*
 375  * calculate agaw for each iommu.
 376  * "SAGAW" may be different across iommus, use a default agaw, and
 377  * get a supported less agaw for iommus that don't support the default agaw.
 378  */
 379 int iommu_calculate_agaw(struct intel_iommu *iommu)
 380 {
 381         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 382 }
 383
 384 static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
 385 {
 386         return sm_supported(iommu) ?
 387                         ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
 388 }
 389
 390 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 391 {
 392         struct iommu_domain_info *info;
 393         struct dmar_drhd_unit *drhd;
 394         struct intel_iommu *iommu;
 395         bool found = false;
 396         unsigned long i;
 397
 398         domain->iommu_coherency = true;
 399         xa_for_each(&domain->iommu_array, i, info) {
 400                 found = true;
 401                 if (!iommu_paging_structure_coherency(info->iommu)) {
 402                         domain->iommu_coherency = false;
 403                         break;
 404                 }
 405         }
 406         if (found)
 407                 return;
 408
 409         /* No hardware attached; use lowest common denominator */
 410         rcu_read_lock();
 411         for_each_active_iommu(iommu, drhd) {
 412                 if (!iommu_paging_structure_coherency(iommu)) {
 413                         domain->iommu_coherency = false;
 414                         break;
 415                 }
 416         }
 417         rcu_read_unlock();
 418 }
 419
 420 static int domain_update_iommu_superpage(struct dmar_domain *domain,
 421                                          struct intel_iommu *skip)
 422 {
 423         struct dmar_drhd_unit *drhd;
 424         struct intel_iommu *iommu;
 425         int mask = 0x3;
 426
 427         if (!intel_iommu_superpage)
 428                 return 0;
 429
 430         /* set iommu_superpage to the smallest common denominator */
 431         rcu_read_lock();
 432         for_each_active_iommu(iommu, drhd) {
 433                 if (iommu != skip) {
 434                         if (domain && domain->use_first_level) {
 435                                 if (!cap_fl1gp_support(iommu->cap))
 436                                         mask = 0x1;
 437                         } else {
 438                                 mask &= cap_super_page_val(iommu->cap);
 439                         }
 440
 441                         if (!mask)
 442                                 break;
 443                 }
 444         }
 445         rcu_read_unlock();
 446
 447         return fls(mask);
 448 }
 449
 450 static int domain_update_device_node(struct dmar_domain *domain)
 451 {
 452         struct device_domain_info *info;
 453         int nid = NUMA_NO_NODE;
 454         unsigned long flags;
 455
 456         spin_lock_irqsave(&domain->lock, flags);
 457         list_for_each_entry(info, &domain->devices, link) {
 458                 /*
 459                  * There could possibly be multiple device numa nodes as devices
 460                  * within the same domain may sit behind different IOMMUs. There
 461                  * isn't perfect answer in such situation, so we select first
 462                  * come first served policy.
 463                  */
 464                 nid = dev_to_node(info->dev);
 465                 if (nid != NUMA_NO_NODE)
 466                         break;
 467         }
 468         spin_unlock_irqrestore(&domain->lock, flags);
 469
 470         return nid;
 471 }
 472
 473 /* Return the super pagesize bitmap if supported. */
 474 static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
 475 {
 476         unsigned long bitmap = 0;
 477
 478         /*
 479          * 1-level super page supports page size of 2MiB, 2-level super page
 480          * supports page size of both 2MiB and 1GiB.
 481          */
 482         if (domain->iommu_superpage == 1)
 483                 bitmap |= SZ_2M;
 484         else if (domain->iommu_superpage == 2)
 485                 bitmap |= SZ_2M | SZ_1G;
 486
 487         return bitmap;
 488 }
 489
 490 /* Some capabilities may be different across iommus */
 491 void domain_update_iommu_cap(struct dmar_domain *domain)
 492 {
 493         domain_update_iommu_coherency(domain);
 494         domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
 495
 496         /*
 497          * If RHSA is missing, we should default to the device numa domain
 498          * as fall back.
 499          */
 500         if (domain->nid == NUMA_NO_NODE)
 501                 domain->nid = domain_update_device_node(domain);
 502
 503         /*
 504          * First-level translation restricts the input-address to a
 505          * canonical address (i.e., address bits 63:N have the same
 506          * value as address bit [N-1], where N is 48-bits with 4-level
 507          * paging and 57-bits with 5-level paging). Hence, skip bit
 508          * [N-1].
 509          */
 510         if (domain->use_first_level)
 511                 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
 512         else
 513                 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
 514
 515         domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
 516         domain_update_iotlb(domain);
 517 }
 518
 519 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
 520                                          u8 devfn, int alloc)
 521 {
 522         struct root_entry *root = &iommu->root_entry[bus];
 523         struct context_entry *context;
 524         u64 *entry;
 525
 526         /*
 527          * Except that the caller requested to allocate a new entry,
 528          * returning a copied context entry makes no sense.
 529          */
 530         if (!alloc && context_copied(iommu, bus, devfn))
 531                 return NULL;
 532
 533         entry = &root->lo;
 534         if (sm_supported(iommu)) {
 535                 if (devfn >= 0x80) {
 536                         devfn -= 0x80;
 537                         entry = &root->hi;
 538                 }
 539                 devfn *= 2;
 540         }
 541         if (*entry & 1)
 542                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 543         else {
 544                 unsigned long phy_addr;
 545                 if (!alloc)
 546                         return NULL;
 547
 548                 context = alloc_pgtable_page(iommu->node, GFP_ATOMIC);
 549                 if (!context)
 550                         return NULL;
 551
 552                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 553                 phy_addr = virt_to_phys((void *)context);
 554                 *entry = phy_addr | 1;
 555                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 556         }
 557         return &context[devfn];
 558 }
 559
 560 /**
 561  * is_downstream_to_pci_bridge - test if a device belongs to the PCI
 562  *                               sub-hierarchy of a candidate PCI-PCI bridge
 563  * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
 564  * @bridge: the candidate PCI-PCI bridge
 565  *
 566  * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
 567  */
 568 static bool
 569 is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
 570 {
 571         struct pci_dev *pdev, *pbridge;
 572
 573         if (!dev_is_pci(dev) || !dev_is_pci(bridge))
 574                 return false;
 575
 576         pdev = to_pci_dev(dev);
 577         pbridge = to_pci_dev(bridge);
 578
 579         if (pbridge->subordinate &&
 580             pbridge->subordinate->number <= pdev->bus->number &&
 581             pbridge->subordinate->busn_res.end >= pdev->bus->number)
 582                 return true;
 583
 584         return false;
 585 }
 586
 587 static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
 588 {
 589         struct dmar_drhd_unit *drhd;
 590         u32 vtbar;
 591         int rc;
 592
 593         /* We know that this device on this chipset has its own IOMMU.
 594          * If we find it under a different IOMMU, then the BIOS is lying
 595          * to us. Hope that the IOMMU for this device is actually
 596          * disabled, and it needs no translation...
 597          */
 598         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
 599         if (rc) {
 600                 /* "can't" happen */
 601                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
 602                 return false;
 603         }
 604         vtbar &= 0xffff0000;
 605
 606         /* we know that the this iommu should be at offset 0xa000 from vtbar */
 607         drhd = dmar_find_matched_drhd_unit(pdev);
 608         if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
 609                 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
 610                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 611                 return true;
 612         }
 613
 614         return false;
 615 }
 616
 617 static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
 618 {
 619         if (!iommu || iommu->drhd->ignored)
 620                 return true;
 621
 622         if (dev_is_pci(dev)) {
 623                 struct pci_dev *pdev = to_pci_dev(dev);
 624
 625                 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
 626                     pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
 627                     quirk_ioat_snb_local_iommu(pdev))
 628                         return true;
 629         }
 630
 631         return false;
 632 }
 633
 634 static struct intel_iommu *device_lookup_iommu(struct device *dev, u8 *bus, u8 *devfn)
 635 {
 636         struct dmar_drhd_unit *drhd = NULL;
 637         struct pci_dev *pdev = NULL;
 638         struct intel_iommu *iommu;
 639         struct device *tmp;
 640         u16 segment = 0;
 641         int i;
 642
 643         if (!dev)
 644                 return NULL;
 645
 646         if (dev_is_pci(dev)) {
 647                 struct pci_dev *pf_pdev;
 648
 649                 pdev = pci_real_dma_dev(to_pci_dev(dev));
 650
 651                 /* VFs aren't listed in scope tables; we need to look up
 652                  * the PF instead to find the IOMMU. */
 653                 pf_pdev = pci_physfn(pdev);
 654                 dev = &pf_pdev->dev;
 655                 segment = pci_domain_nr(pdev->bus);
 656         } else if (has_acpi_companion(dev))
 657                 dev = &ACPI_COMPANION(dev)->dev;
 658
 659         rcu_read_lock();
 660         for_each_iommu(iommu, drhd) {
 661                 if (pdev && segment != drhd->segment)
 662                         continue;
 663
 664                 for_each_active_dev_scope(drhd->devices,
 665                                           drhd->devices_cnt, i, tmp) {
 666                         if (tmp == dev) {
 667                                 /* For a VF use its original BDF# not that of the PF
 668                                  * which we used for the IOMMU lookup. Strictly speaking
 669                                  * we could do this for all PCI devices; we only need to
 670                                  * get the BDF# from the scope table for ACPI matches. */
 671                                 if (pdev && pdev->is_virtfn)
 672                                         goto got_pdev;
 673
 674                                 if (bus && devfn) {
 675                                         *bus = drhd->devices[i].bus;
 676                                         *devfn = drhd->devices[i].devfn;
 677                                 }
 678                                 goto out;
 679                         }
 680
 681                         if (is_downstream_to_pci_bridge(dev, tmp))
 682                                 goto got_pdev;
 683                 }
 684
 685                 if (pdev && drhd->include_all) {
 686 got_pdev:
 687                         if (bus && devfn) {
 688                                 *bus = pdev->bus->number;
 689                                 *devfn = pdev->devfn;
 690                         }
 691                         goto out;
 692                 }
 693         }
 694         iommu = NULL;
 695 out:
 696         if (iommu_is_dummy(iommu, dev))
 697                 iommu = NULL;
 698
 699         rcu_read_unlock();
 700
 701         return iommu;
 702 }
 703
 704 static void domain_flush_cache(struct dmar_domain *domain,
 705                                void *addr, int size)
 706 {
 707         if (!domain->iommu_coherency)
 708                 clflush_cache_range(addr, size);
 709 }
 710
 711 static void free_context_table(struct intel_iommu *iommu)
 712 {
 713         struct context_entry *context;
 714         int i;
 715
 716         if (!iommu->root_entry)
 717                 return;
 718
 719         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 720                 context = iommu_context_addr(iommu, i, 0, 0);
 721                 if (context)
 722                         free_pgtable_page(context);
 723
 724                 if (!sm_supported(iommu))
 725                         continue;
 726
 727                 context = iommu_context_addr(iommu, i, 0x80, 0);
 728                 if (context)
 729                         free_pgtable_page(context);
 730         }
 731
 732         free_pgtable_page(iommu->root_entry);
 733         iommu->root_entry = NULL;
 734 }
 735
 736 #ifdef CONFIG_DMAR_DEBUG
 737 static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,
 738                          u8 bus, u8 devfn, struct dma_pte *parent, int level)
 739 {
 740         struct dma_pte *pte;
 741         int offset;
 742
 743         while (1) {
 744                 offset = pfn_level_offset(pfn, level);
 745                 pte = &parent[offset];
 746                 if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) {
 747                         pr_info("PTE not present at level %d\n", level);
 748                         break;
 749                 }
 750
 751                 pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
 752
 753                 if (level == 1)
 754                         break;
 755
 756                 parent = phys_to_virt(dma_pte_addr(pte));
 757                 level--;
 758         }
 759 }
 760
 761 void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,
 762                           unsigned long long addr, u32 pasid)
 763 {
 764         struct pasid_dir_entry *dir, *pde;
 765         struct pasid_entry *entries, *pte;
 766         struct context_entry *ctx_entry;
 767         struct root_entry *rt_entry;
 768         int i, dir_index, index, level;
 769         u8 devfn = source_id & 0xff;
 770         u8 bus = source_id >> 8;
 771         struct dma_pte *pgtable;
 772
 773         pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
 774
 775         /* root entry dump */
 776         rt_entry = &iommu->root_entry[bus];
 777         if (!rt_entry) {
 778                 pr_info("root table entry is not present\n");
 779                 return;
 780         }
 781
 782         if (sm_supported(iommu))
 783                 pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
 784                         rt_entry->hi, rt_entry->lo);
 785         else
 786                 pr_info("root entry: 0x%016llx", rt_entry->lo);
 787
 788         /* context entry dump */
 789         ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
 790         if (!ctx_entry) {
 791                 pr_info("context table entry is not present\n");
 792                 return;
 793         }
 794
 795         pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",
 796                 ctx_entry->hi, ctx_entry->lo);
 797
 798         /* legacy mode does not require PASID entries */
 799         if (!sm_supported(iommu)) {
 800                 level = agaw_to_level(ctx_entry->hi & 7);
 801                 pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
 802                 goto pgtable_walk;
 803         }
 804
 805         /* get the pointer to pasid directory entry */
 806         dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
 807         if (!dir) {
 808                 pr_info("pasid directory entry is not present\n");
 809                 return;
 810         }
 811         /* For request-without-pasid, get the pasid from context entry */
 812         if (intel_iommu_sm && pasid == IOMMU_PASID_INVALID)
 813                 pasid = IOMMU_NO_PASID;
 814
 815         dir_index = pasid >> PASID_PDE_SHIFT;
 816         pde = &dir[dir_index];
 817         pr_info("pasid dir entry: 0x%016llx\n", pde->val);
 818
 819         /* get the pointer to the pasid table entry */
 820         entries = get_pasid_table_from_pde(pde);
 821         if (!entries) {
 822                 pr_info("pasid table entry is not present\n");
 823                 return;
 824         }
 825         index = pasid & PASID_PTE_MASK;
 826         pte = &entries[index];
 827         for (i = 0; i < ARRAY_SIZE(pte->val); i++)
 828                 pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
 829
 830         if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
 831                 level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
 832                 pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
 833         } else {
 834                 level = agaw_to_level((pte->val[0] >> 2) & 0x7);
 835                 pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
 836         }
 837
 838 pgtable_walk:
 839         pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);
 840 }
 841 #endif
 842
 843 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 844                                       unsigned long pfn, int *target_level,
 845                                       gfp_t gfp)
 846 {
 847         struct dma_pte *parent, *pte;
 848         int level = agaw_to_level(domain->agaw);
 849         int offset;
 850
 851         if (!domain_pfn_supported(domain, pfn))
 852                 /* Address beyond IOMMU's addressing capabilities. */
 853                 return NULL;
 854
 855         parent = domain->pgd;
 856
 857         while (1) {
 858                 void *tmp_page;
 859
 860                 offset = pfn_level_offset(pfn, level);
 861                 pte = &parent[offset];
 862                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 863                         break;
 864                 if (level == *target_level)
 865                         break;
 866
 867                 if (!dma_pte_present(pte)) {
 868                         uint64_t pteval;
 869
 870                         tmp_page = alloc_pgtable_page(domain->nid, gfp);
 871
 872                         if (!tmp_page)
 873                                 return NULL;
 874
 875                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 876                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 877                         if (domain->use_first_level)
 878                                 pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
 879
 880                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 881                                 /* Someone else set it while we were thinking; use theirs. */
 882                                 free_pgtable_page(tmp_page);
 883                         else
 884                                 domain_flush_cache(domain, pte, sizeof(*pte));
 885                 }
 886                 if (level == 1)
 887                         break;
 888
 889                 parent = phys_to_virt(dma_pte_addr(pte));
 890                 level--;
 891         }
 892
 893         if (!*target_level)
 894                 *target_level = level;
 895
 896         return pte;
 897 }
 898
 899 /* return address's pte at specific level */
 900 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 901                                          unsigned long pfn,
 902                                          int level, int *large_page)
 903 {
 904         struct dma_pte *parent, *pte;
 905         int total = agaw_to_level(domain->agaw);
 906         int offset;
 907
 908         parent = domain->pgd;
 909         while (level <= total) {
 910                 offset = pfn_level_offset(pfn, total);
 911                 pte = &parent[offset];
 912                 if (level == total)
 913                         return pte;
 914
 915                 if (!dma_pte_present(pte)) {
 916                         *large_page = total;
 917                         break;
 918                 }
 919
 920                 if (dma_pte_superpage(pte)) {
 921                         *large_page = total;
 922                         return pte;
 923                 }
 924
 925                 parent = phys_to_virt(dma_pte_addr(pte));
 926                 total--;
 927         }
 928         return NULL;
 929 }
 930
 931 /* clear last level pte, a tlb flush should be followed */
 932 static void dma_pte_clear_range(struct dmar_domain *domain,
 933                                 unsigned long start_pfn,
 934                                 unsigned long last_pfn)
 935 {
 936         unsigned int large_page;
 937         struct dma_pte *first_pte, *pte;
 938
 939         if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
 940             WARN_ON(start_pfn > last_pfn))
 941                 return;
 942
 943         /* we don't need lock here; nobody else touches the iova range */
 944         do {
 945                 large_page = 1;
 946                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 947                 if (!pte) {
 948                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 949                         continue;
 950                 }
 951                 do {
 952                         dma_clear_pte(pte);
 953                         start_pfn += lvl_to_nr_pages(large_page);
 954                         pte++;
 955                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 956
 957                 domain_flush_cache(domain, first_pte,
 958                                    (void *)pte - (void *)first_pte);
 959
 960         } while (start_pfn && start_pfn <= last_pfn);
 961 }
 962
 963 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 964                                int retain_level, struct dma_pte *pte,
 965                                unsigned long pfn, unsigned long start_pfn,
 966                                unsigned long last_pfn)
 967 {
 968         pfn = max(start_pfn, pfn);
 969         pte = &pte[pfn_level_offset(pfn, level)];
 970
 971         do {
 972                 unsigned long level_pfn;
 973                 struct dma_pte *level_pte;
 974
 975                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 976                         goto next;
 977
 978                 level_pfn = pfn & level_mask(level);
 979                 level_pte = phys_to_virt(dma_pte_addr(pte));
 980
 981                 if (level > 2) {
 982                         dma_pte_free_level(domain, level - 1, retain_level,
 983                                            level_pte, level_pfn, start_pfn,
 984                                            last_pfn);
 985                 }
 986
 987                 /*
 988                  * Free the page table if we're below the level we want to
 989                  * retain and the range covers the entire table.
 990                  */
 991                 if (level < retain_level && !(start_pfn > level_pfn ||
 992                       last_pfn < level_pfn + level_size(level) - 1)) {
 993                         dma_clear_pte(pte);
 994                         domain_flush_cache(domain, pte, sizeof(*pte));
 995                         free_pgtable_page(level_pte);
 996                 }
 997 next:
 998                 pfn += level_size(level);
 999         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1000 }
1001
1002 /*
1003  * clear last level (leaf) ptes and free page table pages below the
1004  * level we wish to keep intact.
1005  */
1006 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1007                                    unsigned long start_pfn,
1008                                    unsigned long last_pfn,
1009                                    int retain_level)
1010 {
1011         dma_pte_clear_range(domain, start_pfn, last_pfn);
1012
1013         /* We don't need lock here; nobody else touches the iova range */
1014         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1015                            domain->pgd, 0, start_pfn, last_pfn);
1016
1017         /* free pgd */
1018         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1019                 free_pgtable_page(domain->pgd);
1020                 domain->pgd = NULL;
1021         }
1022 }
1023
1024 /* When a page at a given level is being unlinked from its parent, we don't
1025    need to *modify* it at all. All we need to do is make a list of all the
1026    pages which can be freed just as soon as we've flushed the IOTLB and we
1027    know the hardware page-walk will no longer touch them.
1028    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1029    be freed. */
1030 static void dma_pte_list_pagetables(struct dmar_domain *domain,
1031                                     int level, struct dma_pte *pte,
1032                                     struct list_head *freelist)
1033 {
1034         struct page *pg;
1035
1036         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1037         list_add_tail(&pg->lru, freelist);
1038
1039         if (level == 1)
1040                 return;
1041
1042         pte = page_address(pg);
1043         do {
1044                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1045                         dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1046                 pte++;
1047         } while (!first_pte_in_page(pte));
1048 }
1049
1050 static void dma_pte_clear_level(struct dmar_domain *domain, int level,
1051                                 struct dma_pte *pte, unsigned long pfn,
1052                                 unsigned long start_pfn, unsigned long last_pfn,
1053                                 struct list_head *freelist)
1054 {
1055         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1056
1057         pfn = max(start_pfn, pfn);
1058         pte = &pte[pfn_level_offset(pfn, level)];
1059
1060         do {
1061                 unsigned long level_pfn = pfn & level_mask(level);
1062
1063                 if (!dma_pte_present(pte))
1064                         goto next;
1065
1066                 /* If range covers entire pagetable, free it */
1067                 if (start_pfn <= level_pfn &&
1068                     last_pfn >= level_pfn + level_size(level) - 1) {
1069                         /* These suborbinate page tables are going away entirely. Don't
1070                            bother to clear them; we're just going to *free* them. */
1071                         if (level > 1 && !dma_pte_superpage(pte))
1072                                 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1073
1074                         dma_clear_pte(pte);
1075                         if (!first_pte)
1076                                 first_pte = pte;
1077                         last_pte = pte;
1078                 } else if (level > 1) {
1079                         /* Recurse down into a level that isn't *entirely* obsolete */
1080                         dma_pte_clear_level(domain, level - 1,
1081                                             phys_to_virt(dma_pte_addr(pte)),
1082                                             level_pfn, start_pfn, last_pfn,
1083                                             freelist);
1084                 }
1085 next:
1086                 pfn = level_pfn + level_size(level);
1087         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1088
1089         if (first_pte)
1090                 domain_flush_cache(domain, first_pte,
1091                                    (void *)++last_pte - (void *)first_pte);
1092 }
1093
1094 /* We can't just free the pages because the IOMMU may still be walking
1095    the page tables, and may have cached the intermediate levels. The
1096    pages can only be freed after the IOTLB flush has been done. */
1097 static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
1098                          unsigned long last_pfn, struct list_head *freelist)
1099 {
1100         if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
1101             WARN_ON(start_pfn > last_pfn))
1102                 return;
1103
1104         /* we don't need lock here; nobody else touches the iova range */
1105         dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1106                             domain->pgd, 0, start_pfn, last_pfn, freelist);
1107
1108         /* free pgd */
1109         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1110                 struct page *pgd_page = virt_to_page(domain->pgd);
1111                 list_add_tail(&pgd_page->lru, freelist);
1112                 domain->pgd = NULL;
1113         }
1114 }
1115
1116 /* iommu handling */
1117 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1118 {
1119         struct root_entry *root;
1120
1121         root = alloc_pgtable_page(iommu->node, GFP_ATOMIC);
1122         if (!root) {
1123                 pr_err("Allocating root entry for %s failed\n",
1124                         iommu->name);
1125                 return -ENOMEM;
1126         }
1127
1128         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1129         iommu->root_entry = root;
1130
1131         return 0;
1132 }
1133
1134 static void iommu_set_root_entry(struct intel_iommu *iommu)
1135 {
1136         u64 addr;
1137         u32 sts;
1138         unsigned long flag;
1139
1140         addr = virt_to_phys(iommu->root_entry);
1141         if (sm_supported(iommu))
1142                 addr |= DMA_RTADDR_SMT;
1143
1144         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1145         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1146
1147         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1148
1149         /* Make sure hardware complete it */
1150         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1151                       readl, (sts & DMA_GSTS_RTPS), sts);
1152
1153         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1154
1155         /*
1156          * Hardware invalidates all DMA remapping hardware translation
1157          * caches as part of SRTP flow.
1158          */
1159         if (cap_esrtps(iommu->cap))
1160                 return;
1161
1162         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1163         if (sm_supported(iommu))
1164                 qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
1165         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1166 }
1167
1168 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1169 {
1170         u32 val;
1171         unsigned long flag;
1172
1173         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1174                 return;
1175
1176         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1177         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1178
1179         /* Make sure hardware complete it */
1180         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1181                       readl, (!(val & DMA_GSTS_WBFS)), val);
1182
1183         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1184 }
1185
1186 /* return value determine if we need a write buffer flush */
1187 static void __iommu_flush_context(struct intel_iommu *iommu,
1188                                   u16 did, u16 source_id, u8 function_mask,
1189                                   u64 type)
1190 {
1191         u64 val = 0;
1192         unsigned long flag;
1193
1194         switch (type) {
1195         case DMA_CCMD_GLOBAL_INVL:
1196                 val = DMA_CCMD_GLOBAL_INVL;
1197                 break;
1198         case DMA_CCMD_DOMAIN_INVL:
1199                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1200                 break;
1201         case DMA_CCMD_DEVICE_INVL:
1202                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1203                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1204                 break;
1205         default:
1206                 pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n",
1207                         iommu->name, type);
1208                 return;
1209         }
1210         val |= DMA_CCMD_ICC;
1211
1212         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1213         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1214
1215         /* Make sure hardware complete it */
1216         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1217                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1218
1219         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1220 }
1221
1222 /* return value determine if we need a write buffer flush */
1223 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1224                                 u64 addr, unsigned int size_order, u64 type)
1225 {
1226         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1227         u64 val = 0, val_iva = 0;
1228         unsigned long flag;
1229
1230         switch (type) {
1231         case DMA_TLB_GLOBAL_FLUSH:
1232                 /* global flush doesn't need set IVA_REG */
1233                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1234                 break;
1235         case DMA_TLB_DSI_FLUSH:
1236                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1237                 break;
1238         case DMA_TLB_PSI_FLUSH:
1239                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1240                 /* IH bit is passed in as part of address */
1241                 val_iva = size_order | addr;
1242                 break;
1243         default:
1244                 pr_warn("%s: Unexpected iotlb invalidation type 0x%llx\n",
1245                         iommu->name, type);
1246                 return;
1247         }
1248
1249         if (cap_write_drain(iommu->cap))
1250                 val |= DMA_TLB_WRITE_DRAIN;
1251
1252         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1253         /* Note: Only uses first TLB reg currently */
1254         if (val_iva)
1255                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1256         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1257
1258         /* Make sure hardware complete it */
1259         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1260                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1261
1262         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1263
1264         /* check IOTLB invalidation granularity */
1265         if (DMA_TLB_IAIG(val) == 0)
1266                 pr_err("Flush IOTLB failed\n");
1267         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1268                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1269                         (unsigned long long)DMA_TLB_IIRG(type),
1270                         (unsigned long long)DMA_TLB_IAIG(val));
1271 }
1272
1273 static struct device_domain_info *
1274 domain_lookup_dev_info(struct dmar_domain *domain,
1275                        struct intel_iommu *iommu, u8 bus, u8 devfn)
1276 {
1277         struct device_domain_info *info;
1278         unsigned long flags;
1279
1280         spin_lock_irqsave(&domain->lock, flags);
1281         list_for_each_entry(info, &domain->devices, link) {
1282                 if (info->iommu == iommu && info->bus == bus &&
1283                     info->devfn == devfn) {
1284                         spin_unlock_irqrestore(&domain->lock, flags);
1285                         return info;
1286                 }
1287         }
1288         spin_unlock_irqrestore(&domain->lock, flags);
1289
1290         return NULL;
1291 }
1292
1293 void domain_update_iotlb(struct dmar_domain *domain)
1294 {
1295         struct dev_pasid_info *dev_pasid;
1296         struct device_domain_info *info;
1297         bool has_iotlb_device = false;
1298         unsigned long flags;
1299
1300         spin_lock_irqsave(&domain->lock, flags);
1301         list_for_each_entry(info, &domain->devices, link) {
1302                 if (info->ats_enabled) {
1303                         has_iotlb_device = true;
1304                         break;
1305                 }
1306         }
1307
1308         list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) {
1309                 info = dev_iommu_priv_get(dev_pasid->dev);
1310                 if (info->ats_enabled) {
1311                         has_iotlb_device = true;
1312                         break;
1313                 }
1314         }
1315         domain->has_iotlb_device = has_iotlb_device;
1316         spin_unlock_irqrestore(&domain->lock, flags);
1317 }
1318
1319 /*
1320  * The extra devTLB flush quirk impacts those QAT devices with PCI device
1321  * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1322  * check because it applies only to the built-in QAT devices and it doesn't
1323  * grant additional privileges.
1324  */
1325 #define BUGGY_QAT_DEVID_MASK 0x4940
1326 static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev)
1327 {
1328         if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1329                 return false;
1330
1331         if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1332                 return false;
1333
1334         return true;
1335 }
1336
1337 static void iommu_enable_pci_caps(struct device_domain_info *info)
1338 {
1339         struct pci_dev *pdev;
1340
1341         if (!dev_is_pci(info->dev))
1342                 return;
1343
1344         pdev = to_pci_dev(info->dev);
1345
1346         /* The PCIe spec, in its wisdom, declares that the behaviour of
1347            the device if you enable PASID support after ATS support is
1348            undefined. So always enable PASID support on devices which
1349            have it, even if we can't yet know if we're ever going to
1350            use it. */
1351         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1352                 info->pasid_enabled = 1;
1353
1354         if (info->ats_supported && pci_ats_page_aligned(pdev) &&
1355             !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1356                 info->ats_enabled = 1;
1357                 domain_update_iotlb(info->domain);
1358         }
1359 }
1360
1361 static void iommu_disable_pci_caps(struct device_domain_info *info)
1362 {
1363         struct pci_dev *pdev;
1364
1365         if (!dev_is_pci(info->dev))
1366                 return;
1367
1368         pdev = to_pci_dev(info->dev);
1369
1370         if (info->ats_enabled) {
1371                 pci_disable_ats(pdev);
1372                 info->ats_enabled = 0;
1373                 domain_update_iotlb(info->domain);
1374         }
1375
1376         if (info->pasid_enabled) {
1377                 pci_disable_pasid(pdev);
1378                 info->pasid_enabled = 0;
1379         }
1380 }
1381
1382 static void __iommu_flush_dev_iotlb(struct device_domain_info *info,
1383                                     u64 addr, unsigned int mask)
1384 {
1385         u16 sid, qdep;
1386
1387         if (!info || !info->ats_enabled)
1388                 return;
1389
1390         sid = info->bus << 8 | info->devfn;
1391         qdep = info->ats_qdep;
1392         qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1393                            qdep, addr, mask);
1394         quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep);
1395 }
1396
1397 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1398                                   u64 addr, unsigned mask)
1399 {
1400         struct dev_pasid_info *dev_pasid;
1401         struct device_domain_info *info;
1402         unsigned long flags;
1403
1404         if (!domain->has_iotlb_device)
1405                 return;
1406
1407         spin_lock_irqsave(&domain->lock, flags);
1408         list_for_each_entry(info, &domain->devices, link)
1409                 __iommu_flush_dev_iotlb(info, addr, mask);
1410
1411         list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) {
1412                 info = dev_iommu_priv_get(dev_pasid->dev);
1413
1414                 if (!info->ats_enabled)
1415                         continue;
1416
1417                 qi_flush_dev_iotlb_pasid(info->iommu,
1418                                          PCI_DEVID(info->bus, info->devfn),
1419                                          info->pfsid, dev_pasid->pasid,
1420                                          info->ats_qdep, addr,
1421                                          mask);
1422         }
1423         spin_unlock_irqrestore(&domain->lock, flags);
1424 }
1425
1426 static void domain_flush_pasid_iotlb(struct intel_iommu *iommu,
1427                                      struct dmar_domain *domain, u64 addr,
1428                                      unsigned long npages, bool ih)
1429 {
1430         u16 did = domain_id_iommu(domain, iommu);
1431         struct dev_pasid_info *dev_pasid;
1432         unsigned long flags;
1433
1434         spin_lock_irqsave(&domain->lock, flags);
1435         list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain)
1436                 qi_flush_piotlb(iommu, did, dev_pasid->pasid, addr, npages, ih);
1437
1438         if (!list_empty(&domain->devices))
1439                 qi_flush_piotlb(iommu, did, IOMMU_NO_PASID, addr, npages, ih);
1440         spin_unlock_irqrestore(&domain->lock, flags);
1441 }
1442
1443 static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1444                                     unsigned long pfn, unsigned int pages,
1445                                     int ih)
1446 {
1447         unsigned int aligned_pages = __roundup_pow_of_two(pages);
1448         unsigned long bitmask = aligned_pages - 1;
1449         unsigned int mask = ilog2(aligned_pages);
1450         u64 addr = (u64)pfn << VTD_PAGE_SHIFT;
1451
1452         /*
1453          * PSI masks the low order bits of the base address. If the
1454          * address isn't aligned to the mask, then compute a mask value
1455          * needed to ensure the target range is flushed.
1456          */
1457         if (unlikely(bitmask & pfn)) {
1458                 unsigned long end_pfn = pfn + pages - 1, shared_bits;
1459
1460                 /*
1461                  * Since end_pfn <= pfn + bitmask, the only way bits
1462                  * higher than bitmask can differ in pfn and end_pfn is
1463                  * by carrying. This means after masking out bitmask,
1464                  * high bits starting with the first set bit in
1465                  * shared_bits are all equal in both pfn and end_pfn.
1466                  */
1467                 shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
1468                 mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;
1469         }
1470
1471         /*
1472          * Fallback to domain selective flush if no PSI support or
1473          * the size is too big.
1474          */
1475         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1476                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1477                                          DMA_TLB_DSI_FLUSH);
1478         else
1479                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1480                                          DMA_TLB_PSI_FLUSH);
1481 }
1482
1483 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1484                                   struct dmar_domain *domain,
1485                                   unsigned long pfn, unsigned int pages,
1486                                   int ih, int map)
1487 {
1488         unsigned int aligned_pages = __roundup_pow_of_two(pages);
1489         unsigned int mask = ilog2(aligned_pages);
1490         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1491         u16 did = domain_id_iommu(domain, iommu);
1492
1493         if (WARN_ON(!pages))
1494                 return;
1495
1496         if (ih)
1497                 ih = 1 << 6;
1498
1499         if (domain->use_first_level)
1500                 domain_flush_pasid_iotlb(iommu, domain, addr, pages, ih);
1501         else
1502                 __iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih);
1503
1504         /*
1505          * In caching mode, changes of pages from non-present to present require
1506          * flush. However, device IOTLB doesn't need to be flushed in this case.
1507          */
1508         if (!cap_caching_mode(iommu->cap) || !map)
1509                 iommu_flush_dev_iotlb(domain, addr, mask);
1510 }
1511
1512 /* Notification for newly created mappings */
1513 static void __mapping_notify_one(struct intel_iommu *iommu, struct dmar_domain *domain,
1514                                  unsigned long pfn, unsigned int pages)
1515 {
1516         /*
1517          * It's a non-present to present mapping. Only flush if caching mode
1518          * and second level.
1519          */
1520         if (cap_caching_mode(iommu->cap) && !domain->use_first_level)
1521                 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1522         else
1523                 iommu_flush_write_buffer(iommu);
1524 }
1525
1526 /*
1527  * Flush the relevant caches in nested translation if the domain
1528  * also serves as a parent
1529  */
1530 static void parent_domain_flush(struct dmar_domain *domain,
1531                                 unsigned long pfn,
1532                                 unsigned long pages, int ih)
1533 {
1534         struct dmar_domain *s1_domain;
1535
1536         spin_lock(&domain->s1_lock);
1537         list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
1538                 struct device_domain_info *device_info;
1539                 struct iommu_domain_info *info;
1540                 unsigned long flags;
1541                 unsigned long i;
1542
1543                 xa_for_each(&s1_domain->iommu_array, i, info)
1544                         __iommu_flush_iotlb_psi(info->iommu, info->did,
1545                                                 pfn, pages, ih);
1546
1547                 if (!s1_domain->has_iotlb_device)
1548                         continue;
1549
1550                 spin_lock_irqsave(&s1_domain->lock, flags);
1551                 list_for_each_entry(device_info, &s1_domain->devices, link)
1552                         /*
1553                          * Address translation cache in device side caches the
1554                          * result of nested translation. There is no easy way
1555                          * to identify the exact set of nested translations
1556                          * affected by a change in S2. So just flush the entire
1557                          * device cache.
1558                          */
1559                         __iommu_flush_dev_iotlb(device_info, 0,
1560                                                 MAX_AGAW_PFN_WIDTH);
1561                 spin_unlock_irqrestore(&s1_domain->lock, flags);
1562         }
1563         spin_unlock(&domain->s1_lock);
1564 }
1565
1566 static void intel_flush_iotlb_all(struct iommu_domain *domain)
1567 {
1568         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
1569         struct iommu_domain_info *info;
1570         unsigned long idx;
1571
1572         xa_for_each(&dmar_domain->iommu_array, idx, info) {
1573                 struct intel_iommu *iommu = info->iommu;
1574                 u16 did = domain_id_iommu(dmar_domain, iommu);
1575
1576                 if (dmar_domain->use_first_level)
1577                         domain_flush_pasid_iotlb(iommu, dmar_domain, 0, -1, 0);
1578                 else
1579                         iommu->flush.flush_iotlb(iommu, did, 0, 0,
1580                                                  DMA_TLB_DSI_FLUSH);
1581
1582                 if (!cap_caching_mode(iommu->cap))
1583                         iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH);
1584         }
1585
1586         if (dmar_domain->nested_parent)
1587                 parent_domain_flush(dmar_domain, 0, -1, 0);
1588 }
1589
1590 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1591 {
1592         u32 pmen;
1593         unsigned long flags;
1594
1595         if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1596                 return;
1597
1598         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1599         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1600         pmen &= ~DMA_PMEN_EPM;
1601         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1602
1603         /* wait for the protected region status bit to clear */
1604         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1605                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1606
1607         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1608 }
1609
1610 static void iommu_enable_translation(struct intel_iommu *iommu)
1611 {
1612         u32 sts;
1613         unsigned long flags;
1614
1615         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1616         iommu->gcmd |= DMA_GCMD_TE;
1617         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1618
1619         /* Make sure hardware complete it */
1620         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1621                       readl, (sts & DMA_GSTS_TES), sts);
1622
1623         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1624 }
1625
1626 static void iommu_disable_translation(struct intel_iommu *iommu)
1627 {
1628         u32 sts;
1629         unsigned long flag;
1630
1631         if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1632             (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1633                 return;
1634
1635         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1636         iommu->gcmd &= ~DMA_GCMD_TE;
1637         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1638
1639         /* Make sure hardware complete it */
1640         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1641                       readl, (!(sts & DMA_GSTS_TES)), sts);
1642
1643         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1644 }
1645
1646 static int iommu_init_domains(struct intel_iommu *iommu)
1647 {
1648         u32 ndomains;
1649
1650         ndomains = cap_ndoms(iommu->cap);
1651         pr_debug("%s: Number of Domains supported <%d>\n",
1652                  iommu->name, ndomains);
1653
1654         spin_lock_init(&iommu->lock);
1655
1656         iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
1657         if (!iommu->domain_ids)
1658                 return -ENOMEM;
1659
1660         /*
1661          * If Caching mode is set, then invalid translations are tagged
1662          * with domain-id 0, hence we need to pre-allocate it. We also
1663          * use domain-id 0 as a marker for non-allocated domain-id, so
1664          * make sure it is not used for a real domain.
1665          */
1666         set_bit(0, iommu->domain_ids);
1667
1668         /*
1669          * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1670          * entry for first-level or pass-through translation modes should
1671          * be programmed with a domain id different from those used for
1672          * second-level or nested translation. We reserve a domain id for
1673          * this purpose.
1674          */
1675         if (sm_supported(iommu))
1676                 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1677
1678         return 0;
1679 }
1680
1681 static void disable_dmar_iommu(struct intel_iommu *iommu)
1682 {
1683         if (!iommu->domain_ids)
1684                 return;
1685
1686         /*
1687          * All iommu domains must have been detached from the devices,
1688          * hence there should be no domain IDs in use.
1689          */
1690         if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))
1691                     > NUM_RESERVED_DID))
1692                 return;
1693
1694         if (iommu->gcmd & DMA_GCMD_TE)
1695                 iommu_disable_translation(iommu);
1696 }
1697
1698 static void free_dmar_iommu(struct intel_iommu *iommu)
1699 {
1700         if (iommu->domain_ids) {
1701                 bitmap_free(iommu->domain_ids);
1702                 iommu->domain_ids = NULL;
1703         }
1704
1705         if (iommu->copied_tables) {
1706                 bitmap_free(iommu->copied_tables);
1707                 iommu->copied_tables = NULL;
1708         }
1709
1710         /* free context mapping */
1711         free_context_table(iommu);
1712
1713 #ifdef CONFIG_INTEL_IOMMU_SVM
1714         if (pasid_supported(iommu)) {
1715                 if (ecap_prs(iommu->ecap))
1716                         intel_svm_finish_prq(iommu);
1717         }
1718 #endif
1719 }
1720
1721 /*
1722  * Check and return whether first level is used by default for
1723  * DMA translation.
1724  */
1725 static bool first_level_by_default(unsigned int type)
1726 {
1727         /* Only SL is available in legacy mode */
1728         if (!scalable_mode_support())
1729                 return false;
1730
1731         /* Only level (either FL or SL) is available, just use it */
1732         if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity())
1733                 return intel_cap_flts_sanity();
1734
1735         /* Both levels are available, decide it based on domain type */
1736         return type != IOMMU_DOMAIN_UNMANAGED;
1737 }
1738
1739 static struct dmar_domain *alloc_domain(unsigned int type)
1740 {
1741         struct dmar_domain *domain;
1742
1743         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1744         if (!domain)
1745                 return NULL;
1746
1747         domain->nid = NUMA_NO_NODE;
1748         if (first_level_by_default(type))
1749                 domain->use_first_level = true;
1750         domain->has_iotlb_device = false;
1751         INIT_LIST_HEAD(&domain->devices);
1752         INIT_LIST_HEAD(&domain->dev_pasids);
1753         spin_lock_init(&domain->lock);
1754         xa_init(&domain->iommu_array);
1755
1756         return domain;
1757 }
1758
1759 int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1760 {
1761         struct iommu_domain_info *info, *curr;
1762         unsigned long ndomains;
1763         int num, ret = -ENOSPC;
1764
1765         info = kzalloc(sizeof(*info), GFP_KERNEL);
1766         if (!info)
1767                 return -ENOMEM;
1768
1769         spin_lock(&iommu->lock);
1770         curr = xa_load(&domain->iommu_array, iommu->seq_id);
1771         if (curr) {
1772                 curr->refcnt++;
1773                 spin_unlock(&iommu->lock);
1774                 kfree(info);
1775                 return 0;
1776         }
1777
1778         ndomains = cap_ndoms(iommu->cap);
1779         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1780         if (num >= ndomains) {
1781                 pr_err("%s: No free domain ids\n", iommu->name);
1782                 goto err_unlock;
1783         }
1784
1785         set_bit(num, iommu->domain_ids);
1786         info->refcnt    = 1;
1787         info->did       = num;
1788         info->iommu     = iommu;
1789         curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1790                           NULL, info, GFP_ATOMIC);
1791         if (curr) {
1792                 ret = xa_err(curr) ? : -EBUSY;
1793                 goto err_clear;
1794         }
1795         domain_update_iommu_cap(domain);
1796
1797         spin_unlock(&iommu->lock);
1798         return 0;
1799
1800 err_clear:
1801         clear_bit(info->did, iommu->domain_ids);
1802 err_unlock:
1803         spin_unlock(&iommu->lock);
1804         kfree(info);
1805         return ret;
1806 }
1807
1808 void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1809 {
1810         struct iommu_domain_info *info;
1811
1812         spin_lock(&iommu->lock);
1813         info = xa_load(&domain->iommu_array, iommu->seq_id);
1814         if (--info->refcnt == 0) {
1815                 clear_bit(info->did, iommu->domain_ids);
1816                 xa_erase(&domain->iommu_array, iommu->seq_id);
1817                 domain->nid = NUMA_NO_NODE;
1818                 domain_update_iommu_cap(domain);
1819                 kfree(info);
1820         }
1821         spin_unlock(&iommu->lock);
1822 }
1823
1824 static int guestwidth_to_adjustwidth(int gaw)
1825 {
1826         int agaw;
1827         int r = (gaw - 12) % 9;
1828
1829         if (r == 0)
1830                 agaw = gaw;
1831         else
1832                 agaw = gaw + 9 - r;
1833         if (agaw > 64)
1834                 agaw = 64;
1835         return agaw;
1836 }
1837
1838 static void domain_exit(struct dmar_domain *domain)
1839 {
1840         if (domain->pgd) {
1841                 LIST_HEAD(freelist);
1842
1843                 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
1844                 put_pages_list(&freelist);
1845         }
1846
1847         if (WARN_ON(!list_empty(&domain->devices)))
1848                 return;
1849
1850         kfree(domain);
1851 }
1852
1853 static int domain_context_mapping_one(struct dmar_domain *domain,
1854                                       struct intel_iommu *iommu,
1855                                       u8 bus, u8 devfn)
1856 {
1857         struct device_domain_info *info =
1858                         domain_lookup_dev_info(domain, iommu, bus, devfn);
1859         u16 did = domain_id_iommu(domain, iommu);
1860         int translation = CONTEXT_TT_MULTI_LEVEL;
1861         struct dma_pte *pgd = domain->pgd;
1862         struct context_entry *context;
1863         int agaw, ret;
1864
1865         if (hw_pass_through && domain_type_is_si(domain))
1866                 translation = CONTEXT_TT_PASS_THROUGH;
1867
1868         pr_debug("Set context mapping for %02x:%02x.%d\n",
1869                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1870
1871         spin_lock(&iommu->lock);
1872         ret = -ENOMEM;
1873         context = iommu_context_addr(iommu, bus, devfn, 1);
1874         if (!context)
1875                 goto out_unlock;
1876
1877         ret = 0;
1878         if (context_present(context) && !context_copied(iommu, bus, devfn))
1879                 goto out_unlock;
1880
1881         /*
1882          * For kdump cases, old valid entries may be cached due to the
1883          * in-flight DMA and copied pgtable, but there is no unmapping
1884          * behaviour for them, thus we need an explicit cache flush for
1885          * the newly-mapped device. For kdump, at this point, the device
1886          * is supposed to finish reset at its driver probe stage, so no
1887          * in-flight DMA will exist, and we don't need to worry anymore
1888          * hereafter.
1889          */
1890         if (context_copied(iommu, bus, devfn)) {
1891                 u16 did_old = context_domain_id(context);
1892
1893                 if (did_old < cap_ndoms(iommu->cap)) {
1894                         iommu->flush.flush_context(iommu, did_old,
1895                                                    (((u16)bus) << 8) | devfn,
1896                                                    DMA_CCMD_MASK_NOBIT,
1897                                                    DMA_CCMD_DEVICE_INVL);
1898                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1899                                                  DMA_TLB_DSI_FLUSH);
1900                 }
1901
1902                 clear_context_copied(iommu, bus, devfn);
1903         }
1904
1905         context_clear_entry(context);
1906         context_set_domain_id(context, did);
1907
1908         if (translation != CONTEXT_TT_PASS_THROUGH) {
1909                 /*
1910                  * Skip top levels of page tables for iommu which has
1911                  * less agaw than default. Unnecessary for PT mode.
1912                  */
1913                 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
1914                         ret = -ENOMEM;
1915                         pgd = phys_to_virt(dma_pte_addr(pgd));
1916                         if (!dma_pte_present(pgd))
1917                                 goto out_unlock;
1918                 }
1919
1920                 if (info && info->ats_supported)
1921                         translation = CONTEXT_TT_DEV_IOTLB;
1922                 else
1923                         translation = CONTEXT_TT_MULTI_LEVEL;
1924
1925                 context_set_address_root(context, virt_to_phys(pgd));
1926                 context_set_address_width(context, agaw);
1927         } else {
1928                 /*
1929                  * In pass through mode, AW must be programmed to
1930                  * indicate the largest AGAW value supported by
1931                  * hardware. And ASR is ignored by hardware.
1932                  */
1933                 context_set_address_width(context, iommu->msagaw);
1934         }
1935
1936         context_set_translation_type(context, translation);
1937         context_set_fault_enable(context);
1938         context_set_present(context);
1939         if (!ecap_coherent(iommu->ecap))
1940                 clflush_cache_range(context, sizeof(*context));
1941
1942         /*
1943          * It's a non-present to present mapping. If hardware doesn't cache
1944          * non-present entry we only need to flush the write-buffer. If the
1945          * _does_ cache non-present entries, then it does so in the special
1946          * domain #0, which we have to flush:
1947          */
1948         if (cap_caching_mode(iommu->cap)) {
1949                 iommu->flush.flush_context(iommu, 0,
1950                                            (((u16)bus) << 8) | devfn,
1951                                            DMA_CCMD_MASK_NOBIT,
1952                                            DMA_CCMD_DEVICE_INVL);
1953                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1954         } else {
1955                 iommu_flush_write_buffer(iommu);
1956         }
1957
1958         ret = 0;
1959
1960 out_unlock:
1961         spin_unlock(&iommu->lock);
1962
1963         return ret;
1964 }
1965
1966 static int domain_context_mapping_cb(struct pci_dev *pdev,
1967                                      u16 alias, void *opaque)
1968 {
1969         struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev);
1970         struct intel_iommu *iommu = info->iommu;
1971         struct dmar_domain *domain = opaque;
1972
1973         return domain_context_mapping_one(domain, iommu,
1974                                           PCI_BUS_NUM(alias), alias & 0xff);
1975 }
1976
1977 static int
1978 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
1979 {
1980         struct device_domain_info *info = dev_iommu_priv_get(dev);
1981         struct intel_iommu *iommu = info->iommu;
1982         u8 bus = info->bus, devfn = info->devfn;
1983
1984         if (!dev_is_pci(dev))
1985                 return domain_context_mapping_one(domain, iommu, bus, devfn);
1986
1987         return pci_for_each_dma_alias(to_pci_dev(dev),
1988                                       domain_context_mapping_cb, domain);
1989 }
1990
1991 /* Returns a number of VTD pages, but aligned to MM page size */
1992 static unsigned long aligned_nrpages(unsigned long host_addr, size_t size)
1993 {
1994         host_addr &= ~PAGE_MASK;
1995         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1996 }
1997
1998 /* Return largest possible superpage level for a given mapping */
1999 static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn,
2000                                    unsigned long phy_pfn, unsigned long pages)
2001 {
2002         int support, level = 1;
2003         unsigned long pfnmerge;
2004
2005         support = domain->iommu_superpage;
2006
2007         /* To use a large page, the virtual *and* physical addresses
2008            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2009            of them will mean we have to use smaller pages. So just
2010            merge them and check both at once. */
2011         pfnmerge = iov_pfn | phy_pfn;
2012
2013         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2014                 pages >>= VTD_STRIDE_SHIFT;
2015                 if (!pages)
2016                         break;
2017                 pfnmerge >>= VTD_STRIDE_SHIFT;
2018                 level++;
2019                 support--;
2020         }
2021         return level;
2022 }
2023
2024 /*
2025  * Ensure that old small page tables are removed to make room for superpage(s).
2026  * We're going to add new large pages, so make sure we don't remove their parent
2027  * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared.
2028  */
2029 static void switch_to_super_page(struct dmar_domain *domain,
2030                                  unsigned long start_pfn,
2031                                  unsigned long end_pfn, int level)
2032 {
2033         unsigned long lvl_pages = lvl_to_nr_pages(level);
2034         struct iommu_domain_info *info;
2035         struct dma_pte *pte = NULL;
2036         unsigned long i;
2037
2038         while (start_pfn <= end_pfn) {
2039                 if (!pte)
2040                         pte = pfn_to_dma_pte(domain, start_pfn, &level,
2041                                              GFP_ATOMIC);
2042
2043                 if (dma_pte_present(pte)) {
2044                         dma_pte_free_pagetable(domain, start_pfn,
2045                                                start_pfn + lvl_pages - 1,
2046                                                level + 1);
2047
2048                         xa_for_each(&domain->iommu_array, i, info)
2049                                 iommu_flush_iotlb_psi(info->iommu, domain,
2050                                                       start_pfn, lvl_pages,
2051                                                       0, 0);
2052                         if (domain->nested_parent)
2053                                 parent_domain_flush(domain, start_pfn,
2054                                                     lvl_pages, 0);
2055                 }
2056
2057                 pte++;
2058                 start_pfn += lvl_pages;
2059                 if (first_pte_in_page(pte))
2060                         pte = NULL;
2061         }
2062 }
2063
2064 static int
2065 __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2066                  unsigned long phys_pfn, unsigned long nr_pages, int prot,
2067                  gfp_t gfp)
2068 {
2069         struct dma_pte *first_pte = NULL, *pte = NULL;
2070         unsigned int largepage_lvl = 0;
2071         unsigned long lvl_pages = 0;
2072         phys_addr_t pteval;
2073         u64 attr;
2074
2075         if (unlikely(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)))
2076                 return -EINVAL;
2077
2078         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2079                 return -EINVAL;
2080
2081         if (!(prot & DMA_PTE_WRITE) && domain->nested_parent) {
2082                 pr_err_ratelimited("Read-only mapping is disallowed on the domain which serves as the parent in a nested configuration, due to HW errata (ERRATA_772415_SPR17)\n");
2083                 return -EINVAL;
2084         }
2085
2086         attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
2087         attr |= DMA_FL_PTE_PRESENT;
2088         if (domain->use_first_level) {
2089                 attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
2090                 if (prot & DMA_PTE_WRITE)
2091                         attr |= DMA_FL_PTE_DIRTY;
2092         }
2093
2094         domain->has_mappings = true;
2095
2096         pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
2097
2098         while (nr_pages > 0) {
2099                 uint64_t tmp;
2100
2101                 if (!pte) {
2102                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn,
2103                                         phys_pfn, nr_pages);
2104
2105                         pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl,
2106                                              gfp);
2107                         if (!pte)
2108                                 return -ENOMEM;
2109                         first_pte = pte;
2110
2111                         lvl_pages = lvl_to_nr_pages(largepage_lvl);
2112
2113                         /* It is large page*/
2114                         if (largepage_lvl > 1) {
2115                                 unsigned long end_pfn;
2116                                 unsigned long pages_to_remove;
2117
2118                                 pteval |= DMA_PTE_LARGE_PAGE;
2119                                 pages_to_remove = min_t(unsigned long, nr_pages,
2120                                                         nr_pte_to_next_page(pte) * lvl_pages);
2121                                 end_pfn = iov_pfn + pages_to_remove - 1;
2122                                 switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl);
2123                         } else {
2124                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2125                         }
2126
2127                 }
2128                 /* We don't need lock here, nobody else
2129                  * touches the iova range
2130                  */
2131                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2132                 if (tmp) {
2133                         static int dumps = 5;
2134                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2135                                 iov_pfn, tmp, (unsigned long long)pteval);
2136                         if (dumps) {
2137                                 dumps--;
2138                                 debug_dma_dump_mappings(NULL);
2139                         }
2140                         WARN_ON(1);
2141                 }
2142
2143                 nr_pages -= lvl_pages;
2144                 iov_pfn += lvl_pages;
2145                 phys_pfn += lvl_pages;
2146                 pteval += lvl_pages * VTD_PAGE_SIZE;
2147
2148                 /* If the next PTE would be the first in a new page, then we
2149                  * need to flush the cache on the entries we've just written.
2150                  * And then we'll need to recalculate 'pte', so clear it and
2151                  * let it get set again in the if (!pte) block above.
2152                  *
2153                  * If we're done (!nr_pages) we need to flush the cache too.
2154                  *
2155                  * Also if we've been setting superpages, we may need to
2156                  * recalculate 'pte' and switch back to smaller pages for the
2157                  * end of the mapping, if the trailing size is not enough to
2158                  * use another superpage (i.e. nr_pages < lvl_pages).
2159                  */
2160                 pte++;
2161                 if (!nr_pages || first_pte_in_page(pte) ||
2162                     (largepage_lvl > 1 && nr_pages < lvl_pages)) {
2163                         domain_flush_cache(domain, first_pte,
2164                                            (void *)pte - (void *)first_pte);
2165                         pte = NULL;
2166                 }
2167         }
2168
2169         return 0;
2170 }
2171
2172 static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 devfn)
2173 {
2174         struct intel_iommu *iommu = info->iommu;
2175         struct context_entry *context;
2176         u16 did_old;
2177
2178         spin_lock(&iommu->lock);
2179         context = iommu_context_addr(iommu, bus, devfn, 0);
2180         if (!context) {
2181                 spin_unlock(&iommu->lock);
2182                 return;
2183         }
2184
2185         did_old = context_domain_id(context);
2186
2187         context_clear_entry(context);
2188         __iommu_flush_cache(iommu, context, sizeof(*context));
2189         spin_unlock(&iommu->lock);
2190         iommu->flush.flush_context(iommu,
2191                                    did_old,
2192                                    (((u16)bus) << 8) | devfn,
2193                                    DMA_CCMD_MASK_NOBIT,
2194                                    DMA_CCMD_DEVICE_INVL);
2195
2196         iommu->flush.flush_iotlb(iommu,
2197                                  did_old,
2198                                  0,
2199                                  0,
2200                                  DMA_TLB_DSI_FLUSH);
2201
2202         __iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
2203 }
2204
2205 static int domain_setup_first_level(struct intel_iommu *iommu,
2206                                     struct dmar_domain *domain,
2207                                     struct device *dev,
2208                                     u32 pasid)
2209 {
2210         struct dma_pte *pgd = domain->pgd;
2211         int agaw, level;
2212         int flags = 0;
2213
2214         /*
2215          * Skip top levels of page tables for iommu which has
2216          * less agaw than default. Unnecessary for PT mode.
2217          */
2218         for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2219                 pgd = phys_to_virt(dma_pte_addr(pgd));
2220                 if (!dma_pte_present(pgd))
2221                         return -ENOMEM;
2222         }
2223
2224         level = agaw_to_level(agaw);
2225         if (level != 4 && level != 5)
2226                 return -EINVAL;
2227
2228         if (level == 5)
2229                 flags |= PASID_FLAG_FL5LP;
2230
2231         if (domain->force_snooping)
2232                 flags |= PASID_FLAG_PAGE_SNOOP;
2233
2234         return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
2235                                              domain_id_iommu(domain, iommu),
2236                                              flags);
2237 }
2238
2239 static bool dev_is_real_dma_subdevice(struct device *dev)
2240 {
2241         return dev && dev_is_pci(dev) &&
2242                pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
2243 }
2244
2245 static int iommu_domain_identity_map(struct dmar_domain *domain,
2246                                      unsigned long first_vpfn,
2247                                      unsigned long last_vpfn)
2248 {
2249         /*
2250          * RMRR range might have overlap with physical memory range,
2251          * clear it first
2252          */
2253         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2254
2255         return __domain_mapping(domain, first_vpfn,
2256                                 first_vpfn, last_vpfn - first_vpfn + 1,
2257                                 DMA_PTE_READ|DMA_PTE_WRITE, GFP_KERNEL);
2258 }
2259
2260 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2261
2262 static int __init si_domain_init(int hw)
2263 {
2264         struct dmar_rmrr_unit *rmrr;
2265         struct device *dev;
2266         int i, nid, ret;
2267
2268         si_domain = alloc_domain(IOMMU_DOMAIN_IDENTITY);
2269         if (!si_domain)
2270                 return -EFAULT;
2271
2272         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2273                 domain_exit(si_domain);
2274                 si_domain = NULL;
2275                 return -EFAULT;
2276         }
2277
2278         if (hw)
2279                 return 0;
2280
2281         for_each_online_node(nid) {
2282                 unsigned long start_pfn, end_pfn;
2283                 int i;
2284
2285                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2286                         ret = iommu_domain_identity_map(si_domain,
2287                                         mm_to_dma_pfn_start(start_pfn),
2288                                         mm_to_dma_pfn_end(end_pfn));
2289                         if (ret)
2290                                 return ret;
2291                 }
2292         }
2293
2294         /*
2295          * Identity map the RMRRs so that devices with RMRRs could also use
2296          * the si_domain.
2297          */
2298         for_each_rmrr_units(rmrr) {
2299                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2300                                           i, dev) {
2301                         unsigned long long start = rmrr->base_address;
2302                         unsigned long long end = rmrr->end_address;
2303
2304                         if (WARN_ON(end < start ||
2305                                     end >> agaw_to_width(si_domain->agaw)))
2306                                 continue;
2307
2308                         ret = iommu_domain_identity_map(si_domain,
2309                                         mm_to_dma_pfn_start(start >> PAGE_SHIFT),
2310                                         mm_to_dma_pfn_end(end >> PAGE_SHIFT));
2311                         if (ret)
2312                                 return ret;
2313                 }
2314         }
2315
2316         return 0;
2317 }
2318
2319 static int dmar_domain_attach_device(struct dmar_domain *domain,
2320                                      struct device *dev)
2321 {
2322         struct device_domain_info *info = dev_iommu_priv_get(dev);
2323         struct intel_iommu *iommu = info->iommu;
2324         unsigned long flags;
2325         int ret;
2326
2327         ret = domain_attach_iommu(domain, iommu);
2328         if (ret)
2329                 return ret;
2330         info->domain = domain;
2331         spin_lock_irqsave(&domain->lock, flags);
2332         list_add(&info->link, &domain->devices);
2333         spin_unlock_irqrestore(&domain->lock, flags);
2334
2335         if (dev_is_real_dma_subdevice(dev))
2336                 return 0;
2337
2338         if (!sm_supported(iommu))
2339                 ret = domain_context_mapping(domain, dev);
2340         else if (hw_pass_through && domain_type_is_si(domain))
2341                 ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
2342         else if (domain->use_first_level)
2343                 ret = domain_setup_first_level(iommu, domain, dev, IOMMU_NO_PASID);
2344         else
2345                 ret = intel_pasid_setup_second_level(iommu, domain, dev, IOMMU_NO_PASID);
2346
2347         if (ret) {
2348                 device_block_translation(dev);
2349                 return ret;
2350         }
2351
2352         if (sm_supported(info->iommu) || !domain_type_is_si(info->domain))
2353                 iommu_enable_pci_caps(info);
2354
2355         return 0;
2356 }
2357
2358 /**
2359  * device_rmrr_is_relaxable - Test whether the RMRR of this device
2360  * is relaxable (ie. is allowed to be not enforced under some conditions)
2361  * @dev: device handle
2362  *
2363  * We assume that PCI USB devices with RMRRs have them largely
2364  * for historical reasons and that the RMRR space is not actively used post
2365  * boot.  This exclusion may change if vendors begin to abuse it.
2366  *
2367  * The same exception is made for graphics devices, with the requirement that
2368  * any use of the RMRR regions will be torn down before assigning the device
2369  * to a guest.
2370  *
2371  * Return: true if the RMRR is relaxable, false otherwise
2372  */
2373 static bool device_rmrr_is_relaxable(struct device *dev)
2374 {
2375         struct pci_dev *pdev;
2376
2377         if (!dev_is_pci(dev))
2378                 return false;
2379
2380         pdev = to_pci_dev(dev);
2381         if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2382                 return true;
2383         else
2384                 return false;
2385 }
2386
2387 /*
2388  * Return the required default domain type for a specific device.
2389  *
2390  * @dev: the device in query
2391  * @startup: true if this is during early boot
2392  *
2393  * Returns:
2394  *  - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
2395  *  - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
2396  *  - 0: both identity and dynamic domains work for this device
2397  */
2398 static int device_def_domain_type(struct device *dev)
2399 {
2400         if (dev_is_pci(dev)) {
2401                 struct pci_dev *pdev = to_pci_dev(dev);
2402
2403                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2404                         return IOMMU_DOMAIN_IDENTITY;
2405
2406                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2407                         return IOMMU_DOMAIN_IDENTITY;
2408         }
2409
2410         return 0;
2411 }
2412
2413 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2414 {
2415         /*
2416          * Start from the sane iommu hardware state.
2417          * If the queued invalidation is already initialized by us
2418          * (for example, while enabling interrupt-remapping) then
2419          * we got the things already rolling from a sane state.
2420          */
2421         if (!iommu->qi) {
2422                 /*
2423                  * Clear any previous faults.
2424                  */
2425                 dmar_fault(-1, iommu);
2426                 /*
2427                  * Disable queued invalidation if supported and already enabled
2428                  * before OS handover.
2429                  */
2430                 dmar_disable_qi(iommu);
2431         }
2432
2433         if (dmar_enable_qi(iommu)) {
2434                 /*
2435                  * Queued Invalidate not enabled, use Register Based Invalidate
2436                  */
2437                 iommu->flush.flush_context = __iommu_flush_context;
2438                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2439                 pr_info("%s: Using Register based invalidation\n",
2440                         iommu->name);
2441         } else {
2442                 iommu->flush.flush_context = qi_flush_context;
2443                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2444                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2445         }
2446 }
2447
2448 static int copy_context_table(struct intel_iommu *iommu,
2449                               struct root_entry *old_re,
2450                               struct context_entry **tbl,
2451                               int bus, bool ext)
2452 {
2453         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2454         struct context_entry *new_ce = NULL, ce;
2455         struct context_entry *old_ce = NULL;
2456         struct root_entry re;
2457         phys_addr_t old_ce_phys;
2458
2459         tbl_idx = ext ? bus * 2 : bus;
2460         memcpy(&re, old_re, sizeof(re));
2461
2462         for (devfn = 0; devfn < 256; devfn++) {
2463                 /* First calculate the correct index */
2464                 idx = (ext ? devfn * 2 : devfn) % 256;
2465
2466                 if (idx == 0) {
2467                         /* First save what we may have and clean up */
2468                         if (new_ce) {
2469                                 tbl[tbl_idx] = new_ce;
2470                                 __iommu_flush_cache(iommu, new_ce,
2471                                                     VTD_PAGE_SIZE);
2472                                 pos = 1;
2473                         }
2474
2475                         if (old_ce)
2476                                 memunmap(old_ce);
2477
2478                         ret = 0;
2479                         if (devfn < 0x80)
2480                                 old_ce_phys = root_entry_lctp(&re);
2481                         else
2482                                 old_ce_phys = root_entry_uctp(&re);
2483
2484                         if (!old_ce_phys) {
2485                                 if (ext && devfn == 0) {
2486                                         /* No LCTP, try UCTP */
2487                                         devfn = 0x7f;
2488                                         continue;
2489                                 } else {
2490                                         goto out;
2491                                 }
2492                         }
2493
2494                         ret = -ENOMEM;
2495                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
2496                                         MEMREMAP_WB);
2497                         if (!old_ce)
2498                                 goto out;
2499
2500                         new_ce = alloc_pgtable_page(iommu->node, GFP_KERNEL);
2501                         if (!new_ce)
2502                                 goto out_unmap;
2503
2504                         ret = 0;
2505                 }
2506
2507                 /* Now copy the context entry */
2508                 memcpy(&ce, old_ce + idx, sizeof(ce));
2509
2510                 if (!context_present(&ce))
2511                         continue;
2512
2513                 did = context_domain_id(&ce);
2514                 if (did >= 0 && did < cap_ndoms(iommu->cap))
2515                         set_bit(did, iommu->domain_ids);
2516
2517                 set_context_copied(iommu, bus, devfn);
2518                 new_ce[idx] = ce;
2519         }
2520
2521         tbl[tbl_idx + pos] = new_ce;
2522
2523         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2524
2525 out_unmap:
2526         memunmap(old_ce);
2527
2528 out:
2529         return ret;
2530 }
2531
2532 static int copy_translation_tables(struct intel_iommu *iommu)
2533 {
2534         struct context_entry **ctxt_tbls;
2535         struct root_entry *old_rt;
2536         phys_addr_t old_rt_phys;
2537         int ctxt_table_entries;
2538         u64 rtaddr_reg;
2539         int bus, ret;
2540         bool new_ext, ext;
2541
2542         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2543         ext        = !!(rtaddr_reg & DMA_RTADDR_SMT);
2544         new_ext    = !!sm_supported(iommu);
2545
2546         /*
2547          * The RTT bit can only be changed when translation is disabled,
2548          * but disabling translation means to open a window for data
2549          * corruption. So bail out and don't copy anything if we would
2550          * have to change the bit.
2551          */
2552         if (new_ext != ext)
2553                 return -EINVAL;
2554
2555         iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2556         if (!iommu->copied_tables)
2557                 return -ENOMEM;
2558
2559         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2560         if (!old_rt_phys)
2561                 return -EINVAL;
2562
2563         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
2564         if (!old_rt)
2565                 return -ENOMEM;
2566
2567         /* This is too big for the stack - allocate it from slab */
2568         ctxt_table_entries = ext ? 512 : 256;
2569         ret = -ENOMEM;
2570         ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
2571         if (!ctxt_tbls)
2572                 goto out_unmap;
2573
2574         for (bus = 0; bus < 256; bus++) {
2575                 ret = copy_context_table(iommu, &old_rt[bus],
2576                                          ctxt_tbls, bus, ext);
2577                 if (ret) {
2578                         pr_err("%s: Failed to copy context table for bus %d\n",
2579                                 iommu->name, bus);
2580                         continue;
2581                 }
2582         }
2583
2584         spin_lock(&iommu->lock);
2585
2586         /* Context tables are copied, now write them to the root_entry table */
2587         for (bus = 0; bus < 256; bus++) {
2588                 int idx = ext ? bus * 2 : bus;
2589                 u64 val;
2590
2591                 if (ctxt_tbls[idx]) {
2592                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
2593                         iommu->root_entry[bus].lo = val;
2594                 }
2595
2596                 if (!ext || !ctxt_tbls[idx + 1])
2597                         continue;
2598
2599                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2600                 iommu->root_entry[bus].hi = val;
2601         }
2602
2603         spin_unlock(&iommu->lock);
2604
2605         kfree(ctxt_tbls);
2606
2607         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2608
2609         ret = 0;
2610
2611 out_unmap:
2612         memunmap(old_rt);
2613
2614         return ret;
2615 }
2616
2617 static int __init init_dmars(void)
2618 {
2619         struct dmar_drhd_unit *drhd;
2620         struct intel_iommu *iommu;
2621         int ret;
2622
2623         ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
2624         if (ret)
2625                 goto free_iommu;
2626
2627         for_each_iommu(iommu, drhd) {
2628                 if (drhd->ignored) {
2629                         iommu_disable_translation(iommu);
2630                         continue;
2631                 }
2632
2633                 /*
2634                  * Find the max pasid size of all IOMMU's in the system.
2635                  * We need to ensure the system pasid table is no bigger
2636                  * than the smallest supported.
2637                  */
2638                 if (pasid_supported(iommu)) {
2639                         u32 temp = 2 << ecap_pss(iommu->ecap);
2640
2641                         intel_pasid_max_id = min_t(u32, temp,
2642                                                    intel_pasid_max_id);
2643                 }
2644
2645                 intel_iommu_init_qi(iommu);
2646
2647                 ret = iommu_init_domains(iommu);
2648                 if (ret)
2649                         goto free_iommu;
2650
2651                 init_translation_status(iommu);
2652
2653                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2654                         iommu_disable_translation(iommu);
2655                         clear_translation_pre_enabled(iommu);
2656                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
2657                                 iommu->name);
2658                 }
2659
2660                 /*
2661                  * TBD:
2662                  * we could share the same root & context tables
2663                  * among all IOMMU's. Need to Split it later.
2664                  */
2665                 ret = iommu_alloc_root_entry(iommu);
2666                 if (ret)
2667                         goto free_iommu;
2668
2669                 if (translation_pre_enabled(iommu)) {
2670                         pr_info("Translation already enabled - trying to copy translation structures\n");
2671
2672                         ret = copy_translation_tables(iommu);
2673                         if (ret) {
2674                                 /*
2675                                  * We found the IOMMU with translation
2676                                  * enabled - but failed to copy over the
2677                                  * old root-entry table. Try to proceed
2678                                  * by disabling translation now and
2679                                  * allocating a clean root-entry table.
2680                                  * This might cause DMAR faults, but
2681                                  * probably the dump will still succeed.
2682                                  */
2683                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
2684                                        iommu->name);
2685                                 iommu_disable_translation(iommu);
2686                                 clear_translation_pre_enabled(iommu);
2687                         } else {
2688                                 pr_info("Copied translation tables from previous kernel for %s\n",
2689                                         iommu->name);
2690                         }
2691                 }
2692
2693                 if (!ecap_pass_through(iommu->ecap))
2694                         hw_pass_through = 0;
2695                 intel_svm_check(iommu);
2696         }
2697
2698         /*
2699          * Now that qi is enabled on all iommus, set the root entry and flush
2700          * caches. This is required on some Intel X58 chipsets, otherwise the
2701          * flush_context function will loop forever and the boot hangs.
2702          */
2703         for_each_active_iommu(iommu, drhd) {
2704                 iommu_flush_write_buffer(iommu);
2705                 iommu_set_root_entry(iommu);
2706         }
2707
2708         if (!dmar_map_gfx)
2709                 iommu_identity_mapping |= IDENTMAP_GFX;
2710
2711         check_tylersburg_isoch();
2712
2713         ret = si_domain_init(hw_pass_through);
2714         if (ret)
2715                 goto free_iommu;
2716
2717         /*
2718          * for each drhd
2719          *   enable fault log
2720          *   global invalidate context cache
2721          *   global invalidate iotlb
2722          *   enable translation
2723          */
2724         for_each_iommu(iommu, drhd) {
2725                 if (drhd->ignored) {
2726                         /*
2727                          * we always have to disable PMRs or DMA may fail on
2728                          * this device
2729                          */
2730                         if (force_on)
2731                                 iommu_disable_protect_mem_regions(iommu);
2732                         continue;
2733                 }
2734
2735                 iommu_flush_write_buffer(iommu);
2736
2737 #ifdef CONFIG_INTEL_IOMMU_SVM
2738                 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
2739                         /*
2740                          * Call dmar_alloc_hwirq() with dmar_global_lock held,
2741                          * could cause possible lock race condition.
2742                          */
2743                         up_write(&dmar_global_lock);
2744                         ret = intel_svm_enable_prq(iommu);
2745                         down_write(&dmar_global_lock);
2746                         if (ret)
2747                                 goto free_iommu;
2748                 }
2749 #endif
2750                 ret = dmar_set_interrupt(iommu);
2751                 if (ret)
2752                         goto free_iommu;
2753         }
2754
2755         return 0;
2756
2757 free_iommu:
2758         for_each_active_iommu(iommu, drhd) {
2759                 disable_dmar_iommu(iommu);
2760                 free_dmar_iommu(iommu);
2761         }
2762         if (si_domain) {
2763                 domain_exit(si_domain);
2764                 si_domain = NULL;
2765         }
2766
2767         return ret;
2768 }
2769
2770 static void __init init_no_remapping_devices(void)
2771 {
2772         struct dmar_drhd_unit *drhd;
2773         struct device *dev;
2774         int i;
2775
2776         for_each_drhd_unit(drhd) {
2777                 if (!drhd->include_all) {
2778                         for_each_active_dev_scope(drhd->devices,
2779                                                   drhd->devices_cnt, i, dev)
2780                                 break;
2781                         /* ignore DMAR unit if no devices exist */
2782                         if (i == drhd->devices_cnt)
2783                                 drhd->ignored = 1;
2784                 }
2785         }
2786
2787         for_each_active_drhd_unit(drhd) {
2788                 if (drhd->include_all)
2789                         continue;
2790
2791                 for_each_active_dev_scope(drhd->devices,
2792                                           drhd->devices_cnt, i, dev)
2793                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
2794                                 break;
2795                 if (i < drhd->devices_cnt)
2796                         continue;
2797
2798                 /* This IOMMU has *only* gfx devices. Either bypass it or
2799                    set the gfx_mapped flag, as appropriate */
2800                 drhd->gfx_dedicated = 1;
2801                 if (!dmar_map_gfx)
2802                         drhd->ignored = 1;
2803         }
2804 }
2805
2806 #ifdef CONFIG_SUSPEND
2807 static int init_iommu_hw(void)
2808 {
2809         struct dmar_drhd_unit *drhd;
2810         struct intel_iommu *iommu = NULL;
2811         int ret;
2812
2813         for_each_active_iommu(iommu, drhd) {
2814                 if (iommu->qi) {
2815                         ret = dmar_reenable_qi(iommu);
2816                         if (ret)
2817                                 return ret;
2818                 }
2819         }
2820
2821         for_each_iommu(iommu, drhd) {
2822                 if (drhd->ignored) {
2823                         /*
2824                          * we always have to disable PMRs or DMA may fail on
2825                          * this device
2826                          */
2827                         if (force_on)
2828                                 iommu_disable_protect_mem_regions(iommu);
2829                         continue;
2830                 }
2831
2832                 iommu_flush_write_buffer(iommu);
2833                 iommu_set_root_entry(iommu);
2834                 iommu_enable_translation(iommu);
2835                 iommu_disable_protect_mem_regions(iommu);
2836         }
2837
2838         return 0;
2839 }
2840
2841 static void iommu_flush_all(void)
2842 {
2843         struct dmar_drhd_unit *drhd;
2844         struct intel_iommu *iommu;
2845
2846         for_each_active_iommu(iommu, drhd) {
2847                 iommu->flush.flush_context(iommu, 0, 0, 0,
2848                                            DMA_CCMD_GLOBAL_INVL);
2849                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2850                                          DMA_TLB_GLOBAL_FLUSH);
2851         }
2852 }
2853
2854 static int iommu_suspend(void)
2855 {
2856         struct dmar_drhd_unit *drhd;
2857         struct intel_iommu *iommu = NULL;
2858         unsigned long flag;
2859
2860         iommu_flush_all();
2861
2862         for_each_active_iommu(iommu, drhd) {
2863                 iommu_disable_translation(iommu);
2864
2865                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2866
2867                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2868                         readl(iommu->reg + DMAR_FECTL_REG);
2869                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2870                         readl(iommu->reg + DMAR_FEDATA_REG);
2871                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2872                         readl(iommu->reg + DMAR_FEADDR_REG);
2873                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2874                         readl(iommu->reg + DMAR_FEUADDR_REG);
2875
2876                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2877         }
2878         return 0;
2879 }
2880
2881 static void iommu_resume(void)
2882 {
2883         struct dmar_drhd_unit *drhd;
2884         struct intel_iommu *iommu = NULL;
2885         unsigned long flag;
2886
2887         if (init_iommu_hw()) {
2888                 if (force_on)
2889                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
2890                 else
2891                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2892                 return;
2893         }
2894
2895         for_each_active_iommu(iommu, drhd) {
2896
2897                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2898
2899                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2900                         iommu->reg + DMAR_FECTL_REG);
2901                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2902                         iommu->reg + DMAR_FEDATA_REG);
2903                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2904                         iommu->reg + DMAR_FEADDR_REG);
2905                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2906                         iommu->reg + DMAR_FEUADDR_REG);
2907
2908                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2909         }
2910 }
2911
2912 static struct syscore_ops iommu_syscore_ops = {
2913         .resume         = iommu_resume,
2914         .suspend        = iommu_suspend,
2915 };
2916
2917 static void __init init_iommu_pm_ops(void)
2918 {
2919         register_syscore_ops(&iommu_syscore_ops);
2920 }
2921
2922 #else
2923 static inline void init_iommu_pm_ops(void) {}
2924 #endif  /* CONFIG_PM */
2925
2926 static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
2927 {
2928         if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
2929             !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
2930             rmrr->end_address <= rmrr->base_address ||
2931             arch_rmrr_sanity_check(rmrr))
2932                 return -EINVAL;
2933
2934         return 0;
2935 }
2936
2937 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
2938 {
2939         struct acpi_dmar_reserved_memory *rmrr;
2940         struct dmar_rmrr_unit *rmrru;
2941
2942         rmrr = (struct acpi_dmar_reserved_memory *)header;
2943         if (rmrr_sanity_check(rmrr)) {
2944                 pr_warn(FW_BUG
2945                            "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
2946                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2947                            rmrr->base_address, rmrr->end_address,
2948                            dmi_get_system_info(DMI_BIOS_VENDOR),
2949                            dmi_get_system_info(DMI_BIOS_VERSION),
2950                            dmi_get_system_info(DMI_PRODUCT_VERSION));
2951                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
2952         }
2953
2954         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
2955         if (!rmrru)
2956                 goto out;
2957
2958         rmrru->hdr = header;
2959
2960         rmrru->base_address = rmrr->base_address;
2961         rmrru->end_address = rmrr->end_address;
2962
2963         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
2964                                 ((void *)rmrr) + rmrr->header.length,
2965                                 &rmrru->devices_cnt);
2966         if (rmrru->devices_cnt && rmrru->devices == NULL)
2967                 goto free_rmrru;
2968
2969         list_add(&rmrru->list, &dmar_rmrr_units);
2970
2971         return 0;
2972 free_rmrru:
2973         kfree(rmrru);
2974 out:
2975         return -ENOMEM;
2976 }
2977
2978 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
2979 {
2980         struct dmar_atsr_unit *atsru;
2981         struct acpi_dmar_atsr *tmp;
2982
2983         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
2984                                 dmar_rcu_check()) {
2985                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
2986                 if (atsr->segment != tmp->segment)
2987                         continue;
2988                 if (atsr->header.length != tmp->header.length)
2989                         continue;
2990                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
2991                         return atsru;
2992         }
2993
2994         return NULL;
2995 }
2996
2997 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2998 {
2999         struct acpi_dmar_atsr *atsr;
3000         struct dmar_atsr_unit *atsru;
3001
3002         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3003                 return 0;
3004
3005         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3006         atsru = dmar_find_atsr(atsr);
3007         if (atsru)
3008                 return 0;
3009
3010         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3011         if (!atsru)
3012                 return -ENOMEM;
3013
3014         /*
3015          * If memory is allocated from slab by ACPI _DSM method, we need to
3016          * copy the memory content because the memory buffer will be freed
3017          * on return.
3018          */
3019         atsru->hdr = (void *)(atsru + 1);
3020         memcpy(atsru->hdr, hdr, hdr->length);
3021         atsru->include_all = atsr->flags & 0x1;
3022         if (!atsru->include_all) {
3023                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3024                                 (void *)atsr + atsr->header.length,
3025                                 &atsru->devices_cnt);
3026                 if (atsru->devices_cnt && atsru->devices == NULL) {
3027                         kfree(atsru);
3028                         return -ENOMEM;
3029                 }
3030         }
3031
3032         list_add_rcu(&atsru->list, &dmar_atsr_units);
3033
3034         return 0;
3035 }
3036
3037 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3038 {
3039         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3040         kfree(atsru);
3041 }
3042
3043 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3044 {
3045         struct acpi_dmar_atsr *atsr;
3046         struct dmar_atsr_unit *atsru;
3047
3048         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3049         atsru = dmar_find_atsr(atsr);
3050         if (atsru) {
3051                 list_del_rcu(&atsru->list);
3052                 synchronize_rcu();
3053                 intel_iommu_free_atsr(atsru);
3054         }
3055
3056         return 0;
3057 }
3058
3059 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3060 {
3061         int i;
3062         struct device *dev;
3063         struct acpi_dmar_atsr *atsr;
3064         struct dmar_atsr_unit *atsru;
3065
3066         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3067         atsru = dmar_find_atsr(atsr);
3068         if (!atsru)
3069                 return 0;
3070
3071         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
3072                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3073                                           i, dev)
3074                         return -EBUSY;
3075         }
3076
3077         return 0;
3078 }
3079
3080 static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc)
3081 {
3082         struct dmar_satc_unit *satcu;
3083         struct acpi_dmar_satc *tmp;
3084
3085         list_for_each_entry_rcu(satcu, &dmar_satc_units, list,
3086                                 dmar_rcu_check()) {
3087                 tmp = (struct acpi_dmar_satc *)satcu->hdr;
3088                 if (satc->segment != tmp->segment)
3089                         continue;
3090                 if (satc->header.length != tmp->header.length)
3091                         continue;
3092                 if (memcmp(satc, tmp, satc->header.length) == 0)
3093                         return satcu;
3094         }
3095
3096         return NULL;
3097 }
3098
3099 int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg)
3100 {
3101         struct acpi_dmar_satc *satc;
3102         struct dmar_satc_unit *satcu;
3103
3104         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3105                 return 0;
3106
3107         satc = container_of(hdr, struct acpi_dmar_satc, header);
3108         satcu = dmar_find_satc(satc);
3109         if (satcu)
3110                 return 0;
3111
3112         satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
3113         if (!satcu)
3114                 return -ENOMEM;
3115
3116         satcu->hdr = (void *)(satcu + 1);
3117         memcpy(satcu->hdr, hdr, hdr->length);
3118         satcu->atc_required = satc->flags & 0x1;
3119         satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
3120                                               (void *)satc + satc->header.length,
3121                                               &satcu->devices_cnt);
3122         if (satcu->devices_cnt && !satcu->devices) {
3123                 kfree(satcu);
3124                 return -ENOMEM;
3125         }
3126         list_add_rcu(&satcu->list, &dmar_satc_units);
3127
3128         return 0;
3129 }
3130
3131 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3132 {
3133         int sp, ret;
3134         struct intel_iommu *iommu = dmaru->iommu;
3135
3136         ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
3137         if (ret)
3138                 goto out;
3139
3140         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3141                 pr_warn("%s: Doesn't support hardware pass through.\n",
3142                         iommu->name);
3143                 return -ENXIO;
3144         }
3145
3146         sp = domain_update_iommu_superpage(NULL, iommu) - 1;
3147         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3148                 pr_warn("%s: Doesn't support large page.\n",
3149                         iommu->name);
3150                 return -ENXIO;
3151         }
3152
3153         /*
3154          * Disable translation if already enabled prior to OS handover.
3155          */
3156         if (iommu->gcmd & DMA_GCMD_TE)
3157                 iommu_disable_translation(iommu);
3158
3159         ret = iommu_init_domains(iommu);
3160         if (ret == 0)
3161                 ret = iommu_alloc_root_entry(iommu);
3162         if (ret)
3163                 goto out;
3164
3165         intel_svm_check(iommu);
3166
3167         if (dmaru->ignored) {
3168                 /*
3169                  * we always have to disable PMRs or DMA may fail on this device
3170                  */
3171                 if (force_on)
3172                         iommu_disable_protect_mem_regions(iommu);
3173                 return 0;
3174         }
3175
3176         intel_iommu_init_qi(iommu);
3177         iommu_flush_write_buffer(iommu);
3178
3179 #ifdef CONFIG_INTEL_IOMMU_SVM
3180         if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3181                 ret = intel_svm_enable_prq(iommu);
3182                 if (ret)
3183                         goto disable_iommu;
3184         }
3185 #endif
3186         ret = dmar_set_interrupt(iommu);
3187         if (ret)
3188                 goto disable_iommu;
3189
3190         iommu_set_root_entry(iommu);
3191         iommu_enable_translation(iommu);
3192
3193         iommu_disable_protect_mem_regions(iommu);
3194         return 0;
3195
3196 disable_iommu:
3197         disable_dmar_iommu(iommu);
3198 out:
3199         free_dmar_iommu(iommu);
3200         return ret;
3201 }
3202
3203 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3204 {
3205         int ret = 0;
3206         struct intel_iommu *iommu = dmaru->iommu;
3207
3208         if (!intel_iommu_enabled)
3209                 return 0;
3210         if (iommu == NULL)
3211                 return -EINVAL;
3212
3213         if (insert) {
3214                 ret = intel_iommu_add(dmaru);
3215         } else {
3216                 disable_dmar_iommu(iommu);
3217                 free_dmar_iommu(iommu);
3218         }
3219
3220         return ret;
3221 }
3222
3223 static void intel_iommu_free_dmars(void)
3224 {
3225         struct dmar_rmrr_unit *rmrru, *rmrr_n;
3226         struct dmar_atsr_unit *atsru, *atsr_n;
3227         struct dmar_satc_unit *satcu, *satc_n;
3228
3229         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3230                 list_del(&rmrru->list);
3231                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3232                 kfree(rmrru);
3233         }
3234
3235         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3236                 list_del(&atsru->list);
3237                 intel_iommu_free_atsr(atsru);
3238         }
3239         list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) {
3240                 list_del(&satcu->list);
3241                 dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
3242                 kfree(satcu);
3243         }
3244 }
3245
3246 static struct dmar_satc_unit *dmar_find_matched_satc_unit(struct pci_dev *dev)
3247 {
3248         struct dmar_satc_unit *satcu;
3249         struct acpi_dmar_satc *satc;
3250         struct device *tmp;
3251         int i;
3252
3253         dev = pci_physfn(dev);
3254         rcu_read_lock();
3255
3256         list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
3257                 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3258                 if (satc->segment != pci_domain_nr(dev->bus))
3259                         continue;
3260                 for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
3261                         if (to_pci_dev(tmp) == dev)
3262                                 goto out;
3263         }
3264         satcu = NULL;
3265 out:
3266         rcu_read_unlock();
3267         return satcu;
3268 }
3269
3270 static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)
3271 {
3272         int i, ret = 1;
3273         struct pci_bus *bus;
3274         struct pci_dev *bridge = NULL;
3275         struct device *tmp;
3276         struct acpi_dmar_atsr *atsr;
3277         struct dmar_atsr_unit *atsru;
3278         struct dmar_satc_unit *satcu;
3279
3280         dev = pci_physfn(dev);
3281         satcu = dmar_find_matched_satc_unit(dev);
3282         if (satcu)
3283                 /*
3284                  * This device supports ATS as it is in SATC table.
3285                  * When IOMMU is in legacy mode, enabling ATS is done
3286                  * automatically by HW for the device that requires
3287                  * ATS, hence OS should not enable this device ATS
3288                  * to avoid duplicated TLB invalidation.
3289                  */
3290                 return !(satcu->atc_required && !sm_supported(iommu));
3291
3292         for (bus = dev->bus; bus; bus = bus->parent) {
3293                 bridge = bus->self;
3294                 /* If it's an integrated device, allow ATS */
3295                 if (!bridge)
3296                         return 1;
3297                 /* Connected via non-PCIe: no ATS */
3298                 if (!pci_is_pcie(bridge) ||
3299                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3300                         return 0;
3301                 /* If we found the root port, look it up in the ATSR */
3302                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3303                         break;
3304         }
3305
3306         rcu_read_lock();
3307         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3308                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3309                 if (atsr->segment != pci_domain_nr(dev->bus))
3310                         continue;
3311
3312                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3313                         if (tmp == &bridge->dev)
3314                                 goto out;
3315
3316                 if (atsru->include_all)
3317                         goto out;
3318         }
3319         ret = 0;
3320 out:
3321         rcu_read_unlock();
3322
3323         return ret;
3324 }
3325
3326 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3327 {
3328         int ret;
3329         struct dmar_rmrr_unit *rmrru;
3330         struct dmar_atsr_unit *atsru;
3331         struct dmar_satc_unit *satcu;
3332         struct acpi_dmar_atsr *atsr;
3333         struct acpi_dmar_reserved_memory *rmrr;
3334         struct acpi_dmar_satc *satc;
3335
3336         if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
3337                 return 0;
3338
3339         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3340                 rmrr = container_of(rmrru->hdr,
3341                                     struct acpi_dmar_reserved_memory, header);
3342                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3343                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3344                                 ((void *)rmrr) + rmrr->header.length,
3345                                 rmrr->segment, rmrru->devices,
3346                                 rmrru->devices_cnt);
3347                         if (ret < 0)
3348                                 return ret;
3349                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3350                         dmar_remove_dev_scope(info, rmrr->segment,
3351                                 rmrru->devices, rmrru->devices_cnt);
3352                 }
3353         }
3354
3355         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3356                 if (atsru->include_all)
3357                         continue;
3358
3359                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3360                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3361                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3362                                         (void *)atsr + atsr->header.length,
3363                                         atsr->segment, atsru->devices,
3364                                         atsru->devices_cnt);
3365                         if (ret > 0)
3366                                 break;
3367                         else if (ret < 0)
3368                                 return ret;
3369                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3370                         if (dmar_remove_dev_scope(info, atsr->segment,
3371                                         atsru->devices, atsru->devices_cnt))
3372                                 break;
3373                 }
3374         }
3375         list_for_each_entry(satcu, &dmar_satc_units, list) {
3376                 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3377                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3378                         ret = dmar_insert_dev_scope(info, (void *)(satc + 1),
3379                                         (void *)satc + satc->header.length,
3380                                         satc->segment, satcu->devices,
3381                                         satcu->devices_cnt);
3382                         if (ret > 0)
3383                                 break;
3384                         else if (ret < 0)
3385                                 return ret;
3386                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3387                         if (dmar_remove_dev_scope(info, satc->segment,
3388                                         satcu->devices, satcu->devices_cnt))
3389                                 break;
3390                 }
3391         }
3392
3393         return 0;
3394 }
3395
3396 static int intel_iommu_memory_notifier(struct notifier_block *nb,
3397                                        unsigned long val, void *v)
3398 {
3399         struct memory_notify *mhp = v;
3400         unsigned long start_vpfn = mm_to_dma_pfn_start(mhp->start_pfn);
3401         unsigned long last_vpfn = mm_to_dma_pfn_end(mhp->start_pfn +
3402                         mhp->nr_pages - 1);
3403
3404         switch (val) {
3405         case MEM_GOING_ONLINE:
3406                 if (iommu_domain_identity_map(si_domain,
3407                                               start_vpfn, last_vpfn)) {
3408                         pr_warn("Failed to build identity map for [%lx-%lx]\n",
3409                                 start_vpfn, last_vpfn);
3410                         return NOTIFY_BAD;
3411                 }
3412                 break;
3413
3414         case MEM_OFFLINE:
3415         case MEM_CANCEL_ONLINE:
3416                 {
3417                         struct dmar_drhd_unit *drhd;
3418                         struct intel_iommu *iommu;
3419                         LIST_HEAD(freelist);
3420
3421                         domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
3422
3423                         rcu_read_lock();
3424                         for_each_active_iommu(iommu, drhd)
3425                                 iommu_flush_iotlb_psi(iommu, si_domain,
3426                                         start_vpfn, mhp->nr_pages,
3427                                         list_empty(&freelist), 0);
3428                         rcu_read_unlock();
3429                         put_pages_list(&freelist);
3430                 }
3431                 break;
3432         }
3433
3434         return NOTIFY_OK;
3435 }
3436
3437 static struct notifier_block intel_iommu_memory_nb = {
3438         .notifier_call = intel_iommu_memory_notifier,
3439         .priority = 0
3440 };
3441
3442 static void intel_disable_iommus(void)
3443 {
3444         struct intel_iommu *iommu = NULL;
3445         struct dmar_drhd_unit *drhd;
3446
3447         for_each_iommu(iommu, drhd)
3448                 iommu_disable_translation(iommu);
3449 }
3450
3451 void intel_iommu_shutdown(void)
3452 {
3453         struct dmar_drhd_unit *drhd;
3454         struct intel_iommu *iommu = NULL;
3455
3456         if (no_iommu || dmar_disabled)
3457                 return;
3458
3459         down_write(&dmar_global_lock);
3460
3461         /* Disable PMRs explicitly here. */
3462         for_each_iommu(iommu, drhd)
3463                 iommu_disable_protect_mem_regions(iommu);
3464
3465         /* Make sure the IOMMUs are switched off */
3466         intel_disable_iommus();
3467
3468         up_write(&dmar_global_lock);
3469 }
3470
3471 static struct intel_iommu *dev_to_intel_iommu(struct device *dev)
3472 {
3473         struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
3474
3475         return container_of(iommu_dev, struct intel_iommu, iommu);
3476 }
3477
3478 static ssize_t version_show(struct device *dev,
3479                             struct device_attribute *attr, char *buf)
3480 {
3481         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3482         u32 ver = readl(iommu->reg + DMAR_VER_REG);
3483         return sysfs_emit(buf, "%d:%d\n",
3484                           DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3485 }
3486 static DEVICE_ATTR_RO(version);
3487
3488 static ssize_t address_show(struct device *dev,
3489                             struct device_attribute *attr, char *buf)
3490 {
3491         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3492         return sysfs_emit(buf, "%llx\n", iommu->reg_phys);
3493 }
3494 static DEVICE_ATTR_RO(address);
3495
3496 static ssize_t cap_show(struct device *dev,
3497                         struct device_attribute *attr, char *buf)
3498 {
3499         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3500         return sysfs_emit(buf, "%llx\n", iommu->cap);
3501 }
3502 static DEVICE_ATTR_RO(cap);
3503
3504 static ssize_t ecap_show(struct device *dev,
3505                          struct device_attribute *attr, char *buf)
3506 {
3507         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3508         return sysfs_emit(buf, "%llx\n", iommu->ecap);
3509 }
3510 static DEVICE_ATTR_RO(ecap);
3511
3512 static ssize_t domains_supported_show(struct device *dev,
3513                                       struct device_attribute *attr, char *buf)
3514 {
3515         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3516         return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));
3517 }
3518 static DEVICE_ATTR_RO(domains_supported);
3519
3520 static ssize_t domains_used_show(struct device *dev,
3521                                  struct device_attribute *attr, char *buf)
3522 {
3523         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3524         return sysfs_emit(buf, "%d\n",
3525                           bitmap_weight(iommu->domain_ids,
3526                                         cap_ndoms(iommu->cap)));
3527 }
3528 static DEVICE_ATTR_RO(domains_used);
3529
3530 static struct attribute *intel_iommu_attrs[] = {
3531         &dev_attr_version.attr,
3532         &dev_attr_address.attr,
3533         &dev_attr_cap.attr,
3534         &dev_attr_ecap.attr,
3535         &dev_attr_domains_supported.attr,
3536         &dev_attr_domains_used.attr,
3537         NULL,
3538 };
3539
3540 static struct attribute_group intel_iommu_group = {
3541         .name = "intel-iommu",
3542         .attrs = intel_iommu_attrs,
3543 };
3544
3545 const struct attribute_group *intel_iommu_groups[] = {
3546         &intel_iommu_group,
3547         NULL,
3548 };
3549
3550 static bool has_external_pci(void)
3551 {
3552         struct pci_dev *pdev = NULL;
3553
3554         for_each_pci_dev(pdev)
3555                 if (pdev->external_facing) {
3556                         pci_dev_put(pdev);
3557                         return true;
3558                 }
3559
3560         return false;
3561 }
3562
3563 static int __init platform_optin_force_iommu(void)
3564 {
3565         if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
3566                 return 0;
3567
3568         if (no_iommu || dmar_disabled)
3569                 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
3570
3571         /*
3572          * If Intel-IOMMU is disabled by default, we will apply identity
3573          * map for all devices except those marked as being untrusted.
3574          */
3575         if (dmar_disabled)
3576                 iommu_set_default_passthrough(false);
3577
3578         dmar_disabled = 0;
3579         no_iommu = 0;
3580
3581         return 1;
3582 }
3583
3584 static int __init probe_acpi_namespace_devices(void)
3585 {
3586         struct dmar_drhd_unit *drhd;
3587         /* To avoid a -Wunused-but-set-variable warning. */
3588         struct intel_iommu *iommu __maybe_unused;
3589         struct device *dev;
3590         int i, ret = 0;
3591
3592         for_each_active_iommu(iommu, drhd) {
3593                 for_each_active_dev_scope(drhd->devices,
3594                                           drhd->devices_cnt, i, dev) {
3595                         struct acpi_device_physical_node *pn;
3596                         struct acpi_device *adev;
3597
3598                         if (dev->bus != &acpi_bus_type)
3599                                 continue;
3600
3601                         adev = to_acpi_device(dev);
3602                         mutex_lock(&adev->physical_node_lock);
3603                         list_for_each_entry(pn,
3604                                             &adev->physical_node_list, node) {
3605                                 ret = iommu_probe_device(pn->dev);
3606                                 if (ret)
3607                                         break;
3608                         }
3609                         mutex_unlock(&adev->physical_node_lock);
3610
3611                         if (ret)
3612                                 return ret;
3613                 }
3614         }
3615
3616         return 0;
3617 }
3618
3619 static __init int tboot_force_iommu(void)
3620 {
3621         if (!tboot_enabled())
3622                 return 0;
3623
3624         if (no_iommu || dmar_disabled)
3625                 pr_warn("Forcing Intel-IOMMU to enabled\n");
3626
3627         dmar_disabled = 0;
3628         no_iommu = 0;
3629
3630         return 1;
3631 }
3632
3633 int __init intel_iommu_init(void)
3634 {
3635         int ret = -ENODEV;
3636         struct dmar_drhd_unit *drhd;
3637         struct intel_iommu *iommu;
3638
3639         /*
3640          * Intel IOMMU is required for a TXT/tboot launch or platform
3641          * opt in, so enforce that.
3642          */
3643         force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
3644                     platform_optin_force_iommu();
3645
3646         down_write(&dmar_global_lock);
3647         if (dmar_table_init()) {
3648                 if (force_on)
3649                         panic("tboot: Failed to initialize DMAR table\n");
3650                 goto out_free_dmar;
3651         }
3652
3653         if (dmar_dev_scope_init() < 0) {
3654                 if (force_on)
3655                         panic("tboot: Failed to initialize DMAR device scope\n");
3656                 goto out_free_dmar;
3657         }
3658
3659         up_write(&dmar_global_lock);
3660
3661         /*
3662          * The bus notifier takes the dmar_global_lock, so lockdep will
3663          * complain later when we register it under the lock.
3664          */
3665         dmar_register_bus_notifier();
3666
3667         down_write(&dmar_global_lock);
3668
3669         if (!no_iommu)
3670                 intel_iommu_debugfs_init();
3671
3672         if (no_iommu || dmar_disabled) {
3673                 /*
3674                  * We exit the function here to ensure IOMMU's remapping and
3675                  * mempool aren't setup, which means that the IOMMU's PMRs
3676                  * won't be disabled via the call to init_dmars(). So disable
3677                  * it explicitly here. The PMRs were setup by tboot prior to
3678                  * calling SENTER, but the kernel is expected to reset/tear
3679                  * down the PMRs.
3680                  */
3681                 if (intel_iommu_tboot_noforce) {
3682                         for_each_iommu(iommu, drhd)
3683                                 iommu_disable_protect_mem_regions(iommu);
3684                 }
3685
3686                 /*
3687                  * Make sure the IOMMUs are switched off, even when we
3688                  * boot into a kexec kernel and the previous kernel left
3689                  * them enabled
3690                  */
3691                 intel_disable_iommus();
3692                 goto out_free_dmar;
3693         }
3694
3695         if (list_empty(&dmar_rmrr_units))
3696                 pr_info("No RMRR found\n");
3697
3698         if (list_empty(&dmar_atsr_units))
3699                 pr_info("No ATSR found\n");
3700
3701         if (list_empty(&dmar_satc_units))
3702                 pr_info("No SATC found\n");
3703
3704         init_no_remapping_devices();
3705
3706         ret = init_dmars();
3707         if (ret) {
3708                 if (force_on)
3709                         panic("tboot: Failed to initialize DMARs\n");
3710                 pr_err("Initialization failed\n");
3711                 goto out_free_dmar;
3712         }
3713         up_write(&dmar_global_lock);
3714
3715         init_iommu_pm_ops();
3716
3717         down_read(&dmar_global_lock);
3718         for_each_active_iommu(iommu, drhd) {
3719                 /*
3720                  * The flush queue implementation does not perform
3721                  * page-selective invalidations that are required for efficient
3722                  * TLB flushes in virtual environments.  The benefit of batching
3723                  * is likely to be much lower than the overhead of synchronizing
3724                  * the virtual and physical IOMMU page-tables.
3725                  */
3726                 if (cap_caching_mode(iommu->cap) &&
3727                     !first_level_by_default(IOMMU_DOMAIN_DMA)) {
3728                         pr_info_once("IOMMU batching disallowed due to virtualization\n");
3729                         iommu_set_dma_strict();
3730                 }
3731                 iommu_device_sysfs_add(&iommu->iommu, NULL,
3732                                        intel_iommu_groups,
3733                                        "%s", iommu->name);
3734                 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
3735
3736                 iommu_pmu_register(iommu);
3737         }
3738         up_read(&dmar_global_lock);
3739
3740         if (si_domain && !hw_pass_through)
3741                 register_memory_notifier(&intel_iommu_memory_nb);
3742
3743         down_read(&dmar_global_lock);
3744         if (probe_acpi_namespace_devices())
3745                 pr_warn("ACPI name space devices didn't probe correctly\n");
3746
3747         /* Finally, we enable the DMA remapping hardware. */
3748         for_each_iommu(iommu, drhd) {
3749                 if (!drhd->ignored && !translation_pre_enabled(iommu))
3750                         iommu_enable_translation(iommu);
3751
3752                 iommu_disable_protect_mem_regions(iommu);
3753         }
3754         up_read(&dmar_global_lock);
3755
3756         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
3757
3758         intel_iommu_enabled = 1;
3759
3760         return 0;
3761
3762 out_free_dmar:
3763         intel_iommu_free_dmars();
3764         up_write(&dmar_global_lock);
3765         return ret;
3766 }
3767
3768 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
3769 {
3770         struct device_domain_info *info = opaque;
3771
3772         domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);
3773         return 0;
3774 }
3775
3776 /*
3777  * NB - intel-iommu lacks any sort of reference counting for the users of
3778  * dependent devices.  If multiple endpoints have intersecting dependent
3779  * devices, unbinding the driver from any one of them will possibly leave
3780  * the others unable to operate.
3781  */
3782 static void domain_context_clear(struct device_domain_info *info)
3783 {
3784         if (!dev_is_pci(info->dev))
3785                 domain_context_clear_one(info, info->bus, info->devfn);
3786
3787         pci_for_each_dma_alias(to_pci_dev(info->dev),
3788                                &domain_context_clear_one_cb, info);
3789 }
3790
3791 /*
3792  * Clear the page table pointer in context or pasid table entries so that
3793  * all DMA requests without PASID from the device are blocked. If the page
3794  * table has been set, clean up the data structures.
3795  */
3796 void device_block_translation(struct device *dev)
3797 {
3798         struct device_domain_info *info = dev_iommu_priv_get(dev);
3799         struct intel_iommu *iommu = info->iommu;
3800         unsigned long flags;
3801
3802         iommu_disable_pci_caps(info);
3803         if (!dev_is_real_dma_subdevice(dev)) {
3804                 if (sm_supported(iommu))
3805                         intel_pasid_tear_down_entry(iommu, dev,
3806                                                     IOMMU_NO_PASID, false);
3807                 else
3808                         domain_context_clear(info);
3809         }
3810
3811         if (!info->domain)
3812                 return;
3813
3814         spin_lock_irqsave(&info->domain->lock, flags);
3815         list_del(&info->link);
3816         spin_unlock_irqrestore(&info->domain->lock, flags);
3817
3818         domain_detach_iommu(info->domain, iommu);
3819         info->domain = NULL;
3820 }
3821
3822 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3823 {
3824         int adjust_width;
3825
3826         /* calculate AGAW */
3827         domain->gaw = guest_width;
3828         adjust_width = guestwidth_to_adjustwidth(guest_width);
3829         domain->agaw = width_to_agaw(adjust_width);
3830
3831         domain->iommu_coherency = false;
3832         domain->iommu_superpage = 0;
3833         domain->max_addr = 0;
3834
3835         /* always allocate the top pgd */
3836         domain->pgd = alloc_pgtable_page(domain->nid, GFP_ATOMIC);
3837         if (!domain->pgd)
3838                 return -ENOMEM;
3839         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3840         return 0;
3841 }
3842
3843 static int blocking_domain_attach_dev(struct iommu_domain *domain,
3844                                       struct device *dev)
3845 {
3846         device_block_translation(dev);
3847         return 0;
3848 }
3849
3850 static struct iommu_domain blocking_domain = {
3851         .type = IOMMU_DOMAIN_BLOCKED,
3852         .ops = &(const struct iommu_domain_ops) {
3853                 .attach_dev     = blocking_domain_attach_dev,
3854         }
3855 };
3856
3857 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
3858 {
3859         struct dmar_domain *dmar_domain;
3860         struct iommu_domain *domain;
3861
3862         switch (type) {
3863         case IOMMU_DOMAIN_DMA:
3864         case IOMMU_DOMAIN_UNMANAGED:
3865                 dmar_domain = alloc_domain(type);
3866                 if (!dmar_domain) {
3867                         pr_err("Can't allocate dmar_domain\n");
3868                         return NULL;
3869                 }
3870                 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3871                         pr_err("Domain initialization failed\n");
3872                         domain_exit(dmar_domain);
3873                         return NULL;
3874                 }
3875
3876                 domain = &dmar_domain->domain;
3877                 domain->geometry.aperture_start = 0;
3878                 domain->geometry.aperture_end   =
3879                                 __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3880                 domain->geometry.force_aperture = true;
3881
3882                 return domain;
3883         case IOMMU_DOMAIN_IDENTITY:
3884                 return &si_domain->domain;
3885         case IOMMU_DOMAIN_SVA:
3886                 return intel_svm_domain_alloc();
3887         default:
3888                 return NULL;
3889         }
3890
3891         return NULL;
3892 }
3893
3894 static struct iommu_domain *
3895 intel_iommu_domain_alloc_user(struct device *dev, u32 flags,
3896                               struct iommu_domain *parent,
3897                               const struct iommu_user_data *user_data)
3898 {
3899         struct device_domain_info *info = dev_iommu_priv_get(dev);
3900         bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
3901         bool nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
3902         struct intel_iommu *iommu = info->iommu;
3903         struct dmar_domain *dmar_domain;
3904         struct iommu_domain *domain;
3905
3906         /* Must be NESTING domain */
3907         if (parent) {
3908                 if (!nested_supported(iommu) || flags)
3909                         return ERR_PTR(-EOPNOTSUPP);
3910                 return intel_nested_domain_alloc(parent, user_data);
3911         }
3912
3913         if (flags &
3914             (~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING)))
3915                 return ERR_PTR(-EOPNOTSUPP);
3916         if (nested_parent && !nested_supported(iommu))
3917                 return ERR_PTR(-EOPNOTSUPP);
3918         if (user_data || (dirty_tracking && !ssads_supported(iommu)))
3919                 return ERR_PTR(-EOPNOTSUPP);
3920
3921         /*
3922          * domain_alloc_user op needs to fully initialize a domain before
3923          * return, so uses iommu_domain_alloc() here for simple.
3924          */
3925         domain = iommu_domain_alloc(dev->bus);
3926         if (!domain)
3927                 return ERR_PTR(-ENOMEM);
3928
3929         dmar_domain = to_dmar_domain(domain);
3930
3931         if (nested_parent) {
3932                 dmar_domain->nested_parent = true;
3933                 INIT_LIST_HEAD(&dmar_domain->s1_domains);
3934                 spin_lock_init(&dmar_domain->s1_lock);
3935         }
3936
3937         if (dirty_tracking) {
3938                 if (dmar_domain->use_first_level) {
3939                         iommu_domain_free(domain);
3940                         return ERR_PTR(-EOPNOTSUPP);
3941                 }
3942                 domain->dirty_ops = &intel_dirty_ops;
3943         }
3944
3945         return domain;
3946 }
3947
3948 static void intel_iommu_domain_free(struct iommu_domain *domain)
3949 {
3950         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3951
3952         WARN_ON(dmar_domain->nested_parent &&
3953                 !list_empty(&dmar_domain->s1_domains));
3954         if (domain != &si_domain->domain)
3955                 domain_exit(dmar_domain);
3956 }
3957
3958 int prepare_domain_attach_device(struct iommu_domain *domain,
3959                                  struct device *dev)
3960 {
3961         struct device_domain_info *info = dev_iommu_priv_get(dev);
3962         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3963         struct intel_iommu *iommu = info->iommu;
3964         int addr_width;
3965
3966         if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
3967                 return -EINVAL;
3968
3969         if (domain->dirty_ops && !ssads_supported(iommu))
3970                 return -EINVAL;
3971
3972         /* check if this iommu agaw is sufficient for max mapped address */
3973         addr_width = agaw_to_width(iommu->agaw);
3974         if (addr_width > cap_mgaw(iommu->cap))
3975                 addr_width = cap_mgaw(iommu->cap);
3976
3977         if (dmar_domain->max_addr > (1LL << addr_width))
3978                 return -EINVAL;
3979         dmar_domain->gaw = addr_width;
3980
3981         /*
3982          * Knock out extra levels of page tables if necessary
3983          */
3984         while (iommu->agaw < dmar_domain->agaw) {
3985                 struct dma_pte *pte;
3986
3987                 pte = dmar_domain->pgd;
3988                 if (dma_pte_present(pte)) {
3989                         dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte));
3990                         free_pgtable_page(pte);
3991                 }
3992                 dmar_domain->agaw--;
3993         }
3994
3995         if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
3996             context_copied(iommu, info->bus, info->devfn))
3997                 return intel_pasid_setup_sm_context(dev);
3998
3999         return 0;
4000 }
4001
4002 static int intel_iommu_attach_device(struct iommu_domain *domain,
4003                                      struct device *dev)
4004 {
4005         struct device_domain_info *info = dev_iommu_priv_get(dev);
4006         int ret;
4007
4008         if (info->domain)
4009                 device_block_translation(dev);
4010
4011         ret = prepare_domain_attach_device(domain, dev);
4012         if (ret)
4013                 return ret;
4014
4015         return dmar_domain_attach_device(to_dmar_domain(domain), dev);
4016 }
4017
4018 static int intel_iommu_map(struct iommu_domain *domain,
4019                            unsigned long iova, phys_addr_t hpa,
4020                            size_t size, int iommu_prot, gfp_t gfp)
4021 {
4022         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4023         u64 max_addr;
4024         int prot = 0;
4025
4026         if (iommu_prot & IOMMU_READ)
4027                 prot |= DMA_PTE_READ;
4028         if (iommu_prot & IOMMU_WRITE)
4029                 prot |= DMA_PTE_WRITE;
4030         if (dmar_domain->set_pte_snp)
4031                 prot |= DMA_PTE_SNP;
4032
4033         max_addr = iova + size;
4034         if (dmar_domain->max_addr < max_addr) {
4035                 u64 end;
4036
4037                 /* check if minimum agaw is sufficient for mapped address */
4038                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4039                 if (end < max_addr) {
4040                         pr_err("%s: iommu width (%d) is not "
4041                                "sufficient for the mapped address (%llx)\n",
4042                                __func__, dmar_domain->gaw, max_addr);
4043                         return -EFAULT;
4044                 }
4045                 dmar_domain->max_addr = max_addr;
4046         }
4047         /* Round up size to next multiple of PAGE_SIZE, if it and
4048            the low bits of hpa would take us onto the next page */
4049         size = aligned_nrpages(hpa, size);
4050         return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4051                                 hpa >> VTD_PAGE_SHIFT, size, prot, gfp);
4052 }
4053
4054 static int intel_iommu_map_pages(struct iommu_domain *domain,
4055                                  unsigned long iova, phys_addr_t paddr,
4056                                  size_t pgsize, size_t pgcount,
4057                                  int prot, gfp_t gfp, size_t *mapped)
4058 {
4059         unsigned long pgshift = __ffs(pgsize);
4060         size_t size = pgcount << pgshift;
4061         int ret;
4062
4063         if (pgsize != SZ_4K && pgsize != SZ_2M && pgsize != SZ_1G)
4064                 return -EINVAL;
4065
4066         if (!IS_ALIGNED(iova | paddr, pgsize))
4067                 return -EINVAL;
4068
4069         ret = intel_iommu_map(domain, iova, paddr, size, prot, gfp);
4070         if (!ret && mapped)
4071                 *mapped = size;
4072
4073         return ret;
4074 }
4075
4076 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4077                                 unsigned long iova, size_t size,
4078                                 struct iommu_iotlb_gather *gather)
4079 {
4080         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4081         unsigned long start_pfn, last_pfn;
4082         int level = 0;
4083
4084         /* Cope with horrid API which requires us to unmap more than the
4085            size argument if it happens to be a large-page mapping. */
4086         if (unlikely(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT,
4087                                      &level, GFP_ATOMIC)))
4088                 return 0;
4089
4090         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4091                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4092
4093         start_pfn = iova >> VTD_PAGE_SHIFT;
4094         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4095
4096         domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
4097
4098         if (dmar_domain->max_addr == iova + size)
4099                 dmar_domain->max_addr = iova;
4100
4101         /*
4102          * We do not use page-selective IOTLB invalidation in flush queue,
4103          * so there is no need to track page and sync iotlb.
4104          */
4105         if (!iommu_iotlb_gather_queued(gather))
4106                 iommu_iotlb_gather_add_page(domain, gather, iova, size);
4107
4108         return size;
4109 }
4110
4111 static size_t intel_iommu_unmap_pages(struct iommu_domain *domain,
4112                                       unsigned long iova,
4113                                       size_t pgsize, size_t pgcount,
4114                                       struct iommu_iotlb_gather *gather)
4115 {
4116         unsigned long pgshift = __ffs(pgsize);
4117         size_t size = pgcount << pgshift;
4118
4119         return intel_iommu_unmap(domain, iova, size, gather);
4120 }
4121
4122 static void intel_iommu_tlb_sync(struct iommu_domain *domain,
4123                                  struct iommu_iotlb_gather *gather)
4124 {
4125         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4126         unsigned long iova_pfn = IOVA_PFN(gather->start);
4127         size_t size = gather->end - gather->start;
4128         struct iommu_domain_info *info;
4129         unsigned long start_pfn;
4130         unsigned long nrpages;
4131         unsigned long i;
4132
4133         nrpages = aligned_nrpages(gather->start, size);
4134         start_pfn = mm_to_dma_pfn_start(iova_pfn);
4135
4136         xa_for_each(&dmar_domain->iommu_array, i, info)
4137                 iommu_flush_iotlb_psi(info->iommu, dmar_domain,
4138                                       start_pfn, nrpages,
4139                                       list_empty(&gather->freelist), 0);
4140
4141         if (dmar_domain->nested_parent)
4142                 parent_domain_flush(dmar_domain, start_pfn, nrpages,
4143                                     list_empty(&gather->freelist));
4144         put_pages_list(&gather->freelist);
4145 }
4146
4147 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4148                                             dma_addr_t iova)
4149 {
4150         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4151         struct dma_pte *pte;
4152         int level = 0;
4153         u64 phys = 0;
4154
4155         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level,
4156                              GFP_ATOMIC);
4157         if (pte && dma_pte_present(pte))
4158                 phys = dma_pte_addr(pte) +
4159                         (iova & (BIT_MASK(level_to_offset_bits(level) +
4160                                                 VTD_PAGE_SHIFT) - 1));
4161
4162         return phys;
4163 }
4164
4165 static bool domain_support_force_snooping(struct dmar_domain *domain)
4166 {
4167         struct device_domain_info *info;
4168         bool support = true;
4169
4170         assert_spin_locked(&domain->lock);
4171         list_for_each_entry(info, &domain->devices, link) {
4172                 if (!ecap_sc_support(info->iommu->ecap)) {
4173                         support = false;
4174                         break;
4175                 }
4176         }
4177
4178         return support;
4179 }
4180
4181 static void domain_set_force_snooping(struct dmar_domain *domain)
4182 {
4183         struct device_domain_info *info;
4184
4185         assert_spin_locked(&domain->lock);
4186         /*
4187          * Second level page table supports per-PTE snoop control. The
4188          * iommu_map() interface will handle this by setting SNP bit.
4189          */
4190         if (!domain->use_first_level) {
4191                 domain->set_pte_snp = true;
4192                 return;
4193         }
4194
4195         list_for_each_entry(info, &domain->devices, link)
4196                 intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
4197                                                      IOMMU_NO_PASID);
4198 }
4199
4200 static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
4201 {
4202         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4203         unsigned long flags;
4204
4205         if (dmar_domain->force_snooping)
4206                 return true;
4207
4208         spin_lock_irqsave(&dmar_domain->lock, flags);
4209         if (!domain_support_force_snooping(dmar_domain) ||
4210             (!dmar_domain->use_first_level && dmar_domain->has_mappings)) {
4211                 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4212                 return false;
4213         }
4214
4215         domain_set_force_snooping(dmar_domain);
4216         dmar_domain->force_snooping = true;
4217         spin_unlock_irqrestore(&dmar_domain->lock, flags);
4218
4219         return true;
4220 }
4221
4222 static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
4223 {
4224         struct device_domain_info *info = dev_iommu_priv_get(dev);
4225
4226         switch (cap) {
4227         case IOMMU_CAP_CACHE_COHERENCY:
4228         case IOMMU_CAP_DEFERRED_FLUSH:
4229                 return true;
4230         case IOMMU_CAP_PRE_BOOT_PROTECTION:
4231                 return dmar_platform_optin();
4232         case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
4233                 return ecap_sc_support(info->iommu->ecap);
4234         case IOMMU_CAP_DIRTY_TRACKING:
4235                 return ssads_supported(info->iommu);
4236         default:
4237                 return false;
4238         }
4239 }
4240
4241 static struct iommu_device *intel_iommu_probe_device(struct device *dev)
4242 {
4243         struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
4244         struct device_domain_info *info;
4245         struct intel_iommu *iommu;
4246         u8 bus, devfn;
4247         int ret;
4248
4249         iommu = device_lookup_iommu(dev, &bus, &devfn);
4250         if (!iommu || !iommu->iommu.ops)
4251                 return ERR_PTR(-ENODEV);
4252
4253         info = kzalloc(sizeof(*info), GFP_KERNEL);
4254         if (!info)
4255                 return ERR_PTR(-ENOMEM);
4256
4257         if (dev_is_real_dma_subdevice(dev)) {
4258                 info->bus = pdev->bus->number;
4259                 info->devfn = pdev->devfn;
4260                 info->segment = pci_domain_nr(pdev->bus);
4261         } else {
4262                 info->bus = bus;
4263                 info->devfn = devfn;
4264                 info->segment = iommu->segment;
4265         }
4266
4267         info->dev = dev;
4268         info->iommu = iommu;
4269         if (dev_is_pci(dev)) {
4270                 if (ecap_dev_iotlb_support(iommu->ecap) &&
4271                     pci_ats_supported(pdev) &&
4272                     dmar_ats_supported(pdev, iommu)) {
4273                         info->ats_supported = 1;
4274                         info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
4275
4276                         /*
4277                          * For IOMMU that supports device IOTLB throttling
4278                          * (DIT), we assign PFSID to the invalidation desc
4279                          * of a VF such that IOMMU HW can gauge queue depth
4280                          * at PF level. If DIT is not set, PFSID will be
4281                          * treated as reserved, which should be set to 0.
4282                          */
4283                         if (ecap_dit(iommu->ecap))
4284                                 info->pfsid = pci_dev_id(pci_physfn(pdev));
4285                         info->ats_qdep = pci_ats_queue_depth(pdev);
4286                 }
4287                 if (sm_supported(iommu)) {
4288                         if (pasid_supported(iommu)) {
4289                                 int features = pci_pasid_features(pdev);
4290
4291                                 if (features >= 0)
4292                                         info->pasid_supported = features | 1;
4293                         }
4294
4295                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
4296                             pci_pri_supported(pdev))
4297                                 info->pri_supported = 1;
4298                 }
4299         }
4300
4301         dev_iommu_priv_set(dev, info);
4302         ret = device_rbtree_insert(iommu, info);
4303         if (ret)
4304                 goto free;
4305
4306         if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
4307                 ret = intel_pasid_alloc_table(dev);
4308                 if (ret) {
4309                         dev_err(dev, "PASID table allocation failed\n");
4310                         goto clear_rbtree;
4311                 }
4312
4313                 if (!context_copied(iommu, info->bus, info->devfn)) {
4314                         ret = intel_pasid_setup_sm_context(dev);
4315                         if (ret)
4316                                 goto free_table;
4317                 }
4318         }
4319
4320         intel_iommu_debugfs_create_dev(info);
4321
4322         return &iommu->iommu;
4323 free_table:
4324         intel_pasid_free_table(dev);
4325 clear_rbtree:
4326         device_rbtree_remove(info);
4327 free:
4328         kfree(info);
4329
4330         return ERR_PTR(ret);
4331 }
4332
4333 static void intel_iommu_release_device(struct device *dev)
4334 {
4335         struct device_domain_info *info = dev_iommu_priv_get(dev);
4336         struct intel_iommu *iommu = info->iommu;
4337
4338         mutex_lock(&iommu->iopf_lock);
4339         device_rbtree_remove(info);
4340         mutex_unlock(&iommu->iopf_lock);
4341
4342         if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
4343             !context_copied(iommu, info->bus, info->devfn))
4344                 intel_pasid_teardown_sm_context(dev);
4345
4346         intel_pasid_free_table(dev);
4347         intel_iommu_debugfs_remove_dev(info);
4348         kfree(info);
4349         set_dma_ops(dev, NULL);
4350 }
4351
4352 static void intel_iommu_probe_finalize(struct device *dev)
4353 {
4354         set_dma_ops(dev, NULL);
4355         iommu_setup_dma_ops(dev, 0, U64_MAX);
4356 }
4357
4358 static void intel_iommu_get_resv_regions(struct device *device,
4359                                          struct list_head *head)
4360 {
4361         int prot = DMA_PTE_READ | DMA_PTE_WRITE;
4362         struct iommu_resv_region *reg;
4363         struct dmar_rmrr_unit *rmrr;
4364         struct device *i_dev;
4365         int i;
4366
4367         rcu_read_lock();
4368         for_each_rmrr_units(rmrr) {
4369                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
4370                                           i, i_dev) {
4371                         struct iommu_resv_region *resv;
4372                         enum iommu_resv_type type;
4373                         size_t length;
4374
4375                         if (i_dev != device &&
4376                             !is_downstream_to_pci_bridge(device, i_dev))
4377                                 continue;
4378
4379                         length = rmrr->end_address - rmrr->base_address + 1;
4380
4381                         type = device_rmrr_is_relaxable(device) ?
4382                                 IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
4383
4384                         resv = iommu_alloc_resv_region(rmrr->base_address,
4385                                                        length, prot, type,
4386                                                        GFP_ATOMIC);
4387                         if (!resv)
4388                                 break;
4389
4390                         list_add_tail(&resv->list, head);
4391                 }
4392         }
4393         rcu_read_unlock();
4394
4395 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
4396         if (dev_is_pci(device)) {
4397                 struct pci_dev *pdev = to_pci_dev(device);
4398
4399                 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
4400                         reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
4401                                         IOMMU_RESV_DIRECT_RELAXABLE,
4402                                         GFP_KERNEL);
4403                         if (reg)
4404                                 list_add_tail(&reg->list, head);
4405                 }
4406         }
4407 #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
4408
4409         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
4410                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
4411                                       0, IOMMU_RESV_MSI, GFP_KERNEL);
4412         if (!reg)
4413                 return;
4414         list_add_tail(&reg->list, head);
4415 }
4416
4417 static struct iommu_group *intel_iommu_device_group(struct device *dev)
4418 {
4419         if (dev_is_pci(dev))
4420                 return pci_device_group(dev);
4421         return generic_device_group(dev);
4422 }
4423
4424 static int intel_iommu_enable_sva(struct device *dev)
4425 {
4426         struct device_domain_info *info = dev_iommu_priv_get(dev);
4427         struct intel_iommu *iommu;
4428
4429         if (!info || dmar_disabled)
4430                 return -EINVAL;
4431
4432         iommu = info->iommu;
4433         if (!iommu)
4434                 return -EINVAL;
4435
4436         if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))
4437                 return -ENODEV;
4438
4439         if (!info->pasid_enabled || !info->ats_enabled)
4440                 return -EINVAL;
4441
4442         /*
4443          * Devices having device-specific I/O fault handling should not
4444          * support PCI/PRI. The IOMMU side has no means to check the
4445          * capability of device-specific IOPF.  Therefore, IOMMU can only
4446          * default that if the device driver enables SVA on a non-PRI
4447          * device, it will handle IOPF in its own way.
4448          */
4449         if (!info->pri_supported)
4450                 return 0;
4451
4452         /* Devices supporting PRI should have it enabled. */
4453         if (!info->pri_enabled)
4454                 return -EINVAL;
4455
4456         return 0;
4457 }
4458
4459 static int intel_iommu_enable_iopf(struct device *dev)
4460 {
4461         struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
4462         struct device_domain_info *info = dev_iommu_priv_get(dev);
4463         struct intel_iommu *iommu;
4464         int ret;
4465
4466         if (!pdev || !info || !info->ats_enabled || !info->pri_supported)
4467                 return -ENODEV;
4468
4469         if (info->pri_enabled)
4470                 return -EBUSY;
4471
4472         iommu = info->iommu;
4473         if (!iommu)
4474                 return -EINVAL;
4475
4476         /* PASID is required in PRG Response Message. */
4477         if (info->pasid_enabled && !pci_prg_resp_pasid_required(pdev))
4478                 return -EINVAL;
4479
4480         ret = pci_reset_pri(pdev);
4481         if (ret)
4482                 return ret;
4483
4484         ret = iopf_queue_add_device(iommu->iopf_queue, dev);
4485         if (ret)
4486                 return ret;
4487
4488         ret = pci_enable_pri(pdev, PRQ_DEPTH);
4489         if (ret) {
4490                 iopf_queue_remove_device(iommu->iopf_queue, dev);
4491                 return ret;
4492         }
4493
4494         info->pri_enabled = 1;
4495
4496         return 0;
4497 }
4498
4499 static int intel_iommu_disable_iopf(struct device *dev)
4500 {
4501         struct device_domain_info *info = dev_iommu_priv_get(dev);
4502         struct intel_iommu *iommu = info->iommu;
4503
4504         if (!info->pri_enabled)
4505                 return -EINVAL;
4506
4507         /*
4508          * PCIe spec states that by clearing PRI enable bit, the Page
4509          * Request Interface will not issue new page requests, but has
4510          * outstanding page requests that have been transmitted or are
4511          * queued for transmission. This is supposed to be called after
4512          * the device driver has stopped DMA, all PASIDs have been
4513          * unbound and the outstanding PRQs have been drained.
4514          */
4515         pci_disable_pri(to_pci_dev(dev));
4516         info->pri_enabled = 0;
4517         iopf_queue_remove_device(iommu->iopf_queue, dev);
4518
4519         return 0;
4520 }
4521
4522 static int
4523 intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
4524 {
4525         switch (feat) {
4526         case IOMMU_DEV_FEAT_IOPF:
4527                 return intel_iommu_enable_iopf(dev);
4528
4529         case IOMMU_DEV_FEAT_SVA:
4530                 return intel_iommu_enable_sva(dev);
4531
4532         default:
4533                 return -ENODEV;
4534         }
4535 }
4536
4537 static int
4538 intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
4539 {
4540         switch (feat) {
4541         case IOMMU_DEV_FEAT_IOPF:
4542                 return intel_iommu_disable_iopf(dev);
4543
4544         case IOMMU_DEV_FEAT_SVA:
4545                 return 0;
4546
4547         default:
4548                 return -ENODEV;
4549         }
4550 }
4551
4552 static bool intel_iommu_is_attach_deferred(struct device *dev)
4553 {
4554         struct device_domain_info *info = dev_iommu_priv_get(dev);
4555
4556         return translation_pre_enabled(info->iommu) && !info->domain;
4557 }
4558
4559 /*
4560  * Check that the device does not live on an external facing PCI port that is
4561  * marked as untrusted. Such devices should not be able to apply quirks and
4562  * thus not be able to bypass the IOMMU restrictions.
4563  */
4564 static bool risky_device(struct pci_dev *pdev)
4565 {
4566         if (pdev->untrusted) {
4567                 pci_info(pdev,
4568                          "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
4569                          pdev->vendor, pdev->device);
4570                 pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
4571                 return true;
4572         }
4573         return false;
4574 }
4575
4576 static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4577                                       unsigned long iova, size_t size)
4578 {
4579         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4580         unsigned long pages = aligned_nrpages(iova, size);
4581         unsigned long pfn = iova >> VTD_PAGE_SHIFT;
4582         struct iommu_domain_info *info;
4583         unsigned long i;
4584
4585         xa_for_each(&dmar_domain->iommu_array, i, info)
4586                 __mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
4587         return 0;
4588 }
4589
4590 static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
4591 {
4592         struct device_domain_info *info = dev_iommu_priv_get(dev);
4593         struct dev_pasid_info *curr, *dev_pasid = NULL;
4594         struct intel_iommu *iommu = info->iommu;
4595         struct dmar_domain *dmar_domain;
4596         struct iommu_domain *domain;
4597         unsigned long flags;
4598
4599         domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
4600         if (WARN_ON_ONCE(!domain))
4601                 goto out_tear_down;
4602
4603         /*
4604          * The SVA implementation needs to handle its own stuffs like the mm
4605          * notification. Before consolidating that code into iommu core, let
4606          * the intel sva code handle it.
4607          */
4608         if (domain->type == IOMMU_DOMAIN_SVA) {
4609                 intel_svm_remove_dev_pasid(dev, pasid);
4610                 goto out_tear_down;
4611         }
4612
4613         dmar_domain = to_dmar_domain(domain);
4614         spin_lock_irqsave(&dmar_domain->lock, flags);
4615         list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) {
4616                 if (curr->dev == dev && curr->pasid == pasid) {
4617                         list_del(&curr->link_domain);
4618                         dev_pasid = curr;
4619                         break;
4620                 }
4621         }
4622         WARN_ON_ONCE(!dev_pasid);
4623         spin_unlock_irqrestore(&dmar_domain->lock, flags);
4624
4625         domain_detach_iommu(dmar_domain, iommu);
4626         intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
4627         kfree(dev_pasid);
4628 out_tear_down:
4629         intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4630         intel_drain_pasid_prq(dev, pasid);
4631 }
4632
4633 static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
4634                                      struct device *dev, ioasid_t pasid)
4635 {
4636         struct device_domain_info *info = dev_iommu_priv_get(dev);
4637         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4638         struct intel_iommu *iommu = info->iommu;
4639         struct dev_pasid_info *dev_pasid;
4640         unsigned long flags;
4641         int ret;
4642
4643         if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
4644                 return -EOPNOTSUPP;
4645
4646         if (domain->dirty_ops)
4647                 return -EINVAL;
4648
4649         if (context_copied(iommu, info->bus, info->devfn))
4650                 return -EBUSY;
4651
4652         ret = prepare_domain_attach_device(domain, dev);
4653         if (ret)
4654                 return ret;
4655
4656         dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL);
4657         if (!dev_pasid)
4658                 return -ENOMEM;
4659
4660         ret = domain_attach_iommu(dmar_domain, iommu);
4661         if (ret)
4662                 goto out_free;
4663
4664         if (domain_type_is_si(dmar_domain))
4665                 ret = intel_pasid_setup_pass_through(iommu, dev, pasid);
4666         else if (dmar_domain->use_first_level)
4667                 ret = domain_setup_first_level(iommu, dmar_domain,
4668                                                dev, pasid);
4669         else
4670                 ret = intel_pasid_setup_second_level(iommu, dmar_domain,
4671                                                      dev, pasid);
4672         if (ret)
4673                 goto out_detach_iommu;
4674
4675         dev_pasid->dev = dev;
4676         dev_pasid->pasid = pasid;
4677         spin_lock_irqsave(&dmar_domain->lock, flags);
4678         list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
4679         spin_unlock_irqrestore(&dmar_domain->lock, flags);
4680
4681         if (domain->type & __IOMMU_DOMAIN_PAGING)
4682                 intel_iommu_debugfs_create_dev_pasid(dev_pasid);
4683
4684         return 0;
4685 out_detach_iommu:
4686         domain_detach_iommu(dmar_domain, iommu);
4687 out_free:
4688         kfree(dev_pasid);
4689         return ret;
4690 }
4691
4692 static void *intel_iommu_hw_info(struct device *dev, u32 *length, u32 *type)
4693 {
4694         struct device_domain_info *info = dev_iommu_priv_get(dev);
4695         struct intel_iommu *iommu = info->iommu;
4696         struct iommu_hw_info_vtd *vtd;
4697
4698         vtd = kzalloc(sizeof(*vtd), GFP_KERNEL);
4699         if (!vtd)
4700                 return ERR_PTR(-ENOMEM);
4701
4702         vtd->flags = IOMMU_HW_INFO_VTD_ERRATA_772415_SPR17;
4703         vtd->cap_reg = iommu->cap;
4704         vtd->ecap_reg = iommu->ecap;
4705         *length = sizeof(*vtd);
4706         *type = IOMMU_HW_INFO_TYPE_INTEL_VTD;
4707         return vtd;
4708 }
4709
4710 /*
4711  * Set dirty tracking for the device list of a domain. The caller must
4712  * hold the domain->lock when calling it.
4713  */
4714 static int device_set_dirty_tracking(struct list_head *devices, bool enable)
4715 {
4716         struct device_domain_info *info;
4717         int ret = 0;
4718
4719         list_for_each_entry(info, devices, link) {
4720                 ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,
4721                                                        IOMMU_NO_PASID, enable);
4722                 if (ret)
4723                         break;
4724         }
4725
4726         return ret;
4727 }
4728
4729 static int parent_domain_set_dirty_tracking(struct dmar_domain *domain,
4730                                             bool enable)
4731 {
4732         struct dmar_domain *s1_domain;
4733         unsigned long flags;
4734         int ret;
4735
4736         spin_lock(&domain->s1_lock);
4737         list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4738                 spin_lock_irqsave(&s1_domain->lock, flags);
4739                 ret = device_set_dirty_tracking(&s1_domain->devices, enable);
4740                 spin_unlock_irqrestore(&s1_domain->lock, flags);
4741                 if (ret)
4742                         goto err_unwind;
4743         }
4744         spin_unlock(&domain->s1_lock);
4745         return 0;
4746
4747 err_unwind:
4748         list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4749                 spin_lock_irqsave(&s1_domain->lock, flags);
4750                 device_set_dirty_tracking(&s1_domain->devices,
4751                                           domain->dirty_tracking);
4752                 spin_unlock_irqrestore(&s1_domain->lock, flags);
4753         }
4754         spin_unlock(&domain->s1_lock);
4755         return ret;
4756 }
4757
4758 static int intel_iommu_set_dirty_tracking(struct iommu_domain *domain,
4759                                           bool enable)
4760 {
4761         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4762         int ret;
4763
4764         spin_lock(&dmar_domain->lock);
4765         if (dmar_domain->dirty_tracking == enable)
4766                 goto out_unlock;
4767
4768         ret = device_set_dirty_tracking(&dmar_domain->devices, enable);
4769         if (ret)
4770                 goto err_unwind;
4771
4772         if (dmar_domain->nested_parent) {
4773                 ret = parent_domain_set_dirty_tracking(dmar_domain, enable);
4774                 if (ret)
4775                         goto err_unwind;
4776         }
4777
4778         dmar_domain->dirty_tracking = enable;
4779 out_unlock:
4780         spin_unlock(&dmar_domain->lock);
4781
4782         return 0;
4783
4784 err_unwind:
4785         device_set_dirty_tracking(&dmar_domain->devices,
4786                                   dmar_domain->dirty_tracking);
4787         spin_unlock(&dmar_domain->lock);
4788         return ret;
4789 }
4790
4791 static int intel_iommu_read_and_clear_dirty(struct iommu_domain *domain,
4792                                             unsigned long iova, size_t size,
4793                                             unsigned long flags,
4794                                             struct iommu_dirty_bitmap *dirty)
4795 {
4796         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4797         unsigned long end = iova + size - 1;
4798         unsigned long pgsize;
4799
4800         /*
4801          * IOMMUFD core calls into a dirty tracking disabled domain without an
4802          * IOVA bitmap set in order to clean dirty bits in all PTEs that might
4803          * have occurred when we stopped dirty tracking. This ensures that we
4804          * never inherit dirtied bits from a previous cycle.
4805          */
4806         if (!dmar_domain->dirty_tracking && dirty->bitmap)
4807                 return -EINVAL;
4808
4809         do {
4810                 struct dma_pte *pte;
4811                 int lvl = 0;
4812
4813                 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &lvl,
4814                                      GFP_ATOMIC);
4815                 pgsize = level_size(lvl) << VTD_PAGE_SHIFT;
4816                 if (!pte || !dma_pte_present(pte)) {
4817                         iova += pgsize;
4818                         continue;
4819                 }
4820
4821                 if (dma_sl_pte_test_and_clear_dirty(pte, flags))
4822                         iommu_dirty_bitmap_record(dirty, iova, pgsize);
4823                 iova += pgsize;
4824         } while (iova < end);
4825
4826         return 0;
4827 }
4828
4829 static const struct iommu_dirty_ops intel_dirty_ops = {
4830         .set_dirty_tracking = intel_iommu_set_dirty_tracking,
4831         .read_and_clear_dirty = intel_iommu_read_and_clear_dirty,
4832 };
4833
4834 const struct iommu_ops intel_iommu_ops = {
4835         .blocked_domain         = &blocking_domain,
4836         .release_domain         = &blocking_domain,
4837         .capable                = intel_iommu_capable,
4838         .hw_info                = intel_iommu_hw_info,
4839         .domain_alloc           = intel_iommu_domain_alloc,
4840         .domain_alloc_user      = intel_iommu_domain_alloc_user,
4841         .probe_device           = intel_iommu_probe_device,
4842         .probe_finalize         = intel_iommu_probe_finalize,
4843         .release_device         = intel_iommu_release_device,
4844         .get_resv_regions       = intel_iommu_get_resv_regions,
4845         .device_group           = intel_iommu_device_group,
4846         .dev_enable_feat        = intel_iommu_dev_enable_feat,
4847         .dev_disable_feat       = intel_iommu_dev_disable_feat,
4848         .is_attach_deferred     = intel_iommu_is_attach_deferred,
4849         .def_domain_type        = device_def_domain_type,
4850         .remove_dev_pasid       = intel_iommu_remove_dev_pasid,
4851         .pgsize_bitmap          = SZ_4K,
4852 #ifdef CONFIG_INTEL_IOMMU_SVM
4853         .page_response          = intel_svm_page_response,
4854 #endif
4855         .default_domain_ops = &(const struct iommu_domain_ops) {
4856                 .attach_dev             = intel_iommu_attach_device,
4857                 .set_dev_pasid          = intel_iommu_set_dev_pasid,
4858                 .map_pages              = intel_iommu_map_pages,
4859                 .unmap_pages            = intel_iommu_unmap_pages,
4860                 .iotlb_sync_map         = intel_iommu_iotlb_sync_map,
4861                 .flush_iotlb_all        = intel_flush_iotlb_all,
4862                 .iotlb_sync             = intel_iommu_tlb_sync,
4863                 .iova_to_phys           = intel_iommu_iova_to_phys,
4864                 .free                   = intel_iommu_domain_free,
4865                 .enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
4866         }
4867 };
4868
4869 static void quirk_iommu_igfx(struct pci_dev *dev)
4870 {
4871         if (risky_device(dev))
4872                 return;
4873
4874         pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
4875         dmar_map_gfx = 0;
4876 }
4877
4878 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4879 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4880 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4881 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4882 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4883 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4884 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4885 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4886
4887 /* Broadwell igfx malfunctions with dmar */
4888 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4889 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4890 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4891 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4892 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4893 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4894 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4895 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4896 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4897 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4898 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4899 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4900 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4901 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4902 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4903 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4904 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4905 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4906 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4907 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4908 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4909 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4910 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4911 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4912
4913 static void quirk_iommu_rwbf(struct pci_dev *dev)
4914 {
4915         if (risky_device(dev))
4916                 return;
4917
4918         /*
4919          * Mobile 4 Series Chipset neglects to set RWBF capability,
4920          * but needs it. Same seems to hold for the desktop versions.
4921          */
4922         pci_info(dev, "Forcing write-buffer flush capability\n");
4923         rwbf_quirk = 1;
4924 }
4925
4926 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4927 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4928 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4929 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4930 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4931 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4932 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4933
4934 #define GGC 0x52
4935 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4936 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4937 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4938 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4939 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4940 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4941 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4942 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4943
4944 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4945 {
4946         unsigned short ggc;
4947
4948         if (risky_device(dev))
4949                 return;
4950
4951         if (pci_read_config_word(dev, GGC, &ggc))
4952                 return;
4953
4954         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4955                 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4956                 dmar_map_gfx = 0;
4957         } else if (dmar_map_gfx) {
4958                 /* we have to ensure the gfx device is idle before we flush */
4959                 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
4960                 iommu_set_dma_strict();
4961         }
4962 }
4963 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4964 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4965 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4966 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4967
4968 static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
4969 {
4970         unsigned short ver;
4971
4972         if (!IS_GFX_DEVICE(dev))
4973                 return;
4974
4975         ver = (dev->device >> 8) & 0xff;
4976         if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
4977             ver != 0x4e && ver != 0x8a && ver != 0x98 &&
4978             ver != 0x9a && ver != 0xa7 && ver != 0x7d)
4979                 return;
4980
4981         if (risky_device(dev))
4982                 return;
4983
4984         pci_info(dev, "Skip IOMMU disabling for graphics\n");
4985         iommu_skip_te_disable = 1;
4986 }
4987 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
4988
4989 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4990    ISOCH DMAR unit for the Azalia sound device, but not give it any
4991    TLB entries, which causes it to deadlock. Check for that.  We do
4992    this in a function called from init_dmars(), instead of in a PCI
4993    quirk, because we don't want to print the obnoxious "BIOS broken"
4994    message if VT-d is actually disabled.
4995 */
4996 static void __init check_tylersburg_isoch(void)
4997 {
4998         struct pci_dev *pdev;
4999         uint32_t vtisochctrl;
5000
5001         /* If there's no Azalia in the system anyway, forget it. */
5002         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5003         if (!pdev)
5004                 return;
5005
5006         if (risky_device(pdev)) {
5007                 pci_dev_put(pdev);
5008                 return;
5009         }
5010
5011         pci_dev_put(pdev);
5012
5013         /* System Management Registers. Might be hidden, in which case
5014            we can't do the sanity check. But that's OK, because the
5015            known-broken BIOSes _don't_ actually hide it, so far. */
5016         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5017         if (!pdev)
5018                 return;
5019
5020         if (risky_device(pdev)) {
5021                 pci_dev_put(pdev);
5022                 return;
5023         }
5024
5025         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5026                 pci_dev_put(pdev);
5027                 return;
5028         }
5029
5030         pci_dev_put(pdev);
5031
5032         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5033         if (vtisochctrl & 1)
5034                 return;
5035
5036         /* Drop all bits other than the number of TLB entries */
5037         vtisochctrl &= 0x1c;
5038
5039         /* If we have the recommended number of TLB entries (16), fine. */
5040         if (vtisochctrl == 0x10)
5041                 return;
5042
5043         /* Zero TLB entries? You get to ride the short bus to school. */
5044         if (!vtisochctrl) {
5045                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5046                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5047                      dmi_get_system_info(DMI_BIOS_VENDOR),
5048                      dmi_get_system_info(DMI_BIOS_VERSION),
5049                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5050                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5051                 return;
5052         }
5053
5054         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5055                vtisochctrl);
5056 }
5057
5058 /*
5059  * Here we deal with a device TLB defect where device may inadvertently issue ATS
5060  * invalidation completion before posted writes initiated with translated address
5061  * that utilized translations matching the invalidation address range, violating
5062  * the invalidation completion ordering.
5063  * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
5064  * vulnerable to this defect. In other words, any dTLB invalidation initiated not
5065  * under the control of the trusted/privileged host device driver must use this
5066  * quirk.
5067  * Device TLBs are invalidated under the following six conditions:
5068  * 1. Device driver does DMA API unmap IOVA
5069  * 2. Device driver unbind a PASID from a process, sva_unbind_device()
5070  * 3. PASID is torn down, after PASID cache is flushed. e.g. process
5071  *    exit_mmap() due to crash
5072  * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where
5073  *    VM has to free pages that were unmapped
5074  * 5. Userspace driver unmaps a DMA buffer
5075  * 6. Cache invalidation in vSVA usage (upcoming)
5076  *
5077  * For #1 and #2, device drivers are responsible for stopping DMA traffic
5078  * before unmap/unbind. For #3, iommu driver gets mmu_notifier to
5079  * invalidate TLB the same way as normal user unmap which will use this quirk.
5080  * The dTLB invalidation after PASID cache flush does not need this quirk.
5081  *
5082  * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
5083  */
5084 void quirk_extra_dev_tlb_flush(struct device_domain_info *info,
5085                                unsigned long address, unsigned long mask,
5086                                u32 pasid, u16 qdep)
5087 {
5088         u16 sid;
5089
5090         if (likely(!info->dtlb_extra_inval))
5091                 return;
5092
5093         sid = PCI_DEVID(info->bus, info->devfn);
5094         if (pasid == IOMMU_NO_PASID) {
5095                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
5096                                    qdep, address, mask);
5097         } else {
5098                 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
5099                                          pasid, qdep, address, mask);
5100         }
5101 }
5102
5103 #define ecmd_get_status_code(res)       (((res) & 0xff) >> 1)
5104
5105 /*
5106  * Function to submit a command to the enhanced command interface. The
5107  * valid enhanced command descriptions are defined in Table 47 of the
5108  * VT-d spec. The VT-d hardware implementation may support some but not
5109  * all commands, which can be determined by checking the Enhanced
5110  * Command Capability Register.
5111  *
5112  * Return values:
5113  *  - 0: Command successful without any error;
5114  *  - Negative: software error value;
5115  *  - Nonzero positive: failure status code defined in Table 48.
5116  */
5117 int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)
5118 {
5119         unsigned long flags;
5120         u64 res;
5121         int ret;
5122
5123         if (!cap_ecmds(iommu->cap))
5124                 return -ENODEV;
5125
5126         raw_spin_lock_irqsave(&iommu->register_lock, flags);
5127
5128         res = dmar_readq(iommu->reg + DMAR_ECRSP_REG);
5129         if (res & DMA_ECMD_ECRSP_IP) {
5130                 ret = -EBUSY;
5131                 goto err;
5132         }
5133
5134         /*
5135          * Unconditionally write the operand B, because
5136          * - There is no side effect if an ecmd doesn't require an
5137          *   operand B, but we set the register to some value.
5138          * - It's not invoked in any critical path. The extra MMIO
5139          *   write doesn't bring any performance concerns.
5140          */
5141         dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob);
5142         dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT));
5143
5144         IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq,
5145                       !(res & DMA_ECMD_ECRSP_IP), res);
5146
5147         if (res & DMA_ECMD_ECRSP_IP) {
5148                 ret = -ETIMEDOUT;
5149                 goto err;
5150         }
5151
5152         ret = ecmd_get_status_code(res);
5153 err:
5154         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
5155
5156         return ret;
5157 }