drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/dma-direct.h>
  35 #include <linux/mempool.h>
  36 #include <linux/memory.h>
  37 #include <linux/cpu.h>
  38 #include <linux/timer.h>
  39 #include <linux/io.h>
  40 #include <linux/iova.h>
  41 #include <linux/iommu.h>
  42 #include <linux/intel-iommu.h>
  43 #include <linux/syscore_ops.h>
  44 #include <linux/tboot.h>
  45 #include <linux/dmi.h>
  46 #include <linux/pci-ats.h>
  47 #include <linux/memblock.h>
  48 #include <linux/dma-contiguous.h>
  49 #include <linux/dma-direct.h>
  50 #include <linux/crash_dump.h>
  51 #include <asm/irq_remapping.h>
  52 #include <asm/cacheflush.h>
  53 #include <asm/iommu.h>
  54
  55 #include "irq_remapping.h"
  56
  57 #define ROOT_SIZE               VTD_PAGE_SIZE
  58 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  59
  60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  64
  65 #define IOAPIC_RANGE_START      (0xfee00000)
  66 #define IOAPIC_RANGE_END        (0xfeefffff)
  67 #define IOVA_START_ADDR         (0x1000)
  68
  69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  70
  71 #define MAX_AGAW_WIDTH 64
  72 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  73
  74 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  76
  77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  78    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  79 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  80                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  81 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  82
  83 /* IO virtual address start page frame number */
  84 #define IOVA_START_PFN          (1)
  85
  86 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186 int intel_iommu_tboot_noforce;
 187
 188 /*
 189  * 0: Present
 190  * 1-11: Reserved
 191  * 12-63: Context Ptr (12 - (haw-1))
 192  * 64-127: Reserved
 193  */
 194 struct root_entry {
 195         u64     lo;
 196         u64     hi;
 197 };
 198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 199
 200 /*
 201  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 202  * if marked present.
 203  */
 204 static phys_addr_t root_entry_lctp(struct root_entry *re)
 205 {
 206         if (!(re->lo & 1))
 207                 return 0;
 208
 209         return re->lo & VTD_PAGE_MASK;
 210 }
 211
 212 /*
 213  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 214  * if marked present.
 215  */
 216 static phys_addr_t root_entry_uctp(struct root_entry *re)
 217 {
 218         if (!(re->hi & 1))
 219                 return 0;
 220
 221         return re->hi & VTD_PAGE_MASK;
 222 }
 223 /*
 224  * low 64 bits:
 225  * 0: present
 226  * 1: fault processing disable
 227  * 2-3: translation type
 228  * 12-63: address space root
 229  * high 64 bits:
 230  * 0-2: address width
 231  * 3-6: aval
 232  * 8-23: domain id
 233  */
 234 struct context_entry {
 235         u64 lo;
 236         u64 hi;
 237 };
 238
 239 static inline void context_clear_pasid_enable(struct context_entry *context)
 240 {
 241         context->lo &= ~(1ULL << 11);
 242 }
 243
 244 static inline bool context_pasid_enabled(struct context_entry *context)
 245 {
 246         return !!(context->lo & (1ULL << 11));
 247 }
 248
 249 static inline void context_set_copied(struct context_entry *context)
 250 {
 251         context->hi |= (1ull << 3);
 252 }
 253
 254 static inline bool context_copied(struct context_entry *context)
 255 {
 256         return !!(context->hi & (1ULL << 3));
 257 }
 258
 259 static inline bool __context_present(struct context_entry *context)
 260 {
 261         return (context->lo & 1);
 262 }
 263
 264 static inline bool context_present(struct context_entry *context)
 265 {
 266         return context_pasid_enabled(context) ?
 267              __context_present(context) :
 268              __context_present(context) && !context_copied(context);
 269 }
 270
 271 static inline void context_set_present(struct context_entry *context)
 272 {
 273         context->lo |= 1;
 274 }
 275
 276 static inline void context_set_fault_enable(struct context_entry *context)
 277 {
 278         context->lo &= (((u64)-1) << 2) | 1;
 279 }
 280
 281 static inline void context_set_translation_type(struct context_entry *context,
 282                                                 unsigned long value)
 283 {
 284         context->lo &= (((u64)-1) << 4) | 3;
 285         context->lo |= (value & 3) << 2;
 286 }
 287
 288 static inline void context_set_address_root(struct context_entry *context,
 289                                             unsigned long value)
 290 {
 291         context->lo &= ~VTD_PAGE_MASK;
 292         context->lo |= value & VTD_PAGE_MASK;
 293 }
 294
 295 static inline void context_set_address_width(struct context_entry *context,
 296                                              unsigned long value)
 297 {
 298         context->hi |= value & 7;
 299 }
 300
 301 static inline void context_set_domain_id(struct context_entry *context,
 302                                          unsigned long value)
 303 {
 304         context->hi |= (value & ((1 << 16) - 1)) << 8;
 305 }
 306
 307 static inline int context_domain_id(struct context_entry *c)
 308 {
 309         return((c->hi >> 8) & 0xffff);
 310 }
 311
 312 static inline void context_clear_entry(struct context_entry *context)
 313 {
 314         context->lo = 0;
 315         context->hi = 0;
 316 }
 317
 318 /*
 319  * 0: readable
 320  * 1: writable
 321  * 2-6: reserved
 322  * 7: super page
 323  * 8-10: available
 324  * 11: snoop behavior
 325  * 12-63: Host physcial address
 326  */
 327 struct dma_pte {
 328         u64 val;
 329 };
 330
 331 static inline void dma_clear_pte(struct dma_pte *pte)
 332 {
 333         pte->val = 0;
 334 }
 335
 336 static inline u64 dma_pte_addr(struct dma_pte *pte)
 337 {
 338 #ifdef CONFIG_64BIT
 339         return pte->val & VTD_PAGE_MASK;
 340 #else
 341         /* Must have a full atomic 64-bit read */
 342         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 343 #endif
 344 }
 345
 346 static inline bool dma_pte_present(struct dma_pte *pte)
 347 {
 348         return (pte->val & 3) != 0;
 349 }
 350
 351 static inline bool dma_pte_superpage(struct dma_pte *pte)
 352 {
 353         return (pte->val & DMA_PTE_LARGE_PAGE);
 354 }
 355
 356 static inline int first_pte_in_page(struct dma_pte *pte)
 357 {
 358         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 359 }
 360
 361 /*
 362  * This domain is a statically identity mapping domain.
 363  *      1. This domain creats a static 1:1 mapping to all usable memory.
 364  *      2. It maps to each iommu if successful.
 365  *      3. Each iommu mapps to this domain if successful.
 366  */
 367 static struct dmar_domain *si_domain;
 368 static int hw_pass_through = 1;
 369
 370 /*
 371  * Domain represents a virtual machine, more than one devices
 372  * across iommus may be owned in one domain, e.g. kvm guest.
 373  */
 374 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 375
 376 /* si_domain contains mulitple devices */
 377 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 378
 379 #define for_each_domain_iommu(idx, domain)                      \
 380         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 381                 if (domain->iommu_refcnt[idx])
 382
 383 struct dmar_domain {
 384         int     nid;                    /* node id */
 385
 386         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 387                                         /* Refcount of devices per iommu */
 388
 389
 390         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 391                                         /* Domain ids per IOMMU. Use u16 since
 392                                          * domain ids are 16 bit wide according
 393                                          * to VT-d spec, section 9.3 */
 394
 395         bool has_iotlb_device;
 396         struct list_head devices;       /* all devices' list */
 397         struct iova_domain iovad;       /* iova's that belong to this domain */
 398
 399         struct dma_pte  *pgd;           /* virtual address */
 400         int             gaw;            /* max guest address width */
 401
 402         /* adjusted guest address width, 0 is level 2 30-bit */
 403         int             agaw;
 404
 405         int             flags;          /* flags to find out type of domain */
 406
 407         int             iommu_coherency;/* indicate coherency of iommu access */
 408         int             iommu_snooping; /* indicate snooping control feature*/
 409         int             iommu_count;    /* reference count of iommu */
 410         int             iommu_superpage;/* Level of superpages supported:
 411                                            0 == 4KiB (no superpages), 1 == 2MiB,
 412                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 413         u64             max_addr;       /* maximum mapped address */
 414
 415         struct iommu_domain domain;     /* generic domain data structure for
 416                                            iommu core */
 417 };
 418
 419 /* PCI domain-device relationship */
 420 struct device_domain_info {
 421         struct list_head link;  /* link to domain siblings */
 422         struct list_head global; /* link to global list */
 423         u8 bus;                 /* PCI bus number */
 424         u8 devfn;               /* PCI devfn number */
 425         u8 pasid_supported:3;
 426         u8 pasid_enabled:1;
 427         u8 pri_supported:1;
 428         u8 pri_enabled:1;
 429         u8 ats_supported:1;
 430         u8 ats_enabled:1;
 431         u8 ats_qdep;
 432         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 433         struct intel_iommu *iommu; /* IOMMU used by this device */
 434         struct dmar_domain *domain; /* pointer to domain */
 435 };
 436
 437 struct dmar_rmrr_unit {
 438         struct list_head list;          /* list of rmrr units   */
 439         struct acpi_dmar_header *hdr;   /* ACPI header          */
 440         u64     base_address;           /* reserved base address*/
 441         u64     end_address;            /* reserved end address */
 442         struct dmar_dev_scope *devices; /* target devices */
 443         int     devices_cnt;            /* target device count */
 444         struct iommu_resv_region *resv; /* reserved region handle */
 445 };
 446
 447 struct dmar_atsr_unit {
 448         struct list_head list;          /* list of ATSR units */
 449         struct acpi_dmar_header *hdr;   /* ACPI header */
 450         struct dmar_dev_scope *devices; /* target devices */
 451         int devices_cnt;                /* target device count */
 452         u8 include_all:1;               /* include all ports */
 453 };
 454
 455 static LIST_HEAD(dmar_atsr_units);
 456 static LIST_HEAD(dmar_rmrr_units);
 457
 458 #define for_each_rmrr_units(rmrr) \
 459         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 460
 461 /* bitmap for indexing intel_iommus */
 462 static int g_num_of_iommus;
 463
 464 static void domain_exit(struct dmar_domain *domain);
 465 static void domain_remove_dev_info(struct dmar_domain *domain);
 466 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 467                                      struct device *dev);
 468 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 469 static void domain_context_clear(struct intel_iommu *iommu,
 470                                  struct device *dev);
 471 static int domain_detach_iommu(struct dmar_domain *domain,
 472                                struct intel_iommu *iommu);
 473
 474 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 475 int dmar_disabled = 0;
 476 #else
 477 int dmar_disabled = 1;
 478 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 479
 480 int intel_iommu_enabled = 0;
 481 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 482
 483 static int dmar_map_gfx = 1;
 484 static int dmar_forcedac;
 485 static int intel_iommu_strict;
 486 static int intel_iommu_superpage = 1;
 487 static int intel_iommu_ecs = 1;
 488 static int intel_iommu_pasid28;
 489 static int iommu_identity_mapping;
 490
 491 #define IDENTMAP_ALL            1
 492 #define IDENTMAP_GFX            2
 493 #define IDENTMAP_AZALIA         4
 494
 495 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 496  * level" translation of DMA requests-without-PASID doesn't actually happen
 497  * unless you also set the NESTE bit in an extended context-entry. Which of
 498  * course means that SVM doesn't work because it's trying to do nested
 499  * translation of the physical addresses it finds in the process page tables,
 500  * through the IOVA->phys mapping found in the "second level" page tables.
 501  *
 502  * The VT-d specification was retroactively changed to change the definition
 503  * of the capability bits and pretend that Broadwell/Skylake never happened...
 504  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 505  * for some reason it was the PASID capability bit which was redefined (from
 506  * bit 28 on BDW/SKL to bit 40 in future).
 507  *
 508  * So our test for ECS needs to eschew those implementations which set the old
 509  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 510  * Unless we are working around the 'pasid28' limitations, that is, by putting
 511  * the device into passthrough mode for normal DMA and thus masking the bug.
 512  */
 513 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 514                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 515 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 516  * or new capability bits are set. */
 517 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 518                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 519
 520 int intel_iommu_gfx_mapped;
 521 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 522
 523 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 524 static DEFINE_SPINLOCK(device_domain_lock);
 525 static LIST_HEAD(device_domain_list);
 526
 527 const struct iommu_ops intel_iommu_ops;
 528
 529 static bool translation_pre_enabled(struct intel_iommu *iommu)
 530 {
 531         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 532 }
 533
 534 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 535 {
 536         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 537 }
 538
 539 static void init_translation_status(struct intel_iommu *iommu)
 540 {
 541         u32 gsts;
 542
 543         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 544         if (gsts & DMA_GSTS_TES)
 545                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 546 }
 547
 548 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 549 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 550 {
 551         return container_of(dom, struct dmar_domain, domain);
 552 }
 553
 554 static int __init intel_iommu_setup(char *str)
 555 {
 556         if (!str)
 557                 return -EINVAL;
 558         while (*str) {
 559                 if (!strncmp(str, "on", 2)) {
 560                         dmar_disabled = 0;
 561                         pr_info("IOMMU enabled\n");
 562                 } else if (!strncmp(str, "off", 3)) {
 563                         dmar_disabled = 1;
 564                         pr_info("IOMMU disabled\n");
 565                 } else if (!strncmp(str, "igfx_off", 8)) {
 566                         dmar_map_gfx = 0;
 567                         pr_info("Disable GFX device mapping\n");
 568                 } else if (!strncmp(str, "forcedac", 8)) {
 569                         pr_info("Forcing DAC for PCI devices\n");
 570                         dmar_forcedac = 1;
 571                 } else if (!strncmp(str, "strict", 6)) {
 572                         pr_info("Disable batched IOTLB flush\n");
 573                         intel_iommu_strict = 1;
 574                 } else if (!strncmp(str, "sp_off", 6)) {
 575                         pr_info("Disable supported super page\n");
 576                         intel_iommu_superpage = 0;
 577                 } else if (!strncmp(str, "ecs_off", 7)) {
 578                         printk(KERN_INFO
 579                                 "Intel-IOMMU: disable extended context table support\n");
 580                         intel_iommu_ecs = 0;
 581                 } else if (!strncmp(str, "pasid28", 7)) {
 582                         printk(KERN_INFO
 583                                 "Intel-IOMMU: enable pre-production PASID support\n");
 584                         intel_iommu_pasid28 = 1;
 585                         iommu_identity_mapping |= IDENTMAP_GFX;
 586                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 587                         printk(KERN_INFO
 588                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 589                         intel_iommu_tboot_noforce = 1;
 590                 }
 591
 592                 str += strcspn(str, ",");
 593                 while (*str == ',')
 594                         str++;
 595         }
 596         return 0;
 597 }
 598 __setup("intel_iommu=", intel_iommu_setup);
 599
 600 static struct kmem_cache *iommu_domain_cache;
 601 static struct kmem_cache *iommu_devinfo_cache;
 602
 603 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 604 {
 605         struct dmar_domain **domains;
 606         int idx = did >> 8;
 607
 608         domains = iommu->domains[idx];
 609         if (!domains)
 610                 return NULL;
 611
 612         return domains[did & 0xff];
 613 }
 614
 615 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 616                              struct dmar_domain *domain)
 617 {
 618         struct dmar_domain **domains;
 619         int idx = did >> 8;
 620
 621         if (!iommu->domains[idx]) {
 622                 size_t size = 256 * sizeof(struct dmar_domain *);
 623                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 624         }
 625
 626         domains = iommu->domains[idx];
 627         if (WARN_ON(!domains))
 628                 return;
 629         else
 630                 domains[did & 0xff] = domain;
 631 }
 632
 633 static inline void *alloc_pgtable_page(int node)
 634 {
 635         struct page *page;
 636         void *vaddr = NULL;
 637
 638         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 639         if (page)
 640                 vaddr = page_address(page);
 641         return vaddr;
 642 }
 643
 644 static inline void free_pgtable_page(void *vaddr)
 645 {
 646         free_page((unsigned long)vaddr);
 647 }
 648
 649 static inline void *alloc_domain_mem(void)
 650 {
 651         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 652 }
 653
 654 static void free_domain_mem(void *vaddr)
 655 {
 656         kmem_cache_free(iommu_domain_cache, vaddr);
 657 }
 658
 659 static inline void * alloc_devinfo_mem(void)
 660 {
 661         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 662 }
 663
 664 static inline void free_devinfo_mem(void *vaddr)
 665 {
 666         kmem_cache_free(iommu_devinfo_cache, vaddr);
 667 }
 668
 669 static inline int domain_type_is_vm(struct dmar_domain *domain)
 670 {
 671         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 672 }
 673
 674 static inline int domain_type_is_si(struct dmar_domain *domain)
 675 {
 676         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 677 }
 678
 679 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 680 {
 681         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 682                                 DOMAIN_FLAG_STATIC_IDENTITY);
 683 }
 684
 685 static inline int domain_pfn_supported(struct dmar_domain *domain,
 686                                        unsigned long pfn)
 687 {
 688         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 689
 690         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 691 }
 692
 693 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 694 {
 695         unsigned long sagaw;
 696         int agaw = -1;
 697
 698         sagaw = cap_sagaw(iommu->cap);
 699         for (agaw = width_to_agaw(max_gaw);
 700              agaw >= 0; agaw--) {
 701                 if (test_bit(agaw, &sagaw))
 702                         break;
 703         }
 704
 705         return agaw;
 706 }
 707
 708 /*
 709  * Calculate max SAGAW for each iommu.
 710  */
 711 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 712 {
 713         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 714 }
 715
 716 /*
 717  * calculate agaw for each iommu.
 718  * "SAGAW" may be different across iommus, use a default agaw, and
 719  * get a supported less agaw for iommus that don't support the default agaw.
 720  */
 721 int iommu_calculate_agaw(struct intel_iommu *iommu)
 722 {
 723         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 724 }
 725
 726 /* This functionin only returns single iommu in a domain */
 727 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 728 {
 729         int iommu_id;
 730
 731         /* si_domain and vm domain should not get here. */
 732         BUG_ON(domain_type_is_vm_or_si(domain));
 733         for_each_domain_iommu(iommu_id, domain)
 734                 break;
 735
 736         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 737                 return NULL;
 738
 739         return g_iommus[iommu_id];
 740 }
 741
 742 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 743 {
 744         struct dmar_drhd_unit *drhd;
 745         struct intel_iommu *iommu;
 746         bool found = false;
 747         int i;
 748
 749         domain->iommu_coherency = 1;
 750
 751         for_each_domain_iommu(i, domain) {
 752                 found = true;
 753                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 754                         domain->iommu_coherency = 0;
 755                         break;
 756                 }
 757         }
 758         if (found)
 759                 return;
 760
 761         /* No hardware attached; use lowest common denominator */
 762         rcu_read_lock();
 763         for_each_active_iommu(iommu, drhd) {
 764                 if (!ecap_coherent(iommu->ecap)) {
 765                         domain->iommu_coherency = 0;
 766                         break;
 767                 }
 768         }
 769         rcu_read_unlock();
 770 }
 771
 772 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 773 {
 774         struct dmar_drhd_unit *drhd;
 775         struct intel_iommu *iommu;
 776         int ret = 1;
 777
 778         rcu_read_lock();
 779         for_each_active_iommu(iommu, drhd) {
 780                 if (iommu != skip) {
 781                         if (!ecap_sc_support(iommu->ecap)) {
 782                                 ret = 0;
 783                                 break;
 784                         }
 785                 }
 786         }
 787         rcu_read_unlock();
 788
 789         return ret;
 790 }
 791
 792 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 793 {
 794         struct dmar_drhd_unit *drhd;
 795         struct intel_iommu *iommu;
 796         int mask = 0xf;
 797
 798         if (!intel_iommu_superpage) {
 799                 return 0;
 800         }
 801
 802         /* set iommu_superpage to the smallest common denominator */
 803         rcu_read_lock();
 804         for_each_active_iommu(iommu, drhd) {
 805                 if (iommu != skip) {
 806                         mask &= cap_super_page_val(iommu->cap);
 807                         if (!mask)
 808                                 break;
 809                 }
 810         }
 811         rcu_read_unlock();
 812
 813         return fls(mask);
 814 }
 815
 816 /* Some capabilities may be different across iommus */
 817 static void domain_update_iommu_cap(struct dmar_domain *domain)
 818 {
 819         domain_update_iommu_coherency(domain);
 820         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 821         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 822 }
 823
 824 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 825                                                        u8 bus, u8 devfn, int alloc)
 826 {
 827         struct root_entry *root = &iommu->root_entry[bus];
 828         struct context_entry *context;
 829         u64 *entry;
 830
 831         entry = &root->lo;
 832         if (ecs_enabled(iommu)) {
 833                 if (devfn >= 0x80) {
 834                         devfn -= 0x80;
 835                         entry = &root->hi;
 836                 }
 837                 devfn *= 2;
 838         }
 839         if (*entry & 1)
 840                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 841         else {
 842                 unsigned long phy_addr;
 843                 if (!alloc)
 844                         return NULL;
 845
 846                 context = alloc_pgtable_page(iommu->node);
 847                 if (!context)
 848                         return NULL;
 849
 850                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 851                 phy_addr = virt_to_phys((void *)context);
 852                 *entry = phy_addr | 1;
 853                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 854         }
 855         return &context[devfn];
 856 }
 857
 858 static int iommu_dummy(struct device *dev)
 859 {
 860         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 861 }
 862
 863 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 864 {
 865         struct dmar_drhd_unit *drhd = NULL;
 866         struct intel_iommu *iommu;
 867         struct device *tmp;
 868         struct pci_dev *ptmp, *pdev = NULL;
 869         u16 segment = 0;
 870         int i;
 871
 872         if (iommu_dummy(dev))
 873                 return NULL;
 874
 875         if (dev_is_pci(dev)) {
 876                 struct pci_dev *pf_pdev;
 877
 878                 pdev = to_pci_dev(dev);
 879
 880 #ifdef CONFIG_X86
 881                 /* VMD child devices currently cannot be handled individually */
 882                 if (is_vmd(pdev->bus))
 883                         return NULL;
 884 #endif
 885
 886                 /* VFs aren't listed in scope tables; we need to look up
 887                  * the PF instead to find the IOMMU. */
 888                 pf_pdev = pci_physfn(pdev);
 889                 dev = &pf_pdev->dev;
 890                 segment = pci_domain_nr(pdev->bus);
 891         } else if (has_acpi_companion(dev))
 892                 dev = &ACPI_COMPANION(dev)->dev;
 893
 894         rcu_read_lock();
 895         for_each_active_iommu(iommu, drhd) {
 896                 if (pdev && segment != drhd->segment)
 897                         continue;
 898
 899                 for_each_active_dev_scope(drhd->devices,
 900                                           drhd->devices_cnt, i, tmp) {
 901                         if (tmp == dev) {
 902                                 /* For a VF use its original BDF# not that of the PF
 903                                  * which we used for the IOMMU lookup. Strictly speaking
 904                                  * we could do this for all PCI devices; we only need to
 905                                  * get the BDF# from the scope table for ACPI matches. */
 906                                 if (pdev && pdev->is_virtfn)
 907                                         goto got_pdev;
 908
 909                                 *bus = drhd->devices[i].bus;
 910                                 *devfn = drhd->devices[i].devfn;
 911                                 goto out;
 912                         }
 913
 914                         if (!pdev || !dev_is_pci(tmp))
 915                                 continue;
 916
 917                         ptmp = to_pci_dev(tmp);
 918                         if (ptmp->subordinate &&
 919                             ptmp->subordinate->number <= pdev->bus->number &&
 920                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 921                                 goto got_pdev;
 922                 }
 923
 924                 if (pdev && drhd->include_all) {
 925                 got_pdev:
 926                         *bus = pdev->bus->number;
 927                         *devfn = pdev->devfn;
 928                         goto out;
 929                 }
 930         }
 931         iommu = NULL;
 932  out:
 933         rcu_read_unlock();
 934
 935         return iommu;
 936 }
 937
 938 static void domain_flush_cache(struct dmar_domain *domain,
 939                                void *addr, int size)
 940 {
 941         if (!domain->iommu_coherency)
 942                 clflush_cache_range(addr, size);
 943 }
 944
 945 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 946 {
 947         struct context_entry *context;
 948         int ret = 0;
 949         unsigned long flags;
 950
 951         spin_lock_irqsave(&iommu->lock, flags);
 952         context = iommu_context_addr(iommu, bus, devfn, 0);
 953         if (context)
 954                 ret = context_present(context);
 955         spin_unlock_irqrestore(&iommu->lock, flags);
 956         return ret;
 957 }
 958
 959 static void free_context_table(struct intel_iommu *iommu)
 960 {
 961         int i;
 962         unsigned long flags;
 963         struct context_entry *context;
 964
 965         spin_lock_irqsave(&iommu->lock, flags);
 966         if (!iommu->root_entry) {
 967                 goto out;
 968         }
 969         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 970                 context = iommu_context_addr(iommu, i, 0, 0);
 971                 if (context)
 972                         free_pgtable_page(context);
 973
 974                 if (!ecs_enabled(iommu))
 975                         continue;
 976
 977                 context = iommu_context_addr(iommu, i, 0x80, 0);
 978                 if (context)
 979                         free_pgtable_page(context);
 980
 981         }
 982         free_pgtable_page(iommu->root_entry);
 983         iommu->root_entry = NULL;
 984 out:
 985         spin_unlock_irqrestore(&iommu->lock, flags);
 986 }
 987
 988 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 989                                       unsigned long pfn, int *target_level)
 990 {
 991         struct dma_pte *parent, *pte = NULL;
 992         int level = agaw_to_level(domain->agaw);
 993         int offset;
 994
 995         BUG_ON(!domain->pgd);
 996
 997         if (!domain_pfn_supported(domain, pfn))
 998                 /* Address beyond IOMMU's addressing capabilities. */
 999                 return NULL;
1000
1001         parent = domain->pgd;
1002
1003         while (1) {
1004                 void *tmp_page;
1005
1006                 offset = pfn_level_offset(pfn, level);
1007                 pte = &parent[offset];
1008                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1009                         break;
1010                 if (level == *target_level)
1011                         break;
1012
1013                 if (!dma_pte_present(pte)) {
1014                         uint64_t pteval;
1015
1016                         tmp_page = alloc_pgtable_page(domain->nid);
1017
1018                         if (!tmp_page)
1019                                 return NULL;
1020
1021                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1022                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1023                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1024                                 /* Someone else set it while we were thinking; use theirs. */
1025                                 free_pgtable_page(tmp_page);
1026                         else
1027                                 domain_flush_cache(domain, pte, sizeof(*pte));
1028                 }
1029                 if (level == 1)
1030                         break;
1031
1032                 parent = phys_to_virt(dma_pte_addr(pte));
1033                 level--;
1034         }
1035
1036         if (!*target_level)
1037                 *target_level = level;
1038
1039         return pte;
1040 }
1041
1042
1043 /* return address's pte at specific level */
1044 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1045                                          unsigned long pfn,
1046                                          int level, int *large_page)
1047 {
1048         struct dma_pte *parent, *pte = NULL;
1049         int total = agaw_to_level(domain->agaw);
1050         int offset;
1051
1052         parent = domain->pgd;
1053         while (level <= total) {
1054                 offset = pfn_level_offset(pfn, total);
1055                 pte = &parent[offset];
1056                 if (level == total)
1057                         return pte;
1058
1059                 if (!dma_pte_present(pte)) {
1060                         *large_page = total;
1061                         break;
1062                 }
1063
1064                 if (dma_pte_superpage(pte)) {
1065                         *large_page = total;
1066                         return pte;
1067                 }
1068
1069                 parent = phys_to_virt(dma_pte_addr(pte));
1070                 total--;
1071         }
1072         return NULL;
1073 }
1074
1075 /* clear last level pte, a tlb flush should be followed */
1076 static void dma_pte_clear_range(struct dmar_domain *domain,
1077                                 unsigned long start_pfn,
1078                                 unsigned long last_pfn)
1079 {
1080         unsigned int large_page = 1;
1081         struct dma_pte *first_pte, *pte;
1082
1083         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1084         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1085         BUG_ON(start_pfn > last_pfn);
1086
1087         /* we don't need lock here; nobody else touches the iova range */
1088         do {
1089                 large_page = 1;
1090                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1091                 if (!pte) {
1092                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1093                         continue;
1094                 }
1095                 do {
1096                         dma_clear_pte(pte);
1097                         start_pfn += lvl_to_nr_pages(large_page);
1098                         pte++;
1099                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1100
1101                 domain_flush_cache(domain, first_pte,
1102                                    (void *)pte - (void *)first_pte);
1103
1104         } while (start_pfn && start_pfn <= last_pfn);
1105 }
1106
1107 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1108                                int retain_level, struct dma_pte *pte,
1109                                unsigned long pfn, unsigned long start_pfn,
1110                                unsigned long last_pfn)
1111 {
1112         pfn = max(start_pfn, pfn);
1113         pte = &pte[pfn_level_offset(pfn, level)];
1114
1115         do {
1116                 unsigned long level_pfn;
1117                 struct dma_pte *level_pte;
1118
1119                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1120                         goto next;
1121
1122                 level_pfn = pfn & level_mask(level);
1123                 level_pte = phys_to_virt(dma_pte_addr(pte));
1124
1125                 if (level > 2) {
1126                         dma_pte_free_level(domain, level - 1, retain_level,
1127                                            level_pte, level_pfn, start_pfn,
1128                                            last_pfn);
1129                 }
1130
1131                 /*
1132                  * Free the page table if we're below the level we want to
1133                  * retain and the range covers the entire table.
1134                  */
1135                 if (level < retain_level && !(start_pfn > level_pfn ||
1136                       last_pfn < level_pfn + level_size(level) - 1)) {
1137                         dma_clear_pte(pte);
1138                         domain_flush_cache(domain, pte, sizeof(*pte));
1139                         free_pgtable_page(level_pte);
1140                 }
1141 next:
1142                 pfn += level_size(level);
1143         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1144 }
1145
1146 /*
1147  * clear last level (leaf) ptes and free page table pages below the
1148  * level we wish to keep intact.
1149  */
1150 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1151                                    unsigned long start_pfn,
1152                                    unsigned long last_pfn,
1153                                    int retain_level)
1154 {
1155         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1156         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1157         BUG_ON(start_pfn > last_pfn);
1158
1159         dma_pte_clear_range(domain, start_pfn, last_pfn);
1160
1161         /* We don't need lock here; nobody else touches the iova range */
1162         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1163                            domain->pgd, 0, start_pfn, last_pfn);
1164
1165         /* free pgd */
1166         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1167                 free_pgtable_page(domain->pgd);
1168                 domain->pgd = NULL;
1169         }
1170 }
1171
1172 /* When a page at a given level is being unlinked from its parent, we don't
1173    need to *modify* it at all. All we need to do is make a list of all the
1174    pages which can be freed just as soon as we've flushed the IOTLB and we
1175    know the hardware page-walk will no longer touch them.
1176    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1177    be freed. */
1178 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1179                                             int level, struct dma_pte *pte,
1180                                             struct page *freelist)
1181 {
1182         struct page *pg;
1183
1184         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1185         pg->freelist = freelist;
1186         freelist = pg;
1187
1188         if (level == 1)
1189                 return freelist;
1190
1191         pte = page_address(pg);
1192         do {
1193                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1194                         freelist = dma_pte_list_pagetables(domain, level - 1,
1195                                                            pte, freelist);
1196                 pte++;
1197         } while (!first_pte_in_page(pte));
1198
1199         return freelist;
1200 }
1201
1202 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1203                                         struct dma_pte *pte, unsigned long pfn,
1204                                         unsigned long start_pfn,
1205                                         unsigned long last_pfn,
1206                                         struct page *freelist)
1207 {
1208         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1209
1210         pfn = max(start_pfn, pfn);
1211         pte = &pte[pfn_level_offset(pfn, level)];
1212
1213         do {
1214                 unsigned long level_pfn;
1215
1216                 if (!dma_pte_present(pte))
1217                         goto next;
1218
1219                 level_pfn = pfn & level_mask(level);
1220
1221                 /* If range covers entire pagetable, free it */
1222                 if (start_pfn <= level_pfn &&
1223                     last_pfn >= level_pfn + level_size(level) - 1) {
1224                         /* These suborbinate page tables are going away entirely. Don't
1225                            bother to clear them; we're just going to *free* them. */
1226                         if (level > 1 && !dma_pte_superpage(pte))
1227                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1228
1229                         dma_clear_pte(pte);
1230                         if (!first_pte)
1231                                 first_pte = pte;
1232                         last_pte = pte;
1233                 } else if (level > 1) {
1234                         /* Recurse down into a level that isn't *entirely* obsolete */
1235                         freelist = dma_pte_clear_level(domain, level - 1,
1236                                                        phys_to_virt(dma_pte_addr(pte)),
1237                                                        level_pfn, start_pfn, last_pfn,
1238                                                        freelist);
1239                 }
1240 next:
1241                 pfn += level_size(level);
1242         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1243
1244         if (first_pte)
1245                 domain_flush_cache(domain, first_pte,
1246                                    (void *)++last_pte - (void *)first_pte);
1247
1248         return freelist;
1249 }
1250
1251 /* We can't just free the pages because the IOMMU may still be walking
1252    the page tables, and may have cached the intermediate levels. The
1253    pages can only be freed after the IOTLB flush has been done. */
1254 static struct page *domain_unmap(struct dmar_domain *domain,
1255                                  unsigned long start_pfn,
1256                                  unsigned long last_pfn)
1257 {
1258         struct page *freelist = NULL;
1259
1260         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1261         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1262         BUG_ON(start_pfn > last_pfn);
1263
1264         /* we don't need lock here; nobody else touches the iova range */
1265         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1266                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1267
1268         /* free pgd */
1269         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1270                 struct page *pgd_page = virt_to_page(domain->pgd);
1271                 pgd_page->freelist = freelist;
1272                 freelist = pgd_page;
1273
1274                 domain->pgd = NULL;
1275         }
1276
1277         return freelist;
1278 }
1279
1280 static void dma_free_pagelist(struct page *freelist)
1281 {
1282         struct page *pg;
1283
1284         while ((pg = freelist)) {
1285                 freelist = pg->freelist;
1286                 free_pgtable_page(page_address(pg));
1287         }
1288 }
1289
1290 static void iova_entry_free(unsigned long data)
1291 {
1292         struct page *freelist = (struct page *)data;
1293
1294         dma_free_pagelist(freelist);
1295 }
1296
1297 /* iommu handling */
1298 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1299 {
1300         struct root_entry *root;
1301         unsigned long flags;
1302
1303         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1304         if (!root) {
1305                 pr_err("Allocating root entry for %s failed\n",
1306                         iommu->name);
1307                 return -ENOMEM;
1308         }
1309
1310         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1311
1312         spin_lock_irqsave(&iommu->lock, flags);
1313         iommu->root_entry = root;
1314         spin_unlock_irqrestore(&iommu->lock, flags);
1315
1316         return 0;
1317 }
1318
1319 static void iommu_set_root_entry(struct intel_iommu *iommu)
1320 {
1321         u64 addr;
1322         u32 sts;
1323         unsigned long flag;
1324
1325         addr = virt_to_phys(iommu->root_entry);
1326         if (ecs_enabled(iommu))
1327                 addr |= DMA_RTADDR_RTT;
1328
1329         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1330         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1331
1332         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1333
1334         /* Make sure hardware complete it */
1335         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1336                       readl, (sts & DMA_GSTS_RTPS), sts);
1337
1338         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1339 }
1340
1341 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1342 {
1343         u32 val;
1344         unsigned long flag;
1345
1346         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1347                 return;
1348
1349         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1350         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1351
1352         /* Make sure hardware complete it */
1353         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1354                       readl, (!(val & DMA_GSTS_WBFS)), val);
1355
1356         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1357 }
1358
1359 /* return value determine if we need a write buffer flush */
1360 static void __iommu_flush_context(struct intel_iommu *iommu,
1361                                   u16 did, u16 source_id, u8 function_mask,
1362                                   u64 type)
1363 {
1364         u64 val = 0;
1365         unsigned long flag;
1366
1367         switch (type) {
1368         case DMA_CCMD_GLOBAL_INVL:
1369                 val = DMA_CCMD_GLOBAL_INVL;
1370                 break;
1371         case DMA_CCMD_DOMAIN_INVL:
1372                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1373                 break;
1374         case DMA_CCMD_DEVICE_INVL:
1375                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1376                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1377                 break;
1378         default:
1379                 BUG();
1380         }
1381         val |= DMA_CCMD_ICC;
1382
1383         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1384         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1385
1386         /* Make sure hardware complete it */
1387         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1388                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1389
1390         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1391 }
1392
1393 /* return value determine if we need a write buffer flush */
1394 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1395                                 u64 addr, unsigned int size_order, u64 type)
1396 {
1397         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1398         u64 val = 0, val_iva = 0;
1399         unsigned long flag;
1400
1401         switch (type) {
1402         case DMA_TLB_GLOBAL_FLUSH:
1403                 /* global flush doesn't need set IVA_REG */
1404                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1405                 break;
1406         case DMA_TLB_DSI_FLUSH:
1407                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1408                 break;
1409         case DMA_TLB_PSI_FLUSH:
1410                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1411                 /* IH bit is passed in as part of address */
1412                 val_iva = size_order | addr;
1413                 break;
1414         default:
1415                 BUG();
1416         }
1417         /* Note: set drain read/write */
1418 #if 0
1419         /*
1420          * This is probably to be super secure.. Looks like we can
1421          * ignore it without any impact.
1422          */
1423         if (cap_read_drain(iommu->cap))
1424                 val |= DMA_TLB_READ_DRAIN;
1425 #endif
1426         if (cap_write_drain(iommu->cap))
1427                 val |= DMA_TLB_WRITE_DRAIN;
1428
1429         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1430         /* Note: Only uses first TLB reg currently */
1431         if (val_iva)
1432                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1433         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1434
1435         /* Make sure hardware complete it */
1436         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1437                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1438
1439         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1440
1441         /* check IOTLB invalidation granularity */
1442         if (DMA_TLB_IAIG(val) == 0)
1443                 pr_err("Flush IOTLB failed\n");
1444         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1445                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1446                         (unsigned long long)DMA_TLB_IIRG(type),
1447                         (unsigned long long)DMA_TLB_IAIG(val));
1448 }
1449
1450 static struct device_domain_info *
1451 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1452                          u8 bus, u8 devfn)
1453 {
1454         struct device_domain_info *info;
1455
1456         assert_spin_locked(&device_domain_lock);
1457
1458         if (!iommu->qi)
1459                 return NULL;
1460
1461         list_for_each_entry(info, &domain->devices, link)
1462                 if (info->iommu == iommu && info->bus == bus &&
1463                     info->devfn == devfn) {
1464                         if (info->ats_supported && info->dev)
1465                                 return info;
1466                         break;
1467                 }
1468
1469         return NULL;
1470 }
1471
1472 static void domain_update_iotlb(struct dmar_domain *domain)
1473 {
1474         struct device_domain_info *info;
1475         bool has_iotlb_device = false;
1476
1477         assert_spin_locked(&device_domain_lock);
1478
1479         list_for_each_entry(info, &domain->devices, link) {
1480                 struct pci_dev *pdev;
1481
1482                 if (!info->dev || !dev_is_pci(info->dev))
1483                         continue;
1484
1485                 pdev = to_pci_dev(info->dev);
1486                 if (pdev->ats_enabled) {
1487                         has_iotlb_device = true;
1488                         break;
1489                 }
1490         }
1491
1492         domain->has_iotlb_device = has_iotlb_device;
1493 }
1494
1495 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1496 {
1497         struct pci_dev *pdev;
1498
1499         assert_spin_locked(&device_domain_lock);
1500
1501         if (!info || !dev_is_pci(info->dev))
1502                 return;
1503
1504         pdev = to_pci_dev(info->dev);
1505
1506 #ifdef CONFIG_INTEL_IOMMU_SVM
1507         /* The PCIe spec, in its wisdom, declares that the behaviour of
1508            the device if you enable PASID support after ATS support is
1509            undefined. So always enable PASID support on devices which
1510            have it, even if we can't yet know if we're ever going to
1511            use it. */
1512         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1513                 info->pasid_enabled = 1;
1514
1515         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1516                 info->pri_enabled = 1;
1517 #endif
1518         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1519                 info->ats_enabled = 1;
1520                 domain_update_iotlb(info->domain);
1521                 info->ats_qdep = pci_ats_queue_depth(pdev);
1522         }
1523 }
1524
1525 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1526 {
1527         struct pci_dev *pdev;
1528
1529         assert_spin_locked(&device_domain_lock);
1530
1531         if (!dev_is_pci(info->dev))
1532                 return;
1533
1534         pdev = to_pci_dev(info->dev);
1535
1536         if (info->ats_enabled) {
1537                 pci_disable_ats(pdev);
1538                 info->ats_enabled = 0;
1539                 domain_update_iotlb(info->domain);
1540         }
1541 #ifdef CONFIG_INTEL_IOMMU_SVM
1542         if (info->pri_enabled) {
1543                 pci_disable_pri(pdev);
1544                 info->pri_enabled = 0;
1545         }
1546         if (info->pasid_enabled) {
1547                 pci_disable_pasid(pdev);
1548                 info->pasid_enabled = 0;
1549         }
1550 #endif
1551 }
1552
1553 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1554                                   u64 addr, unsigned mask)
1555 {
1556         u16 sid, qdep;
1557         unsigned long flags;
1558         struct device_domain_info *info;
1559
1560         if (!domain->has_iotlb_device)
1561                 return;
1562
1563         spin_lock_irqsave(&device_domain_lock, flags);
1564         list_for_each_entry(info, &domain->devices, link) {
1565                 if (!info->ats_enabled)
1566                         continue;
1567
1568                 sid = info->bus << 8 | info->devfn;
1569                 qdep = info->ats_qdep;
1570                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1571         }
1572         spin_unlock_irqrestore(&device_domain_lock, flags);
1573 }
1574
1575 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1576                                   struct dmar_domain *domain,
1577                                   unsigned long pfn, unsigned int pages,
1578                                   int ih, int map)
1579 {
1580         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1581         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1582         u16 did = domain->iommu_did[iommu->seq_id];
1583
1584         BUG_ON(pages == 0);
1585
1586         if (ih)
1587                 ih = 1 << 6;
1588         /*
1589          * Fallback to domain selective flush if no PSI support or the size is
1590          * too big.
1591          * PSI requires page size to be 2 ^ x, and the base address is naturally
1592          * aligned to the size
1593          */
1594         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1595                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1596                                                 DMA_TLB_DSI_FLUSH);
1597         else
1598                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1599                                                 DMA_TLB_PSI_FLUSH);
1600
1601         /*
1602          * In caching mode, changes of pages from non-present to present require
1603          * flush. However, device IOTLB doesn't need to be flushed in this case.
1604          */
1605         if (!cap_caching_mode(iommu->cap) || !map)
1606                 iommu_flush_dev_iotlb(domain, addr, mask);
1607 }
1608
1609 static void iommu_flush_iova(struct iova_domain *iovad)
1610 {
1611         struct dmar_domain *domain;
1612         int idx;
1613
1614         domain = container_of(iovad, struct dmar_domain, iovad);
1615
1616         for_each_domain_iommu(idx, domain) {
1617                 struct intel_iommu *iommu = g_iommus[idx];
1618                 u16 did = domain->iommu_did[iommu->seq_id];
1619
1620                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1621
1622                 if (!cap_caching_mode(iommu->cap))
1623                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1624                                               0, MAX_AGAW_PFN_WIDTH);
1625         }
1626 }
1627
1628 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1629 {
1630         u32 pmen;
1631         unsigned long flags;
1632
1633         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1634         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1635         pmen &= ~DMA_PMEN_EPM;
1636         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1637
1638         /* wait for the protected region status bit to clear */
1639         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1640                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1641
1642         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1643 }
1644
1645 static void iommu_enable_translation(struct intel_iommu *iommu)
1646 {
1647         u32 sts;
1648         unsigned long flags;
1649
1650         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1651         iommu->gcmd |= DMA_GCMD_TE;
1652         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1653
1654         /* Make sure hardware complete it */
1655         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1656                       readl, (sts & DMA_GSTS_TES), sts);
1657
1658         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1659 }
1660
1661 static void iommu_disable_translation(struct intel_iommu *iommu)
1662 {
1663         u32 sts;
1664         unsigned long flag;
1665
1666         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1667         iommu->gcmd &= ~DMA_GCMD_TE;
1668         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1669
1670         /* Make sure hardware complete it */
1671         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1672                       readl, (!(sts & DMA_GSTS_TES)), sts);
1673
1674         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1675 }
1676
1677
1678 static int iommu_init_domains(struct intel_iommu *iommu)
1679 {
1680         u32 ndomains, nlongs;
1681         size_t size;
1682
1683         ndomains = cap_ndoms(iommu->cap);
1684         pr_debug("%s: Number of Domains supported <%d>\n",
1685                  iommu->name, ndomains);
1686         nlongs = BITS_TO_LONGS(ndomains);
1687
1688         spin_lock_init(&iommu->lock);
1689
1690         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1691         if (!iommu->domain_ids) {
1692                 pr_err("%s: Allocating domain id array failed\n",
1693                        iommu->name);
1694                 return -ENOMEM;
1695         }
1696
1697         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1698         iommu->domains = kzalloc(size, GFP_KERNEL);
1699
1700         if (iommu->domains) {
1701                 size = 256 * sizeof(struct dmar_domain *);
1702                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1703         }
1704
1705         if (!iommu->domains || !iommu->domains[0]) {
1706                 pr_err("%s: Allocating domain array failed\n",
1707                        iommu->name);
1708                 kfree(iommu->domain_ids);
1709                 kfree(iommu->domains);
1710                 iommu->domain_ids = NULL;
1711                 iommu->domains    = NULL;
1712                 return -ENOMEM;
1713         }
1714
1715
1716
1717         /*
1718          * If Caching mode is set, then invalid translations are tagged
1719          * with domain-id 0, hence we need to pre-allocate it. We also
1720          * use domain-id 0 as a marker for non-allocated domain-id, so
1721          * make sure it is not used for a real domain.
1722          */
1723         set_bit(0, iommu->domain_ids);
1724
1725         return 0;
1726 }
1727
1728 static void disable_dmar_iommu(struct intel_iommu *iommu)
1729 {
1730         struct device_domain_info *info, *tmp;
1731         unsigned long flags;
1732
1733         if (!iommu->domains || !iommu->domain_ids)
1734                 return;
1735
1736 again:
1737         spin_lock_irqsave(&device_domain_lock, flags);
1738         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1739                 struct dmar_domain *domain;
1740
1741                 if (info->iommu != iommu)
1742                         continue;
1743
1744                 if (!info->dev || !info->domain)
1745                         continue;
1746
1747                 domain = info->domain;
1748
1749                 __dmar_remove_one_dev_info(info);
1750
1751                 if (!domain_type_is_vm_or_si(domain)) {
1752                         /*
1753                          * The domain_exit() function  can't be called under
1754                          * device_domain_lock, as it takes this lock itself.
1755                          * So release the lock here and re-run the loop
1756                          * afterwards.
1757                          */
1758                         spin_unlock_irqrestore(&device_domain_lock, flags);
1759                         domain_exit(domain);
1760                         goto again;
1761                 }
1762         }
1763         spin_unlock_irqrestore(&device_domain_lock, flags);
1764
1765         if (iommu->gcmd & DMA_GCMD_TE)
1766                 iommu_disable_translation(iommu);
1767 }
1768
1769 static void free_dmar_iommu(struct intel_iommu *iommu)
1770 {
1771         if ((iommu->domains) && (iommu->domain_ids)) {
1772                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1773                 int i;
1774
1775                 for (i = 0; i < elems; i++)
1776                         kfree(iommu->domains[i]);
1777                 kfree(iommu->domains);
1778                 kfree(iommu->domain_ids);
1779                 iommu->domains = NULL;
1780                 iommu->domain_ids = NULL;
1781         }
1782
1783         g_iommus[iommu->seq_id] = NULL;
1784
1785         /* free context mapping */
1786         free_context_table(iommu);
1787
1788 #ifdef CONFIG_INTEL_IOMMU_SVM
1789         if (pasid_enabled(iommu)) {
1790                 if (ecap_prs(iommu->ecap))
1791                         intel_svm_finish_prq(iommu);
1792                 intel_svm_free_pasid_tables(iommu);
1793         }
1794 #endif
1795 }
1796
1797 static struct dmar_domain *alloc_domain(int flags)
1798 {
1799         struct dmar_domain *domain;
1800
1801         domain = alloc_domain_mem();
1802         if (!domain)
1803                 return NULL;
1804
1805         memset(domain, 0, sizeof(*domain));
1806         domain->nid = -1;
1807         domain->flags = flags;
1808         domain->has_iotlb_device = false;
1809         INIT_LIST_HEAD(&domain->devices);
1810
1811         return domain;
1812 }
1813
1814 /* Must be called with iommu->lock */
1815 static int domain_attach_iommu(struct dmar_domain *domain,
1816                                struct intel_iommu *iommu)
1817 {
1818         unsigned long ndomains;
1819         int num;
1820
1821         assert_spin_locked(&device_domain_lock);
1822         assert_spin_locked(&iommu->lock);
1823
1824         domain->iommu_refcnt[iommu->seq_id] += 1;
1825         domain->iommu_count += 1;
1826         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1827                 ndomains = cap_ndoms(iommu->cap);
1828                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1829
1830                 if (num >= ndomains) {
1831                         pr_err("%s: No free domain ids\n", iommu->name);
1832                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1833                         domain->iommu_count -= 1;
1834                         return -ENOSPC;
1835                 }
1836
1837                 set_bit(num, iommu->domain_ids);
1838                 set_iommu_domain(iommu, num, domain);
1839
1840                 domain->iommu_did[iommu->seq_id] = num;
1841                 domain->nid                      = iommu->node;
1842
1843                 domain_update_iommu_cap(domain);
1844         }
1845
1846         return 0;
1847 }
1848
1849 static int domain_detach_iommu(struct dmar_domain *domain,
1850                                struct intel_iommu *iommu)
1851 {
1852         int num, count = INT_MAX;
1853
1854         assert_spin_locked(&device_domain_lock);
1855         assert_spin_locked(&iommu->lock);
1856
1857         domain->iommu_refcnt[iommu->seq_id] -= 1;
1858         count = --domain->iommu_count;
1859         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1860                 num = domain->iommu_did[iommu->seq_id];
1861                 clear_bit(num, iommu->domain_ids);
1862                 set_iommu_domain(iommu, num, NULL);
1863
1864                 domain_update_iommu_cap(domain);
1865                 domain->iommu_did[iommu->seq_id] = 0;
1866         }
1867
1868         return count;
1869 }
1870
1871 static struct iova_domain reserved_iova_list;
1872 static struct lock_class_key reserved_rbtree_key;
1873
1874 static int dmar_init_reserved_ranges(void)
1875 {
1876         struct pci_dev *pdev = NULL;
1877         struct iova *iova;
1878         int i;
1879
1880         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1881
1882         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1883                 &reserved_rbtree_key);
1884
1885         /* IOAPIC ranges shouldn't be accessed by DMA */
1886         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1887                 IOVA_PFN(IOAPIC_RANGE_END));
1888         if (!iova) {
1889                 pr_err("Reserve IOAPIC range failed\n");
1890                 return -ENODEV;
1891         }
1892
1893         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1894         for_each_pci_dev(pdev) {
1895                 struct resource *r;
1896
1897                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1898                         r = &pdev->resource[i];
1899                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1900                                 continue;
1901                         iova = reserve_iova(&reserved_iova_list,
1902                                             IOVA_PFN(r->start),
1903                                             IOVA_PFN(r->end));
1904                         if (!iova) {
1905                                 pr_err("Reserve iova failed\n");
1906                                 return -ENODEV;
1907                         }
1908                 }
1909         }
1910         return 0;
1911 }
1912
1913 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1914 {
1915         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1916 }
1917
1918 static inline int guestwidth_to_adjustwidth(int gaw)
1919 {
1920         int agaw;
1921         int r = (gaw - 12) % 9;
1922
1923         if (r == 0)
1924                 agaw = gaw;
1925         else
1926                 agaw = gaw + 9 - r;
1927         if (agaw > 64)
1928                 agaw = 64;
1929         return agaw;
1930 }
1931
1932 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1933                        int guest_width)
1934 {
1935         int adjust_width, agaw;
1936         unsigned long sagaw;
1937         int err;
1938
1939         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1940
1941         err = init_iova_flush_queue(&domain->iovad,
1942                                     iommu_flush_iova, iova_entry_free);
1943         if (err)
1944                 return err;
1945
1946         domain_reserve_special_ranges(domain);
1947
1948         /* calculate AGAW */
1949         if (guest_width > cap_mgaw(iommu->cap))
1950                 guest_width = cap_mgaw(iommu->cap);
1951         domain->gaw = guest_width;
1952         adjust_width = guestwidth_to_adjustwidth(guest_width);
1953         agaw = width_to_agaw(adjust_width);
1954         sagaw = cap_sagaw(iommu->cap);
1955         if (!test_bit(agaw, &sagaw)) {
1956                 /* hardware doesn't support it, choose a bigger one */
1957                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1958                 agaw = find_next_bit(&sagaw, 5, agaw);
1959                 if (agaw >= 5)
1960                         return -ENODEV;
1961         }
1962         domain->agaw = agaw;
1963
1964         if (ecap_coherent(iommu->ecap))
1965                 domain->iommu_coherency = 1;
1966         else
1967                 domain->iommu_coherency = 0;
1968
1969         if (ecap_sc_support(iommu->ecap))
1970                 domain->iommu_snooping = 1;
1971         else
1972                 domain->iommu_snooping = 0;
1973
1974         if (intel_iommu_superpage)
1975                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1976         else
1977                 domain->iommu_superpage = 0;
1978
1979         domain->nid = iommu->node;
1980
1981         /* always allocate the top pgd */
1982         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1983         if (!domain->pgd)
1984                 return -ENOMEM;
1985         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1986         return 0;
1987 }
1988
1989 static void domain_exit(struct dmar_domain *domain)
1990 {
1991         struct page *freelist = NULL;
1992
1993         /* Domain 0 is reserved, so dont process it */
1994         if (!domain)
1995                 return;
1996
1997         /* Remove associated devices and clear attached or cached domains */
1998         rcu_read_lock();
1999         domain_remove_dev_info(domain);
2000         rcu_read_unlock();
2001
2002         /* destroy iovas */
2003         put_iova_domain(&domain->iovad);
2004
2005         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2006
2007         dma_free_pagelist(freelist);
2008
2009         free_domain_mem(domain);
2010 }
2011
2012 static int domain_context_mapping_one(struct dmar_domain *domain,
2013                                       struct intel_iommu *iommu,
2014                                       u8 bus, u8 devfn)
2015 {
2016         u16 did = domain->iommu_did[iommu->seq_id];
2017         int translation = CONTEXT_TT_MULTI_LEVEL;
2018         struct device_domain_info *info = NULL;
2019         struct context_entry *context;
2020         unsigned long flags;
2021         struct dma_pte *pgd;
2022         int ret, agaw;
2023
2024         WARN_ON(did == 0);
2025
2026         if (hw_pass_through && domain_type_is_si(domain))
2027                 translation = CONTEXT_TT_PASS_THROUGH;
2028
2029         pr_debug("Set context mapping for %02x:%02x.%d\n",
2030                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2031
2032         BUG_ON(!domain->pgd);
2033
2034         spin_lock_irqsave(&device_domain_lock, flags);
2035         spin_lock(&iommu->lock);
2036
2037         ret = -ENOMEM;
2038         context = iommu_context_addr(iommu, bus, devfn, 1);
2039         if (!context)
2040                 goto out_unlock;
2041
2042         ret = 0;
2043         if (context_present(context))
2044                 goto out_unlock;
2045
2046         /*
2047          * For kdump cases, old valid entries may be cached due to the
2048          * in-flight DMA and copied pgtable, but there is no unmapping
2049          * behaviour for them, thus we need an explicit cache flush for
2050          * the newly-mapped device. For kdump, at this point, the device
2051          * is supposed to finish reset at its driver probe stage, so no
2052          * in-flight DMA will exist, and we don't need to worry anymore
2053          * hereafter.
2054          */
2055         if (context_copied(context)) {
2056                 u16 did_old = context_domain_id(context);
2057
2058                 if (did_old < cap_ndoms(iommu->cap)) {
2059                         iommu->flush.flush_context(iommu, did_old,
2060                                                    (((u16)bus) << 8) | devfn,
2061                                                    DMA_CCMD_MASK_NOBIT,
2062                                                    DMA_CCMD_DEVICE_INVL);
2063                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2064                                                  DMA_TLB_DSI_FLUSH);
2065                 }
2066         }
2067
2068         pgd = domain->pgd;
2069
2070         context_clear_entry(context);
2071         context_set_domain_id(context, did);
2072
2073         /*
2074          * Skip top levels of page tables for iommu which has less agaw
2075          * than default.  Unnecessary for PT mode.
2076          */
2077         if (translation != CONTEXT_TT_PASS_THROUGH) {
2078                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2079                         ret = -ENOMEM;
2080                         pgd = phys_to_virt(dma_pte_addr(pgd));
2081                         if (!dma_pte_present(pgd))
2082                                 goto out_unlock;
2083                 }
2084
2085                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2086                 if (info && info->ats_supported)
2087                         translation = CONTEXT_TT_DEV_IOTLB;
2088                 else
2089                         translation = CONTEXT_TT_MULTI_LEVEL;
2090
2091                 context_set_address_root(context, virt_to_phys(pgd));
2092                 context_set_address_width(context, iommu->agaw);
2093         } else {
2094                 /*
2095                  * In pass through mode, AW must be programmed to
2096                  * indicate the largest AGAW value supported by
2097                  * hardware. And ASR is ignored by hardware.
2098                  */
2099                 context_set_address_width(context, iommu->msagaw);
2100         }
2101
2102         context_set_translation_type(context, translation);
2103         context_set_fault_enable(context);
2104         context_set_present(context);
2105         domain_flush_cache(domain, context, sizeof(*context));
2106
2107         /*
2108          * It's a non-present to present mapping. If hardware doesn't cache
2109          * non-present entry we only need to flush the write-buffer. If the
2110          * _does_ cache non-present entries, then it does so in the special
2111          * domain #0, which we have to flush:
2112          */
2113         if (cap_caching_mode(iommu->cap)) {
2114                 iommu->flush.flush_context(iommu, 0,
2115                                            (((u16)bus) << 8) | devfn,
2116                                            DMA_CCMD_MASK_NOBIT,
2117                                            DMA_CCMD_DEVICE_INVL);
2118                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2119         } else {
2120                 iommu_flush_write_buffer(iommu);
2121         }
2122         iommu_enable_dev_iotlb(info);
2123
2124         ret = 0;
2125
2126 out_unlock:
2127         spin_unlock(&iommu->lock);
2128         spin_unlock_irqrestore(&device_domain_lock, flags);
2129
2130         return ret;
2131 }
2132
2133 struct domain_context_mapping_data {
2134         struct dmar_domain *domain;
2135         struct intel_iommu *iommu;
2136 };
2137
2138 static int domain_context_mapping_cb(struct pci_dev *pdev,
2139                                      u16 alias, void *opaque)
2140 {
2141         struct domain_context_mapping_data *data = opaque;
2142
2143         return domain_context_mapping_one(data->domain, data->iommu,
2144                                           PCI_BUS_NUM(alias), alias & 0xff);
2145 }
2146
2147 static int
2148 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2149 {
2150         struct intel_iommu *iommu;
2151         u8 bus, devfn;
2152         struct domain_context_mapping_data data;
2153
2154         iommu = device_to_iommu(dev, &bus, &devfn);
2155         if (!iommu)
2156                 return -ENODEV;
2157
2158         if (!dev_is_pci(dev))
2159                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2160
2161         data.domain = domain;
2162         data.iommu = iommu;
2163
2164         return pci_for_each_dma_alias(to_pci_dev(dev),
2165                                       &domain_context_mapping_cb, &data);
2166 }
2167
2168 static int domain_context_mapped_cb(struct pci_dev *pdev,
2169                                     u16 alias, void *opaque)
2170 {
2171         struct intel_iommu *iommu = opaque;
2172
2173         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2174 }
2175
2176 static int domain_context_mapped(struct device *dev)
2177 {
2178         struct intel_iommu *iommu;
2179         u8 bus, devfn;
2180
2181         iommu = device_to_iommu(dev, &bus, &devfn);
2182         if (!iommu)
2183                 return -ENODEV;
2184
2185         if (!dev_is_pci(dev))
2186                 return device_context_mapped(iommu, bus, devfn);
2187
2188         return !pci_for_each_dma_alias(to_pci_dev(dev),
2189                                        domain_context_mapped_cb, iommu);
2190 }
2191
2192 /* Returns a number of VTD pages, but aligned to MM page size */
2193 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2194                                             size_t size)
2195 {
2196         host_addr &= ~PAGE_MASK;
2197         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2198 }
2199
2200 /* Return largest possible superpage level for a given mapping */
2201 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2202                                           unsigned long iov_pfn,
2203                                           unsigned long phy_pfn,
2204                                           unsigned long pages)
2205 {
2206         int support, level = 1;
2207         unsigned long pfnmerge;
2208
2209         support = domain->iommu_superpage;
2210
2211         /* To use a large page, the virtual *and* physical addresses
2212            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2213            of them will mean we have to use smaller pages. So just
2214            merge them and check both at once. */
2215         pfnmerge = iov_pfn | phy_pfn;
2216
2217         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2218                 pages >>= VTD_STRIDE_SHIFT;
2219                 if (!pages)
2220                         break;
2221                 pfnmerge >>= VTD_STRIDE_SHIFT;
2222                 level++;
2223                 support--;
2224         }
2225         return level;
2226 }
2227
2228 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2229                             struct scatterlist *sg, unsigned long phys_pfn,
2230                             unsigned long nr_pages, int prot)
2231 {
2232         struct dma_pte *first_pte = NULL, *pte = NULL;
2233         phys_addr_t uninitialized_var(pteval);
2234         unsigned long sg_res = 0;
2235         unsigned int largepage_lvl = 0;
2236         unsigned long lvl_pages = 0;
2237
2238         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2239
2240         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2241                 return -EINVAL;
2242
2243         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2244
2245         if (!sg) {
2246                 sg_res = nr_pages;
2247                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2248         }
2249
2250         while (nr_pages > 0) {
2251                 uint64_t tmp;
2252
2253                 if (!sg_res) {
2254                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2255
2256                         sg_res = aligned_nrpages(sg->offset, sg->length);
2257                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2258                         sg->dma_length = sg->length;
2259                         pteval = (sg_phys(sg) - pgoff) | prot;
2260                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2261                 }
2262
2263                 if (!pte) {
2264                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2265
2266                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2267                         if (!pte)
2268                                 return -ENOMEM;
2269                         /* It is large page*/
2270                         if (largepage_lvl > 1) {
2271                                 unsigned long nr_superpages, end_pfn;
2272
2273                                 pteval |= DMA_PTE_LARGE_PAGE;
2274                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2275
2276                                 nr_superpages = sg_res / lvl_pages;
2277                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2278
2279                                 /*
2280                                  * Ensure that old small page tables are
2281                                  * removed to make room for superpage(s).
2282                                  * We're adding new large pages, so make sure
2283                                  * we don't remove their parent tables.
2284                                  */
2285                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2286                                                        largepage_lvl + 1);
2287                         } else {
2288                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2289                         }
2290
2291                 }
2292                 /* We don't need lock here, nobody else
2293                  * touches the iova range
2294                  */
2295                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2296                 if (tmp) {
2297                         static int dumps = 5;
2298                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2299                                 iov_pfn, tmp, (unsigned long long)pteval);
2300                         if (dumps) {
2301                                 dumps--;
2302                                 debug_dma_dump_mappings(NULL);
2303                         }
2304                         WARN_ON(1);
2305                 }
2306
2307                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2308
2309                 BUG_ON(nr_pages < lvl_pages);
2310                 BUG_ON(sg_res < lvl_pages);
2311
2312                 nr_pages -= lvl_pages;
2313                 iov_pfn += lvl_pages;
2314                 phys_pfn += lvl_pages;
2315                 pteval += lvl_pages * VTD_PAGE_SIZE;
2316                 sg_res -= lvl_pages;
2317
2318                 /* If the next PTE would be the first in a new page, then we
2319                    need to flush the cache on the entries we've just written.
2320                    And then we'll need to recalculate 'pte', so clear it and
2321                    let it get set again in the if (!pte) block above.
2322
2323                    If we're done (!nr_pages) we need to flush the cache too.
2324
2325                    Also if we've been setting superpages, we may need to
2326                    recalculate 'pte' and switch back to smaller pages for the
2327                    end of the mapping, if the trailing size is not enough to
2328                    use another superpage (i.e. sg_res < lvl_pages). */
2329                 pte++;
2330                 if (!nr_pages || first_pte_in_page(pte) ||
2331                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2332                         domain_flush_cache(domain, first_pte,
2333                                            (void *)pte - (void *)first_pte);
2334                         pte = NULL;
2335                 }
2336
2337                 if (!sg_res && nr_pages)
2338                         sg = sg_next(sg);
2339         }
2340         return 0;
2341 }
2342
2343 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2344                                     struct scatterlist *sg, unsigned long nr_pages,
2345                                     int prot)
2346 {
2347         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2348 }
2349
2350 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2351                                      unsigned long phys_pfn, unsigned long nr_pages,
2352                                      int prot)
2353 {
2354         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2355 }
2356
2357 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2358 {
2359         unsigned long flags;
2360         struct context_entry *context;
2361         u16 did_old;
2362
2363         if (!iommu)
2364                 return;
2365
2366         spin_lock_irqsave(&iommu->lock, flags);
2367         context = iommu_context_addr(iommu, bus, devfn, 0);
2368         if (!context) {
2369                 spin_unlock_irqrestore(&iommu->lock, flags);
2370                 return;
2371         }
2372         did_old = context_domain_id(context);
2373         context_clear_entry(context);
2374         __iommu_flush_cache(iommu, context, sizeof(*context));
2375         spin_unlock_irqrestore(&iommu->lock, flags);
2376         iommu->flush.flush_context(iommu,
2377                                    did_old,
2378                                    (((u16)bus) << 8) | devfn,
2379                                    DMA_CCMD_MASK_NOBIT,
2380                                    DMA_CCMD_DEVICE_INVL);
2381         iommu->flush.flush_iotlb(iommu,
2382                                  did_old,
2383                                  0,
2384                                  0,
2385                                  DMA_TLB_DSI_FLUSH);
2386 }
2387
2388 static inline void unlink_domain_info(struct device_domain_info *info)
2389 {
2390         assert_spin_locked(&device_domain_lock);
2391         list_del(&info->link);
2392         list_del(&info->global);
2393         if (info->dev)
2394                 info->dev->archdata.iommu = NULL;
2395 }
2396
2397 static void domain_remove_dev_info(struct dmar_domain *domain)
2398 {
2399         struct device_domain_info *info, *tmp;
2400         unsigned long flags;
2401
2402         spin_lock_irqsave(&device_domain_lock, flags);
2403         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2404                 __dmar_remove_one_dev_info(info);
2405         spin_unlock_irqrestore(&device_domain_lock, flags);
2406 }
2407
2408 /*
2409  * find_domain
2410  * Note: we use struct device->archdata.iommu stores the info
2411  */
2412 static struct dmar_domain *find_domain(struct device *dev)
2413 {
2414         struct device_domain_info *info;
2415
2416         /* No lock here, assumes no domain exit in normal case */
2417         info = dev->archdata.iommu;
2418         if (likely(info))
2419                 return info->domain;
2420         return NULL;
2421 }
2422
2423 static inline struct device_domain_info *
2424 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2425 {
2426         struct device_domain_info *info;
2427
2428         list_for_each_entry(info, &device_domain_list, global)
2429                 if (info->iommu->segment == segment && info->bus == bus &&
2430                     info->devfn == devfn)
2431                         return info;
2432
2433         return NULL;
2434 }
2435
2436 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2437                                                     int bus, int devfn,
2438                                                     struct device *dev,
2439                                                     struct dmar_domain *domain)
2440 {
2441         struct dmar_domain *found = NULL;
2442         struct device_domain_info *info;
2443         unsigned long flags;
2444         int ret;
2445
2446         info = alloc_devinfo_mem();
2447         if (!info)
2448                 return NULL;
2449
2450         info->bus = bus;
2451         info->devfn = devfn;
2452         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2453         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2454         info->ats_qdep = 0;
2455         info->dev = dev;
2456         info->domain = domain;
2457         info->iommu = iommu;
2458
2459         if (dev && dev_is_pci(dev)) {
2460                 struct pci_dev *pdev = to_pci_dev(info->dev);
2461
2462                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2463                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2464                     dmar_find_matched_atsr_unit(pdev))
2465                         info->ats_supported = 1;
2466
2467                 if (ecs_enabled(iommu)) {
2468                         if (pasid_enabled(iommu)) {
2469                                 int features = pci_pasid_features(pdev);
2470                                 if (features >= 0)
2471                                         info->pasid_supported = features | 1;
2472                         }
2473
2474                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2475                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2476                                 info->pri_supported = 1;
2477                 }
2478         }
2479
2480         spin_lock_irqsave(&device_domain_lock, flags);
2481         if (dev)
2482                 found = find_domain(dev);
2483
2484         if (!found) {
2485                 struct device_domain_info *info2;
2486                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2487                 if (info2) {
2488                         found      = info2->domain;
2489                         info2->dev = dev;
2490                 }
2491         }
2492
2493         if (found) {
2494                 spin_unlock_irqrestore(&device_domain_lock, flags);
2495                 free_devinfo_mem(info);
2496                 /* Caller must free the original domain */
2497                 return found;
2498         }
2499
2500         spin_lock(&iommu->lock);
2501         ret = domain_attach_iommu(domain, iommu);
2502         spin_unlock(&iommu->lock);
2503
2504         if (ret) {
2505                 spin_unlock_irqrestore(&device_domain_lock, flags);
2506                 free_devinfo_mem(info);
2507                 return NULL;
2508         }
2509
2510         list_add(&info->link, &domain->devices);
2511         list_add(&info->global, &device_domain_list);
2512         if (dev)
2513                 dev->archdata.iommu = info;
2514         spin_unlock_irqrestore(&device_domain_lock, flags);
2515
2516         if (dev && domain_context_mapping(domain, dev)) {
2517                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2518                 dmar_remove_one_dev_info(domain, dev);
2519                 return NULL;
2520         }
2521
2522         return domain;
2523 }
2524
2525 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2526 {
2527         *(u16 *)opaque = alias;
2528         return 0;
2529 }
2530
2531 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2532 {
2533         struct device_domain_info *info = NULL;
2534         struct dmar_domain *domain = NULL;
2535         struct intel_iommu *iommu;
2536         u16 req_id, dma_alias;
2537         unsigned long flags;
2538         u8 bus, devfn;
2539
2540         iommu = device_to_iommu(dev, &bus, &devfn);
2541         if (!iommu)
2542                 return NULL;
2543
2544         req_id = ((u16)bus << 8) | devfn;
2545
2546         if (dev_is_pci(dev)) {
2547                 struct pci_dev *pdev = to_pci_dev(dev);
2548
2549                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2550
2551                 spin_lock_irqsave(&device_domain_lock, flags);
2552                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2553                                                       PCI_BUS_NUM(dma_alias),
2554                                                       dma_alias & 0xff);
2555                 if (info) {
2556                         iommu = info->iommu;
2557                         domain = info->domain;
2558                 }
2559                 spin_unlock_irqrestore(&device_domain_lock, flags);
2560
2561                 /* DMA alias already has a domain, use it */
2562                 if (info)
2563                         goto out;
2564         }
2565
2566         /* Allocate and initialize new domain for the device */
2567         domain = alloc_domain(0);
2568         if (!domain)
2569                 return NULL;
2570         if (domain_init(domain, iommu, gaw)) {
2571                 domain_exit(domain);
2572                 return NULL;
2573         }
2574
2575 out:
2576
2577         return domain;
2578 }
2579
2580 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2581                                               struct dmar_domain *domain)
2582 {
2583         struct intel_iommu *iommu;
2584         struct dmar_domain *tmp;
2585         u16 req_id, dma_alias;
2586         u8 bus, devfn;
2587
2588         iommu = device_to_iommu(dev, &bus, &devfn);
2589         if (!iommu)
2590                 return NULL;
2591
2592         req_id = ((u16)bus << 8) | devfn;
2593
2594         if (dev_is_pci(dev)) {
2595                 struct pci_dev *pdev = to_pci_dev(dev);
2596
2597                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2598
2599                 /* register PCI DMA alias device */
2600                 if (req_id != dma_alias) {
2601                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2602                                         dma_alias & 0xff, NULL, domain);
2603
2604                         if (!tmp || tmp != domain)
2605                                 return tmp;
2606                 }
2607         }
2608
2609         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2610         if (!tmp || tmp != domain)
2611                 return tmp;
2612
2613         return domain;
2614 }
2615
2616 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2617 {
2618         struct dmar_domain *domain, *tmp;
2619
2620         domain = find_domain(dev);
2621         if (domain)
2622                 goto out;
2623
2624         domain = find_or_alloc_domain(dev, gaw);
2625         if (!domain)
2626                 goto out;
2627
2628         tmp = set_domain_for_dev(dev, domain);
2629         if (!tmp || domain != tmp) {
2630                 domain_exit(domain);
2631                 domain = tmp;
2632         }
2633
2634 out:
2635
2636         return domain;
2637 }
2638
2639 static int iommu_domain_identity_map(struct dmar_domain *domain,
2640                                      unsigned long long start,
2641                                      unsigned long long end)
2642 {
2643         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2644         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2645
2646         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2647                           dma_to_mm_pfn(last_vpfn))) {
2648                 pr_err("Reserving iova failed\n");
2649                 return -ENOMEM;
2650         }
2651
2652         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2653         /*
2654          * RMRR range might have overlap with physical memory range,
2655          * clear it first
2656          */
2657         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2658
2659         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2660                                   last_vpfn - first_vpfn + 1,
2661                                   DMA_PTE_READ|DMA_PTE_WRITE);
2662 }
2663
2664 static int domain_prepare_identity_map(struct device *dev,
2665                                        struct dmar_domain *domain,
2666                                        unsigned long long start,
2667                                        unsigned long long end)
2668 {
2669         /* For _hardware_ passthrough, don't bother. But for software
2670            passthrough, we do it anyway -- it may indicate a memory
2671            range which is reserved in E820, so which didn't get set
2672            up to start with in si_domain */
2673         if (domain == si_domain && hw_pass_through) {
2674                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2675                         dev_name(dev), start, end);
2676                 return 0;
2677         }
2678
2679         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2680                 dev_name(dev), start, end);
2681
2682         if (end < start) {
2683                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2684                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2685                         dmi_get_system_info(DMI_BIOS_VENDOR),
2686                         dmi_get_system_info(DMI_BIOS_VERSION),
2687                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2688                 return -EIO;
2689         }
2690
2691         if (end >> agaw_to_width(domain->agaw)) {
2692                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2693                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2694                      agaw_to_width(domain->agaw),
2695                      dmi_get_system_info(DMI_BIOS_VENDOR),
2696                      dmi_get_system_info(DMI_BIOS_VERSION),
2697                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2698                 return -EIO;
2699         }
2700
2701         return iommu_domain_identity_map(domain, start, end);
2702 }
2703
2704 static int iommu_prepare_identity_map(struct device *dev,
2705                                       unsigned long long start,
2706                                       unsigned long long end)
2707 {
2708         struct dmar_domain *domain;
2709         int ret;
2710
2711         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2712         if (!domain)
2713                 return -ENOMEM;
2714
2715         ret = domain_prepare_identity_map(dev, domain, start, end);
2716         if (ret)
2717                 domain_exit(domain);
2718
2719         return ret;
2720 }
2721
2722 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2723                                          struct device *dev)
2724 {
2725         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2726                 return 0;
2727         return iommu_prepare_identity_map(dev, rmrr->base_address,
2728                                           rmrr->end_address);
2729 }
2730
2731 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2732 static inline void iommu_prepare_isa(void)
2733 {
2734         struct pci_dev *pdev;
2735         int ret;
2736
2737         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2738         if (!pdev)
2739                 return;
2740
2741         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2742         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2743
2744         if (ret)
2745                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2746
2747         pci_dev_put(pdev);
2748 }
2749 #else
2750 static inline void iommu_prepare_isa(void)
2751 {
2752         return;
2753 }
2754 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2755
2756 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2757
2758 static int __init si_domain_init(int hw)
2759 {
2760         int nid, ret = 0;
2761
2762         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2763         if (!si_domain)
2764                 return -EFAULT;
2765
2766         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2767                 domain_exit(si_domain);
2768                 return -EFAULT;
2769         }
2770
2771         pr_debug("Identity mapping domain allocated\n");
2772
2773         if (hw)
2774                 return 0;
2775
2776         for_each_online_node(nid) {
2777                 unsigned long start_pfn, end_pfn;
2778                 int i;
2779
2780                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2781                         ret = iommu_domain_identity_map(si_domain,
2782                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2783                         if (ret)
2784                                 return ret;
2785                 }
2786         }
2787
2788         return 0;
2789 }
2790
2791 static int identity_mapping(struct device *dev)
2792 {
2793         struct device_domain_info *info;
2794
2795         if (likely(!iommu_identity_mapping))
2796                 return 0;
2797
2798         info = dev->archdata.iommu;
2799         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2800                 return (info->domain == si_domain);
2801
2802         return 0;
2803 }
2804
2805 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2806 {
2807         struct dmar_domain *ndomain;
2808         struct intel_iommu *iommu;
2809         u8 bus, devfn;
2810
2811         iommu = device_to_iommu(dev, &bus, &devfn);
2812         if (!iommu)
2813                 return -ENODEV;
2814
2815         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2816         if (ndomain != domain)
2817                 return -EBUSY;
2818
2819         return 0;
2820 }
2821
2822 static bool device_has_rmrr(struct device *dev)
2823 {
2824         struct dmar_rmrr_unit *rmrr;
2825         struct device *tmp;
2826         int i;
2827
2828         rcu_read_lock();
2829         for_each_rmrr_units(rmrr) {
2830                 /*
2831                  * Return TRUE if this RMRR contains the device that
2832                  * is passed in.
2833                  */
2834                 for_each_active_dev_scope(rmrr->devices,
2835                                           rmrr->devices_cnt, i, tmp)
2836                         if (tmp == dev) {
2837                                 rcu_read_unlock();
2838                                 return true;
2839                         }
2840         }
2841         rcu_read_unlock();
2842         return false;
2843 }
2844
2845 /*
2846  * There are a couple cases where we need to restrict the functionality of
2847  * devices associated with RMRRs.  The first is when evaluating a device for
2848  * identity mapping because problems exist when devices are moved in and out
2849  * of domains and their respective RMRR information is lost.  This means that
2850  * a device with associated RMRRs will never be in a "passthrough" domain.
2851  * The second is use of the device through the IOMMU API.  This interface
2852  * expects to have full control of the IOVA space for the device.  We cannot
2853  * satisfy both the requirement that RMRR access is maintained and have an
2854  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2855  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2856  * We therefore prevent devices associated with an RMRR from participating in
2857  * the IOMMU API, which eliminates them from device assignment.
2858  *
2859  * In both cases we assume that PCI USB devices with RMRRs have them largely
2860  * for historical reasons and that the RMRR space is not actively used post
2861  * boot.  This exclusion may change if vendors begin to abuse it.
2862  *
2863  * The same exception is made for graphics devices, with the requirement that
2864  * any use of the RMRR regions will be torn down before assigning the device
2865  * to a guest.
2866  */
2867 static bool device_is_rmrr_locked(struct device *dev)
2868 {
2869         if (!device_has_rmrr(dev))
2870                 return false;
2871
2872         if (dev_is_pci(dev)) {
2873                 struct pci_dev *pdev = to_pci_dev(dev);
2874
2875                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2876                         return false;
2877         }
2878
2879         return true;
2880 }
2881
2882 static int iommu_should_identity_map(struct device *dev, int startup)
2883 {
2884
2885         if (dev_is_pci(dev)) {
2886                 struct pci_dev *pdev = to_pci_dev(dev);
2887
2888                 if (device_is_rmrr_locked(dev))
2889                         return 0;
2890
2891                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2892                         return 1;
2893
2894                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2895                         return 1;
2896
2897                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2898                         return 0;
2899
2900                 /*
2901                  * We want to start off with all devices in the 1:1 domain, and
2902                  * take them out later if we find they can't access all of memory.
2903                  *
2904                  * However, we can't do this for PCI devices behind bridges,
2905                  * because all PCI devices behind the same bridge will end up
2906                  * with the same source-id on their transactions.
2907                  *
2908                  * Practically speaking, we can't change things around for these
2909                  * devices at run-time, because we can't be sure there'll be no
2910                  * DMA transactions in flight for any of their siblings.
2911                  *
2912                  * So PCI devices (unless they're on the root bus) as well as
2913                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2914                  * the 1:1 domain, just in _case_ one of their siblings turns out
2915                  * not to be able to map all of memory.
2916                  */
2917                 if (!pci_is_pcie(pdev)) {
2918                         if (!pci_is_root_bus(pdev->bus))
2919                                 return 0;
2920                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2921                                 return 0;
2922                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2923                         return 0;
2924         } else {
2925                 if (device_has_rmrr(dev))
2926                         return 0;
2927         }
2928
2929         /*
2930          * At boot time, we don't yet know if devices will be 64-bit capable.
2931          * Assume that they will — if they turn out not to be, then we can
2932          * take them out of the 1:1 domain later.
2933          */
2934         if (!startup) {
2935                 /*
2936                  * If the device's dma_mask is less than the system's memory
2937                  * size then this is not a candidate for identity mapping.
2938                  */
2939                 u64 dma_mask = *dev->dma_mask;
2940
2941                 if (dev->coherent_dma_mask &&
2942                     dev->coherent_dma_mask < dma_mask)
2943                         dma_mask = dev->coherent_dma_mask;
2944
2945                 return dma_mask >= dma_get_required_mask(dev);
2946         }
2947
2948         return 1;
2949 }
2950
2951 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2952 {
2953         int ret;
2954
2955         if (!iommu_should_identity_map(dev, 1))
2956                 return 0;
2957
2958         ret = domain_add_dev_info(si_domain, dev);
2959         if (!ret)
2960                 pr_info("%s identity mapping for device %s\n",
2961                         hw ? "Hardware" : "Software", dev_name(dev));
2962         else if (ret == -ENODEV)
2963                 /* device not associated with an iommu */
2964                 ret = 0;
2965
2966         return ret;
2967 }
2968
2969
2970 static int __init iommu_prepare_static_identity_mapping(int hw)
2971 {
2972         struct pci_dev *pdev = NULL;
2973         struct dmar_drhd_unit *drhd;
2974         struct intel_iommu *iommu;
2975         struct device *dev;
2976         int i;
2977         int ret = 0;
2978
2979         for_each_pci_dev(pdev) {
2980                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2981                 if (ret)
2982                         return ret;
2983         }
2984
2985         for_each_active_iommu(iommu, drhd)
2986                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2987                         struct acpi_device_physical_node *pn;
2988                         struct acpi_device *adev;
2989
2990                         if (dev->bus != &acpi_bus_type)
2991                                 continue;
2992
2993                         adev= to_acpi_device(dev);
2994                         mutex_lock(&adev->physical_node_lock);
2995                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2996                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2997                                 if (ret)
2998                                         break;
2999                         }
3000                         mutex_unlock(&adev->physical_node_lock);
3001                         if (ret)
3002                                 return ret;
3003                 }
3004
3005         return 0;
3006 }
3007
3008 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3009 {
3010         /*
3011          * Start from the sane iommu hardware state.
3012          * If the queued invalidation is already initialized by us
3013          * (for example, while enabling interrupt-remapping) then
3014          * we got the things already rolling from a sane state.
3015          */
3016         if (!iommu->qi) {
3017                 /*
3018                  * Clear any previous faults.
3019                  */
3020                 dmar_fault(-1, iommu);
3021                 /*
3022                  * Disable queued invalidation if supported and already enabled
3023                  * before OS handover.
3024                  */
3025                 dmar_disable_qi(iommu);
3026         }
3027
3028         if (dmar_enable_qi(iommu)) {
3029                 /*
3030                  * Queued Invalidate not enabled, use Register Based Invalidate
3031                  */
3032                 iommu->flush.flush_context = __iommu_flush_context;
3033                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3034                 pr_info("%s: Using Register based invalidation\n",
3035                         iommu->name);
3036         } else {
3037                 iommu->flush.flush_context = qi_flush_context;
3038                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3039                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3040         }
3041 }
3042
3043 static int copy_context_table(struct intel_iommu *iommu,
3044                               struct root_entry *old_re,
3045                               struct context_entry **tbl,
3046                               int bus, bool ext)
3047 {
3048         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3049         struct context_entry *new_ce = NULL, ce;
3050         struct context_entry *old_ce = NULL;
3051         struct root_entry re;
3052         phys_addr_t old_ce_phys;
3053
3054         tbl_idx = ext ? bus * 2 : bus;
3055         memcpy(&re, old_re, sizeof(re));
3056
3057         for (devfn = 0; devfn < 256; devfn++) {
3058                 /* First calculate the correct index */
3059                 idx = (ext ? devfn * 2 : devfn) % 256;
3060
3061                 if (idx == 0) {
3062                         /* First save what we may have and clean up */
3063                         if (new_ce) {
3064                                 tbl[tbl_idx] = new_ce;
3065                                 __iommu_flush_cache(iommu, new_ce,
3066                                                     VTD_PAGE_SIZE);
3067                                 pos = 1;
3068                         }
3069
3070                         if (old_ce)
3071                                 iounmap(old_ce);
3072
3073                         ret = 0;
3074                         if (devfn < 0x80)
3075                                 old_ce_phys = root_entry_lctp(&re);
3076                         else
3077                                 old_ce_phys = root_entry_uctp(&re);
3078
3079                         if (!old_ce_phys) {
3080                                 if (ext && devfn == 0) {
3081                                         /* No LCTP, try UCTP */
3082                                         devfn = 0x7f;
3083                                         continue;
3084                                 } else {
3085                                         goto out;
3086                                 }
3087                         }
3088
3089                         ret = -ENOMEM;
3090                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3091                                         MEMREMAP_WB);
3092                         if (!old_ce)
3093                                 goto out;
3094
3095                         new_ce = alloc_pgtable_page(iommu->node);
3096                         if (!new_ce)
3097                                 goto out_unmap;
3098
3099                         ret = 0;
3100                 }
3101
3102                 /* Now copy the context entry */
3103                 memcpy(&ce, old_ce + idx, sizeof(ce));
3104
3105                 if (!__context_present(&ce))
3106                         continue;
3107
3108                 did = context_domain_id(&ce);
3109                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3110                         set_bit(did, iommu->domain_ids);
3111
3112                 /*
3113                  * We need a marker for copied context entries. This
3114                  * marker needs to work for the old format as well as
3115                  * for extended context entries.
3116                  *
3117                  * Bit 67 of the context entry is used. In the old
3118                  * format this bit is available to software, in the
3119                  * extended format it is the PGE bit, but PGE is ignored
3120                  * by HW if PASIDs are disabled (and thus still
3121                  * available).
3122                  *
3123                  * So disable PASIDs first and then mark the entry
3124                  * copied. This means that we don't copy PASID
3125                  * translations from the old kernel, but this is fine as
3126                  * faults there are not fatal.
3127                  */
3128                 context_clear_pasid_enable(&ce);
3129                 context_set_copied(&ce);
3130
3131                 new_ce[idx] = ce;
3132         }
3133
3134         tbl[tbl_idx + pos] = new_ce;
3135
3136         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3137
3138 out_unmap:
3139         memunmap(old_ce);
3140
3141 out:
3142         return ret;
3143 }
3144
3145 static int copy_translation_tables(struct intel_iommu *iommu)
3146 {
3147         struct context_entry **ctxt_tbls;
3148         struct root_entry *old_rt;
3149         phys_addr_t old_rt_phys;
3150         int ctxt_table_entries;
3151         unsigned long flags;
3152         u64 rtaddr_reg;
3153         int bus, ret;
3154         bool new_ext, ext;
3155
3156         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3157         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3158         new_ext    = !!ecap_ecs(iommu->ecap);
3159
3160         /*
3161          * The RTT bit can only be changed when translation is disabled,
3162          * but disabling translation means to open a window for data
3163          * corruption. So bail out and don't copy anything if we would
3164          * have to change the bit.
3165          */
3166         if (new_ext != ext)
3167                 return -EINVAL;
3168
3169         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3170         if (!old_rt_phys)
3171                 return -EINVAL;
3172
3173         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3174         if (!old_rt)
3175                 return -ENOMEM;
3176
3177         /* This is too big for the stack - allocate it from slab */
3178         ctxt_table_entries = ext ? 512 : 256;
3179         ret = -ENOMEM;
3180         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3181         if (!ctxt_tbls)
3182                 goto out_unmap;
3183
3184         for (bus = 0; bus < 256; bus++) {
3185                 ret = copy_context_table(iommu, &old_rt[bus],
3186                                          ctxt_tbls, bus, ext);
3187                 if (ret) {
3188                         pr_err("%s: Failed to copy context table for bus %d\n",
3189                                 iommu->name, bus);
3190                         continue;
3191                 }
3192         }
3193
3194         spin_lock_irqsave(&iommu->lock, flags);
3195
3196         /* Context tables are copied, now write them to the root_entry table */
3197         for (bus = 0; bus < 256; bus++) {
3198                 int idx = ext ? bus * 2 : bus;
3199                 u64 val;
3200
3201                 if (ctxt_tbls[idx]) {
3202                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3203                         iommu->root_entry[bus].lo = val;
3204                 }
3205
3206                 if (!ext || !ctxt_tbls[idx + 1])
3207                         continue;
3208
3209                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3210                 iommu->root_entry[bus].hi = val;
3211         }
3212
3213         spin_unlock_irqrestore(&iommu->lock, flags);
3214
3215         kfree(ctxt_tbls);
3216
3217         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3218
3219         ret = 0;
3220
3221 out_unmap:
3222         memunmap(old_rt);
3223
3224         return ret;
3225 }
3226
3227 static int __init init_dmars(void)
3228 {
3229         struct dmar_drhd_unit *drhd;
3230         struct dmar_rmrr_unit *rmrr;
3231         bool copied_tables = false;
3232         struct device *dev;
3233         struct intel_iommu *iommu;
3234         int i, ret;
3235
3236         /*
3237          * for each drhd
3238          *    allocate root
3239          *    initialize and program root entry to not present
3240          * endfor
3241          */
3242         for_each_drhd_unit(drhd) {
3243                 /*
3244                  * lock not needed as this is only incremented in the single
3245                  * threaded kernel __init code path all other access are read
3246                  * only
3247                  */
3248                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3249                         g_num_of_iommus++;
3250                         continue;
3251                 }
3252                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3253         }
3254
3255         /* Preallocate enough resources for IOMMU hot-addition */
3256         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3257                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3258
3259         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3260                         GFP_KERNEL);
3261         if (!g_iommus) {
3262                 pr_err("Allocating global iommu array failed\n");
3263                 ret = -ENOMEM;
3264                 goto error;
3265         }
3266
3267         for_each_active_iommu(iommu, drhd) {
3268                 g_iommus[iommu->seq_id] = iommu;
3269
3270                 intel_iommu_init_qi(iommu);
3271
3272                 ret = iommu_init_domains(iommu);
3273                 if (ret)
3274                         goto free_iommu;
3275
3276                 init_translation_status(iommu);
3277
3278                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3279                         iommu_disable_translation(iommu);
3280                         clear_translation_pre_enabled(iommu);
3281                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3282                                 iommu->name);
3283                 }
3284
3285                 /*
3286                  * TBD:
3287                  * we could share the same root & context tables
3288                  * among all IOMMU's. Need to Split it later.
3289                  */
3290                 ret = iommu_alloc_root_entry(iommu);
3291                 if (ret)
3292                         goto free_iommu;
3293
3294                 if (translation_pre_enabled(iommu)) {
3295                         pr_info("Translation already enabled - trying to copy translation structures\n");
3296
3297                         ret = copy_translation_tables(iommu);
3298                         if (ret) {
3299                                 /*
3300                                  * We found the IOMMU with translation
3301                                  * enabled - but failed to copy over the
3302                                  * old root-entry table. Try to proceed
3303                                  * by disabling translation now and
3304                                  * allocating a clean root-entry table.
3305                                  * This might cause DMAR faults, but
3306                                  * probably the dump will still succeed.
3307                                  */
3308                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3309                                        iommu->name);
3310                                 iommu_disable_translation(iommu);
3311                                 clear_translation_pre_enabled(iommu);
3312                         } else {
3313                                 pr_info("Copied translation tables from previous kernel for %s\n",
3314                                         iommu->name);
3315                                 copied_tables = true;
3316                         }
3317                 }
3318
3319                 if (!ecap_pass_through(iommu->ecap))
3320                         hw_pass_through = 0;
3321 #ifdef CONFIG_INTEL_IOMMU_SVM
3322                 if (pasid_enabled(iommu))
3323                         intel_svm_alloc_pasid_tables(iommu);
3324 #endif
3325         }
3326
3327         /*
3328          * Now that qi is enabled on all iommus, set the root entry and flush
3329          * caches. This is required on some Intel X58 chipsets, otherwise the
3330          * flush_context function will loop forever and the boot hangs.
3331          */
3332         for_each_active_iommu(iommu, drhd) {
3333                 iommu_flush_write_buffer(iommu);
3334                 iommu_set_root_entry(iommu);
3335                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3336                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3337         }
3338
3339         if (iommu_pass_through)
3340                 iommu_identity_mapping |= IDENTMAP_ALL;
3341
3342 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3343         iommu_identity_mapping |= IDENTMAP_GFX;
3344 #endif
3345
3346         check_tylersburg_isoch();
3347
3348         if (iommu_identity_mapping) {
3349                 ret = si_domain_init(hw_pass_through);
3350                 if (ret)
3351                         goto free_iommu;
3352         }
3353
3354
3355         /*
3356          * If we copied translations from a previous kernel in the kdump
3357          * case, we can not assign the devices to domains now, as that
3358          * would eliminate the old mappings. So skip this part and defer
3359          * the assignment to device driver initialization time.
3360          */
3361         if (copied_tables)
3362                 goto domains_done;
3363
3364         /*
3365          * If pass through is not set or not enabled, setup context entries for
3366          * identity mappings for rmrr, gfx, and isa and may fall back to static
3367          * identity mapping if iommu_identity_mapping is set.
3368          */
3369         if (iommu_identity_mapping) {
3370                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3371                 if (ret) {
3372                         pr_crit("Failed to setup IOMMU pass-through\n");
3373                         goto free_iommu;
3374                 }
3375         }
3376         /*
3377          * For each rmrr
3378          *   for each dev attached to rmrr
3379          *   do
3380          *     locate drhd for dev, alloc domain for dev
3381          *     allocate free domain
3382          *     allocate page table entries for rmrr
3383          *     if context not allocated for bus
3384          *           allocate and init context
3385          *           set present in root table for this bus
3386          *     init context with domain, translation etc
3387          *    endfor
3388          * endfor
3389          */
3390         pr_info("Setting RMRR:\n");
3391         for_each_rmrr_units(rmrr) {
3392                 /* some BIOS lists non-exist devices in DMAR table. */
3393                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3394                                           i, dev) {
3395                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3396                         if (ret)
3397                                 pr_err("Mapping reserved region failed\n");
3398                 }
3399         }
3400
3401         iommu_prepare_isa();
3402
3403 domains_done:
3404
3405         /*
3406          * for each drhd
3407          *   enable fault log
3408          *   global invalidate context cache
3409          *   global invalidate iotlb
3410          *   enable translation
3411          */
3412         for_each_iommu(iommu, drhd) {
3413                 if (drhd->ignored) {
3414                         /*
3415                          * we always have to disable PMRs or DMA may fail on
3416                          * this device
3417                          */
3418                         if (force_on)
3419                                 iommu_disable_protect_mem_regions(iommu);
3420                         continue;
3421                 }
3422
3423                 iommu_flush_write_buffer(iommu);
3424
3425 #ifdef CONFIG_INTEL_IOMMU_SVM
3426                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3427                         ret = intel_svm_enable_prq(iommu);
3428                         if (ret)
3429                                 goto free_iommu;
3430                 }
3431 #endif
3432                 ret = dmar_set_interrupt(iommu);
3433                 if (ret)
3434                         goto free_iommu;
3435
3436                 if (!translation_pre_enabled(iommu))
3437                         iommu_enable_translation(iommu);
3438
3439                 iommu_disable_protect_mem_regions(iommu);
3440         }
3441
3442         return 0;
3443
3444 free_iommu:
3445         for_each_active_iommu(iommu, drhd) {
3446                 disable_dmar_iommu(iommu);
3447                 free_dmar_iommu(iommu);
3448         }
3449
3450         kfree(g_iommus);
3451
3452 error:
3453         return ret;
3454 }
3455
3456 /* This takes a number of _MM_ pages, not VTD pages */
3457 static unsigned long intel_alloc_iova(struct device *dev,
3458                                      struct dmar_domain *domain,
3459                                      unsigned long nrpages, uint64_t dma_mask)
3460 {
3461         unsigned long iova_pfn = 0;
3462
3463         /* Restrict dma_mask to the width that the iommu can handle */
3464         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3465         /* Ensure we reserve the whole size-aligned region */
3466         nrpages = __roundup_pow_of_two(nrpages);
3467
3468         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3469                 /*
3470                  * First try to allocate an io virtual address in
3471                  * DMA_BIT_MASK(32) and if that fails then try allocating
3472                  * from higher range
3473                  */
3474                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3475                                            IOVA_PFN(DMA_BIT_MASK(32)), false);
3476                 if (iova_pfn)
3477                         return iova_pfn;
3478         }
3479         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3480                                    IOVA_PFN(dma_mask), true);
3481         if (unlikely(!iova_pfn)) {
3482                 pr_err("Allocating %ld-page iova for %s failed",
3483                        nrpages, dev_name(dev));
3484                 return 0;
3485         }
3486
3487         return iova_pfn;
3488 }
3489
3490 static struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3491 {
3492         struct dmar_domain *domain, *tmp;
3493         struct dmar_rmrr_unit *rmrr;
3494         struct device *i_dev;
3495         int i, ret;
3496
3497         domain = find_domain(dev);
3498         if (domain)
3499                 goto out;
3500
3501         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3502         if (!domain)
3503                 goto out;
3504
3505         /* We have a new domain - setup possible RMRRs for the device */
3506         rcu_read_lock();
3507         for_each_rmrr_units(rmrr) {
3508                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3509                                           i, i_dev) {
3510                         if (i_dev != dev)
3511                                 continue;
3512
3513                         ret = domain_prepare_identity_map(dev, domain,
3514                                                           rmrr->base_address,
3515                                                           rmrr->end_address);
3516                         if (ret)
3517                                 dev_err(dev, "Mapping reserved region failed\n");
3518                 }
3519         }
3520         rcu_read_unlock();
3521
3522         tmp = set_domain_for_dev(dev, domain);
3523         if (!tmp || domain != tmp) {
3524                 domain_exit(domain);
3525                 domain = tmp;
3526         }
3527
3528 out:
3529
3530         if (!domain)
3531                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3532
3533
3534         return domain;
3535 }
3536
3537 /* Check if the dev needs to go through non-identity map and unmap process.*/
3538 static int iommu_no_mapping(struct device *dev)
3539 {
3540         int found;
3541
3542         if (iommu_dummy(dev))
3543                 return 1;
3544
3545         if (!iommu_identity_mapping)
3546                 return 0;
3547
3548         found = identity_mapping(dev);
3549         if (found) {
3550                 if (iommu_should_identity_map(dev, 0))
3551                         return 1;
3552                 else {
3553                         /*
3554                          * 32 bit DMA is removed from si_domain and fall back
3555                          * to non-identity mapping.
3556                          */
3557                         dmar_remove_one_dev_info(si_domain, dev);
3558                         pr_info("32bit %s uses non-identity mapping\n",
3559                                 dev_name(dev));
3560                         return 0;
3561                 }
3562         } else {
3563                 /*
3564                  * In case of a detached 64 bit DMA device from vm, the device
3565                  * is put into si_domain for identity mapping.
3566                  */
3567                 if (iommu_should_identity_map(dev, 0)) {
3568                         int ret;
3569                         ret = domain_add_dev_info(si_domain, dev);
3570                         if (!ret) {
3571                                 pr_info("64bit %s uses identity mapping\n",
3572                                         dev_name(dev));
3573                                 return 1;
3574                         }
3575                 }
3576         }
3577
3578         return 0;
3579 }
3580
3581 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3582                                      size_t size, int dir, u64 dma_mask)
3583 {
3584         struct dmar_domain *domain;
3585         phys_addr_t start_paddr;
3586         unsigned long iova_pfn;
3587         int prot = 0;
3588         int ret;
3589         struct intel_iommu *iommu;
3590         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3591
3592         BUG_ON(dir == DMA_NONE);
3593
3594         if (iommu_no_mapping(dev))
3595                 return paddr;
3596
3597         domain = get_valid_domain_for_dev(dev);
3598         if (!domain)
3599                 return 0;
3600
3601         iommu = domain_get_iommu(domain);
3602         size = aligned_nrpages(paddr, size);
3603
3604         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3605         if (!iova_pfn)
3606                 goto error;
3607
3608         /*
3609          * Check if DMAR supports zero-length reads on write only
3610          * mappings..
3611          */
3612         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3613                         !cap_zlr(iommu->cap))
3614                 prot |= DMA_PTE_READ;
3615         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3616                 prot |= DMA_PTE_WRITE;
3617         /*
3618          * paddr - (paddr + size) might be partial page, we should map the whole
3619          * page.  Note: if two part of one page are separately mapped, we
3620          * might have two guest_addr mapping to the same host paddr, but this
3621          * is not a big problem
3622          */
3623         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3624                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3625         if (ret)
3626                 goto error;
3627
3628         /* it's a non-present to present mapping. Only flush if caching mode */
3629         if (cap_caching_mode(iommu->cap))
3630                 iommu_flush_iotlb_psi(iommu, domain,
3631                                       mm_to_dma_pfn(iova_pfn),
3632                                       size, 0, 1);
3633         else
3634                 iommu_flush_write_buffer(iommu);
3635
3636         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3637         start_paddr += paddr & ~PAGE_MASK;
3638         return start_paddr;
3639
3640 error:
3641         if (iova_pfn)
3642                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3643         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3644                 dev_name(dev), size, (unsigned long long)paddr, dir);
3645         return 0;
3646 }
3647
3648 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3649                                  unsigned long offset, size_t size,
3650                                  enum dma_data_direction dir,
3651                                  unsigned long attrs)
3652 {
3653         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3654                                   dir, *dev->dma_mask);
3655 }
3656
3657 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3658 {
3659         struct dmar_domain *domain;
3660         unsigned long start_pfn, last_pfn;
3661         unsigned long nrpages;
3662         unsigned long iova_pfn;
3663         struct intel_iommu *iommu;
3664         struct page *freelist;
3665
3666         if (iommu_no_mapping(dev))
3667                 return;
3668
3669         domain = find_domain(dev);
3670         BUG_ON(!domain);
3671
3672         iommu = domain_get_iommu(domain);
3673
3674         iova_pfn = IOVA_PFN(dev_addr);
3675
3676         nrpages = aligned_nrpages(dev_addr, size);
3677         start_pfn = mm_to_dma_pfn(iova_pfn);
3678         last_pfn = start_pfn + nrpages - 1;
3679
3680         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3681                  dev_name(dev), start_pfn, last_pfn);
3682
3683         freelist = domain_unmap(domain, start_pfn, last_pfn);
3684
3685         if (intel_iommu_strict) {
3686                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3687                                       nrpages, !freelist, 0);
3688                 /* free iova */
3689                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3690                 dma_free_pagelist(freelist);
3691         } else {
3692                 queue_iova(&domain->iovad, iova_pfn, nrpages,
3693                            (unsigned long)freelist);
3694                 /*
3695                  * queue up the release of the unmap to save the 1/6th of the
3696                  * cpu used up by the iotlb flush operation...
3697                  */
3698         }
3699 }
3700
3701 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3702                              size_t size, enum dma_data_direction dir,
3703                              unsigned long attrs)
3704 {
3705         intel_unmap(dev, dev_addr, size);
3706 }
3707
3708 static void *intel_alloc_coherent(struct device *dev, size_t size,
3709                                   dma_addr_t *dma_handle, gfp_t flags,
3710                                   unsigned long attrs)
3711 {
3712         void *vaddr;
3713
3714         vaddr = dma_direct_alloc(dev, size, dma_handle, flags, attrs);
3715         if (iommu_no_mapping(dev) || !vaddr)
3716                 return vaddr;
3717
3718         *dma_handle = __intel_map_single(dev, virt_to_phys(vaddr),
3719                         PAGE_ALIGN(size), DMA_BIDIRECTIONAL,
3720                         dev->coherent_dma_mask);
3721         if (!*dma_handle)
3722                 goto out_free_pages;
3723         return vaddr;
3724
3725 out_free_pages:
3726         dma_direct_free(dev, size, vaddr, *dma_handle, attrs);
3727         return NULL;
3728 }
3729
3730 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3731                                 dma_addr_t dma_handle, unsigned long attrs)
3732 {
3733         if (!iommu_no_mapping(dev))
3734                 intel_unmap(dev, dma_handle, PAGE_ALIGN(size));
3735         dma_direct_free(dev, size, vaddr, dma_handle, attrs);
3736 }
3737
3738 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3739                            int nelems, enum dma_data_direction dir,
3740                            unsigned long attrs)
3741 {
3742         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3743         unsigned long nrpages = 0;
3744         struct scatterlist *sg;
3745         int i;
3746
3747         for_each_sg(sglist, sg, nelems, i) {
3748                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3749         }
3750
3751         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3752 }
3753
3754 static int intel_nontranslate_map_sg(struct device *hddev,
3755         struct scatterlist *sglist, int nelems, int dir)
3756 {
3757         int i;
3758         struct scatterlist *sg;
3759
3760         for_each_sg(sglist, sg, nelems, i) {
3761                 BUG_ON(!sg_page(sg));
3762                 sg->dma_address = sg_phys(sg);
3763                 sg->dma_length = sg->length;
3764         }
3765         return nelems;
3766 }
3767
3768 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3769                         enum dma_data_direction dir, unsigned long attrs)
3770 {
3771         int i;
3772         struct dmar_domain *domain;
3773         size_t size = 0;
3774         int prot = 0;
3775         unsigned long iova_pfn;
3776         int ret;
3777         struct scatterlist *sg;
3778         unsigned long start_vpfn;
3779         struct intel_iommu *iommu;
3780
3781         BUG_ON(dir == DMA_NONE);
3782         if (iommu_no_mapping(dev))
3783                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3784
3785         domain = get_valid_domain_for_dev(dev);
3786         if (!domain)
3787                 return 0;
3788
3789         iommu = domain_get_iommu(domain);
3790
3791         for_each_sg(sglist, sg, nelems, i)
3792                 size += aligned_nrpages(sg->offset, sg->length);
3793
3794         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3795                                 *dev->dma_mask);
3796         if (!iova_pfn) {
3797                 sglist->dma_length = 0;
3798                 return 0;
3799         }
3800
3801         /*
3802          * Check if DMAR supports zero-length reads on write only
3803          * mappings..
3804          */
3805         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3806                         !cap_zlr(iommu->cap))
3807                 prot |= DMA_PTE_READ;
3808         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3809                 prot |= DMA_PTE_WRITE;
3810
3811         start_vpfn = mm_to_dma_pfn(iova_pfn);
3812
3813         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3814         if (unlikely(ret)) {
3815                 dma_pte_free_pagetable(domain, start_vpfn,
3816                                        start_vpfn + size - 1,
3817                                        agaw_to_level(domain->agaw) + 1);
3818                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3819                 return 0;
3820         }
3821
3822         /* it's a non-present to present mapping. Only flush if caching mode */
3823         if (cap_caching_mode(iommu->cap))
3824                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3825         else
3826                 iommu_flush_write_buffer(iommu);
3827
3828         return nelems;
3829 }
3830
3831 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3832 {
3833         return !dma_addr;
3834 }
3835
3836 const struct dma_map_ops intel_dma_ops = {
3837         .alloc = intel_alloc_coherent,
3838         .free = intel_free_coherent,
3839         .map_sg = intel_map_sg,
3840         .unmap_sg = intel_unmap_sg,
3841         .map_page = intel_map_page,
3842         .unmap_page = intel_unmap_page,
3843         .mapping_error = intel_mapping_error,
3844 #ifdef CONFIG_X86
3845         .dma_supported = dma_direct_supported,
3846 #endif
3847 };
3848
3849 static inline int iommu_domain_cache_init(void)
3850 {
3851         int ret = 0;
3852
3853         iommu_domain_cache = kmem_cache_create("iommu_domain",
3854                                          sizeof(struct dmar_domain),
3855                                          0,
3856                                          SLAB_HWCACHE_ALIGN,
3857
3858                                          NULL);
3859         if (!iommu_domain_cache) {
3860                 pr_err("Couldn't create iommu_domain cache\n");
3861                 ret = -ENOMEM;
3862         }
3863
3864         return ret;
3865 }
3866
3867 static inline int iommu_devinfo_cache_init(void)
3868 {
3869         int ret = 0;
3870
3871         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3872                                          sizeof(struct device_domain_info),
3873                                          0,
3874                                          SLAB_HWCACHE_ALIGN,
3875                                          NULL);
3876         if (!iommu_devinfo_cache) {
3877                 pr_err("Couldn't create devinfo cache\n");
3878                 ret = -ENOMEM;
3879         }
3880
3881         return ret;
3882 }
3883
3884 static int __init iommu_init_mempool(void)
3885 {
3886         int ret;
3887         ret = iova_cache_get();
3888         if (ret)
3889                 return ret;
3890
3891         ret = iommu_domain_cache_init();
3892         if (ret)
3893                 goto domain_error;
3894
3895         ret = iommu_devinfo_cache_init();
3896         if (!ret)
3897                 return ret;
3898
3899         kmem_cache_destroy(iommu_domain_cache);
3900 domain_error:
3901         iova_cache_put();
3902
3903         return -ENOMEM;
3904 }
3905
3906 static void __init iommu_exit_mempool(void)
3907 {
3908         kmem_cache_destroy(iommu_devinfo_cache);
3909         kmem_cache_destroy(iommu_domain_cache);
3910         iova_cache_put();
3911 }
3912
3913 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3914 {
3915         struct dmar_drhd_unit *drhd;
3916         u32 vtbar;
3917         int rc;
3918
3919         /* We know that this device on this chipset has its own IOMMU.
3920          * If we find it under a different IOMMU, then the BIOS is lying
3921          * to us. Hope that the IOMMU for this device is actually
3922          * disabled, and it needs no translation...
3923          */
3924         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3925         if (rc) {
3926                 /* "can't" happen */
3927                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3928                 return;
3929         }
3930         vtbar &= 0xffff0000;
3931
3932         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3933         drhd = dmar_find_matched_drhd_unit(pdev);
3934         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3935                             TAINT_FIRMWARE_WORKAROUND,
3936                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3937                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3938 }
3939 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3940
3941 static void __init init_no_remapping_devices(void)
3942 {
3943         struct dmar_drhd_unit *drhd;
3944         struct device *dev;
3945         int i;
3946
3947         for_each_drhd_unit(drhd) {
3948                 if (!drhd->include_all) {
3949                         for_each_active_dev_scope(drhd->devices,
3950                                                   drhd->devices_cnt, i, dev)
3951                                 break;
3952                         /* ignore DMAR unit if no devices exist */
3953                         if (i == drhd->devices_cnt)
3954                                 drhd->ignored = 1;
3955                 }
3956         }
3957
3958         for_each_active_drhd_unit(drhd) {
3959                 if (drhd->include_all)
3960                         continue;
3961
3962                 for_each_active_dev_scope(drhd->devices,
3963                                           drhd->devices_cnt, i, dev)
3964                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3965                                 break;
3966                 if (i < drhd->devices_cnt)
3967                         continue;
3968
3969                 /* This IOMMU has *only* gfx devices. Either bypass it or
3970                    set the gfx_mapped flag, as appropriate */
3971                 if (dmar_map_gfx) {
3972                         intel_iommu_gfx_mapped = 1;
3973                 } else {
3974                         drhd->ignored = 1;
3975                         for_each_active_dev_scope(drhd->devices,
3976                                                   drhd->devices_cnt, i, dev)
3977                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3978                 }
3979         }
3980 }
3981
3982 #ifdef CONFIG_SUSPEND
3983 static int init_iommu_hw(void)
3984 {
3985         struct dmar_drhd_unit *drhd;
3986         struct intel_iommu *iommu = NULL;
3987
3988         for_each_active_iommu(iommu, drhd)
3989                 if (iommu->qi)
3990                         dmar_reenable_qi(iommu);
3991
3992         for_each_iommu(iommu, drhd) {
3993                 if (drhd->ignored) {
3994                         /*
3995                          * we always have to disable PMRs or DMA may fail on
3996                          * this device
3997                          */
3998                         if (force_on)
3999                                 iommu_disable_protect_mem_regions(iommu);
4000                         continue;
4001                 }
4002
4003                 iommu_flush_write_buffer(iommu);
4004
4005                 iommu_set_root_entry(iommu);
4006
4007                 iommu->flush.flush_context(iommu, 0, 0, 0,
4008                                            DMA_CCMD_GLOBAL_INVL);
4009                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4010                 iommu_enable_translation(iommu);
4011                 iommu_disable_protect_mem_regions(iommu);
4012         }
4013
4014         return 0;
4015 }
4016
4017 static void iommu_flush_all(void)
4018 {
4019         struct dmar_drhd_unit *drhd;
4020         struct intel_iommu *iommu;
4021
4022         for_each_active_iommu(iommu, drhd) {
4023                 iommu->flush.flush_context(iommu, 0, 0, 0,
4024                                            DMA_CCMD_GLOBAL_INVL);
4025                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4026                                          DMA_TLB_GLOBAL_FLUSH);
4027         }
4028 }
4029
4030 static int iommu_suspend(void)
4031 {
4032         struct dmar_drhd_unit *drhd;
4033         struct intel_iommu *iommu = NULL;
4034         unsigned long flag;
4035
4036         for_each_active_iommu(iommu, drhd) {
4037                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4038                                                  GFP_ATOMIC);
4039                 if (!iommu->iommu_state)
4040                         goto nomem;
4041         }
4042
4043         iommu_flush_all();
4044
4045         for_each_active_iommu(iommu, drhd) {
4046                 iommu_disable_translation(iommu);
4047
4048                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4049
4050                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4051                         readl(iommu->reg + DMAR_FECTL_REG);
4052                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4053                         readl(iommu->reg + DMAR_FEDATA_REG);
4054                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4055                         readl(iommu->reg + DMAR_FEADDR_REG);
4056                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4057                         readl(iommu->reg + DMAR_FEUADDR_REG);
4058
4059                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4060         }
4061         return 0;
4062
4063 nomem:
4064         for_each_active_iommu(iommu, drhd)
4065                 kfree(iommu->iommu_state);
4066
4067         return -ENOMEM;
4068 }
4069
4070 static void iommu_resume(void)
4071 {
4072         struct dmar_drhd_unit *drhd;
4073         struct intel_iommu *iommu = NULL;
4074         unsigned long flag;
4075
4076         if (init_iommu_hw()) {
4077                 if (force_on)
4078                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4079                 else
4080                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4081                 return;
4082         }
4083
4084         for_each_active_iommu(iommu, drhd) {
4085
4086                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4087
4088                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4089                         iommu->reg + DMAR_FECTL_REG);
4090                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4091                         iommu->reg + DMAR_FEDATA_REG);
4092                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4093                         iommu->reg + DMAR_FEADDR_REG);
4094                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4095                         iommu->reg + DMAR_FEUADDR_REG);
4096
4097                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4098         }
4099
4100         for_each_active_iommu(iommu, drhd)
4101                 kfree(iommu->iommu_state);
4102 }
4103
4104 static struct syscore_ops iommu_syscore_ops = {
4105         .resume         = iommu_resume,
4106         .suspend        = iommu_suspend,
4107 };
4108
4109 static void __init init_iommu_pm_ops(void)
4110 {
4111         register_syscore_ops(&iommu_syscore_ops);
4112 }
4113
4114 #else
4115 static inline void init_iommu_pm_ops(void) {}
4116 #endif  /* CONFIG_PM */
4117
4118
4119 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4120 {
4121         struct acpi_dmar_reserved_memory *rmrr;
4122         int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4123         struct dmar_rmrr_unit *rmrru;
4124         size_t length;
4125
4126         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4127         if (!rmrru)
4128                 goto out;
4129
4130         rmrru->hdr = header;
4131         rmrr = (struct acpi_dmar_reserved_memory *)header;
4132         rmrru->base_address = rmrr->base_address;
4133         rmrru->end_address = rmrr->end_address;
4134
4135         length = rmrr->end_address - rmrr->base_address + 1;
4136         rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4137                                               IOMMU_RESV_DIRECT);
4138         if (!rmrru->resv)
4139                 goto free_rmrru;
4140
4141         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4142                                 ((void *)rmrr) + rmrr->header.length,
4143                                 &rmrru->devices_cnt);
4144         if (rmrru->devices_cnt && rmrru->devices == NULL)
4145                 goto free_all;
4146
4147         list_add(&rmrru->list, &dmar_rmrr_units);
4148
4149         return 0;
4150 free_all:
4151         kfree(rmrru->resv);
4152 free_rmrru:
4153         kfree(rmrru);
4154 out:
4155         return -ENOMEM;
4156 }
4157
4158 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4159 {
4160         struct dmar_atsr_unit *atsru;
4161         struct acpi_dmar_atsr *tmp;
4162
4163         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4164                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4165                 if (atsr->segment != tmp->segment)
4166                         continue;
4167                 if (atsr->header.length != tmp->header.length)
4168                         continue;
4169                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4170                         return atsru;
4171         }
4172
4173         return NULL;
4174 }
4175
4176 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4177 {
4178         struct acpi_dmar_atsr *atsr;
4179         struct dmar_atsr_unit *atsru;
4180
4181         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4182                 return 0;
4183
4184         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4185         atsru = dmar_find_atsr(atsr);
4186         if (atsru)
4187                 return 0;
4188
4189         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4190         if (!atsru)
4191                 return -ENOMEM;
4192
4193         /*
4194          * If memory is allocated from slab by ACPI _DSM method, we need to
4195          * copy the memory content because the memory buffer will be freed
4196          * on return.
4197          */
4198         atsru->hdr = (void *)(atsru + 1);
4199         memcpy(atsru->hdr, hdr, hdr->length);
4200         atsru->include_all = atsr->flags & 0x1;
4201         if (!atsru->include_all) {
4202                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4203                                 (void *)atsr + atsr->header.length,
4204                                 &atsru->devices_cnt);
4205                 if (atsru->devices_cnt && atsru->devices == NULL) {
4206                         kfree(atsru);
4207                         return -ENOMEM;
4208                 }
4209         }
4210
4211         list_add_rcu(&atsru->list, &dmar_atsr_units);
4212
4213         return 0;
4214 }
4215
4216 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4217 {
4218         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4219         kfree(atsru);
4220 }
4221
4222 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4223 {
4224         struct acpi_dmar_atsr *atsr;
4225         struct dmar_atsr_unit *atsru;
4226
4227         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4228         atsru = dmar_find_atsr(atsr);
4229         if (atsru) {
4230                 list_del_rcu(&atsru->list);
4231                 synchronize_rcu();
4232                 intel_iommu_free_atsr(atsru);
4233         }
4234
4235         return 0;
4236 }
4237
4238 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4239 {
4240         int i;
4241         struct device *dev;
4242         struct acpi_dmar_atsr *atsr;
4243         struct dmar_atsr_unit *atsru;
4244
4245         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4246         atsru = dmar_find_atsr(atsr);
4247         if (!atsru)
4248                 return 0;
4249
4250         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4251                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4252                                           i, dev)
4253                         return -EBUSY;
4254         }
4255
4256         return 0;
4257 }
4258
4259 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4260 {
4261         int sp, ret = 0;
4262         struct intel_iommu *iommu = dmaru->iommu;
4263
4264         if (g_iommus[iommu->seq_id])
4265                 return 0;
4266
4267         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4268                 pr_warn("%s: Doesn't support hardware pass through.\n",
4269                         iommu->name);
4270                 return -ENXIO;
4271         }
4272         if (!ecap_sc_support(iommu->ecap) &&
4273             domain_update_iommu_snooping(iommu)) {
4274                 pr_warn("%s: Doesn't support snooping.\n",
4275                         iommu->name);
4276                 return -ENXIO;
4277         }
4278         sp = domain_update_iommu_superpage(iommu) - 1;
4279         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4280                 pr_warn("%s: Doesn't support large page.\n",
4281                         iommu->name);
4282                 return -ENXIO;
4283         }
4284
4285         /*
4286          * Disable translation if already enabled prior to OS handover.
4287          */
4288         if (iommu->gcmd & DMA_GCMD_TE)
4289                 iommu_disable_translation(iommu);
4290
4291         g_iommus[iommu->seq_id] = iommu;
4292         ret = iommu_init_domains(iommu);
4293         if (ret == 0)
4294                 ret = iommu_alloc_root_entry(iommu);
4295         if (ret)
4296                 goto out;
4297
4298 #ifdef CONFIG_INTEL_IOMMU_SVM
4299         if (pasid_enabled(iommu))
4300                 intel_svm_alloc_pasid_tables(iommu);
4301 #endif
4302
4303         if (dmaru->ignored) {
4304                 /*
4305                  * we always have to disable PMRs or DMA may fail on this device
4306                  */
4307                 if (force_on)
4308                         iommu_disable_protect_mem_regions(iommu);
4309                 return 0;
4310         }
4311
4312         intel_iommu_init_qi(iommu);
4313         iommu_flush_write_buffer(iommu);
4314
4315 #ifdef CONFIG_INTEL_IOMMU_SVM
4316         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4317                 ret = intel_svm_enable_prq(iommu);
4318                 if (ret)
4319                         goto disable_iommu;
4320         }
4321 #endif
4322         ret = dmar_set_interrupt(iommu);
4323         if (ret)
4324                 goto disable_iommu;
4325
4326         iommu_set_root_entry(iommu);
4327         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4328         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4329         iommu_enable_translation(iommu);
4330
4331         iommu_disable_protect_mem_regions(iommu);
4332         return 0;
4333
4334 disable_iommu:
4335         disable_dmar_iommu(iommu);
4336 out:
4337         free_dmar_iommu(iommu);
4338         return ret;
4339 }
4340
4341 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4342 {
4343         int ret = 0;
4344         struct intel_iommu *iommu = dmaru->iommu;
4345
4346         if (!intel_iommu_enabled)
4347                 return 0;
4348         if (iommu == NULL)
4349                 return -EINVAL;
4350
4351         if (insert) {
4352                 ret = intel_iommu_add(dmaru);
4353         } else {
4354                 disable_dmar_iommu(iommu);
4355                 free_dmar_iommu(iommu);
4356         }
4357
4358         return ret;
4359 }
4360
4361 static void intel_iommu_free_dmars(void)
4362 {
4363         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4364         struct dmar_atsr_unit *atsru, *atsr_n;
4365
4366         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4367                 list_del(&rmrru->list);
4368                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4369                 kfree(rmrru->resv);
4370                 kfree(rmrru);
4371         }
4372
4373         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4374                 list_del(&atsru->list);
4375                 intel_iommu_free_atsr(atsru);
4376         }
4377 }
4378
4379 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4380 {
4381         int i, ret = 1;
4382         struct pci_bus *bus;
4383         struct pci_dev *bridge = NULL;
4384         struct device *tmp;
4385         struct acpi_dmar_atsr *atsr;
4386         struct dmar_atsr_unit *atsru;
4387
4388         dev = pci_physfn(dev);
4389         for (bus = dev->bus; bus; bus = bus->parent) {
4390                 bridge = bus->self;
4391                 /* If it's an integrated device, allow ATS */
4392                 if (!bridge)
4393                         return 1;
4394                 /* Connected via non-PCIe: no ATS */
4395                 if (!pci_is_pcie(bridge) ||
4396                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4397                         return 0;
4398                 /* If we found the root port, look it up in the ATSR */
4399                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4400                         break;
4401         }
4402
4403         rcu_read_lock();
4404         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4405                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4406                 if (atsr->segment != pci_domain_nr(dev->bus))
4407                         continue;
4408
4409                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4410                         if (tmp == &bridge->dev)
4411                                 goto out;
4412
4413                 if (atsru->include_all)
4414                         goto out;
4415         }
4416         ret = 0;
4417 out:
4418         rcu_read_unlock();
4419
4420         return ret;
4421 }
4422
4423 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4424 {
4425         int ret = 0;
4426         struct dmar_rmrr_unit *rmrru;
4427         struct dmar_atsr_unit *atsru;
4428         struct acpi_dmar_atsr *atsr;
4429         struct acpi_dmar_reserved_memory *rmrr;
4430
4431         if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4432                 return 0;
4433
4434         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4435                 rmrr = container_of(rmrru->hdr,
4436                                     struct acpi_dmar_reserved_memory, header);
4437                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4438                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4439                                 ((void *)rmrr) + rmrr->header.length,
4440                                 rmrr->segment, rmrru->devices,
4441                                 rmrru->devices_cnt);
4442                         if(ret < 0)
4443                                 return ret;
4444                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4445                         dmar_remove_dev_scope(info, rmrr->segment,
4446                                 rmrru->devices, rmrru->devices_cnt);
4447                 }
4448         }
4449
4450         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4451                 if (atsru->include_all)
4452                         continue;
4453
4454                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4455                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4456                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4457                                         (void *)atsr + atsr->header.length,
4458                                         atsr->segment, atsru->devices,
4459                                         atsru->devices_cnt);
4460                         if (ret > 0)
4461                                 break;
4462                         else if(ret < 0)
4463                                 return ret;
4464                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4465                         if (dmar_remove_dev_scope(info, atsr->segment,
4466                                         atsru->devices, atsru->devices_cnt))
4467                                 break;
4468                 }
4469         }
4470
4471         return 0;
4472 }
4473
4474 /*
4475  * Here we only respond to action of unbound device from driver.
4476  *
4477  * Added device is not attached to its DMAR domain here yet. That will happen
4478  * when mapping the device to iova.
4479  */
4480 static int device_notifier(struct notifier_block *nb,
4481                                   unsigned long action, void *data)
4482 {
4483         struct device *dev = data;
4484         struct dmar_domain *domain;
4485
4486         if (iommu_dummy(dev))
4487                 return 0;
4488
4489         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4490                 return 0;
4491
4492         domain = find_domain(dev);
4493         if (!domain)
4494                 return 0;
4495
4496         dmar_remove_one_dev_info(domain, dev);
4497         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4498                 domain_exit(domain);
4499
4500         return 0;
4501 }
4502
4503 static struct notifier_block device_nb = {
4504         .notifier_call = device_notifier,
4505 };
4506
4507 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4508                                        unsigned long val, void *v)
4509 {
4510         struct memory_notify *mhp = v;
4511         unsigned long long start, end;
4512         unsigned long start_vpfn, last_vpfn;
4513
4514         switch (val) {
4515         case MEM_GOING_ONLINE:
4516                 start = mhp->start_pfn << PAGE_SHIFT;
4517                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4518                 if (iommu_domain_identity_map(si_domain, start, end)) {
4519                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4520                                 start, end);
4521                         return NOTIFY_BAD;
4522                 }
4523                 break;
4524
4525         case MEM_OFFLINE:
4526         case MEM_CANCEL_ONLINE:
4527                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4528                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4529                 while (start_vpfn <= last_vpfn) {
4530                         struct iova *iova;
4531                         struct dmar_drhd_unit *drhd;
4532                         struct intel_iommu *iommu;
4533                         struct page *freelist;
4534
4535                         iova = find_iova(&si_domain->iovad, start_vpfn);
4536                         if (iova == NULL) {
4537                                 pr_debug("Failed get IOVA for PFN %lx\n",
4538                                          start_vpfn);
4539                                 break;
4540                         }
4541
4542                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4543                                                      start_vpfn, last_vpfn);
4544                         if (iova == NULL) {
4545                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4546                                         start_vpfn, last_vpfn);
4547                                 return NOTIFY_BAD;
4548                         }
4549
4550                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4551                                                iova->pfn_hi);
4552
4553                         rcu_read_lock();
4554                         for_each_active_iommu(iommu, drhd)
4555                                 iommu_flush_iotlb_psi(iommu, si_domain,
4556                                         iova->pfn_lo, iova_size(iova),
4557                                         !freelist, 0);
4558                         rcu_read_unlock();
4559                         dma_free_pagelist(freelist);
4560
4561                         start_vpfn = iova->pfn_hi + 1;
4562                         free_iova_mem(iova);
4563                 }
4564                 break;
4565         }
4566
4567         return NOTIFY_OK;
4568 }
4569
4570 static struct notifier_block intel_iommu_memory_nb = {
4571         .notifier_call = intel_iommu_memory_notifier,
4572         .priority = 0
4573 };
4574
4575 static void free_all_cpu_cached_iovas(unsigned int cpu)
4576 {
4577         int i;
4578
4579         for (i = 0; i < g_num_of_iommus; i++) {
4580                 struct intel_iommu *iommu = g_iommus[i];
4581                 struct dmar_domain *domain;
4582                 int did;
4583
4584                 if (!iommu)
4585                         continue;
4586
4587                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4588                         domain = get_iommu_domain(iommu, (u16)did);
4589
4590                         if (!domain)
4591                                 continue;
4592                         free_cpu_cached_iovas(cpu, &domain->iovad);
4593                 }
4594         }
4595 }
4596
4597 static int intel_iommu_cpu_dead(unsigned int cpu)
4598 {
4599         free_all_cpu_cached_iovas(cpu);
4600         return 0;
4601 }
4602
4603 static void intel_disable_iommus(void)
4604 {
4605         struct intel_iommu *iommu = NULL;
4606         struct dmar_drhd_unit *drhd;
4607
4608         for_each_iommu(iommu, drhd)
4609                 iommu_disable_translation(iommu);
4610 }
4611
4612 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4613 {
4614         struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4615
4616         return container_of(iommu_dev, struct intel_iommu, iommu);
4617 }
4618
4619 static ssize_t intel_iommu_show_version(struct device *dev,
4620                                         struct device_attribute *attr,
4621                                         char *buf)
4622 {
4623         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4624         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4625         return sprintf(buf, "%d:%d\n",
4626                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4627 }
4628 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4629
4630 static ssize_t intel_iommu_show_address(struct device *dev,
4631                                         struct device_attribute *attr,
4632                                         char *buf)
4633 {
4634         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4635         return sprintf(buf, "%llx\n", iommu->reg_phys);
4636 }
4637 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4638
4639 static ssize_t intel_iommu_show_cap(struct device *dev,
4640                                     struct device_attribute *attr,
4641                                     char *buf)
4642 {
4643         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4644         return sprintf(buf, "%llx\n", iommu->cap);
4645 }
4646 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4647
4648 static ssize_t intel_iommu_show_ecap(struct device *dev,
4649                                     struct device_attribute *attr,
4650                                     char *buf)
4651 {
4652         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4653         return sprintf(buf, "%llx\n", iommu->ecap);
4654 }
4655 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4656
4657 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4658                                       struct device_attribute *attr,
4659                                       char *buf)
4660 {
4661         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4662         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4663 }
4664 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4665
4666 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4667                                            struct device_attribute *attr,
4668                                            char *buf)
4669 {
4670         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4671         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4672                                                   cap_ndoms(iommu->cap)));
4673 }
4674 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4675
4676 static struct attribute *intel_iommu_attrs[] = {
4677         &dev_attr_version.attr,
4678         &dev_attr_address.attr,
4679         &dev_attr_cap.attr,
4680         &dev_attr_ecap.attr,
4681         &dev_attr_domains_supported.attr,
4682         &dev_attr_domains_used.attr,
4683         NULL,
4684 };
4685
4686 static struct attribute_group intel_iommu_group = {
4687         .name = "intel-iommu",
4688         .attrs = intel_iommu_attrs,
4689 };
4690
4691 const struct attribute_group *intel_iommu_groups[] = {
4692         &intel_iommu_group,
4693         NULL,
4694 };
4695
4696 int __init intel_iommu_init(void)
4697 {
4698         int ret = -ENODEV;
4699         struct dmar_drhd_unit *drhd;
4700         struct intel_iommu *iommu;
4701
4702         /* VT-d is required for a TXT/tboot launch, so enforce that */
4703         force_on = tboot_force_iommu();
4704
4705         if (iommu_init_mempool()) {
4706                 if (force_on)
4707                         panic("tboot: Failed to initialize iommu memory\n");
4708                 return -ENOMEM;
4709         }
4710
4711         down_write(&dmar_global_lock);
4712         if (dmar_table_init()) {
4713                 if (force_on)
4714                         panic("tboot: Failed to initialize DMAR table\n");
4715                 goto out_free_dmar;
4716         }
4717
4718         if (dmar_dev_scope_init() < 0) {
4719                 if (force_on)
4720                         panic("tboot: Failed to initialize DMAR device scope\n");
4721                 goto out_free_dmar;
4722         }
4723
4724         up_write(&dmar_global_lock);
4725
4726         /*
4727          * The bus notifier takes the dmar_global_lock, so lockdep will
4728          * complain later when we register it under the lock.
4729          */
4730         dmar_register_bus_notifier();
4731
4732         down_write(&dmar_global_lock);
4733
4734         if (no_iommu || dmar_disabled) {
4735                 /*
4736                  * We exit the function here to ensure IOMMU's remapping and
4737                  * mempool aren't setup, which means that the IOMMU's PMRs
4738                  * won't be disabled via the call to init_dmars(). So disable
4739                  * it explicitly here. The PMRs were setup by tboot prior to
4740                  * calling SENTER, but the kernel is expected to reset/tear
4741                  * down the PMRs.
4742                  */
4743                 if (intel_iommu_tboot_noforce) {
4744                         for_each_iommu(iommu, drhd)
4745                                 iommu_disable_protect_mem_regions(iommu);
4746                 }
4747
4748                 /*
4749                  * Make sure the IOMMUs are switched off, even when we
4750                  * boot into a kexec kernel and the previous kernel left
4751                  * them enabled
4752                  */
4753                 intel_disable_iommus();
4754                 goto out_free_dmar;
4755         }
4756
4757         if (list_empty(&dmar_rmrr_units))
4758                 pr_info("No RMRR found\n");
4759
4760         if (list_empty(&dmar_atsr_units))
4761                 pr_info("No ATSR found\n");
4762
4763         if (dmar_init_reserved_ranges()) {
4764                 if (force_on)
4765                         panic("tboot: Failed to reserve iommu ranges\n");
4766                 goto out_free_reserved_range;
4767         }
4768
4769         init_no_remapping_devices();
4770
4771         ret = init_dmars();
4772         if (ret) {
4773                 if (force_on)
4774                         panic("tboot: Failed to initialize DMARs\n");
4775                 pr_err("Initialization failed\n");
4776                 goto out_free_reserved_range;
4777         }
4778         up_write(&dmar_global_lock);
4779         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4780
4781 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4782         swiotlb = 0;
4783 #endif
4784         dma_ops = &intel_dma_ops;
4785
4786         init_iommu_pm_ops();
4787
4788         for_each_active_iommu(iommu, drhd) {
4789                 iommu_device_sysfs_add(&iommu->iommu, NULL,
4790                                        intel_iommu_groups,
4791                                        "%s", iommu->name);
4792                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4793                 iommu_device_register(&iommu->iommu);
4794         }
4795
4796         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4797         bus_register_notifier(&pci_bus_type, &device_nb);
4798         if (si_domain && !hw_pass_through)
4799                 register_memory_notifier(&intel_iommu_memory_nb);
4800         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4801                           intel_iommu_cpu_dead);
4802         intel_iommu_enabled = 1;
4803
4804         return 0;
4805
4806 out_free_reserved_range:
4807         put_iova_domain(&reserved_iova_list);
4808 out_free_dmar:
4809         intel_iommu_free_dmars();
4810         up_write(&dmar_global_lock);
4811         iommu_exit_mempool();
4812         return ret;
4813 }
4814
4815 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4816 {
4817         struct intel_iommu *iommu = opaque;
4818
4819         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4820         return 0;
4821 }
4822
4823 /*
4824  * NB - intel-iommu lacks any sort of reference counting for the users of
4825  * dependent devices.  If multiple endpoints have intersecting dependent
4826  * devices, unbinding the driver from any one of them will possibly leave
4827  * the others unable to operate.
4828  */
4829 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4830 {
4831         if (!iommu || !dev || !dev_is_pci(dev))
4832                 return;
4833
4834         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4835 }
4836
4837 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4838 {
4839         struct intel_iommu *iommu;
4840         unsigned long flags;
4841
4842         assert_spin_locked(&device_domain_lock);
4843
4844         if (WARN_ON(!info))
4845                 return;
4846
4847         iommu = info->iommu;
4848
4849         if (info->dev) {
4850                 iommu_disable_dev_iotlb(info);
4851                 domain_context_clear(iommu, info->dev);
4852         }
4853
4854         unlink_domain_info(info);
4855
4856         spin_lock_irqsave(&iommu->lock, flags);
4857         domain_detach_iommu(info->domain, iommu);
4858         spin_unlock_irqrestore(&iommu->lock, flags);
4859
4860         free_devinfo_mem(info);
4861 }
4862
4863 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4864                                      struct device *dev)
4865 {
4866         struct device_domain_info *info;
4867         unsigned long flags;
4868
4869         spin_lock_irqsave(&device_domain_lock, flags);
4870         info = dev->archdata.iommu;
4871         __dmar_remove_one_dev_info(info);
4872         spin_unlock_irqrestore(&device_domain_lock, flags);
4873 }
4874
4875 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4876 {
4877         int adjust_width;
4878
4879         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4880         domain_reserve_special_ranges(domain);
4881
4882         /* calculate AGAW */
4883         domain->gaw = guest_width;
4884         adjust_width = guestwidth_to_adjustwidth(guest_width);
4885         domain->agaw = width_to_agaw(adjust_width);
4886
4887         domain->iommu_coherency = 0;
4888         domain->iommu_snooping = 0;
4889         domain->iommu_superpage = 0;
4890         domain->max_addr = 0;
4891
4892         /* always allocate the top pgd */
4893         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4894         if (!domain->pgd)
4895                 return -ENOMEM;
4896         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4897         return 0;
4898 }
4899
4900 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4901 {
4902         struct dmar_domain *dmar_domain;
4903         struct iommu_domain *domain;
4904
4905         if (type != IOMMU_DOMAIN_UNMANAGED)
4906                 return NULL;
4907
4908         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4909         if (!dmar_domain) {
4910                 pr_err("Can't allocate dmar_domain\n");
4911                 return NULL;
4912         }
4913         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4914                 pr_err("Domain initialization failed\n");
4915                 domain_exit(dmar_domain);
4916                 return NULL;
4917         }
4918         domain_update_iommu_cap(dmar_domain);
4919
4920         domain = &dmar_domain->domain;
4921         domain->geometry.aperture_start = 0;
4922         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4923         domain->geometry.force_aperture = true;
4924
4925         return domain;
4926 }
4927
4928 static void intel_iommu_domain_free(struct iommu_domain *domain)
4929 {
4930         domain_exit(to_dmar_domain(domain));
4931 }
4932
4933 static int intel_iommu_attach_device(struct iommu_domain *domain,
4934                                      struct device *dev)
4935 {
4936         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4937         struct intel_iommu *iommu;
4938         int addr_width;
4939         u8 bus, devfn;
4940
4941         if (device_is_rmrr_locked(dev)) {
4942                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4943                 return -EPERM;
4944         }
4945
4946         /* normally dev is not mapped */
4947         if (unlikely(domain_context_mapped(dev))) {
4948                 struct dmar_domain *old_domain;
4949
4950                 old_domain = find_domain(dev);
4951                 if (old_domain) {
4952                         rcu_read_lock();
4953                         dmar_remove_one_dev_info(old_domain, dev);
4954                         rcu_read_unlock();
4955
4956                         if (!domain_type_is_vm_or_si(old_domain) &&
4957                              list_empty(&old_domain->devices))
4958                                 domain_exit(old_domain);
4959                 }
4960         }
4961
4962         iommu = device_to_iommu(dev, &bus, &devfn);
4963         if (!iommu)
4964                 return -ENODEV;
4965
4966         /* check if this iommu agaw is sufficient for max mapped address */
4967         addr_width = agaw_to_width(iommu->agaw);
4968         if (addr_width > cap_mgaw(iommu->cap))
4969                 addr_width = cap_mgaw(iommu->cap);
4970
4971         if (dmar_domain->max_addr > (1LL << addr_width)) {
4972                 pr_err("%s: iommu width (%d) is not "
4973                        "sufficient for the mapped address (%llx)\n",
4974                        __func__, addr_width, dmar_domain->max_addr);
4975                 return -EFAULT;
4976         }
4977         dmar_domain->gaw = addr_width;
4978
4979         /*
4980          * Knock out extra levels of page tables if necessary
4981          */
4982         while (iommu->agaw < dmar_domain->agaw) {
4983                 struct dma_pte *pte;
4984
4985                 pte = dmar_domain->pgd;
4986                 if (dma_pte_present(pte)) {
4987                         dmar_domain->pgd = (struct dma_pte *)
4988                                 phys_to_virt(dma_pte_addr(pte));
4989                         free_pgtable_page(pte);
4990                 }
4991                 dmar_domain->agaw--;
4992         }
4993
4994         return domain_add_dev_info(dmar_domain, dev);
4995 }
4996
4997 static void intel_iommu_detach_device(struct iommu_domain *domain,
4998                                       struct device *dev)
4999 {
5000         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5001 }
5002
5003 static int intel_iommu_map(struct iommu_domain *domain,
5004                            unsigned long iova, phys_addr_t hpa,
5005                            size_t size, int iommu_prot)
5006 {
5007         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5008         u64 max_addr;
5009         int prot = 0;
5010         int ret;
5011
5012         if (iommu_prot & IOMMU_READ)
5013                 prot |= DMA_PTE_READ;
5014         if (iommu_prot & IOMMU_WRITE)
5015                 prot |= DMA_PTE_WRITE;
5016         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5017                 prot |= DMA_PTE_SNP;
5018
5019         max_addr = iova + size;
5020         if (dmar_domain->max_addr < max_addr) {
5021                 u64 end;
5022
5023                 /* check if minimum agaw is sufficient for mapped address */
5024                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5025                 if (end < max_addr) {
5026                         pr_err("%s: iommu width (%d) is not "
5027                                "sufficient for the mapped address (%llx)\n",
5028                                __func__, dmar_domain->gaw, max_addr);
5029                         return -EFAULT;
5030                 }
5031                 dmar_domain->max_addr = max_addr;
5032         }
5033         /* Round up size to next multiple of PAGE_SIZE, if it and
5034            the low bits of hpa would take us onto the next page */
5035         size = aligned_nrpages(hpa, size);
5036         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5037                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5038         return ret;
5039 }
5040
5041 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5042                                 unsigned long iova, size_t size)
5043 {
5044         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5045         struct page *freelist = NULL;
5046         unsigned long start_pfn, last_pfn;
5047         unsigned int npages;
5048         int iommu_id, level = 0;
5049
5050         /* Cope with horrid API which requires us to unmap more than the
5051            size argument if it happens to be a large-page mapping. */
5052         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5053
5054         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5055                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5056
5057         start_pfn = iova >> VTD_PAGE_SHIFT;
5058         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5059
5060         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5061
5062         npages = last_pfn - start_pfn + 1;
5063
5064         for_each_domain_iommu(iommu_id, dmar_domain)
5065                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5066                                       start_pfn, npages, !freelist, 0);
5067
5068         dma_free_pagelist(freelist);
5069
5070         if (dmar_domain->max_addr == iova + size)
5071                 dmar_domain->max_addr = iova;
5072
5073         return size;
5074 }
5075
5076 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5077                                             dma_addr_t iova)
5078 {
5079         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5080         struct dma_pte *pte;
5081         int level = 0;
5082         u64 phys = 0;
5083
5084         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5085         if (pte)
5086                 phys = dma_pte_addr(pte);
5087
5088         return phys;
5089 }
5090
5091 static bool intel_iommu_capable(enum iommu_cap cap)
5092 {
5093         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5094                 return domain_update_iommu_snooping(NULL) == 1;
5095         if (cap == IOMMU_CAP_INTR_REMAP)
5096                 return irq_remapping_enabled == 1;
5097
5098         return false;
5099 }
5100
5101 static int intel_iommu_add_device(struct device *dev)
5102 {
5103         struct intel_iommu *iommu;
5104         struct iommu_group *group;
5105         u8 bus, devfn;
5106
5107         iommu = device_to_iommu(dev, &bus, &devfn);
5108         if (!iommu)
5109                 return -ENODEV;
5110
5111         iommu_device_link(&iommu->iommu, dev);
5112
5113         group = iommu_group_get_for_dev(dev);
5114
5115         if (IS_ERR(group))
5116                 return PTR_ERR(group);
5117
5118         iommu_group_put(group);
5119         return 0;
5120 }
5121
5122 static void intel_iommu_remove_device(struct device *dev)
5123 {
5124         struct intel_iommu *iommu;
5125         u8 bus, devfn;
5126
5127         iommu = device_to_iommu(dev, &bus, &devfn);
5128         if (!iommu)
5129                 return;
5130
5131         iommu_group_remove_device(dev);
5132
5133         iommu_device_unlink(&iommu->iommu, dev);
5134 }
5135
5136 static void intel_iommu_get_resv_regions(struct device *device,
5137                                          struct list_head *head)
5138 {
5139         struct iommu_resv_region *reg;
5140         struct dmar_rmrr_unit *rmrr;
5141         struct device *i_dev;
5142         int i;
5143
5144         rcu_read_lock();
5145         for_each_rmrr_units(rmrr) {
5146                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5147                                           i, i_dev) {
5148                         if (i_dev != device)
5149                                 continue;
5150
5151                         list_add_tail(&rmrr->resv->list, head);
5152                 }
5153         }
5154         rcu_read_unlock();
5155
5156         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5157                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5158                                       0, IOMMU_RESV_MSI);
5159         if (!reg)
5160                 return;
5161         list_add_tail(&reg->list, head);
5162 }
5163
5164 static void intel_iommu_put_resv_regions(struct device *dev,
5165                                          struct list_head *head)
5166 {
5167         struct iommu_resv_region *entry, *next;
5168
5169         list_for_each_entry_safe(entry, next, head, list) {
5170                 if (entry->type == IOMMU_RESV_RESERVED)
5171                         kfree(entry);
5172         }
5173 }
5174
5175 #ifdef CONFIG_INTEL_IOMMU_SVM
5176 #define MAX_NR_PASID_BITS (20)
5177 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5178 {
5179         /*
5180          * Convert ecap_pss to extend context entry pts encoding, also
5181          * respect the soft pasid_max value set by the iommu.
5182          * - number of PASID bits = ecap_pss + 1
5183          * - number of PASID table entries = 2^(pts + 5)
5184          * Therefore, pts = ecap_pss - 4
5185          * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5186          */
5187         if (ecap_pss(iommu->ecap) < 5)
5188                 return 0;
5189
5190         /* pasid_max is encoded as actual number of entries not the bits */
5191         return find_first_bit((unsigned long *)&iommu->pasid_max,
5192                         MAX_NR_PASID_BITS) - 5;
5193 }
5194
5195 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5196 {
5197         struct device_domain_info *info;
5198         struct context_entry *context;
5199         struct dmar_domain *domain;
5200         unsigned long flags;
5201         u64 ctx_lo;
5202         int ret;
5203
5204         domain = get_valid_domain_for_dev(sdev->dev);
5205         if (!domain)
5206                 return -EINVAL;
5207
5208         spin_lock_irqsave(&device_domain_lock, flags);
5209         spin_lock(&iommu->lock);
5210
5211         ret = -EINVAL;
5212         info = sdev->dev->archdata.iommu;
5213         if (!info || !info->pasid_supported)
5214                 goto out;
5215
5216         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5217         if (WARN_ON(!context))
5218                 goto out;
5219
5220         ctx_lo = context[0].lo;
5221
5222         sdev->did = domain->iommu_did[iommu->seq_id];
5223         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5224
5225         if (!(ctx_lo & CONTEXT_PASIDE)) {
5226                 if (iommu->pasid_state_table)
5227                         context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5228                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5229                         intel_iommu_get_pts(iommu);
5230
5231                 wmb();
5232                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5233                  * extended to permit requests-with-PASID if the PASIDE bit
5234                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5235                  * however, the PASIDE bit is ignored and requests-with-PASID
5236                  * are unconditionally blocked. Which makes less sense.
5237                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5238                  * "guest mode" translation types depending on whether ATS
5239                  * is available or not. Annoyingly, we can't use the new
5240                  * modes *unless* PASIDE is set. */
5241                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5242                         ctx_lo &= ~CONTEXT_TT_MASK;
5243                         if (info->ats_supported)
5244                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5245                         else
5246                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5247                 }
5248                 ctx_lo |= CONTEXT_PASIDE;
5249                 if (iommu->pasid_state_table)
5250                         ctx_lo |= CONTEXT_DINVE;
5251                 if (info->pri_supported)
5252                         ctx_lo |= CONTEXT_PRS;
5253                 context[0].lo = ctx_lo;
5254                 wmb();
5255                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5256                                            DMA_CCMD_MASK_NOBIT,
5257                                            DMA_CCMD_DEVICE_INVL);
5258         }
5259
5260         /* Enable PASID support in the device, if it wasn't already */
5261         if (!info->pasid_enabled)
5262                 iommu_enable_dev_iotlb(info);
5263
5264         if (info->ats_enabled) {
5265                 sdev->dev_iotlb = 1;
5266                 sdev->qdep = info->ats_qdep;
5267                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5268                         sdev->qdep = 0;
5269         }
5270         ret = 0;
5271
5272  out:
5273         spin_unlock(&iommu->lock);
5274         spin_unlock_irqrestore(&device_domain_lock, flags);
5275
5276         return ret;
5277 }
5278
5279 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5280 {
5281         struct intel_iommu *iommu;
5282         u8 bus, devfn;
5283
5284         if (iommu_dummy(dev)) {
5285                 dev_warn(dev,
5286                          "No IOMMU translation for device; cannot enable SVM\n");
5287                 return NULL;
5288         }
5289
5290         iommu = device_to_iommu(dev, &bus, &devfn);
5291         if ((!iommu)) {
5292                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5293                 return NULL;
5294         }
5295
5296         if (!iommu->pasid_table) {
5297                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5298                 return NULL;
5299         }
5300
5301         return iommu;
5302 }
5303 #endif /* CONFIG_INTEL_IOMMU_SVM */
5304
5305 const struct iommu_ops intel_iommu_ops = {
5306         .capable                = intel_iommu_capable,
5307         .domain_alloc           = intel_iommu_domain_alloc,
5308         .domain_free            = intel_iommu_domain_free,
5309         .attach_dev             = intel_iommu_attach_device,
5310         .detach_dev             = intel_iommu_detach_device,
5311         .map                    = intel_iommu_map,
5312         .unmap                  = intel_iommu_unmap,
5313         .map_sg                 = default_iommu_map_sg,
5314         .iova_to_phys           = intel_iommu_iova_to_phys,
5315         .add_device             = intel_iommu_add_device,
5316         .remove_device          = intel_iommu_remove_device,
5317         .get_resv_regions       = intel_iommu_get_resv_regions,
5318         .put_resv_regions       = intel_iommu_put_resv_regions,
5319         .device_group           = pci_device_group,
5320         .pgsize_bitmap          = INTEL_IOMMU_PGSIZES,
5321 };
5322
5323 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5324 {
5325         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5326         pr_info("Disabling IOMMU for graphics on this chipset\n");
5327         dmar_map_gfx = 0;
5328 }
5329
5330 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5331 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5332 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5333 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5334 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5335 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5336 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5337
5338 static void quirk_iommu_rwbf(struct pci_dev *dev)
5339 {
5340         /*
5341          * Mobile 4 Series Chipset neglects to set RWBF capability,
5342          * but needs it. Same seems to hold for the desktop versions.
5343          */
5344         pr_info("Forcing write-buffer flush capability\n");
5345         rwbf_quirk = 1;
5346 }
5347
5348 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5349 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5350 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5351 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5352 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5353 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5354 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5355
5356 #define GGC 0x52
5357 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5358 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5359 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5360 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5361 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5362 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5363 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5364 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5365
5366 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5367 {
5368         unsigned short ggc;
5369
5370         if (pci_read_config_word(dev, GGC, &ggc))
5371                 return;
5372
5373         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5374                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5375                 dmar_map_gfx = 0;
5376         } else if (dmar_map_gfx) {
5377                 /* we have to ensure the gfx device is idle before we flush */
5378                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5379                 intel_iommu_strict = 1;
5380        }
5381 }
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5383 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5384 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5385 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5386
5387 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5388    ISOCH DMAR unit for the Azalia sound device, but not give it any
5389    TLB entries, which causes it to deadlock. Check for that.  We do
5390    this in a function called from init_dmars(), instead of in a PCI
5391    quirk, because we don't want to print the obnoxious "BIOS broken"
5392    message if VT-d is actually disabled.
5393 */
5394 static void __init check_tylersburg_isoch(void)
5395 {
5396         struct pci_dev *pdev;
5397         uint32_t vtisochctrl;
5398
5399         /* If there's no Azalia in the system anyway, forget it. */
5400         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5401         if (!pdev)
5402                 return;
5403         pci_dev_put(pdev);
5404
5405         /* System Management Registers. Might be hidden, in which case
5406            we can't do the sanity check. But that's OK, because the
5407            known-broken BIOSes _don't_ actually hide it, so far. */
5408         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5409         if (!pdev)
5410                 return;
5411
5412         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5413                 pci_dev_put(pdev);
5414                 return;
5415         }
5416
5417         pci_dev_put(pdev);
5418
5419         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5420         if (vtisochctrl & 1)
5421                 return;
5422
5423         /* Drop all bits other than the number of TLB entries */
5424         vtisochctrl &= 0x1c;
5425
5426         /* If we have the recommended number of TLB entries (16), fine. */
5427         if (vtisochctrl == 0x10)
5428                 return;
5429
5430         /* Zero TLB entries? You get to ride the short bus to school. */
5431         if (!vtisochctrl) {
5432                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5433                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5434                      dmi_get_system_info(DMI_BIOS_VENDOR),
5435                      dmi_get_system_info(DMI_BIOS_VERSION),
5436                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5437                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5438                 return;
5439         }
5440
5441         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5442                vtisochctrl);
5443 }