drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <linux/memblock.h>
  45 #include <linux/dma-contiguous.h>
  46 #include <asm/irq_remapping.h>
  47 #include <asm/cacheflush.h>
  48 #include <asm/iommu.h>
  49
  50 #include "irq_remapping.h"
  51
  52 #define ROOT_SIZE               VTD_PAGE_SIZE
  53 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  54
  55 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  56 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  57 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  58 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  59
  60 #define IOAPIC_RANGE_START      (0xfee00000)
  61 #define IOAPIC_RANGE_END        (0xfeefffff)
  62 #define IOVA_START_ADDR         (0x1000)
  63
  64 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  65
  66 #define MAX_AGAW_WIDTH 64
  67 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  68
  69 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  70 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  71
  72 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  73    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  74 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  75                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  76 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  77
  78 /* IO virtual address start page frame number */
  79 #define IOVA_START_PFN          (1)
  80
  81 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  82 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  83 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  84
  85 /* page table handling */
  86 #define LEVEL_STRIDE            (9)
  87 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  88
  89 /*
  90  * This bitmap is used to advertise the page sizes our hardware support
  91  * to the IOMMU core, which will then use this information to split
  92  * physically contiguous memory regions it is mapping into page sizes
  93  * that we support.
  94  *
  95  * Traditionally the IOMMU core just handed us the mappings directly,
  96  * after making sure the size is an order of a 4KiB page and that the
  97  * mapping has natural alignment.
  98  *
  99  * To retain this behavior, we currently advertise that we support
 100  * all page sizes that are an order of 4KiB.
 101  *
 102  * If at some point we'd like to utilize the IOMMU core's new behavior,
 103  * we could change this to advertise the real page sizes we support.
 104  */
 105 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 106
 107 static inline int agaw_to_level(int agaw)
 108 {
 109         return agaw + 2;
 110 }
 111
 112 static inline int agaw_to_width(int agaw)
 113 {
 114         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 115 }
 116
 117 static inline int width_to_agaw(int width)
 118 {
 119         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 120 }
 121
 122 static inline unsigned int level_to_offset_bits(int level)
 123 {
 124         return (level - 1) * LEVEL_STRIDE;
 125 }
 126
 127 static inline int pfn_level_offset(unsigned long pfn, int level)
 128 {
 129         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 130 }
 131
 132 static inline unsigned long level_mask(int level)
 133 {
 134         return -1UL << level_to_offset_bits(level);
 135 }
 136
 137 static inline unsigned long level_size(int level)
 138 {
 139         return 1UL << level_to_offset_bits(level);
 140 }
 141
 142 static inline unsigned long align_to_level(unsigned long pfn, int level)
 143 {
 144         return (pfn + level_size(level) - 1) & level_mask(level);
 145 }
 146
 147 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 148 {
 149         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 150 }
 151
 152 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 153    are never going to work. */
 154 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 155 {
 156         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 157 }
 158
 159 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 160 {
 161         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 162 }
 163 static inline unsigned long page_to_dma_pfn(struct page *pg)
 164 {
 165         return mm_to_dma_pfn(page_to_pfn(pg));
 166 }
 167 static inline unsigned long virt_to_dma_pfn(void *p)
 168 {
 169         return page_to_dma_pfn(virt_to_page(p));
 170 }
 171
 172 /* global iommu list, set NULL for ignored DMAR units */
 173 static struct intel_iommu **g_iommus;
 174
 175 static void __init check_tylersburg_isoch(void);
 176 static int rwbf_quirk;
 177
 178 /*
 179  * set to 1 to panic kernel if can't successfully enable VT-d
 180  * (used when kernel is launched w/ TXT)
 181  */
 182 static int force_on = 0;
 183
 184 /*
 185  * 0: Present
 186  * 1-11: Reserved
 187  * 12-63: Context Ptr (12 - (haw-1))
 188  * 64-127: Reserved
 189  */
 190 struct root_entry {
 191         u64     lo;
 192         u64     hi;
 193 };
 194 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 195
 196
 197 /*
 198  * low 64 bits:
 199  * 0: present
 200  * 1: fault processing disable
 201  * 2-3: translation type
 202  * 12-63: address space root
 203  * high 64 bits:
 204  * 0-2: address width
 205  * 3-6: aval
 206  * 8-23: domain id
 207  */
 208 struct context_entry {
 209         u64 lo;
 210         u64 hi;
 211 };
 212
 213 static inline bool context_present(struct context_entry *context)
 214 {
 215         return (context->lo & 1);
 216 }
 217 static inline void context_set_present(struct context_entry *context)
 218 {
 219         context->lo |= 1;
 220 }
 221
 222 static inline void context_set_fault_enable(struct context_entry *context)
 223 {
 224         context->lo &= (((u64)-1) << 2) | 1;
 225 }
 226
 227 static inline void context_set_translation_type(struct context_entry *context,
 228                                                 unsigned long value)
 229 {
 230         context->lo &= (((u64)-1) << 4) | 3;
 231         context->lo |= (value & 3) << 2;
 232 }
 233
 234 static inline void context_set_address_root(struct context_entry *context,
 235                                             unsigned long value)
 236 {
 237         context->lo &= ~VTD_PAGE_MASK;
 238         context->lo |= value & VTD_PAGE_MASK;
 239 }
 240
 241 static inline void context_set_address_width(struct context_entry *context,
 242                                              unsigned long value)
 243 {
 244         context->hi |= value & 7;
 245 }
 246
 247 static inline void context_set_domain_id(struct context_entry *context,
 248                                          unsigned long value)
 249 {
 250         context->hi |= (value & ((1 << 16) - 1)) << 8;
 251 }
 252
 253 static inline void context_clear_entry(struct context_entry *context)
 254 {
 255         context->lo = 0;
 256         context->hi = 0;
 257 }
 258
 259 /*
 260  * 0: readable
 261  * 1: writable
 262  * 2-6: reserved
 263  * 7: super page
 264  * 8-10: available
 265  * 11: snoop behavior
 266  * 12-63: Host physcial address
 267  */
 268 struct dma_pte {
 269         u64 val;
 270 };
 271
 272 static inline void dma_clear_pte(struct dma_pte *pte)
 273 {
 274         pte->val = 0;
 275 }
 276
 277 static inline u64 dma_pte_addr(struct dma_pte *pte)
 278 {
 279 #ifdef CONFIG_64BIT
 280         return pte->val & VTD_PAGE_MASK;
 281 #else
 282         /* Must have a full atomic 64-bit read */
 283         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 284 #endif
 285 }
 286
 287 static inline bool dma_pte_present(struct dma_pte *pte)
 288 {
 289         return (pte->val & 3) != 0;
 290 }
 291
 292 static inline bool dma_pte_superpage(struct dma_pte *pte)
 293 {
 294         return (pte->val & DMA_PTE_LARGE_PAGE);
 295 }
 296
 297 static inline int first_pte_in_page(struct dma_pte *pte)
 298 {
 299         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 300 }
 301
 302 /*
 303  * This domain is a statically identity mapping domain.
 304  *      1. This domain creats a static 1:1 mapping to all usable memory.
 305  *      2. It maps to each iommu if successful.
 306  *      3. Each iommu mapps to this domain if successful.
 307  */
 308 static struct dmar_domain *si_domain;
 309 static int hw_pass_through = 1;
 310
 311 /* domain represents a virtual machine, more than one devices
 312  * across iommus may be owned in one domain, e.g. kvm guest.
 313  */
 314 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 315
 316 /* si_domain contains mulitple devices */
 317 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 318
 319 struct dmar_domain {
 320         int     id;                     /* domain id */
 321         int     nid;                    /* node id */
 322         DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
 323                                         /* bitmap of iommus this domain uses*/
 324
 325         struct list_head devices;       /* all devices' list */
 326         struct iova_domain iovad;       /* iova's that belong to this domain */
 327
 328         struct dma_pte  *pgd;           /* virtual address */
 329         int             gaw;            /* max guest address width */
 330
 331         /* adjusted guest address width, 0 is level 2 30-bit */
 332         int             agaw;
 333
 334         int             flags;          /* flags to find out type of domain */
 335
 336         int             iommu_coherency;/* indicate coherency of iommu access */
 337         int             iommu_snooping; /* indicate snooping control feature*/
 338         int             iommu_count;    /* reference count of iommu */
 339         int             iommu_superpage;/* Level of superpages supported:
 340                                            0 == 4KiB (no superpages), 1 == 2MiB,
 341                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 342         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 343         u64             max_addr;       /* maximum mapped address */
 344
 345         struct iommu_domain domain;     /* generic domain data structure for
 346                                            iommu core */
 347 };
 348
 349 /* PCI domain-device relationship */
 350 struct device_domain_info {
 351         struct list_head link;  /* link to domain siblings */
 352         struct list_head global; /* link to global list */
 353         u8 bus;                 /* PCI bus number */
 354         u8 devfn;               /* PCI devfn number */
 355         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 356         struct intel_iommu *iommu; /* IOMMU used by this device */
 357         struct dmar_domain *domain; /* pointer to domain */
 358 };
 359
 360 struct dmar_rmrr_unit {
 361         struct list_head list;          /* list of rmrr units   */
 362         struct acpi_dmar_header *hdr;   /* ACPI header          */
 363         u64     base_address;           /* reserved base address*/
 364         u64     end_address;            /* reserved end address */
 365         struct dmar_dev_scope *devices; /* target devices */
 366         int     devices_cnt;            /* target device count */
 367 };
 368
 369 struct dmar_atsr_unit {
 370         struct list_head list;          /* list of ATSR units */
 371         struct acpi_dmar_header *hdr;   /* ACPI header */
 372         struct dmar_dev_scope *devices; /* target devices */
 373         int devices_cnt;                /* target device count */
 374         u8 include_all:1;               /* include all ports */
 375 };
 376
 377 static LIST_HEAD(dmar_atsr_units);
 378 static LIST_HEAD(dmar_rmrr_units);
 379
 380 #define for_each_rmrr_units(rmrr) \
 381         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 382
 383 static void flush_unmaps_timeout(unsigned long data);
 384
 385 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 386
 387 #define HIGH_WATER_MARK 250
 388 struct deferred_flush_tables {
 389         int next;
 390         struct iova *iova[HIGH_WATER_MARK];
 391         struct dmar_domain *domain[HIGH_WATER_MARK];
 392         struct page *freelist[HIGH_WATER_MARK];
 393 };
 394
 395 static struct deferred_flush_tables *deferred_flush;
 396
 397 /* bitmap for indexing intel_iommus */
 398 static int g_num_of_iommus;
 399
 400 static DEFINE_SPINLOCK(async_umap_flush_lock);
 401 static LIST_HEAD(unmaps_to_do);
 402
 403 static int timer_on;
 404 static long list_size;
 405
 406 static void domain_exit(struct dmar_domain *domain);
 407 static void domain_remove_dev_info(struct dmar_domain *domain);
 408 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 409                                        struct device *dev);
 410 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
 411                                            struct device *dev);
 412 static int domain_detach_iommu(struct dmar_domain *domain,
 413                                struct intel_iommu *iommu);
 414
 415 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 416 int dmar_disabled = 0;
 417 #else
 418 int dmar_disabled = 1;
 419 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 420
 421 int intel_iommu_enabled = 0;
 422 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 423
 424 static int dmar_map_gfx = 1;
 425 static int dmar_forcedac;
 426 static int intel_iommu_strict;
 427 static int intel_iommu_superpage = 1;
 428 static int intel_iommu_ecs = 1;
 429
 430 /* We only actually use ECS when PASID support (on the new bit 40)
 431  * is also advertised. Some early implementations — the ones with
 432  * PASID support on bit 28 — have issues even when we *only* use
 433  * extended root/context tables. */
 434 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 435                             ecap_pasid(iommu->ecap))
 436
 437 int intel_iommu_gfx_mapped;
 438 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 439
 440 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 441 static DEFINE_SPINLOCK(device_domain_lock);
 442 static LIST_HEAD(device_domain_list);
 443
 444 static const struct iommu_ops intel_iommu_ops;
 445
 446 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 447 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 448 {
 449         return container_of(dom, struct dmar_domain, domain);
 450 }
 451
 452 static int __init intel_iommu_setup(char *str)
 453 {
 454         if (!str)
 455                 return -EINVAL;
 456         while (*str) {
 457                 if (!strncmp(str, "on", 2)) {
 458                         dmar_disabled = 0;
 459                         pr_info("IOMMU enabled\n");
 460                 } else if (!strncmp(str, "off", 3)) {
 461                         dmar_disabled = 1;
 462                         pr_info("IOMMU disabled\n");
 463                 } else if (!strncmp(str, "igfx_off", 8)) {
 464                         dmar_map_gfx = 0;
 465                         pr_info("Disable GFX device mapping\n");
 466                 } else if (!strncmp(str, "forcedac", 8)) {
 467                         pr_info("Forcing DAC for PCI devices\n");
 468                         dmar_forcedac = 1;
 469                 } else if (!strncmp(str, "strict", 6)) {
 470                         pr_info("Disable batched IOTLB flush\n");
 471                         intel_iommu_strict = 1;
 472                 } else if (!strncmp(str, "sp_off", 6)) {
 473                         pr_info("Disable supported super page\n");
 474                         intel_iommu_superpage = 0;
 475                 } else if (!strncmp(str, "ecs_off", 7)) {
 476                         printk(KERN_INFO
 477                                 "Intel-IOMMU: disable extended context table support\n");
 478                         intel_iommu_ecs = 0;
 479                 }
 480
 481                 str += strcspn(str, ",");
 482                 while (*str == ',')
 483                         str++;
 484         }
 485         return 0;
 486 }
 487 __setup("intel_iommu=", intel_iommu_setup);
 488
 489 static struct kmem_cache *iommu_domain_cache;
 490 static struct kmem_cache *iommu_devinfo_cache;
 491
 492 static inline void *alloc_pgtable_page(int node)
 493 {
 494         struct page *page;
 495         void *vaddr = NULL;
 496
 497         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 498         if (page)
 499                 vaddr = page_address(page);
 500         return vaddr;
 501 }
 502
 503 static inline void free_pgtable_page(void *vaddr)
 504 {
 505         free_page((unsigned long)vaddr);
 506 }
 507
 508 static inline void *alloc_domain_mem(void)
 509 {
 510         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 511 }
 512
 513 static void free_domain_mem(void *vaddr)
 514 {
 515         kmem_cache_free(iommu_domain_cache, vaddr);
 516 }
 517
 518 static inline void * alloc_devinfo_mem(void)
 519 {
 520         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 521 }
 522
 523 static inline void free_devinfo_mem(void *vaddr)
 524 {
 525         kmem_cache_free(iommu_devinfo_cache, vaddr);
 526 }
 527
 528 static inline int domain_type_is_vm(struct dmar_domain *domain)
 529 {
 530         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 531 }
 532
 533 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 534 {
 535         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 536                                 DOMAIN_FLAG_STATIC_IDENTITY);
 537 }
 538
 539 static inline int domain_pfn_supported(struct dmar_domain *domain,
 540                                        unsigned long pfn)
 541 {
 542         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 543
 544         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 545 }
 546
 547 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 548 {
 549         unsigned long sagaw;
 550         int agaw = -1;
 551
 552         sagaw = cap_sagaw(iommu->cap);
 553         for (agaw = width_to_agaw(max_gaw);
 554              agaw >= 0; agaw--) {
 555                 if (test_bit(agaw, &sagaw))
 556                         break;
 557         }
 558
 559         return agaw;
 560 }
 561
 562 /*
 563  * Calculate max SAGAW for each iommu.
 564  */
 565 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 566 {
 567         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 568 }
 569
 570 /*
 571  * calculate agaw for each iommu.
 572  * "SAGAW" may be different across iommus, use a default agaw, and
 573  * get a supported less agaw for iommus that don't support the default agaw.
 574  */
 575 int iommu_calculate_agaw(struct intel_iommu *iommu)
 576 {
 577         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 578 }
 579
 580 /* This functionin only returns single iommu in a domain */
 581 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 582 {
 583         int iommu_id;
 584
 585         /* si_domain and vm domain should not get here. */
 586         BUG_ON(domain_type_is_vm_or_si(domain));
 587         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 588         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 589                 return NULL;
 590
 591         return g_iommus[iommu_id];
 592 }
 593
 594 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 595 {
 596         struct dmar_drhd_unit *drhd;
 597         struct intel_iommu *iommu;
 598         bool found = false;
 599         int i;
 600
 601         domain->iommu_coherency = 1;
 602
 603         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 604                 found = true;
 605                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 606                         domain->iommu_coherency = 0;
 607                         break;
 608                 }
 609         }
 610         if (found)
 611                 return;
 612
 613         /* No hardware attached; use lowest common denominator */
 614         rcu_read_lock();
 615         for_each_active_iommu(iommu, drhd) {
 616                 if (!ecap_coherent(iommu->ecap)) {
 617                         domain->iommu_coherency = 0;
 618                         break;
 619                 }
 620         }
 621         rcu_read_unlock();
 622 }
 623
 624 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 625 {
 626         struct dmar_drhd_unit *drhd;
 627         struct intel_iommu *iommu;
 628         int ret = 1;
 629
 630         rcu_read_lock();
 631         for_each_active_iommu(iommu, drhd) {
 632                 if (iommu != skip) {
 633                         if (!ecap_sc_support(iommu->ecap)) {
 634                                 ret = 0;
 635                                 break;
 636                         }
 637                 }
 638         }
 639         rcu_read_unlock();
 640
 641         return ret;
 642 }
 643
 644 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 645 {
 646         struct dmar_drhd_unit *drhd;
 647         struct intel_iommu *iommu;
 648         int mask = 0xf;
 649
 650         if (!intel_iommu_superpage) {
 651                 return 0;
 652         }
 653
 654         /* set iommu_superpage to the smallest common denominator */
 655         rcu_read_lock();
 656         for_each_active_iommu(iommu, drhd) {
 657                 if (iommu != skip) {
 658                         mask &= cap_super_page_val(iommu->cap);
 659                         if (!mask)
 660                                 break;
 661                 }
 662         }
 663         rcu_read_unlock();
 664
 665         return fls(mask);
 666 }
 667
 668 /* Some capabilities may be different across iommus */
 669 static void domain_update_iommu_cap(struct dmar_domain *domain)
 670 {
 671         domain_update_iommu_coherency(domain);
 672         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 673         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 674 }
 675
 676 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 677                                                        u8 bus, u8 devfn, int alloc)
 678 {
 679         struct root_entry *root = &iommu->root_entry[bus];
 680         struct context_entry *context;
 681         u64 *entry;
 682
 683         if (ecs_enabled(iommu)) {
 684                 if (devfn >= 0x80) {
 685                         devfn -= 0x80;
 686                         entry = &root->hi;
 687                 }
 688                 devfn *= 2;
 689         }
 690         entry = &root->lo;
 691         if (*entry & 1)
 692                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 693         else {
 694                 unsigned long phy_addr;
 695                 if (!alloc)
 696                         return NULL;
 697
 698                 context = alloc_pgtable_page(iommu->node);
 699                 if (!context)
 700                         return NULL;
 701
 702                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 703                 phy_addr = virt_to_phys((void *)context);
 704                 *entry = phy_addr | 1;
 705                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 706         }
 707         return &context[devfn];
 708 }
 709
 710 static int iommu_dummy(struct device *dev)
 711 {
 712         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 713 }
 714
 715 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 716 {
 717         struct dmar_drhd_unit *drhd = NULL;
 718         struct intel_iommu *iommu;
 719         struct device *tmp;
 720         struct pci_dev *ptmp, *pdev = NULL;
 721         u16 segment = 0;
 722         int i;
 723
 724         if (iommu_dummy(dev))
 725                 return NULL;
 726
 727         if (dev_is_pci(dev)) {
 728                 pdev = to_pci_dev(dev);
 729                 segment = pci_domain_nr(pdev->bus);
 730         } else if (has_acpi_companion(dev))
 731                 dev = &ACPI_COMPANION(dev)->dev;
 732
 733         rcu_read_lock();
 734         for_each_active_iommu(iommu, drhd) {
 735                 if (pdev && segment != drhd->segment)
 736                         continue;
 737
 738                 for_each_active_dev_scope(drhd->devices,
 739                                           drhd->devices_cnt, i, tmp) {
 740                         if (tmp == dev) {
 741                                 *bus = drhd->devices[i].bus;
 742                                 *devfn = drhd->devices[i].devfn;
 743                                 goto out;
 744                         }
 745
 746                         if (!pdev || !dev_is_pci(tmp))
 747                                 continue;
 748
 749                         ptmp = to_pci_dev(tmp);
 750                         if (ptmp->subordinate &&
 751                             ptmp->subordinate->number <= pdev->bus->number &&
 752                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 753                                 goto got_pdev;
 754                 }
 755
 756                 if (pdev && drhd->include_all) {
 757                 got_pdev:
 758                         *bus = pdev->bus->number;
 759                         *devfn = pdev->devfn;
 760                         goto out;
 761                 }
 762         }
 763         iommu = NULL;
 764  out:
 765         rcu_read_unlock();
 766
 767         return iommu;
 768 }
 769
 770 static void domain_flush_cache(struct dmar_domain *domain,
 771                                void *addr, int size)
 772 {
 773         if (!domain->iommu_coherency)
 774                 clflush_cache_range(addr, size);
 775 }
 776
 777 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 778 {
 779         struct context_entry *context;
 780         int ret = 0;
 781         unsigned long flags;
 782
 783         spin_lock_irqsave(&iommu->lock, flags);
 784         context = iommu_context_addr(iommu, bus, devfn, 0);
 785         if (context)
 786                 ret = context_present(context);
 787         spin_unlock_irqrestore(&iommu->lock, flags);
 788         return ret;
 789 }
 790
 791 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 792 {
 793         struct context_entry *context;
 794         unsigned long flags;
 795
 796         spin_lock_irqsave(&iommu->lock, flags);
 797         context = iommu_context_addr(iommu, bus, devfn, 0);
 798         if (context) {
 799                 context_clear_entry(context);
 800                 __iommu_flush_cache(iommu, context, sizeof(*context));
 801         }
 802         spin_unlock_irqrestore(&iommu->lock, flags);
 803 }
 804
 805 static void free_context_table(struct intel_iommu *iommu)
 806 {
 807         int i;
 808         unsigned long flags;
 809         struct context_entry *context;
 810
 811         spin_lock_irqsave(&iommu->lock, flags);
 812         if (!iommu->root_entry) {
 813                 goto out;
 814         }
 815         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 816                 context = iommu_context_addr(iommu, i, 0, 0);
 817                 if (context)
 818                         free_pgtable_page(context);
 819
 820                 if (!ecs_enabled(iommu))
 821                         continue;
 822
 823                 context = iommu_context_addr(iommu, i, 0x80, 0);
 824                 if (context)
 825                         free_pgtable_page(context);
 826
 827         }
 828         free_pgtable_page(iommu->root_entry);
 829         iommu->root_entry = NULL;
 830 out:
 831         spin_unlock_irqrestore(&iommu->lock, flags);
 832 }
 833
 834 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 835                                       unsigned long pfn, int *target_level)
 836 {
 837         struct dma_pte *parent, *pte = NULL;
 838         int level = agaw_to_level(domain->agaw);
 839         int offset;
 840
 841         BUG_ON(!domain->pgd);
 842
 843         if (!domain_pfn_supported(domain, pfn))
 844                 /* Address beyond IOMMU's addressing capabilities. */
 845                 return NULL;
 846
 847         parent = domain->pgd;
 848
 849         while (1) {
 850                 void *tmp_page;
 851
 852                 offset = pfn_level_offset(pfn, level);
 853                 pte = &parent[offset];
 854                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 855                         break;
 856                 if (level == *target_level)
 857                         break;
 858
 859                 if (!dma_pte_present(pte)) {
 860                         uint64_t pteval;
 861
 862                         tmp_page = alloc_pgtable_page(domain->nid);
 863
 864                         if (!tmp_page)
 865                                 return NULL;
 866
 867                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 868                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 869                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 870                                 /* Someone else set it while we were thinking; use theirs. */
 871                                 free_pgtable_page(tmp_page);
 872                         else
 873                                 domain_flush_cache(domain, pte, sizeof(*pte));
 874                 }
 875                 if (level == 1)
 876                         break;
 877
 878                 parent = phys_to_virt(dma_pte_addr(pte));
 879                 level--;
 880         }
 881
 882         if (!*target_level)
 883                 *target_level = level;
 884
 885         return pte;
 886 }
 887
 888
 889 /* return address's pte at specific level */
 890 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 891                                          unsigned long pfn,
 892                                          int level, int *large_page)
 893 {
 894         struct dma_pte *parent, *pte = NULL;
 895         int total = agaw_to_level(domain->agaw);
 896         int offset;
 897
 898         parent = domain->pgd;
 899         while (level <= total) {
 900                 offset = pfn_level_offset(pfn, total);
 901                 pte = &parent[offset];
 902                 if (level == total)
 903                         return pte;
 904
 905                 if (!dma_pte_present(pte)) {
 906                         *large_page = total;
 907                         break;
 908                 }
 909
 910                 if (dma_pte_superpage(pte)) {
 911                         *large_page = total;
 912                         return pte;
 913                 }
 914
 915                 parent = phys_to_virt(dma_pte_addr(pte));
 916                 total--;
 917         }
 918         return NULL;
 919 }
 920
 921 /* clear last level pte, a tlb flush should be followed */
 922 static void dma_pte_clear_range(struct dmar_domain *domain,
 923                                 unsigned long start_pfn,
 924                                 unsigned long last_pfn)
 925 {
 926         unsigned int large_page = 1;
 927         struct dma_pte *first_pte, *pte;
 928
 929         BUG_ON(!domain_pfn_supported(domain, start_pfn));
 930         BUG_ON(!domain_pfn_supported(domain, last_pfn));
 931         BUG_ON(start_pfn > last_pfn);
 932
 933         /* we don't need lock here; nobody else touches the iova range */
 934         do {
 935                 large_page = 1;
 936                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 937                 if (!pte) {
 938                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 939                         continue;
 940                 }
 941                 do {
 942                         dma_clear_pte(pte);
 943                         start_pfn += lvl_to_nr_pages(large_page);
 944                         pte++;
 945                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 946
 947                 domain_flush_cache(domain, first_pte,
 948                                    (void *)pte - (void *)first_pte);
 949
 950         } while (start_pfn && start_pfn <= last_pfn);
 951 }
 952
 953 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 954                                struct dma_pte *pte, unsigned long pfn,
 955                                unsigned long start_pfn, unsigned long last_pfn)
 956 {
 957         pfn = max(start_pfn, pfn);
 958         pte = &pte[pfn_level_offset(pfn, level)];
 959
 960         do {
 961                 unsigned long level_pfn;
 962                 struct dma_pte *level_pte;
 963
 964                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 965                         goto next;
 966
 967                 level_pfn = pfn & level_mask(level - 1);
 968                 level_pte = phys_to_virt(dma_pte_addr(pte));
 969
 970                 if (level > 2)
 971                         dma_pte_free_level(domain, level - 1, level_pte,
 972                                            level_pfn, start_pfn, last_pfn);
 973
 974                 /* If range covers entire pagetable, free it */
 975                 if (!(start_pfn > level_pfn ||
 976                       last_pfn < level_pfn + level_size(level) - 1)) {
 977                         dma_clear_pte(pte);
 978                         domain_flush_cache(domain, pte, sizeof(*pte));
 979                         free_pgtable_page(level_pte);
 980                 }
 981 next:
 982                 pfn += level_size(level);
 983         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
 984 }
 985
 986 /* free page table pages. last level pte should already be cleared */
 987 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 988                                    unsigned long start_pfn,
 989                                    unsigned long last_pfn)
 990 {
 991         BUG_ON(!domain_pfn_supported(domain, start_pfn));
 992         BUG_ON(!domain_pfn_supported(domain, last_pfn));
 993         BUG_ON(start_pfn > last_pfn);
 994
 995         dma_pte_clear_range(domain, start_pfn, last_pfn);
 996
 997         /* We don't need lock here; nobody else touches the iova range */
 998         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
 999                            domain->pgd, 0, start_pfn, last_pfn);
1000
1001         /* free pgd */
1002         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1003                 free_pgtable_page(domain->pgd);
1004                 domain->pgd = NULL;
1005         }
1006 }
1007
1008 /* When a page at a given level is being unlinked from its parent, we don't
1009    need to *modify* it at all. All we need to do is make a list of all the
1010    pages which can be freed just as soon as we've flushed the IOTLB and we
1011    know the hardware page-walk will no longer touch them.
1012    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1013    be freed. */
1014 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1015                                             int level, struct dma_pte *pte,
1016                                             struct page *freelist)
1017 {
1018         struct page *pg;
1019
1020         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1021         pg->freelist = freelist;
1022         freelist = pg;
1023
1024         if (level == 1)
1025                 return freelist;
1026
1027         pte = page_address(pg);
1028         do {
1029                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1030                         freelist = dma_pte_list_pagetables(domain, level - 1,
1031                                                            pte, freelist);
1032                 pte++;
1033         } while (!first_pte_in_page(pte));
1034
1035         return freelist;
1036 }
1037
1038 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1039                                         struct dma_pte *pte, unsigned long pfn,
1040                                         unsigned long start_pfn,
1041                                         unsigned long last_pfn,
1042                                         struct page *freelist)
1043 {
1044         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1045
1046         pfn = max(start_pfn, pfn);
1047         pte = &pte[pfn_level_offset(pfn, level)];
1048
1049         do {
1050                 unsigned long level_pfn;
1051
1052                 if (!dma_pte_present(pte))
1053                         goto next;
1054
1055                 level_pfn = pfn & level_mask(level);
1056
1057                 /* If range covers entire pagetable, free it */
1058                 if (start_pfn <= level_pfn &&
1059                     last_pfn >= level_pfn + level_size(level) - 1) {
1060                         /* These suborbinate page tables are going away entirely. Don't
1061                            bother to clear them; we're just going to *free* them. */
1062                         if (level > 1 && !dma_pte_superpage(pte))
1063                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1064
1065                         dma_clear_pte(pte);
1066                         if (!first_pte)
1067                                 first_pte = pte;
1068                         last_pte = pte;
1069                 } else if (level > 1) {
1070                         /* Recurse down into a level that isn't *entirely* obsolete */
1071                         freelist = dma_pte_clear_level(domain, level - 1,
1072                                                        phys_to_virt(dma_pte_addr(pte)),
1073                                                        level_pfn, start_pfn, last_pfn,
1074                                                        freelist);
1075                 }
1076 next:
1077                 pfn += level_size(level);
1078         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1079
1080         if (first_pte)
1081                 domain_flush_cache(domain, first_pte,
1082                                    (void *)++last_pte - (void *)first_pte);
1083
1084         return freelist;
1085 }
1086
1087 /* We can't just free the pages because the IOMMU may still be walking
1088    the page tables, and may have cached the intermediate levels. The
1089    pages can only be freed after the IOTLB flush has been done. */
1090 struct page *domain_unmap(struct dmar_domain *domain,
1091                           unsigned long start_pfn,
1092                           unsigned long last_pfn)
1093 {
1094         struct page *freelist = NULL;
1095
1096         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1098         BUG_ON(start_pfn > last_pfn);
1099
1100         /* we don't need lock here; nobody else touches the iova range */
1101         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1102                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1103
1104         /* free pgd */
1105         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1106                 struct page *pgd_page = virt_to_page(domain->pgd);
1107                 pgd_page->freelist = freelist;
1108                 freelist = pgd_page;
1109
1110                 domain->pgd = NULL;
1111         }
1112
1113         return freelist;
1114 }
1115
1116 void dma_free_pagelist(struct page *freelist)
1117 {
1118         struct page *pg;
1119
1120         while ((pg = freelist)) {
1121                 freelist = pg->freelist;
1122                 free_pgtable_page(page_address(pg));
1123         }
1124 }
1125
1126 /* iommu handling */
1127 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1128 {
1129         struct root_entry *root;
1130         unsigned long flags;
1131
1132         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1133         if (!root) {
1134                 pr_err("Allocating root entry for %s failed\n",
1135                         iommu->name);
1136                 return -ENOMEM;
1137         }
1138
1139         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1140
1141         spin_lock_irqsave(&iommu->lock, flags);
1142         iommu->root_entry = root;
1143         spin_unlock_irqrestore(&iommu->lock, flags);
1144
1145         return 0;
1146 }
1147
1148 static void iommu_set_root_entry(struct intel_iommu *iommu)
1149 {
1150         u64 addr;
1151         u32 sts;
1152         unsigned long flag;
1153
1154         addr = virt_to_phys(iommu->root_entry);
1155         if (ecs_enabled(iommu))
1156                 addr |= DMA_RTADDR_RTT;
1157
1158         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1159         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1160
1161         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1162
1163         /* Make sure hardware complete it */
1164         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1165                       readl, (sts & DMA_GSTS_RTPS), sts);
1166
1167         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1168 }
1169
1170 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1171 {
1172         u32 val;
1173         unsigned long flag;
1174
1175         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1176                 return;
1177
1178         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1179         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1180
1181         /* Make sure hardware complete it */
1182         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1183                       readl, (!(val & DMA_GSTS_WBFS)), val);
1184
1185         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1186 }
1187
1188 /* return value determine if we need a write buffer flush */
1189 static void __iommu_flush_context(struct intel_iommu *iommu,
1190                                   u16 did, u16 source_id, u8 function_mask,
1191                                   u64 type)
1192 {
1193         u64 val = 0;
1194         unsigned long flag;
1195
1196         switch (type) {
1197         case DMA_CCMD_GLOBAL_INVL:
1198                 val = DMA_CCMD_GLOBAL_INVL;
1199                 break;
1200         case DMA_CCMD_DOMAIN_INVL:
1201                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1202                 break;
1203         case DMA_CCMD_DEVICE_INVL:
1204                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1205                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1206                 break;
1207         default:
1208                 BUG();
1209         }
1210         val |= DMA_CCMD_ICC;
1211
1212         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1213         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1214
1215         /* Make sure hardware complete it */
1216         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1217                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1218
1219         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1220 }
1221
1222 /* return value determine if we need a write buffer flush */
1223 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1224                                 u64 addr, unsigned int size_order, u64 type)
1225 {
1226         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1227         u64 val = 0, val_iva = 0;
1228         unsigned long flag;
1229
1230         switch (type) {
1231         case DMA_TLB_GLOBAL_FLUSH:
1232                 /* global flush doesn't need set IVA_REG */
1233                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1234                 break;
1235         case DMA_TLB_DSI_FLUSH:
1236                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1237                 break;
1238         case DMA_TLB_PSI_FLUSH:
1239                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1240                 /* IH bit is passed in as part of address */
1241                 val_iva = size_order | addr;
1242                 break;
1243         default:
1244                 BUG();
1245         }
1246         /* Note: set drain read/write */
1247 #if 0
1248         /*
1249          * This is probably to be super secure.. Looks like we can
1250          * ignore it without any impact.
1251          */
1252         if (cap_read_drain(iommu->cap))
1253                 val |= DMA_TLB_READ_DRAIN;
1254 #endif
1255         if (cap_write_drain(iommu->cap))
1256                 val |= DMA_TLB_WRITE_DRAIN;
1257
1258         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1259         /* Note: Only uses first TLB reg currently */
1260         if (val_iva)
1261                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1262         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1263
1264         /* Make sure hardware complete it */
1265         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1266                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1267
1268         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1269
1270         /* check IOTLB invalidation granularity */
1271         if (DMA_TLB_IAIG(val) == 0)
1272                 pr_err("Flush IOTLB failed\n");
1273         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1274                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1275                         (unsigned long long)DMA_TLB_IIRG(type),
1276                         (unsigned long long)DMA_TLB_IAIG(val));
1277 }
1278
1279 static struct device_domain_info *
1280 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1281                          u8 bus, u8 devfn)
1282 {
1283         bool found = false;
1284         unsigned long flags;
1285         struct device_domain_info *info;
1286         struct pci_dev *pdev;
1287
1288         if (!ecap_dev_iotlb_support(iommu->ecap))
1289                 return NULL;
1290
1291         if (!iommu->qi)
1292                 return NULL;
1293
1294         spin_lock_irqsave(&device_domain_lock, flags);
1295         list_for_each_entry(info, &domain->devices, link)
1296                 if (info->iommu == iommu && info->bus == bus &&
1297                     info->devfn == devfn) {
1298                         found = true;
1299                         break;
1300                 }
1301         spin_unlock_irqrestore(&device_domain_lock, flags);
1302
1303         if (!found || !info->dev || !dev_is_pci(info->dev))
1304                 return NULL;
1305
1306         pdev = to_pci_dev(info->dev);
1307
1308         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1309                 return NULL;
1310
1311         if (!dmar_find_matched_atsr_unit(pdev))
1312                 return NULL;
1313
1314         return info;
1315 }
1316
1317 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1318 {
1319         if (!info || !dev_is_pci(info->dev))
1320                 return;
1321
1322         pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1323 }
1324
1325 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1326 {
1327         if (!info->dev || !dev_is_pci(info->dev) ||
1328             !pci_ats_enabled(to_pci_dev(info->dev)))
1329                 return;
1330
1331         pci_disable_ats(to_pci_dev(info->dev));
1332 }
1333
1334 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1335                                   u64 addr, unsigned mask)
1336 {
1337         u16 sid, qdep;
1338         unsigned long flags;
1339         struct device_domain_info *info;
1340
1341         spin_lock_irqsave(&device_domain_lock, flags);
1342         list_for_each_entry(info, &domain->devices, link) {
1343                 struct pci_dev *pdev;
1344                 if (!info->dev || !dev_is_pci(info->dev))
1345                         continue;
1346
1347                 pdev = to_pci_dev(info->dev);
1348                 if (!pci_ats_enabled(pdev))
1349                         continue;
1350
1351                 sid = info->bus << 8 | info->devfn;
1352                 qdep = pci_ats_queue_depth(pdev);
1353                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1354         }
1355         spin_unlock_irqrestore(&device_domain_lock, flags);
1356 }
1357
1358 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1359                                   unsigned long pfn, unsigned int pages, int ih, int map)
1360 {
1361         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1362         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1363
1364         BUG_ON(pages == 0);
1365
1366         if (ih)
1367                 ih = 1 << 6;
1368         /*
1369          * Fallback to domain selective flush if no PSI support or the size is
1370          * too big.
1371          * PSI requires page size to be 2 ^ x, and the base address is naturally
1372          * aligned to the size
1373          */
1374         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1375                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1376                                                 DMA_TLB_DSI_FLUSH);
1377         else
1378                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1379                                                 DMA_TLB_PSI_FLUSH);
1380
1381         /*
1382          * In caching mode, changes of pages from non-present to present require
1383          * flush. However, device IOTLB doesn't need to be flushed in this case.
1384          */
1385         if (!cap_caching_mode(iommu->cap) || !map)
1386                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1387 }
1388
1389 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1390 {
1391         u32 pmen;
1392         unsigned long flags;
1393
1394         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1395         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1396         pmen &= ~DMA_PMEN_EPM;
1397         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1398
1399         /* wait for the protected region status bit to clear */
1400         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1401                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1402
1403         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1404 }
1405
1406 static void iommu_enable_translation(struct intel_iommu *iommu)
1407 {
1408         u32 sts;
1409         unsigned long flags;
1410
1411         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1412         iommu->gcmd |= DMA_GCMD_TE;
1413         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1414
1415         /* Make sure hardware complete it */
1416         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1417                       readl, (sts & DMA_GSTS_TES), sts);
1418
1419         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1420 }
1421
1422 static void iommu_disable_translation(struct intel_iommu *iommu)
1423 {
1424         u32 sts;
1425         unsigned long flag;
1426
1427         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1428         iommu->gcmd &= ~DMA_GCMD_TE;
1429         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1430
1431         /* Make sure hardware complete it */
1432         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1433                       readl, (!(sts & DMA_GSTS_TES)), sts);
1434
1435         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1436 }
1437
1438
1439 static int iommu_init_domains(struct intel_iommu *iommu)
1440 {
1441         unsigned long ndomains;
1442         unsigned long nlongs;
1443
1444         ndomains = cap_ndoms(iommu->cap);
1445         pr_debug("%s: Number of Domains supported <%ld>\n",
1446                  iommu->name, ndomains);
1447         nlongs = BITS_TO_LONGS(ndomains);
1448
1449         spin_lock_init(&iommu->lock);
1450
1451         /* TBD: there might be 64K domains,
1452          * consider other allocation for future chip
1453          */
1454         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1455         if (!iommu->domain_ids) {
1456                 pr_err("%s: Allocating domain id array failed\n",
1457                        iommu->name);
1458                 return -ENOMEM;
1459         }
1460         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1461                         GFP_KERNEL);
1462         if (!iommu->domains) {
1463                 pr_err("%s: Allocating domain array failed\n",
1464                        iommu->name);
1465                 kfree(iommu->domain_ids);
1466                 iommu->domain_ids = NULL;
1467                 return -ENOMEM;
1468         }
1469
1470         /*
1471          * if Caching mode is set, then invalid translations are tagged
1472          * with domainid 0. Hence we need to pre-allocate it.
1473          */
1474         if (cap_caching_mode(iommu->cap))
1475                 set_bit(0, iommu->domain_ids);
1476         return 0;
1477 }
1478
1479 static void disable_dmar_iommu(struct intel_iommu *iommu)
1480 {
1481         struct dmar_domain *domain;
1482         int i;
1483
1484         if ((iommu->domains) && (iommu->domain_ids)) {
1485                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1486                         /*
1487                          * Domain id 0 is reserved for invalid translation
1488                          * if hardware supports caching mode.
1489                          */
1490                         if (cap_caching_mode(iommu->cap) && i == 0)
1491                                 continue;
1492
1493                         domain = iommu->domains[i];
1494                         clear_bit(i, iommu->domain_ids);
1495                         if (domain_detach_iommu(domain, iommu) == 0 &&
1496                             !domain_type_is_vm(domain))
1497                                 domain_exit(domain);
1498                 }
1499         }
1500
1501         if (iommu->gcmd & DMA_GCMD_TE)
1502                 iommu_disable_translation(iommu);
1503 }
1504
1505 static void free_dmar_iommu(struct intel_iommu *iommu)
1506 {
1507         if ((iommu->domains) && (iommu->domain_ids)) {
1508                 kfree(iommu->domains);
1509                 kfree(iommu->domain_ids);
1510                 iommu->domains = NULL;
1511                 iommu->domain_ids = NULL;
1512         }
1513
1514         g_iommus[iommu->seq_id] = NULL;
1515
1516         /* free context mapping */
1517         free_context_table(iommu);
1518 }
1519
1520 static struct dmar_domain *alloc_domain(int flags)
1521 {
1522         /* domain id for virtual machine, it won't be set in context */
1523         static atomic_t vm_domid = ATOMIC_INIT(0);
1524         struct dmar_domain *domain;
1525
1526         domain = alloc_domain_mem();
1527         if (!domain)
1528                 return NULL;
1529
1530         memset(domain, 0, sizeof(*domain));
1531         domain->nid = -1;
1532         domain->flags = flags;
1533         spin_lock_init(&domain->iommu_lock);
1534         INIT_LIST_HEAD(&domain->devices);
1535         if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1536                 domain->id = atomic_inc_return(&vm_domid);
1537
1538         return domain;
1539 }
1540
1541 static int __iommu_attach_domain(struct dmar_domain *domain,
1542                                  struct intel_iommu *iommu)
1543 {
1544         int num;
1545         unsigned long ndomains;
1546
1547         ndomains = cap_ndoms(iommu->cap);
1548         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1549         if (num < ndomains) {
1550                 set_bit(num, iommu->domain_ids);
1551                 iommu->domains[num] = domain;
1552         } else {
1553                 num = -ENOSPC;
1554         }
1555
1556         return num;
1557 }
1558
1559 static int iommu_attach_domain(struct dmar_domain *domain,
1560                                struct intel_iommu *iommu)
1561 {
1562         int num;
1563         unsigned long flags;
1564
1565         spin_lock_irqsave(&iommu->lock, flags);
1566         num = __iommu_attach_domain(domain, iommu);
1567         spin_unlock_irqrestore(&iommu->lock, flags);
1568         if (num < 0)
1569                 pr_err("%s: No free domain ids\n", iommu->name);
1570
1571         return num;
1572 }
1573
1574 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1575                                   struct intel_iommu *iommu)
1576 {
1577         int num;
1578         unsigned long ndomains;
1579
1580         ndomains = cap_ndoms(iommu->cap);
1581         for_each_set_bit(num, iommu->domain_ids, ndomains)
1582                 if (iommu->domains[num] == domain)
1583                         return num;
1584
1585         return __iommu_attach_domain(domain, iommu);
1586 }
1587
1588 static void iommu_detach_domain(struct dmar_domain *domain,
1589                                 struct intel_iommu *iommu)
1590 {
1591         unsigned long flags;
1592         int num, ndomains;
1593
1594         spin_lock_irqsave(&iommu->lock, flags);
1595         if (domain_type_is_vm_or_si(domain)) {
1596                 ndomains = cap_ndoms(iommu->cap);
1597                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1598                         if (iommu->domains[num] == domain) {
1599                                 clear_bit(num, iommu->domain_ids);
1600                                 iommu->domains[num] = NULL;
1601                                 break;
1602                         }
1603                 }
1604         } else {
1605                 clear_bit(domain->id, iommu->domain_ids);
1606                 iommu->domains[domain->id] = NULL;
1607         }
1608         spin_unlock_irqrestore(&iommu->lock, flags);
1609 }
1610
1611 static void domain_attach_iommu(struct dmar_domain *domain,
1612                                struct intel_iommu *iommu)
1613 {
1614         unsigned long flags;
1615
1616         spin_lock_irqsave(&domain->iommu_lock, flags);
1617         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1618                 domain->iommu_count++;
1619                 if (domain->iommu_count == 1)
1620                         domain->nid = iommu->node;
1621                 domain_update_iommu_cap(domain);
1622         }
1623         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1624 }
1625
1626 static int domain_detach_iommu(struct dmar_domain *domain,
1627                                struct intel_iommu *iommu)
1628 {
1629         unsigned long flags;
1630         int count = INT_MAX;
1631
1632         spin_lock_irqsave(&domain->iommu_lock, flags);
1633         if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1634                 count = --domain->iommu_count;
1635                 domain_update_iommu_cap(domain);
1636         }
1637         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1638
1639         return count;
1640 }
1641
1642 static struct iova_domain reserved_iova_list;
1643 static struct lock_class_key reserved_rbtree_key;
1644
1645 static int dmar_init_reserved_ranges(void)
1646 {
1647         struct pci_dev *pdev = NULL;
1648         struct iova *iova;
1649         int i;
1650
1651         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1652                         DMA_32BIT_PFN);
1653
1654         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1655                 &reserved_rbtree_key);
1656
1657         /* IOAPIC ranges shouldn't be accessed by DMA */
1658         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1659                 IOVA_PFN(IOAPIC_RANGE_END));
1660         if (!iova) {
1661                 pr_err("Reserve IOAPIC range failed\n");
1662                 return -ENODEV;
1663         }
1664
1665         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1666         for_each_pci_dev(pdev) {
1667                 struct resource *r;
1668
1669                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1670                         r = &pdev->resource[i];
1671                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1672                                 continue;
1673                         iova = reserve_iova(&reserved_iova_list,
1674                                             IOVA_PFN(r->start),
1675                                             IOVA_PFN(r->end));
1676                         if (!iova) {
1677                                 pr_err("Reserve iova failed\n");
1678                                 return -ENODEV;
1679                         }
1680                 }
1681         }
1682         return 0;
1683 }
1684
1685 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1686 {
1687         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1688 }
1689
1690 static inline int guestwidth_to_adjustwidth(int gaw)
1691 {
1692         int agaw;
1693         int r = (gaw - 12) % 9;
1694
1695         if (r == 0)
1696                 agaw = gaw;
1697         else
1698                 agaw = gaw + 9 - r;
1699         if (agaw > 64)
1700                 agaw = 64;
1701         return agaw;
1702 }
1703
1704 static int domain_init(struct dmar_domain *domain, int guest_width)
1705 {
1706         struct intel_iommu *iommu;
1707         int adjust_width, agaw;
1708         unsigned long sagaw;
1709
1710         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1711                         DMA_32BIT_PFN);
1712         domain_reserve_special_ranges(domain);
1713
1714         /* calculate AGAW */
1715         iommu = domain_get_iommu(domain);
1716         if (guest_width > cap_mgaw(iommu->cap))
1717                 guest_width = cap_mgaw(iommu->cap);
1718         domain->gaw = guest_width;
1719         adjust_width = guestwidth_to_adjustwidth(guest_width);
1720         agaw = width_to_agaw(adjust_width);
1721         sagaw = cap_sagaw(iommu->cap);
1722         if (!test_bit(agaw, &sagaw)) {
1723                 /* hardware doesn't support it, choose a bigger one */
1724                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1725                 agaw = find_next_bit(&sagaw, 5, agaw);
1726                 if (agaw >= 5)
1727                         return -ENODEV;
1728         }
1729         domain->agaw = agaw;
1730
1731         if (ecap_coherent(iommu->ecap))
1732                 domain->iommu_coherency = 1;
1733         else
1734                 domain->iommu_coherency = 0;
1735
1736         if (ecap_sc_support(iommu->ecap))
1737                 domain->iommu_snooping = 1;
1738         else
1739                 domain->iommu_snooping = 0;
1740
1741         if (intel_iommu_superpage)
1742                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1743         else
1744                 domain->iommu_superpage = 0;
1745
1746         domain->nid = iommu->node;
1747
1748         /* always allocate the top pgd */
1749         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1750         if (!domain->pgd)
1751                 return -ENOMEM;
1752         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1753         return 0;
1754 }
1755
1756 static void domain_exit(struct dmar_domain *domain)
1757 {
1758         struct page *freelist = NULL;
1759         int i;
1760
1761         /* Domain 0 is reserved, so dont process it */
1762         if (!domain)
1763                 return;
1764
1765         /* Flush any lazy unmaps that may reference this domain */
1766         if (!intel_iommu_strict)
1767                 flush_unmaps_timeout(0);
1768
1769         /* remove associated devices */
1770         domain_remove_dev_info(domain);
1771
1772         /* destroy iovas */
1773         put_iova_domain(&domain->iovad);
1774
1775         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1776
1777         /* clear attached or cached domains */
1778         rcu_read_lock();
1779         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1780                 iommu_detach_domain(domain, g_iommus[i]);
1781         rcu_read_unlock();
1782
1783         dma_free_pagelist(freelist);
1784
1785         free_domain_mem(domain);
1786 }
1787
1788 static int domain_context_mapping_one(struct dmar_domain *domain,
1789                                       struct intel_iommu *iommu,
1790                                       u8 bus, u8 devfn, int translation)
1791 {
1792         struct context_entry *context;
1793         unsigned long flags;
1794         struct dma_pte *pgd;
1795         int id;
1796         int agaw;
1797         struct device_domain_info *info = NULL;
1798
1799         pr_debug("Set context mapping for %02x:%02x.%d\n",
1800                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1801
1802         BUG_ON(!domain->pgd);
1803         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1804                translation != CONTEXT_TT_MULTI_LEVEL);
1805
1806         spin_lock_irqsave(&iommu->lock, flags);
1807         context = iommu_context_addr(iommu, bus, devfn, 1);
1808         spin_unlock_irqrestore(&iommu->lock, flags);
1809         if (!context)
1810                 return -ENOMEM;
1811         spin_lock_irqsave(&iommu->lock, flags);
1812         if (context_present(context)) {
1813                 spin_unlock_irqrestore(&iommu->lock, flags);
1814                 return 0;
1815         }
1816
1817         id = domain->id;
1818         pgd = domain->pgd;
1819
1820         if (domain_type_is_vm_or_si(domain)) {
1821                 if (domain_type_is_vm(domain)) {
1822                         id = iommu_attach_vm_domain(domain, iommu);
1823                         if (id < 0) {
1824                                 spin_unlock_irqrestore(&iommu->lock, flags);
1825                                 pr_err("%s: No free domain ids\n", iommu->name);
1826                                 return -EFAULT;
1827                         }
1828                 }
1829
1830                 /* Skip top levels of page tables for
1831                  * iommu which has less agaw than default.
1832                  * Unnecessary for PT mode.
1833                  */
1834                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1835                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1836                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1837                                 if (!dma_pte_present(pgd)) {
1838                                         spin_unlock_irqrestore(&iommu->lock, flags);
1839                                         return -ENOMEM;
1840                                 }
1841                         }
1842                 }
1843         }
1844
1845         context_set_domain_id(context, id);
1846
1847         if (translation != CONTEXT_TT_PASS_THROUGH) {
1848                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1849                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1850                                      CONTEXT_TT_MULTI_LEVEL;
1851         }
1852         /*
1853          * In pass through mode, AW must be programmed to indicate the largest
1854          * AGAW value supported by hardware. And ASR is ignored by hardware.
1855          */
1856         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1857                 context_set_address_width(context, iommu->msagaw);
1858         else {
1859                 context_set_address_root(context, virt_to_phys(pgd));
1860                 context_set_address_width(context, iommu->agaw);
1861         }
1862
1863         context_set_translation_type(context, translation);
1864         context_set_fault_enable(context);
1865         context_set_present(context);
1866         domain_flush_cache(domain, context, sizeof(*context));
1867
1868         /*
1869          * It's a non-present to present mapping. If hardware doesn't cache
1870          * non-present entry we only need to flush the write-buffer. If the
1871          * _does_ cache non-present entries, then it does so in the special
1872          * domain #0, which we have to flush:
1873          */
1874         if (cap_caching_mode(iommu->cap)) {
1875                 iommu->flush.flush_context(iommu, 0,
1876                                            (((u16)bus) << 8) | devfn,
1877                                            DMA_CCMD_MASK_NOBIT,
1878                                            DMA_CCMD_DEVICE_INVL);
1879                 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1880         } else {
1881                 iommu_flush_write_buffer(iommu);
1882         }
1883         iommu_enable_dev_iotlb(info);
1884         spin_unlock_irqrestore(&iommu->lock, flags);
1885
1886         domain_attach_iommu(domain, iommu);
1887
1888         return 0;
1889 }
1890
1891 struct domain_context_mapping_data {
1892         struct dmar_domain *domain;
1893         struct intel_iommu *iommu;
1894         int translation;
1895 };
1896
1897 static int domain_context_mapping_cb(struct pci_dev *pdev,
1898                                      u16 alias, void *opaque)
1899 {
1900         struct domain_context_mapping_data *data = opaque;
1901
1902         return domain_context_mapping_one(data->domain, data->iommu,
1903                                           PCI_BUS_NUM(alias), alias & 0xff,
1904                                           data->translation);
1905 }
1906
1907 static int
1908 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1909                        int translation)
1910 {
1911         struct intel_iommu *iommu;
1912         u8 bus, devfn;
1913         struct domain_context_mapping_data data;
1914
1915         iommu = device_to_iommu(dev, &bus, &devfn);
1916         if (!iommu)
1917                 return -ENODEV;
1918
1919         if (!dev_is_pci(dev))
1920                 return domain_context_mapping_one(domain, iommu, bus, devfn,
1921                                                   translation);
1922
1923         data.domain = domain;
1924         data.iommu = iommu;
1925         data.translation = translation;
1926
1927         return pci_for_each_dma_alias(to_pci_dev(dev),
1928                                       &domain_context_mapping_cb, &data);
1929 }
1930
1931 static int domain_context_mapped_cb(struct pci_dev *pdev,
1932                                     u16 alias, void *opaque)
1933 {
1934         struct intel_iommu *iommu = opaque;
1935
1936         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
1937 }
1938
1939 static int domain_context_mapped(struct device *dev)
1940 {
1941         struct intel_iommu *iommu;
1942         u8 bus, devfn;
1943
1944         iommu = device_to_iommu(dev, &bus, &devfn);
1945         if (!iommu)
1946                 return -ENODEV;
1947
1948         if (!dev_is_pci(dev))
1949                 return device_context_mapped(iommu, bus, devfn);
1950
1951         return !pci_for_each_dma_alias(to_pci_dev(dev),
1952                                        domain_context_mapped_cb, iommu);
1953 }
1954
1955 /* Returns a number of VTD pages, but aligned to MM page size */
1956 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1957                                             size_t size)
1958 {
1959         host_addr &= ~PAGE_MASK;
1960         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1961 }
1962
1963 /* Return largest possible superpage level for a given mapping */
1964 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1965                                           unsigned long iov_pfn,
1966                                           unsigned long phy_pfn,
1967                                           unsigned long pages)
1968 {
1969         int support, level = 1;
1970         unsigned long pfnmerge;
1971
1972         support = domain->iommu_superpage;
1973
1974         /* To use a large page, the virtual *and* physical addresses
1975            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1976            of them will mean we have to use smaller pages. So just
1977            merge them and check both at once. */
1978         pfnmerge = iov_pfn | phy_pfn;
1979
1980         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1981                 pages >>= VTD_STRIDE_SHIFT;
1982                 if (!pages)
1983                         break;
1984                 pfnmerge >>= VTD_STRIDE_SHIFT;
1985                 level++;
1986                 support--;
1987         }
1988         return level;
1989 }
1990
1991 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1992                             struct scatterlist *sg, unsigned long phys_pfn,
1993                             unsigned long nr_pages, int prot)
1994 {
1995         struct dma_pte *first_pte = NULL, *pte = NULL;
1996         phys_addr_t uninitialized_var(pteval);
1997         unsigned long sg_res = 0;
1998         unsigned int largepage_lvl = 0;
1999         unsigned long lvl_pages = 0;
2000
2001         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2002
2003         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2004                 return -EINVAL;
2005
2006         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2007
2008         if (!sg) {
2009                 sg_res = nr_pages;
2010                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2011         }
2012
2013         while (nr_pages > 0) {
2014                 uint64_t tmp;
2015
2016                 if (!sg_res) {
2017                         sg_res = aligned_nrpages(sg->offset, sg->length);
2018                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2019                         sg->dma_length = sg->length;
2020                         pteval = page_to_phys(sg_page(sg)) | prot;
2021                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2022                 }
2023
2024                 if (!pte) {
2025                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2026
2027                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2028                         if (!pte)
2029                                 return -ENOMEM;
2030                         /* It is large page*/
2031                         if (largepage_lvl > 1) {
2032                                 pteval |= DMA_PTE_LARGE_PAGE;
2033                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2034                                 /*
2035                                  * Ensure that old small page tables are
2036                                  * removed to make room for superpage,
2037                                  * if they exist.
2038                                  */
2039                                 dma_pte_free_pagetable(domain, iov_pfn,
2040                                                        iov_pfn + lvl_pages - 1);
2041                         } else {
2042                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2043                         }
2044
2045                 }
2046                 /* We don't need lock here, nobody else
2047                  * touches the iova range
2048                  */
2049                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2050                 if (tmp) {
2051                         static int dumps = 5;
2052                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2053                                 iov_pfn, tmp, (unsigned long long)pteval);
2054                         if (dumps) {
2055                                 dumps--;
2056                                 debug_dma_dump_mappings(NULL);
2057                         }
2058                         WARN_ON(1);
2059                 }
2060
2061                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2062
2063                 BUG_ON(nr_pages < lvl_pages);
2064                 BUG_ON(sg_res < lvl_pages);
2065
2066                 nr_pages -= lvl_pages;
2067                 iov_pfn += lvl_pages;
2068                 phys_pfn += lvl_pages;
2069                 pteval += lvl_pages * VTD_PAGE_SIZE;
2070                 sg_res -= lvl_pages;
2071
2072                 /* If the next PTE would be the first in a new page, then we
2073                    need to flush the cache on the entries we've just written.
2074                    And then we'll need to recalculate 'pte', so clear it and
2075                    let it get set again in the if (!pte) block above.
2076
2077                    If we're done (!nr_pages) we need to flush the cache too.
2078
2079                    Also if we've been setting superpages, we may need to
2080                    recalculate 'pte' and switch back to smaller pages for the
2081                    end of the mapping, if the trailing size is not enough to
2082                    use another superpage (i.e. sg_res < lvl_pages). */
2083                 pte++;
2084                 if (!nr_pages || first_pte_in_page(pte) ||
2085                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2086                         domain_flush_cache(domain, first_pte,
2087                                            (void *)pte - (void *)first_pte);
2088                         pte = NULL;
2089                 }
2090
2091                 if (!sg_res && nr_pages)
2092                         sg = sg_next(sg);
2093         }
2094         return 0;
2095 }
2096
2097 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2098                                     struct scatterlist *sg, unsigned long nr_pages,
2099                                     int prot)
2100 {
2101         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2102 }
2103
2104 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2105                                      unsigned long phys_pfn, unsigned long nr_pages,
2106                                      int prot)
2107 {
2108         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2109 }
2110
2111 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2112 {
2113         if (!iommu)
2114                 return;
2115
2116         clear_context_table(iommu, bus, devfn);
2117         iommu->flush.flush_context(iommu, 0, 0, 0,
2118                                            DMA_CCMD_GLOBAL_INVL);
2119         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2120 }
2121
2122 static inline void unlink_domain_info(struct device_domain_info *info)
2123 {
2124         assert_spin_locked(&device_domain_lock);
2125         list_del(&info->link);
2126         list_del(&info->global);
2127         if (info->dev)
2128                 info->dev->archdata.iommu = NULL;
2129 }
2130
2131 static void domain_remove_dev_info(struct dmar_domain *domain)
2132 {
2133         struct device_domain_info *info, *tmp;
2134         unsigned long flags;
2135
2136         spin_lock_irqsave(&device_domain_lock, flags);
2137         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2138                 unlink_domain_info(info);
2139                 spin_unlock_irqrestore(&device_domain_lock, flags);
2140
2141                 iommu_disable_dev_iotlb(info);
2142                 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2143
2144                 if (domain_type_is_vm(domain)) {
2145                         iommu_detach_dependent_devices(info->iommu, info->dev);
2146                         domain_detach_iommu(domain, info->iommu);
2147                 }
2148
2149                 free_devinfo_mem(info);
2150                 spin_lock_irqsave(&device_domain_lock, flags);
2151         }
2152         spin_unlock_irqrestore(&device_domain_lock, flags);
2153 }
2154
2155 /*
2156  * find_domain
2157  * Note: we use struct device->archdata.iommu stores the info
2158  */
2159 static struct dmar_domain *find_domain(struct device *dev)
2160 {
2161         struct device_domain_info *info;
2162
2163         /* No lock here, assumes no domain exit in normal case */
2164         info = dev->archdata.iommu;
2165         if (info)
2166                 return info->domain;
2167         return NULL;
2168 }
2169
2170 static inline struct device_domain_info *
2171 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2172 {
2173         struct device_domain_info *info;
2174
2175         list_for_each_entry(info, &device_domain_list, global)
2176                 if (info->iommu->segment == segment && info->bus == bus &&
2177                     info->devfn == devfn)
2178                         return info;
2179
2180         return NULL;
2181 }
2182
2183 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2184                                                 int bus, int devfn,
2185                                                 struct device *dev,
2186                                                 struct dmar_domain *domain)
2187 {
2188         struct dmar_domain *found = NULL;
2189         struct device_domain_info *info;
2190         unsigned long flags;
2191
2192         info = alloc_devinfo_mem();
2193         if (!info)
2194                 return NULL;
2195
2196         info->bus = bus;
2197         info->devfn = devfn;
2198         info->dev = dev;
2199         info->domain = domain;
2200         info->iommu = iommu;
2201
2202         spin_lock_irqsave(&device_domain_lock, flags);
2203         if (dev)
2204                 found = find_domain(dev);
2205         else {
2206                 struct device_domain_info *info2;
2207                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2208                 if (info2)
2209                         found = info2->domain;
2210         }
2211         if (found) {
2212                 spin_unlock_irqrestore(&device_domain_lock, flags);
2213                 free_devinfo_mem(info);
2214                 /* Caller must free the original domain */
2215                 return found;
2216         }
2217
2218         list_add(&info->link, &domain->devices);
2219         list_add(&info->global, &device_domain_list);
2220         if (dev)
2221                 dev->archdata.iommu = info;
2222         spin_unlock_irqrestore(&device_domain_lock, flags);
2223
2224         return domain;
2225 }
2226
2227 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2228 {
2229         *(u16 *)opaque = alias;
2230         return 0;
2231 }
2232
2233 /* domain is initialized */
2234 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2235 {
2236         struct dmar_domain *domain, *tmp;
2237         struct intel_iommu *iommu;
2238         struct device_domain_info *info;
2239         u16 dma_alias;
2240         unsigned long flags;
2241         u8 bus, devfn;
2242
2243         domain = find_domain(dev);
2244         if (domain)
2245                 return domain;
2246
2247         iommu = device_to_iommu(dev, &bus, &devfn);
2248         if (!iommu)
2249                 return NULL;
2250
2251         if (dev_is_pci(dev)) {
2252                 struct pci_dev *pdev = to_pci_dev(dev);
2253
2254                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2255
2256                 spin_lock_irqsave(&device_domain_lock, flags);
2257                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2258                                                       PCI_BUS_NUM(dma_alias),
2259                                                       dma_alias & 0xff);
2260                 if (info) {
2261                         iommu = info->iommu;
2262                         domain = info->domain;
2263                 }
2264                 spin_unlock_irqrestore(&device_domain_lock, flags);
2265
2266                 /* DMA alias already has a domain, uses it */
2267                 if (info)
2268                         goto found_domain;
2269         }
2270
2271         /* Allocate and initialize new domain for the device */
2272         domain = alloc_domain(0);
2273         if (!domain)
2274                 return NULL;
2275         domain->id = iommu_attach_domain(domain, iommu);
2276         if (domain->id < 0) {
2277                 free_domain_mem(domain);
2278                 return NULL;
2279         }
2280         domain_attach_iommu(domain, iommu);
2281         if (domain_init(domain, gaw)) {
2282                 domain_exit(domain);
2283                 return NULL;
2284         }
2285
2286         /* register PCI DMA alias device */
2287         if (dev_is_pci(dev)) {
2288                 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2289                                            dma_alias & 0xff, NULL, domain);
2290
2291                 if (!tmp || tmp != domain) {
2292                         domain_exit(domain);
2293                         domain = tmp;
2294                 }
2295
2296                 if (!domain)
2297                         return NULL;
2298         }
2299
2300 found_domain:
2301         tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2302
2303         if (!tmp || tmp != domain) {
2304                 domain_exit(domain);
2305                 domain = tmp;
2306         }
2307
2308         return domain;
2309 }
2310
2311 static int iommu_identity_mapping;
2312 #define IDENTMAP_ALL            1
2313 #define IDENTMAP_GFX            2
2314 #define IDENTMAP_AZALIA         4
2315
2316 static int iommu_domain_identity_map(struct dmar_domain *domain,
2317                                      unsigned long long start,
2318                                      unsigned long long end)
2319 {
2320         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2321         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2322
2323         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2324                           dma_to_mm_pfn(last_vpfn))) {
2325                 pr_err("Reserving iova failed\n");
2326                 return -ENOMEM;
2327         }
2328
2329         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2330                  start, end, domain->id);
2331         /*
2332          * RMRR range might have overlap with physical memory range,
2333          * clear it first
2334          */
2335         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2336
2337         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2338                                   last_vpfn - first_vpfn + 1,
2339                                   DMA_PTE_READ|DMA_PTE_WRITE);
2340 }
2341
2342 static int iommu_prepare_identity_map(struct device *dev,
2343                                       unsigned long long start,
2344                                       unsigned long long end)
2345 {
2346         struct dmar_domain *domain;
2347         int ret;
2348
2349         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2350         if (!domain)
2351                 return -ENOMEM;
2352
2353         /* For _hardware_ passthrough, don't bother. But for software
2354            passthrough, we do it anyway -- it may indicate a memory
2355            range which is reserved in E820, so which didn't get set
2356            up to start with in si_domain */
2357         if (domain == si_domain && hw_pass_through) {
2358                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2359                         dev_name(dev), start, end);
2360                 return 0;
2361         }
2362
2363         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2364                 dev_name(dev), start, end);
2365
2366         if (end < start) {
2367                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2368                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2369                         dmi_get_system_info(DMI_BIOS_VENDOR),
2370                         dmi_get_system_info(DMI_BIOS_VERSION),
2371                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2372                 ret = -EIO;
2373                 goto error;
2374         }
2375
2376         if (end >> agaw_to_width(domain->agaw)) {
2377                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2378                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2379                      agaw_to_width(domain->agaw),
2380                      dmi_get_system_info(DMI_BIOS_VENDOR),
2381                      dmi_get_system_info(DMI_BIOS_VERSION),
2382                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2383                 ret = -EIO;
2384                 goto error;
2385         }
2386
2387         ret = iommu_domain_identity_map(domain, start, end);
2388         if (ret)
2389                 goto error;
2390
2391         /* context entry init */
2392         ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2393         if (ret)
2394                 goto error;
2395
2396         return 0;
2397
2398  error:
2399         domain_exit(domain);
2400         return ret;
2401 }
2402
2403 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2404                                          struct device *dev)
2405 {
2406         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2407                 return 0;
2408         return iommu_prepare_identity_map(dev, rmrr->base_address,
2409                                           rmrr->end_address);
2410 }
2411
2412 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2413 static inline void iommu_prepare_isa(void)
2414 {
2415         struct pci_dev *pdev;
2416         int ret;
2417
2418         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2419         if (!pdev)
2420                 return;
2421
2422         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2423         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2424
2425         if (ret)
2426                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2427
2428         pci_dev_put(pdev);
2429 }
2430 #else
2431 static inline void iommu_prepare_isa(void)
2432 {
2433         return;
2434 }
2435 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2436
2437 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2438
2439 static int __init si_domain_init(int hw)
2440 {
2441         struct dmar_drhd_unit *drhd;
2442         struct intel_iommu *iommu;
2443         int nid, ret = 0;
2444         bool first = true;
2445
2446         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2447         if (!si_domain)
2448                 return -EFAULT;
2449
2450         for_each_active_iommu(iommu, drhd) {
2451                 ret = iommu_attach_domain(si_domain, iommu);
2452                 if (ret < 0) {
2453                         domain_exit(si_domain);
2454                         return -EFAULT;
2455                 } else if (first) {
2456                         si_domain->id = ret;
2457                         first = false;
2458                 } else if (si_domain->id != ret) {
2459                         domain_exit(si_domain);
2460                         return -EFAULT;
2461                 }
2462                 domain_attach_iommu(si_domain, iommu);
2463         }
2464
2465         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2466                 domain_exit(si_domain);
2467                 return -EFAULT;
2468         }
2469
2470         pr_debug("Identity mapping domain is domain %d\n",
2471                  si_domain->id);
2472
2473         if (hw)
2474                 return 0;
2475
2476         for_each_online_node(nid) {
2477                 unsigned long start_pfn, end_pfn;
2478                 int i;
2479
2480                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2481                         ret = iommu_domain_identity_map(si_domain,
2482                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2483                         if (ret)
2484                                 return ret;
2485                 }
2486         }
2487
2488         return 0;
2489 }
2490
2491 static int identity_mapping(struct device *dev)
2492 {
2493         struct device_domain_info *info;
2494
2495         if (likely(!iommu_identity_mapping))
2496                 return 0;
2497
2498         info = dev->archdata.iommu;
2499         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2500                 return (info->domain == si_domain);
2501
2502         return 0;
2503 }
2504
2505 static int domain_add_dev_info(struct dmar_domain *domain,
2506                                struct device *dev, int translation)
2507 {
2508         struct dmar_domain *ndomain;
2509         struct intel_iommu *iommu;
2510         u8 bus, devfn;
2511         int ret;
2512
2513         iommu = device_to_iommu(dev, &bus, &devfn);
2514         if (!iommu)
2515                 return -ENODEV;
2516
2517         ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2518         if (ndomain != domain)
2519                 return -EBUSY;
2520
2521         ret = domain_context_mapping(domain, dev, translation);
2522         if (ret) {
2523                 domain_remove_one_dev_info(domain, dev);
2524                 return ret;
2525         }
2526
2527         return 0;
2528 }
2529
2530 static bool device_has_rmrr(struct device *dev)
2531 {
2532         struct dmar_rmrr_unit *rmrr;
2533         struct device *tmp;
2534         int i;
2535
2536         rcu_read_lock();
2537         for_each_rmrr_units(rmrr) {
2538                 /*
2539                  * Return TRUE if this RMRR contains the device that
2540                  * is passed in.
2541                  */
2542                 for_each_active_dev_scope(rmrr->devices,
2543                                           rmrr->devices_cnt, i, tmp)
2544                         if (tmp == dev) {
2545                                 rcu_read_unlock();
2546                                 return true;
2547                         }
2548         }
2549         rcu_read_unlock();
2550         return false;
2551 }
2552
2553 /*
2554  * There are a couple cases where we need to restrict the functionality of
2555  * devices associated with RMRRs.  The first is when evaluating a device for
2556  * identity mapping because problems exist when devices are moved in and out
2557  * of domains and their respective RMRR information is lost.  This means that
2558  * a device with associated RMRRs will never be in a "passthrough" domain.
2559  * The second is use of the device through the IOMMU API.  This interface
2560  * expects to have full control of the IOVA space for the device.  We cannot
2561  * satisfy both the requirement that RMRR access is maintained and have an
2562  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2563  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2564  * We therefore prevent devices associated with an RMRR from participating in
2565  * the IOMMU API, which eliminates them from device assignment.
2566  *
2567  * In both cases we assume that PCI USB devices with RMRRs have them largely
2568  * for historical reasons and that the RMRR space is not actively used post
2569  * boot.  This exclusion may change if vendors begin to abuse it.
2570  *
2571  * The same exception is made for graphics devices, with the requirement that
2572  * any use of the RMRR regions will be torn down before assigning the device
2573  * to a guest.
2574  */
2575 static bool device_is_rmrr_locked(struct device *dev)
2576 {
2577         if (!device_has_rmrr(dev))
2578                 return false;
2579
2580         if (dev_is_pci(dev)) {
2581                 struct pci_dev *pdev = to_pci_dev(dev);
2582
2583                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2584                         return false;
2585         }
2586
2587         return true;
2588 }
2589
2590 static int iommu_should_identity_map(struct device *dev, int startup)
2591 {
2592
2593         if (dev_is_pci(dev)) {
2594                 struct pci_dev *pdev = to_pci_dev(dev);
2595
2596                 if (device_is_rmrr_locked(dev))
2597                         return 0;
2598
2599                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2600                         return 1;
2601
2602                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2603                         return 1;
2604
2605                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2606                         return 0;
2607
2608                 /*
2609                  * We want to start off with all devices in the 1:1 domain, and
2610                  * take them out later if we find they can't access all of memory.
2611                  *
2612                  * However, we can't do this for PCI devices behind bridges,
2613                  * because all PCI devices behind the same bridge will end up
2614                  * with the same source-id on their transactions.
2615                  *
2616                  * Practically speaking, we can't change things around for these
2617                  * devices at run-time, because we can't be sure there'll be no
2618                  * DMA transactions in flight for any of their siblings.
2619                  *
2620                  * So PCI devices (unless they're on the root bus) as well as
2621                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2622                  * the 1:1 domain, just in _case_ one of their siblings turns out
2623                  * not to be able to map all of memory.
2624                  */
2625                 if (!pci_is_pcie(pdev)) {
2626                         if (!pci_is_root_bus(pdev->bus))
2627                                 return 0;
2628                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2629                                 return 0;
2630                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2631                         return 0;
2632         } else {
2633                 if (device_has_rmrr(dev))
2634                         return 0;
2635         }
2636
2637         /*
2638          * At boot time, we don't yet know if devices will be 64-bit capable.
2639          * Assume that they will — if they turn out not to be, then we can
2640          * take them out of the 1:1 domain later.
2641          */
2642         if (!startup) {
2643                 /*
2644                  * If the device's dma_mask is less than the system's memory
2645                  * size then this is not a candidate for identity mapping.
2646                  */
2647                 u64 dma_mask = *dev->dma_mask;
2648
2649                 if (dev->coherent_dma_mask &&
2650                     dev->coherent_dma_mask < dma_mask)
2651                         dma_mask = dev->coherent_dma_mask;
2652
2653                 return dma_mask >= dma_get_required_mask(dev);
2654         }
2655
2656         return 1;
2657 }
2658
2659 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2660 {
2661         int ret;
2662
2663         if (!iommu_should_identity_map(dev, 1))
2664                 return 0;
2665
2666         ret = domain_add_dev_info(si_domain, dev,
2667                                   hw ? CONTEXT_TT_PASS_THROUGH :
2668                                        CONTEXT_TT_MULTI_LEVEL);
2669         if (!ret)
2670                 pr_info("%s identity mapping for device %s\n",
2671                         hw ? "Hardware" : "Software", dev_name(dev));
2672         else if (ret == -ENODEV)
2673                 /* device not associated with an iommu */
2674                 ret = 0;
2675
2676         return ret;
2677 }
2678
2679
2680 static int __init iommu_prepare_static_identity_mapping(int hw)
2681 {
2682         struct pci_dev *pdev = NULL;
2683         struct dmar_drhd_unit *drhd;
2684         struct intel_iommu *iommu;
2685         struct device *dev;
2686         int i;
2687         int ret = 0;
2688
2689         ret = si_domain_init(hw);
2690         if (ret)
2691                 return -EFAULT;
2692
2693         for_each_pci_dev(pdev) {
2694                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2695                 if (ret)
2696                         return ret;
2697         }
2698
2699         for_each_active_iommu(iommu, drhd)
2700                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2701                         struct acpi_device_physical_node *pn;
2702                         struct acpi_device *adev;
2703
2704                         if (dev->bus != &acpi_bus_type)
2705                                 continue;
2706
2707                         adev= to_acpi_device(dev);
2708                         mutex_lock(&adev->physical_node_lock);
2709                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2710                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2711                                 if (ret)
2712                                         break;
2713                         }
2714                         mutex_unlock(&adev->physical_node_lock);
2715                         if (ret)
2716                                 return ret;
2717                 }
2718
2719         return 0;
2720 }
2721
2722 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2723 {
2724         /*
2725          * Start from the sane iommu hardware state.
2726          * If the queued invalidation is already initialized by us
2727          * (for example, while enabling interrupt-remapping) then
2728          * we got the things already rolling from a sane state.
2729          */
2730         if (!iommu->qi) {
2731                 /*
2732                  * Clear any previous faults.
2733                  */
2734                 dmar_fault(-1, iommu);
2735                 /*
2736                  * Disable queued invalidation if supported and already enabled
2737                  * before OS handover.
2738                  */
2739                 dmar_disable_qi(iommu);
2740         }
2741
2742         if (dmar_enable_qi(iommu)) {
2743                 /*
2744                  * Queued Invalidate not enabled, use Register Based Invalidate
2745                  */
2746                 iommu->flush.flush_context = __iommu_flush_context;
2747                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2748                 pr_info("%s: Using Register based invalidation\n",
2749                         iommu->name);
2750         } else {
2751                 iommu->flush.flush_context = qi_flush_context;
2752                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2753                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2754         }
2755 }
2756
2757 static int __init init_dmars(void)
2758 {
2759         struct dmar_drhd_unit *drhd;
2760         struct dmar_rmrr_unit *rmrr;
2761         struct device *dev;
2762         struct intel_iommu *iommu;
2763         int i, ret;
2764
2765         /*
2766          * for each drhd
2767          *    allocate root
2768          *    initialize and program root entry to not present
2769          * endfor
2770          */
2771         for_each_drhd_unit(drhd) {
2772                 /*
2773                  * lock not needed as this is only incremented in the single
2774                  * threaded kernel __init code path all other access are read
2775                  * only
2776                  */
2777                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
2778                         g_num_of_iommus++;
2779                         continue;
2780                 }
2781                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
2782         }
2783
2784         /* Preallocate enough resources for IOMMU hot-addition */
2785         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
2786                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
2787
2788         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2789                         GFP_KERNEL);
2790         if (!g_iommus) {
2791                 pr_err("Allocating global iommu array failed\n");
2792                 ret = -ENOMEM;
2793                 goto error;
2794         }
2795
2796         deferred_flush = kzalloc(g_num_of_iommus *
2797                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2798         if (!deferred_flush) {
2799                 ret = -ENOMEM;
2800                 goto free_g_iommus;
2801         }
2802
2803         for_each_active_iommu(iommu, drhd) {
2804                 g_iommus[iommu->seq_id] = iommu;
2805
2806                 intel_iommu_init_qi(iommu);
2807
2808                 ret = iommu_init_domains(iommu);
2809                 if (ret)
2810                         goto free_iommu;
2811
2812                 /*
2813                  * TBD:
2814                  * we could share the same root & context tables
2815                  * among all IOMMU's. Need to Split it later.
2816                  */
2817                 ret = iommu_alloc_root_entry(iommu);
2818                 if (ret)
2819                         goto free_iommu;
2820                 if (!ecap_pass_through(iommu->ecap))
2821                         hw_pass_through = 0;
2822         }
2823
2824         if (iommu_pass_through)
2825                 iommu_identity_mapping |= IDENTMAP_ALL;
2826
2827 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2828         iommu_identity_mapping |= IDENTMAP_GFX;
2829 #endif
2830
2831         check_tylersburg_isoch();
2832
2833         /*
2834          * If pass through is not set or not enabled, setup context entries for
2835          * identity mappings for rmrr, gfx, and isa and may fall back to static
2836          * identity mapping if iommu_identity_mapping is set.
2837          */
2838         if (iommu_identity_mapping) {
2839                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2840                 if (ret) {
2841                         pr_crit("Failed to setup IOMMU pass-through\n");
2842                         goto free_iommu;
2843                 }
2844         }
2845         /*
2846          * For each rmrr
2847          *   for each dev attached to rmrr
2848          *   do
2849          *     locate drhd for dev, alloc domain for dev
2850          *     allocate free domain
2851          *     allocate page table entries for rmrr
2852          *     if context not allocated for bus
2853          *           allocate and init context
2854          *           set present in root table for this bus
2855          *     init context with domain, translation etc
2856          *    endfor
2857          * endfor
2858          */
2859         pr_info("Setting RMRR:\n");
2860         for_each_rmrr_units(rmrr) {
2861                 /* some BIOS lists non-exist devices in DMAR table. */
2862                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2863                                           i, dev) {
2864                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
2865                         if (ret)
2866                                 pr_err("Mapping reserved region failed\n");
2867                 }
2868         }
2869
2870         iommu_prepare_isa();
2871
2872         /*
2873          * for each drhd
2874          *   enable fault log
2875          *   global invalidate context cache
2876          *   global invalidate iotlb
2877          *   enable translation
2878          */
2879         for_each_iommu(iommu, drhd) {
2880                 if (drhd->ignored) {
2881                         /*
2882                          * we always have to disable PMRs or DMA may fail on
2883                          * this device
2884                          */
2885                         if (force_on)
2886                                 iommu_disable_protect_mem_regions(iommu);
2887                         continue;
2888                 }
2889
2890                 iommu_flush_write_buffer(iommu);
2891
2892                 ret = dmar_set_interrupt(iommu);
2893                 if (ret)
2894                         goto free_iommu;
2895
2896                 iommu_set_root_entry(iommu);
2897
2898                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2899                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2900                 iommu_enable_translation(iommu);
2901                 iommu_disable_protect_mem_regions(iommu);
2902         }
2903
2904         return 0;
2905
2906 free_iommu:
2907         for_each_active_iommu(iommu, drhd) {
2908                 disable_dmar_iommu(iommu);
2909                 free_dmar_iommu(iommu);
2910         }
2911         kfree(deferred_flush);
2912 free_g_iommus:
2913         kfree(g_iommus);
2914 error:
2915         return ret;
2916 }
2917
2918 /* This takes a number of _MM_ pages, not VTD pages */
2919 static struct iova *intel_alloc_iova(struct device *dev,
2920                                      struct dmar_domain *domain,
2921                                      unsigned long nrpages, uint64_t dma_mask)
2922 {
2923         struct iova *iova = NULL;
2924
2925         /* Restrict dma_mask to the width that the iommu can handle */
2926         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2927
2928         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2929                 /*
2930                  * First try to allocate an io virtual address in
2931                  * DMA_BIT_MASK(32) and if that fails then try allocating
2932                  * from higher range
2933                  */
2934                 iova = alloc_iova(&domain->iovad, nrpages,
2935                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2936                 if (iova)
2937                         return iova;
2938         }
2939         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2940         if (unlikely(!iova)) {
2941                 pr_err("Allocating %ld-page iova for %s failed",
2942                        nrpages, dev_name(dev));
2943                 return NULL;
2944         }
2945
2946         return iova;
2947 }
2948
2949 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2950 {
2951         struct dmar_domain *domain;
2952         int ret;
2953
2954         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2955         if (!domain) {
2956                 pr_err("Allocating domain for %s failed\n",
2957                        dev_name(dev));
2958                 return NULL;
2959         }
2960
2961         /* make sure context mapping is ok */
2962         if (unlikely(!domain_context_mapped(dev))) {
2963                 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2964                 if (ret) {
2965                         pr_err("Domain context map for %s failed\n",
2966                                dev_name(dev));
2967                         return NULL;
2968                 }
2969         }
2970
2971         return domain;
2972 }
2973
2974 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2975 {
2976         struct device_domain_info *info;
2977
2978         /* No lock here, assumes no domain exit in normal case */
2979         info = dev->archdata.iommu;
2980         if (likely(info))
2981                 return info->domain;
2982
2983         return __get_valid_domain_for_dev(dev);
2984 }
2985
2986 /* Check if the dev needs to go through non-identity map and unmap process.*/
2987 static int iommu_no_mapping(struct device *dev)
2988 {
2989         int found;
2990
2991         if (iommu_dummy(dev))
2992                 return 1;
2993
2994         if (!iommu_identity_mapping)
2995                 return 0;
2996
2997         found = identity_mapping(dev);
2998         if (found) {
2999                 if (iommu_should_identity_map(dev, 0))
3000                         return 1;
3001                 else {
3002                         /*
3003                          * 32 bit DMA is removed from si_domain and fall back
3004                          * to non-identity mapping.
3005                          */
3006                         domain_remove_one_dev_info(si_domain, dev);
3007                         pr_info("32bit %s uses non-identity mapping\n",
3008                                 dev_name(dev));
3009                         return 0;
3010                 }
3011         } else {
3012                 /*
3013                  * In case of a detached 64 bit DMA device from vm, the device
3014                  * is put into si_domain for identity mapping.
3015                  */
3016                 if (iommu_should_identity_map(dev, 0)) {
3017                         int ret;
3018                         ret = domain_add_dev_info(si_domain, dev,
3019                                                   hw_pass_through ?
3020                                                   CONTEXT_TT_PASS_THROUGH :
3021                                                   CONTEXT_TT_MULTI_LEVEL);
3022                         if (!ret) {
3023                                 pr_info("64bit %s uses identity mapping\n",
3024                                         dev_name(dev));
3025                                 return 1;
3026                         }
3027                 }
3028         }
3029
3030         return 0;
3031 }
3032
3033 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3034                                      size_t size, int dir, u64 dma_mask)
3035 {
3036         struct dmar_domain *domain;
3037         phys_addr_t start_paddr;
3038         struct iova *iova;
3039         int prot = 0;
3040         int ret;
3041         struct intel_iommu *iommu;
3042         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3043
3044         BUG_ON(dir == DMA_NONE);
3045
3046         if (iommu_no_mapping(dev))
3047                 return paddr;
3048
3049         domain = get_valid_domain_for_dev(dev);
3050         if (!domain)
3051                 return 0;
3052
3053         iommu = domain_get_iommu(domain);
3054         size = aligned_nrpages(paddr, size);
3055
3056         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3057         if (!iova)
3058                 goto error;
3059
3060         /*
3061          * Check if DMAR supports zero-length reads on write only
3062          * mappings..
3063          */
3064         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3065                         !cap_zlr(iommu->cap))
3066                 prot |= DMA_PTE_READ;
3067         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3068                 prot |= DMA_PTE_WRITE;
3069         /*
3070          * paddr - (paddr + size) might be partial page, we should map the whole
3071          * page.  Note: if two part of one page are separately mapped, we
3072          * might have two guest_addr mapping to the same host paddr, but this
3073          * is not a big problem
3074          */
3075         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3076                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3077         if (ret)
3078                 goto error;
3079
3080         /* it's a non-present to present mapping. Only flush if caching mode */
3081         if (cap_caching_mode(iommu->cap))
3082                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3083         else
3084                 iommu_flush_write_buffer(iommu);
3085
3086         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3087         start_paddr += paddr & ~PAGE_MASK;
3088         return start_paddr;
3089
3090 error:
3091         if (iova)
3092                 __free_iova(&domain->iovad, iova);
3093         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3094                 dev_name(dev), size, (unsigned long long)paddr, dir);
3095         return 0;
3096 }
3097
3098 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3099                                  unsigned long offset, size_t size,
3100                                  enum dma_data_direction dir,
3101                                  struct dma_attrs *attrs)
3102 {
3103         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3104                                   dir, *dev->dma_mask);
3105 }
3106
3107 static void flush_unmaps(void)
3108 {
3109         int i, j;
3110
3111         timer_on = 0;
3112
3113         /* just flush them all */
3114         for (i = 0; i < g_num_of_iommus; i++) {
3115                 struct intel_iommu *iommu = g_iommus[i];
3116                 if (!iommu)
3117                         continue;
3118
3119                 if (!deferred_flush[i].next)
3120                         continue;
3121
3122                 /* In caching mode, global flushes turn emulation expensive */
3123                 if (!cap_caching_mode(iommu->cap))
3124                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3125                                          DMA_TLB_GLOBAL_FLUSH);
3126                 for (j = 0; j < deferred_flush[i].next; j++) {
3127                         unsigned long mask;
3128                         struct iova *iova = deferred_flush[i].iova[j];
3129                         struct dmar_domain *domain = deferred_flush[i].domain[j];
3130
3131                         /* On real hardware multiple invalidations are expensive */
3132                         if (cap_caching_mode(iommu->cap))
3133                                 iommu_flush_iotlb_psi(iommu, domain->id,
3134                                         iova->pfn_lo, iova_size(iova),
3135                                         !deferred_flush[i].freelist[j], 0);
3136                         else {
3137                                 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3138                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3139                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3140                         }
3141                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3142                         if (deferred_flush[i].freelist[j])
3143                                 dma_free_pagelist(deferred_flush[i].freelist[j]);
3144                 }
3145                 deferred_flush[i].next = 0;
3146         }
3147
3148         list_size = 0;
3149 }
3150
3151 static void flush_unmaps_timeout(unsigned long data)
3152 {
3153         unsigned long flags;
3154
3155         spin_lock_irqsave(&async_umap_flush_lock, flags);
3156         flush_unmaps();
3157         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3158 }
3159
3160 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3161 {
3162         unsigned long flags;
3163         int next, iommu_id;
3164         struct intel_iommu *iommu;
3165
3166         spin_lock_irqsave(&async_umap_flush_lock, flags);
3167         if (list_size == HIGH_WATER_MARK)
3168                 flush_unmaps();
3169
3170         iommu = domain_get_iommu(dom);
3171         iommu_id = iommu->seq_id;
3172
3173         next = deferred_flush[iommu_id].next;
3174         deferred_flush[iommu_id].domain[next] = dom;
3175         deferred_flush[iommu_id].iova[next] = iova;
3176         deferred_flush[iommu_id].freelist[next] = freelist;
3177         deferred_flush[iommu_id].next++;
3178
3179         if (!timer_on) {
3180                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3181                 timer_on = 1;
3182         }
3183         list_size++;
3184         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3185 }
3186
3187 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3188 {
3189         struct dmar_domain *domain;
3190         unsigned long start_pfn, last_pfn;
3191         struct iova *iova;
3192         struct intel_iommu *iommu;
3193         struct page *freelist;
3194
3195         if (iommu_no_mapping(dev))
3196                 return;
3197
3198         domain = find_domain(dev);
3199         BUG_ON(!domain);
3200
3201         iommu = domain_get_iommu(domain);
3202
3203         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3204         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3205                       (unsigned long long)dev_addr))
3206                 return;
3207
3208         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3209         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3210
3211         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3212                  dev_name(dev), start_pfn, last_pfn);
3213
3214         freelist = domain_unmap(domain, start_pfn, last_pfn);
3215
3216         if (intel_iommu_strict) {
3217                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3218                                       last_pfn - start_pfn + 1, !freelist, 0);
3219                 /* free iova */
3220                 __free_iova(&domain->iovad, iova);
3221                 dma_free_pagelist(freelist);
3222         } else {
3223                 add_unmap(domain, iova, freelist);
3224                 /*
3225                  * queue up the release of the unmap to save the 1/6th of the
3226                  * cpu used up by the iotlb flush operation...
3227                  */
3228         }
3229 }
3230
3231 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3232                              size_t size, enum dma_data_direction dir,
3233                              struct dma_attrs *attrs)
3234 {
3235         intel_unmap(dev, dev_addr);
3236 }
3237
3238 static void *intel_alloc_coherent(struct device *dev, size_t size,
3239                                   dma_addr_t *dma_handle, gfp_t flags,
3240                                   struct dma_attrs *attrs)
3241 {
3242         struct page *page = NULL;
3243         int order;
3244
3245         size = PAGE_ALIGN(size);
3246         order = get_order(size);
3247
3248         if (!iommu_no_mapping(dev))
3249                 flags &= ~(GFP_DMA | GFP_DMA32);
3250         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3251                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3252                         flags |= GFP_DMA;
3253                 else
3254                         flags |= GFP_DMA32;
3255         }
3256
3257         if (flags & __GFP_WAIT) {
3258                 unsigned int count = size >> PAGE_SHIFT;
3259
3260                 page = dma_alloc_from_contiguous(dev, count, order);
3261                 if (page && iommu_no_mapping(dev) &&
3262                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3263                         dma_release_from_contiguous(dev, page, count);
3264                         page = NULL;
3265                 }
3266         }
3267
3268         if (!page)
3269                 page = alloc_pages(flags, order);
3270         if (!page)
3271                 return NULL;
3272         memset(page_address(page), 0, size);
3273
3274         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3275                                          DMA_BIDIRECTIONAL,
3276                                          dev->coherent_dma_mask);
3277         if (*dma_handle)
3278                 return page_address(page);
3279         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3280                 __free_pages(page, order);
3281
3282         return NULL;
3283 }
3284
3285 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3286                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3287 {
3288         int order;
3289         struct page *page = virt_to_page(vaddr);
3290
3291         size = PAGE_ALIGN(size);
3292         order = get_order(size);
3293
3294         intel_unmap(dev, dma_handle);
3295         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3296                 __free_pages(page, order);
3297 }
3298
3299 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3300                            int nelems, enum dma_data_direction dir,
3301                            struct dma_attrs *attrs)
3302 {
3303         intel_unmap(dev, sglist[0].dma_address);
3304 }
3305
3306 static int intel_nontranslate_map_sg(struct device *hddev,
3307         struct scatterlist *sglist, int nelems, int dir)
3308 {
3309         int i;
3310         struct scatterlist *sg;
3311
3312         for_each_sg(sglist, sg, nelems, i) {
3313                 BUG_ON(!sg_page(sg));
3314                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3315                 sg->dma_length = sg->length;
3316         }
3317         return nelems;
3318 }
3319
3320 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3321                         enum dma_data_direction dir, struct dma_attrs *attrs)
3322 {
3323         int i;
3324         struct dmar_domain *domain;
3325         size_t size = 0;
3326         int prot = 0;
3327         struct iova *iova = NULL;
3328         int ret;
3329         struct scatterlist *sg;
3330         unsigned long start_vpfn;
3331         struct intel_iommu *iommu;
3332
3333         BUG_ON(dir == DMA_NONE);
3334         if (iommu_no_mapping(dev))
3335                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3336
3337         domain = get_valid_domain_for_dev(dev);
3338         if (!domain)
3339                 return 0;
3340
3341         iommu = domain_get_iommu(domain);
3342
3343         for_each_sg(sglist, sg, nelems, i)
3344                 size += aligned_nrpages(sg->offset, sg->length);
3345
3346         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3347                                 *dev->dma_mask);
3348         if (!iova) {
3349                 sglist->dma_length = 0;
3350                 return 0;
3351         }
3352
3353         /*
3354          * Check if DMAR supports zero-length reads on write only
3355          * mappings..
3356          */
3357         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3358                         !cap_zlr(iommu->cap))
3359                 prot |= DMA_PTE_READ;
3360         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3361                 prot |= DMA_PTE_WRITE;
3362
3363         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3364
3365         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3366         if (unlikely(ret)) {
3367                 dma_pte_free_pagetable(domain, start_vpfn,
3368                                        start_vpfn + size - 1);
3369                 __free_iova(&domain->iovad, iova);
3370                 return 0;
3371         }
3372
3373         /* it's a non-present to present mapping. Only flush if caching mode */
3374         if (cap_caching_mode(iommu->cap))
3375                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3376         else
3377                 iommu_flush_write_buffer(iommu);
3378
3379         return nelems;
3380 }
3381
3382 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3383 {
3384         return !dma_addr;
3385 }
3386
3387 struct dma_map_ops intel_dma_ops = {
3388         .alloc = intel_alloc_coherent,
3389         .free = intel_free_coherent,
3390         .map_sg = intel_map_sg,
3391         .unmap_sg = intel_unmap_sg,
3392         .map_page = intel_map_page,
3393         .unmap_page = intel_unmap_page,
3394         .mapping_error = intel_mapping_error,
3395 };
3396
3397 static inline int iommu_domain_cache_init(void)
3398 {
3399         int ret = 0;
3400
3401         iommu_domain_cache = kmem_cache_create("iommu_domain",
3402                                          sizeof(struct dmar_domain),
3403                                          0,
3404                                          SLAB_HWCACHE_ALIGN,
3405
3406                                          NULL);
3407         if (!iommu_domain_cache) {
3408                 pr_err("Couldn't create iommu_domain cache\n");
3409                 ret = -ENOMEM;
3410         }
3411
3412         return ret;
3413 }
3414
3415 static inline int iommu_devinfo_cache_init(void)
3416 {
3417         int ret = 0;
3418
3419         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3420                                          sizeof(struct device_domain_info),
3421                                          0,
3422                                          SLAB_HWCACHE_ALIGN,
3423                                          NULL);
3424         if (!iommu_devinfo_cache) {
3425                 pr_err("Couldn't create devinfo cache\n");
3426                 ret = -ENOMEM;
3427         }
3428
3429         return ret;
3430 }
3431
3432 static int __init iommu_init_mempool(void)
3433 {
3434         int ret;
3435         ret = iommu_iova_cache_init();
3436         if (ret)
3437                 return ret;
3438
3439         ret = iommu_domain_cache_init();
3440         if (ret)
3441                 goto domain_error;
3442
3443         ret = iommu_devinfo_cache_init();
3444         if (!ret)
3445                 return ret;
3446
3447         kmem_cache_destroy(iommu_domain_cache);
3448 domain_error:
3449         iommu_iova_cache_destroy();
3450
3451         return -ENOMEM;
3452 }
3453
3454 static void __init iommu_exit_mempool(void)
3455 {
3456         kmem_cache_destroy(iommu_devinfo_cache);
3457         kmem_cache_destroy(iommu_domain_cache);
3458         iommu_iova_cache_destroy();
3459 }
3460
3461 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3462 {
3463         struct dmar_drhd_unit *drhd;
3464         u32 vtbar;
3465         int rc;
3466
3467         /* We know that this device on this chipset has its own IOMMU.
3468          * If we find it under a different IOMMU, then the BIOS is lying
3469          * to us. Hope that the IOMMU for this device is actually
3470          * disabled, and it needs no translation...
3471          */
3472         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3473         if (rc) {
3474                 /* "can't" happen */
3475                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3476                 return;
3477         }
3478         vtbar &= 0xffff0000;
3479
3480         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3481         drhd = dmar_find_matched_drhd_unit(pdev);
3482         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3483                             TAINT_FIRMWARE_WORKAROUND,
3484                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3485                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3486 }
3487 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3488
3489 static void __init init_no_remapping_devices(void)
3490 {
3491         struct dmar_drhd_unit *drhd;
3492         struct device *dev;
3493         int i;
3494
3495         for_each_drhd_unit(drhd) {
3496                 if (!drhd->include_all) {
3497                         for_each_active_dev_scope(drhd->devices,
3498                                                   drhd->devices_cnt, i, dev)
3499                                 break;
3500                         /* ignore DMAR unit if no devices exist */
3501                         if (i == drhd->devices_cnt)
3502                                 drhd->ignored = 1;
3503                 }
3504         }
3505
3506         for_each_active_drhd_unit(drhd) {
3507                 if (drhd->include_all)
3508                         continue;
3509
3510                 for_each_active_dev_scope(drhd->devices,
3511                                           drhd->devices_cnt, i, dev)
3512                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3513                                 break;
3514                 if (i < drhd->devices_cnt)
3515                         continue;
3516
3517                 /* This IOMMU has *only* gfx devices. Either bypass it or
3518                    set the gfx_mapped flag, as appropriate */
3519                 if (dmar_map_gfx) {
3520                         intel_iommu_gfx_mapped = 1;
3521                 } else {
3522                         drhd->ignored = 1;
3523                         for_each_active_dev_scope(drhd->devices,
3524                                                   drhd->devices_cnt, i, dev)
3525                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3526                 }
3527         }
3528 }
3529
3530 #ifdef CONFIG_SUSPEND
3531 static int init_iommu_hw(void)
3532 {
3533         struct dmar_drhd_unit *drhd;
3534         struct intel_iommu *iommu = NULL;
3535
3536         for_each_active_iommu(iommu, drhd)
3537                 if (iommu->qi)
3538                         dmar_reenable_qi(iommu);
3539
3540         for_each_iommu(iommu, drhd) {
3541                 if (drhd->ignored) {
3542                         /*
3543                          * we always have to disable PMRs or DMA may fail on
3544                          * this device
3545                          */
3546                         if (force_on)
3547                                 iommu_disable_protect_mem_regions(iommu);
3548                         continue;
3549                 }
3550
3551                 iommu_flush_write_buffer(iommu);
3552
3553                 iommu_set_root_entry(iommu);
3554
3555                 iommu->flush.flush_context(iommu, 0, 0, 0,
3556                                            DMA_CCMD_GLOBAL_INVL);
3557                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3558                 iommu_enable_translation(iommu);
3559                 iommu_disable_protect_mem_regions(iommu);
3560         }
3561
3562         return 0;
3563 }
3564
3565 static void iommu_flush_all(void)
3566 {
3567         struct dmar_drhd_unit *drhd;
3568         struct intel_iommu *iommu;
3569
3570         for_each_active_iommu(iommu, drhd) {
3571                 iommu->flush.flush_context(iommu, 0, 0, 0,
3572                                            DMA_CCMD_GLOBAL_INVL);
3573                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3574                                          DMA_TLB_GLOBAL_FLUSH);
3575         }
3576 }
3577
3578 static int iommu_suspend(void)
3579 {
3580         struct dmar_drhd_unit *drhd;
3581         struct intel_iommu *iommu = NULL;
3582         unsigned long flag;
3583
3584         for_each_active_iommu(iommu, drhd) {
3585                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3586                                                  GFP_ATOMIC);
3587                 if (!iommu->iommu_state)
3588                         goto nomem;
3589         }
3590
3591         iommu_flush_all();
3592
3593         for_each_active_iommu(iommu, drhd) {
3594                 iommu_disable_translation(iommu);
3595
3596                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3597
3598                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3599                         readl(iommu->reg + DMAR_FECTL_REG);
3600                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3601                         readl(iommu->reg + DMAR_FEDATA_REG);
3602                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3603                         readl(iommu->reg + DMAR_FEADDR_REG);
3604                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3605                         readl(iommu->reg + DMAR_FEUADDR_REG);
3606
3607                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3608         }
3609         return 0;
3610
3611 nomem:
3612         for_each_active_iommu(iommu, drhd)
3613                 kfree(iommu->iommu_state);
3614
3615         return -ENOMEM;
3616 }
3617
3618 static void iommu_resume(void)
3619 {
3620         struct dmar_drhd_unit *drhd;
3621         struct intel_iommu *iommu = NULL;
3622         unsigned long flag;
3623
3624         if (init_iommu_hw()) {
3625                 if (force_on)
3626                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3627                 else
3628                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3629                 return;
3630         }
3631
3632         for_each_active_iommu(iommu, drhd) {
3633
3634                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3635
3636                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3637                         iommu->reg + DMAR_FECTL_REG);
3638                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3639                         iommu->reg + DMAR_FEDATA_REG);
3640                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3641                         iommu->reg + DMAR_FEADDR_REG);
3642                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3643                         iommu->reg + DMAR_FEUADDR_REG);
3644
3645                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3646         }
3647
3648         for_each_active_iommu(iommu, drhd)
3649                 kfree(iommu->iommu_state);
3650 }
3651
3652 static struct syscore_ops iommu_syscore_ops = {
3653         .resume         = iommu_resume,
3654         .suspend        = iommu_suspend,
3655 };
3656
3657 static void __init init_iommu_pm_ops(void)
3658 {
3659         register_syscore_ops(&iommu_syscore_ops);
3660 }
3661
3662 #else
3663 static inline void init_iommu_pm_ops(void) {}
3664 #endif  /* CONFIG_PM */
3665
3666
3667 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3668 {
3669         struct acpi_dmar_reserved_memory *rmrr;
3670         struct dmar_rmrr_unit *rmrru;
3671
3672         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3673         if (!rmrru)
3674                 return -ENOMEM;
3675
3676         rmrru->hdr = header;
3677         rmrr = (struct acpi_dmar_reserved_memory *)header;
3678         rmrru->base_address = rmrr->base_address;
3679         rmrru->end_address = rmrr->end_address;
3680         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3681                                 ((void *)rmrr) + rmrr->header.length,
3682                                 &rmrru->devices_cnt);
3683         if (rmrru->devices_cnt && rmrru->devices == NULL) {
3684                 kfree(rmrru);
3685                 return -ENOMEM;
3686         }
3687
3688         list_add(&rmrru->list, &dmar_rmrr_units);
3689
3690         return 0;
3691 }
3692
3693 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3694 {
3695         struct dmar_atsr_unit *atsru;
3696         struct acpi_dmar_atsr *tmp;
3697
3698         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3699                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3700                 if (atsr->segment != tmp->segment)
3701                         continue;
3702                 if (atsr->header.length != tmp->header.length)
3703                         continue;
3704                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3705                         return atsru;
3706         }
3707
3708         return NULL;
3709 }
3710
3711 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3712 {
3713         struct acpi_dmar_atsr *atsr;
3714         struct dmar_atsr_unit *atsru;
3715
3716         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3717                 return 0;
3718
3719         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3720         atsru = dmar_find_atsr(atsr);
3721         if (atsru)
3722                 return 0;
3723
3724         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3725         if (!atsru)
3726                 return -ENOMEM;
3727
3728         /*
3729          * If memory is allocated from slab by ACPI _DSM method, we need to
3730          * copy the memory content because the memory buffer will be freed
3731          * on return.
3732          */
3733         atsru->hdr = (void *)(atsru + 1);
3734         memcpy(atsru->hdr, hdr, hdr->length);
3735         atsru->include_all = atsr->flags & 0x1;
3736         if (!atsru->include_all) {
3737                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3738                                 (void *)atsr + atsr->header.length,
3739                                 &atsru->devices_cnt);
3740                 if (atsru->devices_cnt && atsru->devices == NULL) {
3741                         kfree(atsru);
3742                         return -ENOMEM;
3743                 }
3744         }
3745
3746         list_add_rcu(&atsru->list, &dmar_atsr_units);
3747
3748         return 0;
3749 }
3750
3751 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3752 {
3753         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3754         kfree(atsru);
3755 }
3756
3757 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3758 {
3759         struct acpi_dmar_atsr *atsr;
3760         struct dmar_atsr_unit *atsru;
3761
3762         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3763         atsru = dmar_find_atsr(atsr);
3764         if (atsru) {
3765                 list_del_rcu(&atsru->list);
3766                 synchronize_rcu();
3767                 intel_iommu_free_atsr(atsru);
3768         }
3769
3770         return 0;
3771 }
3772
3773 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3774 {
3775         int i;
3776         struct device *dev;
3777         struct acpi_dmar_atsr *atsr;
3778         struct dmar_atsr_unit *atsru;
3779
3780         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3781         atsru = dmar_find_atsr(atsr);
3782         if (!atsru)
3783                 return 0;
3784
3785         if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
3786                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3787                                           i, dev)
3788                         return -EBUSY;
3789
3790         return 0;
3791 }
3792
3793 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3794 {
3795         int sp, ret = 0;
3796         struct intel_iommu *iommu = dmaru->iommu;
3797
3798         if (g_iommus[iommu->seq_id])
3799                 return 0;
3800
3801         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3802                 pr_warn("%s: Doesn't support hardware pass through.\n",
3803                         iommu->name);
3804                 return -ENXIO;
3805         }
3806         if (!ecap_sc_support(iommu->ecap) &&
3807             domain_update_iommu_snooping(iommu)) {
3808                 pr_warn("%s: Doesn't support snooping.\n",
3809                         iommu->name);
3810                 return -ENXIO;
3811         }
3812         sp = domain_update_iommu_superpage(iommu) - 1;
3813         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3814                 pr_warn("%s: Doesn't support large page.\n",
3815                         iommu->name);
3816                 return -ENXIO;
3817         }
3818
3819         /*
3820          * Disable translation if already enabled prior to OS handover.
3821          */
3822         if (iommu->gcmd & DMA_GCMD_TE)
3823                 iommu_disable_translation(iommu);
3824
3825         g_iommus[iommu->seq_id] = iommu;
3826         ret = iommu_init_domains(iommu);
3827         if (ret == 0)
3828                 ret = iommu_alloc_root_entry(iommu);
3829         if (ret)
3830                 goto out;
3831
3832         if (dmaru->ignored) {
3833                 /*
3834                  * we always have to disable PMRs or DMA may fail on this device
3835                  */
3836                 if (force_on)
3837                         iommu_disable_protect_mem_regions(iommu);
3838                 return 0;
3839         }
3840
3841         intel_iommu_init_qi(iommu);
3842         iommu_flush_write_buffer(iommu);
3843         ret = dmar_set_interrupt(iommu);
3844         if (ret)
3845                 goto disable_iommu;
3846
3847         iommu_set_root_entry(iommu);
3848         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3849         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3850         iommu_enable_translation(iommu);
3851
3852         if (si_domain) {
3853                 ret = iommu_attach_domain(si_domain, iommu);
3854                 if (ret < 0 || si_domain->id != ret)
3855                         goto disable_iommu;
3856                 domain_attach_iommu(si_domain, iommu);
3857         }
3858
3859         iommu_disable_protect_mem_regions(iommu);
3860         return 0;
3861
3862 disable_iommu:
3863         disable_dmar_iommu(iommu);
3864 out:
3865         free_dmar_iommu(iommu);
3866         return ret;
3867 }
3868
3869 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3870 {
3871         int ret = 0;
3872         struct intel_iommu *iommu = dmaru->iommu;
3873
3874         if (!intel_iommu_enabled)
3875                 return 0;
3876         if (iommu == NULL)
3877                 return -EINVAL;
3878
3879         if (insert) {
3880                 ret = intel_iommu_add(dmaru);
3881         } else {
3882                 disable_dmar_iommu(iommu);
3883                 free_dmar_iommu(iommu);
3884         }
3885
3886         return ret;
3887 }
3888
3889 static void intel_iommu_free_dmars(void)
3890 {
3891         struct dmar_rmrr_unit *rmrru, *rmrr_n;
3892         struct dmar_atsr_unit *atsru, *atsr_n;
3893
3894         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3895                 list_del(&rmrru->list);
3896                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3897                 kfree(rmrru);
3898         }
3899
3900         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3901                 list_del(&atsru->list);
3902                 intel_iommu_free_atsr(atsru);
3903         }
3904 }
3905
3906 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3907 {
3908         int i, ret = 1;
3909         struct pci_bus *bus;
3910         struct pci_dev *bridge = NULL;
3911         struct device *tmp;
3912         struct acpi_dmar_atsr *atsr;
3913         struct dmar_atsr_unit *atsru;
3914
3915         dev = pci_physfn(dev);
3916         for (bus = dev->bus; bus; bus = bus->parent) {
3917                 bridge = bus->self;
3918                 if (!bridge || !pci_is_pcie(bridge) ||
3919                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3920                         return 0;
3921                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3922                         break;
3923         }
3924         if (!bridge)
3925                 return 0;
3926
3927         rcu_read_lock();
3928         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3929                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3930                 if (atsr->segment != pci_domain_nr(dev->bus))
3931                         continue;
3932
3933                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3934                         if (tmp == &bridge->dev)
3935                                 goto out;
3936
3937                 if (atsru->include_all)
3938                         goto out;
3939         }
3940         ret = 0;
3941 out:
3942         rcu_read_unlock();
3943
3944         return ret;
3945 }
3946
3947 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3948 {
3949         int ret = 0;
3950         struct dmar_rmrr_unit *rmrru;
3951         struct dmar_atsr_unit *atsru;
3952         struct acpi_dmar_atsr *atsr;
3953         struct acpi_dmar_reserved_memory *rmrr;
3954
3955         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3956                 return 0;
3957
3958         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3959                 rmrr = container_of(rmrru->hdr,
3960                                     struct acpi_dmar_reserved_memory, header);
3961                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3962                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3963                                 ((void *)rmrr) + rmrr->header.length,
3964                                 rmrr->segment, rmrru->devices,
3965                                 rmrru->devices_cnt);
3966                         if(ret < 0)
3967                                 return ret;
3968                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3969                         dmar_remove_dev_scope(info, rmrr->segment,
3970                                 rmrru->devices, rmrru->devices_cnt);
3971                 }
3972         }
3973
3974         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3975                 if (atsru->include_all)
3976                         continue;
3977
3978                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3979                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3980                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3981                                         (void *)atsr + atsr->header.length,
3982                                         atsr->segment, atsru->devices,
3983                                         atsru->devices_cnt);
3984                         if (ret > 0)
3985                                 break;
3986                         else if(ret < 0)
3987                                 return ret;
3988                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3989                         if (dmar_remove_dev_scope(info, atsr->segment,
3990                                         atsru->devices, atsru->devices_cnt))
3991                                 break;
3992                 }
3993         }
3994
3995         return 0;
3996 }
3997
3998 /*
3999  * Here we only respond to action of unbound device from driver.
4000  *
4001  * Added device is not attached to its DMAR domain here yet. That will happen
4002  * when mapping the device to iova.
4003  */
4004 static int device_notifier(struct notifier_block *nb,
4005                                   unsigned long action, void *data)
4006 {
4007         struct device *dev = data;
4008         struct dmar_domain *domain;
4009
4010         if (iommu_dummy(dev))
4011                 return 0;
4012
4013         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4014                 return 0;
4015
4016         domain = find_domain(dev);
4017         if (!domain)
4018                 return 0;
4019
4020         down_read(&dmar_global_lock);
4021         domain_remove_one_dev_info(domain, dev);
4022         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4023                 domain_exit(domain);
4024         up_read(&dmar_global_lock);
4025
4026         return 0;
4027 }
4028
4029 static struct notifier_block device_nb = {
4030         .notifier_call = device_notifier,
4031 };
4032
4033 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4034                                        unsigned long val, void *v)
4035 {
4036         struct memory_notify *mhp = v;
4037         unsigned long long start, end;
4038         unsigned long start_vpfn, last_vpfn;
4039
4040         switch (val) {
4041         case MEM_GOING_ONLINE:
4042                 start = mhp->start_pfn << PAGE_SHIFT;
4043                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4044                 if (iommu_domain_identity_map(si_domain, start, end)) {
4045                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4046                                 start, end);
4047                         return NOTIFY_BAD;
4048                 }
4049                 break;
4050
4051         case MEM_OFFLINE:
4052         case MEM_CANCEL_ONLINE:
4053                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4054                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4055                 while (start_vpfn <= last_vpfn) {
4056                         struct iova *iova;
4057                         struct dmar_drhd_unit *drhd;
4058                         struct intel_iommu *iommu;
4059                         struct page *freelist;
4060
4061                         iova = find_iova(&si_domain->iovad, start_vpfn);
4062                         if (iova == NULL) {
4063                                 pr_debug("Failed get IOVA for PFN %lx\n",
4064                                          start_vpfn);
4065                                 break;
4066                         }
4067
4068                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4069                                                      start_vpfn, last_vpfn);
4070                         if (iova == NULL) {
4071                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4072                                         start_vpfn, last_vpfn);
4073                                 return NOTIFY_BAD;
4074                         }
4075
4076                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4077                                                iova->pfn_hi);
4078
4079                         rcu_read_lock();
4080                         for_each_active_iommu(iommu, drhd)
4081                                 iommu_flush_iotlb_psi(iommu, si_domain->id,
4082                                         iova->pfn_lo, iova_size(iova),
4083                                         !freelist, 0);
4084                         rcu_read_unlock();
4085                         dma_free_pagelist(freelist);
4086
4087                         start_vpfn = iova->pfn_hi + 1;
4088                         free_iova_mem(iova);
4089                 }
4090                 break;
4091         }
4092
4093         return NOTIFY_OK;
4094 }
4095
4096 static struct notifier_block intel_iommu_memory_nb = {
4097         .notifier_call = intel_iommu_memory_notifier,
4098         .priority = 0
4099 };
4100
4101
4102 static ssize_t intel_iommu_show_version(struct device *dev,
4103                                         struct device_attribute *attr,
4104                                         char *buf)
4105 {
4106         struct intel_iommu *iommu = dev_get_drvdata(dev);
4107         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4108         return sprintf(buf, "%d:%d\n",
4109                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4110 }
4111 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4112
4113 static ssize_t intel_iommu_show_address(struct device *dev,
4114                                         struct device_attribute *attr,
4115                                         char *buf)
4116 {
4117         struct intel_iommu *iommu = dev_get_drvdata(dev);
4118         return sprintf(buf, "%llx\n", iommu->reg_phys);
4119 }
4120 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4121
4122 static ssize_t intel_iommu_show_cap(struct device *dev,
4123                                     struct device_attribute *attr,
4124                                     char *buf)
4125 {
4126         struct intel_iommu *iommu = dev_get_drvdata(dev);
4127         return sprintf(buf, "%llx\n", iommu->cap);
4128 }
4129 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4130
4131 static ssize_t intel_iommu_show_ecap(struct device *dev,
4132                                     struct device_attribute *attr,
4133                                     char *buf)
4134 {
4135         struct intel_iommu *iommu = dev_get_drvdata(dev);
4136         return sprintf(buf, "%llx\n", iommu->ecap);
4137 }
4138 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4139
4140 static struct attribute *intel_iommu_attrs[] = {
4141         &dev_attr_version.attr,
4142         &dev_attr_address.attr,
4143         &dev_attr_cap.attr,
4144         &dev_attr_ecap.attr,
4145         NULL,
4146 };
4147
4148 static struct attribute_group intel_iommu_group = {
4149         .name = "intel-iommu",
4150         .attrs = intel_iommu_attrs,
4151 };
4152
4153 const struct attribute_group *intel_iommu_groups[] = {
4154         &intel_iommu_group,
4155         NULL,
4156 };
4157
4158 int __init intel_iommu_init(void)
4159 {
4160         int ret = -ENODEV;
4161         struct dmar_drhd_unit *drhd;
4162         struct intel_iommu *iommu;
4163
4164         /* VT-d is required for a TXT/tboot launch, so enforce that */
4165         force_on = tboot_force_iommu();
4166
4167         if (iommu_init_mempool()) {
4168                 if (force_on)
4169                         panic("tboot: Failed to initialize iommu memory\n");
4170                 return -ENOMEM;
4171         }
4172
4173         down_write(&dmar_global_lock);
4174         if (dmar_table_init()) {
4175                 if (force_on)
4176                         panic("tboot: Failed to initialize DMAR table\n");
4177                 goto out_free_dmar;
4178         }
4179
4180         /*
4181          * Disable translation if already enabled prior to OS handover.
4182          */
4183         for_each_active_iommu(iommu, drhd)
4184                 if (iommu->gcmd & DMA_GCMD_TE)
4185                         iommu_disable_translation(iommu);
4186
4187         if (dmar_dev_scope_init() < 0) {
4188                 if (force_on)
4189                         panic("tboot: Failed to initialize DMAR device scope\n");
4190                 goto out_free_dmar;
4191         }
4192
4193         if (no_iommu || dmar_disabled)
4194                 goto out_free_dmar;
4195
4196         if (list_empty(&dmar_rmrr_units))
4197                 pr_info("No RMRR found\n");
4198
4199         if (list_empty(&dmar_atsr_units))
4200                 pr_info("No ATSR found\n");
4201
4202         if (dmar_init_reserved_ranges()) {
4203                 if (force_on)
4204                         panic("tboot: Failed to reserve iommu ranges\n");
4205                 goto out_free_reserved_range;
4206         }
4207
4208         init_no_remapping_devices();
4209
4210         ret = init_dmars();
4211         if (ret) {
4212                 if (force_on)
4213                         panic("tboot: Failed to initialize DMARs\n");
4214                 pr_err("Initialization failed\n");
4215                 goto out_free_reserved_range;
4216         }
4217         up_write(&dmar_global_lock);
4218         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4219
4220         init_timer(&unmap_timer);
4221 #ifdef CONFIG_SWIOTLB
4222         swiotlb = 0;
4223 #endif
4224         dma_ops = &intel_dma_ops;
4225
4226         init_iommu_pm_ops();
4227
4228         for_each_active_iommu(iommu, drhd)
4229                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4230                                                        intel_iommu_groups,
4231                                                        iommu->name);
4232
4233         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4234         bus_register_notifier(&pci_bus_type, &device_nb);
4235         if (si_domain && !hw_pass_through)
4236                 register_memory_notifier(&intel_iommu_memory_nb);
4237
4238         intel_iommu_enabled = 1;
4239
4240         return 0;
4241
4242 out_free_reserved_range:
4243         put_iova_domain(&reserved_iova_list);
4244 out_free_dmar:
4245         intel_iommu_free_dmars();
4246         up_write(&dmar_global_lock);
4247         iommu_exit_mempool();
4248         return ret;
4249 }
4250
4251 static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4252 {
4253         struct intel_iommu *iommu = opaque;
4254
4255         iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4256         return 0;
4257 }
4258
4259 /*
4260  * NB - intel-iommu lacks any sort of reference counting for the users of
4261  * dependent devices.  If multiple endpoints have intersecting dependent
4262  * devices, unbinding the driver from any one of them will possibly leave
4263  * the others unable to operate.
4264  */
4265 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4266                                            struct device *dev)
4267 {
4268         if (!iommu || !dev || !dev_is_pci(dev))
4269                 return;
4270
4271         pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4272 }
4273
4274 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4275                                        struct device *dev)
4276 {
4277         struct device_domain_info *info, *tmp;
4278         struct intel_iommu *iommu;
4279         unsigned long flags;
4280         bool found = false;
4281         u8 bus, devfn;
4282
4283         iommu = device_to_iommu(dev, &bus, &devfn);
4284         if (!iommu)
4285                 return;
4286
4287         spin_lock_irqsave(&device_domain_lock, flags);
4288         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4289                 if (info->iommu == iommu && info->bus == bus &&
4290                     info->devfn == devfn) {
4291                         unlink_domain_info(info);
4292                         spin_unlock_irqrestore(&device_domain_lock, flags);
4293
4294                         iommu_disable_dev_iotlb(info);
4295                         iommu_detach_dev(iommu, info->bus, info->devfn);
4296                         iommu_detach_dependent_devices(iommu, dev);
4297                         free_devinfo_mem(info);
4298
4299                         spin_lock_irqsave(&device_domain_lock, flags);
4300
4301                         if (found)
4302                                 break;
4303                         else
4304                                 continue;
4305                 }
4306
4307                 /* if there is no other devices under the same iommu
4308                  * owned by this domain, clear this iommu in iommu_bmp
4309                  * update iommu count and coherency
4310                  */
4311                 if (info->iommu == iommu)
4312                         found = true;
4313         }
4314
4315         spin_unlock_irqrestore(&device_domain_lock, flags);
4316
4317         if (found == 0) {
4318                 domain_detach_iommu(domain, iommu);
4319                 if (!domain_type_is_vm_or_si(domain))
4320                         iommu_detach_domain(domain, iommu);
4321         }
4322 }
4323
4324 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4325 {
4326         int adjust_width;
4327
4328         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4329                         DMA_32BIT_PFN);
4330         domain_reserve_special_ranges(domain);
4331
4332         /* calculate AGAW */
4333         domain->gaw = guest_width;
4334         adjust_width = guestwidth_to_adjustwidth(guest_width);
4335         domain->agaw = width_to_agaw(adjust_width);
4336
4337         domain->iommu_coherency = 0;
4338         domain->iommu_snooping = 0;
4339         domain->iommu_superpage = 0;
4340         domain->max_addr = 0;
4341
4342         /* always allocate the top pgd */
4343         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4344         if (!domain->pgd)
4345                 return -ENOMEM;
4346         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4347         return 0;
4348 }
4349
4350 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4351 {
4352         struct dmar_domain *dmar_domain;
4353         struct iommu_domain *domain;
4354
4355         if (type != IOMMU_DOMAIN_UNMANAGED)
4356                 return NULL;
4357
4358         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4359         if (!dmar_domain) {
4360                 pr_err("Can't allocate dmar_domain\n");
4361                 return NULL;
4362         }
4363         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4364                 pr_err("Domain initialization failed\n");
4365                 domain_exit(dmar_domain);
4366                 return NULL;
4367         }
4368         domain_update_iommu_cap(dmar_domain);
4369
4370         domain = &dmar_domain->domain;
4371         domain->geometry.aperture_start = 0;
4372         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4373         domain->geometry.force_aperture = true;
4374
4375         return domain;
4376 }
4377
4378 static void intel_iommu_domain_free(struct iommu_domain *domain)
4379 {
4380         domain_exit(to_dmar_domain(domain));
4381 }
4382
4383 static int intel_iommu_attach_device(struct iommu_domain *domain,
4384                                      struct device *dev)
4385 {
4386         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4387         struct intel_iommu *iommu;
4388         int addr_width;
4389         u8 bus, devfn;
4390
4391         if (device_is_rmrr_locked(dev)) {
4392                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4393                 return -EPERM;
4394         }
4395
4396         /* normally dev is not mapped */
4397         if (unlikely(domain_context_mapped(dev))) {
4398                 struct dmar_domain *old_domain;
4399
4400                 old_domain = find_domain(dev);
4401                 if (old_domain) {
4402                         if (domain_type_is_vm_or_si(dmar_domain))
4403                                 domain_remove_one_dev_info(old_domain, dev);
4404                         else
4405                                 domain_remove_dev_info(old_domain);
4406
4407                         if (!domain_type_is_vm_or_si(old_domain) &&
4408                              list_empty(&old_domain->devices))
4409                                 domain_exit(old_domain);
4410                 }
4411         }
4412
4413         iommu = device_to_iommu(dev, &bus, &devfn);
4414         if (!iommu)
4415                 return -ENODEV;
4416
4417         /* check if this iommu agaw is sufficient for max mapped address */
4418         addr_width = agaw_to_width(iommu->agaw);
4419         if (addr_width > cap_mgaw(iommu->cap))
4420                 addr_width = cap_mgaw(iommu->cap);
4421
4422         if (dmar_domain->max_addr > (1LL << addr_width)) {
4423                 pr_err("%s: iommu width (%d) is not "
4424                        "sufficient for the mapped address (%llx)\n",
4425                        __func__, addr_width, dmar_domain->max_addr);
4426                 return -EFAULT;
4427         }
4428         dmar_domain->gaw = addr_width;
4429
4430         /*
4431          * Knock out extra levels of page tables if necessary
4432          */
4433         while (iommu->agaw < dmar_domain->agaw) {
4434                 struct dma_pte *pte;
4435
4436                 pte = dmar_domain->pgd;
4437                 if (dma_pte_present(pte)) {
4438                         dmar_domain->pgd = (struct dma_pte *)
4439                                 phys_to_virt(dma_pte_addr(pte));
4440                         free_pgtable_page(pte);
4441                 }
4442                 dmar_domain->agaw--;
4443         }
4444
4445         return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4446 }
4447
4448 static void intel_iommu_detach_device(struct iommu_domain *domain,
4449                                       struct device *dev)
4450 {
4451         domain_remove_one_dev_info(to_dmar_domain(domain), dev);
4452 }
4453
4454 static int intel_iommu_map(struct iommu_domain *domain,
4455                            unsigned long iova, phys_addr_t hpa,
4456                            size_t size, int iommu_prot)
4457 {
4458         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4459         u64 max_addr;
4460         int prot = 0;
4461         int ret;
4462
4463         if (iommu_prot & IOMMU_READ)
4464                 prot |= DMA_PTE_READ;
4465         if (iommu_prot & IOMMU_WRITE)
4466                 prot |= DMA_PTE_WRITE;
4467         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4468                 prot |= DMA_PTE_SNP;
4469
4470         max_addr = iova + size;
4471         if (dmar_domain->max_addr < max_addr) {
4472                 u64 end;
4473
4474                 /* check if minimum agaw is sufficient for mapped address */
4475                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4476                 if (end < max_addr) {
4477                         pr_err("%s: iommu width (%d) is not "
4478                                "sufficient for the mapped address (%llx)\n",
4479                                __func__, dmar_domain->gaw, max_addr);
4480                         return -EFAULT;
4481                 }
4482                 dmar_domain->max_addr = max_addr;
4483         }
4484         /* Round up size to next multiple of PAGE_SIZE, if it and
4485            the low bits of hpa would take us onto the next page */
4486         size = aligned_nrpages(hpa, size);
4487         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4488                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4489         return ret;
4490 }
4491
4492 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4493                                 unsigned long iova, size_t size)
4494 {
4495         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4496         struct page *freelist = NULL;
4497         struct intel_iommu *iommu;
4498         unsigned long start_pfn, last_pfn;
4499         unsigned int npages;
4500         int iommu_id, num, ndomains, level = 0;
4501
4502         /* Cope with horrid API which requires us to unmap more than the
4503            size argument if it happens to be a large-page mapping. */
4504         if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4505                 BUG();
4506
4507         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4508                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4509
4510         start_pfn = iova >> VTD_PAGE_SHIFT;
4511         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4512
4513         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4514
4515         npages = last_pfn - start_pfn + 1;
4516
4517         for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4518                iommu = g_iommus[iommu_id];
4519
4520                /*
4521                 * find bit position of dmar_domain
4522                 */
4523                ndomains = cap_ndoms(iommu->cap);
4524                for_each_set_bit(num, iommu->domain_ids, ndomains) {
4525                        if (iommu->domains[num] == dmar_domain)
4526                                iommu_flush_iotlb_psi(iommu, num, start_pfn,
4527                                                      npages, !freelist, 0);
4528                }
4529
4530         }
4531
4532         dma_free_pagelist(freelist);
4533
4534         if (dmar_domain->max_addr == iova + size)
4535                 dmar_domain->max_addr = iova;
4536
4537         return size;
4538 }
4539
4540 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4541                                             dma_addr_t iova)
4542 {
4543         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4544         struct dma_pte *pte;
4545         int level = 0;
4546         u64 phys = 0;
4547
4548         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4549         if (pte)
4550                 phys = dma_pte_addr(pte);
4551
4552         return phys;
4553 }
4554
4555 static bool intel_iommu_capable(enum iommu_cap cap)
4556 {
4557         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4558                 return domain_update_iommu_snooping(NULL) == 1;
4559         if (cap == IOMMU_CAP_INTR_REMAP)
4560                 return irq_remapping_enabled == 1;
4561
4562         return false;
4563 }
4564
4565 static int intel_iommu_add_device(struct device *dev)
4566 {
4567         struct intel_iommu *iommu;
4568         struct iommu_group *group;
4569         u8 bus, devfn;
4570
4571         iommu = device_to_iommu(dev, &bus, &devfn);
4572         if (!iommu)
4573                 return -ENODEV;
4574
4575         iommu_device_link(iommu->iommu_dev, dev);
4576
4577         group = iommu_group_get_for_dev(dev);
4578
4579         if (IS_ERR(group))
4580                 return PTR_ERR(group);
4581
4582         iommu_group_put(group);
4583         return 0;
4584 }
4585
4586 static void intel_iommu_remove_device(struct device *dev)
4587 {
4588         struct intel_iommu *iommu;
4589         u8 bus, devfn;
4590
4591         iommu = device_to_iommu(dev, &bus, &devfn);
4592         if (!iommu)
4593                 return;
4594
4595         iommu_group_remove_device(dev);
4596
4597         iommu_device_unlink(iommu->iommu_dev, dev);
4598 }
4599
4600 static const struct iommu_ops intel_iommu_ops = {
4601         .capable        = intel_iommu_capable,
4602         .domain_alloc   = intel_iommu_domain_alloc,
4603         .domain_free    = intel_iommu_domain_free,
4604         .attach_dev     = intel_iommu_attach_device,
4605         .detach_dev     = intel_iommu_detach_device,
4606         .map            = intel_iommu_map,
4607         .unmap          = intel_iommu_unmap,
4608         .map_sg         = default_iommu_map_sg,
4609         .iova_to_phys   = intel_iommu_iova_to_phys,
4610         .add_device     = intel_iommu_add_device,
4611         .remove_device  = intel_iommu_remove_device,
4612         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4613 };
4614
4615 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4616 {
4617         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4618         pr_info("Disabling IOMMU for graphics on this chipset\n");
4619         dmar_map_gfx = 0;
4620 }
4621
4622 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4623 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4624 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4625 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4626 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4627 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4628 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4629
4630 static void quirk_iommu_rwbf(struct pci_dev *dev)
4631 {
4632         /*
4633          * Mobile 4 Series Chipset neglects to set RWBF capability,
4634          * but needs it. Same seems to hold for the desktop versions.
4635          */
4636         pr_info("Forcing write-buffer flush capability\n");
4637         rwbf_quirk = 1;
4638 }
4639
4640 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4641 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4642 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4643 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4644 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4645 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4646 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4647
4648 #define GGC 0x52
4649 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4650 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4651 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4652 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4653 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4654 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4655 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4656 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4657
4658 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4659 {
4660         unsigned short ggc;
4661
4662         if (pci_read_config_word(dev, GGC, &ggc))
4663                 return;
4664
4665         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4666                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4667                 dmar_map_gfx = 0;
4668         } else if (dmar_map_gfx) {
4669                 /* we have to ensure the gfx device is idle before we flush */
4670                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
4671                 intel_iommu_strict = 1;
4672        }
4673 }
4674 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4675 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4676 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4677 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4678
4679 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4680    ISOCH DMAR unit for the Azalia sound device, but not give it any
4681    TLB entries, which causes it to deadlock. Check for that.  We do
4682    this in a function called from init_dmars(), instead of in a PCI
4683    quirk, because we don't want to print the obnoxious "BIOS broken"
4684    message if VT-d is actually disabled.
4685 */
4686 static void __init check_tylersburg_isoch(void)
4687 {
4688         struct pci_dev *pdev;
4689         uint32_t vtisochctrl;
4690
4691         /* If there's no Azalia in the system anyway, forget it. */
4692         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4693         if (!pdev)
4694                 return;
4695         pci_dev_put(pdev);
4696
4697         /* System Management Registers. Might be hidden, in which case
4698            we can't do the sanity check. But that's OK, because the
4699            known-broken BIOSes _don't_ actually hide it, so far. */
4700         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4701         if (!pdev)
4702                 return;
4703
4704         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4705                 pci_dev_put(pdev);
4706                 return;
4707         }
4708
4709         pci_dev_put(pdev);
4710
4711         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4712         if (vtisochctrl & 1)
4713                 return;
4714
4715         /* Drop all bits other than the number of TLB entries */
4716         vtisochctrl &= 0x1c;
4717
4718         /* If we have the recommended number of TLB entries (16), fine. */
4719         if (vtisochctrl == 0x10)
4720                 return;
4721
4722         /* Zero TLB entries? You get to ride the short bus to school. */
4723         if (!vtisochctrl) {
4724                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4725                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4726                      dmi_get_system_info(DMI_BIOS_VENDOR),
4727                      dmi_get_system_info(DMI_BIOS_VERSION),
4728                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4729                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4730                 return;
4731         }
4732
4733         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4734                vtisochctrl);
4735 }