drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <linux/sysdev.h>
  40 #include <linux/tboot.h>
  41 #include <linux/dmi.h>
  42 #include <asm/cacheflush.h>
  43 #include <asm/iommu.h>
  44 #include "pci.h"
  45
  46 #define ROOT_SIZE               VTD_PAGE_SIZE
  47 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  48
  49 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  50 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  51 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  52
  53 #define IOAPIC_RANGE_START      (0xfee00000)
  54 #define IOAPIC_RANGE_END        (0xfeefffff)
  55 #define IOVA_START_ADDR         (0x1000)
  56
  57 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  58
  59 #define MAX_AGAW_WIDTH 64
  60
  61 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  62 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  63
  64 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  65    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  66 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  67                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  68 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  69
  70 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  71 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  72 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  73
  74
  75 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
  76    are never going to work. */
  77 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
  78 {
  79         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
  80 }
  81
  82 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
  83 {
  84         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
  85 }
  86 static inline unsigned long page_to_dma_pfn(struct page *pg)
  87 {
  88         return mm_to_dma_pfn(page_to_pfn(pg));
  89 }
  90 static inline unsigned long virt_to_dma_pfn(void *p)
  91 {
  92         return page_to_dma_pfn(virt_to_page(p));
  93 }
  94
  95 /* global iommu list, set NULL for ignored DMAR units */
  96 static struct intel_iommu **g_iommus;
  97
  98 static void __init check_tylersburg_isoch(void);
  99 static int rwbf_quirk;
 100
 101 /*
 102  * 0: Present
 103  * 1-11: Reserved
 104  * 12-63: Context Ptr (12 - (haw-1))
 105  * 64-127: Reserved
 106  */
 107 struct root_entry {
 108         u64     val;
 109         u64     rsvd1;
 110 };
 111 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 112 static inline bool root_present(struct root_entry *root)
 113 {
 114         return (root->val & 1);
 115 }
 116 static inline void set_root_present(struct root_entry *root)
 117 {
 118         root->val |= 1;
 119 }
 120 static inline void set_root_value(struct root_entry *root, unsigned long value)
 121 {
 122         root->val |= value & VTD_PAGE_MASK;
 123 }
 124
 125 static inline struct context_entry *
 126 get_context_addr_from_root(struct root_entry *root)
 127 {
 128         return (struct context_entry *)
 129                 (root_present(root)?phys_to_virt(
 130                 root->val & VTD_PAGE_MASK) :
 131                 NULL);
 132 }
 133
 134 /*
 135  * low 64 bits:
 136  * 0: present
 137  * 1: fault processing disable
 138  * 2-3: translation type
 139  * 12-63: address space root
 140  * high 64 bits:
 141  * 0-2: address width
 142  * 3-6: aval
 143  * 8-23: domain id
 144  */
 145 struct context_entry {
 146         u64 lo;
 147         u64 hi;
 148 };
 149
 150 static inline bool context_present(struct context_entry *context)
 151 {
 152         return (context->lo & 1);
 153 }
 154 static inline void context_set_present(struct context_entry *context)
 155 {
 156         context->lo |= 1;
 157 }
 158
 159 static inline void context_set_fault_enable(struct context_entry *context)
 160 {
 161         context->lo &= (((u64)-1) << 2) | 1;
 162 }
 163
 164 static inline void context_set_translation_type(struct context_entry *context,
 165                                                 unsigned long value)
 166 {
 167         context->lo &= (((u64)-1) << 4) | 3;
 168         context->lo |= (value & 3) << 2;
 169 }
 170
 171 static inline void context_set_address_root(struct context_entry *context,
 172                                             unsigned long value)
 173 {
 174         context->lo |= value & VTD_PAGE_MASK;
 175 }
 176
 177 static inline void context_set_address_width(struct context_entry *context,
 178                                              unsigned long value)
 179 {
 180         context->hi |= value & 7;
 181 }
 182
 183 static inline void context_set_domain_id(struct context_entry *context,
 184                                          unsigned long value)
 185 {
 186         context->hi |= (value & ((1 << 16) - 1)) << 8;
 187 }
 188
 189 static inline void context_clear_entry(struct context_entry *context)
 190 {
 191         context->lo = 0;
 192         context->hi = 0;
 193 }
 194
 195 /*
 196  * 0: readable
 197  * 1: writable
 198  * 2-6: reserved
 199  * 7: super page
 200  * 8-10: available
 201  * 11: snoop behavior
 202  * 12-63: Host physcial address
 203  */
 204 struct dma_pte {
 205         u64 val;
 206 };
 207
 208 static inline void dma_clear_pte(struct dma_pte *pte)
 209 {
 210         pte->val = 0;
 211 }
 212
 213 static inline void dma_set_pte_readable(struct dma_pte *pte)
 214 {
 215         pte->val |= DMA_PTE_READ;
 216 }
 217
 218 static inline void dma_set_pte_writable(struct dma_pte *pte)
 219 {
 220         pte->val |= DMA_PTE_WRITE;
 221 }
 222
 223 static inline void dma_set_pte_snp(struct dma_pte *pte)
 224 {
 225         pte->val |= DMA_PTE_SNP;
 226 }
 227
 228 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 229 {
 230         pte->val = (pte->val & ~3) | (prot & 3);
 231 }
 232
 233 static inline u64 dma_pte_addr(struct dma_pte *pte)
 234 {
 235 #ifdef CONFIG_64BIT
 236         return pte->val & VTD_PAGE_MASK;
 237 #else
 238         /* Must have a full atomic 64-bit read */
 239         return  __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
 240 #endif
 241 }
 242
 243 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 244 {
 245         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 246 }
 247
 248 static inline bool dma_pte_present(struct dma_pte *pte)
 249 {
 250         return (pte->val & 3) != 0;
 251 }
 252
 253 static inline int first_pte_in_page(struct dma_pte *pte)
 254 {
 255         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 256 }
 257
 258 /*
 259  * This domain is a statically identity mapping domain.
 260  *      1. This domain creats a static 1:1 mapping to all usable memory.
 261  *      2. It maps to each iommu if successful.
 262  *      3. Each iommu mapps to this domain if successful.
 263  */
 264 static struct dmar_domain *si_domain;
 265 static int hw_pass_through = 1;
 266
 267 /* devices under the same p2p bridge are owned in one domain */
 268 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 269
 270 /* domain represents a virtual machine, more than one devices
 271  * across iommus may be owned in one domain, e.g. kvm guest.
 272  */
 273 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 274
 275 /* si_domain contains mulitple devices */
 276 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 277
 278 struct dmar_domain {
 279         int     id;                     /* domain id */
 280         int     nid;                    /* node id */
 281         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 282
 283         struct list_head devices;       /* all devices' list */
 284         struct iova_domain iovad;       /* iova's that belong to this domain */
 285
 286         struct dma_pte  *pgd;           /* virtual address */
 287         int             gaw;            /* max guest address width */
 288
 289         /* adjusted guest address width, 0 is level 2 30-bit */
 290         int             agaw;
 291
 292         int             flags;          /* flags to find out type of domain */
 293
 294         int             iommu_coherency;/* indicate coherency of iommu access */
 295         int             iommu_snooping; /* indicate snooping control feature*/
 296         int             iommu_count;    /* reference count of iommu */
 297         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 298         u64             max_addr;       /* maximum mapped address */
 299 };
 300
 301 /* PCI domain-device relationship */
 302 struct device_domain_info {
 303         struct list_head link;  /* link to domain siblings */
 304         struct list_head global; /* link to global list */
 305         int segment;            /* PCI domain */
 306         u8 bus;                 /* PCI bus number */
 307         u8 devfn;               /* PCI devfn number */
 308         struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
 309         struct intel_iommu *iommu; /* IOMMU used by this device */
 310         struct dmar_domain *domain; /* pointer to domain */
 311 };
 312
 313 static void flush_unmaps_timeout(unsigned long data);
 314
 315 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 316
 317 #define HIGH_WATER_MARK 250
 318 struct deferred_flush_tables {
 319         int next;
 320         struct iova *iova[HIGH_WATER_MARK];
 321         struct dmar_domain *domain[HIGH_WATER_MARK];
 322 };
 323
 324 static struct deferred_flush_tables *deferred_flush;
 325
 326 /* bitmap for indexing intel_iommus */
 327 static int g_num_of_iommus;
 328
 329 static DEFINE_SPINLOCK(async_umap_flush_lock);
 330 static LIST_HEAD(unmaps_to_do);
 331
 332 static int timer_on;
 333 static long list_size;
 334
 335 static void domain_remove_dev_info(struct dmar_domain *domain);
 336
 337 #ifdef CONFIG_DMAR_DEFAULT_ON
 338 int dmar_disabled = 0;
 339 #else
 340 int dmar_disabled = 1;
 341 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 342
 343 static int __initdata dmar_map_gfx = 1;
 344 static int dmar_forcedac;
 345 static int intel_iommu_strict;
 346
 347 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 348 static DEFINE_SPINLOCK(device_domain_lock);
 349 static LIST_HEAD(device_domain_list);
 350
 351 static struct iommu_ops intel_iommu_ops;
 352
 353 static int __init intel_iommu_setup(char *str)
 354 {
 355         if (!str)
 356                 return -EINVAL;
 357         while (*str) {
 358                 if (!strncmp(str, "on", 2)) {
 359                         dmar_disabled = 0;
 360                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 361                 } else if (!strncmp(str, "off", 3)) {
 362                         dmar_disabled = 1;
 363                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 364                 } else if (!strncmp(str, "igfx_off", 8)) {
 365                         dmar_map_gfx = 0;
 366                         printk(KERN_INFO
 367                                 "Intel-IOMMU: disable GFX device mapping\n");
 368                 } else if (!strncmp(str, "forcedac", 8)) {
 369                         printk(KERN_INFO
 370                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 371                         dmar_forcedac = 1;
 372                 } else if (!strncmp(str, "strict", 6)) {
 373                         printk(KERN_INFO
 374                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 375                         intel_iommu_strict = 1;
 376                 }
 377
 378                 str += strcspn(str, ",");
 379                 while (*str == ',')
 380                         str++;
 381         }
 382         return 0;
 383 }
 384 __setup("intel_iommu=", intel_iommu_setup);
 385
 386 static struct kmem_cache *iommu_domain_cache;
 387 static struct kmem_cache *iommu_devinfo_cache;
 388 static struct kmem_cache *iommu_iova_cache;
 389
 390 static inline void *alloc_pgtable_page(int node)
 391 {
 392         struct page *page;
 393         void *vaddr = NULL;
 394
 395         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 396         if (page)
 397                 vaddr = page_address(page);
 398         return vaddr;
 399 }
 400
 401 static inline void free_pgtable_page(void *vaddr)
 402 {
 403         free_page((unsigned long)vaddr);
 404 }
 405
 406 static inline void *alloc_domain_mem(void)
 407 {
 408         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 409 }
 410
 411 static void free_domain_mem(void *vaddr)
 412 {
 413         kmem_cache_free(iommu_domain_cache, vaddr);
 414 }
 415
 416 static inline void * alloc_devinfo_mem(void)
 417 {
 418         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 419 }
 420
 421 static inline void free_devinfo_mem(void *vaddr)
 422 {
 423         kmem_cache_free(iommu_devinfo_cache, vaddr);
 424 }
 425
 426 struct iova *alloc_iova_mem(void)
 427 {
 428         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 429 }
 430
 431 void free_iova_mem(struct iova *iova)
 432 {
 433         kmem_cache_free(iommu_iova_cache, iova);
 434 }
 435
 436
 437 static inline int width_to_agaw(int width);
 438
 439 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 440 {
 441         unsigned long sagaw;
 442         int agaw = -1;
 443
 444         sagaw = cap_sagaw(iommu->cap);
 445         for (agaw = width_to_agaw(max_gaw);
 446              agaw >= 0; agaw--) {
 447                 if (test_bit(agaw, &sagaw))
 448                         break;
 449         }
 450
 451         return agaw;
 452 }
 453
 454 /*
 455  * Calculate max SAGAW for each iommu.
 456  */
 457 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 458 {
 459         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 460 }
 461
 462 /*
 463  * calculate agaw for each iommu.
 464  * "SAGAW" may be different across iommus, use a default agaw, and
 465  * get a supported less agaw for iommus that don't support the default agaw.
 466  */
 467 int iommu_calculate_agaw(struct intel_iommu *iommu)
 468 {
 469         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 470 }
 471
 472 /* This functionin only returns single iommu in a domain */
 473 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 474 {
 475         int iommu_id;
 476
 477         /* si_domain and vm domain should not get here. */
 478         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 479         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 480
 481         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 482         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 483                 return NULL;
 484
 485         return g_iommus[iommu_id];
 486 }
 487
 488 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 489 {
 490         int i;
 491
 492         domain->iommu_coherency = 1;
 493
 494         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 495         for (; i < g_num_of_iommus; ) {
 496                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 497                         domain->iommu_coherency = 0;
 498                         break;
 499                 }
 500                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 501         }
 502 }
 503
 504 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 505 {
 506         int i;
 507
 508         domain->iommu_snooping = 1;
 509
 510         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 511         for (; i < g_num_of_iommus; ) {
 512                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 513                         domain->iommu_snooping = 0;
 514                         break;
 515                 }
 516                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 517         }
 518 }
 519
 520 /* Some capabilities may be different across iommus */
 521 static void domain_update_iommu_cap(struct dmar_domain *domain)
 522 {
 523         domain_update_iommu_coherency(domain);
 524         domain_update_iommu_snooping(domain);
 525 }
 526
 527 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 528 {
 529         struct dmar_drhd_unit *drhd = NULL;
 530         int i;
 531
 532         for_each_drhd_unit(drhd) {
 533                 if (drhd->ignored)
 534                         continue;
 535                 if (segment != drhd->segment)
 536                         continue;
 537
 538                 for (i = 0; i < drhd->devices_cnt; i++) {
 539                         if (drhd->devices[i] &&
 540                             drhd->devices[i]->bus->number == bus &&
 541                             drhd->devices[i]->devfn == devfn)
 542                                 return drhd->iommu;
 543                         if (drhd->devices[i] &&
 544                             drhd->devices[i]->subordinate &&
 545                             drhd->devices[i]->subordinate->number <= bus &&
 546                             drhd->devices[i]->subordinate->subordinate >= bus)
 547                                 return drhd->iommu;
 548                 }
 549
 550                 if (drhd->include_all)
 551                         return drhd->iommu;
 552         }
 553
 554         return NULL;
 555 }
 556
 557 static void domain_flush_cache(struct dmar_domain *domain,
 558                                void *addr, int size)
 559 {
 560         if (!domain->iommu_coherency)
 561                 clflush_cache_range(addr, size);
 562 }
 563
 564 /* Gets context entry for a given bus and devfn */
 565 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 566                 u8 bus, u8 devfn)
 567 {
 568         struct root_entry *root;
 569         struct context_entry *context;
 570         unsigned long phy_addr;
 571         unsigned long flags;
 572
 573         spin_lock_irqsave(&iommu->lock, flags);
 574         root = &iommu->root_entry[bus];
 575         context = get_context_addr_from_root(root);
 576         if (!context) {
 577                 context = (struct context_entry *)
 578                                 alloc_pgtable_page(iommu->node);
 579                 if (!context) {
 580                         spin_unlock_irqrestore(&iommu->lock, flags);
 581                         return NULL;
 582                 }
 583                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 584                 phy_addr = virt_to_phys((void *)context);
 585                 set_root_value(root, phy_addr);
 586                 set_root_present(root);
 587                 __iommu_flush_cache(iommu, root, sizeof(*root));
 588         }
 589         spin_unlock_irqrestore(&iommu->lock, flags);
 590         return &context[devfn];
 591 }
 592
 593 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 594 {
 595         struct root_entry *root;
 596         struct context_entry *context;
 597         int ret;
 598         unsigned long flags;
 599
 600         spin_lock_irqsave(&iommu->lock, flags);
 601         root = &iommu->root_entry[bus];
 602         context = get_context_addr_from_root(root);
 603         if (!context) {
 604                 ret = 0;
 605                 goto out;
 606         }
 607         ret = context_present(&context[devfn]);
 608 out:
 609         spin_unlock_irqrestore(&iommu->lock, flags);
 610         return ret;
 611 }
 612
 613 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 614 {
 615         struct root_entry *root;
 616         struct context_entry *context;
 617         unsigned long flags;
 618
 619         spin_lock_irqsave(&iommu->lock, flags);
 620         root = &iommu->root_entry[bus];
 621         context = get_context_addr_from_root(root);
 622         if (context) {
 623                 context_clear_entry(&context[devfn]);
 624                 __iommu_flush_cache(iommu, &context[devfn], \
 625                         sizeof(*context));
 626         }
 627         spin_unlock_irqrestore(&iommu->lock, flags);
 628 }
 629
 630 static void free_context_table(struct intel_iommu *iommu)
 631 {
 632         struct root_entry *root;
 633         int i;
 634         unsigned long flags;
 635         struct context_entry *context;
 636
 637         spin_lock_irqsave(&iommu->lock, flags);
 638         if (!iommu->root_entry) {
 639                 goto out;
 640         }
 641         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 642                 root = &iommu->root_entry[i];
 643                 context = get_context_addr_from_root(root);
 644                 if (context)
 645                         free_pgtable_page(context);
 646         }
 647         free_pgtable_page(iommu->root_entry);
 648         iommu->root_entry = NULL;
 649 out:
 650         spin_unlock_irqrestore(&iommu->lock, flags);
 651 }
 652
 653 /* page table handling */
 654 #define LEVEL_STRIDE            (9)
 655 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 656
 657 static inline int agaw_to_level(int agaw)
 658 {
 659         return agaw + 2;
 660 }
 661
 662 static inline int agaw_to_width(int agaw)
 663 {
 664         return 30 + agaw * LEVEL_STRIDE;
 665
 666 }
 667
 668 static inline int width_to_agaw(int width)
 669 {
 670         return (width - 30) / LEVEL_STRIDE;
 671 }
 672
 673 static inline unsigned int level_to_offset_bits(int level)
 674 {
 675         return (level - 1) * LEVEL_STRIDE;
 676 }
 677
 678 static inline int pfn_level_offset(unsigned long pfn, int level)
 679 {
 680         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 681 }
 682
 683 static inline unsigned long level_mask(int level)
 684 {
 685         return -1UL << level_to_offset_bits(level);
 686 }
 687
 688 static inline unsigned long level_size(int level)
 689 {
 690         return 1UL << level_to_offset_bits(level);
 691 }
 692
 693 static inline unsigned long align_to_level(unsigned long pfn, int level)
 694 {
 695         return (pfn + level_size(level) - 1) & level_mask(level);
 696 }
 697
 698 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 699                                       unsigned long pfn)
 700 {
 701         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 702         struct dma_pte *parent, *pte = NULL;
 703         int level = agaw_to_level(domain->agaw);
 704         int offset;
 705
 706         BUG_ON(!domain->pgd);
 707         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 708         parent = domain->pgd;
 709
 710         while (level > 0) {
 711                 void *tmp_page;
 712
 713                 offset = pfn_level_offset(pfn, level);
 714                 pte = &parent[offset];
 715                 if (level == 1)
 716                         break;
 717
 718                 if (!dma_pte_present(pte)) {
 719                         uint64_t pteval;
 720
 721                         tmp_page = alloc_pgtable_page(domain->nid);
 722
 723                         if (!tmp_page)
 724                                 return NULL;
 725
 726                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 727                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 728                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 729                                 /* Someone else set it while we were thinking; use theirs. */
 730                                 free_pgtable_page(tmp_page);
 731                         } else {
 732                                 dma_pte_addr(pte);
 733                                 domain_flush_cache(domain, pte, sizeof(*pte));
 734                         }
 735                 }
 736                 parent = phys_to_virt(dma_pte_addr(pte));
 737                 level--;
 738         }
 739
 740         return pte;
 741 }
 742
 743 /* return address's pte at specific level */
 744 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 745                                          unsigned long pfn,
 746                                          int level)
 747 {
 748         struct dma_pte *parent, *pte = NULL;
 749         int total = agaw_to_level(domain->agaw);
 750         int offset;
 751
 752         parent = domain->pgd;
 753         while (level <= total) {
 754                 offset = pfn_level_offset(pfn, total);
 755                 pte = &parent[offset];
 756                 if (level == total)
 757                         return pte;
 758
 759                 if (!dma_pte_present(pte))
 760                         break;
 761                 parent = phys_to_virt(dma_pte_addr(pte));
 762                 total--;
 763         }
 764         return NULL;
 765 }
 766
 767 /* clear last level pte, a tlb flush should be followed */
 768 static void dma_pte_clear_range(struct dmar_domain *domain,
 769                                 unsigned long start_pfn,
 770                                 unsigned long last_pfn)
 771 {
 772         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 773         struct dma_pte *first_pte, *pte;
 774
 775         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 776         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 777         BUG_ON(start_pfn > last_pfn);
 778
 779         /* we don't need lock here; nobody else touches the iova range */
 780         do {
 781                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
 782                 if (!pte) {
 783                         start_pfn = align_to_level(start_pfn + 1, 2);
 784                         continue;
 785                 }
 786                 do {
 787                         dma_clear_pte(pte);
 788                         start_pfn++;
 789                         pte++;
 790                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 791
 792                 domain_flush_cache(domain, first_pte,
 793                                    (void *)pte - (void *)first_pte);
 794
 795         } while (start_pfn && start_pfn <= last_pfn);
 796 }
 797
 798 /* free page table pages. last level pte should already be cleared */
 799 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 800                                    unsigned long start_pfn,
 801                                    unsigned long last_pfn)
 802 {
 803         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 804         struct dma_pte *first_pte, *pte;
 805         int total = agaw_to_level(domain->agaw);
 806         int level;
 807         unsigned long tmp;
 808
 809         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 810         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 811         BUG_ON(start_pfn > last_pfn);
 812
 813         /* We don't need lock here; nobody else touches the iova range */
 814         level = 2;
 815         while (level <= total) {
 816                 tmp = align_to_level(start_pfn, level);
 817
 818                 /* If we can't even clear one PTE at this level, we're done */
 819                 if (tmp + level_size(level) - 1 > last_pfn)
 820                         return;
 821
 822                 do {
 823                         first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
 824                         if (!pte) {
 825                                 tmp = align_to_level(tmp + 1, level + 1);
 826                                 continue;
 827                         }
 828                         do {
 829                                 if (dma_pte_present(pte)) {
 830                                         free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 831                                         dma_clear_pte(pte);
 832                                 }
 833                                 pte++;
 834                                 tmp += level_size(level);
 835                         } while (!first_pte_in_page(pte) &&
 836                                  tmp + level_size(level) - 1 <= last_pfn);
 837
 838                         domain_flush_cache(domain, first_pte,
 839                                            (void *)pte - (void *)first_pte);
 840
 841                 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 842                 level++;
 843         }
 844         /* free pgd */
 845         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 846                 free_pgtable_page(domain->pgd);
 847                 domain->pgd = NULL;
 848         }
 849 }
 850
 851 /* iommu handling */
 852 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 853 {
 854         struct root_entry *root;
 855         unsigned long flags;
 856
 857         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 858         if (!root)
 859                 return -ENOMEM;
 860
 861         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 862
 863         spin_lock_irqsave(&iommu->lock, flags);
 864         iommu->root_entry = root;
 865         spin_unlock_irqrestore(&iommu->lock, flags);
 866
 867         return 0;
 868 }
 869
 870 static void iommu_set_root_entry(struct intel_iommu *iommu)
 871 {
 872         void *addr;
 873         u32 sts;
 874         unsigned long flag;
 875
 876         addr = iommu->root_entry;
 877
 878         spin_lock_irqsave(&iommu->register_lock, flag);
 879         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 880
 881         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 882
 883         /* Make sure hardware complete it */
 884         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 885                       readl, (sts & DMA_GSTS_RTPS), sts);
 886
 887         spin_unlock_irqrestore(&iommu->register_lock, flag);
 888 }
 889
 890 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 891 {
 892         u32 val;
 893         unsigned long flag;
 894
 895         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 896                 return;
 897
 898         spin_lock_irqsave(&iommu->register_lock, flag);
 899         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 900
 901         /* Make sure hardware complete it */
 902         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 903                       readl, (!(val & DMA_GSTS_WBFS)), val);
 904
 905         spin_unlock_irqrestore(&iommu->register_lock, flag);
 906 }
 907
 908 /* return value determine if we need a write buffer flush */
 909 static void __iommu_flush_context(struct intel_iommu *iommu,
 910                                   u16 did, u16 source_id, u8 function_mask,
 911                                   u64 type)
 912 {
 913         u64 val = 0;
 914         unsigned long flag;
 915
 916         switch (type) {
 917         case DMA_CCMD_GLOBAL_INVL:
 918                 val = DMA_CCMD_GLOBAL_INVL;
 919                 break;
 920         case DMA_CCMD_DOMAIN_INVL:
 921                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 922                 break;
 923         case DMA_CCMD_DEVICE_INVL:
 924                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 925                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 926                 break;
 927         default:
 928                 BUG();
 929         }
 930         val |= DMA_CCMD_ICC;
 931
 932         spin_lock_irqsave(&iommu->register_lock, flag);
 933         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 934
 935         /* Make sure hardware complete it */
 936         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 937                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 938
 939         spin_unlock_irqrestore(&iommu->register_lock, flag);
 940 }
 941
 942 /* return value determine if we need a write buffer flush */
 943 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 944                                 u64 addr, unsigned int size_order, u64 type)
 945 {
 946         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 947         u64 val = 0, val_iva = 0;
 948         unsigned long flag;
 949
 950         switch (type) {
 951         case DMA_TLB_GLOBAL_FLUSH:
 952                 /* global flush doesn't need set IVA_REG */
 953                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 954                 break;
 955         case DMA_TLB_DSI_FLUSH:
 956                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 957                 break;
 958         case DMA_TLB_PSI_FLUSH:
 959                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 960                 /* Note: always flush non-leaf currently */
 961                 val_iva = size_order | addr;
 962                 break;
 963         default:
 964                 BUG();
 965         }
 966         /* Note: set drain read/write */
 967 #if 0
 968         /*
 969          * This is probably to be super secure.. Looks like we can
 970          * ignore it without any impact.
 971          */
 972         if (cap_read_drain(iommu->cap))
 973                 val |= DMA_TLB_READ_DRAIN;
 974 #endif
 975         if (cap_write_drain(iommu->cap))
 976                 val |= DMA_TLB_WRITE_DRAIN;
 977
 978         spin_lock_irqsave(&iommu->register_lock, flag);
 979         /* Note: Only uses first TLB reg currently */
 980         if (val_iva)
 981                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 982         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 983
 984         /* Make sure hardware complete it */
 985         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 986                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 987
 988         spin_unlock_irqrestore(&iommu->register_lock, flag);
 989
 990         /* check IOTLB invalidation granularity */
 991         if (DMA_TLB_IAIG(val) == 0)
 992                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 993         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 994                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 995                         (unsigned long long)DMA_TLB_IIRG(type),
 996                         (unsigned long long)DMA_TLB_IAIG(val));
 997 }
 998
 999 static struct device_domain_info *iommu_support_dev_iotlb(
1000         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1001 {
1002         int found = 0;
1003         unsigned long flags;
1004         struct device_domain_info *info;
1005         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1006
1007         if (!ecap_dev_iotlb_support(iommu->ecap))
1008                 return NULL;
1009
1010         if (!iommu->qi)
1011                 return NULL;
1012
1013         spin_lock_irqsave(&device_domain_lock, flags);
1014         list_for_each_entry(info, &domain->devices, link)
1015                 if (info->bus == bus && info->devfn == devfn) {
1016                         found = 1;
1017                         break;
1018                 }
1019         spin_unlock_irqrestore(&device_domain_lock, flags);
1020
1021         if (!found || !info->dev)
1022                 return NULL;
1023
1024         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1025                 return NULL;
1026
1027         if (!dmar_find_matched_atsr_unit(info->dev))
1028                 return NULL;
1029
1030         info->iommu = iommu;
1031
1032         return info;
1033 }
1034
1035 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1036 {
1037         if (!info)
1038                 return;
1039
1040         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1041 }
1042
1043 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1044 {
1045         if (!info->dev || !pci_ats_enabled(info->dev))
1046                 return;
1047
1048         pci_disable_ats(info->dev);
1049 }
1050
1051 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1052                                   u64 addr, unsigned mask)
1053 {
1054         u16 sid, qdep;
1055         unsigned long flags;
1056         struct device_domain_info *info;
1057
1058         spin_lock_irqsave(&device_domain_lock, flags);
1059         list_for_each_entry(info, &domain->devices, link) {
1060                 if (!info->dev || !pci_ats_enabled(info->dev))
1061                         continue;
1062
1063                 sid = info->bus << 8 | info->devfn;
1064                 qdep = pci_ats_queue_depth(info->dev);
1065                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1066         }
1067         spin_unlock_irqrestore(&device_domain_lock, flags);
1068 }
1069
1070 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1071                                   unsigned long pfn, unsigned int pages)
1072 {
1073         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1074         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1075
1076         BUG_ON(pages == 0);
1077
1078         /*
1079          * Fallback to domain selective flush if no PSI support or the size is
1080          * too big.
1081          * PSI requires page size to be 2 ^ x, and the base address is naturally
1082          * aligned to the size
1083          */
1084         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1085                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1086                                                 DMA_TLB_DSI_FLUSH);
1087         else
1088                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1089                                                 DMA_TLB_PSI_FLUSH);
1090
1091         /*
1092          * In caching mode, domain ID 0 is reserved for non-present to present
1093          * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1094          */
1095         if (!cap_caching_mode(iommu->cap) || did)
1096                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1097 }
1098
1099 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1100 {
1101         u32 pmen;
1102         unsigned long flags;
1103
1104         spin_lock_irqsave(&iommu->register_lock, flags);
1105         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1106         pmen &= ~DMA_PMEN_EPM;
1107         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1108
1109         /* wait for the protected region status bit to clear */
1110         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1111                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1112
1113         spin_unlock_irqrestore(&iommu->register_lock, flags);
1114 }
1115
1116 static int iommu_enable_translation(struct intel_iommu *iommu)
1117 {
1118         u32 sts;
1119         unsigned long flags;
1120
1121         spin_lock_irqsave(&iommu->register_lock, flags);
1122         iommu->gcmd |= DMA_GCMD_TE;
1123         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1124
1125         /* Make sure hardware complete it */
1126         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1127                       readl, (sts & DMA_GSTS_TES), sts);
1128
1129         spin_unlock_irqrestore(&iommu->register_lock, flags);
1130         return 0;
1131 }
1132
1133 static int iommu_disable_translation(struct intel_iommu *iommu)
1134 {
1135         u32 sts;
1136         unsigned long flag;
1137
1138         spin_lock_irqsave(&iommu->register_lock, flag);
1139         iommu->gcmd &= ~DMA_GCMD_TE;
1140         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1141
1142         /* Make sure hardware complete it */
1143         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1144                       readl, (!(sts & DMA_GSTS_TES)), sts);
1145
1146         spin_unlock_irqrestore(&iommu->register_lock, flag);
1147         return 0;
1148 }
1149
1150
1151 static int iommu_init_domains(struct intel_iommu *iommu)
1152 {
1153         unsigned long ndomains;
1154         unsigned long nlongs;
1155
1156         ndomains = cap_ndoms(iommu->cap);
1157         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1158         nlongs = BITS_TO_LONGS(ndomains);
1159
1160         spin_lock_init(&iommu->lock);
1161
1162         /* TBD: there might be 64K domains,
1163          * consider other allocation for future chip
1164          */
1165         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1166         if (!iommu->domain_ids) {
1167                 printk(KERN_ERR "Allocating domain id array failed\n");
1168                 return -ENOMEM;
1169         }
1170         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1171                         GFP_KERNEL);
1172         if (!iommu->domains) {
1173                 printk(KERN_ERR "Allocating domain array failed\n");
1174                 return -ENOMEM;
1175         }
1176
1177         /*
1178          * if Caching mode is set, then invalid translations are tagged
1179          * with domainid 0. Hence we need to pre-allocate it.
1180          */
1181         if (cap_caching_mode(iommu->cap))
1182                 set_bit(0, iommu->domain_ids);
1183         return 0;
1184 }
1185
1186
1187 static void domain_exit(struct dmar_domain *domain);
1188 static void vm_domain_exit(struct dmar_domain *domain);
1189
1190 void free_dmar_iommu(struct intel_iommu *iommu)
1191 {
1192         struct dmar_domain *domain;
1193         int i;
1194         unsigned long flags;
1195
1196         if ((iommu->domains) && (iommu->domain_ids)) {
1197                 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1198                 for (; i < cap_ndoms(iommu->cap); ) {
1199                         domain = iommu->domains[i];
1200                         clear_bit(i, iommu->domain_ids);
1201
1202                         spin_lock_irqsave(&domain->iommu_lock, flags);
1203                         if (--domain->iommu_count == 0) {
1204                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1205                                         vm_domain_exit(domain);
1206                                 else
1207                                         domain_exit(domain);
1208                         }
1209                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1210
1211                         i = find_next_bit(iommu->domain_ids,
1212                                 cap_ndoms(iommu->cap), i+1);
1213                 }
1214         }
1215
1216         if (iommu->gcmd & DMA_GCMD_TE)
1217                 iommu_disable_translation(iommu);
1218
1219         if (iommu->irq) {
1220                 set_irq_data(iommu->irq, NULL);
1221                 /* This will mask the irq */
1222                 free_irq(iommu->irq, iommu);
1223                 destroy_irq(iommu->irq);
1224         }
1225
1226         kfree(iommu->domains);
1227         kfree(iommu->domain_ids);
1228
1229         g_iommus[iommu->seq_id] = NULL;
1230
1231         /* if all iommus are freed, free g_iommus */
1232         for (i = 0; i < g_num_of_iommus; i++) {
1233                 if (g_iommus[i])
1234                         break;
1235         }
1236
1237         if (i == g_num_of_iommus)
1238                 kfree(g_iommus);
1239
1240         /* free context mapping */
1241         free_context_table(iommu);
1242 }
1243
1244 static struct dmar_domain *alloc_domain(void)
1245 {
1246         struct dmar_domain *domain;
1247
1248         domain = alloc_domain_mem();
1249         if (!domain)
1250                 return NULL;
1251
1252         domain->nid = -1;
1253         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1254         domain->flags = 0;
1255
1256         return domain;
1257 }
1258
1259 static int iommu_attach_domain(struct dmar_domain *domain,
1260                                struct intel_iommu *iommu)
1261 {
1262         int num;
1263         unsigned long ndomains;
1264         unsigned long flags;
1265
1266         ndomains = cap_ndoms(iommu->cap);
1267
1268         spin_lock_irqsave(&iommu->lock, flags);
1269
1270         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1271         if (num >= ndomains) {
1272                 spin_unlock_irqrestore(&iommu->lock, flags);
1273                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1274                 return -ENOMEM;
1275         }
1276
1277         domain->id = num;
1278         set_bit(num, iommu->domain_ids);
1279         set_bit(iommu->seq_id, &domain->iommu_bmp);
1280         iommu->domains[num] = domain;
1281         spin_unlock_irqrestore(&iommu->lock, flags);
1282
1283         return 0;
1284 }
1285
1286 static void iommu_detach_domain(struct dmar_domain *domain,
1287                                 struct intel_iommu *iommu)
1288 {
1289         unsigned long flags;
1290         int num, ndomains;
1291         int found = 0;
1292
1293         spin_lock_irqsave(&iommu->lock, flags);
1294         ndomains = cap_ndoms(iommu->cap);
1295         num = find_first_bit(iommu->domain_ids, ndomains);
1296         for (; num < ndomains; ) {
1297                 if (iommu->domains[num] == domain) {
1298                         found = 1;
1299                         break;
1300                 }
1301                 num = find_next_bit(iommu->domain_ids,
1302                                     cap_ndoms(iommu->cap), num+1);
1303         }
1304
1305         if (found) {
1306                 clear_bit(num, iommu->domain_ids);
1307                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1308                 iommu->domains[num] = NULL;
1309         }
1310         spin_unlock_irqrestore(&iommu->lock, flags);
1311 }
1312
1313 static struct iova_domain reserved_iova_list;
1314 static struct lock_class_key reserved_rbtree_key;
1315
1316 static void dmar_init_reserved_ranges(void)
1317 {
1318         struct pci_dev *pdev = NULL;
1319         struct iova *iova;
1320         int i;
1321
1322         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1323
1324         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1325                 &reserved_rbtree_key);
1326
1327         /* IOAPIC ranges shouldn't be accessed by DMA */
1328         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1329                 IOVA_PFN(IOAPIC_RANGE_END));
1330         if (!iova)
1331                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1332
1333         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1334         for_each_pci_dev(pdev) {
1335                 struct resource *r;
1336
1337                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1338                         r = &pdev->resource[i];
1339                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1340                                 continue;
1341                         iova = reserve_iova(&reserved_iova_list,
1342                                             IOVA_PFN(r->start),
1343                                             IOVA_PFN(r->end));
1344                         if (!iova)
1345                                 printk(KERN_ERR "Reserve iova failed\n");
1346                 }
1347         }
1348
1349 }
1350
1351 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1352 {
1353         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1354 }
1355
1356 static inline int guestwidth_to_adjustwidth(int gaw)
1357 {
1358         int agaw;
1359         int r = (gaw - 12) % 9;
1360
1361         if (r == 0)
1362                 agaw = gaw;
1363         else
1364                 agaw = gaw + 9 - r;
1365         if (agaw > 64)
1366                 agaw = 64;
1367         return agaw;
1368 }
1369
1370 static int domain_init(struct dmar_domain *domain, int guest_width)
1371 {
1372         struct intel_iommu *iommu;
1373         int adjust_width, agaw;
1374         unsigned long sagaw;
1375
1376         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1377         spin_lock_init(&domain->iommu_lock);
1378
1379         domain_reserve_special_ranges(domain);
1380
1381         /* calculate AGAW */
1382         iommu = domain_get_iommu(domain);
1383         if (guest_width > cap_mgaw(iommu->cap))
1384                 guest_width = cap_mgaw(iommu->cap);
1385         domain->gaw = guest_width;
1386         adjust_width = guestwidth_to_adjustwidth(guest_width);
1387         agaw = width_to_agaw(adjust_width);
1388         sagaw = cap_sagaw(iommu->cap);
1389         if (!test_bit(agaw, &sagaw)) {
1390                 /* hardware doesn't support it, choose a bigger one */
1391                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1392                 agaw = find_next_bit(&sagaw, 5, agaw);
1393                 if (agaw >= 5)
1394                         return -ENODEV;
1395         }
1396         domain->agaw = agaw;
1397         INIT_LIST_HEAD(&domain->devices);
1398
1399         if (ecap_coherent(iommu->ecap))
1400                 domain->iommu_coherency = 1;
1401         else
1402                 domain->iommu_coherency = 0;
1403
1404         if (ecap_sc_support(iommu->ecap))
1405                 domain->iommu_snooping = 1;
1406         else
1407                 domain->iommu_snooping = 0;
1408
1409         domain->iommu_count = 1;
1410         domain->nid = iommu->node;
1411
1412         /* always allocate the top pgd */
1413         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1414         if (!domain->pgd)
1415                 return -ENOMEM;
1416         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1417         return 0;
1418 }
1419
1420 static void domain_exit(struct dmar_domain *domain)
1421 {
1422         struct dmar_drhd_unit *drhd;
1423         struct intel_iommu *iommu;
1424
1425         /* Domain 0 is reserved, so dont process it */
1426         if (!domain)
1427                 return;
1428
1429         domain_remove_dev_info(domain);
1430         /* destroy iovas */
1431         put_iova_domain(&domain->iovad);
1432
1433         /* clear ptes */
1434         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1435
1436         /* free page tables */
1437         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1438
1439         for_each_active_iommu(iommu, drhd)
1440                 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1441                         iommu_detach_domain(domain, iommu);
1442
1443         free_domain_mem(domain);
1444 }
1445
1446 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1447                                  u8 bus, u8 devfn, int translation)
1448 {
1449         struct context_entry *context;
1450         unsigned long flags;
1451         struct intel_iommu *iommu;
1452         struct dma_pte *pgd;
1453         unsigned long num;
1454         unsigned long ndomains;
1455         int id;
1456         int agaw;
1457         struct device_domain_info *info = NULL;
1458
1459         pr_debug("Set context mapping for %02x:%02x.%d\n",
1460                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1461
1462         BUG_ON(!domain->pgd);
1463         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1464                translation != CONTEXT_TT_MULTI_LEVEL);
1465
1466         iommu = device_to_iommu(segment, bus, devfn);
1467         if (!iommu)
1468                 return -ENODEV;
1469
1470         context = device_to_context_entry(iommu, bus, devfn);
1471         if (!context)
1472                 return -ENOMEM;
1473         spin_lock_irqsave(&iommu->lock, flags);
1474         if (context_present(context)) {
1475                 spin_unlock_irqrestore(&iommu->lock, flags);
1476                 return 0;
1477         }
1478
1479         id = domain->id;
1480         pgd = domain->pgd;
1481
1482         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1483             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1484                 int found = 0;
1485
1486                 /* find an available domain id for this device in iommu */
1487                 ndomains = cap_ndoms(iommu->cap);
1488                 num = find_first_bit(iommu->domain_ids, ndomains);
1489                 for (; num < ndomains; ) {
1490                         if (iommu->domains[num] == domain) {
1491                                 id = num;
1492                                 found = 1;
1493                                 break;
1494                         }
1495                         num = find_next_bit(iommu->domain_ids,
1496                                             cap_ndoms(iommu->cap), num+1);
1497                 }
1498
1499                 if (found == 0) {
1500                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1501                         if (num >= ndomains) {
1502                                 spin_unlock_irqrestore(&iommu->lock, flags);
1503                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1504                                 return -EFAULT;
1505                         }
1506
1507                         set_bit(num, iommu->domain_ids);
1508                         iommu->domains[num] = domain;
1509                         id = num;
1510                 }
1511
1512                 /* Skip top levels of page tables for
1513                  * iommu which has less agaw than default.
1514                  * Unnecessary for PT mode.
1515                  */
1516                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1517                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1518                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1519                                 if (!dma_pte_present(pgd)) {
1520                                         spin_unlock_irqrestore(&iommu->lock, flags);
1521                                         return -ENOMEM;
1522                                 }
1523                         }
1524                 }
1525         }
1526
1527         context_set_domain_id(context, id);
1528
1529         if (translation != CONTEXT_TT_PASS_THROUGH) {
1530                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1531                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1532                                      CONTEXT_TT_MULTI_LEVEL;
1533         }
1534         /*
1535          * In pass through mode, AW must be programmed to indicate the largest
1536          * AGAW value supported by hardware. And ASR is ignored by hardware.
1537          */
1538         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1539                 context_set_address_width(context, iommu->msagaw);
1540         else {
1541                 context_set_address_root(context, virt_to_phys(pgd));
1542                 context_set_address_width(context, iommu->agaw);
1543         }
1544
1545         context_set_translation_type(context, translation);
1546         context_set_fault_enable(context);
1547         context_set_present(context);
1548         domain_flush_cache(domain, context, sizeof(*context));
1549
1550         /*
1551          * It's a non-present to present mapping. If hardware doesn't cache
1552          * non-present entry we only need to flush the write-buffer. If the
1553          * _does_ cache non-present entries, then it does so in the special
1554          * domain #0, which we have to flush:
1555          */
1556         if (cap_caching_mode(iommu->cap)) {
1557                 iommu->flush.flush_context(iommu, 0,
1558                                            (((u16)bus) << 8) | devfn,
1559                                            DMA_CCMD_MASK_NOBIT,
1560                                            DMA_CCMD_DEVICE_INVL);
1561                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
1562         } else {
1563                 iommu_flush_write_buffer(iommu);
1564         }
1565         iommu_enable_dev_iotlb(info);
1566         spin_unlock_irqrestore(&iommu->lock, flags);
1567
1568         spin_lock_irqsave(&domain->iommu_lock, flags);
1569         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1570                 domain->iommu_count++;
1571                 if (domain->iommu_count == 1)
1572                         domain->nid = iommu->node;
1573                 domain_update_iommu_cap(domain);
1574         }
1575         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1576         return 0;
1577 }
1578
1579 static int
1580 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1581                         int translation)
1582 {
1583         int ret;
1584         struct pci_dev *tmp, *parent;
1585
1586         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1587                                          pdev->bus->number, pdev->devfn,
1588                                          translation);
1589         if (ret)
1590                 return ret;
1591
1592         /* dependent device mapping */
1593         tmp = pci_find_upstream_pcie_bridge(pdev);
1594         if (!tmp)
1595                 return 0;
1596         /* Secondary interface's bus number and devfn 0 */
1597         parent = pdev->bus->self;
1598         while (parent != tmp) {
1599                 ret = domain_context_mapping_one(domain,
1600                                                  pci_domain_nr(parent->bus),
1601                                                  parent->bus->number,
1602                                                  parent->devfn, translation);
1603                 if (ret)
1604                         return ret;
1605                 parent = parent->bus->self;
1606         }
1607         if (pci_is_pcie(tmp)) /* this is a PCIE-to-PCI bridge */
1608                 return domain_context_mapping_one(domain,
1609                                         pci_domain_nr(tmp->subordinate),
1610                                         tmp->subordinate->number, 0,
1611                                         translation);
1612         else /* this is a legacy PCI bridge */
1613                 return domain_context_mapping_one(domain,
1614                                                   pci_domain_nr(tmp->bus),
1615                                                   tmp->bus->number,
1616                                                   tmp->devfn,
1617                                                   translation);
1618 }
1619
1620 static int domain_context_mapped(struct pci_dev *pdev)
1621 {
1622         int ret;
1623         struct pci_dev *tmp, *parent;
1624         struct intel_iommu *iommu;
1625
1626         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1627                                 pdev->devfn);
1628         if (!iommu)
1629                 return -ENODEV;
1630
1631         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1632         if (!ret)
1633                 return ret;
1634         /* dependent device mapping */
1635         tmp = pci_find_upstream_pcie_bridge(pdev);
1636         if (!tmp)
1637                 return ret;
1638         /* Secondary interface's bus number and devfn 0 */
1639         parent = pdev->bus->self;
1640         while (parent != tmp) {
1641                 ret = device_context_mapped(iommu, parent->bus->number,
1642                                             parent->devfn);
1643                 if (!ret)
1644                         return ret;
1645                 parent = parent->bus->self;
1646         }
1647         if (pci_is_pcie(tmp))
1648                 return device_context_mapped(iommu, tmp->subordinate->number,
1649                                              0);
1650         else
1651                 return device_context_mapped(iommu, tmp->bus->number,
1652                                              tmp->devfn);
1653 }
1654
1655 /* Returns a number of VTD pages, but aligned to MM page size */
1656 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1657                                             size_t size)
1658 {
1659         host_addr &= ~PAGE_MASK;
1660         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1661 }
1662
1663 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1664                             struct scatterlist *sg, unsigned long phys_pfn,
1665                             unsigned long nr_pages, int prot)
1666 {
1667         struct dma_pte *first_pte = NULL, *pte = NULL;
1668         phys_addr_t uninitialized_var(pteval);
1669         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1670         unsigned long sg_res;
1671
1672         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1673
1674         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1675                 return -EINVAL;
1676
1677         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1678
1679         if (sg)
1680                 sg_res = 0;
1681         else {
1682                 sg_res = nr_pages + 1;
1683                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1684         }
1685
1686         while (nr_pages--) {
1687                 uint64_t tmp;
1688
1689                 if (!sg_res) {
1690                         sg_res = aligned_nrpages(sg->offset, sg->length);
1691                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1692                         sg->dma_length = sg->length;
1693                         pteval = page_to_phys(sg_page(sg)) | prot;
1694                 }
1695                 if (!pte) {
1696                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1697                         if (!pte)
1698                                 return -ENOMEM;
1699                 }
1700                 /* We don't need lock here, nobody else
1701                  * touches the iova range
1702                  */
1703                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1704                 if (tmp) {
1705                         static int dumps = 5;
1706                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1707                                iov_pfn, tmp, (unsigned long long)pteval);
1708                         if (dumps) {
1709                                 dumps--;
1710                                 debug_dma_dump_mappings(NULL);
1711                         }
1712                         WARN_ON(1);
1713                 }
1714                 pte++;
1715                 if (!nr_pages || first_pte_in_page(pte)) {
1716                         domain_flush_cache(domain, first_pte,
1717                                            (void *)pte - (void *)first_pte);
1718                         pte = NULL;
1719                 }
1720                 iov_pfn++;
1721                 pteval += VTD_PAGE_SIZE;
1722                 sg_res--;
1723                 if (!sg_res)
1724                         sg = sg_next(sg);
1725         }
1726         return 0;
1727 }
1728
1729 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1730                                     struct scatterlist *sg, unsigned long nr_pages,
1731                                     int prot)
1732 {
1733         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1734 }
1735
1736 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1737                                      unsigned long phys_pfn, unsigned long nr_pages,
1738                                      int prot)
1739 {
1740         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1741 }
1742
1743 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1744 {
1745         if (!iommu)
1746                 return;
1747
1748         clear_context_table(iommu, bus, devfn);
1749         iommu->flush.flush_context(iommu, 0, 0, 0,
1750                                            DMA_CCMD_GLOBAL_INVL);
1751         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1752 }
1753
1754 static void domain_remove_dev_info(struct dmar_domain *domain)
1755 {
1756         struct device_domain_info *info;
1757         unsigned long flags;
1758         struct intel_iommu *iommu;
1759
1760         spin_lock_irqsave(&device_domain_lock, flags);
1761         while (!list_empty(&domain->devices)) {
1762                 info = list_entry(domain->devices.next,
1763                         struct device_domain_info, link);
1764                 list_del(&info->link);
1765                 list_del(&info->global);
1766                 if (info->dev)
1767                         info->dev->dev.archdata.iommu = NULL;
1768                 spin_unlock_irqrestore(&device_domain_lock, flags);
1769
1770                 iommu_disable_dev_iotlb(info);
1771                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1772                 iommu_detach_dev(iommu, info->bus, info->devfn);
1773                 free_devinfo_mem(info);
1774
1775                 spin_lock_irqsave(&device_domain_lock, flags);
1776         }
1777         spin_unlock_irqrestore(&device_domain_lock, flags);
1778 }
1779
1780 /*
1781  * find_domain
1782  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1783  */
1784 static struct dmar_domain *
1785 find_domain(struct pci_dev *pdev)
1786 {
1787         struct device_domain_info *info;
1788
1789         /* No lock here, assumes no domain exit in normal case */
1790         info = pdev->dev.archdata.iommu;
1791         if (info)
1792                 return info->domain;
1793         return NULL;
1794 }
1795
1796 /* domain is initialized */
1797 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1798 {
1799         struct dmar_domain *domain, *found = NULL;
1800         struct intel_iommu *iommu;
1801         struct dmar_drhd_unit *drhd;
1802         struct device_domain_info *info, *tmp;
1803         struct pci_dev *dev_tmp;
1804         unsigned long flags;
1805         int bus = 0, devfn = 0;
1806         int segment;
1807         int ret;
1808
1809         domain = find_domain(pdev);
1810         if (domain)
1811                 return domain;
1812
1813         segment = pci_domain_nr(pdev->bus);
1814
1815         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1816         if (dev_tmp) {
1817                 if (pci_is_pcie(dev_tmp)) {
1818                         bus = dev_tmp->subordinate->number;
1819                         devfn = 0;
1820                 } else {
1821                         bus = dev_tmp->bus->number;
1822                         devfn = dev_tmp->devfn;
1823                 }
1824                 spin_lock_irqsave(&device_domain_lock, flags);
1825                 list_for_each_entry(info, &device_domain_list, global) {
1826                         if (info->segment == segment &&
1827                             info->bus == bus && info->devfn == devfn) {
1828                                 found = info->domain;
1829                                 break;
1830                         }
1831                 }
1832                 spin_unlock_irqrestore(&device_domain_lock, flags);
1833                 /* pcie-pci bridge already has a domain, uses it */
1834                 if (found) {
1835                         domain = found;
1836                         goto found_domain;
1837                 }
1838         }
1839
1840         domain = alloc_domain();
1841         if (!domain)
1842                 goto error;
1843
1844         /* Allocate new domain for the device */
1845         drhd = dmar_find_matched_drhd_unit(pdev);
1846         if (!drhd) {
1847                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1848                         pci_name(pdev));
1849                 return NULL;
1850         }
1851         iommu = drhd->iommu;
1852
1853         ret = iommu_attach_domain(domain, iommu);
1854         if (ret) {
1855                 domain_exit(domain);
1856                 goto error;
1857         }
1858
1859         if (domain_init(domain, gaw)) {
1860                 domain_exit(domain);
1861                 goto error;
1862         }
1863
1864         /* register pcie-to-pci device */
1865         if (dev_tmp) {
1866                 info = alloc_devinfo_mem();
1867                 if (!info) {
1868                         domain_exit(domain);
1869                         goto error;
1870                 }
1871                 info->segment = segment;
1872                 info->bus = bus;
1873                 info->devfn = devfn;
1874                 info->dev = NULL;
1875                 info->domain = domain;
1876                 /* This domain is shared by devices under p2p bridge */
1877                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1878
1879                 /* pcie-to-pci bridge already has a domain, uses it */
1880                 found = NULL;
1881                 spin_lock_irqsave(&device_domain_lock, flags);
1882                 list_for_each_entry(tmp, &device_domain_list, global) {
1883                         if (tmp->segment == segment &&
1884                             tmp->bus == bus && tmp->devfn == devfn) {
1885                                 found = tmp->domain;
1886                                 break;
1887                         }
1888                 }
1889                 if (found) {
1890                         free_devinfo_mem(info);
1891                         domain_exit(domain);
1892                         domain = found;
1893                 } else {
1894                         list_add(&info->link, &domain->devices);
1895                         list_add(&info->global, &device_domain_list);
1896                 }
1897                 spin_unlock_irqrestore(&device_domain_lock, flags);
1898         }
1899
1900 found_domain:
1901         info = alloc_devinfo_mem();
1902         if (!info)
1903                 goto error;
1904         info->segment = segment;
1905         info->bus = pdev->bus->number;
1906         info->devfn = pdev->devfn;
1907         info->dev = pdev;
1908         info->domain = domain;
1909         spin_lock_irqsave(&device_domain_lock, flags);
1910         /* somebody is fast */
1911         found = find_domain(pdev);
1912         if (found != NULL) {
1913                 spin_unlock_irqrestore(&device_domain_lock, flags);
1914                 if (found != domain) {
1915                         domain_exit(domain);
1916                         domain = found;
1917                 }
1918                 free_devinfo_mem(info);
1919                 return domain;
1920         }
1921         list_add(&info->link, &domain->devices);
1922         list_add(&info->global, &device_domain_list);
1923         pdev->dev.archdata.iommu = info;
1924         spin_unlock_irqrestore(&device_domain_lock, flags);
1925         return domain;
1926 error:
1927         /* recheck it here, maybe others set it */
1928         return find_domain(pdev);
1929 }
1930
1931 static int iommu_identity_mapping;
1932 #define IDENTMAP_ALL            1
1933 #define IDENTMAP_GFX            2
1934 #define IDENTMAP_AZALIA         4
1935
1936 static int iommu_domain_identity_map(struct dmar_domain *domain,
1937                                      unsigned long long start,
1938                                      unsigned long long end)
1939 {
1940         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1941         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1942
1943         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1944                           dma_to_mm_pfn(last_vpfn))) {
1945                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1946                 return -ENOMEM;
1947         }
1948
1949         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1950                  start, end, domain->id);
1951         /*
1952          * RMRR range might have overlap with physical memory range,
1953          * clear it first
1954          */
1955         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1956
1957         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1958                                   last_vpfn - first_vpfn + 1,
1959                                   DMA_PTE_READ|DMA_PTE_WRITE);
1960 }
1961
1962 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1963                                       unsigned long long start,
1964                                       unsigned long long end)
1965 {
1966         struct dmar_domain *domain;
1967         int ret;
1968
1969         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1970         if (!domain)
1971                 return -ENOMEM;
1972
1973         /* For _hardware_ passthrough, don't bother. But for software
1974            passthrough, we do it anyway -- it may indicate a memory
1975            range which is reserved in E820, so which didn't get set
1976            up to start with in si_domain */
1977         if (domain == si_domain && hw_pass_through) {
1978                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1979                        pci_name(pdev), start, end);
1980                 return 0;
1981         }
1982
1983         printk(KERN_INFO
1984                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1985                pci_name(pdev), start, end);
1986
1987         if (end < start) {
1988                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1989                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1990                         dmi_get_system_info(DMI_BIOS_VENDOR),
1991                         dmi_get_system_info(DMI_BIOS_VERSION),
1992                      dmi_get_system_info(DMI_PRODUCT_VERSION));
1993                 ret = -EIO;
1994                 goto error;
1995         }
1996
1997         if (end >> agaw_to_width(domain->agaw)) {
1998                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1999                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2000                      agaw_to_width(domain->agaw),
2001                      dmi_get_system_info(DMI_BIOS_VENDOR),
2002                      dmi_get_system_info(DMI_BIOS_VERSION),
2003                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2004                 ret = -EIO;
2005                 goto error;
2006         }
2007
2008         ret = iommu_domain_identity_map(domain, start, end);
2009         if (ret)
2010                 goto error;
2011
2012         /* context entry init */
2013         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2014         if (ret)
2015                 goto error;
2016
2017         return 0;
2018
2019  error:
2020         domain_exit(domain);
2021         return ret;
2022 }
2023
2024 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2025         struct pci_dev *pdev)
2026 {
2027         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2028                 return 0;
2029         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2030                 rmrr->end_address + 1);
2031 }
2032
2033 #ifdef CONFIG_DMAR_FLOPPY_WA
2034 static inline void iommu_prepare_isa(void)
2035 {
2036         struct pci_dev *pdev;
2037         int ret;
2038
2039         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2040         if (!pdev)
2041                 return;
2042
2043         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2044         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2045
2046         if (ret)
2047                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2048                        "floppy might not work\n");
2049
2050 }
2051 #else
2052 static inline void iommu_prepare_isa(void)
2053 {
2054         return;
2055 }
2056 #endif /* !CONFIG_DMAR_FLPY_WA */
2057
2058 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2059
2060 static int __init si_domain_work_fn(unsigned long start_pfn,
2061                                     unsigned long end_pfn, void *datax)
2062 {
2063         int *ret = datax;
2064
2065         *ret = iommu_domain_identity_map(si_domain,
2066                                          (uint64_t)start_pfn << PAGE_SHIFT,
2067                                          (uint64_t)end_pfn << PAGE_SHIFT);
2068         return *ret;
2069
2070 }
2071
2072 static int __init si_domain_init(int hw)
2073 {
2074         struct dmar_drhd_unit *drhd;
2075         struct intel_iommu *iommu;
2076         int nid, ret = 0;
2077
2078         si_domain = alloc_domain();
2079         if (!si_domain)
2080                 return -EFAULT;
2081
2082         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2083
2084         for_each_active_iommu(iommu, drhd) {
2085                 ret = iommu_attach_domain(si_domain, iommu);
2086                 if (ret) {
2087                         domain_exit(si_domain);
2088                         return -EFAULT;
2089                 }
2090         }
2091
2092         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2093                 domain_exit(si_domain);
2094                 return -EFAULT;
2095         }
2096
2097         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2098
2099         if (hw)
2100                 return 0;
2101
2102         for_each_online_node(nid) {
2103                 work_with_active_regions(nid, si_domain_work_fn, &ret);
2104                 if (ret)
2105                         return ret;
2106         }
2107
2108         return 0;
2109 }
2110
2111 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2112                                           struct pci_dev *pdev);
2113 static int identity_mapping(struct pci_dev *pdev)
2114 {
2115         struct device_domain_info *info;
2116
2117         if (likely(!iommu_identity_mapping))
2118                 return 0;
2119
2120
2121         list_for_each_entry(info, &si_domain->devices, link)
2122                 if (info->dev == pdev)
2123                         return 1;
2124         return 0;
2125 }
2126
2127 static int domain_add_dev_info(struct dmar_domain *domain,
2128                                struct pci_dev *pdev,
2129                                int translation)
2130 {
2131         struct device_domain_info *info;
2132         unsigned long flags;
2133         int ret;
2134
2135         info = alloc_devinfo_mem();
2136         if (!info)
2137                 return -ENOMEM;
2138
2139         ret = domain_context_mapping(domain, pdev, translation);
2140         if (ret) {
2141                 free_devinfo_mem(info);
2142                 return ret;
2143         }
2144
2145         info->segment = pci_domain_nr(pdev->bus);
2146         info->bus = pdev->bus->number;
2147         info->devfn = pdev->devfn;
2148         info->dev = pdev;
2149         info->domain = domain;
2150
2151         spin_lock_irqsave(&device_domain_lock, flags);
2152         list_add(&info->link, &domain->devices);
2153         list_add(&info->global, &device_domain_list);
2154         pdev->dev.archdata.iommu = info;
2155         spin_unlock_irqrestore(&device_domain_lock, flags);
2156
2157         return 0;
2158 }
2159
2160 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2161 {
2162         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2163                 return 1;
2164
2165         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2166                 return 1;
2167
2168         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2169                 return 0;
2170
2171         /*
2172          * We want to start off with all devices in the 1:1 domain, and
2173          * take them out later if we find they can't access all of memory.
2174          *
2175          * However, we can't do this for PCI devices behind bridges,
2176          * because all PCI devices behind the same bridge will end up
2177          * with the same source-id on their transactions.
2178          *
2179          * Practically speaking, we can't change things around for these
2180          * devices at run-time, because we can't be sure there'll be no
2181          * DMA transactions in flight for any of their siblings.
2182          *
2183          * So PCI devices (unless they're on the root bus) as well as
2184          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2185          * the 1:1 domain, just in _case_ one of their siblings turns out
2186          * not to be able to map all of memory.
2187          */
2188         if (!pci_is_pcie(pdev)) {
2189                 if (!pci_is_root_bus(pdev->bus))
2190                         return 0;
2191                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2192                         return 0;
2193         } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2194                 return 0;
2195
2196         /*
2197          * At boot time, we don't yet know if devices will be 64-bit capable.
2198          * Assume that they will -- if they turn out not to be, then we can
2199          * take them out of the 1:1 domain later.
2200          */
2201         if (!startup)
2202                 return pdev->dma_mask > DMA_BIT_MASK(32);
2203
2204         return 1;
2205 }
2206
2207 static int __init iommu_prepare_static_identity_mapping(int hw)
2208 {
2209         struct pci_dev *pdev = NULL;
2210         int ret;
2211
2212         ret = si_domain_init(hw);
2213         if (ret)
2214                 return -EFAULT;
2215
2216         for_each_pci_dev(pdev) {
2217                 if (iommu_should_identity_map(pdev, 1)) {
2218                         printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2219                                hw ? "hardware" : "software", pci_name(pdev));
2220
2221                         ret = domain_add_dev_info(si_domain, pdev,
2222                                                      hw ? CONTEXT_TT_PASS_THROUGH :
2223                                                      CONTEXT_TT_MULTI_LEVEL);
2224                         if (ret)
2225                                 return ret;
2226                 }
2227         }
2228
2229         return 0;
2230 }
2231
2232 int __init init_dmars(void)
2233 {
2234         struct dmar_drhd_unit *drhd;
2235         struct dmar_rmrr_unit *rmrr;
2236         struct pci_dev *pdev;
2237         struct intel_iommu *iommu;
2238         int i, ret;
2239
2240         /*
2241          * for each drhd
2242          *    allocate root
2243          *    initialize and program root entry to not present
2244          * endfor
2245          */
2246         for_each_drhd_unit(drhd) {
2247                 g_num_of_iommus++;
2248                 /*
2249                  * lock not needed as this is only incremented in the single
2250                  * threaded kernel __init code path all other access are read
2251                  * only
2252                  */
2253         }
2254
2255         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2256                         GFP_KERNEL);
2257         if (!g_iommus) {
2258                 printk(KERN_ERR "Allocating global iommu array failed\n");
2259                 ret = -ENOMEM;
2260                 goto error;
2261         }
2262
2263         deferred_flush = kzalloc(g_num_of_iommus *
2264                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2265         if (!deferred_flush) {
2266                 ret = -ENOMEM;
2267                 goto error;
2268         }
2269
2270         for_each_drhd_unit(drhd) {
2271                 if (drhd->ignored)
2272                         continue;
2273
2274                 iommu = drhd->iommu;
2275                 g_iommus[iommu->seq_id] = iommu;
2276
2277                 ret = iommu_init_domains(iommu);
2278                 if (ret)
2279                         goto error;
2280
2281                 /*
2282                  * TBD:
2283                  * we could share the same root & context tables
2284                  * amoung all IOMMU's. Need to Split it later.
2285                  */
2286                 ret = iommu_alloc_root_entry(iommu);
2287                 if (ret) {
2288                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2289                         goto error;
2290                 }
2291                 if (!ecap_pass_through(iommu->ecap))
2292                         hw_pass_through = 0;
2293         }
2294
2295         /*
2296          * Start from the sane iommu hardware state.
2297          */
2298         for_each_drhd_unit(drhd) {
2299                 if (drhd->ignored)
2300                         continue;
2301
2302                 iommu = drhd->iommu;
2303
2304                 /*
2305                  * If the queued invalidation is already initialized by us
2306                  * (for example, while enabling interrupt-remapping) then
2307                  * we got the things already rolling from a sane state.
2308                  */
2309                 if (iommu->qi)
2310                         continue;
2311
2312                 /*
2313                  * Clear any previous faults.
2314                  */
2315                 dmar_fault(-1, iommu);
2316                 /*
2317                  * Disable queued invalidation if supported and already enabled
2318                  * before OS handover.
2319                  */
2320                 dmar_disable_qi(iommu);
2321         }
2322
2323         for_each_drhd_unit(drhd) {
2324                 if (drhd->ignored)
2325                         continue;
2326
2327                 iommu = drhd->iommu;
2328
2329                 if (dmar_enable_qi(iommu)) {
2330                         /*
2331                          * Queued Invalidate not enabled, use Register Based
2332                          * Invalidate
2333                          */
2334                         iommu->flush.flush_context = __iommu_flush_context;
2335                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2336                         printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2337                                "invalidation\n",
2338                                (unsigned long long)drhd->reg_base_addr);
2339                 } else {
2340                         iommu->flush.flush_context = qi_flush_context;
2341                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2342                         printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2343                                "invalidation\n",
2344                                (unsigned long long)drhd->reg_base_addr);
2345                 }
2346         }
2347
2348         if (iommu_pass_through)
2349                 iommu_identity_mapping |= IDENTMAP_ALL;
2350
2351 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2352         iommu_identity_mapping |= IDENTMAP_GFX;
2353 #endif
2354
2355         check_tylersburg_isoch();
2356
2357         /*
2358          * If pass through is not set or not enabled, setup context entries for
2359          * identity mappings for rmrr, gfx, and isa and may fall back to static
2360          * identity mapping if iommu_identity_mapping is set.
2361          */
2362         if (iommu_identity_mapping) {
2363                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2364                 if (ret) {
2365                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2366                         goto error;
2367                 }
2368         }
2369         /*
2370          * For each rmrr
2371          *   for each dev attached to rmrr
2372          *   do
2373          *     locate drhd for dev, alloc domain for dev
2374          *     allocate free domain
2375          *     allocate page table entries for rmrr
2376          *     if context not allocated for bus
2377          *           allocate and init context
2378          *           set present in root table for this bus
2379          *     init context with domain, translation etc
2380          *    endfor
2381          * endfor
2382          */
2383         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2384         for_each_rmrr_units(rmrr) {
2385                 for (i = 0; i < rmrr->devices_cnt; i++) {
2386                         pdev = rmrr->devices[i];
2387                         /*
2388                          * some BIOS lists non-exist devices in DMAR
2389                          * table.
2390                          */
2391                         if (!pdev)
2392                                 continue;
2393                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2394                         if (ret)
2395                                 printk(KERN_ERR
2396                                        "IOMMU: mapping reserved region failed\n");
2397                 }
2398         }
2399
2400         iommu_prepare_isa();
2401
2402         /*
2403          * for each drhd
2404          *   enable fault log
2405          *   global invalidate context cache
2406          *   global invalidate iotlb
2407          *   enable translation
2408          */
2409         for_each_drhd_unit(drhd) {
2410                 if (drhd->ignored)
2411                         continue;
2412                 iommu = drhd->iommu;
2413
2414                 iommu_flush_write_buffer(iommu);
2415
2416                 ret = dmar_set_interrupt(iommu);
2417                 if (ret)
2418                         goto error;
2419
2420                 iommu_set_root_entry(iommu);
2421
2422                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2423                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2424
2425                 ret = iommu_enable_translation(iommu);
2426                 if (ret)
2427                         goto error;
2428
2429                 iommu_disable_protect_mem_regions(iommu);
2430         }
2431
2432         return 0;
2433 error:
2434         for_each_drhd_unit(drhd) {
2435                 if (drhd->ignored)
2436                         continue;
2437                 iommu = drhd->iommu;
2438                 free_iommu(iommu);
2439         }
2440         kfree(g_iommus);
2441         return ret;
2442 }
2443
2444 /* This takes a number of _MM_ pages, not VTD pages */
2445 static struct iova *intel_alloc_iova(struct device *dev,
2446                                      struct dmar_domain *domain,
2447                                      unsigned long nrpages, uint64_t dma_mask)
2448 {
2449         struct pci_dev *pdev = to_pci_dev(dev);
2450         struct iova *iova = NULL;
2451
2452         /* Restrict dma_mask to the width that the iommu can handle */
2453         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2454
2455         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2456                 /*
2457                  * First try to allocate an io virtual address in
2458                  * DMA_BIT_MASK(32) and if that fails then try allocating
2459                  * from higher range
2460                  */
2461                 iova = alloc_iova(&domain->iovad, nrpages,
2462                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2463                 if (iova)
2464                         return iova;
2465         }
2466         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2467         if (unlikely(!iova)) {
2468                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2469                        nrpages, pci_name(pdev));
2470                 return NULL;
2471         }
2472
2473         return iova;
2474 }
2475
2476 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2477 {
2478         struct dmar_domain *domain;
2479         int ret;
2480
2481         domain = get_domain_for_dev(pdev,
2482                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2483         if (!domain) {
2484                 printk(KERN_ERR
2485                         "Allocating domain for %s failed", pci_name(pdev));
2486                 return NULL;
2487         }
2488
2489         /* make sure context mapping is ok */
2490         if (unlikely(!domain_context_mapped(pdev))) {
2491                 ret = domain_context_mapping(domain, pdev,
2492                                              CONTEXT_TT_MULTI_LEVEL);
2493                 if (ret) {
2494                         printk(KERN_ERR
2495                                 "Domain context map for %s failed",
2496                                 pci_name(pdev));
2497                         return NULL;
2498                 }
2499         }
2500
2501         return domain;
2502 }
2503
2504 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2505 {
2506         struct device_domain_info *info;
2507
2508         /* No lock here, assumes no domain exit in normal case */
2509         info = dev->dev.archdata.iommu;
2510         if (likely(info))
2511                 return info->domain;
2512
2513         return __get_valid_domain_for_dev(dev);
2514 }
2515
2516 static int iommu_dummy(struct pci_dev *pdev)
2517 {
2518         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2519 }
2520
2521 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2522 static int iommu_no_mapping(struct device *dev)
2523 {
2524         struct pci_dev *pdev;
2525         int found;
2526
2527         if (unlikely(dev->bus != &pci_bus_type))
2528                 return 1;
2529
2530         pdev = to_pci_dev(dev);
2531         if (iommu_dummy(pdev))
2532                 return 1;
2533
2534         if (!iommu_identity_mapping)
2535                 return 0;
2536
2537         found = identity_mapping(pdev);
2538         if (found) {
2539                 if (iommu_should_identity_map(pdev, 0))
2540                         return 1;
2541                 else {
2542                         /*
2543                          * 32 bit DMA is removed from si_domain and fall back
2544                          * to non-identity mapping.
2545                          */
2546                         domain_remove_one_dev_info(si_domain, pdev);
2547                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2548                                pci_name(pdev));
2549                         return 0;
2550                 }
2551         } else {
2552                 /*
2553                  * In case of a detached 64 bit DMA device from vm, the device
2554                  * is put into si_domain for identity mapping.
2555                  */
2556                 if (iommu_should_identity_map(pdev, 0)) {
2557                         int ret;
2558                         ret = domain_add_dev_info(si_domain, pdev,
2559                                                   hw_pass_through ?
2560                                                   CONTEXT_TT_PASS_THROUGH :
2561                                                   CONTEXT_TT_MULTI_LEVEL);
2562                         if (!ret) {
2563                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2564                                        pci_name(pdev));
2565                                 return 1;
2566                         }
2567                 }
2568         }
2569
2570         return 0;
2571 }
2572
2573 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2574                                      size_t size, int dir, u64 dma_mask)
2575 {
2576         struct pci_dev *pdev = to_pci_dev(hwdev);
2577         struct dmar_domain *domain;
2578         phys_addr_t start_paddr;
2579         struct iova *iova;
2580         int prot = 0;
2581         int ret;
2582         struct intel_iommu *iommu;
2583         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2584
2585         BUG_ON(dir == DMA_NONE);
2586
2587         if (iommu_no_mapping(hwdev))
2588                 return paddr;
2589
2590         domain = get_valid_domain_for_dev(pdev);
2591         if (!domain)
2592                 return 0;
2593
2594         iommu = domain_get_iommu(domain);
2595         size = aligned_nrpages(paddr, size);
2596
2597         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2598                                 pdev->dma_mask);
2599         if (!iova)
2600                 goto error;
2601
2602         /*
2603          * Check if DMAR supports zero-length reads on write only
2604          * mappings..
2605          */
2606         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2607                         !cap_zlr(iommu->cap))
2608                 prot |= DMA_PTE_READ;
2609         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2610                 prot |= DMA_PTE_WRITE;
2611         /*
2612          * paddr - (paddr + size) might be partial page, we should map the whole
2613          * page.  Note: if two part of one page are separately mapped, we
2614          * might have two guest_addr mapping to the same host paddr, but this
2615          * is not a big problem
2616          */
2617         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2618                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2619         if (ret)
2620                 goto error;
2621
2622         /* it's a non-present to present mapping. Only flush if caching mode */
2623         if (cap_caching_mode(iommu->cap))
2624                 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
2625         else
2626                 iommu_flush_write_buffer(iommu);
2627
2628         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2629         start_paddr += paddr & ~PAGE_MASK;
2630         return start_paddr;
2631
2632 error:
2633         if (iova)
2634                 __free_iova(&domain->iovad, iova);
2635         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2636                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2637         return 0;
2638 }
2639
2640 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2641                                  unsigned long offset, size_t size,
2642                                  enum dma_data_direction dir,
2643                                  struct dma_attrs *attrs)
2644 {
2645         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2646                                   dir, to_pci_dev(dev)->dma_mask);
2647 }
2648
2649 static void flush_unmaps(void)
2650 {
2651         int i, j;
2652
2653         timer_on = 0;
2654
2655         /* just flush them all */
2656         for (i = 0; i < g_num_of_iommus; i++) {
2657                 struct intel_iommu *iommu = g_iommus[i];
2658                 if (!iommu)
2659                         continue;
2660
2661                 if (!deferred_flush[i].next)
2662                         continue;
2663
2664                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2665                                          DMA_TLB_GLOBAL_FLUSH);
2666                 for (j = 0; j < deferred_flush[i].next; j++) {
2667                         unsigned long mask;
2668                         struct iova *iova = deferred_flush[i].iova[j];
2669
2670                         mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2671                         iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2672                                         (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2673                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2674                 }
2675                 deferred_flush[i].next = 0;
2676         }
2677
2678         list_size = 0;
2679 }
2680
2681 static void flush_unmaps_timeout(unsigned long data)
2682 {
2683         unsigned long flags;
2684
2685         spin_lock_irqsave(&async_umap_flush_lock, flags);
2686         flush_unmaps();
2687         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2688 }
2689
2690 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2691 {
2692         unsigned long flags;
2693         int next, iommu_id;
2694         struct intel_iommu *iommu;
2695
2696         spin_lock_irqsave(&async_umap_flush_lock, flags);
2697         if (list_size == HIGH_WATER_MARK)
2698                 flush_unmaps();
2699
2700         iommu = domain_get_iommu(dom);
2701         iommu_id = iommu->seq_id;
2702
2703         next = deferred_flush[iommu_id].next;
2704         deferred_flush[iommu_id].domain[next] = dom;
2705         deferred_flush[iommu_id].iova[next] = iova;
2706         deferred_flush[iommu_id].next++;
2707
2708         if (!timer_on) {
2709                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2710                 timer_on = 1;
2711         }
2712         list_size++;
2713         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2714 }
2715
2716 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2717                              size_t size, enum dma_data_direction dir,
2718                              struct dma_attrs *attrs)
2719 {
2720         struct pci_dev *pdev = to_pci_dev(dev);
2721         struct dmar_domain *domain;
2722         unsigned long start_pfn, last_pfn;
2723         struct iova *iova;
2724         struct intel_iommu *iommu;
2725
2726         if (iommu_no_mapping(dev))
2727                 return;
2728
2729         domain = find_domain(pdev);
2730         BUG_ON(!domain);
2731
2732         iommu = domain_get_iommu(domain);
2733
2734         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2735         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2736                       (unsigned long long)dev_addr))
2737                 return;
2738
2739         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2740         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2741
2742         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2743                  pci_name(pdev), start_pfn, last_pfn);
2744
2745         /*  clear the whole page */
2746         dma_pte_clear_range(domain, start_pfn, last_pfn);
2747
2748         /* free page tables */
2749         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2750
2751         if (intel_iommu_strict) {
2752                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2753                                       last_pfn - start_pfn + 1);
2754                 /* free iova */
2755                 __free_iova(&domain->iovad, iova);
2756         } else {
2757                 add_unmap(domain, iova);
2758                 /*
2759                  * queue up the release of the unmap to save the 1/6th of the
2760                  * cpu used up by the iotlb flush operation...
2761                  */
2762         }
2763 }
2764
2765 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2766                                   dma_addr_t *dma_handle, gfp_t flags)
2767 {
2768         void *vaddr;
2769         int order;
2770
2771         size = PAGE_ALIGN(size);
2772         order = get_order(size);
2773
2774         if (!iommu_no_mapping(hwdev))
2775                 flags &= ~(GFP_DMA | GFP_DMA32);
2776         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2777                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2778                         flags |= GFP_DMA;
2779                 else
2780                         flags |= GFP_DMA32;
2781         }
2782
2783         vaddr = (void *)__get_free_pages(flags, order);
2784         if (!vaddr)
2785                 return NULL;
2786         memset(vaddr, 0, size);
2787
2788         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2789                                          DMA_BIDIRECTIONAL,
2790                                          hwdev->coherent_dma_mask);
2791         if (*dma_handle)
2792                 return vaddr;
2793         free_pages((unsigned long)vaddr, order);
2794         return NULL;
2795 }
2796
2797 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2798                                 dma_addr_t dma_handle)
2799 {
2800         int order;
2801
2802         size = PAGE_ALIGN(size);
2803         order = get_order(size);
2804
2805         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2806         free_pages((unsigned long)vaddr, order);
2807 }
2808
2809 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2810                            int nelems, enum dma_data_direction dir,
2811                            struct dma_attrs *attrs)
2812 {
2813         struct pci_dev *pdev = to_pci_dev(hwdev);
2814         struct dmar_domain *domain;
2815         unsigned long start_pfn, last_pfn;
2816         struct iova *iova;
2817         struct intel_iommu *iommu;
2818
2819         if (iommu_no_mapping(hwdev))
2820                 return;
2821
2822         domain = find_domain(pdev);
2823         BUG_ON(!domain);
2824
2825         iommu = domain_get_iommu(domain);
2826
2827         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2828         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2829                       (unsigned long long)sglist[0].dma_address))
2830                 return;
2831
2832         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2833         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2834
2835         /*  clear the whole page */
2836         dma_pte_clear_range(domain, start_pfn, last_pfn);
2837
2838         /* free page tables */
2839         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2840
2841         if (intel_iommu_strict) {
2842                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2843                                       last_pfn - start_pfn + 1);
2844                 /* free iova */
2845                 __free_iova(&domain->iovad, iova);
2846         } else {
2847                 add_unmap(domain, iova);
2848                 /*
2849                  * queue up the release of the unmap to save the 1/6th of the
2850                  * cpu used up by the iotlb flush operation...
2851                  */
2852         }
2853 }
2854
2855 static int intel_nontranslate_map_sg(struct device *hddev,
2856         struct scatterlist *sglist, int nelems, int dir)
2857 {
2858         int i;
2859         struct scatterlist *sg;
2860
2861         for_each_sg(sglist, sg, nelems, i) {
2862                 BUG_ON(!sg_page(sg));
2863                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2864                 sg->dma_length = sg->length;
2865         }
2866         return nelems;
2867 }
2868
2869 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2870                         enum dma_data_direction dir, struct dma_attrs *attrs)
2871 {
2872         int i;
2873         struct pci_dev *pdev = to_pci_dev(hwdev);
2874         struct dmar_domain *domain;
2875         size_t size = 0;
2876         int prot = 0;
2877         size_t offset_pfn = 0;
2878         struct iova *iova = NULL;
2879         int ret;
2880         struct scatterlist *sg;
2881         unsigned long start_vpfn;
2882         struct intel_iommu *iommu;
2883
2884         BUG_ON(dir == DMA_NONE);
2885         if (iommu_no_mapping(hwdev))
2886                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2887
2888         domain = get_valid_domain_for_dev(pdev);
2889         if (!domain)
2890                 return 0;
2891
2892         iommu = domain_get_iommu(domain);
2893
2894         for_each_sg(sglist, sg, nelems, i)
2895                 size += aligned_nrpages(sg->offset, sg->length);
2896
2897         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2898                                 pdev->dma_mask);
2899         if (!iova) {
2900                 sglist->dma_length = 0;
2901                 return 0;
2902         }
2903
2904         /*
2905          * Check if DMAR supports zero-length reads on write only
2906          * mappings..
2907          */
2908         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2909                         !cap_zlr(iommu->cap))
2910                 prot |= DMA_PTE_READ;
2911         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2912                 prot |= DMA_PTE_WRITE;
2913
2914         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2915
2916         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2917         if (unlikely(ret)) {
2918                 /*  clear the page */
2919                 dma_pte_clear_range(domain, start_vpfn,
2920                                     start_vpfn + size - 1);
2921                 /* free page tables */
2922                 dma_pte_free_pagetable(domain, start_vpfn,
2923                                        start_vpfn + size - 1);
2924                 /* free iova */
2925                 __free_iova(&domain->iovad, iova);
2926                 return 0;
2927         }
2928
2929         /* it's a non-present to present mapping. Only flush if caching mode */
2930         if (cap_caching_mode(iommu->cap))
2931                 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
2932         else
2933                 iommu_flush_write_buffer(iommu);
2934
2935         return nelems;
2936 }
2937
2938 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2939 {
2940         return !dma_addr;
2941 }
2942
2943 struct dma_map_ops intel_dma_ops = {
2944         .alloc_coherent = intel_alloc_coherent,
2945         .free_coherent = intel_free_coherent,
2946         .map_sg = intel_map_sg,
2947         .unmap_sg = intel_unmap_sg,
2948         .map_page = intel_map_page,
2949         .unmap_page = intel_unmap_page,
2950         .mapping_error = intel_mapping_error,
2951 };
2952
2953 static inline int iommu_domain_cache_init(void)
2954 {
2955         int ret = 0;
2956
2957         iommu_domain_cache = kmem_cache_create("iommu_domain",
2958                                          sizeof(struct dmar_domain),
2959                                          0,
2960                                          SLAB_HWCACHE_ALIGN,
2961
2962                                          NULL);
2963         if (!iommu_domain_cache) {
2964                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2965                 ret = -ENOMEM;
2966         }
2967
2968         return ret;
2969 }
2970
2971 static inline int iommu_devinfo_cache_init(void)
2972 {
2973         int ret = 0;
2974
2975         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2976                                          sizeof(struct device_domain_info),
2977                                          0,
2978                                          SLAB_HWCACHE_ALIGN,
2979                                          NULL);
2980         if (!iommu_devinfo_cache) {
2981                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2982                 ret = -ENOMEM;
2983         }
2984
2985         return ret;
2986 }
2987
2988 static inline int iommu_iova_cache_init(void)
2989 {
2990         int ret = 0;
2991
2992         iommu_iova_cache = kmem_cache_create("iommu_iova",
2993                                          sizeof(struct iova),
2994                                          0,
2995                                          SLAB_HWCACHE_ALIGN,
2996                                          NULL);
2997         if (!iommu_iova_cache) {
2998                 printk(KERN_ERR "Couldn't create iova cache\n");
2999                 ret = -ENOMEM;
3000         }
3001
3002         return ret;
3003 }
3004
3005 static int __init iommu_init_mempool(void)
3006 {
3007         int ret;
3008         ret = iommu_iova_cache_init();
3009         if (ret)
3010                 return ret;
3011
3012         ret = iommu_domain_cache_init();
3013         if (ret)
3014                 goto domain_error;
3015
3016         ret = iommu_devinfo_cache_init();
3017         if (!ret)
3018                 return ret;
3019
3020         kmem_cache_destroy(iommu_domain_cache);
3021 domain_error:
3022         kmem_cache_destroy(iommu_iova_cache);
3023
3024         return -ENOMEM;
3025 }
3026
3027 static void __init iommu_exit_mempool(void)
3028 {
3029         kmem_cache_destroy(iommu_devinfo_cache);
3030         kmem_cache_destroy(iommu_domain_cache);
3031         kmem_cache_destroy(iommu_iova_cache);
3032
3033 }
3034
3035 static void __init init_no_remapping_devices(void)
3036 {
3037         struct dmar_drhd_unit *drhd;
3038
3039         for_each_drhd_unit(drhd) {
3040                 if (!drhd->include_all) {
3041                         int i;
3042                         for (i = 0; i < drhd->devices_cnt; i++)
3043                                 if (drhd->devices[i] != NULL)
3044                                         break;
3045                         /* ignore DMAR unit if no pci devices exist */
3046                         if (i == drhd->devices_cnt)
3047                                 drhd->ignored = 1;
3048                 }
3049         }
3050
3051         if (dmar_map_gfx)
3052                 return;
3053
3054         for_each_drhd_unit(drhd) {
3055                 int i;
3056                 if (drhd->ignored || drhd->include_all)
3057                         continue;
3058
3059                 for (i = 0; i < drhd->devices_cnt; i++)
3060                         if (drhd->devices[i] &&
3061                                 !IS_GFX_DEVICE(drhd->devices[i]))
3062                                 break;
3063
3064                 if (i < drhd->devices_cnt)
3065                         continue;
3066
3067                 /* bypass IOMMU if it is just for gfx devices */
3068                 drhd->ignored = 1;
3069                 for (i = 0; i < drhd->devices_cnt; i++) {
3070                         if (!drhd->devices[i])
3071                                 continue;
3072                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3073                 }
3074         }
3075 }
3076
3077 #ifdef CONFIG_SUSPEND
3078 static int init_iommu_hw(void)
3079 {
3080         struct dmar_drhd_unit *drhd;
3081         struct intel_iommu *iommu = NULL;
3082
3083         for_each_active_iommu(iommu, drhd)
3084                 if (iommu->qi)
3085                         dmar_reenable_qi(iommu);
3086
3087         for_each_active_iommu(iommu, drhd) {
3088                 iommu_flush_write_buffer(iommu);
3089
3090                 iommu_set_root_entry(iommu);
3091
3092                 iommu->flush.flush_context(iommu, 0, 0, 0,
3093                                            DMA_CCMD_GLOBAL_INVL);
3094                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3095                                          DMA_TLB_GLOBAL_FLUSH);
3096                 iommu_enable_translation(iommu);
3097                 iommu_disable_protect_mem_regions(iommu);
3098         }
3099
3100         return 0;
3101 }
3102
3103 static void iommu_flush_all(void)
3104 {
3105         struct dmar_drhd_unit *drhd;
3106         struct intel_iommu *iommu;
3107
3108         for_each_active_iommu(iommu, drhd) {
3109                 iommu->flush.flush_context(iommu, 0, 0, 0,
3110                                            DMA_CCMD_GLOBAL_INVL);
3111                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3112                                          DMA_TLB_GLOBAL_FLUSH);
3113         }
3114 }
3115
3116 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3117 {
3118         struct dmar_drhd_unit *drhd;
3119         struct intel_iommu *iommu = NULL;
3120         unsigned long flag;
3121
3122         for_each_active_iommu(iommu, drhd) {
3123                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3124                                                  GFP_ATOMIC);
3125                 if (!iommu->iommu_state)
3126                         goto nomem;
3127         }
3128
3129         iommu_flush_all();
3130
3131         for_each_active_iommu(iommu, drhd) {
3132                 iommu_disable_translation(iommu);
3133
3134                 spin_lock_irqsave(&iommu->register_lock, flag);
3135
3136                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3137                         readl(iommu->reg + DMAR_FECTL_REG);
3138                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3139                         readl(iommu->reg + DMAR_FEDATA_REG);
3140                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3141                         readl(iommu->reg + DMAR_FEADDR_REG);
3142                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3143                         readl(iommu->reg + DMAR_FEUADDR_REG);
3144
3145                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3146         }
3147         return 0;
3148
3149 nomem:
3150         for_each_active_iommu(iommu, drhd)
3151                 kfree(iommu->iommu_state);
3152
3153         return -ENOMEM;
3154 }
3155
3156 static int iommu_resume(struct sys_device *dev)
3157 {
3158         struct dmar_drhd_unit *drhd;
3159         struct intel_iommu *iommu = NULL;
3160         unsigned long flag;
3161
3162         if (init_iommu_hw()) {
3163                 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3164                 return -EIO;
3165         }
3166
3167         for_each_active_iommu(iommu, drhd) {
3168
3169                 spin_lock_irqsave(&iommu->register_lock, flag);
3170
3171                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3172                         iommu->reg + DMAR_FECTL_REG);
3173                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3174                         iommu->reg + DMAR_FEDATA_REG);
3175                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3176                         iommu->reg + DMAR_FEADDR_REG);
3177                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3178                         iommu->reg + DMAR_FEUADDR_REG);
3179
3180                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3181         }
3182
3183         for_each_active_iommu(iommu, drhd)
3184                 kfree(iommu->iommu_state);
3185
3186         return 0;
3187 }
3188
3189 static struct sysdev_class iommu_sysclass = {
3190         .name           = "iommu",
3191         .resume         = iommu_resume,
3192         .suspend        = iommu_suspend,
3193 };
3194
3195 static struct sys_device device_iommu = {
3196         .cls    = &iommu_sysclass,
3197 };
3198
3199 static int __init init_iommu_sysfs(void)
3200 {
3201         int error;
3202
3203         error = sysdev_class_register(&iommu_sysclass);
3204         if (error)
3205                 return error;
3206
3207         error = sysdev_register(&device_iommu);
3208         if (error)
3209                 sysdev_class_unregister(&iommu_sysclass);
3210
3211         return error;
3212 }
3213
3214 #else
3215 static int __init init_iommu_sysfs(void)
3216 {
3217         return 0;
3218 }
3219 #endif  /* CONFIG_PM */
3220
3221 /*
3222  * Here we only respond to action of unbound device from driver.
3223  *
3224  * Added device is not attached to its DMAR domain here yet. That will happen
3225  * when mapping the device to iova.
3226  */
3227 static int device_notifier(struct notifier_block *nb,
3228                                   unsigned long action, void *data)
3229 {
3230         struct device *dev = data;
3231         struct pci_dev *pdev = to_pci_dev(dev);
3232         struct dmar_domain *domain;
3233
3234         if (iommu_no_mapping(dev))
3235                 return 0;
3236
3237         domain = find_domain(pdev);
3238         if (!domain)
3239                 return 0;
3240
3241         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3242                 domain_remove_one_dev_info(domain, pdev);
3243
3244         return 0;
3245 }
3246
3247 static struct notifier_block device_nb = {
3248         .notifier_call = device_notifier,
3249 };
3250
3251 int __init intel_iommu_init(void)
3252 {
3253         int ret = 0;
3254         int force_on = 0;
3255
3256         /* VT-d is required for a TXT/tboot launch, so enforce that */
3257         force_on = tboot_force_iommu();
3258
3259         if (dmar_table_init()) {
3260                 if (force_on)
3261                         panic("tboot: Failed to initialize DMAR table\n");
3262                 return  -ENODEV;
3263         }
3264
3265         if (dmar_dev_scope_init()) {
3266                 if (force_on)
3267                         panic("tboot: Failed to initialize DMAR device scope\n");
3268                 return  -ENODEV;
3269         }
3270
3271         /*
3272          * Check the need for DMA-remapping initialization now.
3273          * Above initialization will also be used by Interrupt-remapping.
3274          */
3275         if (no_iommu || dmar_disabled)
3276                 return -ENODEV;
3277
3278         iommu_init_mempool();
3279         dmar_init_reserved_ranges();
3280
3281         init_no_remapping_devices();
3282
3283         ret = init_dmars();
3284         if (ret) {
3285                 if (force_on)
3286                         panic("tboot: Failed to initialize DMARs\n");
3287                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3288                 put_iova_domain(&reserved_iova_list);
3289                 iommu_exit_mempool();
3290                 return ret;
3291         }
3292         printk(KERN_INFO
3293         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3294
3295         init_timer(&unmap_timer);
3296 #ifdef CONFIG_SWIOTLB
3297         swiotlb = 0;
3298 #endif
3299         dma_ops = &intel_dma_ops;
3300
3301         init_iommu_sysfs();
3302
3303         register_iommu(&intel_iommu_ops);
3304
3305         bus_register_notifier(&pci_bus_type, &device_nb);
3306
3307         return 0;
3308 }
3309
3310 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3311                                            struct pci_dev *pdev)
3312 {
3313         struct pci_dev *tmp, *parent;
3314
3315         if (!iommu || !pdev)
3316                 return;
3317
3318         /* dependent device detach */
3319         tmp = pci_find_upstream_pcie_bridge(pdev);
3320         /* Secondary interface's bus number and devfn 0 */
3321         if (tmp) {
3322                 parent = pdev->bus->self;
3323                 while (parent != tmp) {
3324                         iommu_detach_dev(iommu, parent->bus->number,
3325                                          parent->devfn);
3326                         parent = parent->bus->self;
3327                 }
3328                 if (pci_is_pcie(tmp)) /* this is a PCIE-to-PCI bridge */
3329                         iommu_detach_dev(iommu,
3330                                 tmp->subordinate->number, 0);
3331                 else /* this is a legacy PCI bridge */
3332                         iommu_detach_dev(iommu, tmp->bus->number,
3333                                          tmp->devfn);
3334         }
3335 }
3336
3337 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3338                                           struct pci_dev *pdev)
3339 {
3340         struct device_domain_info *info;
3341         struct intel_iommu *iommu;
3342         unsigned long flags;
3343         int found = 0;
3344         struct list_head *entry, *tmp;
3345
3346         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3347                                 pdev->devfn);
3348         if (!iommu)
3349                 return;
3350
3351         spin_lock_irqsave(&device_domain_lock, flags);
3352         list_for_each_safe(entry, tmp, &domain->devices) {
3353                 info = list_entry(entry, struct device_domain_info, link);
3354                 /* No need to compare PCI domain; it has to be the same */
3355                 if (info->bus == pdev->bus->number &&
3356                     info->devfn == pdev->devfn) {
3357                         list_del(&info->link);
3358                         list_del(&info->global);
3359                         if (info->dev)
3360                                 info->dev->dev.archdata.iommu = NULL;
3361                         spin_unlock_irqrestore(&device_domain_lock, flags);
3362
3363                         iommu_disable_dev_iotlb(info);
3364                         iommu_detach_dev(iommu, info->bus, info->devfn);
3365                         iommu_detach_dependent_devices(iommu, pdev);
3366                         free_devinfo_mem(info);
3367
3368                         spin_lock_irqsave(&device_domain_lock, flags);
3369
3370                         if (found)
3371                                 break;
3372                         else
3373                                 continue;
3374                 }
3375
3376                 /* if there is no other devices under the same iommu
3377                  * owned by this domain, clear this iommu in iommu_bmp
3378                  * update iommu count and coherency
3379                  */
3380                 if (iommu == device_to_iommu(info->segment, info->bus,
3381                                             info->devfn))
3382                         found = 1;
3383         }
3384
3385         if (found == 0) {
3386                 unsigned long tmp_flags;
3387                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3388                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3389                 domain->iommu_count--;
3390                 domain_update_iommu_cap(domain);
3391                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3392         }
3393
3394         spin_unlock_irqrestore(&device_domain_lock, flags);
3395 }
3396
3397 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3398 {
3399         struct device_domain_info *info;
3400         struct intel_iommu *iommu;
3401         unsigned long flags1, flags2;
3402
3403         spin_lock_irqsave(&device_domain_lock, flags1);
3404         while (!list_empty(&domain->devices)) {
3405                 info = list_entry(domain->devices.next,
3406                         struct device_domain_info, link);
3407                 list_del(&info->link);
3408                 list_del(&info->global);
3409                 if (info->dev)
3410                         info->dev->dev.archdata.iommu = NULL;
3411
3412                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3413
3414                 iommu_disable_dev_iotlb(info);
3415                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3416                 iommu_detach_dev(iommu, info->bus, info->devfn);
3417                 iommu_detach_dependent_devices(iommu, info->dev);
3418
3419                 /* clear this iommu in iommu_bmp, update iommu count
3420                  * and capabilities
3421                  */
3422                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3423                 if (test_and_clear_bit(iommu->seq_id,
3424                                        &domain->iommu_bmp)) {
3425                         domain->iommu_count--;
3426                         domain_update_iommu_cap(domain);
3427                 }
3428                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3429
3430                 free_devinfo_mem(info);
3431                 spin_lock_irqsave(&device_domain_lock, flags1);
3432         }
3433         spin_unlock_irqrestore(&device_domain_lock, flags1);
3434 }
3435
3436 /* domain id for virtual machine, it won't be set in context */
3437 static unsigned long vm_domid;
3438
3439 static int vm_domain_min_agaw(struct dmar_domain *domain)
3440 {
3441         int i;
3442         int min_agaw = domain->agaw;
3443
3444         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3445         for (; i < g_num_of_iommus; ) {
3446                 if (min_agaw > g_iommus[i]->agaw)
3447                         min_agaw = g_iommus[i]->agaw;
3448
3449                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3450         }
3451
3452         return min_agaw;
3453 }
3454
3455 static struct dmar_domain *iommu_alloc_vm_domain(void)
3456 {
3457         struct dmar_domain *domain;
3458
3459         domain = alloc_domain_mem();
3460         if (!domain)
3461                 return NULL;
3462
3463         domain->id = vm_domid++;
3464         domain->nid = -1;
3465         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3466         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3467
3468         return domain;
3469 }
3470
3471 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3472 {
3473         int adjust_width;
3474
3475         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3476         spin_lock_init(&domain->iommu_lock);
3477
3478         domain_reserve_special_ranges(domain);
3479
3480         /* calculate AGAW */
3481         domain->gaw = guest_width;
3482         adjust_width = guestwidth_to_adjustwidth(guest_width);
3483         domain->agaw = width_to_agaw(adjust_width);
3484
3485         INIT_LIST_HEAD(&domain->devices);
3486
3487         domain->iommu_count = 0;
3488         domain->iommu_coherency = 0;
3489         domain->iommu_snooping = 0;
3490         domain->max_addr = 0;
3491         domain->nid = -1;
3492
3493         /* always allocate the top pgd */
3494         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3495         if (!domain->pgd)
3496                 return -ENOMEM;
3497         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3498         return 0;
3499 }
3500
3501 static void iommu_free_vm_domain(struct dmar_domain *domain)
3502 {
3503         unsigned long flags;
3504         struct dmar_drhd_unit *drhd;
3505         struct intel_iommu *iommu;
3506         unsigned long i;
3507         unsigned long ndomains;
3508
3509         for_each_drhd_unit(drhd) {
3510                 if (drhd->ignored)
3511                         continue;
3512                 iommu = drhd->iommu;
3513
3514                 ndomains = cap_ndoms(iommu->cap);
3515                 i = find_first_bit(iommu->domain_ids, ndomains);
3516                 for (; i < ndomains; ) {
3517                         if (iommu->domains[i] == domain) {
3518                                 spin_lock_irqsave(&iommu->lock, flags);
3519                                 clear_bit(i, iommu->domain_ids);
3520                                 iommu->domains[i] = NULL;
3521                                 spin_unlock_irqrestore(&iommu->lock, flags);
3522                                 break;
3523                         }
3524                         i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3525                 }
3526         }
3527 }
3528
3529 static void vm_domain_exit(struct dmar_domain *domain)
3530 {
3531         /* Domain 0 is reserved, so dont process it */
3532         if (!domain)
3533                 return;
3534
3535         vm_domain_remove_all_dev_info(domain);
3536         /* destroy iovas */
3537         put_iova_domain(&domain->iovad);
3538
3539         /* clear ptes */
3540         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3541
3542         /* free page tables */
3543         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3544
3545         iommu_free_vm_domain(domain);
3546         free_domain_mem(domain);
3547 }
3548
3549 static int intel_iommu_domain_init(struct iommu_domain *domain)
3550 {
3551         struct dmar_domain *dmar_domain;
3552
3553         dmar_domain = iommu_alloc_vm_domain();
3554         if (!dmar_domain) {
3555                 printk(KERN_ERR
3556                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3557                 return -ENOMEM;
3558         }
3559         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3560                 printk(KERN_ERR
3561                         "intel_iommu_domain_init() failed\n");
3562                 vm_domain_exit(dmar_domain);
3563                 return -ENOMEM;
3564         }
3565         domain->priv = dmar_domain;
3566
3567         return 0;
3568 }
3569
3570 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3571 {
3572         struct dmar_domain *dmar_domain = domain->priv;
3573
3574         domain->priv = NULL;
3575         vm_domain_exit(dmar_domain);
3576 }
3577
3578 static int intel_iommu_attach_device(struct iommu_domain *domain,
3579                                      struct device *dev)
3580 {
3581         struct dmar_domain *dmar_domain = domain->priv;
3582         struct pci_dev *pdev = to_pci_dev(dev);
3583         struct intel_iommu *iommu;
3584         int addr_width;
3585         u64 end;
3586
3587         /* normally pdev is not mapped */
3588         if (unlikely(domain_context_mapped(pdev))) {
3589                 struct dmar_domain *old_domain;
3590
3591                 old_domain = find_domain(pdev);
3592                 if (old_domain) {
3593                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3594                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3595                                 domain_remove_one_dev_info(old_domain, pdev);
3596                         else
3597                                 domain_remove_dev_info(old_domain);
3598                 }
3599         }
3600
3601         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3602                                 pdev->devfn);
3603         if (!iommu)
3604                 return -ENODEV;
3605
3606         /* check if this iommu agaw is sufficient for max mapped address */
3607         addr_width = agaw_to_width(iommu->agaw);
3608         end = DOMAIN_MAX_ADDR(addr_width);
3609         end = end & VTD_PAGE_MASK;
3610         if (end < dmar_domain->max_addr) {
3611                 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3612                        "sufficient for the mapped address (%llx)\n",
3613                        __func__, iommu->agaw, dmar_domain->max_addr);
3614                 return -EFAULT;
3615         }
3616
3617         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3618 }
3619
3620 static void intel_iommu_detach_device(struct iommu_domain *domain,
3621                                       struct device *dev)
3622 {
3623         struct dmar_domain *dmar_domain = domain->priv;
3624         struct pci_dev *pdev = to_pci_dev(dev);
3625
3626         domain_remove_one_dev_info(dmar_domain, pdev);
3627 }
3628
3629 static int intel_iommu_map_range(struct iommu_domain *domain,
3630                                  unsigned long iova, phys_addr_t hpa,
3631                                  size_t size, int iommu_prot)
3632 {
3633         struct dmar_domain *dmar_domain = domain->priv;
3634         u64 max_addr;
3635         int addr_width;
3636         int prot = 0;
3637         int ret;
3638
3639         if (iommu_prot & IOMMU_READ)
3640                 prot |= DMA_PTE_READ;
3641         if (iommu_prot & IOMMU_WRITE)
3642                 prot |= DMA_PTE_WRITE;
3643         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3644                 prot |= DMA_PTE_SNP;
3645
3646         max_addr = iova + size;
3647         if (dmar_domain->max_addr < max_addr) {
3648                 int min_agaw;
3649                 u64 end;
3650
3651                 /* check if minimum agaw is sufficient for mapped address */
3652                 min_agaw = vm_domain_min_agaw(dmar_domain);
3653                 addr_width = agaw_to_width(min_agaw);
3654                 end = DOMAIN_MAX_ADDR(addr_width);
3655                 end = end & VTD_PAGE_MASK;
3656                 if (end < max_addr) {
3657                         printk(KERN_ERR "%s: iommu agaw (%d) is not "
3658                                "sufficient for the mapped address (%llx)\n",
3659                                __func__, min_agaw, max_addr);
3660                         return -EFAULT;
3661                 }
3662                 dmar_domain->max_addr = max_addr;
3663         }
3664         /* Round up size to next multiple of PAGE_SIZE, if it and
3665            the low bits of hpa would take us onto the next page */
3666         size = aligned_nrpages(hpa, size);
3667         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3668                                  hpa >> VTD_PAGE_SHIFT, size, prot);
3669         return ret;
3670 }
3671
3672 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3673                                     unsigned long iova, size_t size)
3674 {
3675         struct dmar_domain *dmar_domain = domain->priv;
3676
3677         if (!size)
3678                 return;
3679
3680         dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3681                             (iova + size - 1) >> VTD_PAGE_SHIFT);
3682
3683         if (dmar_domain->max_addr == iova + size)
3684                 dmar_domain->max_addr = iova;
3685 }
3686
3687 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3688                                             unsigned long iova)
3689 {
3690         struct dmar_domain *dmar_domain = domain->priv;
3691         struct dma_pte *pte;
3692         u64 phys = 0;
3693
3694         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3695         if (pte)
3696                 phys = dma_pte_addr(pte);
3697
3698         return phys;
3699 }
3700
3701 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3702                                       unsigned long cap)
3703 {
3704         struct dmar_domain *dmar_domain = domain->priv;
3705
3706         if (cap == IOMMU_CAP_CACHE_COHERENCY)
3707                 return dmar_domain->iommu_snooping;
3708
3709         return 0;
3710 }
3711
3712 static struct iommu_ops intel_iommu_ops = {
3713         .domain_init    = intel_iommu_domain_init,
3714         .domain_destroy = intel_iommu_domain_destroy,
3715         .attach_dev     = intel_iommu_attach_device,
3716         .detach_dev     = intel_iommu_detach_device,
3717         .map            = intel_iommu_map_range,
3718         .unmap          = intel_iommu_unmap_range,
3719         .iova_to_phys   = intel_iommu_iova_to_phys,
3720         .domain_has_cap = intel_iommu_domain_has_cap,
3721 };
3722
3723 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3724 {
3725         /*
3726          * Mobile 4 Series Chipset neglects to set RWBF capability,
3727          * but needs it:
3728          */
3729         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3730         rwbf_quirk = 1;
3731 }
3732
3733 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3734
3735 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3736    ISOCH DMAR unit for the Azalia sound device, but not give it any
3737    TLB entries, which causes it to deadlock. Check for that.  We do
3738    this in a function called from init_dmars(), instead of in a PCI
3739    quirk, because we don't want to print the obnoxious "BIOS broken"
3740    message if VT-d is actually disabled.
3741 */
3742 static void __init check_tylersburg_isoch(void)
3743 {
3744         struct pci_dev *pdev;
3745         uint32_t vtisochctrl;
3746
3747         /* If there's no Azalia in the system anyway, forget it. */
3748         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3749         if (!pdev)
3750                 return;
3751         pci_dev_put(pdev);
3752
3753         /* System Management Registers. Might be hidden, in which case
3754            we can't do the sanity check. But that's OK, because the
3755            known-broken BIOSes _don't_ actually hide it, so far. */
3756         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3757         if (!pdev)
3758                 return;
3759
3760         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3761                 pci_dev_put(pdev);
3762                 return;
3763         }
3764
3765         pci_dev_put(pdev);
3766
3767         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3768         if (vtisochctrl & 1)
3769                 return;
3770
3771         /* Drop all bits other than the number of TLB entries */
3772         vtisochctrl &= 0x1c;
3773
3774         /* If we have the recommended number of TLB entries (16), fine. */
3775         if (vtisochctrl == 0x10)
3776                 return;
3777
3778         /* Zero TLB entries? You get to ride the short bus to school. */
3779         if (!vtisochctrl) {
3780                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3781                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3782                      dmi_get_system_info(DMI_BIOS_VENDOR),
3783                      dmi_get_system_info(DMI_BIOS_VERSION),
3784                      dmi_get_system_info(DMI_PRODUCT_VERSION));
3785                 iommu_identity_mapping |= IDENTMAP_AZALIA;
3786                 return;
3787         }
3788
3789         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3790                vtisochctrl);
3791 }