drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/dma-direct.h>
  49 #include <linux/crash_dump.h>
  50 #include <asm/irq_remapping.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/iommu.h>
  53
  54 #include "irq_remapping.h"
  55 #include "intel-pasid.h"
  56
  57 #define ROOT_SIZE               VTD_PAGE_SIZE
  58 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  59
  60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  64
  65 #define IOAPIC_RANGE_START      (0xfee00000)
  66 #define IOAPIC_RANGE_END        (0xfeefffff)
  67 #define IOVA_START_ADDR         (0x1000)
  68
  69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  70
  71 #define MAX_AGAW_WIDTH 64
  72 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  73
  74 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  76
  77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  78    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  79 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  80                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  81 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  82
  83 /* IO virtual address start page frame number */
  84 #define IOVA_START_PFN          (1)
  85
  86 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186 int intel_iommu_tboot_noforce;
 187 static int no_platform_optin;
 188
 189 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 190
 191 /*
 192  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 193  * if marked present.
 194  */
 195 static phys_addr_t root_entry_lctp(struct root_entry *re)
 196 {
 197         if (!(re->lo & 1))
 198                 return 0;
 199
 200         return re->lo & VTD_PAGE_MASK;
 201 }
 202
 203 /*
 204  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 205  * if marked present.
 206  */
 207 static phys_addr_t root_entry_uctp(struct root_entry *re)
 208 {
 209         if (!(re->hi & 1))
 210                 return 0;
 211
 212         return re->hi & VTD_PAGE_MASK;
 213 }
 214
 215 static inline void context_clear_pasid_enable(struct context_entry *context)
 216 {
 217         context->lo &= ~(1ULL << 11);
 218 }
 219
 220 static inline bool context_pasid_enabled(struct context_entry *context)
 221 {
 222         return !!(context->lo & (1ULL << 11));
 223 }
 224
 225 static inline void context_set_copied(struct context_entry *context)
 226 {
 227         context->hi |= (1ull << 3);
 228 }
 229
 230 static inline bool context_copied(struct context_entry *context)
 231 {
 232         return !!(context->hi & (1ULL << 3));
 233 }
 234
 235 static inline bool __context_present(struct context_entry *context)
 236 {
 237         return (context->lo & 1);
 238 }
 239
 240 bool context_present(struct context_entry *context)
 241 {
 242         return context_pasid_enabled(context) ?
 243              __context_present(context) :
 244              __context_present(context) && !context_copied(context);
 245 }
 246
 247 static inline void context_set_present(struct context_entry *context)
 248 {
 249         context->lo |= 1;
 250 }
 251
 252 static inline void context_set_fault_enable(struct context_entry *context)
 253 {
 254         context->lo &= (((u64)-1) << 2) | 1;
 255 }
 256
 257 static inline void context_set_translation_type(struct context_entry *context,
 258                                                 unsigned long value)
 259 {
 260         context->lo &= (((u64)-1) << 4) | 3;
 261         context->lo |= (value & 3) << 2;
 262 }
 263
 264 static inline void context_set_address_root(struct context_entry *context,
 265                                             unsigned long value)
 266 {
 267         context->lo &= ~VTD_PAGE_MASK;
 268         context->lo |= value & VTD_PAGE_MASK;
 269 }
 270
 271 static inline void context_set_address_width(struct context_entry *context,
 272                                              unsigned long value)
 273 {
 274         context->hi |= value & 7;
 275 }
 276
 277 static inline void context_set_domain_id(struct context_entry *context,
 278                                          unsigned long value)
 279 {
 280         context->hi |= (value & ((1 << 16) - 1)) << 8;
 281 }
 282
 283 static inline int context_domain_id(struct context_entry *c)
 284 {
 285         return((c->hi >> 8) & 0xffff);
 286 }
 287
 288 static inline void context_clear_entry(struct context_entry *context)
 289 {
 290         context->lo = 0;
 291         context->hi = 0;
 292 }
 293
 294 /*
 295  * This domain is a statically identity mapping domain.
 296  *      1. This domain creats a static 1:1 mapping to all usable memory.
 297  *      2. It maps to each iommu if successful.
 298  *      3. Each iommu mapps to this domain if successful.
 299  */
 300 static struct dmar_domain *si_domain;
 301 static int hw_pass_through = 1;
 302
 303 /*
 304  * Domain represents a virtual machine, more than one devices
 305  * across iommus may be owned in one domain, e.g. kvm guest.
 306  */
 307 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 308
 309 /* si_domain contains mulitple devices */
 310 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 311
 312 #define for_each_domain_iommu(idx, domain)                      \
 313         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 314                 if (domain->iommu_refcnt[idx])
 315
 316 struct dmar_rmrr_unit {
 317         struct list_head list;          /* list of rmrr units   */
 318         struct acpi_dmar_header *hdr;   /* ACPI header          */
 319         u64     base_address;           /* reserved base address*/
 320         u64     end_address;            /* reserved end address */
 321         struct dmar_dev_scope *devices; /* target devices */
 322         int     devices_cnt;            /* target device count */
 323         struct iommu_resv_region *resv; /* reserved region handle */
 324 };
 325
 326 struct dmar_atsr_unit {
 327         struct list_head list;          /* list of ATSR units */
 328         struct acpi_dmar_header *hdr;   /* ACPI header */
 329         struct dmar_dev_scope *devices; /* target devices */
 330         int devices_cnt;                /* target device count */
 331         u8 include_all:1;               /* include all ports */
 332 };
 333
 334 static LIST_HEAD(dmar_atsr_units);
 335 static LIST_HEAD(dmar_rmrr_units);
 336
 337 #define for_each_rmrr_units(rmrr) \
 338         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 339
 340 /* bitmap for indexing intel_iommus */
 341 static int g_num_of_iommus;
 342
 343 static void domain_exit(struct dmar_domain *domain);
 344 static void domain_remove_dev_info(struct dmar_domain *domain);
 345 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 346                                      struct device *dev);
 347 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 348 static void domain_context_clear(struct intel_iommu *iommu,
 349                                  struct device *dev);
 350 static int domain_detach_iommu(struct dmar_domain *domain,
 351                                struct intel_iommu *iommu);
 352
 353 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 354 int dmar_disabled = 0;
 355 #else
 356 int dmar_disabled = 1;
 357 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 358
 359 int intel_iommu_enabled = 0;
 360 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 361
 362 static int dmar_map_gfx = 1;
 363 static int dmar_forcedac;
 364 static int intel_iommu_strict;
 365 static int intel_iommu_superpage = 1;
 366 static int intel_iommu_sm;
 367 static int iommu_identity_mapping;
 368
 369 #define IDENTMAP_ALL            1
 370 #define IDENTMAP_GFX            2
 371 #define IDENTMAP_AZALIA         4
 372
 373 #define sm_supported(iommu)     (intel_iommu_sm && ecap_smts((iommu)->ecap))
 374 #define pasid_supported(iommu)  (sm_supported(iommu) &&                 \
 375                                  ecap_pasid((iommu)->ecap))
 376
 377 int intel_iommu_gfx_mapped;
 378 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 379
 380 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 381 static DEFINE_SPINLOCK(device_domain_lock);
 382 static LIST_HEAD(device_domain_list);
 383
 384 /*
 385  * Iterate over elements in device_domain_list and call the specified
 386  * callback @fn against each element.
 387  */
 388 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
 389                                      void *data), void *data)
 390 {
 391         int ret = 0;
 392         unsigned long flags;
 393         struct device_domain_info *info;
 394
 395         spin_lock_irqsave(&device_domain_lock, flags);
 396         list_for_each_entry(info, &device_domain_list, global) {
 397                 ret = fn(info, data);
 398                 if (ret) {
 399                         spin_unlock_irqrestore(&device_domain_lock, flags);
 400                         return ret;
 401                 }
 402         }
 403         spin_unlock_irqrestore(&device_domain_lock, flags);
 404
 405         return 0;
 406 }
 407
 408 const struct iommu_ops intel_iommu_ops;
 409
 410 static bool translation_pre_enabled(struct intel_iommu *iommu)
 411 {
 412         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 413 }
 414
 415 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 416 {
 417         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 418 }
 419
 420 static void init_translation_status(struct intel_iommu *iommu)
 421 {
 422         u32 gsts;
 423
 424         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 425         if (gsts & DMA_GSTS_TES)
 426                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 427 }
 428
 429 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 430 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 431 {
 432         return container_of(dom, struct dmar_domain, domain);
 433 }
 434
 435 static int __init intel_iommu_setup(char *str)
 436 {
 437         if (!str)
 438                 return -EINVAL;
 439         while (*str) {
 440                 if (!strncmp(str, "on", 2)) {
 441                         dmar_disabled = 0;
 442                         pr_info("IOMMU enabled\n");
 443                 } else if (!strncmp(str, "off", 3)) {
 444                         dmar_disabled = 1;
 445                         no_platform_optin = 1;
 446                         pr_info("IOMMU disabled\n");
 447                 } else if (!strncmp(str, "igfx_off", 8)) {
 448                         dmar_map_gfx = 0;
 449                         pr_info("Disable GFX device mapping\n");
 450                 } else if (!strncmp(str, "forcedac", 8)) {
 451                         pr_info("Forcing DAC for PCI devices\n");
 452                         dmar_forcedac = 1;
 453                 } else if (!strncmp(str, "strict", 6)) {
 454                         pr_info("Disable batched IOTLB flush\n");
 455                         intel_iommu_strict = 1;
 456                 } else if (!strncmp(str, "sp_off", 6)) {
 457                         pr_info("Disable supported super page\n");
 458                         intel_iommu_superpage = 0;
 459                 } else if (!strncmp(str, "sm_on", 5)) {
 460                         pr_info("Intel-IOMMU: scalable mode supported\n");
 461                         intel_iommu_sm = 1;
 462                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 463                         printk(KERN_INFO
 464                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 465                         intel_iommu_tboot_noforce = 1;
 466                 }
 467
 468                 str += strcspn(str, ",");
 469                 while (*str == ',')
 470                         str++;
 471         }
 472         return 0;
 473 }
 474 __setup("intel_iommu=", intel_iommu_setup);
 475
 476 static struct kmem_cache *iommu_domain_cache;
 477 static struct kmem_cache *iommu_devinfo_cache;
 478
 479 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 480 {
 481         struct dmar_domain **domains;
 482         int idx = did >> 8;
 483
 484         domains = iommu->domains[idx];
 485         if (!domains)
 486                 return NULL;
 487
 488         return domains[did & 0xff];
 489 }
 490
 491 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 492                              struct dmar_domain *domain)
 493 {
 494         struct dmar_domain **domains;
 495         int idx = did >> 8;
 496
 497         if (!iommu->domains[idx]) {
 498                 size_t size = 256 * sizeof(struct dmar_domain *);
 499                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 500         }
 501
 502         domains = iommu->domains[idx];
 503         if (WARN_ON(!domains))
 504                 return;
 505         else
 506                 domains[did & 0xff] = domain;
 507 }
 508
 509 void *alloc_pgtable_page(int node)
 510 {
 511         struct page *page;
 512         void *vaddr = NULL;
 513
 514         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 515         if (page)
 516                 vaddr = page_address(page);
 517         return vaddr;
 518 }
 519
 520 void free_pgtable_page(void *vaddr)
 521 {
 522         free_page((unsigned long)vaddr);
 523 }
 524
 525 static inline void *alloc_domain_mem(void)
 526 {
 527         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 528 }
 529
 530 static void free_domain_mem(void *vaddr)
 531 {
 532         kmem_cache_free(iommu_domain_cache, vaddr);
 533 }
 534
 535 static inline void * alloc_devinfo_mem(void)
 536 {
 537         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 538 }
 539
 540 static inline void free_devinfo_mem(void *vaddr)
 541 {
 542         kmem_cache_free(iommu_devinfo_cache, vaddr);
 543 }
 544
 545 static inline int domain_type_is_vm(struct dmar_domain *domain)
 546 {
 547         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 548 }
 549
 550 static inline int domain_type_is_si(struct dmar_domain *domain)
 551 {
 552         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 553 }
 554
 555 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 556 {
 557         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 558                                 DOMAIN_FLAG_STATIC_IDENTITY);
 559 }
 560
 561 static inline int domain_pfn_supported(struct dmar_domain *domain,
 562                                        unsigned long pfn)
 563 {
 564         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 565
 566         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 567 }
 568
 569 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 570 {
 571         unsigned long sagaw;
 572         int agaw = -1;
 573
 574         sagaw = cap_sagaw(iommu->cap);
 575         for (agaw = width_to_agaw(max_gaw);
 576              agaw >= 0; agaw--) {
 577                 if (test_bit(agaw, &sagaw))
 578                         break;
 579         }
 580
 581         return agaw;
 582 }
 583
 584 /*
 585  * Calculate max SAGAW for each iommu.
 586  */
 587 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 588 {
 589         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 590 }
 591
 592 /*
 593  * calculate agaw for each iommu.
 594  * "SAGAW" may be different across iommus, use a default agaw, and
 595  * get a supported less agaw for iommus that don't support the default agaw.
 596  */
 597 int iommu_calculate_agaw(struct intel_iommu *iommu)
 598 {
 599         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 600 }
 601
 602 /* This functionin only returns single iommu in a domain */
 603 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 604 {
 605         int iommu_id;
 606
 607         /* si_domain and vm domain should not get here. */
 608         BUG_ON(domain_type_is_vm_or_si(domain));
 609         for_each_domain_iommu(iommu_id, domain)
 610                 break;
 611
 612         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 613                 return NULL;
 614
 615         return g_iommus[iommu_id];
 616 }
 617
 618 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 619 {
 620         struct dmar_drhd_unit *drhd;
 621         struct intel_iommu *iommu;
 622         bool found = false;
 623         int i;
 624
 625         domain->iommu_coherency = 1;
 626
 627         for_each_domain_iommu(i, domain) {
 628                 found = true;
 629                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 630                         domain->iommu_coherency = 0;
 631                         break;
 632                 }
 633         }
 634         if (found)
 635                 return;
 636
 637         /* No hardware attached; use lowest common denominator */
 638         rcu_read_lock();
 639         for_each_active_iommu(iommu, drhd) {
 640                 if (!ecap_coherent(iommu->ecap)) {
 641                         domain->iommu_coherency = 0;
 642                         break;
 643                 }
 644         }
 645         rcu_read_unlock();
 646 }
 647
 648 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 649 {
 650         struct dmar_drhd_unit *drhd;
 651         struct intel_iommu *iommu;
 652         int ret = 1;
 653
 654         rcu_read_lock();
 655         for_each_active_iommu(iommu, drhd) {
 656                 if (iommu != skip) {
 657                         if (!ecap_sc_support(iommu->ecap)) {
 658                                 ret = 0;
 659                                 break;
 660                         }
 661                 }
 662         }
 663         rcu_read_unlock();
 664
 665         return ret;
 666 }
 667
 668 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 669 {
 670         struct dmar_drhd_unit *drhd;
 671         struct intel_iommu *iommu;
 672         int mask = 0xf;
 673
 674         if (!intel_iommu_superpage) {
 675                 return 0;
 676         }
 677
 678         /* set iommu_superpage to the smallest common denominator */
 679         rcu_read_lock();
 680         for_each_active_iommu(iommu, drhd) {
 681                 if (iommu != skip) {
 682                         mask &= cap_super_page_val(iommu->cap);
 683                         if (!mask)
 684                                 break;
 685                 }
 686         }
 687         rcu_read_unlock();
 688
 689         return fls(mask);
 690 }
 691
 692 /* Some capabilities may be different across iommus */
 693 static void domain_update_iommu_cap(struct dmar_domain *domain)
 694 {
 695         domain_update_iommu_coherency(domain);
 696         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 697         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 698 }
 699
 700 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
 701                                          u8 devfn, int alloc)
 702 {
 703         struct root_entry *root = &iommu->root_entry[bus];
 704         struct context_entry *context;
 705         u64 *entry;
 706
 707         entry = &root->lo;
 708         if (sm_supported(iommu)) {
 709                 if (devfn >= 0x80) {
 710                         devfn -= 0x80;
 711                         entry = &root->hi;
 712                 }
 713                 devfn *= 2;
 714         }
 715         if (*entry & 1)
 716                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 717         else {
 718                 unsigned long phy_addr;
 719                 if (!alloc)
 720                         return NULL;
 721
 722                 context = alloc_pgtable_page(iommu->node);
 723                 if (!context)
 724                         return NULL;
 725
 726                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 727                 phy_addr = virt_to_phys((void *)context);
 728                 *entry = phy_addr | 1;
 729                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 730         }
 731         return &context[devfn];
 732 }
 733
 734 static int iommu_dummy(struct device *dev)
 735 {
 736         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 737 }
 738
 739 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 740 {
 741         struct dmar_drhd_unit *drhd = NULL;
 742         struct intel_iommu *iommu;
 743         struct device *tmp;
 744         struct pci_dev *ptmp, *pdev = NULL;
 745         u16 segment = 0;
 746         int i;
 747
 748         if (iommu_dummy(dev))
 749                 return NULL;
 750
 751         if (dev_is_pci(dev)) {
 752                 struct pci_dev *pf_pdev;
 753
 754                 pdev = to_pci_dev(dev);
 755
 756 #ifdef CONFIG_X86
 757                 /* VMD child devices currently cannot be handled individually */
 758                 if (is_vmd(pdev->bus))
 759                         return NULL;
 760 #endif
 761
 762                 /* VFs aren't listed in scope tables; we need to look up
 763                  * the PF instead to find the IOMMU. */
 764                 pf_pdev = pci_physfn(pdev);
 765                 dev = &pf_pdev->dev;
 766                 segment = pci_domain_nr(pdev->bus);
 767         } else if (has_acpi_companion(dev))
 768                 dev = &ACPI_COMPANION(dev)->dev;
 769
 770         rcu_read_lock();
 771         for_each_active_iommu(iommu, drhd) {
 772                 if (pdev && segment != drhd->segment)
 773                         continue;
 774
 775                 for_each_active_dev_scope(drhd->devices,
 776                                           drhd->devices_cnt, i, tmp) {
 777                         if (tmp == dev) {
 778                                 /* For a VF use its original BDF# not that of the PF
 779                                  * which we used for the IOMMU lookup. Strictly speaking
 780                                  * we could do this for all PCI devices; we only need to
 781                                  * get the BDF# from the scope table for ACPI matches. */
 782                                 if (pdev && pdev->is_virtfn)
 783                                         goto got_pdev;
 784
 785                                 *bus = drhd->devices[i].bus;
 786                                 *devfn = drhd->devices[i].devfn;
 787                                 goto out;
 788                         }
 789
 790                         if (!pdev || !dev_is_pci(tmp))
 791                                 continue;
 792
 793                         ptmp = to_pci_dev(tmp);
 794                         if (ptmp->subordinate &&
 795                             ptmp->subordinate->number <= pdev->bus->number &&
 796                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 797                                 goto got_pdev;
 798                 }
 799
 800                 if (pdev && drhd->include_all) {
 801                 got_pdev:
 802                         *bus = pdev->bus->number;
 803                         *devfn = pdev->devfn;
 804                         goto out;
 805                 }
 806         }
 807         iommu = NULL;
 808  out:
 809         rcu_read_unlock();
 810
 811         return iommu;
 812 }
 813
 814 static void domain_flush_cache(struct dmar_domain *domain,
 815                                void *addr, int size)
 816 {
 817         if (!domain->iommu_coherency)
 818                 clflush_cache_range(addr, size);
 819 }
 820
 821 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 822 {
 823         struct context_entry *context;
 824         int ret = 0;
 825         unsigned long flags;
 826
 827         spin_lock_irqsave(&iommu->lock, flags);
 828         context = iommu_context_addr(iommu, bus, devfn, 0);
 829         if (context)
 830                 ret = context_present(context);
 831         spin_unlock_irqrestore(&iommu->lock, flags);
 832         return ret;
 833 }
 834
 835 static void free_context_table(struct intel_iommu *iommu)
 836 {
 837         int i;
 838         unsigned long flags;
 839         struct context_entry *context;
 840
 841         spin_lock_irqsave(&iommu->lock, flags);
 842         if (!iommu->root_entry) {
 843                 goto out;
 844         }
 845         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 846                 context = iommu_context_addr(iommu, i, 0, 0);
 847                 if (context)
 848                         free_pgtable_page(context);
 849
 850                 if (!sm_supported(iommu))
 851                         continue;
 852
 853                 context = iommu_context_addr(iommu, i, 0x80, 0);
 854                 if (context)
 855                         free_pgtable_page(context);
 856
 857         }
 858         free_pgtable_page(iommu->root_entry);
 859         iommu->root_entry = NULL;
 860 out:
 861         spin_unlock_irqrestore(&iommu->lock, flags);
 862 }
 863
 864 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 865                                       unsigned long pfn, int *target_level)
 866 {
 867         struct dma_pte *parent, *pte = NULL;
 868         int level = agaw_to_level(domain->agaw);
 869         int offset;
 870
 871         BUG_ON(!domain->pgd);
 872
 873         if (!domain_pfn_supported(domain, pfn))
 874                 /* Address beyond IOMMU's addressing capabilities. */
 875                 return NULL;
 876
 877         parent = domain->pgd;
 878
 879         while (1) {
 880                 void *tmp_page;
 881
 882                 offset = pfn_level_offset(pfn, level);
 883                 pte = &parent[offset];
 884                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 885                         break;
 886                 if (level == *target_level)
 887                         break;
 888
 889                 if (!dma_pte_present(pte)) {
 890                         uint64_t pteval;
 891
 892                         tmp_page = alloc_pgtable_page(domain->nid);
 893
 894                         if (!tmp_page)
 895                                 return NULL;
 896
 897                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 898                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 899                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 900                                 /* Someone else set it while we were thinking; use theirs. */
 901                                 free_pgtable_page(tmp_page);
 902                         else
 903                                 domain_flush_cache(domain, pte, sizeof(*pte));
 904                 }
 905                 if (level == 1)
 906                         break;
 907
 908                 parent = phys_to_virt(dma_pte_addr(pte));
 909                 level--;
 910         }
 911
 912         if (!*target_level)
 913                 *target_level = level;
 914
 915         return pte;
 916 }
 917
 918
 919 /* return address's pte at specific level */
 920 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 921                                          unsigned long pfn,
 922                                          int level, int *large_page)
 923 {
 924         struct dma_pte *parent, *pte = NULL;
 925         int total = agaw_to_level(domain->agaw);
 926         int offset;
 927
 928         parent = domain->pgd;
 929         while (level <= total) {
 930                 offset = pfn_level_offset(pfn, total);
 931                 pte = &parent[offset];
 932                 if (level == total)
 933                         return pte;
 934
 935                 if (!dma_pte_present(pte)) {
 936                         *large_page = total;
 937                         break;
 938                 }
 939
 940                 if (dma_pte_superpage(pte)) {
 941                         *large_page = total;
 942                         return pte;
 943                 }
 944
 945                 parent = phys_to_virt(dma_pte_addr(pte));
 946                 total--;
 947         }
 948         return NULL;
 949 }
 950
 951 /* clear last level pte, a tlb flush should be followed */
 952 static void dma_pte_clear_range(struct dmar_domain *domain,
 953                                 unsigned long start_pfn,
 954                                 unsigned long last_pfn)
 955 {
 956         unsigned int large_page = 1;
 957         struct dma_pte *first_pte, *pte;
 958
 959         BUG_ON(!domain_pfn_supported(domain, start_pfn));
 960         BUG_ON(!domain_pfn_supported(domain, last_pfn));
 961         BUG_ON(start_pfn > last_pfn);
 962
 963         /* we don't need lock here; nobody else touches the iova range */
 964         do {
 965                 large_page = 1;
 966                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 967                 if (!pte) {
 968                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 969                         continue;
 970                 }
 971                 do {
 972                         dma_clear_pte(pte);
 973                         start_pfn += lvl_to_nr_pages(large_page);
 974                         pte++;
 975                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 976
 977                 domain_flush_cache(domain, first_pte,
 978                                    (void *)pte - (void *)first_pte);
 979
 980         } while (start_pfn && start_pfn <= last_pfn);
 981 }
 982
 983 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 984                                int retain_level, struct dma_pte *pte,
 985                                unsigned long pfn, unsigned long start_pfn,
 986                                unsigned long last_pfn)
 987 {
 988         pfn = max(start_pfn, pfn);
 989         pte = &pte[pfn_level_offset(pfn, level)];
 990
 991         do {
 992                 unsigned long level_pfn;
 993                 struct dma_pte *level_pte;
 994
 995                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 996                         goto next;
 997
 998                 level_pfn = pfn & level_mask(level);
 999                 level_pte = phys_to_virt(dma_pte_addr(pte));
1000
1001                 if (level > 2) {
1002                         dma_pte_free_level(domain, level - 1, retain_level,
1003                                            level_pte, level_pfn, start_pfn,
1004                                            last_pfn);
1005                 }
1006
1007                 /*
1008                  * Free the page table if we're below the level we want to
1009                  * retain and the range covers the entire table.
1010                  */
1011                 if (level < retain_level && !(start_pfn > level_pfn ||
1012                       last_pfn < level_pfn + level_size(level) - 1)) {
1013                         dma_clear_pte(pte);
1014                         domain_flush_cache(domain, pte, sizeof(*pte));
1015                         free_pgtable_page(level_pte);
1016                 }
1017 next:
1018                 pfn += level_size(level);
1019         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1020 }
1021
1022 /*
1023  * clear last level (leaf) ptes and free page table pages below the
1024  * level we wish to keep intact.
1025  */
1026 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1027                                    unsigned long start_pfn,
1028                                    unsigned long last_pfn,
1029                                    int retain_level)
1030 {
1031         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1032         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1033         BUG_ON(start_pfn > last_pfn);
1034
1035         dma_pte_clear_range(domain, start_pfn, last_pfn);
1036
1037         /* We don't need lock here; nobody else touches the iova range */
1038         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1039                            domain->pgd, 0, start_pfn, last_pfn);
1040
1041         /* free pgd */
1042         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1043                 free_pgtable_page(domain->pgd);
1044                 domain->pgd = NULL;
1045         }
1046 }
1047
1048 /* When a page at a given level is being unlinked from its parent, we don't
1049    need to *modify* it at all. All we need to do is make a list of all the
1050    pages which can be freed just as soon as we've flushed the IOTLB and we
1051    know the hardware page-walk will no longer touch them.
1052    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1053    be freed. */
1054 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1055                                             int level, struct dma_pte *pte,
1056                                             struct page *freelist)
1057 {
1058         struct page *pg;
1059
1060         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1061         pg->freelist = freelist;
1062         freelist = pg;
1063
1064         if (level == 1)
1065                 return freelist;
1066
1067         pte = page_address(pg);
1068         do {
1069                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1070                         freelist = dma_pte_list_pagetables(domain, level - 1,
1071                                                            pte, freelist);
1072                 pte++;
1073         } while (!first_pte_in_page(pte));
1074
1075         return freelist;
1076 }
1077
1078 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1079                                         struct dma_pte *pte, unsigned long pfn,
1080                                         unsigned long start_pfn,
1081                                         unsigned long last_pfn,
1082                                         struct page *freelist)
1083 {
1084         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1085
1086         pfn = max(start_pfn, pfn);
1087         pte = &pte[pfn_level_offset(pfn, level)];
1088
1089         do {
1090                 unsigned long level_pfn;
1091
1092                 if (!dma_pte_present(pte))
1093                         goto next;
1094
1095                 level_pfn = pfn & level_mask(level);
1096
1097                 /* If range covers entire pagetable, free it */
1098                 if (start_pfn <= level_pfn &&
1099                     last_pfn >= level_pfn + level_size(level) - 1) {
1100                         /* These suborbinate page tables are going away entirely. Don't
1101                            bother to clear them; we're just going to *free* them. */
1102                         if (level > 1 && !dma_pte_superpage(pte))
1103                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1104
1105                         dma_clear_pte(pte);
1106                         if (!first_pte)
1107                                 first_pte = pte;
1108                         last_pte = pte;
1109                 } else if (level > 1) {
1110                         /* Recurse down into a level that isn't *entirely* obsolete */
1111                         freelist = dma_pte_clear_level(domain, level - 1,
1112                                                        phys_to_virt(dma_pte_addr(pte)),
1113                                                        level_pfn, start_pfn, last_pfn,
1114                                                        freelist);
1115                 }
1116 next:
1117                 pfn += level_size(level);
1118         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1119
1120         if (first_pte)
1121                 domain_flush_cache(domain, first_pte,
1122                                    (void *)++last_pte - (void *)first_pte);
1123
1124         return freelist;
1125 }
1126
1127 /* We can't just free the pages because the IOMMU may still be walking
1128    the page tables, and may have cached the intermediate levels. The
1129    pages can only be freed after the IOTLB flush has been done. */
1130 static struct page *domain_unmap(struct dmar_domain *domain,
1131                                  unsigned long start_pfn,
1132                                  unsigned long last_pfn)
1133 {
1134         struct page *freelist = NULL;
1135
1136         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1137         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1138         BUG_ON(start_pfn > last_pfn);
1139
1140         /* we don't need lock here; nobody else touches the iova range */
1141         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1142                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1143
1144         /* free pgd */
1145         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1146                 struct page *pgd_page = virt_to_page(domain->pgd);
1147                 pgd_page->freelist = freelist;
1148                 freelist = pgd_page;
1149
1150                 domain->pgd = NULL;
1151         }
1152
1153         return freelist;
1154 }
1155
1156 static void dma_free_pagelist(struct page *freelist)
1157 {
1158         struct page *pg;
1159
1160         while ((pg = freelist)) {
1161                 freelist = pg->freelist;
1162                 free_pgtable_page(page_address(pg));
1163         }
1164 }
1165
1166 static void iova_entry_free(unsigned long data)
1167 {
1168         struct page *freelist = (struct page *)data;
1169
1170         dma_free_pagelist(freelist);
1171 }
1172
1173 /* iommu handling */
1174 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1175 {
1176         struct root_entry *root;
1177         unsigned long flags;
1178
1179         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1180         if (!root) {
1181                 pr_err("Allocating root entry for %s failed\n",
1182                         iommu->name);
1183                 return -ENOMEM;
1184         }
1185
1186         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1187
1188         spin_lock_irqsave(&iommu->lock, flags);
1189         iommu->root_entry = root;
1190         spin_unlock_irqrestore(&iommu->lock, flags);
1191
1192         return 0;
1193 }
1194
1195 static void iommu_set_root_entry(struct intel_iommu *iommu)
1196 {
1197         u64 addr;
1198         u32 sts;
1199         unsigned long flag;
1200
1201         addr = virt_to_phys(iommu->root_entry);
1202         if (sm_supported(iommu))
1203                 addr |= DMA_RTADDR_SMT;
1204
1205         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1206         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1207
1208         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1209
1210         /* Make sure hardware complete it */
1211         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1212                       readl, (sts & DMA_GSTS_RTPS), sts);
1213
1214         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1215 }
1216
1217 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1218 {
1219         u32 val;
1220         unsigned long flag;
1221
1222         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1223                 return;
1224
1225         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1226         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1227
1228         /* Make sure hardware complete it */
1229         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1230                       readl, (!(val & DMA_GSTS_WBFS)), val);
1231
1232         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1233 }
1234
1235 /* return value determine if we need a write buffer flush */
1236 static void __iommu_flush_context(struct intel_iommu *iommu,
1237                                   u16 did, u16 source_id, u8 function_mask,
1238                                   u64 type)
1239 {
1240         u64 val = 0;
1241         unsigned long flag;
1242
1243         switch (type) {
1244         case DMA_CCMD_GLOBAL_INVL:
1245                 val = DMA_CCMD_GLOBAL_INVL;
1246                 break;
1247         case DMA_CCMD_DOMAIN_INVL:
1248                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1249                 break;
1250         case DMA_CCMD_DEVICE_INVL:
1251                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1252                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1253                 break;
1254         default:
1255                 BUG();
1256         }
1257         val |= DMA_CCMD_ICC;
1258
1259         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1260         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1261
1262         /* Make sure hardware complete it */
1263         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1264                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1265
1266         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1267 }
1268
1269 /* return value determine if we need a write buffer flush */
1270 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1271                                 u64 addr, unsigned int size_order, u64 type)
1272 {
1273         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1274         u64 val = 0, val_iva = 0;
1275         unsigned long flag;
1276
1277         switch (type) {
1278         case DMA_TLB_GLOBAL_FLUSH:
1279                 /* global flush doesn't need set IVA_REG */
1280                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1281                 break;
1282         case DMA_TLB_DSI_FLUSH:
1283                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1284                 break;
1285         case DMA_TLB_PSI_FLUSH:
1286                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1287                 /* IH bit is passed in as part of address */
1288                 val_iva = size_order | addr;
1289                 break;
1290         default:
1291                 BUG();
1292         }
1293         /* Note: set drain read/write */
1294 #if 0
1295         /*
1296          * This is probably to be super secure.. Looks like we can
1297          * ignore it without any impact.
1298          */
1299         if (cap_read_drain(iommu->cap))
1300                 val |= DMA_TLB_READ_DRAIN;
1301 #endif
1302         if (cap_write_drain(iommu->cap))
1303                 val |= DMA_TLB_WRITE_DRAIN;
1304
1305         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1306         /* Note: Only uses first TLB reg currently */
1307         if (val_iva)
1308                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1309         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1310
1311         /* Make sure hardware complete it */
1312         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1313                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1314
1315         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1316
1317         /* check IOTLB invalidation granularity */
1318         if (DMA_TLB_IAIG(val) == 0)
1319                 pr_err("Flush IOTLB failed\n");
1320         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1321                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1322                         (unsigned long long)DMA_TLB_IIRG(type),
1323                         (unsigned long long)DMA_TLB_IAIG(val));
1324 }
1325
1326 static struct device_domain_info *
1327 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1328                          u8 bus, u8 devfn)
1329 {
1330         struct device_domain_info *info;
1331
1332         assert_spin_locked(&device_domain_lock);
1333
1334         if (!iommu->qi)
1335                 return NULL;
1336
1337         list_for_each_entry(info, &domain->devices, link)
1338                 if (info->iommu == iommu && info->bus == bus &&
1339                     info->devfn == devfn) {
1340                         if (info->ats_supported && info->dev)
1341                                 return info;
1342                         break;
1343                 }
1344
1345         return NULL;
1346 }
1347
1348 static void domain_update_iotlb(struct dmar_domain *domain)
1349 {
1350         struct device_domain_info *info;
1351         bool has_iotlb_device = false;
1352
1353         assert_spin_locked(&device_domain_lock);
1354
1355         list_for_each_entry(info, &domain->devices, link) {
1356                 struct pci_dev *pdev;
1357
1358                 if (!info->dev || !dev_is_pci(info->dev))
1359                         continue;
1360
1361                 pdev = to_pci_dev(info->dev);
1362                 if (pdev->ats_enabled) {
1363                         has_iotlb_device = true;
1364                         break;
1365                 }
1366         }
1367
1368         domain->has_iotlb_device = has_iotlb_device;
1369 }
1370
1371 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1372 {
1373         struct pci_dev *pdev;
1374
1375         assert_spin_locked(&device_domain_lock);
1376
1377         if (!info || !dev_is_pci(info->dev))
1378                 return;
1379
1380         pdev = to_pci_dev(info->dev);
1381         /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1382          * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1383          * queue depth at PF level. If DIT is not set, PFSID will be treated as
1384          * reserved, which should be set to 0.
1385          */
1386         if (!ecap_dit(info->iommu->ecap))
1387                 info->pfsid = 0;
1388         else {
1389                 struct pci_dev *pf_pdev;
1390
1391                 /* pdev will be returned if device is not a vf */
1392                 pf_pdev = pci_physfn(pdev);
1393                 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1394         }
1395
1396 #ifdef CONFIG_INTEL_IOMMU_SVM
1397         /* The PCIe spec, in its wisdom, declares that the behaviour of
1398            the device if you enable PASID support after ATS support is
1399            undefined. So always enable PASID support on devices which
1400            have it, even if we can't yet know if we're ever going to
1401            use it. */
1402         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1403                 info->pasid_enabled = 1;
1404
1405         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1406                 info->pri_enabled = 1;
1407 #endif
1408         if (!pdev->untrusted && info->ats_supported &&
1409             !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1410                 info->ats_enabled = 1;
1411                 domain_update_iotlb(info->domain);
1412                 info->ats_qdep = pci_ats_queue_depth(pdev);
1413         }
1414 }
1415
1416 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1417 {
1418         struct pci_dev *pdev;
1419
1420         assert_spin_locked(&device_domain_lock);
1421
1422         if (!dev_is_pci(info->dev))
1423                 return;
1424
1425         pdev = to_pci_dev(info->dev);
1426
1427         if (info->ats_enabled) {
1428                 pci_disable_ats(pdev);
1429                 info->ats_enabled = 0;
1430                 domain_update_iotlb(info->domain);
1431         }
1432 #ifdef CONFIG_INTEL_IOMMU_SVM
1433         if (info->pri_enabled) {
1434                 pci_disable_pri(pdev);
1435                 info->pri_enabled = 0;
1436         }
1437         if (info->pasid_enabled) {
1438                 pci_disable_pasid(pdev);
1439                 info->pasid_enabled = 0;
1440         }
1441 #endif
1442 }
1443
1444 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1445                                   u64 addr, unsigned mask)
1446 {
1447         u16 sid, qdep;
1448         unsigned long flags;
1449         struct device_domain_info *info;
1450
1451         if (!domain->has_iotlb_device)
1452                 return;
1453
1454         spin_lock_irqsave(&device_domain_lock, flags);
1455         list_for_each_entry(info, &domain->devices, link) {
1456                 if (!info->ats_enabled)
1457                         continue;
1458
1459                 sid = info->bus << 8 | info->devfn;
1460                 qdep = info->ats_qdep;
1461                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1462                                 qdep, addr, mask);
1463         }
1464         spin_unlock_irqrestore(&device_domain_lock, flags);
1465 }
1466
1467 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1468                                   struct dmar_domain *domain,
1469                                   unsigned long pfn, unsigned int pages,
1470                                   int ih, int map)
1471 {
1472         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1473         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1474         u16 did = domain->iommu_did[iommu->seq_id];
1475
1476         BUG_ON(pages == 0);
1477
1478         if (ih)
1479                 ih = 1 << 6;
1480         /*
1481          * Fallback to domain selective flush if no PSI support or the size is
1482          * too big.
1483          * PSI requires page size to be 2 ^ x, and the base address is naturally
1484          * aligned to the size
1485          */
1486         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1487                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1488                                                 DMA_TLB_DSI_FLUSH);
1489         else
1490                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1491                                                 DMA_TLB_PSI_FLUSH);
1492
1493         /*
1494          * In caching mode, changes of pages from non-present to present require
1495          * flush. However, device IOTLB doesn't need to be flushed in this case.
1496          */
1497         if (!cap_caching_mode(iommu->cap) || !map)
1498                 iommu_flush_dev_iotlb(domain, addr, mask);
1499 }
1500
1501 /* Notification for newly created mappings */
1502 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1503                                         struct dmar_domain *domain,
1504                                         unsigned long pfn, unsigned int pages)
1505 {
1506         /* It's a non-present to present mapping. Only flush if caching mode */
1507         if (cap_caching_mode(iommu->cap))
1508                 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1509         else
1510                 iommu_flush_write_buffer(iommu);
1511 }
1512
1513 static void iommu_flush_iova(struct iova_domain *iovad)
1514 {
1515         struct dmar_domain *domain;
1516         int idx;
1517
1518         domain = container_of(iovad, struct dmar_domain, iovad);
1519
1520         for_each_domain_iommu(idx, domain) {
1521                 struct intel_iommu *iommu = g_iommus[idx];
1522                 u16 did = domain->iommu_did[iommu->seq_id];
1523
1524                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1525
1526                 if (!cap_caching_mode(iommu->cap))
1527                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1528                                               0, MAX_AGAW_PFN_WIDTH);
1529         }
1530 }
1531
1532 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1533 {
1534         u32 pmen;
1535         unsigned long flags;
1536
1537         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1538         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1539         pmen &= ~DMA_PMEN_EPM;
1540         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1541
1542         /* wait for the protected region status bit to clear */
1543         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1544                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1545
1546         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1547 }
1548
1549 static void iommu_enable_translation(struct intel_iommu *iommu)
1550 {
1551         u32 sts;
1552         unsigned long flags;
1553
1554         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1555         iommu->gcmd |= DMA_GCMD_TE;
1556         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1557
1558         /* Make sure hardware complete it */
1559         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1560                       readl, (sts & DMA_GSTS_TES), sts);
1561
1562         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1563 }
1564
1565 static void iommu_disable_translation(struct intel_iommu *iommu)
1566 {
1567         u32 sts;
1568         unsigned long flag;
1569
1570         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1571         iommu->gcmd &= ~DMA_GCMD_TE;
1572         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1573
1574         /* Make sure hardware complete it */
1575         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1576                       readl, (!(sts & DMA_GSTS_TES)), sts);
1577
1578         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1579 }
1580
1581
1582 static int iommu_init_domains(struct intel_iommu *iommu)
1583 {
1584         u32 ndomains, nlongs;
1585         size_t size;
1586
1587         ndomains = cap_ndoms(iommu->cap);
1588         pr_debug("%s: Number of Domains supported <%d>\n",
1589                  iommu->name, ndomains);
1590         nlongs = BITS_TO_LONGS(ndomains);
1591
1592         spin_lock_init(&iommu->lock);
1593
1594         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1595         if (!iommu->domain_ids) {
1596                 pr_err("%s: Allocating domain id array failed\n",
1597                        iommu->name);
1598                 return -ENOMEM;
1599         }
1600
1601         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1602         iommu->domains = kzalloc(size, GFP_KERNEL);
1603
1604         if (iommu->domains) {
1605                 size = 256 * sizeof(struct dmar_domain *);
1606                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1607         }
1608
1609         if (!iommu->domains || !iommu->domains[0]) {
1610                 pr_err("%s: Allocating domain array failed\n",
1611                        iommu->name);
1612                 kfree(iommu->domain_ids);
1613                 kfree(iommu->domains);
1614                 iommu->domain_ids = NULL;
1615                 iommu->domains    = NULL;
1616                 return -ENOMEM;
1617         }
1618
1619
1620
1621         /*
1622          * If Caching mode is set, then invalid translations are tagged
1623          * with domain-id 0, hence we need to pre-allocate it. We also
1624          * use domain-id 0 as a marker for non-allocated domain-id, so
1625          * make sure it is not used for a real domain.
1626          */
1627         set_bit(0, iommu->domain_ids);
1628
1629         /*
1630          * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1631          * entry for first-level or pass-through translation modes should
1632          * be programmed with a domain id different from those used for
1633          * second-level or nested translation. We reserve a domain id for
1634          * this purpose.
1635          */
1636         if (sm_supported(iommu))
1637                 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1638
1639         return 0;
1640 }
1641
1642 static void disable_dmar_iommu(struct intel_iommu *iommu)
1643 {
1644         struct device_domain_info *info, *tmp;
1645         unsigned long flags;
1646
1647         if (!iommu->domains || !iommu->domain_ids)
1648                 return;
1649
1650 again:
1651         spin_lock_irqsave(&device_domain_lock, flags);
1652         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1653                 struct dmar_domain *domain;
1654
1655                 if (info->iommu != iommu)
1656                         continue;
1657
1658                 if (!info->dev || !info->domain)
1659                         continue;
1660
1661                 domain = info->domain;
1662
1663                 __dmar_remove_one_dev_info(info);
1664
1665                 if (!domain_type_is_vm_or_si(domain)) {
1666                         /*
1667                          * The domain_exit() function  can't be called under
1668                          * device_domain_lock, as it takes this lock itself.
1669                          * So release the lock here and re-run the loop
1670                          * afterwards.
1671                          */
1672                         spin_unlock_irqrestore(&device_domain_lock, flags);
1673                         domain_exit(domain);
1674                         goto again;
1675                 }
1676         }
1677         spin_unlock_irqrestore(&device_domain_lock, flags);
1678
1679         if (iommu->gcmd & DMA_GCMD_TE)
1680                 iommu_disable_translation(iommu);
1681 }
1682
1683 static void free_dmar_iommu(struct intel_iommu *iommu)
1684 {
1685         if ((iommu->domains) && (iommu->domain_ids)) {
1686                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1687                 int i;
1688
1689                 for (i = 0; i < elems; i++)
1690                         kfree(iommu->domains[i]);
1691                 kfree(iommu->domains);
1692                 kfree(iommu->domain_ids);
1693                 iommu->domains = NULL;
1694                 iommu->domain_ids = NULL;
1695         }
1696
1697         g_iommus[iommu->seq_id] = NULL;
1698
1699         /* free context mapping */
1700         free_context_table(iommu);
1701
1702 #ifdef CONFIG_INTEL_IOMMU_SVM
1703         if (pasid_supported(iommu)) {
1704                 if (ecap_prs(iommu->ecap))
1705                         intel_svm_finish_prq(iommu);
1706         }
1707 #endif
1708 }
1709
1710 static struct dmar_domain *alloc_domain(int flags)
1711 {
1712         struct dmar_domain *domain;
1713
1714         domain = alloc_domain_mem();
1715         if (!domain)
1716                 return NULL;
1717
1718         memset(domain, 0, sizeof(*domain));
1719         domain->nid = -1;
1720         domain->flags = flags;
1721         domain->has_iotlb_device = false;
1722         INIT_LIST_HEAD(&domain->devices);
1723
1724         return domain;
1725 }
1726
1727 /* Must be called with iommu->lock */
1728 static int domain_attach_iommu(struct dmar_domain *domain,
1729                                struct intel_iommu *iommu)
1730 {
1731         unsigned long ndomains;
1732         int num;
1733
1734         assert_spin_locked(&device_domain_lock);
1735         assert_spin_locked(&iommu->lock);
1736
1737         domain->iommu_refcnt[iommu->seq_id] += 1;
1738         domain->iommu_count += 1;
1739         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1740                 ndomains = cap_ndoms(iommu->cap);
1741                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1742
1743                 if (num >= ndomains) {
1744                         pr_err("%s: No free domain ids\n", iommu->name);
1745                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1746                         domain->iommu_count -= 1;
1747                         return -ENOSPC;
1748                 }
1749
1750                 set_bit(num, iommu->domain_ids);
1751                 set_iommu_domain(iommu, num, domain);
1752
1753                 domain->iommu_did[iommu->seq_id] = num;
1754                 domain->nid                      = iommu->node;
1755
1756                 domain_update_iommu_cap(domain);
1757         }
1758
1759         return 0;
1760 }
1761
1762 static int domain_detach_iommu(struct dmar_domain *domain,
1763                                struct intel_iommu *iommu)
1764 {
1765         int num, count = INT_MAX;
1766
1767         assert_spin_locked(&device_domain_lock);
1768         assert_spin_locked(&iommu->lock);
1769
1770         domain->iommu_refcnt[iommu->seq_id] -= 1;
1771         count = --domain->iommu_count;
1772         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1773                 num = domain->iommu_did[iommu->seq_id];
1774                 clear_bit(num, iommu->domain_ids);
1775                 set_iommu_domain(iommu, num, NULL);
1776
1777                 domain_update_iommu_cap(domain);
1778                 domain->iommu_did[iommu->seq_id] = 0;
1779         }
1780
1781         return count;
1782 }
1783
1784 static struct iova_domain reserved_iova_list;
1785 static struct lock_class_key reserved_rbtree_key;
1786
1787 static int dmar_init_reserved_ranges(void)
1788 {
1789         struct pci_dev *pdev = NULL;
1790         struct iova *iova;
1791         int i;
1792
1793         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1794
1795         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1796                 &reserved_rbtree_key);
1797
1798         /* IOAPIC ranges shouldn't be accessed by DMA */
1799         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1800                 IOVA_PFN(IOAPIC_RANGE_END));
1801         if (!iova) {
1802                 pr_err("Reserve IOAPIC range failed\n");
1803                 return -ENODEV;
1804         }
1805
1806         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1807         for_each_pci_dev(pdev) {
1808                 struct resource *r;
1809
1810                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1811                         r = &pdev->resource[i];
1812                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1813                                 continue;
1814                         iova = reserve_iova(&reserved_iova_list,
1815                                             IOVA_PFN(r->start),
1816                                             IOVA_PFN(r->end));
1817                         if (!iova) {
1818                                 pr_err("Reserve iova failed\n");
1819                                 return -ENODEV;
1820                         }
1821                 }
1822         }
1823         return 0;
1824 }
1825
1826 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1827 {
1828         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1829 }
1830
1831 static inline int guestwidth_to_adjustwidth(int gaw)
1832 {
1833         int agaw;
1834         int r = (gaw - 12) % 9;
1835
1836         if (r == 0)
1837                 agaw = gaw;
1838         else
1839                 agaw = gaw + 9 - r;
1840         if (agaw > 64)
1841                 agaw = 64;
1842         return agaw;
1843 }
1844
1845 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1846                        int guest_width)
1847 {
1848         int adjust_width, agaw;
1849         unsigned long sagaw;
1850         int err;
1851
1852         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1853
1854         err = init_iova_flush_queue(&domain->iovad,
1855                                     iommu_flush_iova, iova_entry_free);
1856         if (err)
1857                 return err;
1858
1859         domain_reserve_special_ranges(domain);
1860
1861         /* calculate AGAW */
1862         if (guest_width > cap_mgaw(iommu->cap))
1863                 guest_width = cap_mgaw(iommu->cap);
1864         domain->gaw = guest_width;
1865         adjust_width = guestwidth_to_adjustwidth(guest_width);
1866         agaw = width_to_agaw(adjust_width);
1867         sagaw = cap_sagaw(iommu->cap);
1868         if (!test_bit(agaw, &sagaw)) {
1869                 /* hardware doesn't support it, choose a bigger one */
1870                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1871                 agaw = find_next_bit(&sagaw, 5, agaw);
1872                 if (agaw >= 5)
1873                         return -ENODEV;
1874         }
1875         domain->agaw = agaw;
1876
1877         if (ecap_coherent(iommu->ecap))
1878                 domain->iommu_coherency = 1;
1879         else
1880                 domain->iommu_coherency = 0;
1881
1882         if (ecap_sc_support(iommu->ecap))
1883                 domain->iommu_snooping = 1;
1884         else
1885                 domain->iommu_snooping = 0;
1886
1887         if (intel_iommu_superpage)
1888                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1889         else
1890                 domain->iommu_superpage = 0;
1891
1892         domain->nid = iommu->node;
1893
1894         /* always allocate the top pgd */
1895         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1896         if (!domain->pgd)
1897                 return -ENOMEM;
1898         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1899         return 0;
1900 }
1901
1902 static void domain_exit(struct dmar_domain *domain)
1903 {
1904         struct page *freelist = NULL;
1905
1906         /* Domain 0 is reserved, so dont process it */
1907         if (!domain)
1908                 return;
1909
1910         /* Remove associated devices and clear attached or cached domains */
1911         rcu_read_lock();
1912         domain_remove_dev_info(domain);
1913         rcu_read_unlock();
1914
1915         /* destroy iovas */
1916         put_iova_domain(&domain->iovad);
1917
1918         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1919
1920         dma_free_pagelist(freelist);
1921
1922         free_domain_mem(domain);
1923 }
1924
1925 /*
1926  * Get the PASID directory size for scalable mode context entry.
1927  * Value of X in the PDTS field of a scalable mode context entry
1928  * indicates PASID directory with 2^(X + 7) entries.
1929  */
1930 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1931 {
1932         int pds, max_pde;
1933
1934         max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1935         pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1936         if (pds < 7)
1937                 return 0;
1938
1939         return pds - 7;
1940 }
1941
1942 /*
1943  * Set the RID_PASID field of a scalable mode context entry. The
1944  * IOMMU hardware will use the PASID value set in this field for
1945  * DMA translations of DMA requests without PASID.
1946  */
1947 static inline void
1948 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1949 {
1950         context->hi |= pasid & ((1 << 20) - 1);
1951         context->hi |= (1 << 20);
1952 }
1953
1954 /*
1955  * Set the DTE(Device-TLB Enable) field of a scalable mode context
1956  * entry.
1957  */
1958 static inline void context_set_sm_dte(struct context_entry *context)
1959 {
1960         context->lo |= (1 << 2);
1961 }
1962
1963 /*
1964  * Set the PRE(Page Request Enable) field of a scalable mode context
1965  * entry.
1966  */
1967 static inline void context_set_sm_pre(struct context_entry *context)
1968 {
1969         context->lo |= (1 << 4);
1970 }
1971
1972 /* Convert value to context PASID directory size field coding. */
1973 #define context_pdts(pds)       (((pds) & 0x7) << 9)
1974
1975 static int domain_context_mapping_one(struct dmar_domain *domain,
1976                                       struct intel_iommu *iommu,
1977                                       struct pasid_table *table,
1978                                       u8 bus, u8 devfn)
1979 {
1980         u16 did = domain->iommu_did[iommu->seq_id];
1981         int translation = CONTEXT_TT_MULTI_LEVEL;
1982         struct device_domain_info *info = NULL;
1983         struct context_entry *context;
1984         unsigned long flags;
1985         int ret;
1986
1987         WARN_ON(did == 0);
1988
1989         if (hw_pass_through && domain_type_is_si(domain))
1990                 translation = CONTEXT_TT_PASS_THROUGH;
1991
1992         pr_debug("Set context mapping for %02x:%02x.%d\n",
1993                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1994
1995         BUG_ON(!domain->pgd);
1996
1997         spin_lock_irqsave(&device_domain_lock, flags);
1998         spin_lock(&iommu->lock);
1999
2000         ret = -ENOMEM;
2001         context = iommu_context_addr(iommu, bus, devfn, 1);
2002         if (!context)
2003                 goto out_unlock;
2004
2005         ret = 0;
2006         if (context_present(context))
2007                 goto out_unlock;
2008
2009         /*
2010          * For kdump cases, old valid entries may be cached due to the
2011          * in-flight DMA and copied pgtable, but there is no unmapping
2012          * behaviour for them, thus we need an explicit cache flush for
2013          * the newly-mapped device. For kdump, at this point, the device
2014          * is supposed to finish reset at its driver probe stage, so no
2015          * in-flight DMA will exist, and we don't need to worry anymore
2016          * hereafter.
2017          */
2018         if (context_copied(context)) {
2019                 u16 did_old = context_domain_id(context);
2020
2021                 if (did_old < cap_ndoms(iommu->cap)) {
2022                         iommu->flush.flush_context(iommu, did_old,
2023                                                    (((u16)bus) << 8) | devfn,
2024                                                    DMA_CCMD_MASK_NOBIT,
2025                                                    DMA_CCMD_DEVICE_INVL);
2026                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2027                                                  DMA_TLB_DSI_FLUSH);
2028                 }
2029         }
2030
2031         context_clear_entry(context);
2032
2033         if (sm_supported(iommu)) {
2034                 unsigned long pds;
2035
2036                 WARN_ON(!table);
2037
2038                 /* Setup the PASID DIR pointer: */
2039                 pds = context_get_sm_pds(table);
2040                 context->lo = (u64)virt_to_phys(table->table) |
2041                                 context_pdts(pds);
2042
2043                 /* Setup the RID_PASID field: */
2044                 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2045
2046                 /*
2047                  * Setup the Device-TLB enable bit and Page request
2048                  * Enable bit:
2049                  */
2050                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2051                 if (info && info->ats_supported)
2052                         context_set_sm_dte(context);
2053                 if (info && info->pri_supported)
2054                         context_set_sm_pre(context);
2055         } else {
2056                 struct dma_pte *pgd = domain->pgd;
2057                 int agaw;
2058
2059                 context_set_domain_id(context, did);
2060                 context_set_translation_type(context, translation);
2061
2062                 if (translation != CONTEXT_TT_PASS_THROUGH) {
2063                         /*
2064                          * Skip top levels of page tables for iommu which has
2065                          * less agaw than default. Unnecessary for PT mode.
2066                          */
2067                         for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2068                                 ret = -ENOMEM;
2069                                 pgd = phys_to_virt(dma_pte_addr(pgd));
2070                                 if (!dma_pte_present(pgd))
2071                                         goto out_unlock;
2072                         }
2073
2074                         info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2075                         if (info && info->ats_supported)
2076                                 translation = CONTEXT_TT_DEV_IOTLB;
2077                         else
2078                                 translation = CONTEXT_TT_MULTI_LEVEL;
2079
2080                         context_set_address_root(context, virt_to_phys(pgd));
2081                         context_set_address_width(context, agaw);
2082                 } else {
2083                         /*
2084                          * In pass through mode, AW must be programmed to
2085                          * indicate the largest AGAW value supported by
2086                          * hardware. And ASR is ignored by hardware.
2087                          */
2088                         context_set_address_width(context, iommu->msagaw);
2089                 }
2090         }
2091
2092         context_set_fault_enable(context);
2093         context_set_present(context);
2094         domain_flush_cache(domain, context, sizeof(*context));
2095
2096         /*
2097          * It's a non-present to present mapping. If hardware doesn't cache
2098          * non-present entry we only need to flush the write-buffer. If the
2099          * _does_ cache non-present entries, then it does so in the special
2100          * domain #0, which we have to flush:
2101          */
2102         if (cap_caching_mode(iommu->cap)) {
2103                 iommu->flush.flush_context(iommu, 0,
2104                                            (((u16)bus) << 8) | devfn,
2105                                            DMA_CCMD_MASK_NOBIT,
2106                                            DMA_CCMD_DEVICE_INVL);
2107                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2108         } else {
2109                 iommu_flush_write_buffer(iommu);
2110         }
2111         iommu_enable_dev_iotlb(info);
2112
2113         ret = 0;
2114
2115 out_unlock:
2116         spin_unlock(&iommu->lock);
2117         spin_unlock_irqrestore(&device_domain_lock, flags);
2118
2119         return ret;
2120 }
2121
2122 struct domain_context_mapping_data {
2123         struct dmar_domain *domain;
2124         struct intel_iommu *iommu;
2125         struct pasid_table *table;
2126 };
2127
2128 static int domain_context_mapping_cb(struct pci_dev *pdev,
2129                                      u16 alias, void *opaque)
2130 {
2131         struct domain_context_mapping_data *data = opaque;
2132
2133         return domain_context_mapping_one(data->domain, data->iommu,
2134                                           data->table, PCI_BUS_NUM(alias),
2135                                           alias & 0xff);
2136 }
2137
2138 static int
2139 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2140 {
2141         struct domain_context_mapping_data data;
2142         struct pasid_table *table;
2143         struct intel_iommu *iommu;
2144         u8 bus, devfn;
2145
2146         iommu = device_to_iommu(dev, &bus, &devfn);
2147         if (!iommu)
2148                 return -ENODEV;
2149
2150         table = intel_pasid_get_table(dev);
2151
2152         if (!dev_is_pci(dev))
2153                 return domain_context_mapping_one(domain, iommu, table,
2154                                                   bus, devfn);
2155
2156         data.domain = domain;
2157         data.iommu = iommu;
2158         data.table = table;
2159
2160         return pci_for_each_dma_alias(to_pci_dev(dev),
2161                                       &domain_context_mapping_cb, &data);
2162 }
2163
2164 static int domain_context_mapped_cb(struct pci_dev *pdev,
2165                                     u16 alias, void *opaque)
2166 {
2167         struct intel_iommu *iommu = opaque;
2168
2169         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2170 }
2171
2172 static int domain_context_mapped(struct device *dev)
2173 {
2174         struct intel_iommu *iommu;
2175         u8 bus, devfn;
2176
2177         iommu = device_to_iommu(dev, &bus, &devfn);
2178         if (!iommu)
2179                 return -ENODEV;
2180
2181         if (!dev_is_pci(dev))
2182                 return device_context_mapped(iommu, bus, devfn);
2183
2184         return !pci_for_each_dma_alias(to_pci_dev(dev),
2185                                        domain_context_mapped_cb, iommu);
2186 }
2187
2188 /* Returns a number of VTD pages, but aligned to MM page size */
2189 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2190                                             size_t size)
2191 {
2192         host_addr &= ~PAGE_MASK;
2193         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2194 }
2195
2196 /* Return largest possible superpage level for a given mapping */
2197 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2198                                           unsigned long iov_pfn,
2199                                           unsigned long phy_pfn,
2200                                           unsigned long pages)
2201 {
2202         int support, level = 1;
2203         unsigned long pfnmerge;
2204
2205         support = domain->iommu_superpage;
2206
2207         /* To use a large page, the virtual *and* physical addresses
2208            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2209            of them will mean we have to use smaller pages. So just
2210            merge them and check both at once. */
2211         pfnmerge = iov_pfn | phy_pfn;
2212
2213         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2214                 pages >>= VTD_STRIDE_SHIFT;
2215                 if (!pages)
2216                         break;
2217                 pfnmerge >>= VTD_STRIDE_SHIFT;
2218                 level++;
2219                 support--;
2220         }
2221         return level;
2222 }
2223
2224 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2225                             struct scatterlist *sg, unsigned long phys_pfn,
2226                             unsigned long nr_pages, int prot)
2227 {
2228         struct dma_pte *first_pte = NULL, *pte = NULL;
2229         phys_addr_t uninitialized_var(pteval);
2230         unsigned long sg_res = 0;
2231         unsigned int largepage_lvl = 0;
2232         unsigned long lvl_pages = 0;
2233
2234         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2235
2236         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2237                 return -EINVAL;
2238
2239         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2240
2241         if (!sg) {
2242                 sg_res = nr_pages;
2243                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2244         }
2245
2246         while (nr_pages > 0) {
2247                 uint64_t tmp;
2248
2249                 if (!sg_res) {
2250                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2251
2252                         sg_res = aligned_nrpages(sg->offset, sg->length);
2253                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2254                         sg->dma_length = sg->length;
2255                         pteval = (sg_phys(sg) - pgoff) | prot;
2256                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2257                 }
2258
2259                 if (!pte) {
2260                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2261
2262                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2263                         if (!pte)
2264                                 return -ENOMEM;
2265                         /* It is large page*/
2266                         if (largepage_lvl > 1) {
2267                                 unsigned long nr_superpages, end_pfn;
2268
2269                                 pteval |= DMA_PTE_LARGE_PAGE;
2270                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2271
2272                                 nr_superpages = sg_res / lvl_pages;
2273                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2274
2275                                 /*
2276                                  * Ensure that old small page tables are
2277                                  * removed to make room for superpage(s).
2278                                  * We're adding new large pages, so make sure
2279                                  * we don't remove their parent tables.
2280                                  */
2281                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2282                                                        largepage_lvl + 1);
2283                         } else {
2284                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2285                         }
2286
2287                 }
2288                 /* We don't need lock here, nobody else
2289                  * touches the iova range
2290                  */
2291                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2292                 if (tmp) {
2293                         static int dumps = 5;
2294                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2295                                 iov_pfn, tmp, (unsigned long long)pteval);
2296                         if (dumps) {
2297                                 dumps--;
2298                                 debug_dma_dump_mappings(NULL);
2299                         }
2300                         WARN_ON(1);
2301                 }
2302
2303                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2304
2305                 BUG_ON(nr_pages < lvl_pages);
2306                 BUG_ON(sg_res < lvl_pages);
2307
2308                 nr_pages -= lvl_pages;
2309                 iov_pfn += lvl_pages;
2310                 phys_pfn += lvl_pages;
2311                 pteval += lvl_pages * VTD_PAGE_SIZE;
2312                 sg_res -= lvl_pages;
2313
2314                 /* If the next PTE would be the first in a new page, then we
2315                    need to flush the cache on the entries we've just written.
2316                    And then we'll need to recalculate 'pte', so clear it and
2317                    let it get set again in the if (!pte) block above.
2318
2319                    If we're done (!nr_pages) we need to flush the cache too.
2320
2321                    Also if we've been setting superpages, we may need to
2322                    recalculate 'pte' and switch back to smaller pages for the
2323                    end of the mapping, if the trailing size is not enough to
2324                    use another superpage (i.e. sg_res < lvl_pages). */
2325                 pte++;
2326                 if (!nr_pages || first_pte_in_page(pte) ||
2327                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2328                         domain_flush_cache(domain, first_pte,
2329                                            (void *)pte - (void *)first_pte);
2330                         pte = NULL;
2331                 }
2332
2333                 if (!sg_res && nr_pages)
2334                         sg = sg_next(sg);
2335         }
2336         return 0;
2337 }
2338
2339 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2340                          struct scatterlist *sg, unsigned long phys_pfn,
2341                          unsigned long nr_pages, int prot)
2342 {
2343        int ret;
2344        struct intel_iommu *iommu;
2345
2346        /* Do the real mapping first */
2347        ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2348        if (ret)
2349                return ret;
2350
2351        /* Notify about the new mapping */
2352        if (domain_type_is_vm(domain)) {
2353                /* VM typed domains can have more than one IOMMUs */
2354                int iommu_id;
2355                for_each_domain_iommu(iommu_id, domain) {
2356                        iommu = g_iommus[iommu_id];
2357                        __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2358                }
2359        } else {
2360                /* General domains only have one IOMMU */
2361                iommu = domain_get_iommu(domain);
2362                __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2363        }
2364
2365        return 0;
2366 }
2367
2368 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2369                                     struct scatterlist *sg, unsigned long nr_pages,
2370                                     int prot)
2371 {
2372         return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2373 }
2374
2375 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2376                                      unsigned long phys_pfn, unsigned long nr_pages,
2377                                      int prot)
2378 {
2379         return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2380 }
2381
2382 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2383 {
2384         unsigned long flags;
2385         struct context_entry *context;
2386         u16 did_old;
2387
2388         if (!iommu)
2389                 return;
2390
2391         spin_lock_irqsave(&iommu->lock, flags);
2392         context = iommu_context_addr(iommu, bus, devfn, 0);
2393         if (!context) {
2394                 spin_unlock_irqrestore(&iommu->lock, flags);
2395                 return;
2396         }
2397         did_old = context_domain_id(context);
2398         context_clear_entry(context);
2399         __iommu_flush_cache(iommu, context, sizeof(*context));
2400         spin_unlock_irqrestore(&iommu->lock, flags);
2401         iommu->flush.flush_context(iommu,
2402                                    did_old,
2403                                    (((u16)bus) << 8) | devfn,
2404                                    DMA_CCMD_MASK_NOBIT,
2405                                    DMA_CCMD_DEVICE_INVL);
2406         iommu->flush.flush_iotlb(iommu,
2407                                  did_old,
2408                                  0,
2409                                  0,
2410                                  DMA_TLB_DSI_FLUSH);
2411 }
2412
2413 static inline void unlink_domain_info(struct device_domain_info *info)
2414 {
2415         assert_spin_locked(&device_domain_lock);
2416         list_del(&info->link);
2417         list_del(&info->global);
2418         if (info->dev)
2419                 info->dev->archdata.iommu = NULL;
2420 }
2421
2422 static void domain_remove_dev_info(struct dmar_domain *domain)
2423 {
2424         struct device_domain_info *info, *tmp;
2425         unsigned long flags;
2426
2427         spin_lock_irqsave(&device_domain_lock, flags);
2428         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2429                 __dmar_remove_one_dev_info(info);
2430         spin_unlock_irqrestore(&device_domain_lock, flags);
2431 }
2432
2433 /*
2434  * find_domain
2435  * Note: we use struct device->archdata.iommu stores the info
2436  */
2437 static struct dmar_domain *find_domain(struct device *dev)
2438 {
2439         struct device_domain_info *info;
2440
2441         /* No lock here, assumes no domain exit in normal case */
2442         info = dev->archdata.iommu;
2443         if (likely(info))
2444                 return info->domain;
2445         return NULL;
2446 }
2447
2448 static inline struct device_domain_info *
2449 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2450 {
2451         struct device_domain_info *info;
2452
2453         list_for_each_entry(info, &device_domain_list, global)
2454                 if (info->iommu->segment == segment && info->bus == bus &&
2455                     info->devfn == devfn)
2456                         return info;
2457
2458         return NULL;
2459 }
2460
2461 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2462                                                     int bus, int devfn,
2463                                                     struct device *dev,
2464                                                     struct dmar_domain *domain)
2465 {
2466         struct dmar_domain *found = NULL;
2467         struct device_domain_info *info;
2468         unsigned long flags;
2469         int ret;
2470
2471         info = alloc_devinfo_mem();
2472         if (!info)
2473                 return NULL;
2474
2475         info->bus = bus;
2476         info->devfn = devfn;
2477         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2478         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2479         info->ats_qdep = 0;
2480         info->dev = dev;
2481         info->domain = domain;
2482         info->iommu = iommu;
2483         info->pasid_table = NULL;
2484
2485         if (dev && dev_is_pci(dev)) {
2486                 struct pci_dev *pdev = to_pci_dev(info->dev);
2487
2488                 if (!pci_ats_disabled() &&
2489                     ecap_dev_iotlb_support(iommu->ecap) &&
2490                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2491                     dmar_find_matched_atsr_unit(pdev))
2492                         info->ats_supported = 1;
2493
2494                 if (sm_supported(iommu)) {
2495                         if (pasid_supported(iommu)) {
2496                                 int features = pci_pasid_features(pdev);
2497                                 if (features >= 0)
2498                                         info->pasid_supported = features | 1;
2499                         }
2500
2501                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2502                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2503                                 info->pri_supported = 1;
2504                 }
2505         }
2506
2507         spin_lock_irqsave(&device_domain_lock, flags);
2508         if (dev)
2509                 found = find_domain(dev);
2510
2511         if (!found) {
2512                 struct device_domain_info *info2;
2513                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2514                 if (info2) {
2515                         found      = info2->domain;
2516                         info2->dev = dev;
2517                 }
2518         }
2519
2520         if (found) {
2521                 spin_unlock_irqrestore(&device_domain_lock, flags);
2522                 free_devinfo_mem(info);
2523                 /* Caller must free the original domain */
2524                 return found;
2525         }
2526
2527         spin_lock(&iommu->lock);
2528         ret = domain_attach_iommu(domain, iommu);
2529         spin_unlock(&iommu->lock);
2530
2531         if (ret) {
2532                 spin_unlock_irqrestore(&device_domain_lock, flags);
2533                 free_devinfo_mem(info);
2534                 return NULL;
2535         }
2536
2537         list_add(&info->link, &domain->devices);
2538         list_add(&info->global, &device_domain_list);
2539         if (dev)
2540                 dev->archdata.iommu = info;
2541         spin_unlock_irqrestore(&device_domain_lock, flags);
2542
2543         /* PASID table is mandatory for a PCI device in scalable mode. */
2544         if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2545                 ret = intel_pasid_alloc_table(dev);
2546                 if (ret) {
2547                         pr_err("PASID table allocation for %s failed\n",
2548                                dev_name(dev));
2549                         dmar_remove_one_dev_info(domain, dev);
2550                         return NULL;
2551                 }
2552
2553                 /* Setup the PASID entry for requests without PASID: */
2554                 spin_lock(&iommu->lock);
2555                 if (hw_pass_through && domain_type_is_si(domain))
2556                         ret = intel_pasid_setup_pass_through(iommu, domain,
2557                                         dev, PASID_RID2PASID);
2558                 else
2559                         ret = intel_pasid_setup_second_level(iommu, domain,
2560                                         dev, PASID_RID2PASID);
2561                 spin_unlock(&iommu->lock);
2562                 if (ret) {
2563                         pr_err("Setup RID2PASID for %s failed\n",
2564                                dev_name(dev));
2565                         dmar_remove_one_dev_info(domain, dev);
2566                         return NULL;
2567                 }
2568         }
2569
2570         if (dev && domain_context_mapping(domain, dev)) {
2571                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2572                 dmar_remove_one_dev_info(domain, dev);
2573                 return NULL;
2574         }
2575
2576         return domain;
2577 }
2578
2579 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2580 {
2581         *(u16 *)opaque = alias;
2582         return 0;
2583 }
2584
2585 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2586 {
2587         struct device_domain_info *info = NULL;
2588         struct dmar_domain *domain = NULL;
2589         struct intel_iommu *iommu;
2590         u16 dma_alias;
2591         unsigned long flags;
2592         u8 bus, devfn;
2593
2594         iommu = device_to_iommu(dev, &bus, &devfn);
2595         if (!iommu)
2596                 return NULL;
2597
2598         if (dev_is_pci(dev)) {
2599                 struct pci_dev *pdev = to_pci_dev(dev);
2600
2601                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2602
2603                 spin_lock_irqsave(&device_domain_lock, flags);
2604                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2605                                                       PCI_BUS_NUM(dma_alias),
2606                                                       dma_alias & 0xff);
2607                 if (info) {
2608                         iommu = info->iommu;
2609                         domain = info->domain;
2610                 }
2611                 spin_unlock_irqrestore(&device_domain_lock, flags);
2612
2613                 /* DMA alias already has a domain, use it */
2614                 if (info)
2615                         goto out;
2616         }
2617
2618         /* Allocate and initialize new domain for the device */
2619         domain = alloc_domain(0);
2620         if (!domain)
2621                 return NULL;
2622         if (domain_init(domain, iommu, gaw)) {
2623                 domain_exit(domain);
2624                 return NULL;
2625         }
2626
2627 out:
2628
2629         return domain;
2630 }
2631
2632 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2633                                               struct dmar_domain *domain)
2634 {
2635         struct intel_iommu *iommu;
2636         struct dmar_domain *tmp;
2637         u16 req_id, dma_alias;
2638         u8 bus, devfn;
2639
2640         iommu = device_to_iommu(dev, &bus, &devfn);
2641         if (!iommu)
2642                 return NULL;
2643
2644         req_id = ((u16)bus << 8) | devfn;
2645
2646         if (dev_is_pci(dev)) {
2647                 struct pci_dev *pdev = to_pci_dev(dev);
2648
2649                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2650
2651                 /* register PCI DMA alias device */
2652                 if (req_id != dma_alias) {
2653                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2654                                         dma_alias & 0xff, NULL, domain);
2655
2656                         if (!tmp || tmp != domain)
2657                                 return tmp;
2658                 }
2659         }
2660
2661         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2662         if (!tmp || tmp != domain)
2663                 return tmp;
2664
2665         return domain;
2666 }
2667
2668 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2669 {
2670         struct dmar_domain *domain, *tmp;
2671
2672         domain = find_domain(dev);
2673         if (domain)
2674                 goto out;
2675
2676         domain = find_or_alloc_domain(dev, gaw);
2677         if (!domain)
2678                 goto out;
2679
2680         tmp = set_domain_for_dev(dev, domain);
2681         if (!tmp || domain != tmp) {
2682                 domain_exit(domain);
2683                 domain = tmp;
2684         }
2685
2686 out:
2687
2688         return domain;
2689 }
2690
2691 static int iommu_domain_identity_map(struct dmar_domain *domain,
2692                                      unsigned long long start,
2693                                      unsigned long long end)
2694 {
2695         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2696         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2697
2698         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2699                           dma_to_mm_pfn(last_vpfn))) {
2700                 pr_err("Reserving iova failed\n");
2701                 return -ENOMEM;
2702         }
2703
2704         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2705         /*
2706          * RMRR range might have overlap with physical memory range,
2707          * clear it first
2708          */
2709         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2710
2711         return __domain_mapping(domain, first_vpfn, NULL,
2712                                 first_vpfn, last_vpfn - first_vpfn + 1,
2713                                 DMA_PTE_READ|DMA_PTE_WRITE);
2714 }
2715
2716 static int domain_prepare_identity_map(struct device *dev,
2717                                        struct dmar_domain *domain,
2718                                        unsigned long long start,
2719                                        unsigned long long end)
2720 {
2721         /* For _hardware_ passthrough, don't bother. But for software
2722            passthrough, we do it anyway -- it may indicate a memory
2723            range which is reserved in E820, so which didn't get set
2724            up to start with in si_domain */
2725         if (domain == si_domain && hw_pass_through) {
2726                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2727                         dev_name(dev), start, end);
2728                 return 0;
2729         }
2730
2731         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2732                 dev_name(dev), start, end);
2733
2734         if (end < start) {
2735                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2736                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2737                         dmi_get_system_info(DMI_BIOS_VENDOR),
2738                         dmi_get_system_info(DMI_BIOS_VERSION),
2739                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2740                 return -EIO;
2741         }
2742
2743         if (end >> agaw_to_width(domain->agaw)) {
2744                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2745                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2746                      agaw_to_width(domain->agaw),
2747                      dmi_get_system_info(DMI_BIOS_VENDOR),
2748                      dmi_get_system_info(DMI_BIOS_VERSION),
2749                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2750                 return -EIO;
2751         }
2752
2753         return iommu_domain_identity_map(domain, start, end);
2754 }
2755
2756 static int iommu_prepare_identity_map(struct device *dev,
2757                                       unsigned long long start,
2758                                       unsigned long long end)
2759 {
2760         struct dmar_domain *domain;
2761         int ret;
2762
2763         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2764         if (!domain)
2765                 return -ENOMEM;
2766
2767         ret = domain_prepare_identity_map(dev, domain, start, end);
2768         if (ret)
2769                 domain_exit(domain);
2770
2771         return ret;
2772 }
2773
2774 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2775                                          struct device *dev)
2776 {
2777         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2778                 return 0;
2779         return iommu_prepare_identity_map(dev, rmrr->base_address,
2780                                           rmrr->end_address);
2781 }
2782
2783 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2784 static inline void iommu_prepare_isa(void)
2785 {
2786         struct pci_dev *pdev;
2787         int ret;
2788
2789         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2790         if (!pdev)
2791                 return;
2792
2793         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2794         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2795
2796         if (ret)
2797                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2798
2799         pci_dev_put(pdev);
2800 }
2801 #else
2802 static inline void iommu_prepare_isa(void)
2803 {
2804         return;
2805 }
2806 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2807
2808 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2809
2810 static int __init si_domain_init(int hw)
2811 {
2812         int nid, ret = 0;
2813
2814         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2815         if (!si_domain)
2816                 return -EFAULT;
2817
2818         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2819                 domain_exit(si_domain);
2820                 return -EFAULT;
2821         }
2822
2823         pr_debug("Identity mapping domain allocated\n");
2824
2825         if (hw)
2826                 return 0;
2827
2828         for_each_online_node(nid) {
2829                 unsigned long start_pfn, end_pfn;
2830                 int i;
2831
2832                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2833                         ret = iommu_domain_identity_map(si_domain,
2834                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2835                         if (ret)
2836                                 return ret;
2837                 }
2838         }
2839
2840         return 0;
2841 }
2842
2843 static int identity_mapping(struct device *dev)
2844 {
2845         struct device_domain_info *info;
2846
2847         if (likely(!iommu_identity_mapping))
2848                 return 0;
2849
2850         info = dev->archdata.iommu;
2851         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2852                 return (info->domain == si_domain);
2853
2854         return 0;
2855 }
2856
2857 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2858 {
2859         struct dmar_domain *ndomain;
2860         struct intel_iommu *iommu;
2861         u8 bus, devfn;
2862
2863         iommu = device_to_iommu(dev, &bus, &devfn);
2864         if (!iommu)
2865                 return -ENODEV;
2866
2867         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2868         if (ndomain != domain)
2869                 return -EBUSY;
2870
2871         return 0;
2872 }
2873
2874 static bool device_has_rmrr(struct device *dev)
2875 {
2876         struct dmar_rmrr_unit *rmrr;
2877         struct device *tmp;
2878         int i;
2879
2880         rcu_read_lock();
2881         for_each_rmrr_units(rmrr) {
2882                 /*
2883                  * Return TRUE if this RMRR contains the device that
2884                  * is passed in.
2885                  */
2886                 for_each_active_dev_scope(rmrr->devices,
2887                                           rmrr->devices_cnt, i, tmp)
2888                         if (tmp == dev) {
2889                                 rcu_read_unlock();
2890                                 return true;
2891                         }
2892         }
2893         rcu_read_unlock();
2894         return false;
2895 }
2896
2897 /*
2898  * There are a couple cases where we need to restrict the functionality of
2899  * devices associated with RMRRs.  The first is when evaluating a device for
2900  * identity mapping because problems exist when devices are moved in and out
2901  * of domains and their respective RMRR information is lost.  This means that
2902  * a device with associated RMRRs will never be in a "passthrough" domain.
2903  * The second is use of the device through the IOMMU API.  This interface
2904  * expects to have full control of the IOVA space for the device.  We cannot
2905  * satisfy both the requirement that RMRR access is maintained and have an
2906  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2907  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2908  * We therefore prevent devices associated with an RMRR from participating in
2909  * the IOMMU API, which eliminates them from device assignment.
2910  *
2911  * In both cases we assume that PCI USB devices with RMRRs have them largely
2912  * for historical reasons and that the RMRR space is not actively used post
2913  * boot.  This exclusion may change if vendors begin to abuse it.
2914  *
2915  * The same exception is made for graphics devices, with the requirement that
2916  * any use of the RMRR regions will be torn down before assigning the device
2917  * to a guest.
2918  */
2919 static bool device_is_rmrr_locked(struct device *dev)
2920 {
2921         if (!device_has_rmrr(dev))
2922                 return false;
2923
2924         if (dev_is_pci(dev)) {
2925                 struct pci_dev *pdev = to_pci_dev(dev);
2926
2927                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2928                         return false;
2929         }
2930
2931         return true;
2932 }
2933
2934 static int iommu_should_identity_map(struct device *dev, int startup)
2935 {
2936
2937         if (dev_is_pci(dev)) {
2938                 struct pci_dev *pdev = to_pci_dev(dev);
2939
2940                 if (device_is_rmrr_locked(dev))
2941                         return 0;
2942
2943                 /*
2944                  * Prevent any device marked as untrusted from getting
2945                  * placed into the statically identity mapping domain.
2946                  */
2947                 if (pdev->untrusted)
2948                         return 0;
2949
2950                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2951                         return 1;
2952
2953                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2954                         return 1;
2955
2956                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2957                         return 0;
2958
2959                 /*
2960                  * We want to start off with all devices in the 1:1 domain, and
2961                  * take them out later if we find they can't access all of memory.
2962                  *
2963                  * However, we can't do this for PCI devices behind bridges,
2964                  * because all PCI devices behind the same bridge will end up
2965                  * with the same source-id on their transactions.
2966                  *
2967                  * Practically speaking, we can't change things around for these
2968                  * devices at run-time, because we can't be sure there'll be no
2969                  * DMA transactions in flight for any of their siblings.
2970                  *
2971                  * So PCI devices (unless they're on the root bus) as well as
2972                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2973                  * the 1:1 domain, just in _case_ one of their siblings turns out
2974                  * not to be able to map all of memory.
2975                  */
2976                 if (!pci_is_pcie(pdev)) {
2977                         if (!pci_is_root_bus(pdev->bus))
2978                                 return 0;
2979                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2980                                 return 0;
2981                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2982                         return 0;
2983         } else {
2984                 if (device_has_rmrr(dev))
2985                         return 0;
2986         }
2987
2988         /*
2989          * At boot time, we don't yet know if devices will be 64-bit capable.
2990          * Assume that they will — if they turn out not to be, then we can
2991          * take them out of the 1:1 domain later.
2992          */
2993         if (!startup) {
2994                 /*
2995                  * If the device's dma_mask is less than the system's memory
2996                  * size then this is not a candidate for identity mapping.
2997                  */
2998                 u64 dma_mask = *dev->dma_mask;
2999
3000                 if (dev->coherent_dma_mask &&
3001                     dev->coherent_dma_mask < dma_mask)
3002                         dma_mask = dev->coherent_dma_mask;
3003
3004                 return dma_mask >= dma_get_required_mask(dev);
3005         }
3006
3007         return 1;
3008 }
3009
3010 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
3011 {
3012         int ret;
3013
3014         if (!iommu_should_identity_map(dev, 1))
3015                 return 0;
3016
3017         ret = domain_add_dev_info(si_domain, dev);
3018         if (!ret)
3019                 pr_info("%s identity mapping for device %s\n",
3020                         hw ? "Hardware" : "Software", dev_name(dev));
3021         else if (ret == -ENODEV)
3022                 /* device not associated with an iommu */
3023                 ret = 0;
3024
3025         return ret;
3026 }
3027
3028
3029 static int __init iommu_prepare_static_identity_mapping(int hw)
3030 {
3031         struct pci_dev *pdev = NULL;
3032         struct dmar_drhd_unit *drhd;
3033         struct intel_iommu *iommu;
3034         struct device *dev;
3035         int i;
3036         int ret = 0;
3037
3038         for_each_pci_dev(pdev) {
3039                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3040                 if (ret)
3041                         return ret;
3042         }
3043
3044         for_each_active_iommu(iommu, drhd)
3045                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3046                         struct acpi_device_physical_node *pn;
3047                         struct acpi_device *adev;
3048
3049                         if (dev->bus != &acpi_bus_type)
3050                                 continue;
3051
3052                         adev= to_acpi_device(dev);
3053                         mutex_lock(&adev->physical_node_lock);
3054                         list_for_each_entry(pn, &adev->physical_node_list, node) {
3055                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3056                                 if (ret)
3057                                         break;
3058                         }
3059                         mutex_unlock(&adev->physical_node_lock);
3060                         if (ret)
3061                                 return ret;
3062                 }
3063
3064         return 0;
3065 }
3066
3067 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3068 {
3069         /*
3070          * Start from the sane iommu hardware state.
3071          * If the queued invalidation is already initialized by us
3072          * (for example, while enabling interrupt-remapping) then
3073          * we got the things already rolling from a sane state.
3074          */
3075         if (!iommu->qi) {
3076                 /*
3077                  * Clear any previous faults.
3078                  */
3079                 dmar_fault(-1, iommu);
3080                 /*
3081                  * Disable queued invalidation if supported and already enabled
3082                  * before OS handover.
3083                  */
3084                 dmar_disable_qi(iommu);
3085         }
3086
3087         if (dmar_enable_qi(iommu)) {
3088                 /*
3089                  * Queued Invalidate not enabled, use Register Based Invalidate
3090                  */
3091                 iommu->flush.flush_context = __iommu_flush_context;
3092                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3093                 pr_info("%s: Using Register based invalidation\n",
3094                         iommu->name);
3095         } else {
3096                 iommu->flush.flush_context = qi_flush_context;
3097                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3098                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3099         }
3100 }
3101
3102 static int copy_context_table(struct intel_iommu *iommu,
3103                               struct root_entry *old_re,
3104                               struct context_entry **tbl,
3105                               int bus, bool ext)
3106 {
3107         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3108         struct context_entry *new_ce = NULL, ce;
3109         struct context_entry *old_ce = NULL;
3110         struct root_entry re;
3111         phys_addr_t old_ce_phys;
3112
3113         tbl_idx = ext ? bus * 2 : bus;
3114         memcpy(&re, old_re, sizeof(re));
3115
3116         for (devfn = 0; devfn < 256; devfn++) {
3117                 /* First calculate the correct index */
3118                 idx = (ext ? devfn * 2 : devfn) % 256;
3119
3120                 if (idx == 0) {
3121                         /* First save what we may have and clean up */
3122                         if (new_ce) {
3123                                 tbl[tbl_idx] = new_ce;
3124                                 __iommu_flush_cache(iommu, new_ce,
3125                                                     VTD_PAGE_SIZE);
3126                                 pos = 1;
3127                         }
3128
3129                         if (old_ce)
3130                                 memunmap(old_ce);
3131
3132                         ret = 0;
3133                         if (devfn < 0x80)
3134                                 old_ce_phys = root_entry_lctp(&re);
3135                         else
3136                                 old_ce_phys = root_entry_uctp(&re);
3137
3138                         if (!old_ce_phys) {
3139                                 if (ext && devfn == 0) {
3140                                         /* No LCTP, try UCTP */
3141                                         devfn = 0x7f;
3142                                         continue;
3143                                 } else {
3144                                         goto out;
3145                                 }
3146                         }
3147
3148                         ret = -ENOMEM;
3149                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3150                                         MEMREMAP_WB);
3151                         if (!old_ce)
3152                                 goto out;
3153
3154                         new_ce = alloc_pgtable_page(iommu->node);
3155                         if (!new_ce)
3156                                 goto out_unmap;
3157
3158                         ret = 0;
3159                 }
3160
3161                 /* Now copy the context entry */
3162                 memcpy(&ce, old_ce + idx, sizeof(ce));
3163
3164                 if (!__context_present(&ce))
3165                         continue;
3166
3167                 did = context_domain_id(&ce);
3168                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3169                         set_bit(did, iommu->domain_ids);
3170
3171                 /*
3172                  * We need a marker for copied context entries. This
3173                  * marker needs to work for the old format as well as
3174                  * for extended context entries.
3175                  *
3176                  * Bit 67 of the context entry is used. In the old
3177                  * format this bit is available to software, in the
3178                  * extended format it is the PGE bit, but PGE is ignored
3179                  * by HW if PASIDs are disabled (and thus still
3180                  * available).
3181                  *
3182                  * So disable PASIDs first and then mark the entry
3183                  * copied. This means that we don't copy PASID
3184                  * translations from the old kernel, but this is fine as
3185                  * faults there are not fatal.
3186                  */
3187                 context_clear_pasid_enable(&ce);
3188                 context_set_copied(&ce);
3189
3190                 new_ce[idx] = ce;
3191         }
3192
3193         tbl[tbl_idx + pos] = new_ce;
3194
3195         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3196
3197 out_unmap:
3198         memunmap(old_ce);
3199
3200 out:
3201         return ret;
3202 }
3203
3204 static int copy_translation_tables(struct intel_iommu *iommu)
3205 {
3206         struct context_entry **ctxt_tbls;
3207         struct root_entry *old_rt;
3208         phys_addr_t old_rt_phys;
3209         int ctxt_table_entries;
3210         unsigned long flags;
3211         u64 rtaddr_reg;
3212         int bus, ret;
3213         bool new_ext, ext;
3214
3215         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3216         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3217         new_ext    = !!ecap_ecs(iommu->ecap);
3218
3219         /*
3220          * The RTT bit can only be changed when translation is disabled,
3221          * but disabling translation means to open a window for data
3222          * corruption. So bail out and don't copy anything if we would
3223          * have to change the bit.
3224          */
3225         if (new_ext != ext)
3226                 return -EINVAL;
3227
3228         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3229         if (!old_rt_phys)
3230                 return -EINVAL;
3231
3232         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3233         if (!old_rt)
3234                 return -ENOMEM;
3235
3236         /* This is too big for the stack - allocate it from slab */
3237         ctxt_table_entries = ext ? 512 : 256;
3238         ret = -ENOMEM;
3239         ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3240         if (!ctxt_tbls)
3241                 goto out_unmap;
3242
3243         for (bus = 0; bus < 256; bus++) {
3244                 ret = copy_context_table(iommu, &old_rt[bus],
3245                                          ctxt_tbls, bus, ext);
3246                 if (ret) {
3247                         pr_err("%s: Failed to copy context table for bus %d\n",
3248                                 iommu->name, bus);
3249                         continue;
3250                 }
3251         }
3252
3253         spin_lock_irqsave(&iommu->lock, flags);
3254
3255         /* Context tables are copied, now write them to the root_entry table */
3256         for (bus = 0; bus < 256; bus++) {
3257                 int idx = ext ? bus * 2 : bus;
3258                 u64 val;
3259
3260                 if (ctxt_tbls[idx]) {
3261                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3262                         iommu->root_entry[bus].lo = val;
3263                 }
3264
3265                 if (!ext || !ctxt_tbls[idx + 1])
3266                         continue;
3267
3268                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3269                 iommu->root_entry[bus].hi = val;
3270         }
3271
3272         spin_unlock_irqrestore(&iommu->lock, flags);
3273
3274         kfree(ctxt_tbls);
3275
3276         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3277
3278         ret = 0;
3279
3280 out_unmap:
3281         memunmap(old_rt);
3282
3283         return ret;
3284 }
3285
3286 static int __init init_dmars(void)
3287 {
3288         struct dmar_drhd_unit *drhd;
3289         struct dmar_rmrr_unit *rmrr;
3290         bool copied_tables = false;
3291         struct device *dev;
3292         struct intel_iommu *iommu;
3293         int i, ret;
3294
3295         /*
3296          * for each drhd
3297          *    allocate root
3298          *    initialize and program root entry to not present
3299          * endfor
3300          */
3301         for_each_drhd_unit(drhd) {
3302                 /*
3303                  * lock not needed as this is only incremented in the single
3304                  * threaded kernel __init code path all other access are read
3305                  * only
3306                  */
3307                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3308                         g_num_of_iommus++;
3309                         continue;
3310                 }
3311                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3312         }
3313
3314         /* Preallocate enough resources for IOMMU hot-addition */
3315         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3316                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3317
3318         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3319                         GFP_KERNEL);
3320         if (!g_iommus) {
3321                 pr_err("Allocating global iommu array failed\n");
3322                 ret = -ENOMEM;
3323                 goto error;
3324         }
3325
3326         for_each_active_iommu(iommu, drhd) {
3327                 /*
3328                  * Find the max pasid size of all IOMMU's in the system.
3329                  * We need to ensure the system pasid table is no bigger
3330                  * than the smallest supported.
3331                  */
3332                 if (pasid_supported(iommu)) {
3333                         u32 temp = 2 << ecap_pss(iommu->ecap);
3334
3335                         intel_pasid_max_id = min_t(u32, temp,
3336                                                    intel_pasid_max_id);
3337                 }
3338
3339                 g_iommus[iommu->seq_id] = iommu;
3340
3341                 intel_iommu_init_qi(iommu);
3342
3343                 ret = iommu_init_domains(iommu);
3344                 if (ret)
3345                         goto free_iommu;
3346
3347                 init_translation_status(iommu);
3348
3349                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3350                         iommu_disable_translation(iommu);
3351                         clear_translation_pre_enabled(iommu);
3352                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3353                                 iommu->name);
3354                 }
3355
3356                 /*
3357                  * TBD:
3358                  * we could share the same root & context tables
3359                  * among all IOMMU's. Need to Split it later.
3360                  */
3361                 ret = iommu_alloc_root_entry(iommu);
3362                 if (ret)
3363                         goto free_iommu;
3364
3365                 if (translation_pre_enabled(iommu)) {
3366                         pr_info("Translation already enabled - trying to copy translation structures\n");
3367
3368                         ret = copy_translation_tables(iommu);
3369                         if (ret) {
3370                                 /*
3371                                  * We found the IOMMU with translation
3372                                  * enabled - but failed to copy over the
3373                                  * old root-entry table. Try to proceed
3374                                  * by disabling translation now and
3375                                  * allocating a clean root-entry table.
3376                                  * This might cause DMAR faults, but
3377                                  * probably the dump will still succeed.
3378                                  */
3379                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3380                                        iommu->name);
3381                                 iommu_disable_translation(iommu);
3382                                 clear_translation_pre_enabled(iommu);
3383                         } else {
3384                                 pr_info("Copied translation tables from previous kernel for %s\n",
3385                                         iommu->name);
3386                                 copied_tables = true;
3387                         }
3388                 }
3389
3390                 if (!ecap_pass_through(iommu->ecap))
3391                         hw_pass_through = 0;
3392 #ifdef CONFIG_INTEL_IOMMU_SVM
3393                 if (pasid_supported(iommu))
3394                         intel_svm_init(iommu);
3395 #endif
3396         }
3397
3398         /*
3399          * Now that qi is enabled on all iommus, set the root entry and flush
3400          * caches. This is required on some Intel X58 chipsets, otherwise the
3401          * flush_context function will loop forever and the boot hangs.
3402          */
3403         for_each_active_iommu(iommu, drhd) {
3404                 iommu_flush_write_buffer(iommu);
3405                 iommu_set_root_entry(iommu);
3406                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3407                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3408         }
3409
3410         if (iommu_pass_through)
3411                 iommu_identity_mapping |= IDENTMAP_ALL;
3412
3413 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3414         iommu_identity_mapping |= IDENTMAP_GFX;
3415 #endif
3416
3417         check_tylersburg_isoch();
3418
3419         if (iommu_identity_mapping) {
3420                 ret = si_domain_init(hw_pass_through);
3421                 if (ret)
3422                         goto free_iommu;
3423         }
3424
3425
3426         /*
3427          * If we copied translations from a previous kernel in the kdump
3428          * case, we can not assign the devices to domains now, as that
3429          * would eliminate the old mappings. So skip this part and defer
3430          * the assignment to device driver initialization time.
3431          */
3432         if (copied_tables)
3433                 goto domains_done;
3434
3435         /*
3436          * If pass through is not set or not enabled, setup context entries for
3437          * identity mappings for rmrr, gfx, and isa and may fall back to static
3438          * identity mapping if iommu_identity_mapping is set.
3439          */
3440         if (iommu_identity_mapping) {
3441                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3442                 if (ret) {
3443                         pr_crit("Failed to setup IOMMU pass-through\n");
3444                         goto free_iommu;
3445                 }
3446         }
3447         /*
3448          * For each rmrr
3449          *   for each dev attached to rmrr
3450          *   do
3451          *     locate drhd for dev, alloc domain for dev
3452          *     allocate free domain
3453          *     allocate page table entries for rmrr
3454          *     if context not allocated for bus
3455          *           allocate and init context
3456          *           set present in root table for this bus
3457          *     init context with domain, translation etc
3458          *    endfor
3459          * endfor
3460          */
3461         pr_info("Setting RMRR:\n");
3462         for_each_rmrr_units(rmrr) {
3463                 /* some BIOS lists non-exist devices in DMAR table. */
3464                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3465                                           i, dev) {
3466                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3467                         if (ret)
3468                                 pr_err("Mapping reserved region failed\n");
3469                 }
3470         }
3471
3472         iommu_prepare_isa();
3473
3474 domains_done:
3475
3476         /*
3477          * for each drhd
3478          *   enable fault log
3479          *   global invalidate context cache
3480          *   global invalidate iotlb
3481          *   enable translation
3482          */
3483         for_each_iommu(iommu, drhd) {
3484                 if (drhd->ignored) {
3485                         /*
3486                          * we always have to disable PMRs or DMA may fail on
3487                          * this device
3488                          */
3489                         if (force_on)
3490                                 iommu_disable_protect_mem_regions(iommu);
3491                         continue;
3492                 }
3493
3494                 iommu_flush_write_buffer(iommu);
3495
3496 #ifdef CONFIG_INTEL_IOMMU_SVM
3497                 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3498                         ret = intel_svm_enable_prq(iommu);
3499                         if (ret)
3500                                 goto free_iommu;
3501                 }
3502 #endif
3503                 ret = dmar_set_interrupt(iommu);
3504                 if (ret)
3505                         goto free_iommu;
3506
3507                 if (!translation_pre_enabled(iommu))
3508                         iommu_enable_translation(iommu);
3509
3510                 iommu_disable_protect_mem_regions(iommu);
3511         }
3512
3513         return 0;
3514
3515 free_iommu:
3516         for_each_active_iommu(iommu, drhd) {
3517                 disable_dmar_iommu(iommu);
3518                 free_dmar_iommu(iommu);
3519         }
3520
3521         kfree(g_iommus);
3522
3523 error:
3524         return ret;
3525 }
3526
3527 /* This takes a number of _MM_ pages, not VTD pages */
3528 static unsigned long intel_alloc_iova(struct device *dev,
3529                                      struct dmar_domain *domain,
3530                                      unsigned long nrpages, uint64_t dma_mask)
3531 {
3532         unsigned long iova_pfn = 0;
3533
3534         /* Restrict dma_mask to the width that the iommu can handle */
3535         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3536         /* Ensure we reserve the whole size-aligned region */
3537         nrpages = __roundup_pow_of_two(nrpages);
3538
3539         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3540                 /*
3541                  * First try to allocate an io virtual address in
3542                  * DMA_BIT_MASK(32) and if that fails then try allocating
3543                  * from higher range
3544                  */
3545                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3546                                            IOVA_PFN(DMA_BIT_MASK(32)), false);
3547                 if (iova_pfn)
3548                         return iova_pfn;
3549         }
3550         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3551                                    IOVA_PFN(dma_mask), true);
3552         if (unlikely(!iova_pfn)) {
3553                 pr_err("Allocating %ld-page iova for %s failed",
3554                        nrpages, dev_name(dev));
3555                 return 0;
3556         }
3557
3558         return iova_pfn;
3559 }
3560
3561 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3562 {
3563         struct dmar_domain *domain, *tmp;
3564         struct dmar_rmrr_unit *rmrr;
3565         struct device *i_dev;
3566         int i, ret;
3567
3568         domain = find_domain(dev);
3569         if (domain)
3570                 goto out;
3571
3572         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3573         if (!domain)
3574                 goto out;
3575
3576         /* We have a new domain - setup possible RMRRs for the device */
3577         rcu_read_lock();
3578         for_each_rmrr_units(rmrr) {
3579                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3580                                           i, i_dev) {
3581                         if (i_dev != dev)
3582                                 continue;
3583
3584                         ret = domain_prepare_identity_map(dev, domain,
3585                                                           rmrr->base_address,
3586                                                           rmrr->end_address);
3587                         if (ret)
3588                                 dev_err(dev, "Mapping reserved region failed\n");
3589                 }
3590         }
3591         rcu_read_unlock();
3592
3593         tmp = set_domain_for_dev(dev, domain);
3594         if (!tmp || domain != tmp) {
3595                 domain_exit(domain);
3596                 domain = tmp;
3597         }
3598
3599 out:
3600
3601         if (!domain)
3602                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3603
3604
3605         return domain;
3606 }
3607
3608 /* Check if the dev needs to go through non-identity map and unmap process.*/
3609 static int iommu_no_mapping(struct device *dev)
3610 {
3611         int found;
3612
3613         if (iommu_dummy(dev))
3614                 return 1;
3615
3616         if (!iommu_identity_mapping)
3617                 return 0;
3618
3619         found = identity_mapping(dev);
3620         if (found) {
3621                 if (iommu_should_identity_map(dev, 0))
3622                         return 1;
3623                 else {
3624                         /*
3625                          * 32 bit DMA is removed from si_domain and fall back
3626                          * to non-identity mapping.
3627                          */
3628                         dmar_remove_one_dev_info(si_domain, dev);
3629                         pr_info("32bit %s uses non-identity mapping\n",
3630                                 dev_name(dev));
3631                         return 0;
3632                 }
3633         } else {
3634                 /*
3635                  * In case of a detached 64 bit DMA device from vm, the device
3636                  * is put into si_domain for identity mapping.
3637                  */
3638                 if (iommu_should_identity_map(dev, 0)) {
3639                         int ret;
3640                         ret = domain_add_dev_info(si_domain, dev);
3641                         if (!ret) {
3642                                 pr_info("64bit %s uses identity mapping\n",
3643                                         dev_name(dev));
3644                                 return 1;
3645                         }
3646                 }
3647         }
3648
3649         return 0;
3650 }
3651
3652 static dma_addr_t __intel_map_page(struct device *dev, struct page *page,
3653                                    unsigned long offset, size_t size, int dir,
3654                                    u64 dma_mask)
3655 {
3656         phys_addr_t paddr = page_to_phys(page) + offset;
3657         struct dmar_domain *domain;
3658         phys_addr_t start_paddr;
3659         unsigned long iova_pfn;
3660         int prot = 0;
3661         int ret;
3662         struct intel_iommu *iommu;
3663         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3664
3665         BUG_ON(dir == DMA_NONE);
3666
3667         if (iommu_no_mapping(dev))
3668                 return paddr;
3669
3670         domain = get_valid_domain_for_dev(dev);
3671         if (!domain)
3672                 return DMA_MAPPING_ERROR;
3673
3674         iommu = domain_get_iommu(domain);
3675         size = aligned_nrpages(paddr, size);
3676
3677         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3678         if (!iova_pfn)
3679                 goto error;
3680
3681         /*
3682          * Check if DMAR supports zero-length reads on write only
3683          * mappings..
3684          */
3685         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3686                         !cap_zlr(iommu->cap))
3687                 prot |= DMA_PTE_READ;
3688         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3689                 prot |= DMA_PTE_WRITE;
3690         /*
3691          * paddr - (paddr + size) might be partial page, we should map the whole
3692          * page.  Note: if two part of one page are separately mapped, we
3693          * might have two guest_addr mapping to the same host paddr, but this
3694          * is not a big problem
3695          */
3696         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3697                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3698         if (ret)
3699                 goto error;
3700
3701         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3702         start_paddr += paddr & ~PAGE_MASK;
3703         return start_paddr;
3704
3705 error:
3706         if (iova_pfn)
3707                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3708         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3709                 dev_name(dev), size, (unsigned long long)paddr, dir);
3710         return DMA_MAPPING_ERROR;
3711 }
3712
3713 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3714                                  unsigned long offset, size_t size,
3715                                  enum dma_data_direction dir,
3716                                  unsigned long attrs)
3717 {
3718         return __intel_map_page(dev, page, offset, size, dir, *dev->dma_mask);
3719 }
3720
3721 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3722 {
3723         struct dmar_domain *domain;
3724         unsigned long start_pfn, last_pfn;
3725         unsigned long nrpages;
3726         unsigned long iova_pfn;
3727         struct intel_iommu *iommu;
3728         struct page *freelist;
3729
3730         if (iommu_no_mapping(dev))
3731                 return;
3732
3733         domain = find_domain(dev);
3734         BUG_ON(!domain);
3735
3736         iommu = domain_get_iommu(domain);
3737
3738         iova_pfn = IOVA_PFN(dev_addr);
3739
3740         nrpages = aligned_nrpages(dev_addr, size);
3741         start_pfn = mm_to_dma_pfn(iova_pfn);
3742         last_pfn = start_pfn + nrpages - 1;
3743
3744         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3745                  dev_name(dev), start_pfn, last_pfn);
3746
3747         freelist = domain_unmap(domain, start_pfn, last_pfn);
3748
3749         if (intel_iommu_strict) {
3750                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3751                                       nrpages, !freelist, 0);
3752                 /* free iova */
3753                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3754                 dma_free_pagelist(freelist);
3755         } else {
3756                 queue_iova(&domain->iovad, iova_pfn, nrpages,
3757                            (unsigned long)freelist);
3758                 /*
3759                  * queue up the release of the unmap to save the 1/6th of the
3760                  * cpu used up by the iotlb flush operation...
3761                  */
3762         }
3763 }
3764
3765 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3766                              size_t size, enum dma_data_direction dir,
3767                              unsigned long attrs)
3768 {
3769         intel_unmap(dev, dev_addr, size);
3770 }
3771
3772 static void *intel_alloc_coherent(struct device *dev, size_t size,
3773                                   dma_addr_t *dma_handle, gfp_t flags,
3774                                   unsigned long attrs)
3775 {
3776         struct page *page = NULL;
3777         int order;
3778
3779         size = PAGE_ALIGN(size);
3780         order = get_order(size);
3781
3782         if (!iommu_no_mapping(dev))
3783                 flags &= ~(GFP_DMA | GFP_DMA32);
3784         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3785                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3786                         flags |= GFP_DMA;
3787                 else
3788                         flags |= GFP_DMA32;
3789         }
3790
3791         if (gfpflags_allow_blocking(flags)) {
3792                 unsigned int count = size >> PAGE_SHIFT;
3793
3794                 page = dma_alloc_from_contiguous(dev, count, order,
3795                                                  flags & __GFP_NOWARN);
3796                 if (page && iommu_no_mapping(dev) &&
3797                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3798                         dma_release_from_contiguous(dev, page, count);
3799                         page = NULL;
3800                 }
3801         }
3802
3803         if (!page)
3804                 page = alloc_pages(flags, order);
3805         if (!page)
3806                 return NULL;
3807         memset(page_address(page), 0, size);
3808
3809         *dma_handle = __intel_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL,
3810                                        dev->coherent_dma_mask);
3811         if (*dma_handle != DMA_MAPPING_ERROR)
3812                 return page_address(page);
3813         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3814                 __free_pages(page, order);
3815
3816         return NULL;
3817 }
3818
3819 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3820                                 dma_addr_t dma_handle, unsigned long attrs)
3821 {
3822         int order;
3823         struct page *page = virt_to_page(vaddr);
3824
3825         size = PAGE_ALIGN(size);
3826         order = get_order(size);
3827
3828         intel_unmap(dev, dma_handle, size);
3829         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3830                 __free_pages(page, order);
3831 }
3832
3833 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3834                            int nelems, enum dma_data_direction dir,
3835                            unsigned long attrs)
3836 {
3837         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3838         unsigned long nrpages = 0;
3839         struct scatterlist *sg;
3840         int i;
3841
3842         for_each_sg(sglist, sg, nelems, i) {
3843                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3844         }
3845
3846         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3847 }
3848
3849 static int intel_nontranslate_map_sg(struct device *hddev,
3850         struct scatterlist *sglist, int nelems, int dir)
3851 {
3852         int i;
3853         struct scatterlist *sg;
3854
3855         for_each_sg(sglist, sg, nelems, i) {
3856                 BUG_ON(!sg_page(sg));
3857                 sg->dma_address = sg_phys(sg);
3858                 sg->dma_length = sg->length;
3859         }
3860         return nelems;
3861 }
3862
3863 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3864                         enum dma_data_direction dir, unsigned long attrs)
3865 {
3866         int i;
3867         struct dmar_domain *domain;
3868         size_t size = 0;
3869         int prot = 0;
3870         unsigned long iova_pfn;
3871         int ret;
3872         struct scatterlist *sg;
3873         unsigned long start_vpfn;
3874         struct intel_iommu *iommu;
3875
3876         BUG_ON(dir == DMA_NONE);
3877         if (iommu_no_mapping(dev))
3878                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3879
3880         domain = get_valid_domain_for_dev(dev);
3881         if (!domain)
3882                 return 0;
3883
3884         iommu = domain_get_iommu(domain);
3885
3886         for_each_sg(sglist, sg, nelems, i)
3887                 size += aligned_nrpages(sg->offset, sg->length);
3888
3889         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3890                                 *dev->dma_mask);
3891         if (!iova_pfn) {
3892                 sglist->dma_length = 0;
3893                 return 0;
3894         }
3895
3896         /*
3897          * Check if DMAR supports zero-length reads on write only
3898          * mappings..
3899          */
3900         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3901                         !cap_zlr(iommu->cap))
3902                 prot |= DMA_PTE_READ;
3903         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3904                 prot |= DMA_PTE_WRITE;
3905
3906         start_vpfn = mm_to_dma_pfn(iova_pfn);
3907
3908         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3909         if (unlikely(ret)) {
3910                 dma_pte_free_pagetable(domain, start_vpfn,
3911                                        start_vpfn + size - 1,
3912                                        agaw_to_level(domain->agaw) + 1);
3913                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3914                 return 0;
3915         }
3916
3917         return nelems;
3918 }
3919
3920 static const struct dma_map_ops intel_dma_ops = {
3921         .alloc = intel_alloc_coherent,
3922         .free = intel_free_coherent,
3923         .map_sg = intel_map_sg,
3924         .unmap_sg = intel_unmap_sg,
3925         .map_page = intel_map_page,
3926         .unmap_page = intel_unmap_page,
3927         .dma_supported = dma_direct_supported,
3928 };
3929
3930 static inline int iommu_domain_cache_init(void)
3931 {
3932         int ret = 0;
3933
3934         iommu_domain_cache = kmem_cache_create("iommu_domain",
3935                                          sizeof(struct dmar_domain),
3936                                          0,
3937                                          SLAB_HWCACHE_ALIGN,
3938
3939                                          NULL);
3940         if (!iommu_domain_cache) {
3941                 pr_err("Couldn't create iommu_domain cache\n");
3942                 ret = -ENOMEM;
3943         }
3944
3945         return ret;
3946 }
3947
3948 static inline int iommu_devinfo_cache_init(void)
3949 {
3950         int ret = 0;
3951
3952         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3953                                          sizeof(struct device_domain_info),
3954                                          0,
3955                                          SLAB_HWCACHE_ALIGN,
3956                                          NULL);
3957         if (!iommu_devinfo_cache) {
3958                 pr_err("Couldn't create devinfo cache\n");
3959                 ret = -ENOMEM;
3960         }
3961
3962         return ret;
3963 }
3964
3965 static int __init iommu_init_mempool(void)
3966 {
3967         int ret;
3968         ret = iova_cache_get();
3969         if (ret)
3970                 return ret;
3971
3972         ret = iommu_domain_cache_init();
3973         if (ret)
3974                 goto domain_error;
3975
3976         ret = iommu_devinfo_cache_init();
3977         if (!ret)
3978                 return ret;
3979
3980         kmem_cache_destroy(iommu_domain_cache);
3981 domain_error:
3982         iova_cache_put();
3983
3984         return -ENOMEM;
3985 }
3986
3987 static void __init iommu_exit_mempool(void)
3988 {
3989         kmem_cache_destroy(iommu_devinfo_cache);
3990         kmem_cache_destroy(iommu_domain_cache);
3991         iova_cache_put();
3992 }
3993
3994 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3995 {
3996         struct dmar_drhd_unit *drhd;
3997         u32 vtbar;
3998         int rc;
3999
4000         /* We know that this device on this chipset has its own IOMMU.
4001          * If we find it under a different IOMMU, then the BIOS is lying
4002          * to us. Hope that the IOMMU for this device is actually
4003          * disabled, and it needs no translation...
4004          */
4005         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4006         if (rc) {
4007                 /* "can't" happen */
4008                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4009                 return;
4010         }
4011         vtbar &= 0xffff0000;
4012
4013         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4014         drhd = dmar_find_matched_drhd_unit(pdev);
4015         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4016                             TAINT_FIRMWARE_WORKAROUND,
4017                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4018                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4019 }
4020 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4021
4022 static void __init init_no_remapping_devices(void)
4023 {
4024         struct dmar_drhd_unit *drhd;
4025         struct device *dev;
4026         int i;
4027
4028         for_each_drhd_unit(drhd) {
4029                 if (!drhd->include_all) {
4030                         for_each_active_dev_scope(drhd->devices,
4031                                                   drhd->devices_cnt, i, dev)
4032                                 break;
4033                         /* ignore DMAR unit if no devices exist */
4034                         if (i == drhd->devices_cnt)
4035                                 drhd->ignored = 1;
4036                 }
4037         }
4038
4039         for_each_active_drhd_unit(drhd) {
4040                 if (drhd->include_all)
4041                         continue;
4042
4043                 for_each_active_dev_scope(drhd->devices,
4044                                           drhd->devices_cnt, i, dev)
4045                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4046                                 break;
4047                 if (i < drhd->devices_cnt)
4048                         continue;
4049
4050                 /* This IOMMU has *only* gfx devices. Either bypass it or
4051                    set the gfx_mapped flag, as appropriate */
4052                 if (dmar_map_gfx) {
4053                         intel_iommu_gfx_mapped = 1;
4054                 } else {
4055                         drhd->ignored = 1;
4056                         for_each_active_dev_scope(drhd->devices,
4057                                                   drhd->devices_cnt, i, dev)
4058                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4059                 }
4060         }
4061 }
4062
4063 #ifdef CONFIG_SUSPEND
4064 static int init_iommu_hw(void)
4065 {
4066         struct dmar_drhd_unit *drhd;
4067         struct intel_iommu *iommu = NULL;
4068
4069         for_each_active_iommu(iommu, drhd)
4070                 if (iommu->qi)
4071                         dmar_reenable_qi(iommu);
4072
4073         for_each_iommu(iommu, drhd) {
4074                 if (drhd->ignored) {
4075                         /*
4076                          * we always have to disable PMRs or DMA may fail on
4077                          * this device
4078                          */
4079                         if (force_on)
4080                                 iommu_disable_protect_mem_regions(iommu);
4081                         continue;
4082                 }
4083
4084                 iommu_flush_write_buffer(iommu);
4085
4086                 iommu_set_root_entry(iommu);
4087
4088                 iommu->flush.flush_context(iommu, 0, 0, 0,
4089                                            DMA_CCMD_GLOBAL_INVL);
4090                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4091                 iommu_enable_translation(iommu);
4092                 iommu_disable_protect_mem_regions(iommu);
4093         }
4094
4095         return 0;
4096 }
4097
4098 static void iommu_flush_all(void)
4099 {
4100         struct dmar_drhd_unit *drhd;
4101         struct intel_iommu *iommu;
4102
4103         for_each_active_iommu(iommu, drhd) {
4104                 iommu->flush.flush_context(iommu, 0, 0, 0,
4105                                            DMA_CCMD_GLOBAL_INVL);
4106                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4107                                          DMA_TLB_GLOBAL_FLUSH);
4108         }
4109 }
4110
4111 static int iommu_suspend(void)
4112 {
4113         struct dmar_drhd_unit *drhd;
4114         struct intel_iommu *iommu = NULL;
4115         unsigned long flag;
4116
4117         for_each_active_iommu(iommu, drhd) {
4118                 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4119                                                  GFP_ATOMIC);
4120                 if (!iommu->iommu_state)
4121                         goto nomem;
4122         }
4123
4124         iommu_flush_all();
4125
4126         for_each_active_iommu(iommu, drhd) {
4127                 iommu_disable_translation(iommu);
4128
4129                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4130
4131                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4132                         readl(iommu->reg + DMAR_FECTL_REG);
4133                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4134                         readl(iommu->reg + DMAR_FEDATA_REG);
4135                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4136                         readl(iommu->reg + DMAR_FEADDR_REG);
4137                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4138                         readl(iommu->reg + DMAR_FEUADDR_REG);
4139
4140                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4141         }
4142         return 0;
4143
4144 nomem:
4145         for_each_active_iommu(iommu, drhd)
4146                 kfree(iommu->iommu_state);
4147
4148         return -ENOMEM;
4149 }
4150
4151 static void iommu_resume(void)
4152 {
4153         struct dmar_drhd_unit *drhd;
4154         struct intel_iommu *iommu = NULL;
4155         unsigned long flag;
4156
4157         if (init_iommu_hw()) {
4158                 if (force_on)
4159                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4160                 else
4161                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4162                 return;
4163         }
4164
4165         for_each_active_iommu(iommu, drhd) {
4166
4167                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4168
4169                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4170                         iommu->reg + DMAR_FECTL_REG);
4171                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4172                         iommu->reg + DMAR_FEDATA_REG);
4173                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4174                         iommu->reg + DMAR_FEADDR_REG);
4175                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4176                         iommu->reg + DMAR_FEUADDR_REG);
4177
4178                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4179         }
4180
4181         for_each_active_iommu(iommu, drhd)
4182                 kfree(iommu->iommu_state);
4183 }
4184
4185 static struct syscore_ops iommu_syscore_ops = {
4186         .resume         = iommu_resume,
4187         .suspend        = iommu_suspend,
4188 };
4189
4190 static void __init init_iommu_pm_ops(void)
4191 {
4192         register_syscore_ops(&iommu_syscore_ops);
4193 }
4194
4195 #else
4196 static inline void init_iommu_pm_ops(void) {}
4197 #endif  /* CONFIG_PM */
4198
4199
4200 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4201 {
4202         struct acpi_dmar_reserved_memory *rmrr;
4203         int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4204         struct dmar_rmrr_unit *rmrru;
4205         size_t length;
4206
4207         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4208         if (!rmrru)
4209                 goto out;
4210
4211         rmrru->hdr = header;
4212         rmrr = (struct acpi_dmar_reserved_memory *)header;
4213         rmrru->base_address = rmrr->base_address;
4214         rmrru->end_address = rmrr->end_address;
4215
4216         length = rmrr->end_address - rmrr->base_address + 1;
4217         rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4218                                               IOMMU_RESV_DIRECT);
4219         if (!rmrru->resv)
4220                 goto free_rmrru;
4221
4222         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4223                                 ((void *)rmrr) + rmrr->header.length,
4224                                 &rmrru->devices_cnt);
4225         if (rmrru->devices_cnt && rmrru->devices == NULL)
4226                 goto free_all;
4227
4228         list_add(&rmrru->list, &dmar_rmrr_units);
4229
4230         return 0;
4231 free_all:
4232         kfree(rmrru->resv);
4233 free_rmrru:
4234         kfree(rmrru);
4235 out:
4236         return -ENOMEM;
4237 }
4238
4239 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4240 {
4241         struct dmar_atsr_unit *atsru;
4242         struct acpi_dmar_atsr *tmp;
4243
4244         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4245                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4246                 if (atsr->segment != tmp->segment)
4247                         continue;
4248                 if (atsr->header.length != tmp->header.length)
4249                         continue;
4250                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4251                         return atsru;
4252         }
4253
4254         return NULL;
4255 }
4256
4257 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4258 {
4259         struct acpi_dmar_atsr *atsr;
4260         struct dmar_atsr_unit *atsru;
4261
4262         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4263                 return 0;
4264
4265         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4266         atsru = dmar_find_atsr(atsr);
4267         if (atsru)
4268                 return 0;
4269
4270         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4271         if (!atsru)
4272                 return -ENOMEM;
4273
4274         /*
4275          * If memory is allocated from slab by ACPI _DSM method, we need to
4276          * copy the memory content because the memory buffer will be freed
4277          * on return.
4278          */
4279         atsru->hdr = (void *)(atsru + 1);
4280         memcpy(atsru->hdr, hdr, hdr->length);
4281         atsru->include_all = atsr->flags & 0x1;
4282         if (!atsru->include_all) {
4283                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4284                                 (void *)atsr + atsr->header.length,
4285                                 &atsru->devices_cnt);
4286                 if (atsru->devices_cnt && atsru->devices == NULL) {
4287                         kfree(atsru);
4288                         return -ENOMEM;
4289                 }
4290         }
4291
4292         list_add_rcu(&atsru->list, &dmar_atsr_units);
4293
4294         return 0;
4295 }
4296
4297 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4298 {
4299         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4300         kfree(atsru);
4301 }
4302
4303 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4304 {
4305         struct acpi_dmar_atsr *atsr;
4306         struct dmar_atsr_unit *atsru;
4307
4308         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4309         atsru = dmar_find_atsr(atsr);
4310         if (atsru) {
4311                 list_del_rcu(&atsru->list);
4312                 synchronize_rcu();
4313                 intel_iommu_free_atsr(atsru);
4314         }
4315
4316         return 0;
4317 }
4318
4319 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4320 {
4321         int i;
4322         struct device *dev;
4323         struct acpi_dmar_atsr *atsr;
4324         struct dmar_atsr_unit *atsru;
4325
4326         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4327         atsru = dmar_find_atsr(atsr);
4328         if (!atsru)
4329                 return 0;
4330
4331         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4332                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4333                                           i, dev)
4334                         return -EBUSY;
4335         }
4336
4337         return 0;
4338 }
4339
4340 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4341 {
4342         int sp, ret = 0;
4343         struct intel_iommu *iommu = dmaru->iommu;
4344
4345         if (g_iommus[iommu->seq_id])
4346                 return 0;
4347
4348         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4349                 pr_warn("%s: Doesn't support hardware pass through.\n",
4350                         iommu->name);
4351                 return -ENXIO;
4352         }
4353         if (!ecap_sc_support(iommu->ecap) &&
4354             domain_update_iommu_snooping(iommu)) {
4355                 pr_warn("%s: Doesn't support snooping.\n",
4356                         iommu->name);
4357                 return -ENXIO;
4358         }
4359         sp = domain_update_iommu_superpage(iommu) - 1;
4360         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4361                 pr_warn("%s: Doesn't support large page.\n",
4362                         iommu->name);
4363                 return -ENXIO;
4364         }
4365
4366         /*
4367          * Disable translation if already enabled prior to OS handover.
4368          */
4369         if (iommu->gcmd & DMA_GCMD_TE)
4370                 iommu_disable_translation(iommu);
4371
4372         g_iommus[iommu->seq_id] = iommu;
4373         ret = iommu_init_domains(iommu);
4374         if (ret == 0)
4375                 ret = iommu_alloc_root_entry(iommu);
4376         if (ret)
4377                 goto out;
4378
4379 #ifdef CONFIG_INTEL_IOMMU_SVM
4380         if (pasid_supported(iommu))
4381                 intel_svm_init(iommu);
4382 #endif
4383
4384         if (dmaru->ignored) {
4385                 /*
4386                  * we always have to disable PMRs or DMA may fail on this device
4387                  */
4388                 if (force_on)
4389                         iommu_disable_protect_mem_regions(iommu);
4390                 return 0;
4391         }
4392
4393         intel_iommu_init_qi(iommu);
4394         iommu_flush_write_buffer(iommu);
4395
4396 #ifdef CONFIG_INTEL_IOMMU_SVM
4397         if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
4398                 ret = intel_svm_enable_prq(iommu);
4399                 if (ret)
4400                         goto disable_iommu;
4401         }
4402 #endif
4403         ret = dmar_set_interrupt(iommu);
4404         if (ret)
4405                 goto disable_iommu;
4406
4407         iommu_set_root_entry(iommu);
4408         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4409         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4410         iommu_enable_translation(iommu);
4411
4412         iommu_disable_protect_mem_regions(iommu);
4413         return 0;
4414
4415 disable_iommu:
4416         disable_dmar_iommu(iommu);
4417 out:
4418         free_dmar_iommu(iommu);
4419         return ret;
4420 }
4421
4422 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4423 {
4424         int ret = 0;
4425         struct intel_iommu *iommu = dmaru->iommu;
4426
4427         if (!intel_iommu_enabled)
4428                 return 0;
4429         if (iommu == NULL)
4430                 return -EINVAL;
4431
4432         if (insert) {
4433                 ret = intel_iommu_add(dmaru);
4434         } else {
4435                 disable_dmar_iommu(iommu);
4436                 free_dmar_iommu(iommu);
4437         }
4438
4439         return ret;
4440 }
4441
4442 static void intel_iommu_free_dmars(void)
4443 {
4444         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4445         struct dmar_atsr_unit *atsru, *atsr_n;
4446
4447         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4448                 list_del(&rmrru->list);
4449                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4450                 kfree(rmrru->resv);
4451                 kfree(rmrru);
4452         }
4453
4454         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4455                 list_del(&atsru->list);
4456                 intel_iommu_free_atsr(atsru);
4457         }
4458 }
4459
4460 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4461 {
4462         int i, ret = 1;
4463         struct pci_bus *bus;
4464         struct pci_dev *bridge = NULL;
4465         struct device *tmp;
4466         struct acpi_dmar_atsr *atsr;
4467         struct dmar_atsr_unit *atsru;
4468
4469         dev = pci_physfn(dev);
4470         for (bus = dev->bus; bus; bus = bus->parent) {
4471                 bridge = bus->self;
4472                 /* If it's an integrated device, allow ATS */
4473                 if (!bridge)
4474                         return 1;
4475                 /* Connected via non-PCIe: no ATS */
4476                 if (!pci_is_pcie(bridge) ||
4477                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4478                         return 0;
4479                 /* If we found the root port, look it up in the ATSR */
4480                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4481                         break;
4482         }
4483
4484         rcu_read_lock();
4485         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4486                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4487                 if (atsr->segment != pci_domain_nr(dev->bus))
4488                         continue;
4489
4490                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4491                         if (tmp == &bridge->dev)
4492                                 goto out;
4493
4494                 if (atsru->include_all)
4495                         goto out;
4496         }
4497         ret = 0;
4498 out:
4499         rcu_read_unlock();
4500
4501         return ret;
4502 }
4503
4504 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4505 {
4506         int ret = 0;
4507         struct dmar_rmrr_unit *rmrru;
4508         struct dmar_atsr_unit *atsru;
4509         struct acpi_dmar_atsr *atsr;
4510         struct acpi_dmar_reserved_memory *rmrr;
4511
4512         if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4513                 return 0;
4514
4515         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4516                 rmrr = container_of(rmrru->hdr,
4517                                     struct acpi_dmar_reserved_memory, header);
4518                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4519                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4520                                 ((void *)rmrr) + rmrr->header.length,
4521                                 rmrr->segment, rmrru->devices,
4522                                 rmrru->devices_cnt);
4523                         if(ret < 0)
4524                                 return ret;
4525                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4526                         dmar_remove_dev_scope(info, rmrr->segment,
4527                                 rmrru->devices, rmrru->devices_cnt);
4528                 }
4529         }
4530
4531         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4532                 if (atsru->include_all)
4533                         continue;
4534
4535                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4536                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4537                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4538                                         (void *)atsr + atsr->header.length,
4539                                         atsr->segment, atsru->devices,
4540                                         atsru->devices_cnt);
4541                         if (ret > 0)
4542                                 break;
4543                         else if(ret < 0)
4544                                 return ret;
4545                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4546                         if (dmar_remove_dev_scope(info, atsr->segment,
4547                                         atsru->devices, atsru->devices_cnt))
4548                                 break;
4549                 }
4550         }
4551
4552         return 0;
4553 }
4554
4555 /*
4556  * Here we only respond to action of unbound device from driver.
4557  *
4558  * Added device is not attached to its DMAR domain here yet. That will happen
4559  * when mapping the device to iova.
4560  */
4561 static int device_notifier(struct notifier_block *nb,
4562                                   unsigned long action, void *data)
4563 {
4564         struct device *dev = data;
4565         struct dmar_domain *domain;
4566
4567         if (iommu_dummy(dev))
4568                 return 0;
4569
4570         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4571                 return 0;
4572
4573         domain = find_domain(dev);
4574         if (!domain)
4575                 return 0;
4576
4577         dmar_remove_one_dev_info(domain, dev);
4578         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4579                 domain_exit(domain);
4580
4581         return 0;
4582 }
4583
4584 static struct notifier_block device_nb = {
4585         .notifier_call = device_notifier,
4586 };
4587
4588 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4589                                        unsigned long val, void *v)
4590 {
4591         struct memory_notify *mhp = v;
4592         unsigned long long start, end;
4593         unsigned long start_vpfn, last_vpfn;
4594
4595         switch (val) {
4596         case MEM_GOING_ONLINE:
4597                 start = mhp->start_pfn << PAGE_SHIFT;
4598                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4599                 if (iommu_domain_identity_map(si_domain, start, end)) {
4600                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4601                                 start, end);
4602                         return NOTIFY_BAD;
4603                 }
4604                 break;
4605
4606         case MEM_OFFLINE:
4607         case MEM_CANCEL_ONLINE:
4608                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4609                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4610                 while (start_vpfn <= last_vpfn) {
4611                         struct iova *iova;
4612                         struct dmar_drhd_unit *drhd;
4613                         struct intel_iommu *iommu;
4614                         struct page *freelist;
4615
4616                         iova = find_iova(&si_domain->iovad, start_vpfn);
4617                         if (iova == NULL) {
4618                                 pr_debug("Failed get IOVA for PFN %lx\n",
4619                                          start_vpfn);
4620                                 break;
4621                         }
4622
4623                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4624                                                      start_vpfn, last_vpfn);
4625                         if (iova == NULL) {
4626                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4627                                         start_vpfn, last_vpfn);
4628                                 return NOTIFY_BAD;
4629                         }
4630
4631                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4632                                                iova->pfn_hi);
4633
4634                         rcu_read_lock();
4635                         for_each_active_iommu(iommu, drhd)
4636                                 iommu_flush_iotlb_psi(iommu, si_domain,
4637                                         iova->pfn_lo, iova_size(iova),
4638                                         !freelist, 0);
4639                         rcu_read_unlock();
4640                         dma_free_pagelist(freelist);
4641
4642                         start_vpfn = iova->pfn_hi + 1;
4643                         free_iova_mem(iova);
4644                 }
4645                 break;
4646         }
4647
4648         return NOTIFY_OK;
4649 }
4650
4651 static struct notifier_block intel_iommu_memory_nb = {
4652         .notifier_call = intel_iommu_memory_notifier,
4653         .priority = 0
4654 };
4655
4656 static void free_all_cpu_cached_iovas(unsigned int cpu)
4657 {
4658         int i;
4659
4660         for (i = 0; i < g_num_of_iommus; i++) {
4661                 struct intel_iommu *iommu = g_iommus[i];
4662                 struct dmar_domain *domain;
4663                 int did;
4664
4665                 if (!iommu)
4666                         continue;
4667
4668                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4669                         domain = get_iommu_domain(iommu, (u16)did);
4670
4671                         if (!domain)
4672                                 continue;
4673                         free_cpu_cached_iovas(cpu, &domain->iovad);
4674                 }
4675         }
4676 }
4677
4678 static int intel_iommu_cpu_dead(unsigned int cpu)
4679 {
4680         free_all_cpu_cached_iovas(cpu);
4681         return 0;
4682 }
4683
4684 static void intel_disable_iommus(void)
4685 {
4686         struct intel_iommu *iommu = NULL;
4687         struct dmar_drhd_unit *drhd;
4688
4689         for_each_iommu(iommu, drhd)
4690                 iommu_disable_translation(iommu);
4691 }
4692
4693 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4694 {
4695         struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4696
4697         return container_of(iommu_dev, struct intel_iommu, iommu);
4698 }
4699
4700 static ssize_t intel_iommu_show_version(struct device *dev,
4701                                         struct device_attribute *attr,
4702                                         char *buf)
4703 {
4704         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4705         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4706         return sprintf(buf, "%d:%d\n",
4707                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4708 }
4709 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4710
4711 static ssize_t intel_iommu_show_address(struct device *dev,
4712                                         struct device_attribute *attr,
4713                                         char *buf)
4714 {
4715         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4716         return sprintf(buf, "%llx\n", iommu->reg_phys);
4717 }
4718 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4719
4720 static ssize_t intel_iommu_show_cap(struct device *dev,
4721                                     struct device_attribute *attr,
4722                                     char *buf)
4723 {
4724         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4725         return sprintf(buf, "%llx\n", iommu->cap);
4726 }
4727 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4728
4729 static ssize_t intel_iommu_show_ecap(struct device *dev,
4730                                     struct device_attribute *attr,
4731                                     char *buf)
4732 {
4733         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4734         return sprintf(buf, "%llx\n", iommu->ecap);
4735 }
4736 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4737
4738 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4739                                       struct device_attribute *attr,
4740                                       char *buf)
4741 {
4742         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4743         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4744 }
4745 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4746
4747 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4748                                            struct device_attribute *attr,
4749                                            char *buf)
4750 {
4751         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4752         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4753                                                   cap_ndoms(iommu->cap)));
4754 }
4755 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4756
4757 static struct attribute *intel_iommu_attrs[] = {
4758         &dev_attr_version.attr,
4759         &dev_attr_address.attr,
4760         &dev_attr_cap.attr,
4761         &dev_attr_ecap.attr,
4762         &dev_attr_domains_supported.attr,
4763         &dev_attr_domains_used.attr,
4764         NULL,
4765 };
4766
4767 static struct attribute_group intel_iommu_group = {
4768         .name = "intel-iommu",
4769         .attrs = intel_iommu_attrs,
4770 };
4771
4772 const struct attribute_group *intel_iommu_groups[] = {
4773         &intel_iommu_group,
4774         NULL,
4775 };
4776
4777 static int __init platform_optin_force_iommu(void)
4778 {
4779         struct pci_dev *pdev = NULL;
4780         bool has_untrusted_dev = false;
4781
4782         if (!dmar_platform_optin() || no_platform_optin)
4783                 return 0;
4784
4785         for_each_pci_dev(pdev) {
4786                 if (pdev->untrusted) {
4787                         has_untrusted_dev = true;
4788                         break;
4789                 }
4790         }
4791
4792         if (!has_untrusted_dev)
4793                 return 0;
4794
4795         if (no_iommu || dmar_disabled)
4796                 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
4797
4798         /*
4799          * If Intel-IOMMU is disabled by default, we will apply identity
4800          * map for all devices except those marked as being untrusted.
4801          */
4802         if (dmar_disabled)
4803                 iommu_identity_mapping |= IDENTMAP_ALL;
4804
4805         dmar_disabled = 0;
4806 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4807         swiotlb = 0;
4808 #endif
4809         no_iommu = 0;
4810
4811         return 1;
4812 }
4813
4814 int __init intel_iommu_init(void)
4815 {
4816         int ret = -ENODEV;
4817         struct dmar_drhd_unit *drhd;
4818         struct intel_iommu *iommu;
4819
4820         /*
4821          * Intel IOMMU is required for a TXT/tboot launch or platform
4822          * opt in, so enforce that.
4823          */
4824         force_on = tboot_force_iommu() || platform_optin_force_iommu();
4825
4826         if (iommu_init_mempool()) {
4827                 if (force_on)
4828                         panic("tboot: Failed to initialize iommu memory\n");
4829                 return -ENOMEM;
4830         }
4831
4832         down_write(&dmar_global_lock);
4833         if (dmar_table_init()) {
4834                 if (force_on)
4835                         panic("tboot: Failed to initialize DMAR table\n");
4836                 goto out_free_dmar;
4837         }
4838
4839         if (dmar_dev_scope_init() < 0) {
4840                 if (force_on)
4841                         panic("tboot: Failed to initialize DMAR device scope\n");
4842                 goto out_free_dmar;
4843         }
4844
4845         up_write(&dmar_global_lock);
4846
4847         /*
4848          * The bus notifier takes the dmar_global_lock, so lockdep will
4849          * complain later when we register it under the lock.
4850          */
4851         dmar_register_bus_notifier();
4852
4853         down_write(&dmar_global_lock);
4854
4855         if (no_iommu || dmar_disabled) {
4856                 /*
4857                  * We exit the function here to ensure IOMMU's remapping and
4858                  * mempool aren't setup, which means that the IOMMU's PMRs
4859                  * won't be disabled via the call to init_dmars(). So disable
4860                  * it explicitly here. The PMRs were setup by tboot prior to
4861                  * calling SENTER, but the kernel is expected to reset/tear
4862                  * down the PMRs.
4863                  */
4864                 if (intel_iommu_tboot_noforce) {
4865                         for_each_iommu(iommu, drhd)
4866                                 iommu_disable_protect_mem_regions(iommu);
4867                 }
4868
4869                 /*
4870                  * Make sure the IOMMUs are switched off, even when we
4871                  * boot into a kexec kernel and the previous kernel left
4872                  * them enabled
4873                  */
4874                 intel_disable_iommus();
4875                 goto out_free_dmar;
4876         }
4877
4878         if (list_empty(&dmar_rmrr_units))
4879                 pr_info("No RMRR found\n");
4880
4881         if (list_empty(&dmar_atsr_units))
4882                 pr_info("No ATSR found\n");
4883
4884         if (dmar_init_reserved_ranges()) {
4885                 if (force_on)
4886                         panic("tboot: Failed to reserve iommu ranges\n");
4887                 goto out_free_reserved_range;
4888         }
4889
4890         init_no_remapping_devices();
4891
4892         ret = init_dmars();
4893         if (ret) {
4894                 if (force_on)
4895                         panic("tboot: Failed to initialize DMARs\n");
4896                 pr_err("Initialization failed\n");
4897                 goto out_free_reserved_range;
4898         }
4899         up_write(&dmar_global_lock);
4900         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4901
4902 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4903         swiotlb = 0;
4904 #endif
4905         dma_ops = &intel_dma_ops;
4906
4907         init_iommu_pm_ops();
4908
4909         for_each_active_iommu(iommu, drhd) {
4910                 iommu_device_sysfs_add(&iommu->iommu, NULL,
4911                                        intel_iommu_groups,
4912                                        "%s", iommu->name);
4913                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4914                 iommu_device_register(&iommu->iommu);
4915         }
4916
4917         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4918         bus_register_notifier(&pci_bus_type, &device_nb);
4919         if (si_domain && !hw_pass_through)
4920                 register_memory_notifier(&intel_iommu_memory_nb);
4921         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4922                           intel_iommu_cpu_dead);
4923         intel_iommu_enabled = 1;
4924         intel_iommu_debugfs_init();
4925
4926         return 0;
4927
4928 out_free_reserved_range:
4929         put_iova_domain(&reserved_iova_list);
4930 out_free_dmar:
4931         intel_iommu_free_dmars();
4932         up_write(&dmar_global_lock);
4933         iommu_exit_mempool();
4934         return ret;
4935 }
4936
4937 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4938 {
4939         struct intel_iommu *iommu = opaque;
4940
4941         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4942         return 0;
4943 }
4944
4945 /*
4946  * NB - intel-iommu lacks any sort of reference counting for the users of
4947  * dependent devices.  If multiple endpoints have intersecting dependent
4948  * devices, unbinding the driver from any one of them will possibly leave
4949  * the others unable to operate.
4950  */
4951 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4952 {
4953         if (!iommu || !dev || !dev_is_pci(dev))
4954                 return;
4955
4956         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4957 }
4958
4959 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4960 {
4961         struct intel_iommu *iommu;
4962         unsigned long flags;
4963
4964         assert_spin_locked(&device_domain_lock);
4965
4966         if (WARN_ON(!info))
4967                 return;
4968
4969         iommu = info->iommu;
4970
4971         if (info->dev) {
4972                 if (dev_is_pci(info->dev) && sm_supported(iommu))
4973                         intel_pasid_tear_down_entry(iommu, info->dev,
4974                                         PASID_RID2PASID);
4975
4976                 iommu_disable_dev_iotlb(info);
4977                 domain_context_clear(iommu, info->dev);
4978                 intel_pasid_free_table(info->dev);
4979         }
4980
4981         unlink_domain_info(info);
4982
4983         spin_lock_irqsave(&iommu->lock, flags);
4984         domain_detach_iommu(info->domain, iommu);
4985         spin_unlock_irqrestore(&iommu->lock, flags);
4986
4987         free_devinfo_mem(info);
4988 }
4989
4990 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4991                                      struct device *dev)
4992 {
4993         struct device_domain_info *info;
4994         unsigned long flags;
4995
4996         spin_lock_irqsave(&device_domain_lock, flags);
4997         info = dev->archdata.iommu;
4998         __dmar_remove_one_dev_info(info);
4999         spin_unlock_irqrestore(&device_domain_lock, flags);
5000 }
5001
5002 static int md_domain_init(struct dmar_domain *domain, int guest_width)
5003 {
5004         int adjust_width;
5005
5006         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5007         domain_reserve_special_ranges(domain);
5008
5009         /* calculate AGAW */
5010         domain->gaw = guest_width;
5011         adjust_width = guestwidth_to_adjustwidth(guest_width);
5012         domain->agaw = width_to_agaw(adjust_width);
5013
5014         domain->iommu_coherency = 0;
5015         domain->iommu_snooping = 0;
5016         domain->iommu_superpage = 0;
5017         domain->max_addr = 0;
5018
5019         /* always allocate the top pgd */
5020         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5021         if (!domain->pgd)
5022                 return -ENOMEM;
5023         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5024         return 0;
5025 }
5026
5027 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5028 {
5029         struct dmar_domain *dmar_domain;
5030         struct iommu_domain *domain;
5031
5032         if (type != IOMMU_DOMAIN_UNMANAGED)
5033                 return NULL;
5034
5035         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5036         if (!dmar_domain) {
5037                 pr_err("Can't allocate dmar_domain\n");
5038                 return NULL;
5039         }
5040         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5041                 pr_err("Domain initialization failed\n");
5042                 domain_exit(dmar_domain);
5043                 return NULL;
5044         }
5045         domain_update_iommu_cap(dmar_domain);
5046
5047         domain = &dmar_domain->domain;
5048         domain->geometry.aperture_start = 0;
5049         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5050         domain->geometry.force_aperture = true;
5051
5052         return domain;
5053 }
5054
5055 static void intel_iommu_domain_free(struct iommu_domain *domain)
5056 {
5057         domain_exit(to_dmar_domain(domain));
5058 }
5059
5060 static int intel_iommu_attach_device(struct iommu_domain *domain,
5061                                      struct device *dev)
5062 {
5063         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5064         struct intel_iommu *iommu;
5065         int addr_width;
5066         u8 bus, devfn;
5067
5068         if (device_is_rmrr_locked(dev)) {
5069                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5070                 return -EPERM;
5071         }
5072
5073         /* normally dev is not mapped */
5074         if (unlikely(domain_context_mapped(dev))) {
5075                 struct dmar_domain *old_domain;
5076
5077                 old_domain = find_domain(dev);
5078                 if (old_domain) {
5079                         rcu_read_lock();
5080                         dmar_remove_one_dev_info(old_domain, dev);
5081                         rcu_read_unlock();
5082
5083                         if (!domain_type_is_vm_or_si(old_domain) &&
5084                              list_empty(&old_domain->devices))
5085                                 domain_exit(old_domain);
5086                 }
5087         }
5088
5089         iommu = device_to_iommu(dev, &bus, &devfn);
5090         if (!iommu)
5091                 return -ENODEV;
5092
5093         /* check if this iommu agaw is sufficient for max mapped address */
5094         addr_width = agaw_to_width(iommu->agaw);
5095         if (addr_width > cap_mgaw(iommu->cap))
5096                 addr_width = cap_mgaw(iommu->cap);
5097
5098         if (dmar_domain->max_addr > (1LL << addr_width)) {
5099                 pr_err("%s: iommu width (%d) is not "
5100                        "sufficient for the mapped address (%llx)\n",
5101                        __func__, addr_width, dmar_domain->max_addr);
5102                 return -EFAULT;
5103         }
5104         dmar_domain->gaw = addr_width;
5105
5106         /*
5107          * Knock out extra levels of page tables if necessary
5108          */
5109         while (iommu->agaw < dmar_domain->agaw) {
5110                 struct dma_pte *pte;
5111
5112                 pte = dmar_domain->pgd;
5113                 if (dma_pte_present(pte)) {
5114                         dmar_domain->pgd = (struct dma_pte *)
5115                                 phys_to_virt(dma_pte_addr(pte));
5116                         free_pgtable_page(pte);
5117                 }
5118                 dmar_domain->agaw--;
5119         }
5120
5121         return domain_add_dev_info(dmar_domain, dev);
5122 }
5123
5124 static void intel_iommu_detach_device(struct iommu_domain *domain,
5125                                       struct device *dev)
5126 {
5127         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5128 }
5129
5130 static int intel_iommu_map(struct iommu_domain *domain,
5131                            unsigned long iova, phys_addr_t hpa,
5132                            size_t size, int iommu_prot)
5133 {
5134         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5135         u64 max_addr;
5136         int prot = 0;
5137         int ret;
5138
5139         if (iommu_prot & IOMMU_READ)
5140                 prot |= DMA_PTE_READ;
5141         if (iommu_prot & IOMMU_WRITE)
5142                 prot |= DMA_PTE_WRITE;
5143         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5144                 prot |= DMA_PTE_SNP;
5145
5146         max_addr = iova + size;
5147         if (dmar_domain->max_addr < max_addr) {
5148                 u64 end;
5149
5150                 /* check if minimum agaw is sufficient for mapped address */
5151                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5152                 if (end < max_addr) {
5153                         pr_err("%s: iommu width (%d) is not "
5154                                "sufficient for the mapped address (%llx)\n",
5155                                __func__, dmar_domain->gaw, max_addr);
5156                         return -EFAULT;
5157                 }
5158                 dmar_domain->max_addr = max_addr;
5159         }
5160         /* Round up size to next multiple of PAGE_SIZE, if it and
5161            the low bits of hpa would take us onto the next page */
5162         size = aligned_nrpages(hpa, size);
5163         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5164                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5165         return ret;
5166 }
5167
5168 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5169                                 unsigned long iova, size_t size)
5170 {
5171         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5172         struct page *freelist = NULL;
5173         unsigned long start_pfn, last_pfn;
5174         unsigned int npages;
5175         int iommu_id, level = 0;
5176
5177         /* Cope with horrid API which requires us to unmap more than the
5178            size argument if it happens to be a large-page mapping. */
5179         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5180
5181         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5182                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5183
5184         start_pfn = iova >> VTD_PAGE_SHIFT;
5185         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5186
5187         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5188
5189         npages = last_pfn - start_pfn + 1;
5190
5191         for_each_domain_iommu(iommu_id, dmar_domain)
5192                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5193                                       start_pfn, npages, !freelist, 0);
5194
5195         dma_free_pagelist(freelist);
5196
5197         if (dmar_domain->max_addr == iova + size)
5198                 dmar_domain->max_addr = iova;
5199
5200         return size;
5201 }
5202
5203 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5204                                             dma_addr_t iova)
5205 {
5206         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5207         struct dma_pte *pte;
5208         int level = 0;
5209         u64 phys = 0;
5210
5211         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5212         if (pte)
5213                 phys = dma_pte_addr(pte);
5214
5215         return phys;
5216 }
5217
5218 static bool intel_iommu_capable(enum iommu_cap cap)
5219 {
5220         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5221                 return domain_update_iommu_snooping(NULL) == 1;
5222         if (cap == IOMMU_CAP_INTR_REMAP)
5223                 return irq_remapping_enabled == 1;
5224
5225         return false;
5226 }
5227
5228 static int intel_iommu_add_device(struct device *dev)
5229 {
5230         struct intel_iommu *iommu;
5231         struct iommu_group *group;
5232         u8 bus, devfn;
5233
5234         iommu = device_to_iommu(dev, &bus, &devfn);
5235         if (!iommu)
5236                 return -ENODEV;
5237
5238         iommu_device_link(&iommu->iommu, dev);
5239
5240         group = iommu_group_get_for_dev(dev);
5241
5242         if (IS_ERR(group))
5243                 return PTR_ERR(group);
5244
5245         iommu_group_put(group);
5246         return 0;
5247 }
5248
5249 static void intel_iommu_remove_device(struct device *dev)
5250 {
5251         struct intel_iommu *iommu;
5252         u8 bus, devfn;
5253
5254         iommu = device_to_iommu(dev, &bus, &devfn);
5255         if (!iommu)
5256                 return;
5257
5258         iommu_group_remove_device(dev);
5259
5260         iommu_device_unlink(&iommu->iommu, dev);
5261 }
5262
5263 static void intel_iommu_get_resv_regions(struct device *device,
5264                                          struct list_head *head)
5265 {
5266         struct iommu_resv_region *reg;
5267         struct dmar_rmrr_unit *rmrr;
5268         struct device *i_dev;
5269         int i;
5270
5271         rcu_read_lock();
5272         for_each_rmrr_units(rmrr) {
5273                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5274                                           i, i_dev) {
5275                         if (i_dev != device)
5276                                 continue;
5277
5278                         list_add_tail(&rmrr->resv->list, head);
5279                 }
5280         }
5281         rcu_read_unlock();
5282
5283         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5284                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5285                                       0, IOMMU_RESV_MSI);
5286         if (!reg)
5287                 return;
5288         list_add_tail(&reg->list, head);
5289 }
5290
5291 static void intel_iommu_put_resv_regions(struct device *dev,
5292                                          struct list_head *head)
5293 {
5294         struct iommu_resv_region *entry, *next;
5295
5296         list_for_each_entry_safe(entry, next, head, list) {
5297                 if (entry->type == IOMMU_RESV_MSI)
5298                         kfree(entry);
5299         }
5300 }
5301
5302 #ifdef CONFIG_INTEL_IOMMU_SVM
5303 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5304 {
5305         struct device_domain_info *info;
5306         struct context_entry *context;
5307         struct dmar_domain *domain;
5308         unsigned long flags;
5309         u64 ctx_lo;
5310         int ret;
5311
5312         domain = get_valid_domain_for_dev(sdev->dev);
5313         if (!domain)
5314                 return -EINVAL;
5315
5316         spin_lock_irqsave(&device_domain_lock, flags);
5317         spin_lock(&iommu->lock);
5318
5319         ret = -EINVAL;
5320         info = sdev->dev->archdata.iommu;
5321         if (!info || !info->pasid_supported)
5322                 goto out;
5323
5324         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5325         if (WARN_ON(!context))
5326                 goto out;
5327
5328         ctx_lo = context[0].lo;
5329
5330         sdev->did = domain->iommu_did[iommu->seq_id];
5331         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5332
5333         if (!(ctx_lo & CONTEXT_PASIDE)) {
5334                 ctx_lo |= CONTEXT_PASIDE;
5335                 context[0].lo = ctx_lo;
5336                 wmb();
5337                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5338                                            DMA_CCMD_MASK_NOBIT,
5339                                            DMA_CCMD_DEVICE_INVL);
5340         }
5341
5342         /* Enable PASID support in the device, if it wasn't already */
5343         if (!info->pasid_enabled)
5344                 iommu_enable_dev_iotlb(info);
5345
5346         if (info->ats_enabled) {
5347                 sdev->dev_iotlb = 1;
5348                 sdev->qdep = info->ats_qdep;
5349                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5350                         sdev->qdep = 0;
5351         }
5352         ret = 0;
5353
5354  out:
5355         spin_unlock(&iommu->lock);
5356         spin_unlock_irqrestore(&device_domain_lock, flags);
5357
5358         return ret;
5359 }
5360
5361 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5362 {
5363         struct intel_iommu *iommu;
5364         u8 bus, devfn;
5365
5366         if (iommu_dummy(dev)) {
5367                 dev_warn(dev,
5368                          "No IOMMU translation for device; cannot enable SVM\n");
5369                 return NULL;
5370         }
5371
5372         iommu = device_to_iommu(dev, &bus, &devfn);
5373         if ((!iommu)) {
5374                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5375                 return NULL;
5376         }
5377
5378         return iommu;
5379 }
5380 #endif /* CONFIG_INTEL_IOMMU_SVM */
5381
5382 const struct iommu_ops intel_iommu_ops = {
5383         .capable                = intel_iommu_capable,
5384         .domain_alloc           = intel_iommu_domain_alloc,
5385         .domain_free            = intel_iommu_domain_free,
5386         .attach_dev             = intel_iommu_attach_device,
5387         .detach_dev             = intel_iommu_detach_device,
5388         .map                    = intel_iommu_map,
5389         .unmap                  = intel_iommu_unmap,
5390         .iova_to_phys           = intel_iommu_iova_to_phys,
5391         .add_device             = intel_iommu_add_device,
5392         .remove_device          = intel_iommu_remove_device,
5393         .get_resv_regions       = intel_iommu_get_resv_regions,
5394         .put_resv_regions       = intel_iommu_put_resv_regions,
5395         .device_group           = pci_device_group,
5396         .pgsize_bitmap          = INTEL_IOMMU_PGSIZES,
5397 };
5398
5399 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5400 {
5401         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5402         pr_info("Disabling IOMMU for graphics on this chipset\n");
5403         dmar_map_gfx = 0;
5404 }
5405
5406 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5407 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5408 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5413
5414 static void quirk_iommu_rwbf(struct pci_dev *dev)
5415 {
5416         /*
5417          * Mobile 4 Series Chipset neglects to set RWBF capability,
5418          * but needs it. Same seems to hold for the desktop versions.
5419          */
5420         pr_info("Forcing write-buffer flush capability\n");
5421         rwbf_quirk = 1;
5422 }
5423
5424 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5428 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5429 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5430 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5431
5432 #define GGC 0x52
5433 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5434 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5435 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5436 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5437 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5438 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5439 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5440 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5441
5442 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5443 {
5444         unsigned short ggc;
5445
5446         if (pci_read_config_word(dev, GGC, &ggc))
5447                 return;
5448
5449         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5450                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5451                 dmar_map_gfx = 0;
5452         } else if (dmar_map_gfx) {
5453                 /* we have to ensure the gfx device is idle before we flush */
5454                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5455                 intel_iommu_strict = 1;
5456        }
5457 }
5458 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5459 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5460 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5461 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5462
5463 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5464    ISOCH DMAR unit for the Azalia sound device, but not give it any
5465    TLB entries, which causes it to deadlock. Check for that.  We do
5466    this in a function called from init_dmars(), instead of in a PCI
5467    quirk, because we don't want to print the obnoxious "BIOS broken"
5468    message if VT-d is actually disabled.
5469 */
5470 static void __init check_tylersburg_isoch(void)
5471 {
5472         struct pci_dev *pdev;
5473         uint32_t vtisochctrl;
5474
5475         /* If there's no Azalia in the system anyway, forget it. */
5476         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5477         if (!pdev)
5478                 return;
5479         pci_dev_put(pdev);
5480
5481         /* System Management Registers. Might be hidden, in which case
5482            we can't do the sanity check. But that's OK, because the
5483            known-broken BIOSes _don't_ actually hide it, so far. */
5484         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5485         if (!pdev)
5486                 return;
5487
5488         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5489                 pci_dev_put(pdev);
5490                 return;
5491         }
5492
5493         pci_dev_put(pdev);
5494
5495         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5496         if (vtisochctrl & 1)
5497                 return;
5498
5499         /* Drop all bits other than the number of TLB entries */
5500         vtisochctrl &= 0x1c;
5501
5502         /* If we have the recommended number of TLB entries (16), fine. */
5503         if (vtisochctrl == 0x10)
5504                 return;
5505
5506         /* Zero TLB entries? You get to ride the short bus to school. */
5507         if (!vtisochctrl) {
5508                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5509                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5510                      dmi_get_system_info(DMI_BIOS_VENDOR),
5511                      dmi_get_system_info(DMI_BIOS_VERSION),
5512                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5513                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5514                 return;
5515         }
5516
5517         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5518                vtisochctrl);
5519 }