drivers/iommu/intel-iommu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright © 2006-2014 Intel Corporation.
   4  *
   5  * Authors: David Woodhouse <dwmw2@infradead.org>,
   6  *          Ashok Raj <ashok.raj@intel.com>,
   7  *          Shaohua Li <shaohua.li@intel.com>,
   8  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
   9  *          Fenghua Yu <fenghua.yu@intel.com>
  10  *          Joerg Roedel <jroedel@suse.de>
  11  */
  12
  13 #define pr_fmt(fmt)     "DMAR: " fmt
  14 #define dev_fmt(fmt)    pr_fmt(fmt)
  15
  16 #include <linux/init.h>
  17 #include <linux/bitmap.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/export.h>
  20 #include <linux/slab.h>
  21 #include <linux/irq.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/pci.h>
  25 #include <linux/dmar.h>
  26 #include <linux/dma-mapping.h>
  27 #include <linux/mempool.h>
  28 #include <linux/memory.h>
  29 #include <linux/cpu.h>
  30 #include <linux/timer.h>
  31 #include <linux/io.h>
  32 #include <linux/iova.h>
  33 #include <linux/iommu.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/syscore_ops.h>
  36 #include <linux/tboot.h>
  37 #include <linux/dmi.h>
  38 #include <linux/pci-ats.h>
  39 #include <linux/memblock.h>
  40 #include <linux/dma-contiguous.h>
  41 #include <linux/dma-direct.h>
  42 #include <linux/crash_dump.h>
  43 #include <linux/numa.h>
  44 #include <asm/irq_remapping.h>
  45 #include <asm/cacheflush.h>
  46 #include <asm/iommu.h>
  47
  48 #include "irq_remapping.h"
  49 #include "intel-pasid.h"
  50
  51 #define ROOT_SIZE               VTD_PAGE_SIZE
  52 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  53
  54 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  55 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  56 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  57 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  58
  59 #define IOAPIC_RANGE_START      (0xfee00000)
  60 #define IOAPIC_RANGE_END        (0xfeefffff)
  61 #define IOVA_START_ADDR         (0x1000)
  62
  63 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  64
  65 #define MAX_AGAW_WIDTH 64
  66 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  67
  68 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  69 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  70
  71 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  72    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  73 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  74                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  75 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  76
  77 /* IO virtual address start page frame number */
  78 #define IOVA_START_PFN          (1)
  79
  80 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  81
  82 /* page table handling */
  83 #define LEVEL_STRIDE            (9)
  84 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  85
  86 /*
  87  * This bitmap is used to advertise the page sizes our hardware support
  88  * to the IOMMU core, which will then use this information to split
  89  * physically contiguous memory regions it is mapping into page sizes
  90  * that we support.
  91  *
  92  * Traditionally the IOMMU core just handed us the mappings directly,
  93  * after making sure the size is an order of a 4KiB page and that the
  94  * mapping has natural alignment.
  95  *
  96  * To retain this behavior, we currently advertise that we support
  97  * all page sizes that are an order of 4KiB.
  98  *
  99  * If at some point we'd like to utilize the IOMMU core's new behavior,
 100  * we could change this to advertise the real page sizes we support.
 101  */
 102 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 103
 104 static inline int agaw_to_level(int agaw)
 105 {
 106         return agaw + 2;
 107 }
 108
 109 static inline int agaw_to_width(int agaw)
 110 {
 111         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 112 }
 113
 114 static inline int width_to_agaw(int width)
 115 {
 116         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 117 }
 118
 119 static inline unsigned int level_to_offset_bits(int level)
 120 {
 121         return (level - 1) * LEVEL_STRIDE;
 122 }
 123
 124 static inline int pfn_level_offset(unsigned long pfn, int level)
 125 {
 126         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 127 }
 128
 129 static inline unsigned long level_mask(int level)
 130 {
 131         return -1UL << level_to_offset_bits(level);
 132 }
 133
 134 static inline unsigned long level_size(int level)
 135 {
 136         return 1UL << level_to_offset_bits(level);
 137 }
 138
 139 static inline unsigned long align_to_level(unsigned long pfn, int level)
 140 {
 141         return (pfn + level_size(level) - 1) & level_mask(level);
 142 }
 143
 144 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 145 {
 146         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 147 }
 148
 149 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 150    are never going to work. */
 151 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 152 {
 153         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 154 }
 155
 156 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 157 {
 158         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 159 }
 160 static inline unsigned long page_to_dma_pfn(struct page *pg)
 161 {
 162         return mm_to_dma_pfn(page_to_pfn(pg));
 163 }
 164 static inline unsigned long virt_to_dma_pfn(void *p)
 165 {
 166         return page_to_dma_pfn(virt_to_page(p));
 167 }
 168
 169 /* global iommu list, set NULL for ignored DMAR units */
 170 static struct intel_iommu **g_iommus;
 171
 172 static void __init check_tylersburg_isoch(void);
 173 static int rwbf_quirk;
 174
 175 /*
 176  * set to 1 to panic kernel if can't successfully enable VT-d
 177  * (used when kernel is launched w/ TXT)
 178  */
 179 static int force_on = 0;
 180 int intel_iommu_tboot_noforce;
 181 static int no_platform_optin;
 182
 183 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 184
 185 /*
 186  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 187  * if marked present.
 188  */
 189 static phys_addr_t root_entry_lctp(struct root_entry *re)
 190 {
 191         if (!(re->lo & 1))
 192                 return 0;
 193
 194         return re->lo & VTD_PAGE_MASK;
 195 }
 196
 197 /*
 198  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 199  * if marked present.
 200  */
 201 static phys_addr_t root_entry_uctp(struct root_entry *re)
 202 {
 203         if (!(re->hi & 1))
 204                 return 0;
 205
 206         return re->hi & VTD_PAGE_MASK;
 207 }
 208
 209 static inline void context_clear_pasid_enable(struct context_entry *context)
 210 {
 211         context->lo &= ~(1ULL << 11);
 212 }
 213
 214 static inline bool context_pasid_enabled(struct context_entry *context)
 215 {
 216         return !!(context->lo & (1ULL << 11));
 217 }
 218
 219 static inline void context_set_copied(struct context_entry *context)
 220 {
 221         context->hi |= (1ull << 3);
 222 }
 223
 224 static inline bool context_copied(struct context_entry *context)
 225 {
 226         return !!(context->hi & (1ULL << 3));
 227 }
 228
 229 static inline bool __context_present(struct context_entry *context)
 230 {
 231         return (context->lo & 1);
 232 }
 233
 234 bool context_present(struct context_entry *context)
 235 {
 236         return context_pasid_enabled(context) ?
 237              __context_present(context) :
 238              __context_present(context) && !context_copied(context);
 239 }
 240
 241 static inline void context_set_present(struct context_entry *context)
 242 {
 243         context->lo |= 1;
 244 }
 245
 246 static inline void context_set_fault_enable(struct context_entry *context)
 247 {
 248         context->lo &= (((u64)-1) << 2) | 1;
 249 }
 250
 251 static inline void context_set_translation_type(struct context_entry *context,
 252                                                 unsigned long value)
 253 {
 254         context->lo &= (((u64)-1) << 4) | 3;
 255         context->lo |= (value & 3) << 2;
 256 }
 257
 258 static inline void context_set_address_root(struct context_entry *context,
 259                                             unsigned long value)
 260 {
 261         context->lo &= ~VTD_PAGE_MASK;
 262         context->lo |= value & VTD_PAGE_MASK;
 263 }
 264
 265 static inline void context_set_address_width(struct context_entry *context,
 266                                              unsigned long value)
 267 {
 268         context->hi |= value & 7;
 269 }
 270
 271 static inline void context_set_domain_id(struct context_entry *context,
 272                                          unsigned long value)
 273 {
 274         context->hi |= (value & ((1 << 16) - 1)) << 8;
 275 }
 276
 277 static inline int context_domain_id(struct context_entry *c)
 278 {
 279         return((c->hi >> 8) & 0xffff);
 280 }
 281
 282 static inline void context_clear_entry(struct context_entry *context)
 283 {
 284         context->lo = 0;
 285         context->hi = 0;
 286 }
 287
 288 /*
 289  * This domain is a statically identity mapping domain.
 290  *      1. This domain creats a static 1:1 mapping to all usable memory.
 291  *      2. It maps to each iommu if successful.
 292  *      3. Each iommu mapps to this domain if successful.
 293  */
 294 static struct dmar_domain *si_domain;
 295 static int hw_pass_through = 1;
 296
 297 /* si_domain contains mulitple devices */
 298 #define DOMAIN_FLAG_STATIC_IDENTITY             BIT(0)
 299
 300 /*
 301  * This is a DMA domain allocated through the iommu domain allocation
 302  * interface. But one or more devices belonging to this domain have
 303  * been chosen to use a private domain. We should avoid to use the
 304  * map/unmap/iova_to_phys APIs on it.
 305  */
 306 #define DOMAIN_FLAG_LOSE_CHILDREN               BIT(1)
 307
 308 #define for_each_domain_iommu(idx, domain)                      \
 309         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 310                 if (domain->iommu_refcnt[idx])
 311
 312 struct dmar_rmrr_unit {
 313         struct list_head list;          /* list of rmrr units   */
 314         struct acpi_dmar_header *hdr;   /* ACPI header          */
 315         u64     base_address;           /* reserved base address*/
 316         u64     end_address;            /* reserved end address */
 317         struct dmar_dev_scope *devices; /* target devices */
 318         int     devices_cnt;            /* target device count */
 319 };
 320
 321 struct dmar_atsr_unit {
 322         struct list_head list;          /* list of ATSR units */
 323         struct acpi_dmar_header *hdr;   /* ACPI header */
 324         struct dmar_dev_scope *devices; /* target devices */
 325         int devices_cnt;                /* target device count */
 326         u8 include_all:1;               /* include all ports */
 327 };
 328
 329 static LIST_HEAD(dmar_atsr_units);
 330 static LIST_HEAD(dmar_rmrr_units);
 331
 332 #define for_each_rmrr_units(rmrr) \
 333         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 334
 335 /* bitmap for indexing intel_iommus */
 336 static int g_num_of_iommus;
 337
 338 static void domain_exit(struct dmar_domain *domain);
 339 static void domain_remove_dev_info(struct dmar_domain *domain);
 340 static void dmar_remove_one_dev_info(struct device *dev);
 341 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 342 static int domain_detach_iommu(struct dmar_domain *domain,
 343                                struct intel_iommu *iommu);
 344 static bool device_is_rmrr_locked(struct device *dev);
 345 static int intel_iommu_attach_device(struct iommu_domain *domain,
 346                                      struct device *dev);
 347
 348 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 349 int dmar_disabled = 0;
 350 #else
 351 int dmar_disabled = 1;
 352 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 353
 354 int intel_iommu_sm;
 355 int intel_iommu_enabled = 0;
 356 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 357
 358 static int dmar_map_gfx = 1;
 359 static int dmar_forcedac;
 360 static int intel_iommu_strict;
 361 static int intel_iommu_superpage = 1;
 362 static int iommu_identity_mapping;
 363
 364 #define IDENTMAP_ALL            1
 365 #define IDENTMAP_GFX            2
 366 #define IDENTMAP_AZALIA         4
 367
 368 int intel_iommu_gfx_mapped;
 369 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 370
 371 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 372 #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
 373 static DEFINE_SPINLOCK(device_domain_lock);
 374 static LIST_HEAD(device_domain_list);
 375
 376 /*
 377  * Iterate over elements in device_domain_list and call the specified
 378  * callback @fn against each element.
 379  */
 380 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
 381                                      void *data), void *data)
 382 {
 383         int ret = 0;
 384         unsigned long flags;
 385         struct device_domain_info *info;
 386
 387         spin_lock_irqsave(&device_domain_lock, flags);
 388         list_for_each_entry(info, &device_domain_list, global) {
 389                 ret = fn(info, data);
 390                 if (ret) {
 391                         spin_unlock_irqrestore(&device_domain_lock, flags);
 392                         return ret;
 393                 }
 394         }
 395         spin_unlock_irqrestore(&device_domain_lock, flags);
 396
 397         return 0;
 398 }
 399
 400 const struct iommu_ops intel_iommu_ops;
 401
 402 static bool translation_pre_enabled(struct intel_iommu *iommu)
 403 {
 404         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 405 }
 406
 407 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 408 {
 409         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 410 }
 411
 412 static void init_translation_status(struct intel_iommu *iommu)
 413 {
 414         u32 gsts;
 415
 416         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 417         if (gsts & DMA_GSTS_TES)
 418                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 419 }
 420
 421 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 422 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 423 {
 424         return container_of(dom, struct dmar_domain, domain);
 425 }
 426
 427 static int __init intel_iommu_setup(char *str)
 428 {
 429         if (!str)
 430                 return -EINVAL;
 431         while (*str) {
 432                 if (!strncmp(str, "on", 2)) {
 433                         dmar_disabled = 0;
 434                         pr_info("IOMMU enabled\n");
 435                 } else if (!strncmp(str, "off", 3)) {
 436                         dmar_disabled = 1;
 437                         no_platform_optin = 1;
 438                         pr_info("IOMMU disabled\n");
 439                 } else if (!strncmp(str, "igfx_off", 8)) {
 440                         dmar_map_gfx = 0;
 441                         pr_info("Disable GFX device mapping\n");
 442                 } else if (!strncmp(str, "forcedac", 8)) {
 443                         pr_info("Forcing DAC for PCI devices\n");
 444                         dmar_forcedac = 1;
 445                 } else if (!strncmp(str, "strict", 6)) {
 446                         pr_info("Disable batched IOTLB flush\n");
 447                         intel_iommu_strict = 1;
 448                 } else if (!strncmp(str, "sp_off", 6)) {
 449                         pr_info("Disable supported super page\n");
 450                         intel_iommu_superpage = 0;
 451                 } else if (!strncmp(str, "sm_on", 5)) {
 452                         pr_info("Intel-IOMMU: scalable mode supported\n");
 453                         intel_iommu_sm = 1;
 454                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 455                         printk(KERN_INFO
 456                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 457                         intel_iommu_tboot_noforce = 1;
 458                 }
 459
 460                 str += strcspn(str, ",");
 461                 while (*str == ',')
 462                         str++;
 463         }
 464         return 0;
 465 }
 466 __setup("intel_iommu=", intel_iommu_setup);
 467
 468 static struct kmem_cache *iommu_domain_cache;
 469 static struct kmem_cache *iommu_devinfo_cache;
 470
 471 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 472 {
 473         struct dmar_domain **domains;
 474         int idx = did >> 8;
 475
 476         domains = iommu->domains[idx];
 477         if (!domains)
 478                 return NULL;
 479
 480         return domains[did & 0xff];
 481 }
 482
 483 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 484                              struct dmar_domain *domain)
 485 {
 486         struct dmar_domain **domains;
 487         int idx = did >> 8;
 488
 489         if (!iommu->domains[idx]) {
 490                 size_t size = 256 * sizeof(struct dmar_domain *);
 491                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 492         }
 493
 494         domains = iommu->domains[idx];
 495         if (WARN_ON(!domains))
 496                 return;
 497         else
 498                 domains[did & 0xff] = domain;
 499 }
 500
 501 void *alloc_pgtable_page(int node)
 502 {
 503         struct page *page;
 504         void *vaddr = NULL;
 505
 506         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 507         if (page)
 508                 vaddr = page_address(page);
 509         return vaddr;
 510 }
 511
 512 void free_pgtable_page(void *vaddr)
 513 {
 514         free_page((unsigned long)vaddr);
 515 }
 516
 517 static inline void *alloc_domain_mem(void)
 518 {
 519         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 520 }
 521
 522 static void free_domain_mem(void *vaddr)
 523 {
 524         kmem_cache_free(iommu_domain_cache, vaddr);
 525 }
 526
 527 static inline void * alloc_devinfo_mem(void)
 528 {
 529         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 530 }
 531
 532 static inline void free_devinfo_mem(void *vaddr)
 533 {
 534         kmem_cache_free(iommu_devinfo_cache, vaddr);
 535 }
 536
 537 static inline int domain_type_is_si(struct dmar_domain *domain)
 538 {
 539         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 540 }
 541
 542 static inline int domain_pfn_supported(struct dmar_domain *domain,
 543                                        unsigned long pfn)
 544 {
 545         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 546
 547         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 548 }
 549
 550 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 551 {
 552         unsigned long sagaw;
 553         int agaw = -1;
 554
 555         sagaw = cap_sagaw(iommu->cap);
 556         for (agaw = width_to_agaw(max_gaw);
 557              agaw >= 0; agaw--) {
 558                 if (test_bit(agaw, &sagaw))
 559                         break;
 560         }
 561
 562         return agaw;
 563 }
 564
 565 /*
 566  * Calculate max SAGAW for each iommu.
 567  */
 568 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 569 {
 570         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 571 }
 572
 573 /*
 574  * calculate agaw for each iommu.
 575  * "SAGAW" may be different across iommus, use a default agaw, and
 576  * get a supported less agaw for iommus that don't support the default agaw.
 577  */
 578 int iommu_calculate_agaw(struct intel_iommu *iommu)
 579 {
 580         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 581 }
 582
 583 /* This functionin only returns single iommu in a domain */
 584 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 585 {
 586         int iommu_id;
 587
 588         /* si_domain and vm domain should not get here. */
 589         if (WARN_ON(domain->domain.type != IOMMU_DOMAIN_DMA))
 590                 return NULL;
 591
 592         for_each_domain_iommu(iommu_id, domain)
 593                 break;
 594
 595         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 596                 return NULL;
 597
 598         return g_iommus[iommu_id];
 599 }
 600
 601 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 602 {
 603         struct dmar_drhd_unit *drhd;
 604         struct intel_iommu *iommu;
 605         bool found = false;
 606         int i;
 607
 608         domain->iommu_coherency = 1;
 609
 610         for_each_domain_iommu(i, domain) {
 611                 found = true;
 612                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 613                         domain->iommu_coherency = 0;
 614                         break;
 615                 }
 616         }
 617         if (found)
 618                 return;
 619
 620         /* No hardware attached; use lowest common denominator */
 621         rcu_read_lock();
 622         for_each_active_iommu(iommu, drhd) {
 623                 if (!ecap_coherent(iommu->ecap)) {
 624                         domain->iommu_coherency = 0;
 625                         break;
 626                 }
 627         }
 628         rcu_read_unlock();
 629 }
 630
 631 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 632 {
 633         struct dmar_drhd_unit *drhd;
 634         struct intel_iommu *iommu;
 635         int ret = 1;
 636
 637         rcu_read_lock();
 638         for_each_active_iommu(iommu, drhd) {
 639                 if (iommu != skip) {
 640                         if (!ecap_sc_support(iommu->ecap)) {
 641                                 ret = 0;
 642                                 break;
 643                         }
 644                 }
 645         }
 646         rcu_read_unlock();
 647
 648         return ret;
 649 }
 650
 651 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 652 {
 653         struct dmar_drhd_unit *drhd;
 654         struct intel_iommu *iommu;
 655         int mask = 0xf;
 656
 657         if (!intel_iommu_superpage) {
 658                 return 0;
 659         }
 660
 661         /* set iommu_superpage to the smallest common denominator */
 662         rcu_read_lock();
 663         for_each_active_iommu(iommu, drhd) {
 664                 if (iommu != skip) {
 665                         mask &= cap_super_page_val(iommu->cap);
 666                         if (!mask)
 667                                 break;
 668                 }
 669         }
 670         rcu_read_unlock();
 671
 672         return fls(mask);
 673 }
 674
 675 /* Some capabilities may be different across iommus */
 676 static void domain_update_iommu_cap(struct dmar_domain *domain)
 677 {
 678         domain_update_iommu_coherency(domain);
 679         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 680         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 681 }
 682
 683 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
 684                                          u8 devfn, int alloc)
 685 {
 686         struct root_entry *root = &iommu->root_entry[bus];
 687         struct context_entry *context;
 688         u64 *entry;
 689
 690         entry = &root->lo;
 691         if (sm_supported(iommu)) {
 692                 if (devfn >= 0x80) {
 693                         devfn -= 0x80;
 694                         entry = &root->hi;
 695                 }
 696                 devfn *= 2;
 697         }
 698         if (*entry & 1)
 699                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 700         else {
 701                 unsigned long phy_addr;
 702                 if (!alloc)
 703                         return NULL;
 704
 705                 context = alloc_pgtable_page(iommu->node);
 706                 if (!context)
 707                         return NULL;
 708
 709                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 710                 phy_addr = virt_to_phys((void *)context);
 711                 *entry = phy_addr | 1;
 712                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 713         }
 714         return &context[devfn];
 715 }
 716
 717 static int iommu_dummy(struct device *dev)
 718 {
 719         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 720 }
 721
 722 /**
 723  * is_downstream_to_pci_bridge - test if a device belongs to the PCI
 724  *                               sub-hierarchy of a candidate PCI-PCI bridge
 725  * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
 726  * @bridge: the candidate PCI-PCI bridge
 727  *
 728  * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
 729  */
 730 static bool
 731 is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
 732 {
 733         struct pci_dev *pdev, *pbridge;
 734
 735         if (!dev_is_pci(dev) || !dev_is_pci(bridge))
 736                 return false;
 737
 738         pdev = to_pci_dev(dev);
 739         pbridge = to_pci_dev(bridge);
 740
 741         if (pbridge->subordinate &&
 742             pbridge->subordinate->number <= pdev->bus->number &&
 743             pbridge->subordinate->busn_res.end >= pdev->bus->number)
 744                 return true;
 745
 746         return false;
 747 }
 748
 749 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 750 {
 751         struct dmar_drhd_unit *drhd = NULL;
 752         struct intel_iommu *iommu;
 753         struct device *tmp;
 754         struct pci_dev *pdev = NULL;
 755         u16 segment = 0;
 756         int i;
 757
 758         if (iommu_dummy(dev))
 759                 return NULL;
 760
 761         if (dev_is_pci(dev)) {
 762                 struct pci_dev *pf_pdev;
 763
 764                 pdev = to_pci_dev(dev);
 765
 766 #ifdef CONFIG_X86
 767                 /* VMD child devices currently cannot be handled individually */
 768                 if (is_vmd(pdev->bus))
 769                         return NULL;
 770 #endif
 771
 772                 /* VFs aren't listed in scope tables; we need to look up
 773                  * the PF instead to find the IOMMU. */
 774                 pf_pdev = pci_physfn(pdev);
 775                 dev = &pf_pdev->dev;
 776                 segment = pci_domain_nr(pdev->bus);
 777         } else if (has_acpi_companion(dev))
 778                 dev = &ACPI_COMPANION(dev)->dev;
 779
 780         rcu_read_lock();
 781         for_each_active_iommu(iommu, drhd) {
 782                 if (pdev && segment != drhd->segment)
 783                         continue;
 784
 785                 for_each_active_dev_scope(drhd->devices,
 786                                           drhd->devices_cnt, i, tmp) {
 787                         if (tmp == dev) {
 788                                 /* For a VF use its original BDF# not that of the PF
 789                                  * which we used for the IOMMU lookup. Strictly speaking
 790                                  * we could do this for all PCI devices; we only need to
 791                                  * get the BDF# from the scope table for ACPI matches. */
 792                                 if (pdev && pdev->is_virtfn)
 793                                         goto got_pdev;
 794
 795                                 *bus = drhd->devices[i].bus;
 796                                 *devfn = drhd->devices[i].devfn;
 797                                 goto out;
 798                         }
 799
 800                         if (is_downstream_to_pci_bridge(dev, tmp))
 801                                 goto got_pdev;
 802                 }
 803
 804                 if (pdev && drhd->include_all) {
 805                 got_pdev:
 806                         *bus = pdev->bus->number;
 807                         *devfn = pdev->devfn;
 808                         goto out;
 809                 }
 810         }
 811         iommu = NULL;
 812  out:
 813         rcu_read_unlock();
 814
 815         return iommu;
 816 }
 817
 818 static void domain_flush_cache(struct dmar_domain *domain,
 819                                void *addr, int size)
 820 {
 821         if (!domain->iommu_coherency)
 822                 clflush_cache_range(addr, size);
 823 }
 824
 825 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 826 {
 827         struct context_entry *context;
 828         int ret = 0;
 829         unsigned long flags;
 830
 831         spin_lock_irqsave(&iommu->lock, flags);
 832         context = iommu_context_addr(iommu, bus, devfn, 0);
 833         if (context)
 834                 ret = context_present(context);
 835         spin_unlock_irqrestore(&iommu->lock, flags);
 836         return ret;
 837 }
 838
 839 static void free_context_table(struct intel_iommu *iommu)
 840 {
 841         int i;
 842         unsigned long flags;
 843         struct context_entry *context;
 844
 845         spin_lock_irqsave(&iommu->lock, flags);
 846         if (!iommu->root_entry) {
 847                 goto out;
 848         }
 849         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 850                 context = iommu_context_addr(iommu, i, 0, 0);
 851                 if (context)
 852                         free_pgtable_page(context);
 853
 854                 if (!sm_supported(iommu))
 855                         continue;
 856
 857                 context = iommu_context_addr(iommu, i, 0x80, 0);
 858                 if (context)
 859                         free_pgtable_page(context);
 860
 861         }
 862         free_pgtable_page(iommu->root_entry);
 863         iommu->root_entry = NULL;
 864 out:
 865         spin_unlock_irqrestore(&iommu->lock, flags);
 866 }
 867
 868 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 869                                       unsigned long pfn, int *target_level)
 870 {
 871         struct dma_pte *parent, *pte;
 872         int level = agaw_to_level(domain->agaw);
 873         int offset;
 874
 875         BUG_ON(!domain->pgd);
 876
 877         if (!domain_pfn_supported(domain, pfn))
 878                 /* Address beyond IOMMU's addressing capabilities. */
 879                 return NULL;
 880
 881         parent = domain->pgd;
 882
 883         while (1) {
 884                 void *tmp_page;
 885
 886                 offset = pfn_level_offset(pfn, level);
 887                 pte = &parent[offset];
 888                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 889                         break;
 890                 if (level == *target_level)
 891                         break;
 892
 893                 if (!dma_pte_present(pte)) {
 894                         uint64_t pteval;
 895
 896                         tmp_page = alloc_pgtable_page(domain->nid);
 897
 898                         if (!tmp_page)
 899                                 return NULL;
 900
 901                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 902                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 903                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 904                                 /* Someone else set it while we were thinking; use theirs. */
 905                                 free_pgtable_page(tmp_page);
 906                         else
 907                                 domain_flush_cache(domain, pte, sizeof(*pte));
 908                 }
 909                 if (level == 1)
 910                         break;
 911
 912                 parent = phys_to_virt(dma_pte_addr(pte));
 913                 level--;
 914         }
 915
 916         if (!*target_level)
 917                 *target_level = level;
 918
 919         return pte;
 920 }
 921
 922 /* return address's pte at specific level */
 923 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 924                                          unsigned long pfn,
 925                                          int level, int *large_page)
 926 {
 927         struct dma_pte *parent, *pte;
 928         int total = agaw_to_level(domain->agaw);
 929         int offset;
 930
 931         parent = domain->pgd;
 932         while (level <= total) {
 933                 offset = pfn_level_offset(pfn, total);
 934                 pte = &parent[offset];
 935                 if (level == total)
 936                         return pte;
 937
 938                 if (!dma_pte_present(pte)) {
 939                         *large_page = total;
 940                         break;
 941                 }
 942
 943                 if (dma_pte_superpage(pte)) {
 944                         *large_page = total;
 945                         return pte;
 946                 }
 947
 948                 parent = phys_to_virt(dma_pte_addr(pte));
 949                 total--;
 950         }
 951         return NULL;
 952 }
 953
 954 /* clear last level pte, a tlb flush should be followed */
 955 static void dma_pte_clear_range(struct dmar_domain *domain,
 956                                 unsigned long start_pfn,
 957                                 unsigned long last_pfn)
 958 {
 959         unsigned int large_page;
 960         struct dma_pte *first_pte, *pte;
 961
 962         BUG_ON(!domain_pfn_supported(domain, start_pfn));
 963         BUG_ON(!domain_pfn_supported(domain, last_pfn));
 964         BUG_ON(start_pfn > last_pfn);
 965
 966         /* we don't need lock here; nobody else touches the iova range */
 967         do {
 968                 large_page = 1;
 969                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 970                 if (!pte) {
 971                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 972                         continue;
 973                 }
 974                 do {
 975                         dma_clear_pte(pte);
 976                         start_pfn += lvl_to_nr_pages(large_page);
 977                         pte++;
 978                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 979
 980                 domain_flush_cache(domain, first_pte,
 981                                    (void *)pte - (void *)first_pte);
 982
 983         } while (start_pfn && start_pfn <= last_pfn);
 984 }
 985
 986 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 987                                int retain_level, struct dma_pte *pte,
 988                                unsigned long pfn, unsigned long start_pfn,
 989                                unsigned long last_pfn)
 990 {
 991         pfn = max(start_pfn, pfn);
 992         pte = &pte[pfn_level_offset(pfn, level)];
 993
 994         do {
 995                 unsigned long level_pfn;
 996                 struct dma_pte *level_pte;
 997
 998                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 999                         goto next;
1000
1001                 level_pfn = pfn & level_mask(level);
1002                 level_pte = phys_to_virt(dma_pte_addr(pte));
1003
1004                 if (level > 2) {
1005                         dma_pte_free_level(domain, level - 1, retain_level,
1006                                            level_pte, level_pfn, start_pfn,
1007                                            last_pfn);
1008                 }
1009
1010                 /*
1011                  * Free the page table if we're below the level we want to
1012                  * retain and the range covers the entire table.
1013                  */
1014                 if (level < retain_level && !(start_pfn > level_pfn ||
1015                       last_pfn < level_pfn + level_size(level) - 1)) {
1016                         dma_clear_pte(pte);
1017                         domain_flush_cache(domain, pte, sizeof(*pte));
1018                         free_pgtable_page(level_pte);
1019                 }
1020 next:
1021                 pfn += level_size(level);
1022         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1023 }
1024
1025 /*
1026  * clear last level (leaf) ptes and free page table pages below the
1027  * level we wish to keep intact.
1028  */
1029 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1030                                    unsigned long start_pfn,
1031                                    unsigned long last_pfn,
1032                                    int retain_level)
1033 {
1034         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1035         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1036         BUG_ON(start_pfn > last_pfn);
1037
1038         dma_pte_clear_range(domain, start_pfn, last_pfn);
1039
1040         /* We don't need lock here; nobody else touches the iova range */
1041         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1042                            domain->pgd, 0, start_pfn, last_pfn);
1043
1044         /* free pgd */
1045         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1046                 free_pgtable_page(domain->pgd);
1047                 domain->pgd = NULL;
1048         }
1049 }
1050
1051 /* When a page at a given level is being unlinked from its parent, we don't
1052    need to *modify* it at all. All we need to do is make a list of all the
1053    pages which can be freed just as soon as we've flushed the IOTLB and we
1054    know the hardware page-walk will no longer touch them.
1055    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1056    be freed. */
1057 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1058                                             int level, struct dma_pte *pte,
1059                                             struct page *freelist)
1060 {
1061         struct page *pg;
1062
1063         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1064         pg->freelist = freelist;
1065         freelist = pg;
1066
1067         if (level == 1)
1068                 return freelist;
1069
1070         pte = page_address(pg);
1071         do {
1072                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1073                         freelist = dma_pte_list_pagetables(domain, level - 1,
1074                                                            pte, freelist);
1075                 pte++;
1076         } while (!first_pte_in_page(pte));
1077
1078         return freelist;
1079 }
1080
1081 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1082                                         struct dma_pte *pte, unsigned long pfn,
1083                                         unsigned long start_pfn,
1084                                         unsigned long last_pfn,
1085                                         struct page *freelist)
1086 {
1087         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1088
1089         pfn = max(start_pfn, pfn);
1090         pte = &pte[pfn_level_offset(pfn, level)];
1091
1092         do {
1093                 unsigned long level_pfn;
1094
1095                 if (!dma_pte_present(pte))
1096                         goto next;
1097
1098                 level_pfn = pfn & level_mask(level);
1099
1100                 /* If range covers entire pagetable, free it */
1101                 if (start_pfn <= level_pfn &&
1102                     last_pfn >= level_pfn + level_size(level) - 1) {
1103                         /* These suborbinate page tables are going away entirely. Don't
1104                            bother to clear them; we're just going to *free* them. */
1105                         if (level > 1 && !dma_pte_superpage(pte))
1106                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1107
1108                         dma_clear_pte(pte);
1109                         if (!first_pte)
1110                                 first_pte = pte;
1111                         last_pte = pte;
1112                 } else if (level > 1) {
1113                         /* Recurse down into a level that isn't *entirely* obsolete */
1114                         freelist = dma_pte_clear_level(domain, level - 1,
1115                                                        phys_to_virt(dma_pte_addr(pte)),
1116                                                        level_pfn, start_pfn, last_pfn,
1117                                                        freelist);
1118                 }
1119 next:
1120                 pfn += level_size(level);
1121         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1122
1123         if (first_pte)
1124                 domain_flush_cache(domain, first_pte,
1125                                    (void *)++last_pte - (void *)first_pte);
1126
1127         return freelist;
1128 }
1129
1130 /* We can't just free the pages because the IOMMU may still be walking
1131    the page tables, and may have cached the intermediate levels. The
1132    pages can only be freed after the IOTLB flush has been done. */
1133 static struct page *domain_unmap(struct dmar_domain *domain,
1134                                  unsigned long start_pfn,
1135                                  unsigned long last_pfn)
1136 {
1137         struct page *freelist;
1138
1139         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1140         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1141         BUG_ON(start_pfn > last_pfn);
1142
1143         /* we don't need lock here; nobody else touches the iova range */
1144         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1145                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1146
1147         /* free pgd */
1148         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1149                 struct page *pgd_page = virt_to_page(domain->pgd);
1150                 pgd_page->freelist = freelist;
1151                 freelist = pgd_page;
1152
1153                 domain->pgd = NULL;
1154         }
1155
1156         return freelist;
1157 }
1158
1159 static void dma_free_pagelist(struct page *freelist)
1160 {
1161         struct page *pg;
1162
1163         while ((pg = freelist)) {
1164                 freelist = pg->freelist;
1165                 free_pgtable_page(page_address(pg));
1166         }
1167 }
1168
1169 static void iova_entry_free(unsigned long data)
1170 {
1171         struct page *freelist = (struct page *)data;
1172
1173         dma_free_pagelist(freelist);
1174 }
1175
1176 /* iommu handling */
1177 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1178 {
1179         struct root_entry *root;
1180         unsigned long flags;
1181
1182         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1183         if (!root) {
1184                 pr_err("Allocating root entry for %s failed\n",
1185                         iommu->name);
1186                 return -ENOMEM;
1187         }
1188
1189         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1190
1191         spin_lock_irqsave(&iommu->lock, flags);
1192         iommu->root_entry = root;
1193         spin_unlock_irqrestore(&iommu->lock, flags);
1194
1195         return 0;
1196 }
1197
1198 static void iommu_set_root_entry(struct intel_iommu *iommu)
1199 {
1200         u64 addr;
1201         u32 sts;
1202         unsigned long flag;
1203
1204         addr = virt_to_phys(iommu->root_entry);
1205         if (sm_supported(iommu))
1206                 addr |= DMA_RTADDR_SMT;
1207
1208         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1209         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1210
1211         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1212
1213         /* Make sure hardware complete it */
1214         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1215                       readl, (sts & DMA_GSTS_RTPS), sts);
1216
1217         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1218 }
1219
1220 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1221 {
1222         u32 val;
1223         unsigned long flag;
1224
1225         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1226                 return;
1227
1228         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1229         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1230
1231         /* Make sure hardware complete it */
1232         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1233                       readl, (!(val & DMA_GSTS_WBFS)), val);
1234
1235         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1236 }
1237
1238 /* return value determine if we need a write buffer flush */
1239 static void __iommu_flush_context(struct intel_iommu *iommu,
1240                                   u16 did, u16 source_id, u8 function_mask,
1241                                   u64 type)
1242 {
1243         u64 val = 0;
1244         unsigned long flag;
1245
1246         switch (type) {
1247         case DMA_CCMD_GLOBAL_INVL:
1248                 val = DMA_CCMD_GLOBAL_INVL;
1249                 break;
1250         case DMA_CCMD_DOMAIN_INVL:
1251                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1252                 break;
1253         case DMA_CCMD_DEVICE_INVL:
1254                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1255                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1256                 break;
1257         default:
1258                 BUG();
1259         }
1260         val |= DMA_CCMD_ICC;
1261
1262         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1263         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1264
1265         /* Make sure hardware complete it */
1266         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1267                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1268
1269         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1270 }
1271
1272 /* return value determine if we need a write buffer flush */
1273 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1274                                 u64 addr, unsigned int size_order, u64 type)
1275 {
1276         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1277         u64 val = 0, val_iva = 0;
1278         unsigned long flag;
1279
1280         switch (type) {
1281         case DMA_TLB_GLOBAL_FLUSH:
1282                 /* global flush doesn't need set IVA_REG */
1283                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1284                 break;
1285         case DMA_TLB_DSI_FLUSH:
1286                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1287                 break;
1288         case DMA_TLB_PSI_FLUSH:
1289                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1290                 /* IH bit is passed in as part of address */
1291                 val_iva = size_order | addr;
1292                 break;
1293         default:
1294                 BUG();
1295         }
1296         /* Note: set drain read/write */
1297 #if 0
1298         /*
1299          * This is probably to be super secure.. Looks like we can
1300          * ignore it without any impact.
1301          */
1302         if (cap_read_drain(iommu->cap))
1303                 val |= DMA_TLB_READ_DRAIN;
1304 #endif
1305         if (cap_write_drain(iommu->cap))
1306                 val |= DMA_TLB_WRITE_DRAIN;
1307
1308         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1309         /* Note: Only uses first TLB reg currently */
1310         if (val_iva)
1311                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1312         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1313
1314         /* Make sure hardware complete it */
1315         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1316                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1317
1318         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1319
1320         /* check IOTLB invalidation granularity */
1321         if (DMA_TLB_IAIG(val) == 0)
1322                 pr_err("Flush IOTLB failed\n");
1323         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1324                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1325                         (unsigned long long)DMA_TLB_IIRG(type),
1326                         (unsigned long long)DMA_TLB_IAIG(val));
1327 }
1328
1329 static struct device_domain_info *
1330 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1331                          u8 bus, u8 devfn)
1332 {
1333         struct device_domain_info *info;
1334
1335         assert_spin_locked(&device_domain_lock);
1336
1337         if (!iommu->qi)
1338                 return NULL;
1339
1340         list_for_each_entry(info, &domain->devices, link)
1341                 if (info->iommu == iommu && info->bus == bus &&
1342                     info->devfn == devfn) {
1343                         if (info->ats_supported && info->dev)
1344                                 return info;
1345                         break;
1346                 }
1347
1348         return NULL;
1349 }
1350
1351 static void domain_update_iotlb(struct dmar_domain *domain)
1352 {
1353         struct device_domain_info *info;
1354         bool has_iotlb_device = false;
1355
1356         assert_spin_locked(&device_domain_lock);
1357
1358         list_for_each_entry(info, &domain->devices, link) {
1359                 struct pci_dev *pdev;
1360
1361                 if (!info->dev || !dev_is_pci(info->dev))
1362                         continue;
1363
1364                 pdev = to_pci_dev(info->dev);
1365                 if (pdev->ats_enabled) {
1366                         has_iotlb_device = true;
1367                         break;
1368                 }
1369         }
1370
1371         domain->has_iotlb_device = has_iotlb_device;
1372 }
1373
1374 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1375 {
1376         struct pci_dev *pdev;
1377
1378         assert_spin_locked(&device_domain_lock);
1379
1380         if (!info || !dev_is_pci(info->dev))
1381                 return;
1382
1383         pdev = to_pci_dev(info->dev);
1384         /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1385          * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1386          * queue depth at PF level. If DIT is not set, PFSID will be treated as
1387          * reserved, which should be set to 0.
1388          */
1389         if (!ecap_dit(info->iommu->ecap))
1390                 info->pfsid = 0;
1391         else {
1392                 struct pci_dev *pf_pdev;
1393
1394                 /* pdev will be returned if device is not a vf */
1395                 pf_pdev = pci_physfn(pdev);
1396                 info->pfsid = pci_dev_id(pf_pdev);
1397         }
1398
1399 #ifdef CONFIG_INTEL_IOMMU_SVM
1400         /* The PCIe spec, in its wisdom, declares that the behaviour of
1401            the device if you enable PASID support after ATS support is
1402            undefined. So always enable PASID support on devices which
1403            have it, even if we can't yet know if we're ever going to
1404            use it. */
1405         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1406                 info->pasid_enabled = 1;
1407
1408         if (info->pri_supported &&
1409             (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1)  &&
1410             !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1411                 info->pri_enabled = 1;
1412 #endif
1413         if (!pdev->untrusted && info->ats_supported &&
1414             pci_ats_page_aligned(pdev) &&
1415             !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1416                 info->ats_enabled = 1;
1417                 domain_update_iotlb(info->domain);
1418                 info->ats_qdep = pci_ats_queue_depth(pdev);
1419         }
1420 }
1421
1422 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1423 {
1424         struct pci_dev *pdev;
1425
1426         assert_spin_locked(&device_domain_lock);
1427
1428         if (!dev_is_pci(info->dev))
1429                 return;
1430
1431         pdev = to_pci_dev(info->dev);
1432
1433         if (info->ats_enabled) {
1434                 pci_disable_ats(pdev);
1435                 info->ats_enabled = 0;
1436                 domain_update_iotlb(info->domain);
1437         }
1438 #ifdef CONFIG_INTEL_IOMMU_SVM
1439         if (info->pri_enabled) {
1440                 pci_disable_pri(pdev);
1441                 info->pri_enabled = 0;
1442         }
1443         if (info->pasid_enabled) {
1444                 pci_disable_pasid(pdev);
1445                 info->pasid_enabled = 0;
1446         }
1447 #endif
1448 }
1449
1450 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1451                                   u64 addr, unsigned mask)
1452 {
1453         u16 sid, qdep;
1454         unsigned long flags;
1455         struct device_domain_info *info;
1456
1457         if (!domain->has_iotlb_device)
1458                 return;
1459
1460         spin_lock_irqsave(&device_domain_lock, flags);
1461         list_for_each_entry(info, &domain->devices, link) {
1462                 if (!info->ats_enabled)
1463                         continue;
1464
1465                 sid = info->bus << 8 | info->devfn;
1466                 qdep = info->ats_qdep;
1467                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1468                                 qdep, addr, mask);
1469         }
1470         spin_unlock_irqrestore(&device_domain_lock, flags);
1471 }
1472
1473 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1474                                   struct dmar_domain *domain,
1475                                   unsigned long pfn, unsigned int pages,
1476                                   int ih, int map)
1477 {
1478         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1479         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1480         u16 did = domain->iommu_did[iommu->seq_id];
1481
1482         BUG_ON(pages == 0);
1483
1484         if (ih)
1485                 ih = 1 << 6;
1486         /*
1487          * Fallback to domain selective flush if no PSI support or the size is
1488          * too big.
1489          * PSI requires page size to be 2 ^ x, and the base address is naturally
1490          * aligned to the size
1491          */
1492         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1493                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1494                                                 DMA_TLB_DSI_FLUSH);
1495         else
1496                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1497                                                 DMA_TLB_PSI_FLUSH);
1498
1499         /*
1500          * In caching mode, changes of pages from non-present to present require
1501          * flush. However, device IOTLB doesn't need to be flushed in this case.
1502          */
1503         if (!cap_caching_mode(iommu->cap) || !map)
1504                 iommu_flush_dev_iotlb(domain, addr, mask);
1505 }
1506
1507 /* Notification for newly created mappings */
1508 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1509                                         struct dmar_domain *domain,
1510                                         unsigned long pfn, unsigned int pages)
1511 {
1512         /* It's a non-present to present mapping. Only flush if caching mode */
1513         if (cap_caching_mode(iommu->cap))
1514                 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1515         else
1516                 iommu_flush_write_buffer(iommu);
1517 }
1518
1519 static void iommu_flush_iova(struct iova_domain *iovad)
1520 {
1521         struct dmar_domain *domain;
1522         int idx;
1523
1524         domain = container_of(iovad, struct dmar_domain, iovad);
1525
1526         for_each_domain_iommu(idx, domain) {
1527                 struct intel_iommu *iommu = g_iommus[idx];
1528                 u16 did = domain->iommu_did[iommu->seq_id];
1529
1530                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1531
1532                 if (!cap_caching_mode(iommu->cap))
1533                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1534                                               0, MAX_AGAW_PFN_WIDTH);
1535         }
1536 }
1537
1538 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1539 {
1540         u32 pmen;
1541         unsigned long flags;
1542
1543         if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1544                 return;
1545
1546         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1547         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1548         pmen &= ~DMA_PMEN_EPM;
1549         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1550
1551         /* wait for the protected region status bit to clear */
1552         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1553                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1554
1555         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1556 }
1557
1558 static void iommu_enable_translation(struct intel_iommu *iommu)
1559 {
1560         u32 sts;
1561         unsigned long flags;
1562
1563         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1564         iommu->gcmd |= DMA_GCMD_TE;
1565         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1566
1567         /* Make sure hardware complete it */
1568         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1569                       readl, (sts & DMA_GSTS_TES), sts);
1570
1571         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1572 }
1573
1574 static void iommu_disable_translation(struct intel_iommu *iommu)
1575 {
1576         u32 sts;
1577         unsigned long flag;
1578
1579         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1580         iommu->gcmd &= ~DMA_GCMD_TE;
1581         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1582
1583         /* Make sure hardware complete it */
1584         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1585                       readl, (!(sts & DMA_GSTS_TES)), sts);
1586
1587         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1588 }
1589
1590 static int iommu_init_domains(struct intel_iommu *iommu)
1591 {
1592         u32 ndomains, nlongs;
1593         size_t size;
1594
1595         ndomains = cap_ndoms(iommu->cap);
1596         pr_debug("%s: Number of Domains supported <%d>\n",
1597                  iommu->name, ndomains);
1598         nlongs = BITS_TO_LONGS(ndomains);
1599
1600         spin_lock_init(&iommu->lock);
1601
1602         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1603         if (!iommu->domain_ids) {
1604                 pr_err("%s: Allocating domain id array failed\n",
1605                        iommu->name);
1606                 return -ENOMEM;
1607         }
1608
1609         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1610         iommu->domains = kzalloc(size, GFP_KERNEL);
1611
1612         if (iommu->domains) {
1613                 size = 256 * sizeof(struct dmar_domain *);
1614                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1615         }
1616
1617         if (!iommu->domains || !iommu->domains[0]) {
1618                 pr_err("%s: Allocating domain array failed\n",
1619                        iommu->name);
1620                 kfree(iommu->domain_ids);
1621                 kfree(iommu->domains);
1622                 iommu->domain_ids = NULL;
1623                 iommu->domains    = NULL;
1624                 return -ENOMEM;
1625         }
1626
1627         /*
1628          * If Caching mode is set, then invalid translations are tagged
1629          * with domain-id 0, hence we need to pre-allocate it. We also
1630          * use domain-id 0 as a marker for non-allocated domain-id, so
1631          * make sure it is not used for a real domain.
1632          */
1633         set_bit(0, iommu->domain_ids);
1634
1635         /*
1636          * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1637          * entry for first-level or pass-through translation modes should
1638          * be programmed with a domain id different from those used for
1639          * second-level or nested translation. We reserve a domain id for
1640          * this purpose.
1641          */
1642         if (sm_supported(iommu))
1643                 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1644
1645         return 0;
1646 }
1647
1648 static void disable_dmar_iommu(struct intel_iommu *iommu)
1649 {
1650         struct device_domain_info *info, *tmp;
1651         unsigned long flags;
1652
1653         if (!iommu->domains || !iommu->domain_ids)
1654                 return;
1655
1656         spin_lock_irqsave(&device_domain_lock, flags);
1657         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1658                 if (info->iommu != iommu)
1659                         continue;
1660
1661                 if (!info->dev || !info->domain)
1662                         continue;
1663
1664                 __dmar_remove_one_dev_info(info);
1665         }
1666         spin_unlock_irqrestore(&device_domain_lock, flags);
1667
1668         if (iommu->gcmd & DMA_GCMD_TE)
1669                 iommu_disable_translation(iommu);
1670 }
1671
1672 static void free_dmar_iommu(struct intel_iommu *iommu)
1673 {
1674         if ((iommu->domains) && (iommu->domain_ids)) {
1675                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1676                 int i;
1677
1678                 for (i = 0; i < elems; i++)
1679                         kfree(iommu->domains[i]);
1680                 kfree(iommu->domains);
1681                 kfree(iommu->domain_ids);
1682                 iommu->domains = NULL;
1683                 iommu->domain_ids = NULL;
1684         }
1685
1686         g_iommus[iommu->seq_id] = NULL;
1687
1688         /* free context mapping */
1689         free_context_table(iommu);
1690
1691 #ifdef CONFIG_INTEL_IOMMU_SVM
1692         if (pasid_supported(iommu)) {
1693                 if (ecap_prs(iommu->ecap))
1694                         intel_svm_finish_prq(iommu);
1695         }
1696 #endif
1697 }
1698
1699 static struct dmar_domain *alloc_domain(int flags)
1700 {
1701         struct dmar_domain *domain;
1702
1703         domain = alloc_domain_mem();
1704         if (!domain)
1705                 return NULL;
1706
1707         memset(domain, 0, sizeof(*domain));
1708         domain->nid = NUMA_NO_NODE;
1709         domain->flags = flags;
1710         domain->has_iotlb_device = false;
1711         INIT_LIST_HEAD(&domain->devices);
1712
1713         return domain;
1714 }
1715
1716 /* Must be called with iommu->lock */
1717 static int domain_attach_iommu(struct dmar_domain *domain,
1718                                struct intel_iommu *iommu)
1719 {
1720         unsigned long ndomains;
1721         int num;
1722
1723         assert_spin_locked(&device_domain_lock);
1724         assert_spin_locked(&iommu->lock);
1725
1726         domain->iommu_refcnt[iommu->seq_id] += 1;
1727         domain->iommu_count += 1;
1728         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1729                 ndomains = cap_ndoms(iommu->cap);
1730                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1731
1732                 if (num >= ndomains) {
1733                         pr_err("%s: No free domain ids\n", iommu->name);
1734                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1735                         domain->iommu_count -= 1;
1736                         return -ENOSPC;
1737                 }
1738
1739                 set_bit(num, iommu->domain_ids);
1740                 set_iommu_domain(iommu, num, domain);
1741
1742                 domain->iommu_did[iommu->seq_id] = num;
1743                 domain->nid                      = iommu->node;
1744
1745                 domain_update_iommu_cap(domain);
1746         }
1747
1748         return 0;
1749 }
1750
1751 static int domain_detach_iommu(struct dmar_domain *domain,
1752                                struct intel_iommu *iommu)
1753 {
1754         int num, count;
1755
1756         assert_spin_locked(&device_domain_lock);
1757         assert_spin_locked(&iommu->lock);
1758
1759         domain->iommu_refcnt[iommu->seq_id] -= 1;
1760         count = --domain->iommu_count;
1761         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1762                 num = domain->iommu_did[iommu->seq_id];
1763                 clear_bit(num, iommu->domain_ids);
1764                 set_iommu_domain(iommu, num, NULL);
1765
1766                 domain_update_iommu_cap(domain);
1767                 domain->iommu_did[iommu->seq_id] = 0;
1768         }
1769
1770         return count;
1771 }
1772
1773 static struct iova_domain reserved_iova_list;
1774 static struct lock_class_key reserved_rbtree_key;
1775
1776 static int dmar_init_reserved_ranges(void)
1777 {
1778         struct pci_dev *pdev = NULL;
1779         struct iova *iova;
1780         int i;
1781
1782         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1783
1784         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1785                 &reserved_rbtree_key);
1786
1787         /* IOAPIC ranges shouldn't be accessed by DMA */
1788         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1789                 IOVA_PFN(IOAPIC_RANGE_END));
1790         if (!iova) {
1791                 pr_err("Reserve IOAPIC range failed\n");
1792                 return -ENODEV;
1793         }
1794
1795         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1796         for_each_pci_dev(pdev) {
1797                 struct resource *r;
1798
1799                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1800                         r = &pdev->resource[i];
1801                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1802                                 continue;
1803                         iova = reserve_iova(&reserved_iova_list,
1804                                             IOVA_PFN(r->start),
1805                                             IOVA_PFN(r->end));
1806                         if (!iova) {
1807                                 pci_err(pdev, "Reserve iova for %pR failed\n", r);
1808                                 return -ENODEV;
1809                         }
1810                 }
1811         }
1812         return 0;
1813 }
1814
1815 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1816 {
1817         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1818 }
1819
1820 static inline int guestwidth_to_adjustwidth(int gaw)
1821 {
1822         int agaw;
1823         int r = (gaw - 12) % 9;
1824
1825         if (r == 0)
1826                 agaw = gaw;
1827         else
1828                 agaw = gaw + 9 - r;
1829         if (agaw > 64)
1830                 agaw = 64;
1831         return agaw;
1832 }
1833
1834 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1835                        int guest_width)
1836 {
1837         int adjust_width, agaw;
1838         unsigned long sagaw;
1839         int err;
1840
1841         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1842
1843         err = init_iova_flush_queue(&domain->iovad,
1844                                     iommu_flush_iova, iova_entry_free);
1845         if (err)
1846                 return err;
1847
1848         domain_reserve_special_ranges(domain);
1849
1850         /* calculate AGAW */
1851         if (guest_width > cap_mgaw(iommu->cap))
1852                 guest_width = cap_mgaw(iommu->cap);
1853         domain->gaw = guest_width;
1854         adjust_width = guestwidth_to_adjustwidth(guest_width);
1855         agaw = width_to_agaw(adjust_width);
1856         sagaw = cap_sagaw(iommu->cap);
1857         if (!test_bit(agaw, &sagaw)) {
1858                 /* hardware doesn't support it, choose a bigger one */
1859                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1860                 agaw = find_next_bit(&sagaw, 5, agaw);
1861                 if (agaw >= 5)
1862                         return -ENODEV;
1863         }
1864         domain->agaw = agaw;
1865
1866         if (ecap_coherent(iommu->ecap))
1867                 domain->iommu_coherency = 1;
1868         else
1869                 domain->iommu_coherency = 0;
1870
1871         if (ecap_sc_support(iommu->ecap))
1872                 domain->iommu_snooping = 1;
1873         else
1874                 domain->iommu_snooping = 0;
1875
1876         if (intel_iommu_superpage)
1877                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1878         else
1879                 domain->iommu_superpage = 0;
1880
1881         domain->nid = iommu->node;
1882
1883         /* always allocate the top pgd */
1884         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1885         if (!domain->pgd)
1886                 return -ENOMEM;
1887         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1888         return 0;
1889 }
1890
1891 static void domain_exit(struct dmar_domain *domain)
1892 {
1893
1894         /* Remove associated devices and clear attached or cached domains */
1895         domain_remove_dev_info(domain);
1896
1897         /* destroy iovas */
1898         put_iova_domain(&domain->iovad);
1899
1900         if (domain->pgd) {
1901                 struct page *freelist;
1902
1903                 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1904                 dma_free_pagelist(freelist);
1905         }
1906
1907         free_domain_mem(domain);
1908 }
1909
1910 /*
1911  * Get the PASID directory size for scalable mode context entry.
1912  * Value of X in the PDTS field of a scalable mode context entry
1913  * indicates PASID directory with 2^(X + 7) entries.
1914  */
1915 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1916 {
1917         int pds, max_pde;
1918
1919         max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1920         pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1921         if (pds < 7)
1922                 return 0;
1923
1924         return pds - 7;
1925 }
1926
1927 /*
1928  * Set the RID_PASID field of a scalable mode context entry. The
1929  * IOMMU hardware will use the PASID value set in this field for
1930  * DMA translations of DMA requests without PASID.
1931  */
1932 static inline void
1933 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1934 {
1935         context->hi |= pasid & ((1 << 20) - 1);
1936         context->hi |= (1 << 20);
1937 }
1938
1939 /*
1940  * Set the DTE(Device-TLB Enable) field of a scalable mode context
1941  * entry.
1942  */
1943 static inline void context_set_sm_dte(struct context_entry *context)
1944 {
1945         context->lo |= (1 << 2);
1946 }
1947
1948 /*
1949  * Set the PRE(Page Request Enable) field of a scalable mode context
1950  * entry.
1951  */
1952 static inline void context_set_sm_pre(struct context_entry *context)
1953 {
1954         context->lo |= (1 << 4);
1955 }
1956
1957 /* Convert value to context PASID directory size field coding. */
1958 #define context_pdts(pds)       (((pds) & 0x7) << 9)
1959
1960 static int domain_context_mapping_one(struct dmar_domain *domain,
1961                                       struct intel_iommu *iommu,
1962                                       struct pasid_table *table,
1963                                       u8 bus, u8 devfn)
1964 {
1965         u16 did = domain->iommu_did[iommu->seq_id];
1966         int translation = CONTEXT_TT_MULTI_LEVEL;
1967         struct device_domain_info *info = NULL;
1968         struct context_entry *context;
1969         unsigned long flags;
1970         int ret;
1971
1972         WARN_ON(did == 0);
1973
1974         if (hw_pass_through && domain_type_is_si(domain))
1975                 translation = CONTEXT_TT_PASS_THROUGH;
1976
1977         pr_debug("Set context mapping for %02x:%02x.%d\n",
1978                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1979
1980         BUG_ON(!domain->pgd);
1981
1982         spin_lock_irqsave(&device_domain_lock, flags);
1983         spin_lock(&iommu->lock);
1984
1985         ret = -ENOMEM;
1986         context = iommu_context_addr(iommu, bus, devfn, 1);
1987         if (!context)
1988                 goto out_unlock;
1989
1990         ret = 0;
1991         if (context_present(context))
1992                 goto out_unlock;
1993
1994         /*
1995          * For kdump cases, old valid entries may be cached due to the
1996          * in-flight DMA and copied pgtable, but there is no unmapping
1997          * behaviour for them, thus we need an explicit cache flush for
1998          * the newly-mapped device. For kdump, at this point, the device
1999          * is supposed to finish reset at its driver probe stage, so no
2000          * in-flight DMA will exist, and we don't need to worry anymore
2001          * hereafter.
2002          */
2003         if (context_copied(context)) {
2004                 u16 did_old = context_domain_id(context);
2005
2006                 if (did_old < cap_ndoms(iommu->cap)) {
2007                         iommu->flush.flush_context(iommu, did_old,
2008                                                    (((u16)bus) << 8) | devfn,
2009                                                    DMA_CCMD_MASK_NOBIT,
2010                                                    DMA_CCMD_DEVICE_INVL);
2011                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2012                                                  DMA_TLB_DSI_FLUSH);
2013                 }
2014         }
2015
2016         context_clear_entry(context);
2017
2018         if (sm_supported(iommu)) {
2019                 unsigned long pds;
2020
2021                 WARN_ON(!table);
2022
2023                 /* Setup the PASID DIR pointer: */
2024                 pds = context_get_sm_pds(table);
2025                 context->lo = (u64)virt_to_phys(table->table) |
2026                                 context_pdts(pds);
2027
2028                 /* Setup the RID_PASID field: */
2029                 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2030
2031                 /*
2032                  * Setup the Device-TLB enable bit and Page request
2033                  * Enable bit:
2034                  */
2035                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2036                 if (info && info->ats_supported)
2037                         context_set_sm_dte(context);
2038                 if (info && info->pri_supported)
2039                         context_set_sm_pre(context);
2040         } else {
2041                 struct dma_pte *pgd = domain->pgd;
2042                 int agaw;
2043
2044                 context_set_domain_id(context, did);
2045
2046                 if (translation != CONTEXT_TT_PASS_THROUGH) {
2047                         /*
2048                          * Skip top levels of page tables for iommu which has
2049                          * less agaw than default. Unnecessary for PT mode.
2050                          */
2051                         for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2052                                 ret = -ENOMEM;
2053                                 pgd = phys_to_virt(dma_pte_addr(pgd));
2054                                 if (!dma_pte_present(pgd))
2055                                         goto out_unlock;
2056                         }
2057
2058                         info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2059                         if (info && info->ats_supported)
2060                                 translation = CONTEXT_TT_DEV_IOTLB;
2061                         else
2062                                 translation = CONTEXT_TT_MULTI_LEVEL;
2063
2064                         context_set_address_root(context, virt_to_phys(pgd));
2065                         context_set_address_width(context, agaw);
2066                 } else {
2067                         /*
2068                          * In pass through mode, AW must be programmed to
2069                          * indicate the largest AGAW value supported by
2070                          * hardware. And ASR is ignored by hardware.
2071                          */
2072                         context_set_address_width(context, iommu->msagaw);
2073                 }
2074
2075                 context_set_translation_type(context, translation);
2076         }
2077
2078         context_set_fault_enable(context);
2079         context_set_present(context);
2080         domain_flush_cache(domain, context, sizeof(*context));
2081
2082         /*
2083          * It's a non-present to present mapping. If hardware doesn't cache
2084          * non-present entry we only need to flush the write-buffer. If the
2085          * _does_ cache non-present entries, then it does so in the special
2086          * domain #0, which we have to flush:
2087          */
2088         if (cap_caching_mode(iommu->cap)) {
2089                 iommu->flush.flush_context(iommu, 0,
2090                                            (((u16)bus) << 8) | devfn,
2091                                            DMA_CCMD_MASK_NOBIT,
2092                                            DMA_CCMD_DEVICE_INVL);
2093                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2094         } else {
2095                 iommu_flush_write_buffer(iommu);
2096         }
2097         iommu_enable_dev_iotlb(info);
2098
2099         ret = 0;
2100
2101 out_unlock:
2102         spin_unlock(&iommu->lock);
2103         spin_unlock_irqrestore(&device_domain_lock, flags);
2104
2105         return ret;
2106 }
2107
2108 static int
2109 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2110 {
2111         struct pasid_table *table;
2112         struct intel_iommu *iommu;
2113         u8 bus, devfn;
2114
2115         iommu = device_to_iommu(dev, &bus, &devfn);
2116         if (!iommu)
2117                 return -ENODEV;
2118
2119         table = intel_pasid_get_table(dev);
2120         return domain_context_mapping_one(domain, iommu, table, bus, devfn);
2121 }
2122
2123 static int domain_context_mapped_cb(struct pci_dev *pdev,
2124                                     u16 alias, void *opaque)
2125 {
2126         struct intel_iommu *iommu = opaque;
2127
2128         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2129 }
2130
2131 static int domain_context_mapped(struct device *dev)
2132 {
2133         struct intel_iommu *iommu;
2134         u8 bus, devfn;
2135
2136         iommu = device_to_iommu(dev, &bus, &devfn);
2137         if (!iommu)
2138                 return -ENODEV;
2139
2140         if (!dev_is_pci(dev))
2141                 return device_context_mapped(iommu, bus, devfn);
2142
2143         return !pci_for_each_dma_alias(to_pci_dev(dev),
2144                                        domain_context_mapped_cb, iommu);
2145 }
2146
2147 /* Returns a number of VTD pages, but aligned to MM page size */
2148 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2149                                             size_t size)
2150 {
2151         host_addr &= ~PAGE_MASK;
2152         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2153 }
2154
2155 /* Return largest possible superpage level for a given mapping */
2156 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2157                                           unsigned long iov_pfn,
2158                                           unsigned long phy_pfn,
2159                                           unsigned long pages)
2160 {
2161         int support, level = 1;
2162         unsigned long pfnmerge;
2163
2164         support = domain->iommu_superpage;
2165
2166         /* To use a large page, the virtual *and* physical addresses
2167            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2168            of them will mean we have to use smaller pages. So just
2169            merge them and check both at once. */
2170         pfnmerge = iov_pfn | phy_pfn;
2171
2172         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2173                 pages >>= VTD_STRIDE_SHIFT;
2174                 if (!pages)
2175                         break;
2176                 pfnmerge >>= VTD_STRIDE_SHIFT;
2177                 level++;
2178                 support--;
2179         }
2180         return level;
2181 }
2182
2183 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2184                             struct scatterlist *sg, unsigned long phys_pfn,
2185                             unsigned long nr_pages, int prot)
2186 {
2187         struct dma_pte *first_pte = NULL, *pte = NULL;
2188         phys_addr_t uninitialized_var(pteval);
2189         unsigned long sg_res = 0;
2190         unsigned int largepage_lvl = 0;
2191         unsigned long lvl_pages = 0;
2192
2193         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2194
2195         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2196                 return -EINVAL;
2197
2198         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2199
2200         if (!sg) {
2201                 sg_res = nr_pages;
2202                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2203         }
2204
2205         while (nr_pages > 0) {
2206                 uint64_t tmp;
2207
2208                 if (!sg_res) {
2209                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2210
2211                         sg_res = aligned_nrpages(sg->offset, sg->length);
2212                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2213                         sg->dma_length = sg->length;
2214                         pteval = (sg_phys(sg) - pgoff) | prot;
2215                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2216                 }
2217
2218                 if (!pte) {
2219                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2220
2221                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2222                         if (!pte)
2223                                 return -ENOMEM;
2224                         /* It is large page*/
2225                         if (largepage_lvl > 1) {
2226                                 unsigned long nr_superpages, end_pfn;
2227
2228                                 pteval |= DMA_PTE_LARGE_PAGE;
2229                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2230
2231                                 nr_superpages = sg_res / lvl_pages;
2232                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2233
2234                                 /*
2235                                  * Ensure that old small page tables are
2236                                  * removed to make room for superpage(s).
2237                                  * We're adding new large pages, so make sure
2238                                  * we don't remove their parent tables.
2239                                  */
2240                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2241                                                        largepage_lvl + 1);
2242                         } else {
2243                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2244                         }
2245
2246                 }
2247                 /* We don't need lock here, nobody else
2248                  * touches the iova range
2249                  */
2250                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2251                 if (tmp) {
2252                         static int dumps = 5;
2253                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2254                                 iov_pfn, tmp, (unsigned long long)pteval);
2255                         if (dumps) {
2256                                 dumps--;
2257                                 debug_dma_dump_mappings(NULL);
2258                         }
2259                         WARN_ON(1);
2260                 }
2261
2262                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2263
2264                 BUG_ON(nr_pages < lvl_pages);
2265                 BUG_ON(sg_res < lvl_pages);
2266
2267                 nr_pages -= lvl_pages;
2268                 iov_pfn += lvl_pages;
2269                 phys_pfn += lvl_pages;
2270                 pteval += lvl_pages * VTD_PAGE_SIZE;
2271                 sg_res -= lvl_pages;
2272
2273                 /* If the next PTE would be the first in a new page, then we
2274                    need to flush the cache on the entries we've just written.
2275                    And then we'll need to recalculate 'pte', so clear it and
2276                    let it get set again in the if (!pte) block above.
2277
2278                    If we're done (!nr_pages) we need to flush the cache too.
2279
2280                    Also if we've been setting superpages, we may need to
2281                    recalculate 'pte' and switch back to smaller pages for the
2282                    end of the mapping, if the trailing size is not enough to
2283                    use another superpage (i.e. sg_res < lvl_pages). */
2284                 pte++;
2285                 if (!nr_pages || first_pte_in_page(pte) ||
2286                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2287                         domain_flush_cache(domain, first_pte,
2288                                            (void *)pte - (void *)first_pte);
2289                         pte = NULL;
2290                 }
2291
2292                 if (!sg_res && nr_pages)
2293                         sg = sg_next(sg);
2294         }
2295         return 0;
2296 }
2297
2298 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2299                           struct scatterlist *sg, unsigned long phys_pfn,
2300                           unsigned long nr_pages, int prot)
2301 {
2302         int iommu_id, ret;
2303         struct intel_iommu *iommu;
2304
2305         /* Do the real mapping first */
2306         ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2307         if (ret)
2308                 return ret;
2309
2310         for_each_domain_iommu(iommu_id, domain) {
2311                 iommu = g_iommus[iommu_id];
2312                 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2313         }
2314
2315         return 0;
2316 }
2317
2318 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2319                                     struct scatterlist *sg, unsigned long nr_pages,
2320                                     int prot)
2321 {
2322         return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2323 }
2324
2325 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2326                                      unsigned long phys_pfn, unsigned long nr_pages,
2327                                      int prot)
2328 {
2329         return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2330 }
2331
2332 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2333 {
2334         unsigned long flags;
2335         struct context_entry *context;
2336         u16 did_old;
2337
2338         if (!iommu)
2339                 return;
2340
2341         spin_lock_irqsave(&iommu->lock, flags);
2342         context = iommu_context_addr(iommu, bus, devfn, 0);
2343         if (!context) {
2344                 spin_unlock_irqrestore(&iommu->lock, flags);
2345                 return;
2346         }
2347         did_old = context_domain_id(context);
2348         context_clear_entry(context);
2349         __iommu_flush_cache(iommu, context, sizeof(*context));
2350         spin_unlock_irqrestore(&iommu->lock, flags);
2351         iommu->flush.flush_context(iommu,
2352                                    did_old,
2353                                    (((u16)bus) << 8) | devfn,
2354                                    DMA_CCMD_MASK_NOBIT,
2355                                    DMA_CCMD_DEVICE_INVL);
2356         iommu->flush.flush_iotlb(iommu,
2357                                  did_old,
2358                                  0,
2359                                  0,
2360                                  DMA_TLB_DSI_FLUSH);
2361 }
2362
2363 static inline void unlink_domain_info(struct device_domain_info *info)
2364 {
2365         assert_spin_locked(&device_domain_lock);
2366         list_del(&info->link);
2367         list_del(&info->global);
2368         if (info->dev)
2369                 info->dev->archdata.iommu = NULL;
2370 }
2371
2372 static void domain_remove_dev_info(struct dmar_domain *domain)
2373 {
2374         struct device_domain_info *info, *tmp;
2375         unsigned long flags;
2376
2377         spin_lock_irqsave(&device_domain_lock, flags);
2378         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2379                 __dmar_remove_one_dev_info(info);
2380         spin_unlock_irqrestore(&device_domain_lock, flags);
2381 }
2382
2383 /*
2384  * find_domain
2385  * Note: we use struct device->archdata.iommu stores the info
2386  */
2387 static struct dmar_domain *find_domain(struct device *dev)
2388 {
2389         struct device_domain_info *info;
2390
2391         if (unlikely(dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO)) {
2392                 struct iommu_domain *domain;
2393
2394                 dev->archdata.iommu = NULL;
2395                 domain = iommu_get_domain_for_dev(dev);
2396                 if (domain)
2397                         intel_iommu_attach_device(domain, dev);
2398         }
2399
2400         /* No lock here, assumes no domain exit in normal case */
2401         info = dev->archdata.iommu;
2402
2403         if (likely(info))
2404                 return info->domain;
2405         return NULL;
2406 }
2407
2408 static inline struct device_domain_info *
2409 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2410 {
2411         struct device_domain_info *info;
2412
2413         list_for_each_entry(info, &device_domain_list, global)
2414                 if (info->iommu->segment == segment && info->bus == bus &&
2415                     info->devfn == devfn)
2416                         return info;
2417
2418         return NULL;
2419 }
2420
2421 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2422                                                     int bus, int devfn,
2423                                                     struct device *dev,
2424                                                     struct dmar_domain *domain)
2425 {
2426         struct dmar_domain *found = NULL;
2427         struct device_domain_info *info;
2428         unsigned long flags;
2429         int ret;
2430
2431         info = alloc_devinfo_mem();
2432         if (!info)
2433                 return NULL;
2434
2435         info->bus = bus;
2436         info->devfn = devfn;
2437         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2438         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2439         info->ats_qdep = 0;
2440         info->dev = dev;
2441         info->domain = domain;
2442         info->iommu = iommu;
2443         info->pasid_table = NULL;
2444         info->auxd_enabled = 0;
2445         INIT_LIST_HEAD(&info->auxiliary_domains);
2446
2447         if (dev && dev_is_pci(dev)) {
2448                 struct pci_dev *pdev = to_pci_dev(info->dev);
2449
2450                 if (!pdev->untrusted &&
2451                     !pci_ats_disabled() &&
2452                     ecap_dev_iotlb_support(iommu->ecap) &&
2453                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2454                     dmar_find_matched_atsr_unit(pdev))
2455                         info->ats_supported = 1;
2456
2457                 if (sm_supported(iommu)) {
2458                         if (pasid_supported(iommu)) {
2459                                 int features = pci_pasid_features(pdev);
2460                                 if (features >= 0)
2461                                         info->pasid_supported = features | 1;
2462                         }
2463
2464                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2465                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2466                                 info->pri_supported = 1;
2467                 }
2468         }
2469
2470         spin_lock_irqsave(&device_domain_lock, flags);
2471         if (dev)
2472                 found = find_domain(dev);
2473
2474         if (!found) {
2475                 struct device_domain_info *info2;
2476                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2477                 if (info2) {
2478                         found      = info2->domain;
2479                         info2->dev = dev;
2480                 }
2481         }
2482
2483         if (found) {
2484                 spin_unlock_irqrestore(&device_domain_lock, flags);
2485                 free_devinfo_mem(info);
2486                 /* Caller must free the original domain */
2487                 return found;
2488         }
2489
2490         spin_lock(&iommu->lock);
2491         ret = domain_attach_iommu(domain, iommu);
2492         spin_unlock(&iommu->lock);
2493
2494         if (ret) {
2495                 spin_unlock_irqrestore(&device_domain_lock, flags);
2496                 free_devinfo_mem(info);
2497                 return NULL;
2498         }
2499
2500         list_add(&info->link, &domain->devices);
2501         list_add(&info->global, &device_domain_list);
2502         if (dev)
2503                 dev->archdata.iommu = info;
2504         spin_unlock_irqrestore(&device_domain_lock, flags);
2505
2506         /* PASID table is mandatory for a PCI device in scalable mode. */
2507         if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2508                 ret = intel_pasid_alloc_table(dev);
2509                 if (ret) {
2510                         dev_err(dev, "PASID table allocation failed\n");
2511                         dmar_remove_one_dev_info(dev);
2512                         return NULL;
2513                 }
2514
2515                 /* Setup the PASID entry for requests without PASID: */
2516                 spin_lock(&iommu->lock);
2517                 if (hw_pass_through && domain_type_is_si(domain))
2518                         ret = intel_pasid_setup_pass_through(iommu, domain,
2519                                         dev, PASID_RID2PASID);
2520                 else
2521                         ret = intel_pasid_setup_second_level(iommu, domain,
2522                                         dev, PASID_RID2PASID);
2523                 spin_unlock(&iommu->lock);
2524                 if (ret) {
2525                         dev_err(dev, "Setup RID2PASID failed\n");
2526                         dmar_remove_one_dev_info(dev);
2527                         return NULL;
2528                 }
2529         }
2530
2531         if (dev && domain_context_mapping(domain, dev)) {
2532                 dev_err(dev, "Domain context map failed\n");
2533                 dmar_remove_one_dev_info(dev);
2534                 return NULL;
2535         }
2536
2537         return domain;
2538 }
2539
2540 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2541 {
2542         *(u16 *)opaque = alias;
2543         return 0;
2544 }
2545
2546 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2547 {
2548         struct device_domain_info *info;
2549         struct dmar_domain *domain = NULL;
2550         struct intel_iommu *iommu;
2551         u16 dma_alias;
2552         unsigned long flags;
2553         u8 bus, devfn;
2554
2555         iommu = device_to_iommu(dev, &bus, &devfn);
2556         if (!iommu)
2557                 return NULL;
2558
2559         if (dev_is_pci(dev)) {
2560                 struct pci_dev *pdev = to_pci_dev(dev);
2561
2562                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2563
2564                 spin_lock_irqsave(&device_domain_lock, flags);
2565                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2566                                                       PCI_BUS_NUM(dma_alias),
2567                                                       dma_alias & 0xff);
2568                 if (info) {
2569                         iommu = info->iommu;
2570                         domain = info->domain;
2571                 }
2572                 spin_unlock_irqrestore(&device_domain_lock, flags);
2573
2574                 /* DMA alias already has a domain, use it */
2575                 if (info)
2576                         goto out;
2577         }
2578
2579         /* Allocate and initialize new domain for the device */
2580         domain = alloc_domain(0);
2581         if (!domain)
2582                 return NULL;
2583         if (domain_init(domain, iommu, gaw)) {
2584                 domain_exit(domain);
2585                 return NULL;
2586         }
2587
2588 out:
2589         return domain;
2590 }
2591
2592 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2593                                               struct dmar_domain *domain)
2594 {
2595         struct intel_iommu *iommu;
2596         struct dmar_domain *tmp;
2597         u16 req_id, dma_alias;
2598         u8 bus, devfn;
2599
2600         iommu = device_to_iommu(dev, &bus, &devfn);
2601         if (!iommu)
2602                 return NULL;
2603
2604         req_id = ((u16)bus << 8) | devfn;
2605
2606         if (dev_is_pci(dev)) {
2607                 struct pci_dev *pdev = to_pci_dev(dev);
2608
2609                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2610
2611                 /* register PCI DMA alias device */
2612                 if (req_id != dma_alias) {
2613                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2614                                         dma_alias & 0xff, NULL, domain);
2615
2616                         if (!tmp || tmp != domain)
2617                                 return tmp;
2618                 }
2619         }
2620
2621         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2622         if (!tmp || tmp != domain)
2623                 return tmp;
2624
2625         return domain;
2626 }
2627
2628 static int iommu_domain_identity_map(struct dmar_domain *domain,
2629                                      unsigned long long start,
2630                                      unsigned long long end)
2631 {
2632         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2633         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2634
2635         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2636                           dma_to_mm_pfn(last_vpfn))) {
2637                 pr_err("Reserving iova failed\n");
2638                 return -ENOMEM;
2639         }
2640
2641         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2642         /*
2643          * RMRR range might have overlap with physical memory range,
2644          * clear it first
2645          */
2646         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2647
2648         return __domain_mapping(domain, first_vpfn, NULL,
2649                                 first_vpfn, last_vpfn - first_vpfn + 1,
2650                                 DMA_PTE_READ|DMA_PTE_WRITE);
2651 }
2652
2653 static int domain_prepare_identity_map(struct device *dev,
2654                                        struct dmar_domain *domain,
2655                                        unsigned long long start,
2656                                        unsigned long long end)
2657 {
2658         /* For _hardware_ passthrough, don't bother. But for software
2659            passthrough, we do it anyway -- it may indicate a memory
2660            range which is reserved in E820, so which didn't get set
2661            up to start with in si_domain */
2662         if (domain == si_domain && hw_pass_through) {
2663                 dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
2664                          start, end);
2665                 return 0;
2666         }
2667
2668         dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
2669
2670         if (end < start) {
2671                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2672                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2673                         dmi_get_system_info(DMI_BIOS_VENDOR),
2674                         dmi_get_system_info(DMI_BIOS_VERSION),
2675                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2676                 return -EIO;
2677         }
2678
2679         if (end >> agaw_to_width(domain->agaw)) {
2680                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2681                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2682                      agaw_to_width(domain->agaw),
2683                      dmi_get_system_info(DMI_BIOS_VENDOR),
2684                      dmi_get_system_info(DMI_BIOS_VERSION),
2685                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2686                 return -EIO;
2687         }
2688
2689         return iommu_domain_identity_map(domain, start, end);
2690 }
2691
2692 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2693
2694 static int __init si_domain_init(int hw)
2695 {
2696         struct dmar_rmrr_unit *rmrr;
2697         struct device *dev;
2698         int i, nid, ret;
2699
2700         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2701         if (!si_domain)
2702                 return -EFAULT;
2703
2704         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2705                 domain_exit(si_domain);
2706                 return -EFAULT;
2707         }
2708
2709         if (hw)
2710                 return 0;
2711
2712         for_each_online_node(nid) {
2713                 unsigned long start_pfn, end_pfn;
2714                 int i;
2715
2716                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2717                         ret = iommu_domain_identity_map(si_domain,
2718                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2719                         if (ret)
2720                                 return ret;
2721                 }
2722         }
2723
2724         /*
2725          * Normally we use DMA domains for devices which have RMRRs. But we
2726          * loose this requirement for graphic and usb devices. Identity map
2727          * the RMRRs for graphic and USB devices so that they could use the
2728          * si_domain.
2729          */
2730         for_each_rmrr_units(rmrr) {
2731                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2732                                           i, dev) {
2733                         unsigned long long start = rmrr->base_address;
2734                         unsigned long long end = rmrr->end_address;
2735
2736                         if (device_is_rmrr_locked(dev))
2737                                 continue;
2738
2739                         if (WARN_ON(end < start ||
2740                                     end >> agaw_to_width(si_domain->agaw)))
2741                                 continue;
2742
2743                         ret = iommu_domain_identity_map(si_domain, start, end);
2744                         if (ret)
2745                                 return ret;
2746                 }
2747         }
2748
2749         return 0;
2750 }
2751
2752 static int identity_mapping(struct device *dev)
2753 {
2754         struct device_domain_info *info;
2755
2756         info = dev->archdata.iommu;
2757         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2758                 return (info->domain == si_domain);
2759
2760         return 0;
2761 }
2762
2763 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2764 {
2765         struct dmar_domain *ndomain;
2766         struct intel_iommu *iommu;
2767         u8 bus, devfn;
2768
2769         iommu = device_to_iommu(dev, &bus, &devfn);
2770         if (!iommu)
2771                 return -ENODEV;
2772
2773         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2774         if (ndomain != domain)
2775                 return -EBUSY;
2776
2777         return 0;
2778 }
2779
2780 static bool device_has_rmrr(struct device *dev)
2781 {
2782         struct dmar_rmrr_unit *rmrr;
2783         struct device *tmp;
2784         int i;
2785
2786         rcu_read_lock();
2787         for_each_rmrr_units(rmrr) {
2788                 /*
2789                  * Return TRUE if this RMRR contains the device that
2790                  * is passed in.
2791                  */
2792                 for_each_active_dev_scope(rmrr->devices,
2793                                           rmrr->devices_cnt, i, tmp)
2794                         if (tmp == dev ||
2795                             is_downstream_to_pci_bridge(dev, tmp)) {
2796                                 rcu_read_unlock();
2797                                 return true;
2798                         }
2799         }
2800         rcu_read_unlock();
2801         return false;
2802 }
2803
2804 /**
2805  * device_rmrr_is_relaxable - Test whether the RMRR of this device
2806  * is relaxable (ie. is allowed to be not enforced under some conditions)
2807  * @dev: device handle
2808  *
2809  * We assume that PCI USB devices with RMRRs have them largely
2810  * for historical reasons and that the RMRR space is not actively used post
2811  * boot.  This exclusion may change if vendors begin to abuse it.
2812  *
2813  * The same exception is made for graphics devices, with the requirement that
2814  * any use of the RMRR regions will be torn down before assigning the device
2815  * to a guest.
2816  *
2817  * Return: true if the RMRR is relaxable, false otherwise
2818  */
2819 static bool device_rmrr_is_relaxable(struct device *dev)
2820 {
2821         struct pci_dev *pdev;
2822
2823         if (!dev_is_pci(dev))
2824                 return false;
2825
2826         pdev = to_pci_dev(dev);
2827         if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2828                 return true;
2829         else
2830                 return false;
2831 }
2832
2833 /*
2834  * There are a couple cases where we need to restrict the functionality of
2835  * devices associated with RMRRs.  The first is when evaluating a device for
2836  * identity mapping because problems exist when devices are moved in and out
2837  * of domains and their respective RMRR information is lost.  This means that
2838  * a device with associated RMRRs will never be in a "passthrough" domain.
2839  * The second is use of the device through the IOMMU API.  This interface
2840  * expects to have full control of the IOVA space for the device.  We cannot
2841  * satisfy both the requirement that RMRR access is maintained and have an
2842  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2843  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2844  * We therefore prevent devices associated with an RMRR from participating in
2845  * the IOMMU API, which eliminates them from device assignment.
2846  *
2847  * In both cases, devices which have relaxable RMRRs are not concerned by this
2848  * restriction. See device_rmrr_is_relaxable comment.
2849  */
2850 static bool device_is_rmrr_locked(struct device *dev)
2851 {
2852         if (!device_has_rmrr(dev))
2853                 return false;
2854
2855         if (device_rmrr_is_relaxable(dev))
2856                 return false;
2857
2858         return true;
2859 }
2860
2861 /*
2862  * Return the required default domain type for a specific device.
2863  *
2864  * @dev: the device in query
2865  * @startup: true if this is during early boot
2866  *
2867  * Returns:
2868  *  - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
2869  *  - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
2870  *  - 0: both identity and dynamic domains work for this device
2871  */
2872 static int device_def_domain_type(struct device *dev)
2873 {
2874         if (dev_is_pci(dev)) {
2875                 struct pci_dev *pdev = to_pci_dev(dev);
2876
2877                 if (device_is_rmrr_locked(dev))
2878                         return IOMMU_DOMAIN_DMA;
2879
2880                 /*
2881                  * Prevent any device marked as untrusted from getting
2882                  * placed into the statically identity mapping domain.
2883                  */
2884                 if (pdev->untrusted)
2885                         return IOMMU_DOMAIN_DMA;
2886
2887                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2888                         return IOMMU_DOMAIN_IDENTITY;
2889
2890                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2891                         return IOMMU_DOMAIN_IDENTITY;
2892
2893                 /*
2894                  * We want to start off with all devices in the 1:1 domain, and
2895                  * take them out later if we find they can't access all of memory.
2896                  *
2897                  * However, we can't do this for PCI devices behind bridges,
2898                  * because all PCI devices behind the same bridge will end up
2899                  * with the same source-id on their transactions.
2900                  *
2901                  * Practically speaking, we can't change things around for these
2902                  * devices at run-time, because we can't be sure there'll be no
2903                  * DMA transactions in flight for any of their siblings.
2904                  *
2905                  * So PCI devices (unless they're on the root bus) as well as
2906                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2907                  * the 1:1 domain, just in _case_ one of their siblings turns out
2908                  * not to be able to map all of memory.
2909                  */
2910                 if (!pci_is_pcie(pdev)) {
2911                         if (!pci_is_root_bus(pdev->bus))
2912                                 return IOMMU_DOMAIN_DMA;
2913                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2914                                 return IOMMU_DOMAIN_DMA;
2915                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2916                         return IOMMU_DOMAIN_DMA;
2917         } else {
2918                 if (device_has_rmrr(dev))
2919                         return IOMMU_DOMAIN_DMA;
2920         }
2921
2922         return (iommu_identity_mapping & IDENTMAP_ALL) ?
2923                         IOMMU_DOMAIN_IDENTITY : 0;
2924 }
2925
2926 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2927 {
2928         /*
2929          * Start from the sane iommu hardware state.
2930          * If the queued invalidation is already initialized by us
2931          * (for example, while enabling interrupt-remapping) then
2932          * we got the things already rolling from a sane state.
2933          */
2934         if (!iommu->qi) {
2935                 /*
2936                  * Clear any previous faults.
2937                  */
2938                 dmar_fault(-1, iommu);
2939                 /*
2940                  * Disable queued invalidation if supported and already enabled
2941                  * before OS handover.
2942                  */
2943                 dmar_disable_qi(iommu);
2944         }
2945
2946         if (dmar_enable_qi(iommu)) {
2947                 /*
2948                  * Queued Invalidate not enabled, use Register Based Invalidate
2949                  */
2950                 iommu->flush.flush_context = __iommu_flush_context;
2951                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2952                 pr_info("%s: Using Register based invalidation\n",
2953                         iommu->name);
2954         } else {
2955                 iommu->flush.flush_context = qi_flush_context;
2956                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2957                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2958         }
2959 }
2960
2961 static int copy_context_table(struct intel_iommu *iommu,
2962                               struct root_entry *old_re,
2963                               struct context_entry **tbl,
2964                               int bus, bool ext)
2965 {
2966         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2967         struct context_entry *new_ce = NULL, ce;
2968         struct context_entry *old_ce = NULL;
2969         struct root_entry re;
2970         phys_addr_t old_ce_phys;
2971
2972         tbl_idx = ext ? bus * 2 : bus;
2973         memcpy(&re, old_re, sizeof(re));
2974
2975         for (devfn = 0; devfn < 256; devfn++) {
2976                 /* First calculate the correct index */
2977                 idx = (ext ? devfn * 2 : devfn) % 256;
2978
2979                 if (idx == 0) {
2980                         /* First save what we may have and clean up */
2981                         if (new_ce) {
2982                                 tbl[tbl_idx] = new_ce;
2983                                 __iommu_flush_cache(iommu, new_ce,
2984                                                     VTD_PAGE_SIZE);
2985                                 pos = 1;
2986                         }
2987
2988                         if (old_ce)
2989                                 memunmap(old_ce);
2990
2991                         ret = 0;
2992                         if (devfn < 0x80)
2993                                 old_ce_phys = root_entry_lctp(&re);
2994                         else
2995                                 old_ce_phys = root_entry_uctp(&re);
2996
2997                         if (!old_ce_phys) {
2998                                 if (ext && devfn == 0) {
2999                                         /* No LCTP, try UCTP */
3000                                         devfn = 0x7f;
3001                                         continue;
3002                                 } else {
3003                                         goto out;
3004                                 }
3005                         }
3006
3007                         ret = -ENOMEM;
3008                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3009                                         MEMREMAP_WB);
3010                         if (!old_ce)
3011                                 goto out;
3012
3013                         new_ce = alloc_pgtable_page(iommu->node);
3014                         if (!new_ce)
3015                                 goto out_unmap;
3016
3017                         ret = 0;
3018                 }
3019
3020                 /* Now copy the context entry */
3021                 memcpy(&ce, old_ce + idx, sizeof(ce));
3022
3023                 if (!__context_present(&ce))
3024                         continue;
3025
3026                 did = context_domain_id(&ce);
3027                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3028                         set_bit(did, iommu->domain_ids);
3029
3030                 /*
3031                  * We need a marker for copied context entries. This
3032                  * marker needs to work for the old format as well as
3033                  * for extended context entries.
3034                  *
3035                  * Bit 67 of the context entry is used. In the old
3036                  * format this bit is available to software, in the
3037                  * extended format it is the PGE bit, but PGE is ignored
3038                  * by HW if PASIDs are disabled (and thus still
3039                  * available).
3040                  *
3041                  * So disable PASIDs first and then mark the entry
3042                  * copied. This means that we don't copy PASID
3043                  * translations from the old kernel, but this is fine as
3044                  * faults there are not fatal.
3045                  */
3046                 context_clear_pasid_enable(&ce);
3047                 context_set_copied(&ce);
3048
3049                 new_ce[idx] = ce;
3050         }
3051
3052         tbl[tbl_idx + pos] = new_ce;
3053
3054         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3055
3056 out_unmap:
3057         memunmap(old_ce);
3058
3059 out:
3060         return ret;
3061 }
3062
3063 static int copy_translation_tables(struct intel_iommu *iommu)
3064 {
3065         struct context_entry **ctxt_tbls;
3066         struct root_entry *old_rt;
3067         phys_addr_t old_rt_phys;
3068         int ctxt_table_entries;
3069         unsigned long flags;
3070         u64 rtaddr_reg;
3071         int bus, ret;
3072         bool new_ext, ext;
3073
3074         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3075         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3076         new_ext    = !!ecap_ecs(iommu->ecap);
3077
3078         /*
3079          * The RTT bit can only be changed when translation is disabled,
3080          * but disabling translation means to open a window for data
3081          * corruption. So bail out and don't copy anything if we would
3082          * have to change the bit.
3083          */
3084         if (new_ext != ext)
3085                 return -EINVAL;
3086
3087         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3088         if (!old_rt_phys)
3089                 return -EINVAL;
3090
3091         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3092         if (!old_rt)
3093                 return -ENOMEM;
3094
3095         /* This is too big for the stack - allocate it from slab */
3096         ctxt_table_entries = ext ? 512 : 256;
3097         ret = -ENOMEM;
3098         ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3099         if (!ctxt_tbls)
3100                 goto out_unmap;
3101
3102         for (bus = 0; bus < 256; bus++) {
3103                 ret = copy_context_table(iommu, &old_rt[bus],
3104                                          ctxt_tbls, bus, ext);
3105                 if (ret) {
3106                         pr_err("%s: Failed to copy context table for bus %d\n",
3107                                 iommu->name, bus);
3108                         continue;
3109                 }
3110         }
3111
3112         spin_lock_irqsave(&iommu->lock, flags);
3113
3114         /* Context tables are copied, now write them to the root_entry table */
3115         for (bus = 0; bus < 256; bus++) {
3116                 int idx = ext ? bus * 2 : bus;
3117                 u64 val;
3118
3119                 if (ctxt_tbls[idx]) {
3120                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3121                         iommu->root_entry[bus].lo = val;
3122                 }
3123
3124                 if (!ext || !ctxt_tbls[idx + 1])
3125                         continue;
3126
3127                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3128                 iommu->root_entry[bus].hi = val;
3129         }
3130
3131         spin_unlock_irqrestore(&iommu->lock, flags);
3132
3133         kfree(ctxt_tbls);
3134
3135         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3136
3137         ret = 0;
3138
3139 out_unmap:
3140         memunmap(old_rt);
3141
3142         return ret;
3143 }
3144
3145 static int __init init_dmars(void)
3146 {
3147         struct dmar_drhd_unit *drhd;
3148         struct intel_iommu *iommu;
3149         int ret;
3150
3151         /*
3152          * for each drhd
3153          *    allocate root
3154          *    initialize and program root entry to not present
3155          * endfor
3156          */
3157         for_each_drhd_unit(drhd) {
3158                 /*
3159                  * lock not needed as this is only incremented in the single
3160                  * threaded kernel __init code path all other access are read
3161                  * only
3162                  */
3163                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3164                         g_num_of_iommus++;
3165                         continue;
3166                 }
3167                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3168         }
3169
3170         /* Preallocate enough resources for IOMMU hot-addition */
3171         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3172                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3173
3174         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3175                         GFP_KERNEL);
3176         if (!g_iommus) {
3177                 pr_err("Allocating global iommu array failed\n");
3178                 ret = -ENOMEM;
3179                 goto error;
3180         }
3181
3182         for_each_iommu(iommu, drhd) {
3183                 if (drhd->ignored) {
3184                         iommu_disable_translation(iommu);
3185                         continue;
3186                 }
3187
3188                 /*
3189                  * Find the max pasid size of all IOMMU's in the system.
3190                  * We need to ensure the system pasid table is no bigger
3191                  * than the smallest supported.
3192                  */
3193                 if (pasid_supported(iommu)) {
3194                         u32 temp = 2 << ecap_pss(iommu->ecap);
3195
3196                         intel_pasid_max_id = min_t(u32, temp,
3197                                                    intel_pasid_max_id);
3198                 }
3199
3200                 g_iommus[iommu->seq_id] = iommu;
3201
3202                 intel_iommu_init_qi(iommu);