2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
22 #define dev_fmt(fmt) pr_fmt(fmt)
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/export.h>
28 #include <linux/slab.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/memory.h>
37 #include <linux/cpu.h>
38 #include <linux/timer.h>
40 #include <linux/iova.h>
41 #include <linux/iommu.h>
42 #include <linux/intel-iommu.h>
43 #include <linux/syscore_ops.h>
44 #include <linux/tboot.h>
45 #include <linux/dmi.h>
46 #include <linux/pci-ats.h>
47 #include <linux/memblock.h>
48 #include <linux/dma-contiguous.h>
49 #include <linux/dma-direct.h>
50 #include <linux/crash_dump.h>
51 #include <asm/irq_remapping.h>
52 #include <asm/cacheflush.h>
53 #include <asm/iommu.h>
55 #include "irq_remapping.h"
56 #include "intel-pasid.h"
58 #define ROOT_SIZE VTD_PAGE_SIZE
59 #define CONTEXT_SIZE VTD_PAGE_SIZE
61 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
62 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
63 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
64 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
66 #define IOAPIC_RANGE_START (0xfee00000)
67 #define IOAPIC_RANGE_END (0xfeefffff)
68 #define IOVA_START_ADDR (0x1000)
70 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
72 #define MAX_AGAW_WIDTH 64
73 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
75 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
76 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
78 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
79 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
80 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
81 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
82 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
84 /* IO virtual address start page frame number */
85 #define IOVA_START_PFN (1)
87 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
89 /* page table handling */
90 #define LEVEL_STRIDE (9)
91 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
94 * This bitmap is used to advertise the page sizes our hardware support
95 * to the IOMMU core, which will then use this information to split
96 * physically contiguous memory regions it is mapping into page sizes
99 * Traditionally the IOMMU core just handed us the mappings directly,
100 * after making sure the size is an order of a 4KiB page and that the
101 * mapping has natural alignment.
103 * To retain this behavior, we currently advertise that we support
104 * all page sizes that are an order of 4KiB.
106 * If at some point we'd like to utilize the IOMMU core's new behavior,
107 * we could change this to advertise the real page sizes we support.
109 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
111 static inline int agaw_to_level(int agaw)
116 static inline int agaw_to_width(int agaw)
118 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
121 static inline int width_to_agaw(int width)
123 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
126 static inline unsigned int level_to_offset_bits(int level)
128 return (level - 1) * LEVEL_STRIDE;
131 static inline int pfn_level_offset(unsigned long pfn, int level)
133 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
136 static inline unsigned long level_mask(int level)
138 return -1UL << level_to_offset_bits(level);
141 static inline unsigned long level_size(int level)
143 return 1UL << level_to_offset_bits(level);
146 static inline unsigned long align_to_level(unsigned long pfn, int level)
148 return (pfn + level_size(level) - 1) & level_mask(level);
151 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
153 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
156 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
157 are never going to work. */
158 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
160 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
163 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
165 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
167 static inline unsigned long page_to_dma_pfn(struct page *pg)
169 return mm_to_dma_pfn(page_to_pfn(pg));
171 static inline unsigned long virt_to_dma_pfn(void *p)
173 return page_to_dma_pfn(virt_to_page(p));
176 /* global iommu list, set NULL for ignored DMAR units */
177 static struct intel_iommu **g_iommus;
179 static void __init check_tylersburg_isoch(void);
180 static int rwbf_quirk;
183 * set to 1 to panic kernel if can't successfully enable VT-d
184 * (used when kernel is launched w/ TXT)
186 static int force_on = 0;
187 int intel_iommu_tboot_noforce;
188 static int no_platform_optin;
190 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
193 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
196 static phys_addr_t root_entry_lctp(struct root_entry *re)
201 return re->lo & VTD_PAGE_MASK;
205 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
208 static phys_addr_t root_entry_uctp(struct root_entry *re)
213 return re->hi & VTD_PAGE_MASK;
216 static inline void context_clear_pasid_enable(struct context_entry *context)
218 context->lo &= ~(1ULL << 11);
221 static inline bool context_pasid_enabled(struct context_entry *context)
223 return !!(context->lo & (1ULL << 11));
226 static inline void context_set_copied(struct context_entry *context)
228 context->hi |= (1ull << 3);
231 static inline bool context_copied(struct context_entry *context)
233 return !!(context->hi & (1ULL << 3));
236 static inline bool __context_present(struct context_entry *context)
238 return (context->lo & 1);
241 bool context_present(struct context_entry *context)
243 return context_pasid_enabled(context) ?
244 __context_present(context) :
245 __context_present(context) && !context_copied(context);
248 static inline void context_set_present(struct context_entry *context)
253 static inline void context_set_fault_enable(struct context_entry *context)
255 context->lo &= (((u64)-1) << 2) | 1;
258 static inline void context_set_translation_type(struct context_entry *context,
261 context->lo &= (((u64)-1) << 4) | 3;
262 context->lo |= (value & 3) << 2;
265 static inline void context_set_address_root(struct context_entry *context,
268 context->lo &= ~VTD_PAGE_MASK;
269 context->lo |= value & VTD_PAGE_MASK;
272 static inline void context_set_address_width(struct context_entry *context,
275 context->hi |= value & 7;
278 static inline void context_set_domain_id(struct context_entry *context,
281 context->hi |= (value & ((1 << 16) - 1)) << 8;
284 static inline int context_domain_id(struct context_entry *c)
286 return((c->hi >> 8) & 0xffff);
289 static inline void context_clear_entry(struct context_entry *context)
296 * This domain is a statically identity mapping domain.
297 * 1. This domain creats a static 1:1 mapping to all usable memory.
298 * 2. It maps to each iommu if successful.
299 * 3. Each iommu mapps to this domain if successful.
301 static struct dmar_domain *si_domain;
302 static int hw_pass_through = 1;
305 * Domain represents a virtual machine, more than one devices
306 * across iommus may be owned in one domain, e.g. kvm guest.
308 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
310 /* si_domain contains mulitple devices */
311 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
313 #define for_each_domain_iommu(idx, domain) \
314 for (idx = 0; idx < g_num_of_iommus; idx++) \
315 if (domain->iommu_refcnt[idx])
317 struct dmar_rmrr_unit {
318 struct list_head list; /* list of rmrr units */
319 struct acpi_dmar_header *hdr; /* ACPI header */
320 u64 base_address; /* reserved base address*/
321 u64 end_address; /* reserved end address */
322 struct dmar_dev_scope *devices; /* target devices */
323 int devices_cnt; /* target device count */
324 struct iommu_resv_region *resv; /* reserved region handle */
327 struct dmar_atsr_unit {
328 struct list_head list; /* list of ATSR units */
329 struct acpi_dmar_header *hdr; /* ACPI header */
330 struct dmar_dev_scope *devices; /* target devices */
331 int devices_cnt; /* target device count */
332 u8 include_all:1; /* include all ports */
335 static LIST_HEAD(dmar_atsr_units);
336 static LIST_HEAD(dmar_rmrr_units);
338 #define for_each_rmrr_units(rmrr) \
339 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
341 /* bitmap for indexing intel_iommus */
342 static int g_num_of_iommus;
344 static void domain_exit(struct dmar_domain *domain);
345 static void domain_remove_dev_info(struct dmar_domain *domain);
346 static void dmar_remove_one_dev_info(struct device *dev);
347 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
348 static void domain_context_clear(struct intel_iommu *iommu,
350 static int domain_detach_iommu(struct dmar_domain *domain,
351 struct intel_iommu *iommu);
353 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
354 int dmar_disabled = 0;
356 int dmar_disabled = 1;
357 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
359 int intel_iommu_enabled = 0;
360 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
362 static int dmar_map_gfx = 1;
363 static int dmar_forcedac;
364 static int intel_iommu_strict;
365 static int intel_iommu_superpage = 1;
366 static int intel_iommu_sm = 1;
367 static int iommu_identity_mapping;
369 #define IDENTMAP_ALL 1
370 #define IDENTMAP_GFX 2
371 #define IDENTMAP_AZALIA 4
373 #define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
374 #define pasid_supported(iommu) (sm_supported(iommu) && \
375 ecap_pasid((iommu)->ecap))
377 int intel_iommu_gfx_mapped;
378 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
380 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
381 static DEFINE_SPINLOCK(device_domain_lock);
382 static LIST_HEAD(device_domain_list);
385 * Iterate over elements in device_domain_list and call the specified
386 * callback @fn against each element.
388 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
389 void *data), void *data)
393 struct device_domain_info *info;
395 spin_lock_irqsave(&device_domain_lock, flags);
396 list_for_each_entry(info, &device_domain_list, global) {
397 ret = fn(info, data);
399 spin_unlock_irqrestore(&device_domain_lock, flags);
403 spin_unlock_irqrestore(&device_domain_lock, flags);
408 const struct iommu_ops intel_iommu_ops;
410 static bool translation_pre_enabled(struct intel_iommu *iommu)
412 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
415 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
417 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
420 static void init_translation_status(struct intel_iommu *iommu)
424 gsts = readl(iommu->reg + DMAR_GSTS_REG);
425 if (gsts & DMA_GSTS_TES)
426 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
429 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
430 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
432 return container_of(dom, struct dmar_domain, domain);
435 static int __init intel_iommu_setup(char *str)
440 if (!strncmp(str, "on", 2)) {
442 pr_info("IOMMU enabled\n");
443 } else if (!strncmp(str, "off", 3)) {
445 no_platform_optin = 1;
446 pr_info("IOMMU disabled\n");
447 } else if (!strncmp(str, "igfx_off", 8)) {
449 pr_info("Disable GFX device mapping\n");
450 } else if (!strncmp(str, "forcedac", 8)) {
451 pr_info("Forcing DAC for PCI devices\n");
453 } else if (!strncmp(str, "strict", 6)) {
454 pr_info("Disable batched IOTLB flush\n");
455 intel_iommu_strict = 1;
456 } else if (!strncmp(str, "sp_off", 6)) {
457 pr_info("Disable supported super page\n");
458 intel_iommu_superpage = 0;
459 } else if (!strncmp(str, "sm_off", 6)) {
460 pr_info("Intel-IOMMU: disable scalable mode support\n");
462 } else if (!strncmp(str, "tboot_noforce", 13)) {
464 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
465 intel_iommu_tboot_noforce = 1;
468 str += strcspn(str, ",");
474 __setup("intel_iommu=", intel_iommu_setup);
476 static struct kmem_cache *iommu_domain_cache;
477 static struct kmem_cache *iommu_devinfo_cache;
479 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
481 struct dmar_domain **domains;
484 domains = iommu->domains[idx];
488 return domains[did & 0xff];
491 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
492 struct dmar_domain *domain)
494 struct dmar_domain **domains;
497 if (!iommu->domains[idx]) {
498 size_t size = 256 * sizeof(struct dmar_domain *);
499 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
502 domains = iommu->domains[idx];
503 if (WARN_ON(!domains))
506 domains[did & 0xff] = domain;
509 void *alloc_pgtable_page(int node)
514 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
516 vaddr = page_address(page);
520 void free_pgtable_page(void *vaddr)
522 free_page((unsigned long)vaddr);
525 static inline void *alloc_domain_mem(void)
527 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
530 static void free_domain_mem(void *vaddr)
532 kmem_cache_free(iommu_domain_cache, vaddr);
535 static inline void * alloc_devinfo_mem(void)
537 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
540 static inline void free_devinfo_mem(void *vaddr)
542 kmem_cache_free(iommu_devinfo_cache, vaddr);
545 static inline int domain_type_is_vm(struct dmar_domain *domain)
547 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
550 static inline int domain_type_is_si(struct dmar_domain *domain)
552 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
555 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
557 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
558 DOMAIN_FLAG_STATIC_IDENTITY);
561 static inline int domain_pfn_supported(struct dmar_domain *domain,
564 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
566 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
569 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
574 sagaw = cap_sagaw(iommu->cap);
575 for (agaw = width_to_agaw(max_gaw);
577 if (test_bit(agaw, &sagaw))
585 * Calculate max SAGAW for each iommu.
587 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
589 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
593 * calculate agaw for each iommu.
594 * "SAGAW" may be different across iommus, use a default agaw, and
595 * get a supported less agaw for iommus that don't support the default agaw.
597 int iommu_calculate_agaw(struct intel_iommu *iommu)
599 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
602 /* This functionin only returns single iommu in a domain */
603 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
607 /* si_domain and vm domain should not get here. */
608 BUG_ON(domain_type_is_vm_or_si(domain));
609 for_each_domain_iommu(iommu_id, domain)
612 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
615 return g_iommus[iommu_id];
618 static void domain_update_iommu_coherency(struct dmar_domain *domain)
620 struct dmar_drhd_unit *drhd;
621 struct intel_iommu *iommu;
625 domain->iommu_coherency = 1;
627 for_each_domain_iommu(i, domain) {
629 if (!ecap_coherent(g_iommus[i]->ecap)) {
630 domain->iommu_coherency = 0;
637 /* No hardware attached; use lowest common denominator */
639 for_each_active_iommu(iommu, drhd) {
640 if (!ecap_coherent(iommu->ecap)) {
641 domain->iommu_coherency = 0;
648 static int domain_update_iommu_snooping(struct intel_iommu *skip)
650 struct dmar_drhd_unit *drhd;
651 struct intel_iommu *iommu;
655 for_each_active_iommu(iommu, drhd) {
657 if (!ecap_sc_support(iommu->ecap)) {
668 static int domain_update_iommu_superpage(struct intel_iommu *skip)
670 struct dmar_drhd_unit *drhd;
671 struct intel_iommu *iommu;
674 if (!intel_iommu_superpage) {
678 /* set iommu_superpage to the smallest common denominator */
680 for_each_active_iommu(iommu, drhd) {
682 mask &= cap_super_page_val(iommu->cap);
692 /* Some capabilities may be different across iommus */
693 static void domain_update_iommu_cap(struct dmar_domain *domain)
695 domain_update_iommu_coherency(domain);
696 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
697 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
700 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
703 struct root_entry *root = &iommu->root_entry[bus];
704 struct context_entry *context;
708 if (sm_supported(iommu)) {
716 context = phys_to_virt(*entry & VTD_PAGE_MASK);
718 unsigned long phy_addr;
722 context = alloc_pgtable_page(iommu->node);
726 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
727 phy_addr = virt_to_phys((void *)context);
728 *entry = phy_addr | 1;
729 __iommu_flush_cache(iommu, entry, sizeof(*entry));
731 return &context[devfn];
734 static int iommu_dummy(struct device *dev)
736 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
739 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
741 struct dmar_drhd_unit *drhd = NULL;
742 struct intel_iommu *iommu;
744 struct pci_dev *ptmp, *pdev = NULL;
748 if (iommu_dummy(dev))
751 if (dev_is_pci(dev)) {
752 struct pci_dev *pf_pdev;
754 pdev = to_pci_dev(dev);
757 /* VMD child devices currently cannot be handled individually */
758 if (is_vmd(pdev->bus))
762 /* VFs aren't listed in scope tables; we need to look up
763 * the PF instead to find the IOMMU. */
764 pf_pdev = pci_physfn(pdev);
766 segment = pci_domain_nr(pdev->bus);
767 } else if (has_acpi_companion(dev))
768 dev = &ACPI_COMPANION(dev)->dev;
771 for_each_active_iommu(iommu, drhd) {
772 if (pdev && segment != drhd->segment)
775 for_each_active_dev_scope(drhd->devices,
776 drhd->devices_cnt, i, tmp) {
778 /* For a VF use its original BDF# not that of the PF
779 * which we used for the IOMMU lookup. Strictly speaking
780 * we could do this for all PCI devices; we only need to
781 * get the BDF# from the scope table for ACPI matches. */
782 if (pdev && pdev->is_virtfn)
785 *bus = drhd->devices[i].bus;
786 *devfn = drhd->devices[i].devfn;
790 if (!pdev || !dev_is_pci(tmp))
793 ptmp = to_pci_dev(tmp);
794 if (ptmp->subordinate &&
795 ptmp->subordinate->number <= pdev->bus->number &&
796 ptmp->subordinate->busn_res.end >= pdev->bus->number)
800 if (pdev && drhd->include_all) {
802 *bus = pdev->bus->number;
803 *devfn = pdev->devfn;
814 static void domain_flush_cache(struct dmar_domain *domain,
815 void *addr, int size)
817 if (!domain->iommu_coherency)
818 clflush_cache_range(addr, size);
821 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
823 struct context_entry *context;
827 spin_lock_irqsave(&iommu->lock, flags);
828 context = iommu_context_addr(iommu, bus, devfn, 0);
830 ret = context_present(context);
831 spin_unlock_irqrestore(&iommu->lock, flags);
835 static void free_context_table(struct intel_iommu *iommu)
839 struct context_entry *context;
841 spin_lock_irqsave(&iommu->lock, flags);
842 if (!iommu->root_entry) {
845 for (i = 0; i < ROOT_ENTRY_NR; i++) {
846 context = iommu_context_addr(iommu, i, 0, 0);
848 free_pgtable_page(context);
850 if (!sm_supported(iommu))
853 context = iommu_context_addr(iommu, i, 0x80, 0);
855 free_pgtable_page(context);
858 free_pgtable_page(iommu->root_entry);
859 iommu->root_entry = NULL;
861 spin_unlock_irqrestore(&iommu->lock, flags);
864 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
865 unsigned long pfn, int *target_level)
867 struct dma_pte *parent, *pte;
868 int level = agaw_to_level(domain->agaw);
871 BUG_ON(!domain->pgd);
873 if (!domain_pfn_supported(domain, pfn))
874 /* Address beyond IOMMU's addressing capabilities. */
877 parent = domain->pgd;
882 offset = pfn_level_offset(pfn, level);
883 pte = &parent[offset];
884 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
886 if (level == *target_level)
889 if (!dma_pte_present(pte)) {
892 tmp_page = alloc_pgtable_page(domain->nid);
897 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
898 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
899 if (cmpxchg64(&pte->val, 0ULL, pteval))
900 /* Someone else set it while we were thinking; use theirs. */
901 free_pgtable_page(tmp_page);
903 domain_flush_cache(domain, pte, sizeof(*pte));
908 parent = phys_to_virt(dma_pte_addr(pte));
913 *target_level = level;
919 /* return address's pte at specific level */
920 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
922 int level, int *large_page)
924 struct dma_pte *parent, *pte;
925 int total = agaw_to_level(domain->agaw);
928 parent = domain->pgd;
929 while (level <= total) {
930 offset = pfn_level_offset(pfn, total);
931 pte = &parent[offset];
935 if (!dma_pte_present(pte)) {
940 if (dma_pte_superpage(pte)) {
945 parent = phys_to_virt(dma_pte_addr(pte));
951 /* clear last level pte, a tlb flush should be followed */
952 static void dma_pte_clear_range(struct dmar_domain *domain,
953 unsigned long start_pfn,
954 unsigned long last_pfn)
956 unsigned int large_page;
957 struct dma_pte *first_pte, *pte;
959 BUG_ON(!domain_pfn_supported(domain, start_pfn));
960 BUG_ON(!domain_pfn_supported(domain, last_pfn));
961 BUG_ON(start_pfn > last_pfn);
963 /* we don't need lock here; nobody else touches the iova range */
966 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
968 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
973 start_pfn += lvl_to_nr_pages(large_page);
975 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
977 domain_flush_cache(domain, first_pte,
978 (void *)pte - (void *)first_pte);
980 } while (start_pfn && start_pfn <= last_pfn);
983 static void dma_pte_free_level(struct dmar_domain *domain, int level,
984 int retain_level, struct dma_pte *pte,
985 unsigned long pfn, unsigned long start_pfn,
986 unsigned long last_pfn)
988 pfn = max(start_pfn, pfn);
989 pte = &pte[pfn_level_offset(pfn, level)];
992 unsigned long level_pfn;
993 struct dma_pte *level_pte;
995 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
998 level_pfn = pfn & level_mask(level);
999 level_pte = phys_to_virt(dma_pte_addr(pte));
1002 dma_pte_free_level(domain, level - 1, retain_level,
1003 level_pte, level_pfn, start_pfn,
1008 * Free the page table if we're below the level we want to
1009 * retain and the range covers the entire table.
1011 if (level < retain_level && !(start_pfn > level_pfn ||
1012 last_pfn < level_pfn + level_size(level) - 1)) {
1014 domain_flush_cache(domain, pte, sizeof(*pte));
1015 free_pgtable_page(level_pte);
1018 pfn += level_size(level);
1019 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1023 * clear last level (leaf) ptes and free page table pages below the
1024 * level we wish to keep intact.
1026 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1027 unsigned long start_pfn,
1028 unsigned long last_pfn,
1031 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1032 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1033 BUG_ON(start_pfn > last_pfn);
1035 dma_pte_clear_range(domain, start_pfn, last_pfn);
1037 /* We don't need lock here; nobody else touches the iova range */
1038 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1039 domain->pgd, 0, start_pfn, last_pfn);
1042 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1043 free_pgtable_page(domain->pgd);
1048 /* When a page at a given level is being unlinked from its parent, we don't
1049 need to *modify* it at all. All we need to do is make a list of all the
1050 pages which can be freed just as soon as we've flushed the IOTLB and we
1051 know the hardware page-walk will no longer touch them.
1052 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1054 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1055 int level, struct dma_pte *pte,
1056 struct page *freelist)
1060 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1061 pg->freelist = freelist;
1067 pte = page_address(pg);
1069 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1070 freelist = dma_pte_list_pagetables(domain, level - 1,
1073 } while (!first_pte_in_page(pte));
1078 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1079 struct dma_pte *pte, unsigned long pfn,
1080 unsigned long start_pfn,
1081 unsigned long last_pfn,
1082 struct page *freelist)
1084 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1086 pfn = max(start_pfn, pfn);
1087 pte = &pte[pfn_level_offset(pfn, level)];
1090 unsigned long level_pfn;
1092 if (!dma_pte_present(pte))
1095 level_pfn = pfn & level_mask(level);
1097 /* If range covers entire pagetable, free it */
1098 if (start_pfn <= level_pfn &&
1099 last_pfn >= level_pfn + level_size(level) - 1) {
1100 /* These suborbinate page tables are going away entirely. Don't
1101 bother to clear them; we're just going to *free* them. */
1102 if (level > 1 && !dma_pte_superpage(pte))
1103 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1109 } else if (level > 1) {
1110 /* Recurse down into a level that isn't *entirely* obsolete */
1111 freelist = dma_pte_clear_level(domain, level - 1,
1112 phys_to_virt(dma_pte_addr(pte)),
1113 level_pfn, start_pfn, last_pfn,
1117 pfn += level_size(level);
1118 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1121 domain_flush_cache(domain, first_pte,
1122 (void *)++last_pte - (void *)first_pte);
1127 /* We can't just free the pages because the IOMMU may still be walking
1128 the page tables, and may have cached the intermediate levels. The
1129 pages can only be freed after the IOTLB flush has been done. */
1130 static struct page *domain_unmap(struct dmar_domain *domain,
1131 unsigned long start_pfn,
1132 unsigned long last_pfn)
1134 struct page *freelist;
1136 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1137 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1138 BUG_ON(start_pfn > last_pfn);
1140 /* we don't need lock here; nobody else touches the iova range */
1141 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1142 domain->pgd, 0, start_pfn, last_pfn, NULL);
1145 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1146 struct page *pgd_page = virt_to_page(domain->pgd);
1147 pgd_page->freelist = freelist;
1148 freelist = pgd_page;
1156 static void dma_free_pagelist(struct page *freelist)
1160 while ((pg = freelist)) {
1161 freelist = pg->freelist;
1162 free_pgtable_page(page_address(pg));
1166 static void iova_entry_free(unsigned long data)
1168 struct page *freelist = (struct page *)data;
1170 dma_free_pagelist(freelist);
1173 /* iommu handling */
1174 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1176 struct root_entry *root;
1177 unsigned long flags;
1179 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1181 pr_err("Allocating root entry for %s failed\n",
1186 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1188 spin_lock_irqsave(&iommu->lock, flags);
1189 iommu->root_entry = root;
1190 spin_unlock_irqrestore(&iommu->lock, flags);
1195 static void iommu_set_root_entry(struct intel_iommu *iommu)
1201 addr = virt_to_phys(iommu->root_entry);
1202 if (sm_supported(iommu))
1203 addr |= DMA_RTADDR_SMT;
1205 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1206 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1208 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1210 /* Make sure hardware complete it */
1211 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1212 readl, (sts & DMA_GSTS_RTPS), sts);
1214 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1217 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1222 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1225 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1226 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1228 /* Make sure hardware complete it */
1229 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1230 readl, (!(val & DMA_GSTS_WBFS)), val);
1232 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1235 /* return value determine if we need a write buffer flush */
1236 static void __iommu_flush_context(struct intel_iommu *iommu,
1237 u16 did, u16 source_id, u8 function_mask,
1244 case DMA_CCMD_GLOBAL_INVL:
1245 val = DMA_CCMD_GLOBAL_INVL;
1247 case DMA_CCMD_DOMAIN_INVL:
1248 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1250 case DMA_CCMD_DEVICE_INVL:
1251 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1252 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1257 val |= DMA_CCMD_ICC;
1259 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1260 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1262 /* Make sure hardware complete it */
1263 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1264 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1266 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1269 /* return value determine if we need a write buffer flush */
1270 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1271 u64 addr, unsigned int size_order, u64 type)
1273 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1274 u64 val = 0, val_iva = 0;
1278 case DMA_TLB_GLOBAL_FLUSH:
1279 /* global flush doesn't need set IVA_REG */
1280 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1282 case DMA_TLB_DSI_FLUSH:
1283 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1285 case DMA_TLB_PSI_FLUSH:
1286 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1287 /* IH bit is passed in as part of address */
1288 val_iva = size_order | addr;
1293 /* Note: set drain read/write */
1296 * This is probably to be super secure.. Looks like we can
1297 * ignore it without any impact.
1299 if (cap_read_drain(iommu->cap))
1300 val |= DMA_TLB_READ_DRAIN;
1302 if (cap_write_drain(iommu->cap))
1303 val |= DMA_TLB_WRITE_DRAIN;
1305 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1306 /* Note: Only uses first TLB reg currently */
1308 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1309 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1311 /* Make sure hardware complete it */
1312 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1313 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1315 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1317 /* check IOTLB invalidation granularity */
1318 if (DMA_TLB_IAIG(val) == 0)
1319 pr_err("Flush IOTLB failed\n");
1320 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1321 pr_debug("TLB flush request %Lx, actual %Lx\n",
1322 (unsigned long long)DMA_TLB_IIRG(type),
1323 (unsigned long long)DMA_TLB_IAIG(val));
1326 static struct device_domain_info *
1327 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1330 struct device_domain_info *info;
1332 assert_spin_locked(&device_domain_lock);
1337 list_for_each_entry(info, &domain->devices, link)
1338 if (info->iommu == iommu && info->bus == bus &&
1339 info->devfn == devfn) {
1340 if (info->ats_supported && info->dev)
1348 static void domain_update_iotlb(struct dmar_domain *domain)
1350 struct device_domain_info *info;
1351 bool has_iotlb_device = false;
1353 assert_spin_locked(&device_domain_lock);
1355 list_for_each_entry(info, &domain->devices, link) {
1356 struct pci_dev *pdev;
1358 if (!info->dev || !dev_is_pci(info->dev))
1361 pdev = to_pci_dev(info->dev);
1362 if (pdev->ats_enabled) {
1363 has_iotlb_device = true;
1368 domain->has_iotlb_device = has_iotlb_device;
1371 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1373 struct pci_dev *pdev;
1375 assert_spin_locked(&device_domain_lock);
1377 if (!info || !dev_is_pci(info->dev))
1380 pdev = to_pci_dev(info->dev);
1381 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1382 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1383 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1384 * reserved, which should be set to 0.
1386 if (!ecap_dit(info->iommu->ecap))
1389 struct pci_dev *pf_pdev;
1391 /* pdev will be returned if device is not a vf */
1392 pf_pdev = pci_physfn(pdev);
1393 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1396 #ifdef CONFIG_INTEL_IOMMU_SVM
1397 /* The PCIe spec, in its wisdom, declares that the behaviour of
1398 the device if you enable PASID support after ATS support is
1399 undefined. So always enable PASID support on devices which
1400 have it, even if we can't yet know if we're ever going to
1402 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1403 info->pasid_enabled = 1;
1405 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1406 info->pri_enabled = 1;
1408 if (!pdev->untrusted && info->ats_supported &&
1409 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1410 info->ats_enabled = 1;
1411 domain_update_iotlb(info->domain);
1412 info->ats_qdep = pci_ats_queue_depth(pdev);
1416 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1418 struct pci_dev *pdev;
1420 assert_spin_locked(&device_domain_lock);
1422 if (!dev_is_pci(info->dev))
1425 pdev = to_pci_dev(info->dev);
1427 if (info->ats_enabled) {
1428 pci_disable_ats(pdev);
1429 info->ats_enabled = 0;
1430 domain_update_iotlb(info->domain);
1432 #ifdef CONFIG_INTEL_IOMMU_SVM
1433 if (info->pri_enabled) {
1434 pci_disable_pri(pdev);
1435 info->pri_enabled = 0;
1437 if (info->pasid_enabled) {
1438 pci_disable_pasid(pdev);
1439 info->pasid_enabled = 0;
1444 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1445 u64 addr, unsigned mask)
1448 unsigned long flags;
1449 struct device_domain_info *info;
1451 if (!domain->has_iotlb_device)
1454 spin_lock_irqsave(&device_domain_lock, flags);
1455 list_for_each_entry(info, &domain->devices, link) {
1456 if (!info->ats_enabled)
1459 sid = info->bus << 8 | info->devfn;
1460 qdep = info->ats_qdep;
1461 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1464 spin_unlock_irqrestore(&device_domain_lock, flags);
1467 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1468 struct dmar_domain *domain,
1469 unsigned long pfn, unsigned int pages,
1472 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1473 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1474 u16 did = domain->iommu_did[iommu->seq_id];
1481 * Fallback to domain selective flush if no PSI support or the size is
1483 * PSI requires page size to be 2 ^ x, and the base address is naturally
1484 * aligned to the size
1486 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1487 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1490 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1494 * In caching mode, changes of pages from non-present to present require
1495 * flush. However, device IOTLB doesn't need to be flushed in this case.
1497 if (!cap_caching_mode(iommu->cap) || !map)
1498 iommu_flush_dev_iotlb(domain, addr, mask);
1501 /* Notification for newly created mappings */
1502 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1503 struct dmar_domain *domain,
1504 unsigned long pfn, unsigned int pages)
1506 /* It's a non-present to present mapping. Only flush if caching mode */
1507 if (cap_caching_mode(iommu->cap))
1508 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1510 iommu_flush_write_buffer(iommu);
1513 static void iommu_flush_iova(struct iova_domain *iovad)
1515 struct dmar_domain *domain;
1518 domain = container_of(iovad, struct dmar_domain, iovad);
1520 for_each_domain_iommu(idx, domain) {
1521 struct intel_iommu *iommu = g_iommus[idx];
1522 u16 did = domain->iommu_did[iommu->seq_id];
1524 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1526 if (!cap_caching_mode(iommu->cap))
1527 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1528 0, MAX_AGAW_PFN_WIDTH);
1532 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1535 unsigned long flags;
1537 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1538 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1539 pmen &= ~DMA_PMEN_EPM;
1540 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1542 /* wait for the protected region status bit to clear */
1543 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1544 readl, !(pmen & DMA_PMEN_PRS), pmen);
1546 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1549 static void iommu_enable_translation(struct intel_iommu *iommu)
1552 unsigned long flags;
1554 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1555 iommu->gcmd |= DMA_GCMD_TE;
1556 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1558 /* Make sure hardware complete it */
1559 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1560 readl, (sts & DMA_GSTS_TES), sts);
1562 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1565 static void iommu_disable_translation(struct intel_iommu *iommu)
1570 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1571 iommu->gcmd &= ~DMA_GCMD_TE;
1572 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1574 /* Make sure hardware complete it */
1575 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1576 readl, (!(sts & DMA_GSTS_TES)), sts);
1578 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1582 static int iommu_init_domains(struct intel_iommu *iommu)
1584 u32 ndomains, nlongs;
1587 ndomains = cap_ndoms(iommu->cap);
1588 pr_debug("%s: Number of Domains supported <%d>\n",
1589 iommu->name, ndomains);
1590 nlongs = BITS_TO_LONGS(ndomains);
1592 spin_lock_init(&iommu->lock);
1594 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1595 if (!iommu->domain_ids) {
1596 pr_err("%s: Allocating domain id array failed\n",
1601 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1602 iommu->domains = kzalloc(size, GFP_KERNEL);
1604 if (iommu->domains) {
1605 size = 256 * sizeof(struct dmar_domain *);
1606 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1609 if (!iommu->domains || !iommu->domains[0]) {
1610 pr_err("%s: Allocating domain array failed\n",
1612 kfree(iommu->domain_ids);
1613 kfree(iommu->domains);
1614 iommu->domain_ids = NULL;
1615 iommu->domains = NULL;
1622 * If Caching mode is set, then invalid translations are tagged
1623 * with domain-id 0, hence we need to pre-allocate it. We also
1624 * use domain-id 0 as a marker for non-allocated domain-id, so
1625 * make sure it is not used for a real domain.
1627 set_bit(0, iommu->domain_ids);
1630 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1631 * entry for first-level or pass-through translation modes should
1632 * be programmed with a domain id different from those used for
1633 * second-level or nested translation. We reserve a domain id for
1636 if (sm_supported(iommu))
1637 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1642 static void disable_dmar_iommu(struct intel_iommu *iommu)
1644 struct device_domain_info *info, *tmp;
1645 unsigned long flags;
1647 if (!iommu->domains || !iommu->domain_ids)
1651 spin_lock_irqsave(&device_domain_lock, flags);
1652 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1653 struct dmar_domain *domain;
1655 if (info->iommu != iommu)
1658 if (!info->dev || !info->domain)
1661 domain = info->domain;
1663 __dmar_remove_one_dev_info(info);
1665 if (!domain_type_is_vm_or_si(domain)) {
1667 * The domain_exit() function can't be called under
1668 * device_domain_lock, as it takes this lock itself.
1669 * So release the lock here and re-run the loop
1672 spin_unlock_irqrestore(&device_domain_lock, flags);
1673 domain_exit(domain);
1677 spin_unlock_irqrestore(&device_domain_lock, flags);
1679 if (iommu->gcmd & DMA_GCMD_TE)
1680 iommu_disable_translation(iommu);
1683 static void free_dmar_iommu(struct intel_iommu *iommu)
1685 if ((iommu->domains) && (iommu->domain_ids)) {
1686 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1689 for (i = 0; i < elems; i++)
1690 kfree(iommu->domains[i]);
1691 kfree(iommu->domains);
1692 kfree(iommu->domain_ids);
1693 iommu->domains = NULL;
1694 iommu->domain_ids = NULL;
1697 g_iommus[iommu->seq_id] = NULL;
1699 /* free context mapping */
1700 free_context_table(iommu);
1702 #ifdef CONFIG_INTEL_IOMMU_SVM
1703 if (pasid_supported(iommu)) {
1704 if (ecap_prs(iommu->ecap))
1705 intel_svm_finish_prq(iommu);
1710 static struct dmar_domain *alloc_domain(int flags)
1712 struct dmar_domain *domain;
1714 domain = alloc_domain_mem();
1718 memset(domain, 0, sizeof(*domain));
1720 domain->flags = flags;
1721 domain->has_iotlb_device = false;
1722 INIT_LIST_HEAD(&domain->devices);
1727 /* Must be called with iommu->lock */
1728 static int domain_attach_iommu(struct dmar_domain *domain,
1729 struct intel_iommu *iommu)
1731 unsigned long ndomains;
1734 assert_spin_locked(&device_domain_lock);
1735 assert_spin_locked(&iommu->lock);
1737 domain->iommu_refcnt[iommu->seq_id] += 1;
1738 domain->iommu_count += 1;
1739 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1740 ndomains = cap_ndoms(iommu->cap);
1741 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1743 if (num >= ndomains) {
1744 pr_err("%s: No free domain ids\n", iommu->name);
1745 domain->iommu_refcnt[iommu->seq_id] -= 1;
1746 domain->iommu_count -= 1;
1750 set_bit(num, iommu->domain_ids);
1751 set_iommu_domain(iommu, num, domain);
1753 domain->iommu_did[iommu->seq_id] = num;
1754 domain->nid = iommu->node;
1756 domain_update_iommu_cap(domain);
1762 static int domain_detach_iommu(struct dmar_domain *domain,
1763 struct intel_iommu *iommu)
1767 assert_spin_locked(&device_domain_lock);
1768 assert_spin_locked(&iommu->lock);
1770 domain->iommu_refcnt[iommu->seq_id] -= 1;
1771 count = --domain->iommu_count;
1772 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1773 num = domain->iommu_did[iommu->seq_id];
1774 clear_bit(num, iommu->domain_ids);
1775 set_iommu_domain(iommu, num, NULL);
1777 domain_update_iommu_cap(domain);
1778 domain->iommu_did[iommu->seq_id] = 0;
1784 static struct iova_domain reserved_iova_list;
1785 static struct lock_class_key reserved_rbtree_key;
1787 static int dmar_init_reserved_ranges(void)
1789 struct pci_dev *pdev = NULL;
1793 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1795 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1796 &reserved_rbtree_key);
1798 /* IOAPIC ranges shouldn't be accessed by DMA */
1799 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1800 IOVA_PFN(IOAPIC_RANGE_END));
1802 pr_err("Reserve IOAPIC range failed\n");
1806 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1807 for_each_pci_dev(pdev) {
1810 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1811 r = &pdev->resource[i];
1812 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1814 iova = reserve_iova(&reserved_iova_list,
1818 pci_err(pdev, "Reserve iova for %pR failed\n", r);
1826 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1828 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1831 static inline int guestwidth_to_adjustwidth(int gaw)
1834 int r = (gaw - 12) % 9;
1845 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1848 int adjust_width, agaw;
1849 unsigned long sagaw;
1852 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1854 err = init_iova_flush_queue(&domain->iovad,
1855 iommu_flush_iova, iova_entry_free);
1859 domain_reserve_special_ranges(domain);
1861 /* calculate AGAW */
1862 if (guest_width > cap_mgaw(iommu->cap))
1863 guest_width = cap_mgaw(iommu->cap);
1864 domain->gaw = guest_width;
1865 adjust_width = guestwidth_to_adjustwidth(guest_width);
1866 agaw = width_to_agaw(adjust_width);
1867 sagaw = cap_sagaw(iommu->cap);
1868 if (!test_bit(agaw, &sagaw)) {
1869 /* hardware doesn't support it, choose a bigger one */
1870 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1871 agaw = find_next_bit(&sagaw, 5, agaw);
1875 domain->agaw = agaw;
1877 if (ecap_coherent(iommu->ecap))
1878 domain->iommu_coherency = 1;
1880 domain->iommu_coherency = 0;
1882 if (ecap_sc_support(iommu->ecap))
1883 domain->iommu_snooping = 1;
1885 domain->iommu_snooping = 0;
1887 if (intel_iommu_superpage)
1888 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1890 domain->iommu_superpage = 0;
1892 domain->nid = iommu->node;
1894 /* always allocate the top pgd */
1895 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1898 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1902 static void domain_exit(struct dmar_domain *domain)
1904 struct page *freelist;
1906 /* Domain 0 is reserved, so dont process it */
1910 /* Remove associated devices and clear attached or cached domains */
1912 domain_remove_dev_info(domain);
1916 put_iova_domain(&domain->iovad);
1918 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1920 dma_free_pagelist(freelist);
1922 free_domain_mem(domain);
1926 * Get the PASID directory size for scalable mode context entry.
1927 * Value of X in the PDTS field of a scalable mode context entry
1928 * indicates PASID directory with 2^(X + 7) entries.
1930 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1934 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1935 pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1943 * Set the RID_PASID field of a scalable mode context entry. The
1944 * IOMMU hardware will use the PASID value set in this field for
1945 * DMA translations of DMA requests without PASID.
1948 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1950 context->hi |= pasid & ((1 << 20) - 1);
1951 context->hi |= (1 << 20);
1955 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1958 static inline void context_set_sm_dte(struct context_entry *context)
1960 context->lo |= (1 << 2);
1964 * Set the PRE(Page Request Enable) field of a scalable mode context
1967 static inline void context_set_sm_pre(struct context_entry *context)
1969 context->lo |= (1 << 4);
1972 /* Convert value to context PASID directory size field coding. */
1973 #define context_pdts(pds) (((pds) & 0x7) << 9)
1975 static int domain_context_mapping_one(struct dmar_domain *domain,
1976 struct intel_iommu *iommu,
1977 struct pasid_table *table,
1980 u16 did = domain->iommu_did[iommu->seq_id];
1981 int translation = CONTEXT_TT_MULTI_LEVEL;
1982 struct device_domain_info *info = NULL;
1983 struct context_entry *context;
1984 unsigned long flags;
1989 if (hw_pass_through && domain_type_is_si(domain))
1990 translation = CONTEXT_TT_PASS_THROUGH;
1992 pr_debug("Set context mapping for %02x:%02x.%d\n",
1993 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1995 BUG_ON(!domain->pgd);
1997 spin_lock_irqsave(&device_domain_lock, flags);
1998 spin_lock(&iommu->lock);
2001 context = iommu_context_addr(iommu, bus, devfn, 1);
2006 if (context_present(context))
2010 * For kdump cases, old valid entries may be cached due to the
2011 * in-flight DMA and copied pgtable, but there is no unmapping
2012 * behaviour for them, thus we need an explicit cache flush for
2013 * the newly-mapped device. For kdump, at this point, the device
2014 * is supposed to finish reset at its driver probe stage, so no
2015 * in-flight DMA will exist, and we don't need to worry anymore
2018 if (context_copied(context)) {
2019 u16 did_old = context_domain_id(context);
2021 if (did_old < cap_ndoms(iommu->cap)) {
2022 iommu->flush.flush_context(iommu, did_old,
2023 (((u16)bus) << 8) | devfn,
2024 DMA_CCMD_MASK_NOBIT,
2025 DMA_CCMD_DEVICE_INVL);
2026 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2031 context_clear_entry(context);
2033 if (sm_supported(iommu)) {
2038 /* Setup the PASID DIR pointer: */
2039 pds = context_get_sm_pds(table);
2040 context->lo = (u64)virt_to_phys(table->table) |
2043 /* Setup the RID_PASID field: */
2044 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2047 * Setup the Device-TLB enable bit and Page request
2050 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2051 if (info && info->ats_supported)
2052 context_set_sm_dte(context);
2053 if (info && info->pri_supported)
2054 context_set_sm_pre(context);
2056 struct dma_pte *pgd = domain->pgd;
2059 context_set_domain_id(context, did);
2060 context_set_translation_type(context, translation);
2062 if (translation != CONTEXT_TT_PASS_THROUGH) {
2064 * Skip top levels of page tables for iommu which has
2065 * less agaw than default. Unnecessary for PT mode.
2067 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2069 pgd = phys_to_virt(dma_pte_addr(pgd));
2070 if (!dma_pte_present(pgd))
2074 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2075 if (info && info->ats_supported)
2076 translation = CONTEXT_TT_DEV_IOTLB;
2078 translation = CONTEXT_TT_MULTI_LEVEL;
2080 context_set_address_root(context, virt_to_phys(pgd));
2081 context_set_address_width(context, agaw);
2084 * In pass through mode, AW must be programmed to
2085 * indicate the largest AGAW value supported by
2086 * hardware. And ASR is ignored by hardware.
2088 context_set_address_width(context, iommu->msagaw);
2092 context_set_fault_enable(context);
2093 context_set_present(context);
2094 domain_flush_cache(domain, context, sizeof(*context));
2097 * It's a non-present to present mapping. If hardware doesn't cache
2098 * non-present entry we only need to flush the write-buffer. If the
2099 * _does_ cache non-present entries, then it does so in the special
2100 * domain #0, which we have to flush:
2102 if (cap_caching_mode(iommu->cap)) {
2103 iommu->flush.flush_context(iommu, 0,
2104 (((u16)bus) << 8) | devfn,
2105 DMA_CCMD_MASK_NOBIT,
2106 DMA_CCMD_DEVICE_INVL);
2107 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2109 iommu_flush_write_buffer(iommu);
2111 iommu_enable_dev_iotlb(info);
2116 spin_unlock(&iommu->lock);
2117 spin_unlock_irqrestore(&device_domain_lock, flags);
2122 struct domain_context_mapping_data {
2123 struct dmar_domain *domain;
2124 struct intel_iommu *iommu;
2125 struct pasid_table *table;
2128 static int domain_context_mapping_cb(struct pci_dev *pdev,
2129 u16 alias, void *opaque)
2131 struct domain_context_mapping_data *data = opaque;
2133 return domain_context_mapping_one(data->domain, data->iommu,
2134 data->table, PCI_BUS_NUM(alias),
2139 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2141 struct domain_context_mapping_data data;
2142 struct pasid_table *table;
2143 struct intel_iommu *iommu;
2146 iommu = device_to_iommu(dev, &bus, &devfn);
2150 table = intel_pasid_get_table(dev);
2152 if (!dev_is_pci(dev))
2153 return domain_context_mapping_one(domain, iommu, table,
2156 data.domain = domain;
2160 return pci_for_each_dma_alias(to_pci_dev(dev),
2161 &domain_context_mapping_cb, &data);
2164 static int domain_context_mapped_cb(struct pci_dev *pdev,
2165 u16 alias, void *opaque)
2167 struct intel_iommu *iommu = opaque;
2169 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2172 static int domain_context_mapped(struct device *dev)
2174 struct intel_iommu *iommu;
2177 iommu = device_to_iommu(dev, &bus, &devfn);
2181 if (!dev_is_pci(dev))
2182 return device_context_mapped(iommu, bus, devfn);
2184 return !pci_for_each_dma_alias(to_pci_dev(dev),
2185 domain_context_mapped_cb, iommu);
2188 /* Returns a number of VTD pages, but aligned to MM page size */
2189 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2192 host_addr &= ~PAGE_MASK;
2193 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2196 /* Return largest possible superpage level for a given mapping */
2197 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2198 unsigned long iov_pfn,
2199 unsigned long phy_pfn,
2200 unsigned long pages)
2202 int support, level = 1;
2203 unsigned long pfnmerge;
2205 support = domain->iommu_superpage;
2207 /* To use a large page, the virtual *and* physical addresses
2208 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2209 of them will mean we have to use smaller pages. So just
2210 merge them and check both at once. */
2211 pfnmerge = iov_pfn | phy_pfn;
2213 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2214 pages >>= VTD_STRIDE_SHIFT;
2217 pfnmerge >>= VTD_STRIDE_SHIFT;
2224 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2225 struct scatterlist *sg, unsigned long phys_pfn,
2226 unsigned long nr_pages, int prot)
2228 struct dma_pte *first_pte = NULL, *pte = NULL;
2229 phys_addr_t uninitialized_var(pteval);
2230 unsigned long sg_res = 0;
2231 unsigned int largepage_lvl = 0;
2232 unsigned long lvl_pages = 0;
2234 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2236 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2239 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2243 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2246 while (nr_pages > 0) {
2250 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2252 sg_res = aligned_nrpages(sg->offset, sg->length);
2253 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2254 sg->dma_length = sg->length;
2255 pteval = (sg_phys(sg) - pgoff) | prot;
2256 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2260 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2262 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2265 /* It is large page*/
2266 if (largepage_lvl > 1) {
2267 unsigned long nr_superpages, end_pfn;
2269 pteval |= DMA_PTE_LARGE_PAGE;
2270 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2272 nr_superpages = sg_res / lvl_pages;
2273 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2276 * Ensure that old small page tables are
2277 * removed to make room for superpage(s).
2278 * We're adding new large pages, so make sure
2279 * we don't remove their parent tables.
2281 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2284 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2288 /* We don't need lock here, nobody else
2289 * touches the iova range
2291 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2293 static int dumps = 5;
2294 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2295 iov_pfn, tmp, (unsigned long long)pteval);
2298 debug_dma_dump_mappings(NULL);
2303 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2305 BUG_ON(nr_pages < lvl_pages);
2306 BUG_ON(sg_res < lvl_pages);
2308 nr_pages -= lvl_pages;
2309 iov_pfn += lvl_pages;
2310 phys_pfn += lvl_pages;
2311 pteval += lvl_pages * VTD_PAGE_SIZE;
2312 sg_res -= lvl_pages;
2314 /* If the next PTE would be the first in a new page, then we
2315 need to flush the cache on the entries we've just written.
2316 And then we'll need to recalculate 'pte', so clear it and
2317 let it get set again in the if (!pte) block above.
2319 If we're done (!nr_pages) we need to flush the cache too.
2321 Also if we've been setting superpages, we may need to
2322 recalculate 'pte' and switch back to smaller pages for the
2323 end of the mapping, if the trailing size is not enough to
2324 use another superpage (i.e. sg_res < lvl_pages). */
2326 if (!nr_pages || first_pte_in_page(pte) ||
2327 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2328 domain_flush_cache(domain, first_pte,
2329 (void *)pte - (void *)first_pte);
2333 if (!sg_res && nr_pages)
2339 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2340 struct scatterlist *sg, unsigned long phys_pfn,
2341 unsigned long nr_pages, int prot)
2344 struct intel_iommu *iommu;
2346 /* Do the real mapping first */
2347 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2351 /* Notify about the new mapping */
2352 if (domain_type_is_vm(domain)) {
2353 /* VM typed domains can have more than one IOMMUs */
2355 for_each_domain_iommu(iommu_id, domain) {
2356 iommu = g_iommus[iommu_id];
2357 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2360 /* General domains only have one IOMMU */
2361 iommu = domain_get_iommu(domain);
2362 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2368 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2369 struct scatterlist *sg, unsigned long nr_pages,
2372 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2375 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2376 unsigned long phys_pfn, unsigned long nr_pages,
2379 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2382 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2384 unsigned long flags;
2385 struct context_entry *context;
2391 spin_lock_irqsave(&iommu->lock, flags);
2392 context = iommu_context_addr(iommu, bus, devfn, 0);
2394 spin_unlock_irqrestore(&iommu->lock, flags);
2397 did_old = context_domain_id(context);
2398 context_clear_entry(context);
2399 __iommu_flush_cache(iommu, context, sizeof(*context));
2400 spin_unlock_irqrestore(&iommu->lock, flags);
2401 iommu->flush.flush_context(iommu,
2403 (((u16)bus) << 8) | devfn,
2404 DMA_CCMD_MASK_NOBIT,
2405 DMA_CCMD_DEVICE_INVL);
2406 iommu->flush.flush_iotlb(iommu,
2413 static inline void unlink_domain_info(struct device_domain_info *info)
2415 assert_spin_locked(&device_domain_lock);
2416 list_del(&info->link);
2417 list_del(&info->global);
2419 info->dev->archdata.iommu = NULL;
2422 static void domain_remove_dev_info(struct dmar_domain *domain)
2424 struct device_domain_info *info, *tmp;
2425 unsigned long flags;
2427 spin_lock_irqsave(&device_domain_lock, flags);
2428 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2429 __dmar_remove_one_dev_info(info);
2430 spin_unlock_irqrestore(&device_domain_lock, flags);
2435 * Note: we use struct device->archdata.iommu stores the info
2437 static struct dmar_domain *find_domain(struct device *dev)
2439 struct device_domain_info *info;
2441 /* No lock here, assumes no domain exit in normal case */
2442 info = dev->archdata.iommu;
2444 return info->domain;
2448 static inline struct device_domain_info *
2449 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2451 struct device_domain_info *info;
2453 list_for_each_entry(info, &device_domain_list, global)
2454 if (info->iommu->segment == segment && info->bus == bus &&
2455 info->devfn == devfn)
2461 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2464 struct dmar_domain *domain)
2466 struct dmar_domain *found = NULL;
2467 struct device_domain_info *info;
2468 unsigned long flags;
2471 info = alloc_devinfo_mem();
2476 info->devfn = devfn;
2477 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2478 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2481 info->domain = domain;
2482 info->iommu = iommu;
2483 info->pasid_table = NULL;
2485 if (dev && dev_is_pci(dev)) {
2486 struct pci_dev *pdev = to_pci_dev(info->dev);
2488 if (!pci_ats_disabled() &&
2489 ecap_dev_iotlb_support(iommu->ecap) &&
2490 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2491 dmar_find_matched_atsr_unit(pdev))
2492 info->ats_supported = 1;
2494 if (sm_supported(iommu)) {
2495 if (pasid_supported(iommu)) {
2496 int features = pci_pasid_features(pdev);
2498 info->pasid_supported = features | 1;
2501 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2502 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2503 info->pri_supported = 1;
2507 spin_lock_irqsave(&device_domain_lock, flags);
2509 found = find_domain(dev);
2512 struct device_domain_info *info2;
2513 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2515 found = info2->domain;
2521 spin_unlock_irqrestore(&device_domain_lock, flags);
2522 free_devinfo_mem(info);
2523 /* Caller must free the original domain */
2527 spin_lock(&iommu->lock);
2528 ret = domain_attach_iommu(domain, iommu);
2529 spin_unlock(&iommu->lock);
2532 spin_unlock_irqrestore(&device_domain_lock, flags);
2533 free_devinfo_mem(info);
2537 list_add(&info->link, &domain->devices);
2538 list_add(&info->global, &device_domain_list);
2540 dev->archdata.iommu = info;
2541 spin_unlock_irqrestore(&device_domain_lock, flags);
2543 /* PASID table is mandatory for a PCI device in scalable mode. */
2544 if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2545 ret = intel_pasid_alloc_table(dev);
2547 dev_err(dev, "PASID table allocation failed\n");
2548 dmar_remove_one_dev_info(dev);
2552 /* Setup the PASID entry for requests without PASID: */
2553 spin_lock(&iommu->lock);
2554 if (hw_pass_through && domain_type_is_si(domain))
2555 ret = intel_pasid_setup_pass_through(iommu, domain,
2556 dev, PASID_RID2PASID);
2558 ret = intel_pasid_setup_second_level(iommu, domain,
2559 dev, PASID_RID2PASID);
2560 spin_unlock(&iommu->lock);
2562 dev_err(dev, "Setup RID2PASID failed\n");
2563 dmar_remove_one_dev_info(dev);
2568 if (dev && domain_context_mapping(domain, dev)) {
2569 dev_err(dev, "Domain context map failed\n");
2570 dmar_remove_one_dev_info(dev);
2577 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2579 *(u16 *)opaque = alias;
2583 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2585 struct device_domain_info *info;
2586 struct dmar_domain *domain = NULL;
2587 struct intel_iommu *iommu;
2589 unsigned long flags;
2592 iommu = device_to_iommu(dev, &bus, &devfn);
2596 if (dev_is_pci(dev)) {
2597 struct pci_dev *pdev = to_pci_dev(dev);
2599 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2601 spin_lock_irqsave(&device_domain_lock, flags);
2602 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2603 PCI_BUS_NUM(dma_alias),
2606 iommu = info->iommu;
2607 domain = info->domain;
2609 spin_unlock_irqrestore(&device_domain_lock, flags);
2611 /* DMA alias already has a domain, use it */
2616 /* Allocate and initialize new domain for the device */
2617 domain = alloc_domain(0);
2620 if (domain_init(domain, iommu, gaw)) {
2621 domain_exit(domain);
2630 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2631 struct dmar_domain *domain)
2633 struct intel_iommu *iommu;
2634 struct dmar_domain *tmp;
2635 u16 req_id, dma_alias;
2638 iommu = device_to_iommu(dev, &bus, &devfn);
2642 req_id = ((u16)bus << 8) | devfn;
2644 if (dev_is_pci(dev)) {
2645 struct pci_dev *pdev = to_pci_dev(dev);
2647 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2649 /* register PCI DMA alias device */
2650 if (req_id != dma_alias) {
2651 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2652 dma_alias & 0xff, NULL, domain);
2654 if (!tmp || tmp != domain)
2659 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2660 if (!tmp || tmp != domain)
2666 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2668 struct dmar_domain *domain, *tmp;
2670 domain = find_domain(dev);
2674 domain = find_or_alloc_domain(dev, gaw);
2678 tmp = set_domain_for_dev(dev, domain);
2679 if (!tmp || domain != tmp) {
2680 domain_exit(domain);
2689 static int iommu_domain_identity_map(struct dmar_domain *domain,
2690 unsigned long long start,
2691 unsigned long long end)
2693 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2694 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2696 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2697 dma_to_mm_pfn(last_vpfn))) {
2698 pr_err("Reserving iova failed\n");
2702 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2704 * RMRR range might have overlap with physical memory range,
2707 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2709 return __domain_mapping(domain, first_vpfn, NULL,
2710 first_vpfn, last_vpfn - first_vpfn + 1,
2711 DMA_PTE_READ|DMA_PTE_WRITE);
2714 static int domain_prepare_identity_map(struct device *dev,
2715 struct dmar_domain *domain,
2716 unsigned long long start,
2717 unsigned long long end)
2719 /* For _hardware_ passthrough, don't bother. But for software
2720 passthrough, we do it anyway -- it may indicate a memory
2721 range which is reserved in E820, so which didn't get set
2722 up to start with in si_domain */
2723 if (domain == si_domain && hw_pass_through) {
2724 dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
2729 dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
2732 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2733 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2734 dmi_get_system_info(DMI_BIOS_VENDOR),
2735 dmi_get_system_info(DMI_BIOS_VERSION),
2736 dmi_get_system_info(DMI_PRODUCT_VERSION));
2740 if (end >> agaw_to_width(domain->agaw)) {
2741 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2742 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2743 agaw_to_width(domain->agaw),
2744 dmi_get_system_info(DMI_BIOS_VENDOR),
2745 dmi_get_system_info(DMI_BIOS_VERSION),
2746 dmi_get_system_info(DMI_PRODUCT_VERSION));
2750 return iommu_domain_identity_map(domain, start, end);
2753 static int iommu_prepare_identity_map(struct device *dev,
2754 unsigned long long start,
2755 unsigned long long end)
2757 struct dmar_domain *domain;
2760 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2764 ret = domain_prepare_identity_map(dev, domain, start, end);
2766 domain_exit(domain);
2771 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2774 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2776 return iommu_prepare_identity_map(dev, rmrr->base_address,
2780 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2781 static inline void iommu_prepare_isa(void)
2783 struct pci_dev *pdev;
2786 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2790 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2791 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2794 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2799 static inline void iommu_prepare_isa(void)
2803 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2805 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2807 static int __init si_domain_init(int hw)
2811 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2815 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2816 domain_exit(si_domain);
2820 pr_debug("Identity mapping domain allocated\n");
2825 for_each_online_node(nid) {
2826 unsigned long start_pfn, end_pfn;
2829 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2830 ret = iommu_domain_identity_map(si_domain,
2831 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2840 static int identity_mapping(struct device *dev)
2842 struct device_domain_info *info;
2844 if (likely(!iommu_identity_mapping))
2847 info = dev->archdata.iommu;
2848 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2849 return (info->domain == si_domain);
2854 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2856 struct dmar_domain *ndomain;
2857 struct intel_iommu *iommu;
2860 iommu = device_to_iommu(dev, &bus, &devfn);
2864 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2865 if (ndomain != domain)
2871 static bool device_has_rmrr(struct device *dev)
2873 struct dmar_rmrr_unit *rmrr;
2878 for_each_rmrr_units(rmrr) {
2880 * Return TRUE if this RMRR contains the device that
2883 for_each_active_dev_scope(rmrr->devices,
2884 rmrr->devices_cnt, i, tmp)
2895 * There are a couple cases where we need to restrict the functionality of
2896 * devices associated with RMRRs. The first is when evaluating a device for
2897 * identity mapping because problems exist when devices are moved in and out
2898 * of domains and their respective RMRR information is lost. This means that
2899 * a device with associated RMRRs will never be in a "passthrough" domain.
2900 * The second is use of the device through the IOMMU API. This interface
2901 * expects to have full control of the IOVA space for the device. We cannot
2902 * satisfy both the requirement that RMRR access is maintained and have an
2903 * unencumbered IOVA space. We also have no ability to quiesce the device's
2904 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2905 * We therefore prevent devices associated with an RMRR from participating in
2906 * the IOMMU API, which eliminates them from device assignment.
2908 * In both cases we assume that PCI USB devices with RMRRs have them largely
2909 * for historical reasons and that the RMRR space is not actively used post
2910 * boot. This exclusion may change if vendors begin to abuse it.
2912 * The same exception is made for graphics devices, with the requirement that
2913 * any use of the RMRR regions will be torn down before assigning the device
2916 static bool device_is_rmrr_locked(struct device *dev)
2918 if (!device_has_rmrr(dev))
2921 if (dev_is_pci(dev)) {
2922 struct pci_dev *pdev = to_pci_dev(dev);
2924 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2931 static int iommu_should_identity_map(struct device *dev, int startup)
2933 if (dev_is_pci(dev)) {
2934 struct pci_dev *pdev = to_pci_dev(dev);
2936 if (device_is_rmrr_locked(dev))
2940 * Prevent any device marked as untrusted from getting
2941 * placed into the statically identity mapping domain.
2943 if (pdev->untrusted)
2946 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2949 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2952 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2956 * We want to start off with all devices in the 1:1 domain, and
2957 * take them out later if we find they can't access all of memory.
2959 * However, we can't do this for PCI devices behind bridges,
2960 * because all PCI devices behind the same bridge will end up
2961 * with the same source-id on their transactions.
2963 * Practically speaking, we can't change things around for these
2964 * devices at run-time, because we can't be sure there'll be no
2965 * DMA transactions in flight for any of their siblings.
2967 * So PCI devices (unless they're on the root bus) as well as
2968 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2969 * the 1:1 domain, just in _case_ one of their siblings turns out
2970 * not to be able to map all of memory.
2972 if (!pci_is_pcie(pdev)) {
2973 if (!pci_is_root_bus(pdev->bus))
2975 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2977 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2980 if (device_has_rmrr(dev))
2985 * At boot time, we don't yet know if devices will be 64-bit capable.
2986 * Assume that they will — if they turn out not to be, then we can
2987 * take them out of the 1:1 domain later.
2991 * If the device's dma_mask is less than the system's memory
2992 * size then this is not a candidate for identity mapping.
2994 u64 dma_mask = *dev->dma_mask;
2996 if (dev->coherent_dma_mask &&
2997 dev->coherent_dma_mask < dma_mask)
2998 dma_mask = dev->coherent_dma_mask;
3000 return dma_mask >= dma_get_required_mask(dev);
3006 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
3010 if (!iommu_should_identity_map(dev, 1))
3013 ret = domain_add_dev_info(si_domain, dev);
3015 dev_info(dev, "%s identity mapping\n",
3016 hw ? "Hardware" : "Software");
3017 else if (ret == -ENODEV)
3018 /* device not associated with an iommu */
3025 static int __init iommu_prepare_static_identity_mapping(int hw)
3027 struct pci_dev *pdev = NULL;
3028 struct dmar_drhd_unit *drhd;
3029 struct intel_iommu *iommu;
3034 for_each_pci_dev(pdev) {
3035 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3040 for_each_active_iommu(iommu, drhd)
3041 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3042 struct acpi_device_physical_node *pn;
3043 struct acpi_device *adev;
3045 if (dev->bus != &acpi_bus_type)
3048 adev= to_acpi_device(dev);
3049 mutex_lock(&adev->physical_node_lock);
3050 list_for_each_entry(pn, &adev->physical_node_list, node) {
3051 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3055 mutex_unlock(&adev->physical_node_lock);
3063 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3066 * Start from the sane iommu hardware state.
3067 * If the queued invalidation is already initialized by us
3068 * (for example, while enabling interrupt-remapping) then
3069 * we got the things already rolling from a sane state.
3073 * Clear any previous faults.
3075 dmar_fault(-1, iommu);
3077 * Disable queued invalidation if supported and already enabled
3078 * before OS handover.
3080 dmar_disable_qi(iommu);
3083 if (dmar_enable_qi(iommu)) {
3085 * Queued Invalidate not enabled, use Register Based Invalidate
3087 iommu->flush.flush_context = __iommu_flush_context;
3088 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3089 pr_info("%s: Using Register based invalidation\n",
3092 iommu->flush.flush_context = qi_flush_context;
3093 iommu->flush.flush_iotlb = qi_flush_iotlb;
3094 pr_info("%s: Using Queued invalidation\n", iommu->name);
3098 static int copy_context_table(struct intel_iommu *iommu,
3099 struct root_entry *old_re,
3100 struct context_entry **tbl,
3103 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3104 struct context_entry *new_ce = NULL, ce;
3105 struct context_entry *old_ce = NULL;
3106 struct root_entry re;
3107 phys_addr_t old_ce_phys;
3109 tbl_idx = ext ? bus * 2 : bus;
3110 memcpy(&re, old_re, sizeof(re));
3112 for (devfn = 0; devfn < 256; devfn++) {
3113 /* First calculate the correct index */
3114 idx = (ext ? devfn * 2 : devfn) % 256;
3117 /* First save what we may have and clean up */
3119 tbl[tbl_idx] = new_ce;
3120 __iommu_flush_cache(iommu, new_ce,
3130 old_ce_phys = root_entry_lctp(&re);
3132 old_ce_phys = root_entry_uctp(&re);
3135 if (ext && devfn == 0) {
3136 /* No LCTP, try UCTP */
3145 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3150 new_ce = alloc_pgtable_page(iommu->node);
3157 /* Now copy the context entry */
3158 memcpy(&ce, old_ce + idx, sizeof(ce));
3160 if (!__context_present(&ce))
3163 did = context_domain_id(&ce);
3164 if (did >= 0 && did < cap_ndoms(iommu->cap))
3165 set_bit(did, iommu->domain_ids);
3168 * We need a marker for copied context entries. This
3169 * marker needs to work for the old format as well as
3170 * for extended context entries.
3172 * Bit 67 of the context entry is used. In the old
3173 * format this bit is available to software, in the
3174 * extended format it is the PGE bit, but PGE is ignored
3175 * by HW if PASIDs are disabled (and thus still
3178 * So disable PASIDs first and then mark the entry
3179 * copied. This means that we don't copy PASID
3180 * translations from the old kernel, but this is fine as
3181 * faults there are not fatal.
3183 context_clear_pasid_enable(&ce);
3184 context_set_copied(&ce);
3189 tbl[tbl_idx + pos] = new_ce;
3191 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3200 static int copy_translation_tables(struct intel_iommu *iommu)
3202 struct context_entry **ctxt_tbls;
3203 struct root_entry *old_rt;
3204 phys_addr_t old_rt_phys;
3205 int ctxt_table_entries;
3206 unsigned long flags;
3211 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3212 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3213 new_ext = !!ecap_ecs(iommu->ecap);
3216 * The RTT bit can only be changed when translation is disabled,
3217 * but disabling translation means to open a window for data
3218 * corruption. So bail out and don't copy anything if we would
3219 * have to change the bit.
3224 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3228 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3232 /* This is too big for the stack - allocate it from slab */
3233 ctxt_table_entries = ext ? 512 : 256;
3235 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3239 for (bus = 0; bus < 256; bus++) {
3240 ret = copy_context_table(iommu, &old_rt[bus],
3241 ctxt_tbls, bus, ext);
3243 pr_err("%s: Failed to copy context table for bus %d\n",
3249 spin_lock_irqsave(&iommu->lock, flags);
3251 /* Context tables are copied, now write them to the root_entry table */
3252 for (bus = 0; bus < 256; bus++) {
3253 int idx = ext ? bus * 2 : bus;
3256 if (ctxt_tbls[idx]) {
3257 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3258 iommu->root_entry[bus].lo = val;
3261 if (!ext || !ctxt_tbls[idx + 1])
3264 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3265 iommu->root_entry[bus].hi = val;
3268 spin_unlock_irqrestore(&iommu->lock, flags);
3272 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3282 static int __init init_dmars(void)
3284 struct dmar_drhd_unit *drhd;
3285 struct dmar_rmrr_unit *rmrr;
3286 bool copied_tables = false;
3288 struct intel_iommu *iommu;
3294 * initialize and program root entry to not present
3297 for_each_drhd_unit(drhd) {
3299 * lock not needed as this is only incremented in the single
3300 * threaded kernel __init code path all other access are read
3303 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3307 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3310 /* Preallocate enough resources for IOMMU hot-addition */
3311 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3312 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3314 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3317 pr_err("Allocating global iommu array failed\n");
3322 for_each_active_iommu(iommu, drhd) {
3324 * Find the max pasid size of all IOMMU's in the system.
3325 * We need to ensure the system pasid table is no bigger
3326 * than the smallest supported.
3328 if (pasid_supported(iommu)) {
3329 u32 temp = 2 << ecap_pss(iommu->ecap);
3331 intel_pasid_max_id = min_t(u32, temp,
3332 intel_pasid_max_id);
3335 g_iommus[iommu->seq_id] = iommu;
3337 intel_iommu_init_qi(iommu);
3339 ret = iommu_init_domains(iommu);
3343 init_translation_status(iommu);
3345 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3346 iommu_disable_translation(iommu);
3347 clear_translation_pre_enabled(iommu);
3348 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3354 * we could share the same root & context tables
3355 * among all IOMMU's. Need to Split it later.
3357 ret = iommu_alloc_root_entry(iommu);
3361 if (translation_pre_enabled(iommu)) {
3362 pr_info("Translation already enabled - trying to copy translation structures\n");
3364 ret = copy_translation_tables(iommu);
3367 * We found the IOMMU with translation
3368 * enabled - but failed to copy over the
3369 * old root-entry table. Try to proceed
3370 * by disabling translation now and
3371 * allocating a clean root-entry table.
3372 * This might cause DMAR faults, but
3373 * probably the dump will still succeed.
3375 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3377 iommu_disable_translation(iommu);
3378 clear_translation_pre_enabled(iommu);
3380 pr_info("Copied translation tables from previous kernel for %s\n",
3382 copied_tables = true;
3386 if (!ecap_pass_through(iommu->ecap))
3387 hw_pass_through = 0;
3388 #ifdef CONFIG_INTEL_IOMMU_SVM
3389 if (pasid_supported(iommu))
3390 intel_svm_init(iommu);
3395 * Now that qi is enabled on all iommus, set the root entry and flush
3396 * caches. This is required on some Intel X58 chipsets, otherwise the
3397 * flush_context function will loop forever and the boot hangs.
3399 for_each_active_iommu(iommu, drhd) {
3400 iommu_flush_write_buffer(iommu);
3401 iommu_set_root_entry(iommu);
3402 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3403 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3406 if (iommu_pass_through)
3407 iommu_identity_mapping |= IDENTMAP_ALL;
3409 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3410 iommu_identity_mapping |= IDENTMAP_GFX;
3413 check_tylersburg_isoch();
3415 if (iommu_identity_mapping) {
3416 ret = si_domain_init(hw_pass_through);
3423 * If we copied translations from a previous kernel in the kdump
3424 * case, we can not assign the devices to domains now, as that
3425 * would eliminate the old mappings. So skip this part and defer
3426 * the assignment to device driver initialization time.
3432 * If pass through is not set or not enabled, setup context entries for
3433 * identity mappings for rmrr, gfx, and isa and may fall back to static
3434 * identity mapping if iommu_identity_mapping is set.
3436 if (iommu_identity_mapping) {
3437 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3439 pr_crit("Failed to setup IOMMU pass-through\n");
3445 * for each dev attached to rmrr
3447 * locate drhd for dev, alloc domain for dev
3448 * allocate free domain
3449 * allocate page table entries for rmrr
3450 * if context not allocated for bus
3451 * allocate and init context
3452 * set present in root table for this bus
3453 * init context with domain, translation etc
3457 pr_info("Setting RMRR:\n");
3458 for_each_rmrr_units(rmrr) {
3459 /* some BIOS lists non-exist devices in DMAR table. */
3460 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3462 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3464 pr_err("Mapping reserved region failed\n");
3468 iommu_prepare_isa();
3475 * global invalidate context cache
3476 * global invalidate iotlb
3477 * enable translation
3479 for_each_iommu(iommu, drhd) {
3480 if (drhd->ignored) {
3482 * we always have to disable PMRs or DMA may fail on
3486 iommu_disable_protect_mem_regions(iommu);
3490 iommu_flush_write_buffer(iommu);
3492 #ifdef CONFIG_INTEL_IOMMU_SVM
3493 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3494 ret = intel_svm_enable_prq(iommu);
3499 ret = dmar_set_interrupt(iommu);
3503 if (!translation_pre_enabled(iommu))
3504 iommu_enable_translation(iommu);
3506 iommu_disable_protect_mem_regions(iommu);
3512 for_each_active_iommu(iommu, drhd) {
3513 disable_dmar_iommu(iommu);
3514 free_dmar_iommu(iommu);
3523 /* This takes a number of _MM_ pages, not VTD pages */
3524 static unsigned long intel_alloc_iova(struct device *dev,
3525 struct dmar_domain *domain,
3526 unsigned long nrpages, uint64_t dma_mask)
3528 unsigned long iova_pfn;
3530 /* Restrict dma_mask to the width that the iommu can handle */
3531 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3532 /* Ensure we reserve the whole size-aligned region */
3533 nrpages = __roundup_pow_of_two(nrpages);
3535 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3537 * First try to allocate an io virtual address in
3538 * DMA_BIT_MASK(32) and if that fails then try allocating
3541 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3542 IOVA_PFN(DMA_BIT_MASK(32)), false);
3546 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3547 IOVA_PFN(dma_mask), true);
3548 if (unlikely(!iova_pfn)) {
3549 dev_err(dev, "Allocating %ld-page iova failed", nrpages);
3556 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3558 struct dmar_domain *domain, *tmp;
3559 struct dmar_rmrr_unit *rmrr;
3560 struct device *i_dev;
3563 domain = find_domain(dev);
3567 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3571 /* We have a new domain - setup possible RMRRs for the device */
3573 for_each_rmrr_units(rmrr) {
3574 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3579 ret = domain_prepare_identity_map(dev, domain,
3583 dev_err(dev, "Mapping reserved region failed\n");
3588 tmp = set_domain_for_dev(dev, domain);
3589 if (!tmp || domain != tmp) {
3590 domain_exit(domain);
3597 dev_err(dev, "Allocating domain failed\n");
3603 /* Check if the dev needs to go through non-identity map and unmap process.*/
3604 static int iommu_no_mapping(struct device *dev)
3608 if (iommu_dummy(dev))
3611 if (!iommu_identity_mapping)
3614 found = identity_mapping(dev);
3616 if (iommu_should_identity_map(dev, 0))
3620 * 32 bit DMA is removed from si_domain and fall back
3621 * to non-identity mapping.
3623 dmar_remove_one_dev_info(dev);
3624 dev_info(dev, "32bit DMA uses non-identity mapping\n");
3629 * In case of a detached 64 bit DMA device from vm, the device
3630 * is put into si_domain for identity mapping.
3632 if (iommu_should_identity_map(dev, 0)) {
3634 ret = domain_add_dev_info(si_domain, dev);
3636 dev_info(dev, "64bit DMA uses identity mapping\n");
3645 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3646 size_t size, int dir, u64 dma_mask)
3648 struct dmar_domain *domain;
3649 phys_addr_t start_paddr;
3650 unsigned long iova_pfn;
3653 struct intel_iommu *iommu;
3654 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3656 BUG_ON(dir == DMA_NONE);
3658 if (iommu_no_mapping(dev))
3661 domain = get_valid_domain_for_dev(dev);
3663 return DMA_MAPPING_ERROR;
3665 iommu = domain_get_iommu(domain);
3666 size = aligned_nrpages(paddr, size);
3668 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3673 * Check if DMAR supports zero-length reads on write only
3676 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3677 !cap_zlr(iommu->cap))
3678 prot |= DMA_PTE_READ;
3679 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3680 prot |= DMA_PTE_WRITE;
3682 * paddr - (paddr + size) might be partial page, we should map the whole
3683 * page. Note: if two part of one page are separately mapped, we
3684 * might have two guest_addr mapping to the same host paddr, but this
3685 * is not a big problem
3687 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3688 mm_to_dma_pfn(paddr_pfn), size, prot);
3692 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3693 start_paddr += paddr & ~PAGE_MASK;
3698 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3699 dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
3700 size, (unsigned long long)paddr, dir);
3701 return DMA_MAPPING_ERROR;
3704 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3705 unsigned long offset, size_t size,
3706 enum dma_data_direction dir,
3707 unsigned long attrs)
3709 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3710 dir, *dev->dma_mask);
3713 static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
3714 size_t size, enum dma_data_direction dir,
3715 unsigned long attrs)
3717 return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
3720 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3722 struct dmar_domain *domain;
3723 unsigned long start_pfn, last_pfn;
3724 unsigned long nrpages;
3725 unsigned long iova_pfn;
3726 struct intel_iommu *iommu;
3727 struct page *freelist;
3729 if (iommu_no_mapping(dev))
3732 domain = find_domain(dev);
3735 iommu = domain_get_iommu(domain);
3737 iova_pfn = IOVA_PFN(dev_addr);
3739 nrpages = aligned_nrpages(dev_addr, size);
3740 start_pfn = mm_to_dma_pfn(iova_pfn);
3741 last_pfn = start_pfn + nrpages - 1;
3743 dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn);
3745 freelist = domain_unmap(domain, start_pfn, last_pfn);
3747 if (intel_iommu_strict) {
3748 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3749 nrpages, !freelist, 0);
3751 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3752 dma_free_pagelist(freelist);
3754 queue_iova(&domain->iovad, iova_pfn, nrpages,
3755 (unsigned long)freelist);
3757 * queue up the release of the unmap to save the 1/6th of the
3758 * cpu used up by the iotlb flush operation...
3763 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3764 size_t size, enum dma_data_direction dir,
3765 unsigned long attrs)
3767 intel_unmap(dev, dev_addr, size);
3770 static void *intel_alloc_coherent(struct device *dev, size_t size,
3771 dma_addr_t *dma_handle, gfp_t flags,
3772 unsigned long attrs)
3774 struct page *page = NULL;
3777 size = PAGE_ALIGN(size);
3778 order = get_order(size);
3780 if (!iommu_no_mapping(dev))
3781 flags &= ~(GFP_DMA | GFP_DMA32);
3782 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3783 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3789 if (gfpflags_allow_blocking(flags)) {
3790 unsigned int count = size >> PAGE_SHIFT;
3792 page = dma_alloc_from_contiguous(dev, count, order,
3793 flags & __GFP_NOWARN);
3794 if (page && iommu_no_mapping(dev) &&
3795 page_to_phys(page) + size > dev->coherent_dma_mask) {
3796 dma_release_from_contiguous(dev, page, count);
3802 page = alloc_pages(flags, order);
3805 memset(page_address(page), 0, size);
3807 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3809 dev->coherent_dma_mask);
3810 if (*dma_handle != DMA_MAPPING_ERROR)
3811 return page_address(page);
3812 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3813 __free_pages(page, order);
3818 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3819 dma_addr_t dma_handle, unsigned long attrs)
3822 struct page *page = virt_to_page(vaddr);
3824 size = PAGE_ALIGN(size);
3825 order = get_order(size);
3827 intel_unmap(dev, dma_handle, size);
3828 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3829 __free_pages(page, order);
3832 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3833 int nelems, enum dma_data_direction dir,
3834 unsigned long attrs)
3836 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3837 unsigned long nrpages = 0;
3838 struct scatterlist *sg;
3841 for_each_sg(sglist, sg, nelems, i) {
3842 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3845 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3848 static int intel_nontranslate_map_sg(struct device *hddev,
3849 struct scatterlist *sglist, int nelems, int dir)
3852 struct scatterlist *sg;
3854 for_each_sg(sglist, sg, nelems, i) {
3855 BUG_ON(!sg_page(sg));
3856 sg->dma_address = sg_phys(sg);
3857 sg->dma_length = sg->length;
3862 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3863 enum dma_data_direction dir, unsigned long attrs)
3866 struct dmar_domain *domain;
3869 unsigned long iova_pfn;
3871 struct scatterlist *sg;
3872 unsigned long start_vpfn;
3873 struct intel_iommu *iommu;
3875 BUG_ON(dir == DMA_NONE);
3876 if (iommu_no_mapping(dev))
3877 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3879 domain = get_valid_domain_for_dev(dev);
3883 iommu = domain_get_iommu(domain);
3885 for_each_sg(sglist, sg, nelems, i)
3886 size += aligned_nrpages(sg->offset, sg->length);
3888 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3891 sglist->dma_length = 0;
3896 * Check if DMAR supports zero-length reads on write only
3899 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3900 !cap_zlr(iommu->cap))
3901 prot |= DMA_PTE_READ;
3902 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3903 prot |= DMA_PTE_WRITE;
3905 start_vpfn = mm_to_dma_pfn(iova_pfn);
3907 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3908 if (unlikely(ret)) {
3909 dma_pte_free_pagetable(domain, start_vpfn,
3910 start_vpfn + size - 1,
3911 agaw_to_level(domain->agaw) + 1);
3912 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3919 static const struct dma_map_ops intel_dma_ops = {
3920 .alloc = intel_alloc_coherent,
3921 .free = intel_free_coherent,
3922 .map_sg = intel_map_sg,
3923 .unmap_sg = intel_unmap_sg,
3924 .map_page = intel_map_page,
3925 .unmap_page = intel_unmap_page,
3926 .map_resource = intel_map_resource,
3927 .unmap_resource = intel_unmap_page,
3928 .dma_supported = dma_direct_supported,
3931 static inline int iommu_domain_cache_init(void)
3935 iommu_domain_cache = kmem_cache_create("iommu_domain",
3936 sizeof(struct dmar_domain),
3941 if (!iommu_domain_cache) {
3942 pr_err("Couldn't create iommu_domain cache\n");
3949 static inline int iommu_devinfo_cache_init(void)
3953 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3954 sizeof(struct device_domain_info),
3958 if (!iommu_devinfo_cache) {
3959 pr_err("Couldn't create devinfo cache\n");
3966 static int __init iommu_init_mempool(void)
3969 ret = iova_cache_get();
3973 ret = iommu_domain_cache_init();
3977 ret = iommu_devinfo_cache_init();
3981 kmem_cache_destroy(iommu_domain_cache);
3988 static void __init iommu_exit_mempool(void)
3990 kmem_cache_destroy(iommu_devinfo_cache);
3991 kmem_cache_destroy(iommu_domain_cache);
3995 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3997 struct dmar_drhd_unit *drhd;
4001 /* We know that this device on this chipset has its own IOMMU.
4002 * If we find it under a different IOMMU, then the BIOS is lying
4003 * to us. Hope that the IOMMU for this device is actually
4004 * disabled, and it needs no translation...
4006 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4008 /* "can't" happen */
4009 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4012 vtbar &= 0xffff0000;
4014 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4015 drhd = dmar_find_matched_drhd_unit(pdev);
4016 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4017 TAINT_FIRMWARE_WORKAROUND,
4018 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4019 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4021 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4023 static void __init init_no_remapping_devices(void)
4025 struct dmar_drhd_unit *drhd;
4029 for_each_drhd_unit(drhd) {
4030 if (!drhd->include_all) {
4031 for_each_active_dev_scope(drhd->devices,
4032 drhd->devices_cnt, i, dev)
4034 /* ignore DMAR unit if no devices exist */
4035 if (i == drhd->devices_cnt)
4040 for_each_active_drhd_unit(drhd) {
4041 if (drhd->include_all)
4044 for_each_active_dev_scope(drhd->devices,
4045 drhd->devices_cnt, i, dev)
4046 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4048 if (i < drhd->devices_cnt)
4051 /* This IOMMU has *only* gfx devices. Either bypass it or
4052 set the gfx_mapped flag, as appropriate */
4054 intel_iommu_gfx_mapped = 1;
4057 for_each_active_dev_scope(drhd->devices,
4058 drhd->devices_cnt, i, dev)
4059 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4064 #ifdef CONFIG_SUSPEND
4065 static int init_iommu_hw(void)
4067 struct dmar_drhd_unit *drhd;
4068 struct intel_iommu *iommu = NULL;
4070 for_each_active_iommu(iommu, drhd)
4072 dmar_reenable_qi(iommu);
4074 for_each_iommu(iommu, drhd) {
4075 if (drhd->ignored) {
4077 * we always have to disable PMRs or DMA may fail on
4081 iommu_disable_protect_mem_regions(iommu);
4085 iommu_flush_write_buffer(iommu);
4087 iommu_set_root_entry(iommu);
4089 iommu->flush.flush_context(iommu, 0, 0, 0,
4090 DMA_CCMD_GLOBAL_INVL);
4091 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4092 iommu_enable_translation(iommu);
4093 iommu_disable_protect_mem_regions(iommu);
4099 static void iommu_flush_all(void)
4101 struct dmar_drhd_unit *drhd;
4102 struct intel_iommu *iommu;
4104 for_each_active_iommu(iommu, drhd) {
4105 iommu->flush.flush_context(iommu, 0, 0, 0,
4106 DMA_CCMD_GLOBAL_INVL);
4107 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4108 DMA_TLB_GLOBAL_FLUSH);
4112 static int iommu_suspend(void)
4114 struct dmar_drhd_unit *drhd;
4115 struct intel_iommu *iommu = NULL;
4118 for_each_active_iommu(iommu, drhd) {
4119 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4121 if (!iommu->iommu_state)
4127 for_each_active_iommu(iommu, drhd) {
4128 iommu_disable_translation(iommu);
4130 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4132 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4133 readl(iommu->reg + DMAR_FECTL_REG);
4134 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4135 readl(iommu->reg + DMAR_FEDATA_REG);
4136 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4137 readl(iommu->reg + DMAR_FEADDR_REG);
4138 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4139 readl(iommu->reg + DMAR_FEUADDR_REG);
4141 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4146 for_each_active_iommu(iommu, drhd)
4147 kfree(iommu->iommu_state);
4152 static void iommu_resume(void)
4154 struct dmar_drhd_unit *drhd;
4155 struct intel_iommu *iommu = NULL;
4158 if (init_iommu_hw()) {
4160 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4162 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4166 for_each_active_iommu(iommu, drhd) {
4168 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4170 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4171 iommu->reg + DMAR_FECTL_REG);
4172 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4173 iommu->reg + DMAR_FEDATA_REG);
4174 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4175 iommu->reg + DMAR_FEADDR_REG);
4176 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4177 iommu->reg + DMAR_FEUADDR_REG);
4179 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4182 for_each_active_iommu(iommu, drhd)
4183 kfree(iommu->iommu_state);
4186 static struct syscore_ops iommu_syscore_ops = {
4187 .resume = iommu_resume,
4188 .suspend = iommu_suspend,
4191 static void __init init_iommu_pm_ops(void)
4193 register_syscore_ops(&iommu_syscore_ops);
4197 static inline void init_iommu_pm_ops(void) {}
4198 #endif /* CONFIG_PM */
4201 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4203 struct acpi_dmar_reserved_memory *rmrr;
4204 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4205 struct dmar_rmrr_unit *rmrru;
4208 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4212 rmrru->hdr = header;
4213 rmrr = (struct acpi_dmar_reserved_memory *)header;
4214 rmrru->base_address = rmrr->base_address;
4215 rmrru->end_address = rmrr->end_address;
4217 length = rmrr->end_address - rmrr->base_address + 1;
4218 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4223 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4224 ((void *)rmrr) + rmrr->header.length,
4225 &rmrru->devices_cnt);
4226 if (rmrru->devices_cnt && rmrru->devices == NULL)
4229 list_add(&rmrru->list, &dmar_rmrr_units);
4240 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4242 struct dmar_atsr_unit *atsru;
4243 struct acpi_dmar_atsr *tmp;
4245 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4246 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4247 if (atsr->segment != tmp->segment)
4249 if (atsr->header.length != tmp->header.length)
4251 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4258 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4260 struct acpi_dmar_atsr *atsr;
4261 struct dmar_atsr_unit *atsru;
4263 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4266 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4267 atsru = dmar_find_atsr(atsr);
4271 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4276 * If memory is allocated from slab by ACPI _DSM method, we need to
4277 * copy the memory content because the memory buffer will be freed
4280 atsru->hdr = (void *)(atsru + 1);
4281 memcpy(atsru->hdr, hdr, hdr->length);
4282 atsru->include_all = atsr->flags & 0x1;
4283 if (!atsru->include_all) {
4284 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4285 (void *)atsr + atsr->header.length,
4286 &atsru->devices_cnt);
4287 if (atsru->devices_cnt && atsru->devices == NULL) {
4293 list_add_rcu(&atsru->list, &dmar_atsr_units);
4298 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4300 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4304 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4306 struct acpi_dmar_atsr *atsr;
4307 struct dmar_atsr_unit *atsru;
4309 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4310 atsru = dmar_find_atsr(atsr);
4312 list_del_rcu(&atsru->list);
4314 intel_iommu_free_atsr(atsru);
4320 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4324 struct acpi_dmar_atsr *atsr;
4325 struct dmar_atsr_unit *atsru;
4327 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4328 atsru = dmar_find_atsr(atsr);
4332 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4333 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4341 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4344 struct intel_iommu *iommu = dmaru->iommu;
4346 if (g_iommus[iommu->seq_id])
4349 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4350 pr_warn("%s: Doesn't support hardware pass through.\n",
4354 if (!ecap_sc_support(iommu->ecap) &&
4355 domain_update_iommu_snooping(iommu)) {
4356 pr_warn("%s: Doesn't support snooping.\n",
4360 sp = domain_update_iommu_superpage(iommu) - 1;
4361 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4362 pr_warn("%s: Doesn't support large page.\n",
4368 * Disable translation if already enabled prior to OS handover.
4370 if (iommu->gcmd & DMA_GCMD_TE)
4371 iommu_disable_translation(iommu);
4373 g_iommus[iommu->seq_id] = iommu;
4374 ret = iommu_init_domains(iommu);
4376 ret = iommu_alloc_root_entry(iommu);
4380 #ifdef CONFIG_INTEL_IOMMU_SVM
4381 if (pasid_supported(iommu))
4382 intel_svm_init(iommu);
4385 if (dmaru->ignored) {
4387 * we always have to disable PMRs or DMA may fail on this device
4390 iommu_disable_protect_mem_regions(iommu);
4394 intel_iommu_init_qi(iommu);
4395 iommu_flush_write_buffer(iommu);
4397 #ifdef CONFIG_INTEL_IOMMU_SVM
4398 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
4399 ret = intel_svm_enable_prq(iommu);
4404 ret = dmar_set_interrupt(iommu);
4408 iommu_set_root_entry(iommu);
4409 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4410 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4411 iommu_enable_translation(iommu);
4413 iommu_disable_protect_mem_regions(iommu);
4417 disable_dmar_iommu(iommu);
4419 free_dmar_iommu(iommu);
4423 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4426 struct intel_iommu *iommu = dmaru->iommu;
4428 if (!intel_iommu_enabled)
4434 ret = intel_iommu_add(dmaru);
4436 disable_dmar_iommu(iommu);
4437 free_dmar_iommu(iommu);
4443 static void intel_iommu_free_dmars(void)
4445 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4446 struct dmar_atsr_unit *atsru, *atsr_n;
4448 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4449 list_del(&rmrru->list);
4450 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4455 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4456 list_del(&atsru->list);
4457 intel_iommu_free_atsr(atsru);
4461 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4464 struct pci_bus *bus;
4465 struct pci_dev *bridge = NULL;
4467 struct acpi_dmar_atsr *atsr;
4468 struct dmar_atsr_unit *atsru;
4470 dev = pci_physfn(dev);
4471 for (bus = dev->bus; bus; bus = bus->parent) {
4473 /* If it's an integrated device, allow ATS */
4476 /* Connected via non-PCIe: no ATS */
4477 if (!pci_is_pcie(bridge) ||
4478 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4480 /* If we found the root port, look it up in the ATSR */
4481 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4486 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4487 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4488 if (atsr->segment != pci_domain_nr(dev->bus))
4491 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4492 if (tmp == &bridge->dev)
4495 if (atsru->include_all)
4505 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4508 struct dmar_rmrr_unit *rmrru;
4509 struct dmar_atsr_unit *atsru;
4510 struct acpi_dmar_atsr *atsr;
4511 struct acpi_dmar_reserved_memory *rmrr;
4513 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4516 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4517 rmrr = container_of(rmrru->hdr,
4518 struct acpi_dmar_reserved_memory, header);
4519 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4520 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4521 ((void *)rmrr) + rmrr->header.length,
4522 rmrr->segment, rmrru->devices,
4523 rmrru->devices_cnt);
4526 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4527 dmar_remove_dev_scope(info, rmrr->segment,
4528 rmrru->devices, rmrru->devices_cnt);
4532 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4533 if (atsru->include_all)
4536 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4537 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4538 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4539 (void *)atsr + atsr->header.length,
4540 atsr->segment, atsru->devices,
4541 atsru->devices_cnt);
4546 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4547 if (dmar_remove_dev_scope(info, atsr->segment,
4548 atsru->devices, atsru->devices_cnt))
4557 * Here we only respond to action of unbound device from driver.
4559 * Added device is not attached to its DMAR domain here yet. That will happen
4560 * when mapping the device to iova.
4562 static int device_notifier(struct notifier_block *nb,
4563 unsigned long action, void *data)
4565 struct device *dev = data;
4566 struct dmar_domain *domain;
4568 if (iommu_dummy(dev))
4571 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4574 domain = find_domain(dev);
4578 dmar_remove_one_dev_info(dev);
4579 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4580 domain_exit(domain);
4585 static struct notifier_block device_nb = {
4586 .notifier_call = device_notifier,
4589 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4590 unsigned long val, void *v)
4592 struct memory_notify *mhp = v;
4593 unsigned long long start, end;
4594 unsigned long start_vpfn, last_vpfn;
4597 case MEM_GOING_ONLINE:
4598 start = mhp->start_pfn << PAGE_SHIFT;
4599 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4600 if (iommu_domain_identity_map(si_domain, start, end)) {
4601 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4608 case MEM_CANCEL_ONLINE:
4609 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4610 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4611 while (start_vpfn <= last_vpfn) {
4613 struct dmar_drhd_unit *drhd;
4614 struct intel_iommu *iommu;
4615 struct page *freelist;
4617 iova = find_iova(&si_domain->iovad, start_vpfn);
4619 pr_debug("Failed get IOVA for PFN %lx\n",
4624 iova = split_and_remove_iova(&si_domain->iovad, iova,
4625 start_vpfn, last_vpfn);
4627 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4628 start_vpfn, last_vpfn);
4632 freelist = domain_unmap(si_domain, iova->pfn_lo,
4636 for_each_active_iommu(iommu, drhd)
4637 iommu_flush_iotlb_psi(iommu, si_domain,
4638 iova->pfn_lo, iova_size(iova),
4641 dma_free_pagelist(freelist);
4643 start_vpfn = iova->pfn_hi + 1;
4644 free_iova_mem(iova);
4652 static struct notifier_block intel_iommu_memory_nb = {
4653 .notifier_call = intel_iommu_memory_notifier,
4657 static void free_all_cpu_cached_iovas(unsigned int cpu)
4661 for (i = 0; i < g_num_of_iommus; i++) {
4662 struct intel_iommu *iommu = g_iommus[i];
4663 struct dmar_domain *domain;
4669 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4670 domain = get_iommu_domain(iommu, (u16)did);
4674 free_cpu_cached_iovas(cpu, &domain->iovad);
4679 static int intel_iommu_cpu_dead(unsigned int cpu)
4681 free_all_cpu_cached_iovas(cpu);
4685 static void intel_disable_iommus(void)
4687 struct intel_iommu *iommu = NULL;
4688 struct dmar_drhd_unit *drhd;
4690 for_each_iommu(iommu, drhd)
4691 iommu_disable_translation(iommu);
4694 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4696 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4698 return container_of(iommu_dev, struct intel_iommu, iommu);
4701 static ssize_t intel_iommu_show_version(struct device *dev,
4702 struct device_attribute *attr,
4705 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4706 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4707 return sprintf(buf, "%d:%d\n",
4708 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4710 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4712 static ssize_t intel_iommu_show_address(struct device *dev,
4713 struct device_attribute *attr,
4716 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4717 return sprintf(buf, "%llx\n", iommu->reg_phys);
4719 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4721 static ssize_t intel_iommu_show_cap(struct device *dev,
4722 struct device_attribute *attr,
4725 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4726 return sprintf(buf, "%llx\n", iommu->cap);
4728 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4730 static ssize_t intel_iommu_show_ecap(struct device *dev,
4731 struct device_attribute *attr,
4734 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4735 return sprintf(buf, "%llx\n", iommu->ecap);
4737 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4739 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4740 struct device_attribute *attr,
4743 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4744 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4746 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4748 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4749 struct device_attribute *attr,
4752 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4753 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4754 cap_ndoms(iommu->cap)));
4756 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4758 static struct attribute *intel_iommu_attrs[] = {
4759 &dev_attr_version.attr,
4760 &dev_attr_address.attr,
4762 &dev_attr_ecap.attr,
4763 &dev_attr_domains_supported.attr,
4764 &dev_attr_domains_used.attr,
4768 static struct attribute_group intel_iommu_group = {
4769 .name = "intel-iommu",
4770 .attrs = intel_iommu_attrs,
4773 const struct attribute_group *intel_iommu_groups[] = {
4778 static int __init platform_optin_force_iommu(void)
4780 struct pci_dev *pdev = NULL;
4781 bool has_untrusted_dev = false;
4783 if (!dmar_platform_optin() || no_platform_optin)
4786 for_each_pci_dev(pdev) {
4787 if (pdev->untrusted) {
4788 has_untrusted_dev = true;
4793 if (!has_untrusted_dev)
4796 if (no_iommu || dmar_disabled)
4797 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
4800 * If Intel-IOMMU is disabled by default, we will apply identity
4801 * map for all devices except those marked as being untrusted.
4804 iommu_identity_mapping |= IDENTMAP_ALL;
4807 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4815 int __init intel_iommu_init(void)
4818 struct dmar_drhd_unit *drhd;
4819 struct intel_iommu *iommu;
4822 * Intel IOMMU is required for a TXT/tboot launch or platform
4823 * opt in, so enforce that.
4825 force_on = tboot_force_iommu() || platform_optin_force_iommu();
4827 if (iommu_init_mempool()) {
4829 panic("tboot: Failed to initialize iommu memory\n");
4833 down_write(&dmar_global_lock);
4834 if (dmar_table_init()) {
4836 panic("tboot: Failed to initialize DMAR table\n");
4840 if (dmar_dev_scope_init() < 0) {
4842 panic("tboot: Failed to initialize DMAR device scope\n");
4846 up_write(&dmar_global_lock);
4849 * The bus notifier takes the dmar_global_lock, so lockdep will
4850 * complain later when we register it under the lock.
4852 dmar_register_bus_notifier();
4854 down_write(&dmar_global_lock);
4856 if (no_iommu || dmar_disabled) {
4858 * We exit the function here to ensure IOMMU's remapping and
4859 * mempool aren't setup, which means that the IOMMU's PMRs
4860 * won't be disabled via the call to init_dmars(). So disable
4861 * it explicitly here. The PMRs were setup by tboot prior to
4862 * calling SENTER, but the kernel is expected to reset/tear
4865 if (intel_iommu_tboot_noforce) {
4866 for_each_iommu(iommu, drhd)
4867 iommu_disable_protect_mem_regions(iommu);
4871 * Make sure the IOMMUs are switched off, even when we
4872 * boot into a kexec kernel and the previous kernel left
4875 intel_disable_iommus();
4879 if (list_empty(&dmar_rmrr_units))
4880 pr_info("No RMRR found\n");
4882 if (list_empty(&dmar_atsr_units))
4883 pr_info("No ATSR found\n");
4885 if (dmar_init_reserved_ranges()) {
4887 panic("tboot: Failed to reserve iommu ranges\n");
4888 goto out_free_reserved_range;
4891 init_no_remapping_devices();
4896 panic("tboot: Failed to initialize DMARs\n");
4897 pr_err("Initialization failed\n");
4898 goto out_free_reserved_range;
4900 up_write(&dmar_global_lock);
4901 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4903 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4906 dma_ops = &intel_dma_ops;
4908 init_iommu_pm_ops();
4910 for_each_active_iommu(iommu, drhd) {
4911 iommu_device_sysfs_add(&iommu->iommu, NULL,
4914 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4915 iommu_device_register(&iommu->iommu);
4918 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4919 bus_register_notifier(&pci_bus_type, &device_nb);
4920 if (si_domain && !hw_pass_through)
4921 register_memory_notifier(&intel_iommu_memory_nb);
4922 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4923 intel_iommu_cpu_dead);
4924 intel_iommu_enabled = 1;
4925 intel_iommu_debugfs_init();
4929 out_free_reserved_range:
4930 put_iova_domain(&reserved_iova_list);
4932 intel_iommu_free_dmars();
4933 up_write(&dmar_global_lock);
4934 iommu_exit_mempool();
4938 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4940 struct intel_iommu *iommu = opaque;
4942 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4947 * NB - intel-iommu lacks any sort of reference counting for the users of
4948 * dependent devices. If multiple endpoints have intersecting dependent
4949 * devices, unbinding the driver from any one of them will possibly leave
4950 * the others unable to operate.
4952 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4954 if (!iommu || !dev || !dev_is_pci(dev))
4957 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4960 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4962 struct intel_iommu *iommu;
4963 unsigned long flags;
4965 assert_spin_locked(&device_domain_lock);
4970 iommu = info->iommu;
4973 if (dev_is_pci(info->dev) && sm_supported(iommu))
4974 intel_pasid_tear_down_entry(iommu, info->dev,
4977 iommu_disable_dev_iotlb(info);
4978 domain_context_clear(iommu, info->dev);
4979 intel_pasid_free_table(info->dev);
4982 unlink_domain_info(info);
4984 spin_lock_irqsave(&iommu->lock, flags);
4985 domain_detach_iommu(info->domain, iommu);
4986 spin_unlock_irqrestore(&iommu->lock, flags);
4988 free_devinfo_mem(info);
4991 static void dmar_remove_one_dev_info(struct device *dev)
4993 struct device_domain_info *info;
4994 unsigned long flags;
4996 spin_lock_irqsave(&device_domain_lock, flags);
4997 info = dev->archdata.iommu;
4998 __dmar_remove_one_dev_info(info);
4999 spin_unlock_irqrestore(&device_domain_lock, flags);
5002 static int md_domain_init(struct dmar_domain *domain, int guest_width)
5006 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5007 domain_reserve_special_ranges(domain);
5009 /* calculate AGAW */
5010 domain->gaw = guest_width;
5011 adjust_width = guestwidth_to_adjustwidth(guest_width);
5012 domain->agaw = width_to_agaw(adjust_width);
5014 domain->iommu_coherency = 0;
5015 domain->iommu_snooping = 0;
5016 domain->iommu_superpage = 0;
5017 domain->max_addr = 0;
5019 /* always allocate the top pgd */
5020 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5023 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5027 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5029 struct dmar_domain *dmar_domain;
5030 struct iommu_domain *domain;
5032 if (type != IOMMU_DOMAIN_UNMANAGED)
5035 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5037 pr_err("Can't allocate dmar_domain\n");
5040 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5041 pr_err("Domain initialization failed\n");
5042 domain_exit(dmar_domain);
5045 domain_update_iommu_cap(dmar_domain);
5047 domain = &dmar_domain->domain;
5048 domain->geometry.aperture_start = 0;
5049 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5050 domain->geometry.force_aperture = true;
5055 static void intel_iommu_domain_free(struct iommu_domain *domain)
5057 domain_exit(to_dmar_domain(domain));
5060 static int intel_iommu_attach_device(struct iommu_domain *domain,
5063 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5064 struct intel_iommu *iommu;
5068 if (device_is_rmrr_locked(dev)) {
5069 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5073 /* normally dev is not mapped */
5074 if (unlikely(domain_context_mapped(dev))) {
5075 struct dmar_domain *old_domain;
5077 old_domain = find_domain(dev);
5080 dmar_remove_one_dev_info(dev);
5083 if (!domain_type_is_vm_or_si(old_domain) &&
5084 list_empty(&old_domain->devices))
5085 domain_exit(old_domain);
5089 iommu = device_to_iommu(dev, &bus, &devfn);
5093 /* check if this iommu agaw is sufficient for max mapped address */
5094 addr_width = agaw_to_width(iommu->agaw);
5095 if (addr_width > cap_mgaw(iommu->cap))
5096 addr_width = cap_mgaw(iommu->cap);
5098 if (dmar_domain->max_addr > (1LL << addr_width)) {
5099 dev_err(dev, "%s: iommu width (%d) is not "
5100 "sufficient for the mapped address (%llx)\n",
5101 __func__, addr_width, dmar_domain->max_addr);
5104 dmar_domain->gaw = addr_width;
5107 * Knock out extra levels of page tables if necessary
5109 while (iommu->agaw < dmar_domain->agaw) {
5110 struct dma_pte *pte;
5112 pte = dmar_domain->pgd;
5113 if (dma_pte_present(pte)) {
5114 dmar_domain->pgd = (struct dma_pte *)
5115 phys_to_virt(dma_pte_addr(pte));
5116 free_pgtable_page(pte);
5118 dmar_domain->agaw--;
5121 return domain_add_dev_info(dmar_domain, dev);
5124 static void intel_iommu_detach_device(struct iommu_domain *domain,
5127 dmar_remove_one_dev_info(dev);
5130 static int intel_iommu_map(struct iommu_domain *domain,
5131 unsigned long iova, phys_addr_t hpa,
5132 size_t size, int iommu_prot)
5134 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5139 if (iommu_prot & IOMMU_READ)
5140 prot |= DMA_PTE_READ;
5141 if (iommu_prot & IOMMU_WRITE)
5142 prot |= DMA_PTE_WRITE;
5143 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5144 prot |= DMA_PTE_SNP;
5146 max_addr = iova + size;
5147 if (dmar_domain->max_addr < max_addr) {
5150 /* check if minimum agaw is sufficient for mapped address */
5151 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5152 if (end < max_addr) {
5153 pr_err("%s: iommu width (%d) is not "
5154 "sufficient for the mapped address (%llx)\n",
5155 __func__, dmar_domain->gaw, max_addr);
5158 dmar_domain->max_addr = max_addr;
5160 /* Round up size to next multiple of PAGE_SIZE, if it and
5161 the low bits of hpa would take us onto the next page */
5162 size = aligned_nrpages(hpa, size);
5163 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5164 hpa >> VTD_PAGE_SHIFT, size, prot);
5168 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5169 unsigned long iova, size_t size)
5171 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5172 struct page *freelist = NULL;
5173 unsigned long start_pfn, last_pfn;
5174 unsigned int npages;
5175 int iommu_id, level = 0;
5177 /* Cope with horrid API which requires us to unmap more than the
5178 size argument if it happens to be a large-page mapping. */
5179 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5181 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5182 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5184 start_pfn = iova >> VTD_PAGE_SHIFT;
5185 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5187 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5189 npages = last_pfn - start_pfn + 1;
5191 for_each_domain_iommu(iommu_id, dmar_domain)
5192 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5193 start_pfn, npages, !freelist, 0);
5195 dma_free_pagelist(freelist);
5197 if (dmar_domain->max_addr == iova + size)
5198 dmar_domain->max_addr = iova;
5203 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5206 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5207 struct dma_pte *pte;
5211 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5213 phys = dma_pte_addr(pte);
5218 static bool intel_iommu_capable(enum iommu_cap cap)
5220 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5221 return domain_update_iommu_snooping(NULL) == 1;
5222 if (cap == IOMMU_CAP_INTR_REMAP)
5223 return irq_remapping_enabled == 1;
5228 static int intel_iommu_add_device(struct device *dev)
5230 struct intel_iommu *iommu;
5231 struct iommu_group *group;
5234 iommu = device_to_iommu(dev, &bus, &devfn);
5238 iommu_device_link(&iommu->iommu, dev);
5240 group = iommu_group_get_for_dev(dev);
5243 return PTR_ERR(group);
5245 iommu_group_put(group);
5249 static void intel_iommu_remove_device(struct device *dev)
5251 struct intel_iommu *iommu;
5254 iommu = device_to_iommu(dev, &bus, &devfn);
5258 iommu_group_remove_device(dev);
5260 iommu_device_unlink(&iommu->iommu, dev);
5263 static void intel_iommu_get_resv_regions(struct device *device,
5264 struct list_head *head)
5266 struct iommu_resv_region *reg;
5267 struct dmar_rmrr_unit *rmrr;
5268 struct device *i_dev;
5272 for_each_rmrr_units(rmrr) {
5273 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5275 if (i_dev != device)
5278 list_add_tail(&rmrr->resv->list, head);
5283 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5284 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5288 list_add_tail(®->list, head);
5291 static void intel_iommu_put_resv_regions(struct device *dev,
5292 struct list_head *head)
5294 struct iommu_resv_region *entry, *next;
5296 list_for_each_entry_safe(entry, next, head, list) {
5297 if (entry->type == IOMMU_RESV_RESERVED)
5302 #ifdef CONFIG_INTEL_IOMMU_SVM
5303 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5305 struct device_domain_info *info;
5306 struct context_entry *context;
5307 struct dmar_domain *domain;
5308 unsigned long flags;
5312 domain = get_valid_domain_for_dev(sdev->dev);
5316 spin_lock_irqsave(&device_domain_lock, flags);
5317 spin_lock(&iommu->lock);
5320 info = sdev->dev->archdata.iommu;
5321 if (!info || !info->pasid_supported)
5324 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5325 if (WARN_ON(!context))
5328 ctx_lo = context[0].lo;
5330 sdev->did = domain->iommu_did[iommu->seq_id];
5331 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5333 if (!(ctx_lo & CONTEXT_PASIDE)) {
5334 ctx_lo |= CONTEXT_PASIDE;
5335 context[0].lo = ctx_lo;
5337 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5338 DMA_CCMD_MASK_NOBIT,
5339 DMA_CCMD_DEVICE_INVL);
5342 /* Enable PASID support in the device, if it wasn't already */
5343 if (!info->pasid_enabled)
5344 iommu_enable_dev_iotlb(info);
5346 if (info->ats_enabled) {
5347 sdev->dev_iotlb = 1;
5348 sdev->qdep = info->ats_qdep;
5349 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5355 spin_unlock(&iommu->lock);
5356 spin_unlock_irqrestore(&device_domain_lock, flags);
5361 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5363 struct intel_iommu *iommu;
5366 if (iommu_dummy(dev)) {
5368 "No IOMMU translation for device; cannot enable SVM\n");
5372 iommu = device_to_iommu(dev, &bus, &devfn);
5374 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5380 #endif /* CONFIG_INTEL_IOMMU_SVM */
5382 const struct iommu_ops intel_iommu_ops = {
5383 .capable = intel_iommu_capable,
5384 .domain_alloc = intel_iommu_domain_alloc,
5385 .domain_free = intel_iommu_domain_free,
5386 .attach_dev = intel_iommu_attach_device,
5387 .detach_dev = intel_iommu_detach_device,
5388 .map = intel_iommu_map,
5389 .unmap = intel_iommu_unmap,
5390 .iova_to_phys = intel_iommu_iova_to_phys,
5391 .add_device = intel_iommu_add_device,
5392 .remove_device = intel_iommu_remove_device,
5393 .get_resv_regions = intel_iommu_get_resv_regions,
5394 .put_resv_regions = intel_iommu_put_resv_regions,
5395 .device_group = pci_device_group,
5396 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5399 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5401 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5402 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
5406 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5407 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5408 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5414 static void quirk_iommu_rwbf(struct pci_dev *dev)
5417 * Mobile 4 Series Chipset neglects to set RWBF capability,
5418 * but needs it. Same seems to hold for the desktop versions.
5420 pci_info(dev, "Forcing write-buffer flush capability\n");
5424 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5428 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5429 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5430 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5433 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5434 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5435 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5436 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5437 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5438 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5439 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5440 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5442 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5446 if (pci_read_config_word(dev, GGC, &ggc))
5449 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5450 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5452 } else if (dmar_map_gfx) {
5453 /* we have to ensure the gfx device is idle before we flush */
5454 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
5455 intel_iommu_strict = 1;
5458 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5459 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5460 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5461 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5463 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5464 ISOCH DMAR unit for the Azalia sound device, but not give it any
5465 TLB entries, which causes it to deadlock. Check for that. We do
5466 this in a function called from init_dmars(), instead of in a PCI
5467 quirk, because we don't want to print the obnoxious "BIOS broken"
5468 message if VT-d is actually disabled.
5470 static void __init check_tylersburg_isoch(void)
5472 struct pci_dev *pdev;
5473 uint32_t vtisochctrl;
5475 /* If there's no Azalia in the system anyway, forget it. */
5476 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5481 /* System Management Registers. Might be hidden, in which case
5482 we can't do the sanity check. But that's OK, because the
5483 known-broken BIOSes _don't_ actually hide it, so far. */
5484 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5488 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5495 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5496 if (vtisochctrl & 1)
5499 /* Drop all bits other than the number of TLB entries */
5500 vtisochctrl &= 0x1c;
5502 /* If we have the recommended number of TLB entries (16), fine. */
5503 if (vtisochctrl == 0x10)
5506 /* Zero TLB entries? You get to ride the short bus to school. */
5508 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5509 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5510 dmi_get_system_info(DMI_BIOS_VENDOR),
5511 dmi_get_system_info(DMI_BIOS_VERSION),
5512 dmi_get_system_info(DMI_PRODUCT_VERSION));
5513 iommu_identity_mapping |= IDENTMAP_AZALIA;
5517 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",