Merge branches 'arm/rockchip', 'arm/exynos', 'arm/smmu', 'x86/vt-d', 'x86/amd', ...
[sfrench/cifs-2.6.git] / drivers / iommu / intel-iommu.c
1 /*
2  * Copyright © 2006-2014 Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * Authors: David Woodhouse <dwmw2@infradead.org>,
14  *          Ashok Raj <ashok.raj@intel.com>,
15  *          Shaohua Li <shaohua.li@intel.com>,
16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17  *          Fenghua Yu <fenghua.yu@intel.com>
18  *          Joerg Roedel <jroedel@suse.de>
19  */
20
21 #define pr_fmt(fmt)     "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/timer.h>
37 #include <linux/iova.h>
38 #include <linux/iommu.h>
39 #include <linux/intel-iommu.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/tboot.h>
42 #include <linux/dmi.h>
43 #include <linux/pci-ats.h>
44 #include <linux/memblock.h>
45 #include <linux/dma-contiguous.h>
46 #include <linux/crash_dump.h>
47 #include <asm/irq_remapping.h>
48 #include <asm/cacheflush.h>
49 #include <asm/iommu.h>
50
51 #include "irq_remapping.h"
52
53 #define ROOT_SIZE               VTD_PAGE_SIZE
54 #define CONTEXT_SIZE            VTD_PAGE_SIZE
55
56 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
57 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
58 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
59 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
60
61 #define IOAPIC_RANGE_START      (0xfee00000)
62 #define IOAPIC_RANGE_END        (0xfeefffff)
63 #define IOVA_START_ADDR         (0x1000)
64
65 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
67 #define MAX_AGAW_WIDTH 64
68 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
69
70 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
76                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
78
79 /* IO virtual address start page frame number */
80 #define IOVA_START_PFN          (1)
81
82 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
83 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
84 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
85
86 /* page table handling */
87 #define LEVEL_STRIDE            (9)
88 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
89
90 /*
91  * This bitmap is used to advertise the page sizes our hardware support
92  * to the IOMMU core, which will then use this information to split
93  * physically contiguous memory regions it is mapping into page sizes
94  * that we support.
95  *
96  * Traditionally the IOMMU core just handed us the mappings directly,
97  * after making sure the size is an order of a 4KiB page and that the
98  * mapping has natural alignment.
99  *
100  * To retain this behavior, we currently advertise that we support
101  * all page sizes that are an order of 4KiB.
102  *
103  * If at some point we'd like to utilize the IOMMU core's new behavior,
104  * we could change this to advertise the real page sizes we support.
105  */
106 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
107
108 static inline int agaw_to_level(int agaw)
109 {
110         return agaw + 2;
111 }
112
113 static inline int agaw_to_width(int agaw)
114 {
115         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
116 }
117
118 static inline int width_to_agaw(int width)
119 {
120         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
121 }
122
123 static inline unsigned int level_to_offset_bits(int level)
124 {
125         return (level - 1) * LEVEL_STRIDE;
126 }
127
128 static inline int pfn_level_offset(unsigned long pfn, int level)
129 {
130         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131 }
132
133 static inline unsigned long level_mask(int level)
134 {
135         return -1UL << level_to_offset_bits(level);
136 }
137
138 static inline unsigned long level_size(int level)
139 {
140         return 1UL << level_to_offset_bits(level);
141 }
142
143 static inline unsigned long align_to_level(unsigned long pfn, int level)
144 {
145         return (pfn + level_size(level) - 1) & level_mask(level);
146 }
147
148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149 {
150         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
151 }
152
153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154    are never going to work. */
155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156 {
157         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158 }
159
160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161 {
162         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163 }
164 static inline unsigned long page_to_dma_pfn(struct page *pg)
165 {
166         return mm_to_dma_pfn(page_to_pfn(pg));
167 }
168 static inline unsigned long virt_to_dma_pfn(void *p)
169 {
170         return page_to_dma_pfn(virt_to_page(p));
171 }
172
173 /* global iommu list, set NULL for ignored DMAR units */
174 static struct intel_iommu **g_iommus;
175
176 static void __init check_tylersburg_isoch(void);
177 static int rwbf_quirk;
178
179 /*
180  * set to 1 to panic kernel if can't successfully enable VT-d
181  * (used when kernel is launched w/ TXT)
182  */
183 static int force_on = 0;
184
185 /*
186  * 0: Present
187  * 1-11: Reserved
188  * 12-63: Context Ptr (12 - (haw-1))
189  * 64-127: Reserved
190  */
191 struct root_entry {
192         u64     lo;
193         u64     hi;
194 };
195 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
196
197 /*
198  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199  * if marked present.
200  */
201 static phys_addr_t root_entry_lctp(struct root_entry *re)
202 {
203         if (!(re->lo & 1))
204                 return 0;
205
206         return re->lo & VTD_PAGE_MASK;
207 }
208
209 /*
210  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211  * if marked present.
212  */
213 static phys_addr_t root_entry_uctp(struct root_entry *re)
214 {
215         if (!(re->hi & 1))
216                 return 0;
217
218         return re->hi & VTD_PAGE_MASK;
219 }
220 /*
221  * low 64 bits:
222  * 0: present
223  * 1: fault processing disable
224  * 2-3: translation type
225  * 12-63: address space root
226  * high 64 bits:
227  * 0-2: address width
228  * 3-6: aval
229  * 8-23: domain id
230  */
231 struct context_entry {
232         u64 lo;
233         u64 hi;
234 };
235
236 static inline void context_clear_pasid_enable(struct context_entry *context)
237 {
238         context->lo &= ~(1ULL << 11);
239 }
240
241 static inline bool context_pasid_enabled(struct context_entry *context)
242 {
243         return !!(context->lo & (1ULL << 11));
244 }
245
246 static inline void context_set_copied(struct context_entry *context)
247 {
248         context->hi |= (1ull << 3);
249 }
250
251 static inline bool context_copied(struct context_entry *context)
252 {
253         return !!(context->hi & (1ULL << 3));
254 }
255
256 static inline bool __context_present(struct context_entry *context)
257 {
258         return (context->lo & 1);
259 }
260
261 static inline bool context_present(struct context_entry *context)
262 {
263         return context_pasid_enabled(context) ?
264              __context_present(context) :
265              __context_present(context) && !context_copied(context);
266 }
267
268 static inline void context_set_present(struct context_entry *context)
269 {
270         context->lo |= 1;
271 }
272
273 static inline void context_set_fault_enable(struct context_entry *context)
274 {
275         context->lo &= (((u64)-1) << 2) | 1;
276 }
277
278 static inline void context_set_translation_type(struct context_entry *context,
279                                                 unsigned long value)
280 {
281         context->lo &= (((u64)-1) << 4) | 3;
282         context->lo |= (value & 3) << 2;
283 }
284
285 static inline void context_set_address_root(struct context_entry *context,
286                                             unsigned long value)
287 {
288         context->lo &= ~VTD_PAGE_MASK;
289         context->lo |= value & VTD_PAGE_MASK;
290 }
291
292 static inline void context_set_address_width(struct context_entry *context,
293                                              unsigned long value)
294 {
295         context->hi |= value & 7;
296 }
297
298 static inline void context_set_domain_id(struct context_entry *context,
299                                          unsigned long value)
300 {
301         context->hi |= (value & ((1 << 16) - 1)) << 8;
302 }
303
304 static inline int context_domain_id(struct context_entry *c)
305 {
306         return((c->hi >> 8) & 0xffff);
307 }
308
309 static inline void context_clear_entry(struct context_entry *context)
310 {
311         context->lo = 0;
312         context->hi = 0;
313 }
314
315 /*
316  * 0: readable
317  * 1: writable
318  * 2-6: reserved
319  * 7: super page
320  * 8-10: available
321  * 11: snoop behavior
322  * 12-63: Host physcial address
323  */
324 struct dma_pte {
325         u64 val;
326 };
327
328 static inline void dma_clear_pte(struct dma_pte *pte)
329 {
330         pte->val = 0;
331 }
332
333 static inline u64 dma_pte_addr(struct dma_pte *pte)
334 {
335 #ifdef CONFIG_64BIT
336         return pte->val & VTD_PAGE_MASK;
337 #else
338         /* Must have a full atomic 64-bit read */
339         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
340 #endif
341 }
342
343 static inline bool dma_pte_present(struct dma_pte *pte)
344 {
345         return (pte->val & 3) != 0;
346 }
347
348 static inline bool dma_pte_superpage(struct dma_pte *pte)
349 {
350         return (pte->val & DMA_PTE_LARGE_PAGE);
351 }
352
353 static inline int first_pte_in_page(struct dma_pte *pte)
354 {
355         return !((unsigned long)pte & ~VTD_PAGE_MASK);
356 }
357
358 /*
359  * This domain is a statically identity mapping domain.
360  *      1. This domain creats a static 1:1 mapping to all usable memory.
361  *      2. It maps to each iommu if successful.
362  *      3. Each iommu mapps to this domain if successful.
363  */
364 static struct dmar_domain *si_domain;
365 static int hw_pass_through = 1;
366
367 /* domain represents a virtual machine, more than one devices
368  * across iommus may be owned in one domain, e.g. kvm guest.
369  */
370 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
371
372 /* si_domain contains mulitple devices */
373 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
374
375 struct dmar_domain {
376         int     id;                     /* domain id */
377         int     nid;                    /* node id */
378         DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
379                                         /* bitmap of iommus this domain uses*/
380
381         struct list_head devices;       /* all devices' list */
382         struct iova_domain iovad;       /* iova's that belong to this domain */
383
384         struct dma_pte  *pgd;           /* virtual address */
385         int             gaw;            /* max guest address width */
386
387         /* adjusted guest address width, 0 is level 2 30-bit */
388         int             agaw;
389
390         int             flags;          /* flags to find out type of domain */
391
392         int             iommu_coherency;/* indicate coherency of iommu access */
393         int             iommu_snooping; /* indicate snooping control feature*/
394         int             iommu_count;    /* reference count of iommu */
395         int             iommu_superpage;/* Level of superpages supported:
396                                            0 == 4KiB (no superpages), 1 == 2MiB,
397                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
398         spinlock_t      iommu_lock;     /* protect iommu set in domain */
399         u64             max_addr;       /* maximum mapped address */
400
401         struct iommu_domain domain;     /* generic domain data structure for
402                                            iommu core */
403 };
404
405 /* PCI domain-device relationship */
406 struct device_domain_info {
407         struct list_head link;  /* link to domain siblings */
408         struct list_head global; /* link to global list */
409         u8 bus;                 /* PCI bus number */
410         u8 devfn;               /* PCI devfn number */
411         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
412         struct intel_iommu *iommu; /* IOMMU used by this device */
413         struct dmar_domain *domain; /* pointer to domain */
414 };
415
416 struct dmar_rmrr_unit {
417         struct list_head list;          /* list of rmrr units   */
418         struct acpi_dmar_header *hdr;   /* ACPI header          */
419         u64     base_address;           /* reserved base address*/
420         u64     end_address;            /* reserved end address */
421         struct dmar_dev_scope *devices; /* target devices */
422         int     devices_cnt;            /* target device count */
423 };
424
425 struct dmar_atsr_unit {
426         struct list_head list;          /* list of ATSR units */
427         struct acpi_dmar_header *hdr;   /* ACPI header */
428         struct dmar_dev_scope *devices; /* target devices */
429         int devices_cnt;                /* target device count */
430         u8 include_all:1;               /* include all ports */
431 };
432
433 static LIST_HEAD(dmar_atsr_units);
434 static LIST_HEAD(dmar_rmrr_units);
435
436 #define for_each_rmrr_units(rmrr) \
437         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
438
439 static void flush_unmaps_timeout(unsigned long data);
440
441 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
442
443 #define HIGH_WATER_MARK 250
444 struct deferred_flush_tables {
445         int next;
446         struct iova *iova[HIGH_WATER_MARK];
447         struct dmar_domain *domain[HIGH_WATER_MARK];
448         struct page *freelist[HIGH_WATER_MARK];
449 };
450
451 static struct deferred_flush_tables *deferred_flush;
452
453 /* bitmap for indexing intel_iommus */
454 static int g_num_of_iommus;
455
456 static DEFINE_SPINLOCK(async_umap_flush_lock);
457 static LIST_HEAD(unmaps_to_do);
458
459 static int timer_on;
460 static long list_size;
461
462 static void domain_exit(struct dmar_domain *domain);
463 static void domain_remove_dev_info(struct dmar_domain *domain);
464 static void domain_remove_one_dev_info(struct dmar_domain *domain,
465                                        struct device *dev);
466 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
467                                            struct device *dev);
468 static int domain_detach_iommu(struct dmar_domain *domain,
469                                struct intel_iommu *iommu);
470
471 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
472 int dmar_disabled = 0;
473 #else
474 int dmar_disabled = 1;
475 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
476
477 int intel_iommu_enabled = 0;
478 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
479
480 static int dmar_map_gfx = 1;
481 static int dmar_forcedac;
482 static int intel_iommu_strict;
483 static int intel_iommu_superpage = 1;
484 static int intel_iommu_ecs = 1;
485
486 /* We only actually use ECS when PASID support (on the new bit 40)
487  * is also advertised. Some early implementations — the ones with
488  * PASID support on bit 28 — have issues even when we *only* use
489  * extended root/context tables. */
490 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
491                             ecap_pasid(iommu->ecap))
492
493 int intel_iommu_gfx_mapped;
494 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
495
496 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
497 static DEFINE_SPINLOCK(device_domain_lock);
498 static LIST_HEAD(device_domain_list);
499
500 static const struct iommu_ops intel_iommu_ops;
501
502 static bool translation_pre_enabled(struct intel_iommu *iommu)
503 {
504         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
505 }
506
507 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
508 {
509         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
510 }
511
512 static void init_translation_status(struct intel_iommu *iommu)
513 {
514         u32 gsts;
515
516         gsts = readl(iommu->reg + DMAR_GSTS_REG);
517         if (gsts & DMA_GSTS_TES)
518                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
519 }
520
521 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
522 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
523 {
524         return container_of(dom, struct dmar_domain, domain);
525 }
526
527 static int __init intel_iommu_setup(char *str)
528 {
529         if (!str)
530                 return -EINVAL;
531         while (*str) {
532                 if (!strncmp(str, "on", 2)) {
533                         dmar_disabled = 0;
534                         pr_info("IOMMU enabled\n");
535                 } else if (!strncmp(str, "off", 3)) {
536                         dmar_disabled = 1;
537                         pr_info("IOMMU disabled\n");
538                 } else if (!strncmp(str, "igfx_off", 8)) {
539                         dmar_map_gfx = 0;
540                         pr_info("Disable GFX device mapping\n");
541                 } else if (!strncmp(str, "forcedac", 8)) {
542                         pr_info("Forcing DAC for PCI devices\n");
543                         dmar_forcedac = 1;
544                 } else if (!strncmp(str, "strict", 6)) {
545                         pr_info("Disable batched IOTLB flush\n");
546                         intel_iommu_strict = 1;
547                 } else if (!strncmp(str, "sp_off", 6)) {
548                         pr_info("Disable supported super page\n");
549                         intel_iommu_superpage = 0;
550                 } else if (!strncmp(str, "ecs_off", 7)) {
551                         printk(KERN_INFO
552                                 "Intel-IOMMU: disable extended context table support\n");
553                         intel_iommu_ecs = 0;
554                 }
555
556                 str += strcspn(str, ",");
557                 while (*str == ',')
558                         str++;
559         }
560         return 0;
561 }
562 __setup("intel_iommu=", intel_iommu_setup);
563
564 static struct kmem_cache *iommu_domain_cache;
565 static struct kmem_cache *iommu_devinfo_cache;
566
567 static inline void *alloc_pgtable_page(int node)
568 {
569         struct page *page;
570         void *vaddr = NULL;
571
572         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
573         if (page)
574                 vaddr = page_address(page);
575         return vaddr;
576 }
577
578 static inline void free_pgtable_page(void *vaddr)
579 {
580         free_page((unsigned long)vaddr);
581 }
582
583 static inline void *alloc_domain_mem(void)
584 {
585         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
586 }
587
588 static void free_domain_mem(void *vaddr)
589 {
590         kmem_cache_free(iommu_domain_cache, vaddr);
591 }
592
593 static inline void * alloc_devinfo_mem(void)
594 {
595         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
596 }
597
598 static inline void free_devinfo_mem(void *vaddr)
599 {
600         kmem_cache_free(iommu_devinfo_cache, vaddr);
601 }
602
603 static inline int domain_type_is_vm(struct dmar_domain *domain)
604 {
605         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
606 }
607
608 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
609 {
610         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
611                                 DOMAIN_FLAG_STATIC_IDENTITY);
612 }
613
614 static inline int domain_pfn_supported(struct dmar_domain *domain,
615                                        unsigned long pfn)
616 {
617         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
618
619         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
620 }
621
622 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
623 {
624         unsigned long sagaw;
625         int agaw = -1;
626
627         sagaw = cap_sagaw(iommu->cap);
628         for (agaw = width_to_agaw(max_gaw);
629              agaw >= 0; agaw--) {
630                 if (test_bit(agaw, &sagaw))
631                         break;
632         }
633
634         return agaw;
635 }
636
637 /*
638  * Calculate max SAGAW for each iommu.
639  */
640 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
641 {
642         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
643 }
644
645 /*
646  * calculate agaw for each iommu.
647  * "SAGAW" may be different across iommus, use a default agaw, and
648  * get a supported less agaw for iommus that don't support the default agaw.
649  */
650 int iommu_calculate_agaw(struct intel_iommu *iommu)
651 {
652         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
653 }
654
655 /* This functionin only returns single iommu in a domain */
656 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
657 {
658         int iommu_id;
659
660         /* si_domain and vm domain should not get here. */
661         BUG_ON(domain_type_is_vm_or_si(domain));
662         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
663         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
664                 return NULL;
665
666         return g_iommus[iommu_id];
667 }
668
669 static void domain_update_iommu_coherency(struct dmar_domain *domain)
670 {
671         struct dmar_drhd_unit *drhd;
672         struct intel_iommu *iommu;
673         bool found = false;
674         int i;
675
676         domain->iommu_coherency = 1;
677
678         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
679                 found = true;
680                 if (!ecap_coherent(g_iommus[i]->ecap)) {
681                         domain->iommu_coherency = 0;
682                         break;
683                 }
684         }
685         if (found)
686                 return;
687
688         /* No hardware attached; use lowest common denominator */
689         rcu_read_lock();
690         for_each_active_iommu(iommu, drhd) {
691                 if (!ecap_coherent(iommu->ecap)) {
692                         domain->iommu_coherency = 0;
693                         break;
694                 }
695         }
696         rcu_read_unlock();
697 }
698
699 static int domain_update_iommu_snooping(struct intel_iommu *skip)
700 {
701         struct dmar_drhd_unit *drhd;
702         struct intel_iommu *iommu;
703         int ret = 1;
704
705         rcu_read_lock();
706         for_each_active_iommu(iommu, drhd) {
707                 if (iommu != skip) {
708                         if (!ecap_sc_support(iommu->ecap)) {
709                                 ret = 0;
710                                 break;
711                         }
712                 }
713         }
714         rcu_read_unlock();
715
716         return ret;
717 }
718
719 static int domain_update_iommu_superpage(struct intel_iommu *skip)
720 {
721         struct dmar_drhd_unit *drhd;
722         struct intel_iommu *iommu;
723         int mask = 0xf;
724
725         if (!intel_iommu_superpage) {
726                 return 0;
727         }
728
729         /* set iommu_superpage to the smallest common denominator */
730         rcu_read_lock();
731         for_each_active_iommu(iommu, drhd) {
732                 if (iommu != skip) {
733                         mask &= cap_super_page_val(iommu->cap);
734                         if (!mask)
735                                 break;
736                 }
737         }
738         rcu_read_unlock();
739
740         return fls(mask);
741 }
742
743 /* Some capabilities may be different across iommus */
744 static void domain_update_iommu_cap(struct dmar_domain *domain)
745 {
746         domain_update_iommu_coherency(domain);
747         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
748         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
749 }
750
751 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
752                                                        u8 bus, u8 devfn, int alloc)
753 {
754         struct root_entry *root = &iommu->root_entry[bus];
755         struct context_entry *context;
756         u64 *entry;
757
758         if (ecs_enabled(iommu)) {
759                 if (devfn >= 0x80) {
760                         devfn -= 0x80;
761                         entry = &root->hi;
762                 }
763                 devfn *= 2;
764         }
765         entry = &root->lo;
766         if (*entry & 1)
767                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
768         else {
769                 unsigned long phy_addr;
770                 if (!alloc)
771                         return NULL;
772
773                 context = alloc_pgtable_page(iommu->node);
774                 if (!context)
775                         return NULL;
776
777                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
778                 phy_addr = virt_to_phys((void *)context);
779                 *entry = phy_addr | 1;
780                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
781         }
782         return &context[devfn];
783 }
784
785 static int iommu_dummy(struct device *dev)
786 {
787         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
788 }
789
790 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
791 {
792         struct dmar_drhd_unit *drhd = NULL;
793         struct intel_iommu *iommu;
794         struct device *tmp;
795         struct pci_dev *ptmp, *pdev = NULL;
796         u16 segment = 0;
797         int i;
798
799         if (iommu_dummy(dev))
800                 return NULL;
801
802         if (dev_is_pci(dev)) {
803                 pdev = to_pci_dev(dev);
804                 segment = pci_domain_nr(pdev->bus);
805         } else if (has_acpi_companion(dev))
806                 dev = &ACPI_COMPANION(dev)->dev;
807
808         rcu_read_lock();
809         for_each_active_iommu(iommu, drhd) {
810                 if (pdev && segment != drhd->segment)
811                         continue;
812
813                 for_each_active_dev_scope(drhd->devices,
814                                           drhd->devices_cnt, i, tmp) {
815                         if (tmp == dev) {
816                                 *bus = drhd->devices[i].bus;
817                                 *devfn = drhd->devices[i].devfn;
818                                 goto out;
819                         }
820
821                         if (!pdev || !dev_is_pci(tmp))
822                                 continue;
823
824                         ptmp = to_pci_dev(tmp);
825                         if (ptmp->subordinate &&
826                             ptmp->subordinate->number <= pdev->bus->number &&
827                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
828                                 goto got_pdev;
829                 }
830
831                 if (pdev && drhd->include_all) {
832                 got_pdev:
833                         *bus = pdev->bus->number;
834                         *devfn = pdev->devfn;
835                         goto out;
836                 }
837         }
838         iommu = NULL;
839  out:
840         rcu_read_unlock();
841
842         return iommu;
843 }
844
845 static void domain_flush_cache(struct dmar_domain *domain,
846                                void *addr, int size)
847 {
848         if (!domain->iommu_coherency)
849                 clflush_cache_range(addr, size);
850 }
851
852 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
853 {
854         struct context_entry *context;
855         int ret = 0;
856         unsigned long flags;
857
858         spin_lock_irqsave(&iommu->lock, flags);
859         context = iommu_context_addr(iommu, bus, devfn, 0);
860         if (context)
861                 ret = context_present(context);
862         spin_unlock_irqrestore(&iommu->lock, flags);
863         return ret;
864 }
865
866 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
867 {
868         struct context_entry *context;
869         unsigned long flags;
870
871         spin_lock_irqsave(&iommu->lock, flags);
872         context = iommu_context_addr(iommu, bus, devfn, 0);
873         if (context) {
874                 context_clear_entry(context);
875                 __iommu_flush_cache(iommu, context, sizeof(*context));
876         }
877         spin_unlock_irqrestore(&iommu->lock, flags);
878 }
879
880 static void free_context_table(struct intel_iommu *iommu)
881 {
882         int i;
883         unsigned long flags;
884         struct context_entry *context;
885
886         spin_lock_irqsave(&iommu->lock, flags);
887         if (!iommu->root_entry) {
888                 goto out;
889         }
890         for (i = 0; i < ROOT_ENTRY_NR; i++) {
891                 context = iommu_context_addr(iommu, i, 0, 0);
892                 if (context)
893                         free_pgtable_page(context);
894
895                 if (!ecs_enabled(iommu))
896                         continue;
897
898                 context = iommu_context_addr(iommu, i, 0x80, 0);
899                 if (context)
900                         free_pgtable_page(context);
901
902         }
903         free_pgtable_page(iommu->root_entry);
904         iommu->root_entry = NULL;
905 out:
906         spin_unlock_irqrestore(&iommu->lock, flags);
907 }
908
909 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
910                                       unsigned long pfn, int *target_level)
911 {
912         struct dma_pte *parent, *pte = NULL;
913         int level = agaw_to_level(domain->agaw);
914         int offset;
915
916         BUG_ON(!domain->pgd);
917
918         if (!domain_pfn_supported(domain, pfn))
919                 /* Address beyond IOMMU's addressing capabilities. */
920                 return NULL;
921
922         parent = domain->pgd;
923
924         while (1) {
925                 void *tmp_page;
926
927                 offset = pfn_level_offset(pfn, level);
928                 pte = &parent[offset];
929                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
930                         break;
931                 if (level == *target_level)
932                         break;
933
934                 if (!dma_pte_present(pte)) {
935                         uint64_t pteval;
936
937                         tmp_page = alloc_pgtable_page(domain->nid);
938
939                         if (!tmp_page)
940                                 return NULL;
941
942                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
943                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
944                         if (cmpxchg64(&pte->val, 0ULL, pteval))
945                                 /* Someone else set it while we were thinking; use theirs. */
946                                 free_pgtable_page(tmp_page);
947                         else
948                                 domain_flush_cache(domain, pte, sizeof(*pte));
949                 }
950                 if (level == 1)
951                         break;
952
953                 parent = phys_to_virt(dma_pte_addr(pte));
954                 level--;
955         }
956
957         if (!*target_level)
958                 *target_level = level;
959
960         return pte;
961 }
962
963
964 /* return address's pte at specific level */
965 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
966                                          unsigned long pfn,
967                                          int level, int *large_page)
968 {
969         struct dma_pte *parent, *pte = NULL;
970         int total = agaw_to_level(domain->agaw);
971         int offset;
972
973         parent = domain->pgd;
974         while (level <= total) {
975                 offset = pfn_level_offset(pfn, total);
976                 pte = &parent[offset];
977                 if (level == total)
978                         return pte;
979
980                 if (!dma_pte_present(pte)) {
981                         *large_page = total;
982                         break;
983                 }
984
985                 if (dma_pte_superpage(pte)) {
986                         *large_page = total;
987                         return pte;
988                 }
989
990                 parent = phys_to_virt(dma_pte_addr(pte));
991                 total--;
992         }
993         return NULL;
994 }
995
996 /* clear last level pte, a tlb flush should be followed */
997 static void dma_pte_clear_range(struct dmar_domain *domain,
998                                 unsigned long start_pfn,
999                                 unsigned long last_pfn)
1000 {
1001         unsigned int large_page = 1;
1002         struct dma_pte *first_pte, *pte;
1003
1004         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1005         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1006         BUG_ON(start_pfn > last_pfn);
1007
1008         /* we don't need lock here; nobody else touches the iova range */
1009         do {
1010                 large_page = 1;
1011                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1012                 if (!pte) {
1013                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1014                         continue;
1015                 }
1016                 do {
1017                         dma_clear_pte(pte);
1018                         start_pfn += lvl_to_nr_pages(large_page);
1019                         pte++;
1020                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1021
1022                 domain_flush_cache(domain, first_pte,
1023                                    (void *)pte - (void *)first_pte);
1024
1025         } while (start_pfn && start_pfn <= last_pfn);
1026 }
1027
1028 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1029                                struct dma_pte *pte, unsigned long pfn,
1030                                unsigned long start_pfn, unsigned long last_pfn)
1031 {
1032         pfn = max(start_pfn, pfn);
1033         pte = &pte[pfn_level_offset(pfn, level)];
1034
1035         do {
1036                 unsigned long level_pfn;
1037                 struct dma_pte *level_pte;
1038
1039                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1040                         goto next;
1041
1042                 level_pfn = pfn & level_mask(level - 1);
1043                 level_pte = phys_to_virt(dma_pte_addr(pte));
1044
1045                 if (level > 2)
1046                         dma_pte_free_level(domain, level - 1, level_pte,
1047                                            level_pfn, start_pfn, last_pfn);
1048
1049                 /* If range covers entire pagetable, free it */
1050                 if (!(start_pfn > level_pfn ||
1051                       last_pfn < level_pfn + level_size(level) - 1)) {
1052                         dma_clear_pte(pte);
1053                         domain_flush_cache(domain, pte, sizeof(*pte));
1054                         free_pgtable_page(level_pte);
1055                 }
1056 next:
1057                 pfn += level_size(level);
1058         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1059 }
1060
1061 /* free page table pages. last level pte should already be cleared */
1062 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1063                                    unsigned long start_pfn,
1064                                    unsigned long last_pfn)
1065 {
1066         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1067         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1068         BUG_ON(start_pfn > last_pfn);
1069
1070         dma_pte_clear_range(domain, start_pfn, last_pfn);
1071
1072         /* We don't need lock here; nobody else touches the iova range */
1073         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1074                            domain->pgd, 0, start_pfn, last_pfn);
1075
1076         /* free pgd */
1077         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1078                 free_pgtable_page(domain->pgd);
1079                 domain->pgd = NULL;
1080         }
1081 }
1082
1083 /* When a page at a given level is being unlinked from its parent, we don't
1084    need to *modify* it at all. All we need to do is make a list of all the
1085    pages which can be freed just as soon as we've flushed the IOTLB and we
1086    know the hardware page-walk will no longer touch them.
1087    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1088    be freed. */
1089 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1090                                             int level, struct dma_pte *pte,
1091                                             struct page *freelist)
1092 {
1093         struct page *pg;
1094
1095         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1096         pg->freelist = freelist;
1097         freelist = pg;
1098
1099         if (level == 1)
1100                 return freelist;
1101
1102         pte = page_address(pg);
1103         do {
1104                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1105                         freelist = dma_pte_list_pagetables(domain, level - 1,
1106                                                            pte, freelist);
1107                 pte++;
1108         } while (!first_pte_in_page(pte));
1109
1110         return freelist;
1111 }
1112
1113 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1114                                         struct dma_pte *pte, unsigned long pfn,
1115                                         unsigned long start_pfn,
1116                                         unsigned long last_pfn,
1117                                         struct page *freelist)
1118 {
1119         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1120
1121         pfn = max(start_pfn, pfn);
1122         pte = &pte[pfn_level_offset(pfn, level)];
1123
1124         do {
1125                 unsigned long level_pfn;
1126
1127                 if (!dma_pte_present(pte))
1128                         goto next;
1129
1130                 level_pfn = pfn & level_mask(level);
1131
1132                 /* If range covers entire pagetable, free it */
1133                 if (start_pfn <= level_pfn &&
1134                     last_pfn >= level_pfn + level_size(level) - 1) {
1135                         /* These suborbinate page tables are going away entirely. Don't
1136                            bother to clear them; we're just going to *free* them. */
1137                         if (level > 1 && !dma_pte_superpage(pte))
1138                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1139
1140                         dma_clear_pte(pte);
1141                         if (!first_pte)
1142                                 first_pte = pte;
1143                         last_pte = pte;
1144                 } else if (level > 1) {
1145                         /* Recurse down into a level that isn't *entirely* obsolete */
1146                         freelist = dma_pte_clear_level(domain, level - 1,
1147                                                        phys_to_virt(dma_pte_addr(pte)),
1148                                                        level_pfn, start_pfn, last_pfn,
1149                                                        freelist);
1150                 }
1151 next:
1152                 pfn += level_size(level);
1153         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1154
1155         if (first_pte)
1156                 domain_flush_cache(domain, first_pte,
1157                                    (void *)++last_pte - (void *)first_pte);
1158
1159         return freelist;
1160 }
1161
1162 /* We can't just free the pages because the IOMMU may still be walking
1163    the page tables, and may have cached the intermediate levels. The
1164    pages can only be freed after the IOTLB flush has been done. */
1165 struct page *domain_unmap(struct dmar_domain *domain,
1166                           unsigned long start_pfn,
1167                           unsigned long last_pfn)
1168 {
1169         struct page *freelist = NULL;
1170
1171         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173         BUG_ON(start_pfn > last_pfn);
1174
1175         /* we don't need lock here; nobody else touches the iova range */
1176         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1177                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1178
1179         /* free pgd */
1180         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1181                 struct page *pgd_page = virt_to_page(domain->pgd);
1182                 pgd_page->freelist = freelist;
1183                 freelist = pgd_page;
1184
1185                 domain->pgd = NULL;
1186         }
1187
1188         return freelist;
1189 }
1190
1191 void dma_free_pagelist(struct page *freelist)
1192 {
1193         struct page *pg;
1194
1195         while ((pg = freelist)) {
1196                 freelist = pg->freelist;
1197                 free_pgtable_page(page_address(pg));
1198         }
1199 }
1200
1201 /* iommu handling */
1202 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1203 {
1204         struct root_entry *root;
1205         unsigned long flags;
1206
1207         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1208         if (!root) {
1209                 pr_err("Allocating root entry for %s failed\n",
1210                         iommu->name);
1211                 return -ENOMEM;
1212         }
1213
1214         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1215
1216         spin_lock_irqsave(&iommu->lock, flags);
1217         iommu->root_entry = root;
1218         spin_unlock_irqrestore(&iommu->lock, flags);
1219
1220         return 0;
1221 }
1222
1223 static void iommu_set_root_entry(struct intel_iommu *iommu)
1224 {
1225         u64 addr;
1226         u32 sts;
1227         unsigned long flag;
1228
1229         addr = virt_to_phys(iommu->root_entry);
1230         if (ecs_enabled(iommu))
1231                 addr |= DMA_RTADDR_RTT;
1232
1233         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1234         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1235
1236         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1237
1238         /* Make sure hardware complete it */
1239         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240                       readl, (sts & DMA_GSTS_RTPS), sts);
1241
1242         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1243 }
1244
1245 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1246 {
1247         u32 val;
1248         unsigned long flag;
1249
1250         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1251                 return;
1252
1253         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1254         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1255
1256         /* Make sure hardware complete it */
1257         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1258                       readl, (!(val & DMA_GSTS_WBFS)), val);
1259
1260         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1261 }
1262
1263 /* return value determine if we need a write buffer flush */
1264 static void __iommu_flush_context(struct intel_iommu *iommu,
1265                                   u16 did, u16 source_id, u8 function_mask,
1266                                   u64 type)
1267 {
1268         u64 val = 0;
1269         unsigned long flag;
1270
1271         switch (type) {
1272         case DMA_CCMD_GLOBAL_INVL:
1273                 val = DMA_CCMD_GLOBAL_INVL;
1274                 break;
1275         case DMA_CCMD_DOMAIN_INVL:
1276                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1277                 break;
1278         case DMA_CCMD_DEVICE_INVL:
1279                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1280                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1281                 break;
1282         default:
1283                 BUG();
1284         }
1285         val |= DMA_CCMD_ICC;
1286
1287         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1288         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1289
1290         /* Make sure hardware complete it */
1291         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1292                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1293
1294         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1295 }
1296
1297 /* return value determine if we need a write buffer flush */
1298 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1299                                 u64 addr, unsigned int size_order, u64 type)
1300 {
1301         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1302         u64 val = 0, val_iva = 0;
1303         unsigned long flag;
1304
1305         switch (type) {
1306         case DMA_TLB_GLOBAL_FLUSH:
1307                 /* global flush doesn't need set IVA_REG */
1308                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1309                 break;
1310         case DMA_TLB_DSI_FLUSH:
1311                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1312                 break;
1313         case DMA_TLB_PSI_FLUSH:
1314                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1315                 /* IH bit is passed in as part of address */
1316                 val_iva = size_order | addr;
1317                 break;
1318         default:
1319                 BUG();
1320         }
1321         /* Note: set drain read/write */
1322 #if 0
1323         /*
1324          * This is probably to be super secure.. Looks like we can
1325          * ignore it without any impact.
1326          */
1327         if (cap_read_drain(iommu->cap))
1328                 val |= DMA_TLB_READ_DRAIN;
1329 #endif
1330         if (cap_write_drain(iommu->cap))
1331                 val |= DMA_TLB_WRITE_DRAIN;
1332
1333         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1334         /* Note: Only uses first TLB reg currently */
1335         if (val_iva)
1336                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1337         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1338
1339         /* Make sure hardware complete it */
1340         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1341                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1342
1343         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1344
1345         /* check IOTLB invalidation granularity */
1346         if (DMA_TLB_IAIG(val) == 0)
1347                 pr_err("Flush IOTLB failed\n");
1348         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1349                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1350                         (unsigned long long)DMA_TLB_IIRG(type),
1351                         (unsigned long long)DMA_TLB_IAIG(val));
1352 }
1353
1354 static struct device_domain_info *
1355 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1356                          u8 bus, u8 devfn)
1357 {
1358         bool found = false;
1359         unsigned long flags;
1360         struct device_domain_info *info;
1361         struct pci_dev *pdev;
1362
1363         if (!ecap_dev_iotlb_support(iommu->ecap))
1364                 return NULL;
1365
1366         if (!iommu->qi)
1367                 return NULL;
1368
1369         spin_lock_irqsave(&device_domain_lock, flags);
1370         list_for_each_entry(info, &domain->devices, link)
1371                 if (info->iommu == iommu && info->bus == bus &&
1372                     info->devfn == devfn) {
1373                         found = true;
1374                         break;
1375                 }
1376         spin_unlock_irqrestore(&device_domain_lock, flags);
1377
1378         if (!found || !info->dev || !dev_is_pci(info->dev))
1379                 return NULL;
1380
1381         pdev = to_pci_dev(info->dev);
1382
1383         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1384                 return NULL;
1385
1386         if (!dmar_find_matched_atsr_unit(pdev))
1387                 return NULL;
1388
1389         return info;
1390 }
1391
1392 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1393 {
1394         if (!info || !dev_is_pci(info->dev))
1395                 return;
1396
1397         pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1398 }
1399
1400 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1401 {
1402         if (!info->dev || !dev_is_pci(info->dev) ||
1403             !pci_ats_enabled(to_pci_dev(info->dev)))
1404                 return;
1405
1406         pci_disable_ats(to_pci_dev(info->dev));
1407 }
1408
1409 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1410                                   u64 addr, unsigned mask)
1411 {
1412         u16 sid, qdep;
1413         unsigned long flags;
1414         struct device_domain_info *info;
1415
1416         spin_lock_irqsave(&device_domain_lock, flags);
1417         list_for_each_entry(info, &domain->devices, link) {
1418                 struct pci_dev *pdev;
1419                 if (!info->dev || !dev_is_pci(info->dev))
1420                         continue;
1421
1422                 pdev = to_pci_dev(info->dev);
1423                 if (!pci_ats_enabled(pdev))
1424                         continue;
1425
1426                 sid = info->bus << 8 | info->devfn;
1427                 qdep = pci_ats_queue_depth(pdev);
1428                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1429         }
1430         spin_unlock_irqrestore(&device_domain_lock, flags);
1431 }
1432
1433 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1434                                   unsigned long pfn, unsigned int pages, int ih, int map)
1435 {
1436         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1437         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1438
1439         BUG_ON(pages == 0);
1440
1441         if (ih)
1442                 ih = 1 << 6;
1443         /*
1444          * Fallback to domain selective flush if no PSI support or the size is
1445          * too big.
1446          * PSI requires page size to be 2 ^ x, and the base address is naturally
1447          * aligned to the size
1448          */
1449         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1450                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1451                                                 DMA_TLB_DSI_FLUSH);
1452         else
1453                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1454                                                 DMA_TLB_PSI_FLUSH);
1455
1456         /*
1457          * In caching mode, changes of pages from non-present to present require
1458          * flush. However, device IOTLB doesn't need to be flushed in this case.
1459          */
1460         if (!cap_caching_mode(iommu->cap) || !map)
1461                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1462 }
1463
1464 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1465 {
1466         u32 pmen;
1467         unsigned long flags;
1468
1469         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1470         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1471         pmen &= ~DMA_PMEN_EPM;
1472         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1473
1474         /* wait for the protected region status bit to clear */
1475         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1476                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1477
1478         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1479 }
1480
1481 static void iommu_enable_translation(struct intel_iommu *iommu)
1482 {
1483         u32 sts;
1484         unsigned long flags;
1485
1486         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1487         iommu->gcmd |= DMA_GCMD_TE;
1488         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1489
1490         /* Make sure hardware complete it */
1491         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1492                       readl, (sts & DMA_GSTS_TES), sts);
1493
1494         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1495 }
1496
1497 static void iommu_disable_translation(struct intel_iommu *iommu)
1498 {
1499         u32 sts;
1500         unsigned long flag;
1501
1502         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1503         iommu->gcmd &= ~DMA_GCMD_TE;
1504         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1505
1506         /* Make sure hardware complete it */
1507         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1508                       readl, (!(sts & DMA_GSTS_TES)), sts);
1509
1510         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1511 }
1512
1513
1514 static int iommu_init_domains(struct intel_iommu *iommu)
1515 {
1516         unsigned long ndomains;
1517         unsigned long nlongs;
1518
1519         ndomains = cap_ndoms(iommu->cap);
1520         pr_debug("%s: Number of Domains supported <%ld>\n",
1521                  iommu->name, ndomains);
1522         nlongs = BITS_TO_LONGS(ndomains);
1523
1524         spin_lock_init(&iommu->lock);
1525
1526         /* TBD: there might be 64K domains,
1527          * consider other allocation for future chip
1528          */
1529         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1530         if (!iommu->domain_ids) {
1531                 pr_err("%s: Allocating domain id array failed\n",
1532                        iommu->name);
1533                 return -ENOMEM;
1534         }
1535         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1536                         GFP_KERNEL);
1537         if (!iommu->domains) {
1538                 pr_err("%s: Allocating domain array failed\n",
1539                        iommu->name);
1540                 kfree(iommu->domain_ids);
1541                 iommu->domain_ids = NULL;
1542                 return -ENOMEM;
1543         }
1544
1545         /*
1546          * if Caching mode is set, then invalid translations are tagged
1547          * with domainid 0. Hence we need to pre-allocate it.
1548          */
1549         if (cap_caching_mode(iommu->cap))
1550                 set_bit(0, iommu->domain_ids);
1551         return 0;
1552 }
1553
1554 static void disable_dmar_iommu(struct intel_iommu *iommu)
1555 {
1556         struct dmar_domain *domain;
1557         int i;
1558
1559         if ((iommu->domains) && (iommu->domain_ids)) {
1560                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1561                         /*
1562                          * Domain id 0 is reserved for invalid translation
1563                          * if hardware supports caching mode.
1564                          */
1565                         if (cap_caching_mode(iommu->cap) && i == 0)
1566                                 continue;
1567
1568                         domain = iommu->domains[i];
1569                         clear_bit(i, iommu->domain_ids);
1570                         if (domain_detach_iommu(domain, iommu) == 0 &&
1571                             !domain_type_is_vm(domain))
1572                                 domain_exit(domain);
1573                 }
1574         }
1575
1576         if (iommu->gcmd & DMA_GCMD_TE)
1577                 iommu_disable_translation(iommu);
1578 }
1579
1580 static void free_dmar_iommu(struct intel_iommu *iommu)
1581 {
1582         if ((iommu->domains) && (iommu->domain_ids)) {
1583                 kfree(iommu->domains);
1584                 kfree(iommu->domain_ids);
1585                 iommu->domains = NULL;
1586                 iommu->domain_ids = NULL;
1587         }
1588
1589         g_iommus[iommu->seq_id] = NULL;
1590
1591         /* free context mapping */
1592         free_context_table(iommu);
1593 }
1594
1595 static struct dmar_domain *alloc_domain(int flags)
1596 {
1597         /* domain id for virtual machine, it won't be set in context */
1598         static atomic_t vm_domid = ATOMIC_INIT(0);
1599         struct dmar_domain *domain;
1600
1601         domain = alloc_domain_mem();
1602         if (!domain)
1603                 return NULL;
1604
1605         memset(domain, 0, sizeof(*domain));
1606         domain->nid = -1;
1607         domain->flags = flags;
1608         spin_lock_init(&domain->iommu_lock);
1609         INIT_LIST_HEAD(&domain->devices);
1610         if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1611                 domain->id = atomic_inc_return(&vm_domid);
1612
1613         return domain;
1614 }
1615
1616 static int __iommu_attach_domain(struct dmar_domain *domain,
1617                                  struct intel_iommu *iommu)
1618 {
1619         int num;
1620         unsigned long ndomains;
1621
1622         ndomains = cap_ndoms(iommu->cap);
1623         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1624         if (num < ndomains) {
1625                 set_bit(num, iommu->domain_ids);
1626                 iommu->domains[num] = domain;
1627         } else {
1628                 num = -ENOSPC;
1629         }
1630
1631         return num;
1632 }
1633
1634 static int iommu_attach_domain(struct dmar_domain *domain,
1635                                struct intel_iommu *iommu)
1636 {
1637         int num;
1638         unsigned long flags;
1639
1640         spin_lock_irqsave(&iommu->lock, flags);
1641         num = __iommu_attach_domain(domain, iommu);
1642         spin_unlock_irqrestore(&iommu->lock, flags);
1643         if (num < 0)
1644                 pr_err("%s: No free domain ids\n", iommu->name);
1645
1646         return num;
1647 }
1648
1649 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1650                                   struct intel_iommu *iommu)
1651 {
1652         int num;
1653         unsigned long ndomains;
1654
1655         ndomains = cap_ndoms(iommu->cap);
1656         for_each_set_bit(num, iommu->domain_ids, ndomains)
1657                 if (iommu->domains[num] == domain)
1658                         return num;
1659
1660         return __iommu_attach_domain(domain, iommu);
1661 }
1662
1663 static void iommu_detach_domain(struct dmar_domain *domain,
1664                                 struct intel_iommu *iommu)
1665 {
1666         unsigned long flags;
1667         int num, ndomains;
1668
1669         spin_lock_irqsave(&iommu->lock, flags);
1670         if (domain_type_is_vm_or_si(domain)) {
1671                 ndomains = cap_ndoms(iommu->cap);
1672                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1673                         if (iommu->domains[num] == domain) {
1674                                 clear_bit(num, iommu->domain_ids);
1675                                 iommu->domains[num] = NULL;
1676                                 break;
1677                         }
1678                 }
1679         } else {
1680                 clear_bit(domain->id, iommu->domain_ids);
1681                 iommu->domains[domain->id] = NULL;
1682         }
1683         spin_unlock_irqrestore(&iommu->lock, flags);
1684 }
1685
1686 static void domain_attach_iommu(struct dmar_domain *domain,
1687                                struct intel_iommu *iommu)
1688 {
1689         unsigned long flags;
1690
1691         spin_lock_irqsave(&domain->iommu_lock, flags);
1692         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1693                 domain->iommu_count++;
1694                 if (domain->iommu_count == 1)
1695                         domain->nid = iommu->node;
1696                 domain_update_iommu_cap(domain);
1697         }
1698         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1699 }
1700
1701 static int domain_detach_iommu(struct dmar_domain *domain,
1702                                struct intel_iommu *iommu)
1703 {
1704         unsigned long flags;
1705         int count = INT_MAX;
1706
1707         spin_lock_irqsave(&domain->iommu_lock, flags);
1708         if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1709                 count = --domain->iommu_count;
1710                 domain_update_iommu_cap(domain);
1711         }
1712         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1713
1714         return count;
1715 }
1716
1717 static struct iova_domain reserved_iova_list;
1718 static struct lock_class_key reserved_rbtree_key;
1719
1720 static int dmar_init_reserved_ranges(void)
1721 {
1722         struct pci_dev *pdev = NULL;
1723         struct iova *iova;
1724         int i;
1725
1726         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1727                         DMA_32BIT_PFN);
1728
1729         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1730                 &reserved_rbtree_key);
1731
1732         /* IOAPIC ranges shouldn't be accessed by DMA */
1733         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1734                 IOVA_PFN(IOAPIC_RANGE_END));
1735         if (!iova) {
1736                 pr_err("Reserve IOAPIC range failed\n");
1737                 return -ENODEV;
1738         }
1739
1740         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1741         for_each_pci_dev(pdev) {
1742                 struct resource *r;
1743
1744                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1745                         r = &pdev->resource[i];
1746                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1747                                 continue;
1748                         iova = reserve_iova(&reserved_iova_list,
1749                                             IOVA_PFN(r->start),
1750                                             IOVA_PFN(r->end));
1751                         if (!iova) {
1752                                 pr_err("Reserve iova failed\n");
1753                                 return -ENODEV;
1754                         }
1755                 }
1756         }
1757         return 0;
1758 }
1759
1760 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1761 {
1762         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1763 }
1764
1765 static inline int guestwidth_to_adjustwidth(int gaw)
1766 {
1767         int agaw;
1768         int r = (gaw - 12) % 9;
1769
1770         if (r == 0)
1771                 agaw = gaw;
1772         else
1773                 agaw = gaw + 9 - r;
1774         if (agaw > 64)
1775                 agaw = 64;
1776         return agaw;
1777 }
1778
1779 static int domain_init(struct dmar_domain *domain, int guest_width)
1780 {
1781         struct intel_iommu *iommu;
1782         int adjust_width, agaw;
1783         unsigned long sagaw;
1784
1785         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1786                         DMA_32BIT_PFN);
1787         domain_reserve_special_ranges(domain);
1788
1789         /* calculate AGAW */
1790         iommu = domain_get_iommu(domain);
1791         if (guest_width > cap_mgaw(iommu->cap))
1792                 guest_width = cap_mgaw(iommu->cap);
1793         domain->gaw = guest_width;
1794         adjust_width = guestwidth_to_adjustwidth(guest_width);
1795         agaw = width_to_agaw(adjust_width);
1796         sagaw = cap_sagaw(iommu->cap);
1797         if (!test_bit(agaw, &sagaw)) {
1798                 /* hardware doesn't support it, choose a bigger one */
1799                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1800                 agaw = find_next_bit(&sagaw, 5, agaw);
1801                 if (agaw >= 5)
1802                         return -ENODEV;
1803         }
1804         domain->agaw = agaw;
1805
1806         if (ecap_coherent(iommu->ecap))
1807                 domain->iommu_coherency = 1;
1808         else
1809                 domain->iommu_coherency = 0;
1810
1811         if (ecap_sc_support(iommu->ecap))
1812                 domain->iommu_snooping = 1;
1813         else
1814                 domain->iommu_snooping = 0;
1815
1816         if (intel_iommu_superpage)
1817                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1818         else
1819                 domain->iommu_superpage = 0;
1820
1821         domain->nid = iommu->node;
1822
1823         /* always allocate the top pgd */
1824         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1825         if (!domain->pgd)
1826                 return -ENOMEM;
1827         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1828         return 0;
1829 }
1830
1831 static void domain_exit(struct dmar_domain *domain)
1832 {
1833         struct page *freelist = NULL;
1834         int i;
1835
1836         /* Domain 0 is reserved, so dont process it */
1837         if (!domain)
1838                 return;
1839
1840         /* Flush any lazy unmaps that may reference this domain */
1841         if (!intel_iommu_strict)
1842                 flush_unmaps_timeout(0);
1843
1844         /* remove associated devices */
1845         domain_remove_dev_info(domain);
1846
1847         /* destroy iovas */
1848         put_iova_domain(&domain->iovad);
1849
1850         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1851
1852         /* clear attached or cached domains */
1853         rcu_read_lock();
1854         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1855                 iommu_detach_domain(domain, g_iommus[i]);
1856         rcu_read_unlock();
1857
1858         dma_free_pagelist(freelist);
1859
1860         free_domain_mem(domain);
1861 }
1862
1863 static int domain_context_mapping_one(struct dmar_domain *domain,
1864                                       struct intel_iommu *iommu,
1865                                       u8 bus, u8 devfn, int translation)
1866 {
1867         struct context_entry *context;
1868         unsigned long flags;
1869         struct dma_pte *pgd;
1870         int id;
1871         int agaw;
1872         struct device_domain_info *info = NULL;
1873
1874         pr_debug("Set context mapping for %02x:%02x.%d\n",
1875                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1876
1877         BUG_ON(!domain->pgd);
1878         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1879                translation != CONTEXT_TT_MULTI_LEVEL);
1880
1881         spin_lock_irqsave(&iommu->lock, flags);
1882         context = iommu_context_addr(iommu, bus, devfn, 1);
1883         spin_unlock_irqrestore(&iommu->lock, flags);
1884         if (!context)
1885                 return -ENOMEM;
1886         spin_lock_irqsave(&iommu->lock, flags);
1887         if (context_present(context)) {
1888                 spin_unlock_irqrestore(&iommu->lock, flags);
1889                 return 0;
1890         }
1891
1892         context_clear_entry(context);
1893
1894         id = domain->id;
1895         pgd = domain->pgd;
1896
1897         if (domain_type_is_vm_or_si(domain)) {
1898                 if (domain_type_is_vm(domain)) {
1899                         id = iommu_attach_vm_domain(domain, iommu);
1900                         if (id < 0) {
1901                                 spin_unlock_irqrestore(&iommu->lock, flags);
1902                                 pr_err("%s: No free domain ids\n", iommu->name);
1903                                 return -EFAULT;
1904                         }
1905                 }
1906
1907                 /* Skip top levels of page tables for
1908                  * iommu which has less agaw than default.
1909                  * Unnecessary for PT mode.
1910                  */
1911                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1912                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1913                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1914                                 if (!dma_pte_present(pgd)) {
1915                                         spin_unlock_irqrestore(&iommu->lock, flags);
1916                                         return -ENOMEM;
1917                                 }
1918                         }
1919                 }
1920         }
1921
1922         context_set_domain_id(context, id);
1923
1924         if (translation != CONTEXT_TT_PASS_THROUGH) {
1925                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1926                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1927                                      CONTEXT_TT_MULTI_LEVEL;
1928         }
1929         /*
1930          * In pass through mode, AW must be programmed to indicate the largest
1931          * AGAW value supported by hardware. And ASR is ignored by hardware.
1932          */
1933         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1934                 context_set_address_width(context, iommu->msagaw);
1935         else {
1936                 context_set_address_root(context, virt_to_phys(pgd));
1937                 context_set_address_width(context, iommu->agaw);
1938         }
1939
1940         context_set_translation_type(context, translation);
1941         context_set_fault_enable(context);
1942         context_set_present(context);
1943         domain_flush_cache(domain, context, sizeof(*context));
1944
1945         /*
1946          * It's a non-present to present mapping. If hardware doesn't cache
1947          * non-present entry we only need to flush the write-buffer. If the
1948          * _does_ cache non-present entries, then it does so in the special
1949          * domain #0, which we have to flush:
1950          */
1951         if (cap_caching_mode(iommu->cap)) {
1952                 iommu->flush.flush_context(iommu, 0,
1953                                            (((u16)bus) << 8) | devfn,
1954                                            DMA_CCMD_MASK_NOBIT,
1955                                            DMA_CCMD_DEVICE_INVL);
1956                 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1957         } else {
1958                 iommu_flush_write_buffer(iommu);
1959         }
1960         iommu_enable_dev_iotlb(info);
1961         spin_unlock_irqrestore(&iommu->lock, flags);
1962
1963         domain_attach_iommu(domain, iommu);
1964
1965         return 0;
1966 }
1967
1968 struct domain_context_mapping_data {
1969         struct dmar_domain *domain;
1970         struct intel_iommu *iommu;
1971         int translation;
1972 };
1973
1974 static int domain_context_mapping_cb(struct pci_dev *pdev,
1975                                      u16 alias, void *opaque)
1976 {
1977         struct domain_context_mapping_data *data = opaque;
1978
1979         return domain_context_mapping_one(data->domain, data->iommu,
1980                                           PCI_BUS_NUM(alias), alias & 0xff,
1981                                           data->translation);
1982 }
1983
1984 static int
1985 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1986                        int translation)
1987 {
1988         struct intel_iommu *iommu;
1989         u8 bus, devfn;
1990         struct domain_context_mapping_data data;
1991
1992         iommu = device_to_iommu(dev, &bus, &devfn);
1993         if (!iommu)
1994                 return -ENODEV;
1995
1996         if (!dev_is_pci(dev))
1997                 return domain_context_mapping_one(domain, iommu, bus, devfn,
1998                                                   translation);
1999
2000         data.domain = domain;
2001         data.iommu = iommu;
2002         data.translation = translation;
2003
2004         return pci_for_each_dma_alias(to_pci_dev(dev),
2005                                       &domain_context_mapping_cb, &data);
2006 }
2007
2008 static int domain_context_mapped_cb(struct pci_dev *pdev,
2009                                     u16 alias, void *opaque)
2010 {
2011         struct intel_iommu *iommu = opaque;
2012
2013         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2014 }
2015
2016 static int domain_context_mapped(struct device *dev)
2017 {
2018         struct intel_iommu *iommu;
2019         u8 bus, devfn;
2020
2021         iommu = device_to_iommu(dev, &bus, &devfn);
2022         if (!iommu)
2023                 return -ENODEV;
2024
2025         if (!dev_is_pci(dev))
2026                 return device_context_mapped(iommu, bus, devfn);
2027
2028         return !pci_for_each_dma_alias(to_pci_dev(dev),
2029                                        domain_context_mapped_cb, iommu);
2030 }
2031
2032 /* Returns a number of VTD pages, but aligned to MM page size */
2033 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2034                                             size_t size)
2035 {
2036         host_addr &= ~PAGE_MASK;
2037         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2038 }
2039
2040 /* Return largest possible superpage level for a given mapping */
2041 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2042                                           unsigned long iov_pfn,
2043                                           unsigned long phy_pfn,
2044                                           unsigned long pages)
2045 {
2046         int support, level = 1;
2047         unsigned long pfnmerge;
2048
2049         support = domain->iommu_superpage;
2050
2051         /* To use a large page, the virtual *and* physical addresses
2052            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2053            of them will mean we have to use smaller pages. So just
2054            merge them and check both at once. */
2055         pfnmerge = iov_pfn | phy_pfn;
2056
2057         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2058                 pages >>= VTD_STRIDE_SHIFT;
2059                 if (!pages)
2060                         break;
2061                 pfnmerge >>= VTD_STRIDE_SHIFT;
2062                 level++;
2063                 support--;
2064         }
2065         return level;
2066 }
2067
2068 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2069                             struct scatterlist *sg, unsigned long phys_pfn,
2070                             unsigned long nr_pages, int prot)
2071 {
2072         struct dma_pte *first_pte = NULL, *pte = NULL;
2073         phys_addr_t uninitialized_var(pteval);
2074         unsigned long sg_res = 0;
2075         unsigned int largepage_lvl = 0;
2076         unsigned long lvl_pages = 0;
2077
2078         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2079
2080         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2081                 return -EINVAL;
2082
2083         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2084
2085         if (!sg) {
2086                 sg_res = nr_pages;
2087                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2088         }
2089
2090         while (nr_pages > 0) {
2091                 uint64_t tmp;
2092
2093                 if (!sg_res) {
2094                         sg_res = aligned_nrpages(sg->offset, sg->length);
2095                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2096                         sg->dma_length = sg->length;
2097                         pteval = page_to_phys(sg_page(sg)) | prot;
2098                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2099                 }
2100
2101                 if (!pte) {
2102                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2103
2104                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2105                         if (!pte)
2106                                 return -ENOMEM;
2107                         /* It is large page*/
2108                         if (largepage_lvl > 1) {
2109                                 pteval |= DMA_PTE_LARGE_PAGE;
2110                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2111                                 /*
2112                                  * Ensure that old small page tables are
2113                                  * removed to make room for superpage,
2114                                  * if they exist.
2115                                  */
2116                                 dma_pte_free_pagetable(domain, iov_pfn,
2117                                                        iov_pfn + lvl_pages - 1);
2118                         } else {
2119                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2120                         }
2121
2122                 }
2123                 /* We don't need lock here, nobody else
2124                  * touches the iova range
2125                  */
2126                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2127                 if (tmp) {
2128                         static int dumps = 5;
2129                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2130                                 iov_pfn, tmp, (unsigned long long)pteval);
2131                         if (dumps) {
2132                                 dumps--;
2133                                 debug_dma_dump_mappings(NULL);
2134                         }
2135                         WARN_ON(1);
2136                 }
2137
2138                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2139
2140                 BUG_ON(nr_pages < lvl_pages);
2141                 BUG_ON(sg_res < lvl_pages);
2142
2143                 nr_pages -= lvl_pages;
2144                 iov_pfn += lvl_pages;
2145                 phys_pfn += lvl_pages;
2146                 pteval += lvl_pages * VTD_PAGE_SIZE;
2147                 sg_res -= lvl_pages;
2148
2149                 /* If the next PTE would be the first in a new page, then we
2150                    need to flush the cache on the entries we've just written.
2151                    And then we'll need to recalculate 'pte', so clear it and
2152                    let it get set again in the if (!pte) block above.
2153
2154                    If we're done (!nr_pages) we need to flush the cache too.
2155
2156                    Also if we've been setting superpages, we may need to
2157                    recalculate 'pte' and switch back to smaller pages for the
2158                    end of the mapping, if the trailing size is not enough to
2159                    use another superpage (i.e. sg_res < lvl_pages). */
2160                 pte++;
2161                 if (!nr_pages || first_pte_in_page(pte) ||
2162                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2163                         domain_flush_cache(domain, first_pte,
2164                                            (void *)pte - (void *)first_pte);
2165                         pte = NULL;
2166                 }
2167
2168                 if (!sg_res && nr_pages)
2169                         sg = sg_next(sg);
2170         }
2171         return 0;
2172 }
2173
2174 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175                                     struct scatterlist *sg, unsigned long nr_pages,
2176                                     int prot)
2177 {
2178         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2179 }
2180
2181 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2182                                      unsigned long phys_pfn, unsigned long nr_pages,
2183                                      int prot)
2184 {
2185         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2186 }
2187
2188 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2189 {
2190         if (!iommu)
2191                 return;
2192
2193         clear_context_table(iommu, bus, devfn);
2194         iommu->flush.flush_context(iommu, 0, 0, 0,
2195                                            DMA_CCMD_GLOBAL_INVL);
2196         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2197 }
2198
2199 static inline void unlink_domain_info(struct device_domain_info *info)
2200 {
2201         assert_spin_locked(&device_domain_lock);
2202         list_del(&info->link);
2203         list_del(&info->global);
2204         if (info->dev)
2205                 info->dev->archdata.iommu = NULL;
2206 }
2207
2208 static void domain_remove_dev_info(struct dmar_domain *domain)
2209 {
2210         struct device_domain_info *info, *tmp;
2211         unsigned long flags;
2212
2213         spin_lock_irqsave(&device_domain_lock, flags);
2214         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2215                 unlink_domain_info(info);
2216                 spin_unlock_irqrestore(&device_domain_lock, flags);
2217
2218                 iommu_disable_dev_iotlb(info);
2219                 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2220
2221                 if (domain_type_is_vm(domain)) {
2222                         iommu_detach_dependent_devices(info->iommu, info->dev);
2223                         domain_detach_iommu(domain, info->iommu);
2224                 }
2225
2226                 free_devinfo_mem(info);
2227                 spin_lock_irqsave(&device_domain_lock, flags);
2228         }
2229         spin_unlock_irqrestore(&device_domain_lock, flags);
2230 }
2231
2232 /*
2233  * find_domain
2234  * Note: we use struct device->archdata.iommu stores the info
2235  */
2236 static struct dmar_domain *find_domain(struct device *dev)
2237 {
2238         struct device_domain_info *info;
2239
2240         /* No lock here, assumes no domain exit in normal case */
2241         info = dev->archdata.iommu;
2242         if (info)
2243                 return info->domain;
2244         return NULL;
2245 }
2246
2247 static inline struct device_domain_info *
2248 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2249 {
2250         struct device_domain_info *info;
2251
2252         list_for_each_entry(info, &device_domain_list, global)
2253                 if (info->iommu->segment == segment && info->bus == bus &&
2254                     info->devfn == devfn)
2255                         return info;
2256
2257         return NULL;
2258 }
2259
2260 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2261                                                 int bus, int devfn,
2262                                                 struct device *dev,
2263                                                 struct dmar_domain *domain)
2264 {
2265         struct dmar_domain *found = NULL;
2266         struct device_domain_info *info;
2267         unsigned long flags;
2268
2269         info = alloc_devinfo_mem();
2270         if (!info)
2271                 return NULL;
2272
2273         info->bus = bus;
2274         info->devfn = devfn;
2275         info->dev = dev;
2276         info->domain = domain;
2277         info->iommu = iommu;
2278
2279         spin_lock_irqsave(&device_domain_lock, flags);
2280         if (dev)
2281                 found = find_domain(dev);
2282         else {
2283                 struct device_domain_info *info2;
2284                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2285                 if (info2)
2286                         found = info2->domain;
2287         }
2288         if (found) {
2289                 spin_unlock_irqrestore(&device_domain_lock, flags);
2290                 free_devinfo_mem(info);
2291                 /* Caller must free the original domain */
2292                 return found;
2293         }
2294
2295         list_add(&info->link, &domain->devices);
2296         list_add(&info->global, &device_domain_list);
2297         if (dev)
2298                 dev->archdata.iommu = info;
2299         spin_unlock_irqrestore(&device_domain_lock, flags);
2300
2301         return domain;
2302 }
2303
2304 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2305 {
2306         *(u16 *)opaque = alias;
2307         return 0;
2308 }
2309
2310 /* domain is initialized */
2311 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2312 {
2313         struct dmar_domain *domain, *tmp;
2314         struct intel_iommu *iommu;
2315         struct device_domain_info *info;
2316         u16 dma_alias;
2317         unsigned long flags;
2318         u8 bus, devfn;
2319
2320         domain = find_domain(dev);
2321         if (domain)
2322                 return domain;
2323
2324         iommu = device_to_iommu(dev, &bus, &devfn);
2325         if (!iommu)
2326                 return NULL;
2327
2328         if (dev_is_pci(dev)) {
2329                 struct pci_dev *pdev = to_pci_dev(dev);
2330
2331                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2332
2333                 spin_lock_irqsave(&device_domain_lock, flags);
2334                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2335                                                       PCI_BUS_NUM(dma_alias),
2336                                                       dma_alias & 0xff);
2337                 if (info) {
2338                         iommu = info->iommu;
2339                         domain = info->domain;
2340                 }
2341                 spin_unlock_irqrestore(&device_domain_lock, flags);
2342
2343                 /* DMA alias already has a domain, uses it */
2344                 if (info)
2345                         goto found_domain;
2346         }
2347
2348         /* Allocate and initialize new domain for the device */
2349         domain = alloc_domain(0);
2350         if (!domain)
2351                 return NULL;
2352         domain->id = iommu_attach_domain(domain, iommu);
2353         if (domain->id < 0) {
2354                 free_domain_mem(domain);
2355                 return NULL;
2356         }
2357         domain_attach_iommu(domain, iommu);
2358         if (domain_init(domain, gaw)) {
2359                 domain_exit(domain);
2360                 return NULL;
2361         }
2362
2363         /* register PCI DMA alias device */
2364         if (dev_is_pci(dev)) {
2365                 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2366                                            dma_alias & 0xff, NULL, domain);
2367
2368                 if (!tmp || tmp != domain) {
2369                         domain_exit(domain);
2370                         domain = tmp;
2371                 }
2372
2373                 if (!domain)
2374                         return NULL;
2375         }
2376
2377 found_domain:
2378         tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2379
2380         if (!tmp || tmp != domain) {
2381                 domain_exit(domain);
2382                 domain = tmp;
2383         }
2384
2385         return domain;
2386 }
2387
2388 static int iommu_identity_mapping;
2389 #define IDENTMAP_ALL            1
2390 #define IDENTMAP_GFX            2
2391 #define IDENTMAP_AZALIA         4
2392
2393 static int iommu_domain_identity_map(struct dmar_domain *domain,
2394                                      unsigned long long start,
2395                                      unsigned long long end)
2396 {
2397         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2398         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2399
2400         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2401                           dma_to_mm_pfn(last_vpfn))) {
2402                 pr_err("Reserving iova failed\n");
2403                 return -ENOMEM;
2404         }
2405
2406         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2407                  start, end, domain->id);
2408         /*
2409          * RMRR range might have overlap with physical memory range,
2410          * clear it first
2411          */
2412         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2413
2414         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2415                                   last_vpfn - first_vpfn + 1,
2416                                   DMA_PTE_READ|DMA_PTE_WRITE);
2417 }
2418
2419 static int iommu_prepare_identity_map(struct device *dev,
2420                                       unsigned long long start,
2421                                       unsigned long long end)
2422 {
2423         struct dmar_domain *domain;
2424         int ret;
2425
2426         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2427         if (!domain)
2428                 return -ENOMEM;
2429
2430         /* For _hardware_ passthrough, don't bother. But for software
2431            passthrough, we do it anyway -- it may indicate a memory
2432            range which is reserved in E820, so which didn't get set
2433            up to start with in si_domain */
2434         if (domain == si_domain && hw_pass_through) {
2435                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2436                         dev_name(dev), start, end);
2437                 return 0;
2438         }
2439
2440         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2441                 dev_name(dev), start, end);
2442
2443         if (end < start) {
2444                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2445                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2446                         dmi_get_system_info(DMI_BIOS_VENDOR),
2447                         dmi_get_system_info(DMI_BIOS_VERSION),
2448                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2449                 ret = -EIO;
2450                 goto error;
2451         }
2452
2453         if (end >> agaw_to_width(domain->agaw)) {
2454                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2455                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2456                      agaw_to_width(domain->agaw),
2457                      dmi_get_system_info(DMI_BIOS_VENDOR),
2458                      dmi_get_system_info(DMI_BIOS_VERSION),
2459                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2460                 ret = -EIO;
2461                 goto error;
2462         }
2463
2464         ret = iommu_domain_identity_map(domain, start, end);
2465         if (ret)
2466                 goto error;
2467
2468         /* context entry init */
2469         ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2470         if (ret)
2471                 goto error;
2472
2473         return 0;
2474
2475  error:
2476         domain_exit(domain);
2477         return ret;
2478 }
2479
2480 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2481                                          struct device *dev)
2482 {
2483         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2484                 return 0;
2485         return iommu_prepare_identity_map(dev, rmrr->base_address,
2486                                           rmrr->end_address);
2487 }
2488
2489 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2490 static inline void iommu_prepare_isa(void)
2491 {
2492         struct pci_dev *pdev;
2493         int ret;
2494
2495         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2496         if (!pdev)
2497                 return;
2498
2499         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2500         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2501
2502         if (ret)
2503                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2504
2505         pci_dev_put(pdev);
2506 }
2507 #else
2508 static inline void iommu_prepare_isa(void)
2509 {
2510         return;
2511 }
2512 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2513
2514 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2515
2516 static int __init si_domain_init(int hw)
2517 {
2518         struct dmar_drhd_unit *drhd;
2519         struct intel_iommu *iommu;
2520         int nid, ret = 0;
2521         bool first = true;
2522
2523         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2524         if (!si_domain)
2525                 return -EFAULT;
2526
2527         for_each_active_iommu(iommu, drhd) {
2528                 ret = iommu_attach_domain(si_domain, iommu);
2529                 if (ret < 0) {
2530                         domain_exit(si_domain);
2531                         return -EFAULT;
2532                 } else if (first) {
2533                         si_domain->id = ret;
2534                         first = false;
2535                 } else if (si_domain->id != ret) {
2536                         domain_exit(si_domain);
2537                         return -EFAULT;
2538                 }
2539                 domain_attach_iommu(si_domain, iommu);
2540         }
2541
2542         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2543                 domain_exit(si_domain);
2544                 return -EFAULT;
2545         }
2546
2547         pr_debug("Identity mapping domain is domain %d\n",
2548                  si_domain->id);
2549
2550         if (hw)
2551                 return 0;
2552
2553         for_each_online_node(nid) {
2554                 unsigned long start_pfn, end_pfn;
2555                 int i;
2556
2557                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2558                         ret = iommu_domain_identity_map(si_domain,
2559                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2560                         if (ret)
2561                                 return ret;
2562                 }
2563         }
2564
2565         return 0;
2566 }
2567
2568 static int identity_mapping(struct device *dev)
2569 {
2570         struct device_domain_info *info;
2571
2572         if (likely(!iommu_identity_mapping))
2573                 return 0;
2574
2575         info = dev->archdata.iommu;
2576         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2577                 return (info->domain == si_domain);
2578
2579         return 0;
2580 }
2581
2582 static int domain_add_dev_info(struct dmar_domain *domain,
2583                                struct device *dev, int translation)
2584 {
2585         struct dmar_domain *ndomain;
2586         struct intel_iommu *iommu;
2587         u8 bus, devfn;
2588         int ret;
2589
2590         iommu = device_to_iommu(dev, &bus, &devfn);
2591         if (!iommu)
2592                 return -ENODEV;
2593
2594         ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2595         if (ndomain != domain)
2596                 return -EBUSY;
2597
2598         ret = domain_context_mapping(domain, dev, translation);
2599         if (ret) {
2600                 domain_remove_one_dev_info(domain, dev);
2601                 return ret;
2602         }
2603
2604         return 0;
2605 }
2606
2607 static bool device_has_rmrr(struct device *dev)
2608 {
2609         struct dmar_rmrr_unit *rmrr;
2610         struct device *tmp;
2611         int i;
2612
2613         rcu_read_lock();
2614         for_each_rmrr_units(rmrr) {
2615                 /*
2616                  * Return TRUE if this RMRR contains the device that
2617                  * is passed in.
2618                  */
2619                 for_each_active_dev_scope(rmrr->devices,
2620                                           rmrr->devices_cnt, i, tmp)
2621                         if (tmp == dev) {
2622                                 rcu_read_unlock();
2623                                 return true;
2624                         }
2625         }
2626         rcu_read_unlock();
2627         return false;
2628 }
2629
2630 /*
2631  * There are a couple cases where we need to restrict the functionality of
2632  * devices associated with RMRRs.  The first is when evaluating a device for
2633  * identity mapping because problems exist when devices are moved in and out
2634  * of domains and their respective RMRR information is lost.  This means that
2635  * a device with associated RMRRs will never be in a "passthrough" domain.
2636  * The second is use of the device through the IOMMU API.  This interface
2637  * expects to have full control of the IOVA space for the device.  We cannot
2638  * satisfy both the requirement that RMRR access is maintained and have an
2639  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2640  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2641  * We therefore prevent devices associated with an RMRR from participating in
2642  * the IOMMU API, which eliminates them from device assignment.
2643  *
2644  * In both cases we assume that PCI USB devices with RMRRs have them largely
2645  * for historical reasons and that the RMRR space is not actively used post
2646  * boot.  This exclusion may change if vendors begin to abuse it.
2647  *
2648  * The same exception is made for graphics devices, with the requirement that
2649  * any use of the RMRR regions will be torn down before assigning the device
2650  * to a guest.
2651  */
2652 static bool device_is_rmrr_locked(struct device *dev)
2653 {
2654         if (!device_has_rmrr(dev))
2655                 return false;
2656
2657         if (dev_is_pci(dev)) {
2658                 struct pci_dev *pdev = to_pci_dev(dev);
2659
2660                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2661                         return false;
2662         }
2663
2664         return true;
2665 }
2666
2667 static int iommu_should_identity_map(struct device *dev, int startup)
2668 {
2669
2670         if (dev_is_pci(dev)) {
2671                 struct pci_dev *pdev = to_pci_dev(dev);
2672
2673                 if (device_is_rmrr_locked(dev))
2674                         return 0;
2675
2676                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2677                         return 1;
2678
2679                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2680                         return 1;
2681
2682                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2683                         return 0;
2684
2685                 /*
2686                  * We want to start off with all devices in the 1:1 domain, and
2687                  * take them out later if we find they can't access all of memory.
2688                  *
2689                  * However, we can't do this for PCI devices behind bridges,
2690                  * because all PCI devices behind the same bridge will end up
2691                  * with the same source-id on their transactions.
2692                  *
2693                  * Practically speaking, we can't change things around for these
2694                  * devices at run-time, because we can't be sure there'll be no
2695                  * DMA transactions in flight for any of their siblings.
2696                  *
2697                  * So PCI devices (unless they're on the root bus) as well as
2698                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2699                  * the 1:1 domain, just in _case_ one of their siblings turns out
2700                  * not to be able to map all of memory.
2701                  */
2702                 if (!pci_is_pcie(pdev)) {
2703                         if (!pci_is_root_bus(pdev->bus))
2704                                 return 0;
2705                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2706                                 return 0;
2707                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2708                         return 0;
2709         } else {
2710                 if (device_has_rmrr(dev))
2711                         return 0;
2712         }
2713
2714         /*
2715          * At boot time, we don't yet know if devices will be 64-bit capable.
2716          * Assume that they will — if they turn out not to be, then we can
2717          * take them out of the 1:1 domain later.
2718          */
2719         if (!startup) {
2720                 /*
2721                  * If the device's dma_mask is less than the system's memory
2722                  * size then this is not a candidate for identity mapping.
2723                  */
2724                 u64 dma_mask = *dev->dma_mask;
2725
2726                 if (dev->coherent_dma_mask &&
2727                     dev->coherent_dma_mask < dma_mask)
2728                         dma_mask = dev->coherent_dma_mask;
2729
2730                 return dma_mask >= dma_get_required_mask(dev);
2731         }
2732
2733         return 1;
2734 }
2735
2736 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2737 {
2738         int ret;
2739
2740         if (!iommu_should_identity_map(dev, 1))
2741                 return 0;
2742
2743         ret = domain_add_dev_info(si_domain, dev,
2744                                   hw ? CONTEXT_TT_PASS_THROUGH :
2745                                        CONTEXT_TT_MULTI_LEVEL);
2746         if (!ret)
2747                 pr_info("%s identity mapping for device %s\n",
2748                         hw ? "Hardware" : "Software", dev_name(dev));
2749         else if (ret == -ENODEV)
2750                 /* device not associated with an iommu */
2751                 ret = 0;
2752
2753         return ret;
2754 }
2755
2756
2757 static int __init iommu_prepare_static_identity_mapping(int hw)
2758 {
2759         struct pci_dev *pdev = NULL;
2760         struct dmar_drhd_unit *drhd;
2761         struct intel_iommu *iommu;
2762         struct device *dev;
2763         int i;
2764         int ret = 0;
2765
2766         for_each_pci_dev(pdev) {
2767                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2768                 if (ret)
2769                         return ret;
2770         }
2771
2772         for_each_active_iommu(iommu, drhd)
2773                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2774                         struct acpi_device_physical_node *pn;
2775                         struct acpi_device *adev;
2776
2777                         if (dev->bus != &acpi_bus_type)
2778                                 continue;
2779
2780                         adev= to_acpi_device(dev);
2781                         mutex_lock(&adev->physical_node_lock);
2782                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2783                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2784                                 if (ret)
2785                                         break;
2786                         }
2787                         mutex_unlock(&adev->physical_node_lock);
2788                         if (ret)
2789                                 return ret;
2790                 }
2791
2792         return 0;
2793 }
2794
2795 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2796 {
2797         /*
2798          * Start from the sane iommu hardware state.
2799          * If the queued invalidation is already initialized by us
2800          * (for example, while enabling interrupt-remapping) then
2801          * we got the things already rolling from a sane state.
2802          */
2803         if (!iommu->qi) {
2804                 /*
2805                  * Clear any previous faults.
2806                  */
2807                 dmar_fault(-1, iommu);
2808                 /*
2809                  * Disable queued invalidation if supported and already enabled
2810                  * before OS handover.
2811                  */
2812                 dmar_disable_qi(iommu);
2813         }
2814
2815         if (dmar_enable_qi(iommu)) {
2816                 /*
2817                  * Queued Invalidate not enabled, use Register Based Invalidate
2818                  */
2819                 iommu->flush.flush_context = __iommu_flush_context;
2820                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2821                 pr_info("%s: Using Register based invalidation\n",
2822                         iommu->name);
2823         } else {
2824                 iommu->flush.flush_context = qi_flush_context;
2825                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2826                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2827         }
2828 }
2829
2830 static int copy_context_table(struct intel_iommu *iommu,
2831                               struct root_entry *old_re,
2832                               struct context_entry **tbl,
2833                               int bus, bool ext)
2834 {
2835         struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
2836         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2837         phys_addr_t old_ce_phys;
2838
2839         tbl_idx = ext ? bus * 2 : bus;
2840
2841         for (devfn = 0; devfn < 256; devfn++) {
2842                 /* First calculate the correct index */
2843                 idx = (ext ? devfn * 2 : devfn) % 256;
2844
2845                 if (idx == 0) {
2846                         /* First save what we may have and clean up */
2847                         if (new_ce) {
2848                                 tbl[tbl_idx] = new_ce;
2849                                 __iommu_flush_cache(iommu, new_ce,
2850                                                     VTD_PAGE_SIZE);
2851                                 pos = 1;
2852                         }
2853
2854                         if (old_ce)
2855                                 iounmap(old_ce);
2856
2857                         ret = 0;
2858                         if (devfn < 0x80)
2859                                 old_ce_phys = root_entry_lctp(old_re);
2860                         else
2861                                 old_ce_phys = root_entry_uctp(old_re);
2862
2863                         if (!old_ce_phys) {
2864                                 if (ext && devfn == 0) {
2865                                         /* No LCTP, try UCTP */
2866                                         devfn = 0x7f;
2867                                         continue;
2868                                 } else {
2869                                         goto out;
2870                                 }
2871                         }
2872
2873                         ret = -ENOMEM;
2874                         old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2875                         if (!old_ce)
2876                                 goto out;
2877
2878                         new_ce = alloc_pgtable_page(iommu->node);
2879                         if (!new_ce)
2880                                 goto out_unmap;
2881
2882                         ret = 0;
2883                 }
2884
2885                 /* Now copy the context entry */
2886                 ce = old_ce[idx];
2887
2888                 if (!__context_present(&ce))
2889                         continue;
2890
2891                 did = context_domain_id(&ce);
2892                 if (did >= 0 && did < cap_ndoms(iommu->cap))
2893                         set_bit(did, iommu->domain_ids);
2894
2895                 /*
2896                  * We need a marker for copied context entries. This
2897                  * marker needs to work for the old format as well as
2898                  * for extended context entries.
2899                  *
2900                  * Bit 67 of the context entry is used. In the old
2901                  * format this bit is available to software, in the
2902                  * extended format it is the PGE bit, but PGE is ignored
2903                  * by HW if PASIDs are disabled (and thus still
2904                  * available).
2905                  *
2906                  * So disable PASIDs first and then mark the entry
2907                  * copied. This means that we don't copy PASID
2908                  * translations from the old kernel, but this is fine as
2909                  * faults there are not fatal.
2910                  */
2911                 context_clear_pasid_enable(&ce);
2912                 context_set_copied(&ce);
2913
2914                 new_ce[idx] = ce;
2915         }
2916
2917         tbl[tbl_idx + pos] = new_ce;
2918
2919         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2920
2921 out_unmap:
2922         iounmap(old_ce);
2923
2924 out:
2925         return ret;
2926 }
2927
2928 static int copy_translation_tables(struct intel_iommu *iommu)
2929 {
2930         struct context_entry **ctxt_tbls;
2931         struct root_entry *old_rt;
2932         phys_addr_t old_rt_phys;
2933         int ctxt_table_entries;
2934         unsigned long flags;
2935         u64 rtaddr_reg;
2936         int bus, ret;
2937         bool new_ext, ext;
2938
2939         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2940         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
2941         new_ext    = !!ecap_ecs(iommu->ecap);
2942
2943         /*
2944          * The RTT bit can only be changed when translation is disabled,
2945          * but disabling translation means to open a window for data
2946          * corruption. So bail out and don't copy anything if we would
2947          * have to change the bit.
2948          */
2949         if (new_ext != ext)
2950                 return -EINVAL;
2951
2952         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2953         if (!old_rt_phys)
2954                 return -EINVAL;
2955
2956         old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2957         if (!old_rt)
2958                 return -ENOMEM;
2959
2960         /* This is too big for the stack - allocate it from slab */
2961         ctxt_table_entries = ext ? 512 : 256;
2962         ret = -ENOMEM;
2963         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2964         if (!ctxt_tbls)
2965                 goto out_unmap;
2966
2967         for (bus = 0; bus < 256; bus++) {
2968                 ret = copy_context_table(iommu, &old_rt[bus],
2969                                          ctxt_tbls, bus, ext);
2970                 if (ret) {
2971                         pr_err("%s: Failed to copy context table for bus %d\n",
2972                                 iommu->name, bus);
2973                         continue;
2974                 }
2975         }
2976
2977         spin_lock_irqsave(&iommu->lock, flags);
2978
2979         /* Context tables are copied, now write them to the root_entry table */
2980         for (bus = 0; bus < 256; bus++) {
2981                 int idx = ext ? bus * 2 : bus;
2982                 u64 val;
2983
2984                 if (ctxt_tbls[idx]) {
2985                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
2986                         iommu->root_entry[bus].lo = val;
2987                 }
2988
2989                 if (!ext || !ctxt_tbls[idx + 1])
2990                         continue;
2991
2992                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2993                 iommu->root_entry[bus].hi = val;
2994         }
2995
2996         spin_unlock_irqrestore(&iommu->lock, flags);
2997
2998         kfree(ctxt_tbls);
2999
3000         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3001
3002         ret = 0;
3003
3004 out_unmap:
3005         iounmap(old_rt);
3006
3007         return ret;
3008 }
3009
3010 static int __init init_dmars(void)
3011 {
3012         struct dmar_drhd_unit *drhd;
3013         struct dmar_rmrr_unit *rmrr;
3014         bool copied_tables = false;
3015         struct device *dev;
3016         struct intel_iommu *iommu;
3017         int i, ret;
3018
3019         /*
3020          * for each drhd
3021          *    allocate root
3022          *    initialize and program root entry to not present
3023          * endfor
3024          */
3025         for_each_drhd_unit(drhd) {
3026                 /*
3027                  * lock not needed as this is only incremented in the single
3028                  * threaded kernel __init code path all other access are read
3029                  * only
3030                  */
3031                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3032                         g_num_of_iommus++;
3033                         continue;
3034                 }
3035                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3036         }
3037
3038         /* Preallocate enough resources for IOMMU hot-addition */
3039         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3040                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3041
3042         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3043                         GFP_KERNEL);
3044         if (!g_iommus) {
3045                 pr_err("Allocating global iommu array failed\n");
3046                 ret = -ENOMEM;
3047                 goto error;
3048         }
3049
3050         deferred_flush = kzalloc(g_num_of_iommus *
3051                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3052         if (!deferred_flush) {
3053                 ret = -ENOMEM;
3054                 goto free_g_iommus;
3055         }
3056
3057         for_each_active_iommu(iommu, drhd) {
3058                 g_iommus[iommu->seq_id] = iommu;
3059
3060                 intel_iommu_init_qi(iommu);
3061
3062                 ret = iommu_init_domains(iommu);
3063                 if (ret)
3064                         goto free_iommu;
3065
3066                 init_translation_status(iommu);
3067
3068                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3069                         iommu_disable_translation(iommu);
3070                         clear_translation_pre_enabled(iommu);
3071                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3072                                 iommu->name);
3073                 }
3074
3075                 /*
3076                  * TBD:
3077                  * we could share the same root & context tables
3078                  * among all IOMMU's. Need to Split it later.
3079                  */
3080                 ret = iommu_alloc_root_entry(iommu);
3081                 if (ret)
3082                         goto free_iommu;
3083
3084                 if (translation_pre_enabled(iommu)) {
3085                         pr_info("Translation already enabled - trying to copy translation structures\n");
3086
3087                         ret = copy_translation_tables(iommu);
3088                         if (ret) {
3089                                 /*
3090                                  * We found the IOMMU with translation
3091                                  * enabled - but failed to copy over the
3092                                  * old root-entry table. Try to proceed
3093                                  * by disabling translation now and
3094                                  * allocating a clean root-entry table.
3095                                  * This might cause DMAR faults, but
3096                                  * probably the dump will still succeed.
3097                                  */
3098                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3099                                        iommu->name);
3100                                 iommu_disable_translation(iommu);
3101                                 clear_translation_pre_enabled(iommu);
3102                         } else {
3103                                 pr_info("Copied translation tables from previous kernel for %s\n",
3104                                         iommu->name);
3105                                 copied_tables = true;
3106                         }
3107                 }
3108
3109                 iommu_flush_write_buffer(iommu);
3110                 iommu_set_root_entry(iommu);
3111                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3112                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3113
3114                 if (!ecap_pass_through(iommu->ecap))
3115                         hw_pass_through = 0;
3116         }
3117
3118         if (iommu_pass_through)
3119                 iommu_identity_mapping |= IDENTMAP_ALL;
3120
3121 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3122         iommu_identity_mapping |= IDENTMAP_GFX;
3123 #endif
3124
3125         if (iommu_identity_mapping) {
3126                 ret = si_domain_init(hw_pass_through);
3127                 if (ret)
3128                         goto free_iommu;
3129         }
3130
3131         check_tylersburg_isoch();
3132
3133         /*
3134          * If we copied translations from a previous kernel in the kdump
3135          * case, we can not assign the devices to domains now, as that
3136          * would eliminate the old mappings. So skip this part and defer
3137          * the assignment to device driver initialization time.
3138          */
3139         if (copied_tables)
3140                 goto domains_done;
3141
3142         /*
3143          * If pass through is not set or not enabled, setup context entries for
3144          * identity mappings for rmrr, gfx, and isa and may fall back to static
3145          * identity mapping if iommu_identity_mapping is set.
3146          */
3147         if (iommu_identity_mapping) {
3148                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3149                 if (ret) {
3150                         pr_crit("Failed to setup IOMMU pass-through\n");
3151                         goto free_iommu;
3152                 }
3153         }
3154         /*
3155          * For each rmrr
3156          *   for each dev attached to rmrr
3157          *   do
3158          *     locate drhd for dev, alloc domain for dev
3159          *     allocate free domain
3160          *     allocate page table entries for rmrr
3161          *     if context not allocated for bus
3162          *           allocate and init context
3163          *           set present in root table for this bus
3164          *     init context with domain, translation etc
3165          *    endfor
3166          * endfor
3167          */
3168         pr_info("Setting RMRR:\n");
3169         for_each_rmrr_units(rmrr) {
3170                 /* some BIOS lists non-exist devices in DMAR table. */
3171                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3172                                           i, dev) {
3173                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3174                         if (ret)
3175                                 pr_err("Mapping reserved region failed\n");
3176                 }
3177         }
3178
3179         iommu_prepare_isa();
3180
3181 domains_done:
3182
3183         /*
3184          * for each drhd
3185          *   enable fault log
3186          *   global invalidate context cache
3187          *   global invalidate iotlb
3188          *   enable translation
3189          */
3190         for_each_iommu(iommu, drhd) {
3191                 if (drhd->ignored) {
3192                         /*
3193                          * we always have to disable PMRs or DMA may fail on
3194                          * this device
3195                          */
3196                         if (force_on)
3197                                 iommu_disable_protect_mem_regions(iommu);
3198                         continue;
3199                 }
3200
3201                 iommu_flush_write_buffer(iommu);
3202
3203                 ret = dmar_set_interrupt(iommu);
3204                 if (ret)
3205                         goto free_iommu;
3206