Merge branch 'pcmcia' of git://git.kernel.org/pub/scm/linux/kernel/git/brodo/pcmcia
[sfrench/cifs-2.6.git] / drivers / iommu / intel-iommu.c
1 /*
2  * Copyright © 2006-2014 Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * Authors: David Woodhouse <dwmw2@infradead.org>,
14  *          Ashok Raj <ashok.raj@intel.com>,
15  *          Shaohua Li <shaohua.li@intel.com>,
16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17  *          Fenghua Yu <fenghua.yu@intel.com>
18  *          Joerg Roedel <jroedel@suse.de>
19  */
20
21 #define pr_fmt(fmt)     "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE               VTD_PAGE_SIZE
56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START      (0xfee00000)
64 #define IOAPIC_RANGE_END        (0xfeefffff)
65 #define IOVA_START_ADDR         (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN          (1)
83
84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
85
86 /* page table handling */
87 #define LEVEL_STRIDE            (9)
88 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
89
90 /*
91  * This bitmap is used to advertise the page sizes our hardware support
92  * to the IOMMU core, which will then use this information to split
93  * physically contiguous memory regions it is mapping into page sizes
94  * that we support.
95  *
96  * Traditionally the IOMMU core just handed us the mappings directly,
97  * after making sure the size is an order of a 4KiB page and that the
98  * mapping has natural alignment.
99  *
100  * To retain this behavior, we currently advertise that we support
101  * all page sizes that are an order of 4KiB.
102  *
103  * If at some point we'd like to utilize the IOMMU core's new behavior,
104  * we could change this to advertise the real page sizes we support.
105  */
106 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
107
108 static inline int agaw_to_level(int agaw)
109 {
110         return agaw + 2;
111 }
112
113 static inline int agaw_to_width(int agaw)
114 {
115         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
116 }
117
118 static inline int width_to_agaw(int width)
119 {
120         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
121 }
122
123 static inline unsigned int level_to_offset_bits(int level)
124 {
125         return (level - 1) * LEVEL_STRIDE;
126 }
127
128 static inline int pfn_level_offset(unsigned long pfn, int level)
129 {
130         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131 }
132
133 static inline unsigned long level_mask(int level)
134 {
135         return -1UL << level_to_offset_bits(level);
136 }
137
138 static inline unsigned long level_size(int level)
139 {
140         return 1UL << level_to_offset_bits(level);
141 }
142
143 static inline unsigned long align_to_level(unsigned long pfn, int level)
144 {
145         return (pfn + level_size(level) - 1) & level_mask(level);
146 }
147
148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149 {
150         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
151 }
152
153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154    are never going to work. */
155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156 {
157         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158 }
159
160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161 {
162         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163 }
164 static inline unsigned long page_to_dma_pfn(struct page *pg)
165 {
166         return mm_to_dma_pfn(page_to_pfn(pg));
167 }
168 static inline unsigned long virt_to_dma_pfn(void *p)
169 {
170         return page_to_dma_pfn(virt_to_page(p));
171 }
172
173 /* global iommu list, set NULL for ignored DMAR units */
174 static struct intel_iommu **g_iommus;
175
176 static void __init check_tylersburg_isoch(void);
177 static int rwbf_quirk;
178
179 /*
180  * set to 1 to panic kernel if can't successfully enable VT-d
181  * (used when kernel is launched w/ TXT)
182  */
183 static int force_on = 0;
184 int intel_iommu_tboot_noforce;
185
186 /*
187  * 0: Present
188  * 1-11: Reserved
189  * 12-63: Context Ptr (12 - (haw-1))
190  * 64-127: Reserved
191  */
192 struct root_entry {
193         u64     lo;
194         u64     hi;
195 };
196 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
197
198 /*
199  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
200  * if marked present.
201  */
202 static phys_addr_t root_entry_lctp(struct root_entry *re)
203 {
204         if (!(re->lo & 1))
205                 return 0;
206
207         return re->lo & VTD_PAGE_MASK;
208 }
209
210 /*
211  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
212  * if marked present.
213  */
214 static phys_addr_t root_entry_uctp(struct root_entry *re)
215 {
216         if (!(re->hi & 1))
217                 return 0;
218
219         return re->hi & VTD_PAGE_MASK;
220 }
221 /*
222  * low 64 bits:
223  * 0: present
224  * 1: fault processing disable
225  * 2-3: translation type
226  * 12-63: address space root
227  * high 64 bits:
228  * 0-2: address width
229  * 3-6: aval
230  * 8-23: domain id
231  */
232 struct context_entry {
233         u64 lo;
234         u64 hi;
235 };
236
237 static inline void context_clear_pasid_enable(struct context_entry *context)
238 {
239         context->lo &= ~(1ULL << 11);
240 }
241
242 static inline bool context_pasid_enabled(struct context_entry *context)
243 {
244         return !!(context->lo & (1ULL << 11));
245 }
246
247 static inline void context_set_copied(struct context_entry *context)
248 {
249         context->hi |= (1ull << 3);
250 }
251
252 static inline bool context_copied(struct context_entry *context)
253 {
254         return !!(context->hi & (1ULL << 3));
255 }
256
257 static inline bool __context_present(struct context_entry *context)
258 {
259         return (context->lo & 1);
260 }
261
262 static inline bool context_present(struct context_entry *context)
263 {
264         return context_pasid_enabled(context) ?
265              __context_present(context) :
266              __context_present(context) && !context_copied(context);
267 }
268
269 static inline void context_set_present(struct context_entry *context)
270 {
271         context->lo |= 1;
272 }
273
274 static inline void context_set_fault_enable(struct context_entry *context)
275 {
276         context->lo &= (((u64)-1) << 2) | 1;
277 }
278
279 static inline void context_set_translation_type(struct context_entry *context,
280                                                 unsigned long value)
281 {
282         context->lo &= (((u64)-1) << 4) | 3;
283         context->lo |= (value & 3) << 2;
284 }
285
286 static inline void context_set_address_root(struct context_entry *context,
287                                             unsigned long value)
288 {
289         context->lo &= ~VTD_PAGE_MASK;
290         context->lo |= value & VTD_PAGE_MASK;
291 }
292
293 static inline void context_set_address_width(struct context_entry *context,
294                                              unsigned long value)
295 {
296         context->hi |= value & 7;
297 }
298
299 static inline void context_set_domain_id(struct context_entry *context,
300                                          unsigned long value)
301 {
302         context->hi |= (value & ((1 << 16) - 1)) << 8;
303 }
304
305 static inline int context_domain_id(struct context_entry *c)
306 {
307         return((c->hi >> 8) & 0xffff);
308 }
309
310 static inline void context_clear_entry(struct context_entry *context)
311 {
312         context->lo = 0;
313         context->hi = 0;
314 }
315
316 /*
317  * 0: readable
318  * 1: writable
319  * 2-6: reserved
320  * 7: super page
321  * 8-10: available
322  * 11: snoop behavior
323  * 12-63: Host physcial address
324  */
325 struct dma_pte {
326         u64 val;
327 };
328
329 static inline void dma_clear_pte(struct dma_pte *pte)
330 {
331         pte->val = 0;
332 }
333
334 static inline u64 dma_pte_addr(struct dma_pte *pte)
335 {
336 #ifdef CONFIG_64BIT
337         return pte->val & VTD_PAGE_MASK;
338 #else
339         /* Must have a full atomic 64-bit read */
340         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
341 #endif
342 }
343
344 static inline bool dma_pte_present(struct dma_pte *pte)
345 {
346         return (pte->val & 3) != 0;
347 }
348
349 static inline bool dma_pte_superpage(struct dma_pte *pte)
350 {
351         return (pte->val & DMA_PTE_LARGE_PAGE);
352 }
353
354 static inline int first_pte_in_page(struct dma_pte *pte)
355 {
356         return !((unsigned long)pte & ~VTD_PAGE_MASK);
357 }
358
359 /*
360  * This domain is a statically identity mapping domain.
361  *      1. This domain creats a static 1:1 mapping to all usable memory.
362  *      2. It maps to each iommu if successful.
363  *      3. Each iommu mapps to this domain if successful.
364  */
365 static struct dmar_domain *si_domain;
366 static int hw_pass_through = 1;
367
368 /*
369  * Domain represents a virtual machine, more than one devices
370  * across iommus may be owned in one domain, e.g. kvm guest.
371  */
372 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
373
374 /* si_domain contains mulitple devices */
375 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
376
377 #define for_each_domain_iommu(idx, domain)                      \
378         for (idx = 0; idx < g_num_of_iommus; idx++)             \
379                 if (domain->iommu_refcnt[idx])
380
381 struct dmar_domain {
382         int     nid;                    /* node id */
383
384         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
385                                         /* Refcount of devices per iommu */
386
387
388         u16             iommu_did[DMAR_UNITS_SUPPORTED];
389                                         /* Domain ids per IOMMU. Use u16 since
390                                          * domain ids are 16 bit wide according
391                                          * to VT-d spec, section 9.3 */
392
393         bool has_iotlb_device;
394         struct list_head devices;       /* all devices' list */
395         struct iova_domain iovad;       /* iova's that belong to this domain */
396
397         struct dma_pte  *pgd;           /* virtual address */
398         int             gaw;            /* max guest address width */
399
400         /* adjusted guest address width, 0 is level 2 30-bit */
401         int             agaw;
402
403         int             flags;          /* flags to find out type of domain */
404
405         int             iommu_coherency;/* indicate coherency of iommu access */
406         int             iommu_snooping; /* indicate snooping control feature*/
407         int             iommu_count;    /* reference count of iommu */
408         int             iommu_superpage;/* Level of superpages supported:
409                                            0 == 4KiB (no superpages), 1 == 2MiB,
410                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
411         u64             max_addr;       /* maximum mapped address */
412
413         struct iommu_domain domain;     /* generic domain data structure for
414                                            iommu core */
415 };
416
417 /* PCI domain-device relationship */
418 struct device_domain_info {
419         struct list_head link;  /* link to domain siblings */
420         struct list_head global; /* link to global list */
421         u8 bus;                 /* PCI bus number */
422         u8 devfn;               /* PCI devfn number */
423         u8 pasid_supported:3;
424         u8 pasid_enabled:1;
425         u8 pri_supported:1;
426         u8 pri_enabled:1;
427         u8 ats_supported:1;
428         u8 ats_enabled:1;
429         u8 ats_qdep;
430         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
431         struct intel_iommu *iommu; /* IOMMU used by this device */
432         struct dmar_domain *domain; /* pointer to domain */
433 };
434
435 struct dmar_rmrr_unit {
436         struct list_head list;          /* list of rmrr units   */
437         struct acpi_dmar_header *hdr;   /* ACPI header          */
438         u64     base_address;           /* reserved base address*/
439         u64     end_address;            /* reserved end address */
440         struct dmar_dev_scope *devices; /* target devices */
441         int     devices_cnt;            /* target device count */
442         struct iommu_resv_region *resv; /* reserved region handle */
443 };
444
445 struct dmar_atsr_unit {
446         struct list_head list;          /* list of ATSR units */
447         struct acpi_dmar_header *hdr;   /* ACPI header */
448         struct dmar_dev_scope *devices; /* target devices */
449         int devices_cnt;                /* target device count */
450         u8 include_all:1;               /* include all ports */
451 };
452
453 static LIST_HEAD(dmar_atsr_units);
454 static LIST_HEAD(dmar_rmrr_units);
455
456 #define for_each_rmrr_units(rmrr) \
457         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
459 /* bitmap for indexing intel_iommus */
460 static int g_num_of_iommus;
461
462 static void domain_exit(struct dmar_domain *domain);
463 static void domain_remove_dev_info(struct dmar_domain *domain);
464 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
465                                      struct device *dev);
466 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
467 static void domain_context_clear(struct intel_iommu *iommu,
468                                  struct device *dev);
469 static int domain_detach_iommu(struct dmar_domain *domain,
470                                struct intel_iommu *iommu);
471
472 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
473 int dmar_disabled = 0;
474 #else
475 int dmar_disabled = 1;
476 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
477
478 int intel_iommu_enabled = 0;
479 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
480
481 static int dmar_map_gfx = 1;
482 static int dmar_forcedac;
483 static int intel_iommu_strict;
484 static int intel_iommu_superpage = 1;
485 static int intel_iommu_ecs = 1;
486 static int intel_iommu_pasid28;
487 static int iommu_identity_mapping;
488
489 #define IDENTMAP_ALL            1
490 #define IDENTMAP_GFX            2
491 #define IDENTMAP_AZALIA         4
492
493 /* Broadwell and Skylake have broken ECS support — normal so-called "second
494  * level" translation of DMA requests-without-PASID doesn't actually happen
495  * unless you also set the NESTE bit in an extended context-entry. Which of
496  * course means that SVM doesn't work because it's trying to do nested
497  * translation of the physical addresses it finds in the process page tables,
498  * through the IOVA->phys mapping found in the "second level" page tables.
499  *
500  * The VT-d specification was retroactively changed to change the definition
501  * of the capability bits and pretend that Broadwell/Skylake never happened...
502  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
503  * for some reason it was the PASID capability bit which was redefined (from
504  * bit 28 on BDW/SKL to bit 40 in future).
505  *
506  * So our test for ECS needs to eschew those implementations which set the old
507  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
508  * Unless we are working around the 'pasid28' limitations, that is, by putting
509  * the device into passthrough mode for normal DMA and thus masking the bug.
510  */
511 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
512                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
513 /* PASID support is thus enabled if ECS is enabled and *either* of the old
514  * or new capability bits are set. */
515 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
516                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
517
518 int intel_iommu_gfx_mapped;
519 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
520
521 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
522 static DEFINE_SPINLOCK(device_domain_lock);
523 static LIST_HEAD(device_domain_list);
524
525 const struct iommu_ops intel_iommu_ops;
526
527 static bool translation_pre_enabled(struct intel_iommu *iommu)
528 {
529         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
530 }
531
532 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
533 {
534         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
535 }
536
537 static void init_translation_status(struct intel_iommu *iommu)
538 {
539         u32 gsts;
540
541         gsts = readl(iommu->reg + DMAR_GSTS_REG);
542         if (gsts & DMA_GSTS_TES)
543                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
544 }
545
546 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
547 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
548 {
549         return container_of(dom, struct dmar_domain, domain);
550 }
551
552 static int __init intel_iommu_setup(char *str)
553 {
554         if (!str)
555                 return -EINVAL;
556         while (*str) {
557                 if (!strncmp(str, "on", 2)) {
558                         dmar_disabled = 0;
559                         pr_info("IOMMU enabled\n");
560                 } else if (!strncmp(str, "off", 3)) {
561                         dmar_disabled = 1;
562                         pr_info("IOMMU disabled\n");
563                 } else if (!strncmp(str, "igfx_off", 8)) {
564                         dmar_map_gfx = 0;
565                         pr_info("Disable GFX device mapping\n");
566                 } else if (!strncmp(str, "forcedac", 8)) {
567                         pr_info("Forcing DAC for PCI devices\n");
568                         dmar_forcedac = 1;
569                 } else if (!strncmp(str, "strict", 6)) {
570                         pr_info("Disable batched IOTLB flush\n");
571                         intel_iommu_strict = 1;
572                 } else if (!strncmp(str, "sp_off", 6)) {
573                         pr_info("Disable supported super page\n");
574                         intel_iommu_superpage = 0;
575                 } else if (!strncmp(str, "ecs_off", 7)) {
576                         printk(KERN_INFO
577                                 "Intel-IOMMU: disable extended context table support\n");
578                         intel_iommu_ecs = 0;
579                 } else if (!strncmp(str, "pasid28", 7)) {
580                         printk(KERN_INFO
581                                 "Intel-IOMMU: enable pre-production PASID support\n");
582                         intel_iommu_pasid28 = 1;
583                         iommu_identity_mapping |= IDENTMAP_GFX;
584                 } else if (!strncmp(str, "tboot_noforce", 13)) {
585                         printk(KERN_INFO
586                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
587                         intel_iommu_tboot_noforce = 1;
588                 }
589
590                 str += strcspn(str, ",");
591                 while (*str == ',')
592                         str++;
593         }
594         return 0;
595 }
596 __setup("intel_iommu=", intel_iommu_setup);
597
598 static struct kmem_cache *iommu_domain_cache;
599 static struct kmem_cache *iommu_devinfo_cache;
600
601 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
602 {
603         struct dmar_domain **domains;
604         int idx = did >> 8;
605
606         domains = iommu->domains[idx];
607         if (!domains)
608                 return NULL;
609
610         return domains[did & 0xff];
611 }
612
613 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
614                              struct dmar_domain *domain)
615 {
616         struct dmar_domain **domains;
617         int idx = did >> 8;
618
619         if (!iommu->domains[idx]) {
620                 size_t size = 256 * sizeof(struct dmar_domain *);
621                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
622         }
623
624         domains = iommu->domains[idx];
625         if (WARN_ON(!domains))
626                 return;
627         else
628                 domains[did & 0xff] = domain;
629 }
630
631 static inline void *alloc_pgtable_page(int node)
632 {
633         struct page *page;
634         void *vaddr = NULL;
635
636         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
637         if (page)
638                 vaddr = page_address(page);
639         return vaddr;
640 }
641
642 static inline void free_pgtable_page(void *vaddr)
643 {
644         free_page((unsigned long)vaddr);
645 }
646
647 static inline void *alloc_domain_mem(void)
648 {
649         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
650 }
651
652 static void free_domain_mem(void *vaddr)
653 {
654         kmem_cache_free(iommu_domain_cache, vaddr);
655 }
656
657 static inline void * alloc_devinfo_mem(void)
658 {
659         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
660 }
661
662 static inline void free_devinfo_mem(void *vaddr)
663 {
664         kmem_cache_free(iommu_devinfo_cache, vaddr);
665 }
666
667 static inline int domain_type_is_vm(struct dmar_domain *domain)
668 {
669         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
670 }
671
672 static inline int domain_type_is_si(struct dmar_domain *domain)
673 {
674         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
675 }
676
677 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
678 {
679         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
680                                 DOMAIN_FLAG_STATIC_IDENTITY);
681 }
682
683 static inline int domain_pfn_supported(struct dmar_domain *domain,
684                                        unsigned long pfn)
685 {
686         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
687
688         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
689 }
690
691 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
692 {
693         unsigned long sagaw;
694         int agaw = -1;
695
696         sagaw = cap_sagaw(iommu->cap);
697         for (agaw = width_to_agaw(max_gaw);
698              agaw >= 0; agaw--) {
699                 if (test_bit(agaw, &sagaw))
700                         break;
701         }
702
703         return agaw;
704 }
705
706 /*
707  * Calculate max SAGAW for each iommu.
708  */
709 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
710 {
711         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
712 }
713
714 /*
715  * calculate agaw for each iommu.
716  * "SAGAW" may be different across iommus, use a default agaw, and
717  * get a supported less agaw for iommus that don't support the default agaw.
718  */
719 int iommu_calculate_agaw(struct intel_iommu *iommu)
720 {
721         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
722 }
723
724 /* This functionin only returns single iommu in a domain */
725 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
726 {
727         int iommu_id;
728
729         /* si_domain and vm domain should not get here. */
730         BUG_ON(domain_type_is_vm_or_si(domain));
731         for_each_domain_iommu(iommu_id, domain)
732                 break;
733
734         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
735                 return NULL;
736
737         return g_iommus[iommu_id];
738 }
739
740 static void domain_update_iommu_coherency(struct dmar_domain *domain)
741 {
742         struct dmar_drhd_unit *drhd;
743         struct intel_iommu *iommu;
744         bool found = false;
745         int i;
746
747         domain->iommu_coherency = 1;
748
749         for_each_domain_iommu(i, domain) {
750                 found = true;
751                 if (!ecap_coherent(g_iommus[i]->ecap)) {
752                         domain->iommu_coherency = 0;
753                         break;
754                 }
755         }
756         if (found)
757                 return;
758
759         /* No hardware attached; use lowest common denominator */
760         rcu_read_lock();
761         for_each_active_iommu(iommu, drhd) {
762                 if (!ecap_coherent(iommu->ecap)) {
763                         domain->iommu_coherency = 0;
764                         break;
765                 }
766         }
767         rcu_read_unlock();
768 }
769
770 static int domain_update_iommu_snooping(struct intel_iommu *skip)
771 {
772         struct dmar_drhd_unit *drhd;
773         struct intel_iommu *iommu;
774         int ret = 1;
775
776         rcu_read_lock();
777         for_each_active_iommu(iommu, drhd) {
778                 if (iommu != skip) {
779                         if (!ecap_sc_support(iommu->ecap)) {
780                                 ret = 0;
781                                 break;
782                         }
783                 }
784         }
785         rcu_read_unlock();
786
787         return ret;
788 }
789
790 static int domain_update_iommu_superpage(struct intel_iommu *skip)
791 {
792         struct dmar_drhd_unit *drhd;
793         struct intel_iommu *iommu;
794         int mask = 0xf;
795
796         if (!intel_iommu_superpage) {
797                 return 0;
798         }
799
800         /* set iommu_superpage to the smallest common denominator */
801         rcu_read_lock();
802         for_each_active_iommu(iommu, drhd) {
803                 if (iommu != skip) {
804                         mask &= cap_super_page_val(iommu->cap);
805                         if (!mask)
806                                 break;
807                 }
808         }
809         rcu_read_unlock();
810
811         return fls(mask);
812 }
813
814 /* Some capabilities may be different across iommus */
815 static void domain_update_iommu_cap(struct dmar_domain *domain)
816 {
817         domain_update_iommu_coherency(domain);
818         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
819         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
820 }
821
822 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
823                                                        u8 bus, u8 devfn, int alloc)
824 {
825         struct root_entry *root = &iommu->root_entry[bus];
826         struct context_entry *context;
827         u64 *entry;
828
829         entry = &root->lo;
830         if (ecs_enabled(iommu)) {
831                 if (devfn >= 0x80) {
832                         devfn -= 0x80;
833                         entry = &root->hi;
834                 }
835                 devfn *= 2;
836         }
837         if (*entry & 1)
838                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
839         else {
840                 unsigned long phy_addr;
841                 if (!alloc)
842                         return NULL;
843
844                 context = alloc_pgtable_page(iommu->node);
845                 if (!context)
846                         return NULL;
847
848                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
849                 phy_addr = virt_to_phys((void *)context);
850                 *entry = phy_addr | 1;
851                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
852         }
853         return &context[devfn];
854 }
855
856 static int iommu_dummy(struct device *dev)
857 {
858         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
859 }
860
861 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
862 {
863         struct dmar_drhd_unit *drhd = NULL;
864         struct intel_iommu *iommu;
865         struct device *tmp;
866         struct pci_dev *ptmp, *pdev = NULL;
867         u16 segment = 0;
868         int i;
869
870         if (iommu_dummy(dev))
871                 return NULL;
872
873         if (dev_is_pci(dev)) {
874                 struct pci_dev *pf_pdev;
875
876                 pdev = to_pci_dev(dev);
877
878 #ifdef CONFIG_X86
879                 /* VMD child devices currently cannot be handled individually */
880                 if (is_vmd(pdev->bus))
881                         return NULL;
882 #endif
883
884                 /* VFs aren't listed in scope tables; we need to look up
885                  * the PF instead to find the IOMMU. */
886                 pf_pdev = pci_physfn(pdev);
887                 dev = &pf_pdev->dev;
888                 segment = pci_domain_nr(pdev->bus);
889         } else if (has_acpi_companion(dev))
890                 dev = &ACPI_COMPANION(dev)->dev;
891
892         rcu_read_lock();
893         for_each_active_iommu(iommu, drhd) {
894                 if (pdev && segment != drhd->segment)
895                         continue;
896
897                 for_each_active_dev_scope(drhd->devices,
898                                           drhd->devices_cnt, i, tmp) {
899                         if (tmp == dev) {
900                                 /* For a VF use its original BDF# not that of the PF
901                                  * which we used for the IOMMU lookup. Strictly speaking
902                                  * we could do this for all PCI devices; we only need to
903                                  * get the BDF# from the scope table for ACPI matches. */
904                                 if (pdev && pdev->is_virtfn)
905                                         goto got_pdev;
906
907                                 *bus = drhd->devices[i].bus;
908                                 *devfn = drhd->devices[i].devfn;
909                                 goto out;
910                         }
911
912                         if (!pdev || !dev_is_pci(tmp))
913                                 continue;
914
915                         ptmp = to_pci_dev(tmp);
916                         if (ptmp->subordinate &&
917                             ptmp->subordinate->number <= pdev->bus->number &&
918                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
919                                 goto got_pdev;
920                 }
921
922                 if (pdev && drhd->include_all) {
923                 got_pdev:
924                         *bus = pdev->bus->number;
925                         *devfn = pdev->devfn;
926                         goto out;
927                 }
928         }
929         iommu = NULL;
930  out:
931         rcu_read_unlock();
932
933         return iommu;
934 }
935
936 static void domain_flush_cache(struct dmar_domain *domain,
937                                void *addr, int size)
938 {
939         if (!domain->iommu_coherency)
940                 clflush_cache_range(addr, size);
941 }
942
943 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
944 {
945         struct context_entry *context;
946         int ret = 0;
947         unsigned long flags;
948
949         spin_lock_irqsave(&iommu->lock, flags);
950         context = iommu_context_addr(iommu, bus, devfn, 0);
951         if (context)
952                 ret = context_present(context);
953         spin_unlock_irqrestore(&iommu->lock, flags);
954         return ret;
955 }
956
957 static void free_context_table(struct intel_iommu *iommu)
958 {
959         int i;
960         unsigned long flags;
961         struct context_entry *context;
962
963         spin_lock_irqsave(&iommu->lock, flags);
964         if (!iommu->root_entry) {
965                 goto out;
966         }
967         for (i = 0; i < ROOT_ENTRY_NR; i++) {
968                 context = iommu_context_addr(iommu, i, 0, 0);
969                 if (context)
970                         free_pgtable_page(context);
971
972                 if (!ecs_enabled(iommu))
973                         continue;
974
975                 context = iommu_context_addr(iommu, i, 0x80, 0);
976                 if (context)
977                         free_pgtable_page(context);
978
979         }
980         free_pgtable_page(iommu->root_entry);
981         iommu->root_entry = NULL;
982 out:
983         spin_unlock_irqrestore(&iommu->lock, flags);
984 }
985
986 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
987                                       unsigned long pfn, int *target_level)
988 {
989         struct dma_pte *parent, *pte = NULL;
990         int level = agaw_to_level(domain->agaw);
991         int offset;
992
993         BUG_ON(!domain->pgd);
994
995         if (!domain_pfn_supported(domain, pfn))
996                 /* Address beyond IOMMU's addressing capabilities. */
997                 return NULL;
998
999         parent = domain->pgd;
1000
1001         while (1) {
1002                 void *tmp_page;
1003
1004                 offset = pfn_level_offset(pfn, level);
1005                 pte = &parent[offset];
1006                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1007                         break;
1008                 if (level == *target_level)
1009                         break;
1010
1011                 if (!dma_pte_present(pte)) {
1012                         uint64_t pteval;
1013
1014                         tmp_page = alloc_pgtable_page(domain->nid);
1015
1016                         if (!tmp_page)
1017                                 return NULL;
1018
1019                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1020                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1021                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1022                                 /* Someone else set it while we were thinking; use theirs. */
1023                                 free_pgtable_page(tmp_page);
1024                         else
1025                                 domain_flush_cache(domain, pte, sizeof(*pte));
1026                 }
1027                 if (level == 1)
1028                         break;
1029
1030                 parent = phys_to_virt(dma_pte_addr(pte));
1031                 level--;
1032         }
1033
1034         if (!*target_level)
1035                 *target_level = level;
1036
1037         return pte;
1038 }
1039
1040
1041 /* return address's pte at specific level */
1042 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1043                                          unsigned long pfn,
1044                                          int level, int *large_page)
1045 {
1046         struct dma_pte *parent, *pte = NULL;
1047         int total = agaw_to_level(domain->agaw);
1048         int offset;
1049
1050         parent = domain->pgd;
1051         while (level <= total) {
1052                 offset = pfn_level_offset(pfn, total);
1053                 pte = &parent[offset];
1054                 if (level == total)
1055                         return pte;
1056
1057                 if (!dma_pte_present(pte)) {
1058                         *large_page = total;
1059                         break;
1060                 }
1061
1062                 if (dma_pte_superpage(pte)) {
1063                         *large_page = total;
1064                         return pte;
1065                 }
1066
1067                 parent = phys_to_virt(dma_pte_addr(pte));
1068                 total--;
1069         }
1070         return NULL;
1071 }
1072
1073 /* clear last level pte, a tlb flush should be followed */
1074 static void dma_pte_clear_range(struct dmar_domain *domain,
1075                                 unsigned long start_pfn,
1076                                 unsigned long last_pfn)
1077 {
1078         unsigned int large_page = 1;
1079         struct dma_pte *first_pte, *pte;
1080
1081         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1082         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1083         BUG_ON(start_pfn > last_pfn);
1084
1085         /* we don't need lock here; nobody else touches the iova range */
1086         do {
1087                 large_page = 1;
1088                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1089                 if (!pte) {
1090                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1091                         continue;
1092                 }
1093                 do {
1094                         dma_clear_pte(pte);
1095                         start_pfn += lvl_to_nr_pages(large_page);
1096                         pte++;
1097                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1098
1099                 domain_flush_cache(domain, first_pte,
1100                                    (void *)pte - (void *)first_pte);
1101
1102         } while (start_pfn && start_pfn <= last_pfn);
1103 }
1104
1105 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1106                                int retain_level, struct dma_pte *pte,
1107                                unsigned long pfn, unsigned long start_pfn,
1108                                unsigned long last_pfn)
1109 {
1110         pfn = max(start_pfn, pfn);
1111         pte = &pte[pfn_level_offset(pfn, level)];
1112
1113         do {
1114                 unsigned long level_pfn;
1115                 struct dma_pte *level_pte;
1116
1117                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1118                         goto next;
1119
1120                 level_pfn = pfn & level_mask(level);
1121                 level_pte = phys_to_virt(dma_pte_addr(pte));
1122
1123                 if (level > 2) {
1124                         dma_pte_free_level(domain, level - 1, retain_level,
1125                                            level_pte, level_pfn, start_pfn,
1126                                            last_pfn);
1127                 }
1128
1129                 /*
1130                  * Free the page table if we're below the level we want to
1131                  * retain and the range covers the entire table.
1132                  */
1133                 if (level < retain_level && !(start_pfn > level_pfn ||
1134                       last_pfn < level_pfn + level_size(level) - 1)) {
1135                         dma_clear_pte(pte);
1136                         domain_flush_cache(domain, pte, sizeof(*pte));
1137                         free_pgtable_page(level_pte);
1138                 }
1139 next:
1140                 pfn += level_size(level);
1141         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1142 }
1143
1144 /*
1145  * clear last level (leaf) ptes and free page table pages below the
1146  * level we wish to keep intact.
1147  */
1148 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1149                                    unsigned long start_pfn,
1150                                    unsigned long last_pfn,
1151                                    int retain_level)
1152 {
1153         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1154         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1155         BUG_ON(start_pfn > last_pfn);
1156
1157         dma_pte_clear_range(domain, start_pfn, last_pfn);
1158
1159         /* We don't need lock here; nobody else touches the iova range */
1160         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1161                            domain->pgd, 0, start_pfn, last_pfn);
1162
1163         /* free pgd */
1164         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1165                 free_pgtable_page(domain->pgd);
1166                 domain->pgd = NULL;
1167         }
1168 }
1169
1170 /* When a page at a given level is being unlinked from its parent, we don't
1171    need to *modify* it at all. All we need to do is make a list of all the
1172    pages which can be freed just as soon as we've flushed the IOTLB and we
1173    know the hardware page-walk will no longer touch them.
1174    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1175    be freed. */
1176 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1177                                             int level, struct dma_pte *pte,
1178                                             struct page *freelist)
1179 {
1180         struct page *pg;
1181
1182         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1183         pg->freelist = freelist;
1184         freelist = pg;
1185
1186         if (level == 1)
1187                 return freelist;
1188
1189         pte = page_address(pg);
1190         do {
1191                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1192                         freelist = dma_pte_list_pagetables(domain, level - 1,
1193                                                            pte, freelist);
1194                 pte++;
1195         } while (!first_pte_in_page(pte));
1196
1197         return freelist;
1198 }
1199
1200 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1201                                         struct dma_pte *pte, unsigned long pfn,
1202                                         unsigned long start_pfn,
1203                                         unsigned long last_pfn,
1204                                         struct page *freelist)
1205 {
1206         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1207
1208         pfn = max(start_pfn, pfn);
1209         pte = &pte[pfn_level_offset(pfn, level)];
1210
1211         do {
1212                 unsigned long level_pfn;
1213
1214                 if (!dma_pte_present(pte))
1215                         goto next;
1216
1217                 level_pfn = pfn & level_mask(level);
1218
1219                 /* If range covers entire pagetable, free it */
1220                 if (start_pfn <= level_pfn &&
1221                     last_pfn >= level_pfn + level_size(level) - 1) {
1222                         /* These suborbinate page tables are going away entirely. Don't
1223                            bother to clear them; we're just going to *free* them. */
1224                         if (level > 1 && !dma_pte_superpage(pte))
1225                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1226
1227                         dma_clear_pte(pte);
1228                         if (!first_pte)
1229                                 first_pte = pte;
1230                         last_pte = pte;
1231                 } else if (level > 1) {
1232                         /* Recurse down into a level that isn't *entirely* obsolete */
1233                         freelist = dma_pte_clear_level(domain, level - 1,
1234                                                        phys_to_virt(dma_pte_addr(pte)),
1235                                                        level_pfn, start_pfn, last_pfn,
1236                                                        freelist);
1237                 }
1238 next:
1239                 pfn += level_size(level);
1240         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1241
1242         if (first_pte)
1243                 domain_flush_cache(domain, first_pte,
1244                                    (void *)++last_pte - (void *)first_pte);
1245
1246         return freelist;
1247 }
1248
1249 /* We can't just free the pages because the IOMMU may still be walking
1250    the page tables, and may have cached the intermediate levels. The
1251    pages can only be freed after the IOTLB flush has been done. */
1252 static struct page *domain_unmap(struct dmar_domain *domain,
1253                                  unsigned long start_pfn,
1254                                  unsigned long last_pfn)
1255 {
1256         struct page *freelist = NULL;
1257
1258         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1259         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1260         BUG_ON(start_pfn > last_pfn);
1261
1262         /* we don't need lock here; nobody else touches the iova range */
1263         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1264                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1265
1266         /* free pgd */
1267         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1268                 struct page *pgd_page = virt_to_page(domain->pgd);
1269                 pgd_page->freelist = freelist;
1270                 freelist = pgd_page;
1271
1272                 domain->pgd = NULL;
1273         }
1274
1275         return freelist;
1276 }
1277
1278 static void dma_free_pagelist(struct page *freelist)
1279 {
1280         struct page *pg;
1281
1282         while ((pg = freelist)) {
1283                 freelist = pg->freelist;
1284                 free_pgtable_page(page_address(pg));
1285         }
1286 }
1287
1288 static void iova_entry_free(unsigned long data)
1289 {
1290         struct page *freelist = (struct page *)data;
1291
1292         dma_free_pagelist(freelist);
1293 }
1294
1295 /* iommu handling */
1296 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1297 {
1298         struct root_entry *root;
1299         unsigned long flags;
1300
1301         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1302         if (!root) {
1303                 pr_err("Allocating root entry for %s failed\n",
1304                         iommu->name);
1305                 return -ENOMEM;
1306         }
1307
1308         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1309
1310         spin_lock_irqsave(&iommu->lock, flags);
1311         iommu->root_entry = root;
1312         spin_unlock_irqrestore(&iommu->lock, flags);
1313
1314         return 0;
1315 }
1316
1317 static void iommu_set_root_entry(struct intel_iommu *iommu)
1318 {
1319         u64 addr;
1320         u32 sts;
1321         unsigned long flag;
1322
1323         addr = virt_to_phys(iommu->root_entry);
1324         if (ecs_enabled(iommu))
1325                 addr |= DMA_RTADDR_RTT;
1326
1327         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1328         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1329
1330         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1331
1332         /* Make sure hardware complete it */
1333         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1334                       readl, (sts & DMA_GSTS_RTPS), sts);
1335
1336         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1337 }
1338
1339 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1340 {
1341         u32 val;
1342         unsigned long flag;
1343
1344         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1345                 return;
1346
1347         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1348         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1349
1350         /* Make sure hardware complete it */
1351         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1352                       readl, (!(val & DMA_GSTS_WBFS)), val);
1353
1354         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1355 }
1356
1357 /* return value determine if we need a write buffer flush */
1358 static void __iommu_flush_context(struct intel_iommu *iommu,
1359                                   u16 did, u16 source_id, u8 function_mask,
1360                                   u64 type)
1361 {
1362         u64 val = 0;
1363         unsigned long flag;
1364
1365         switch (type) {
1366         case DMA_CCMD_GLOBAL_INVL:
1367                 val = DMA_CCMD_GLOBAL_INVL;
1368                 break;
1369         case DMA_CCMD_DOMAIN_INVL:
1370                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1371                 break;
1372         case DMA_CCMD_DEVICE_INVL:
1373                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1374                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1375                 break;
1376         default:
1377                 BUG();
1378         }
1379         val |= DMA_CCMD_ICC;
1380
1381         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1382         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1383
1384         /* Make sure hardware complete it */
1385         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1386                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1387
1388         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1389 }
1390
1391 /* return value determine if we need a write buffer flush */
1392 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1393                                 u64 addr, unsigned int size_order, u64 type)
1394 {
1395         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1396         u64 val = 0, val_iva = 0;
1397         unsigned long flag;
1398
1399         switch (type) {
1400         case DMA_TLB_GLOBAL_FLUSH:
1401                 /* global flush doesn't need set IVA_REG */
1402                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1403                 break;
1404         case DMA_TLB_DSI_FLUSH:
1405                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1406                 break;
1407         case DMA_TLB_PSI_FLUSH:
1408                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1409                 /* IH bit is passed in as part of address */
1410                 val_iva = size_order | addr;
1411                 break;
1412         default:
1413                 BUG();
1414         }
1415         /* Note: set drain read/write */
1416 #if 0
1417         /*
1418          * This is probably to be super secure.. Looks like we can
1419          * ignore it without any impact.
1420          */
1421         if (cap_read_drain(iommu->cap))
1422                 val |= DMA_TLB_READ_DRAIN;
1423 #endif
1424         if (cap_write_drain(iommu->cap))
1425                 val |= DMA_TLB_WRITE_DRAIN;
1426
1427         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1428         /* Note: Only uses first TLB reg currently */
1429         if (val_iva)
1430                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1431         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1432
1433         /* Make sure hardware complete it */
1434         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1435                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1436
1437         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1438
1439         /* check IOTLB invalidation granularity */
1440         if (DMA_TLB_IAIG(val) == 0)
1441                 pr_err("Flush IOTLB failed\n");
1442         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1443                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1444                         (unsigned long long)DMA_TLB_IIRG(type),
1445                         (unsigned long long)DMA_TLB_IAIG(val));
1446 }
1447
1448 static struct device_domain_info *
1449 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1450                          u8 bus, u8 devfn)
1451 {
1452         struct device_domain_info *info;
1453
1454         assert_spin_locked(&device_domain_lock);
1455
1456         if (!iommu->qi)
1457                 return NULL;
1458
1459         list_for_each_entry(info, &domain->devices, link)
1460                 if (info->iommu == iommu && info->bus == bus &&
1461                     info->devfn == devfn) {
1462                         if (info->ats_supported && info->dev)
1463                                 return info;
1464                         break;
1465                 }
1466
1467         return NULL;
1468 }
1469
1470 static void domain_update_iotlb(struct dmar_domain *domain)
1471 {
1472         struct device_domain_info *info;
1473         bool has_iotlb_device = false;
1474
1475         assert_spin_locked(&device_domain_lock);
1476
1477         list_for_each_entry(info, &domain->devices, link) {
1478                 struct pci_dev *pdev;
1479
1480                 if (!info->dev || !dev_is_pci(info->dev))
1481                         continue;
1482
1483                 pdev = to_pci_dev(info->dev);
1484                 if (pdev->ats_enabled) {
1485                         has_iotlb_device = true;
1486                         break;
1487                 }
1488         }
1489
1490         domain->has_iotlb_device = has_iotlb_device;
1491 }
1492
1493 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1494 {
1495         struct pci_dev *pdev;
1496
1497         assert_spin_locked(&device_domain_lock);
1498
1499         if (!info || !dev_is_pci(info->dev))
1500                 return;
1501
1502         pdev = to_pci_dev(info->dev);
1503
1504 #ifdef CONFIG_INTEL_IOMMU_SVM
1505         /* The PCIe spec, in its wisdom, declares that the behaviour of
1506            the device if you enable PASID support after ATS support is
1507            undefined. So always enable PASID support on devices which
1508            have it, even if we can't yet know if we're ever going to
1509            use it. */
1510         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1511                 info->pasid_enabled = 1;
1512
1513         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1514                 info->pri_enabled = 1;
1515 #endif
1516         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1517                 info->ats_enabled = 1;
1518                 domain_update_iotlb(info->domain);
1519                 info->ats_qdep = pci_ats_queue_depth(pdev);
1520         }
1521 }
1522
1523 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1524 {
1525         struct pci_dev *pdev;
1526
1527         assert_spin_locked(&device_domain_lock);
1528
1529         if (!dev_is_pci(info->dev))
1530                 return;
1531
1532         pdev = to_pci_dev(info->dev);
1533
1534         if (info->ats_enabled) {
1535                 pci_disable_ats(pdev);
1536                 info->ats_enabled = 0;
1537                 domain_update_iotlb(info->domain);
1538         }
1539 #ifdef CONFIG_INTEL_IOMMU_SVM
1540         if (info->pri_enabled) {
1541                 pci_disable_pri(pdev);
1542                 info->pri_enabled = 0;
1543         }
1544         if (info->pasid_enabled) {
1545                 pci_disable_pasid(pdev);
1546                 info->pasid_enabled = 0;
1547         }
1548 #endif
1549 }
1550
1551 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1552                                   u64 addr, unsigned mask)
1553 {
1554         u16 sid, qdep;
1555         unsigned long flags;
1556         struct device_domain_info *info;
1557
1558         if (!domain->has_iotlb_device)
1559                 return;
1560
1561         spin_lock_irqsave(&device_domain_lock, flags);
1562         list_for_each_entry(info, &domain->devices, link) {
1563                 if (!info->ats_enabled)
1564                         continue;
1565
1566                 sid = info->bus << 8 | info->devfn;
1567                 qdep = info->ats_qdep;
1568                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1569         }
1570         spin_unlock_irqrestore(&device_domain_lock, flags);
1571 }
1572
1573 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1574                                   struct dmar_domain *domain,
1575                                   unsigned long pfn, unsigned int pages,
1576                                   int ih, int map)
1577 {
1578         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1579         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1580         u16 did = domain->iommu_did[iommu->seq_id];
1581
1582         BUG_ON(pages == 0);
1583
1584         if (ih)
1585                 ih = 1 << 6;
1586         /*
1587          * Fallback to domain selective flush if no PSI support or the size is
1588          * too big.
1589          * PSI requires page size to be 2 ^ x, and the base address is naturally
1590          * aligned to the size
1591          */
1592         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1593                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1594                                                 DMA_TLB_DSI_FLUSH);
1595         else
1596                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1597                                                 DMA_TLB_PSI_FLUSH);
1598
1599         /*
1600          * In caching mode, changes of pages from non-present to present require
1601          * flush. However, device IOTLB doesn't need to be flushed in this case.
1602          */
1603         if (!cap_caching_mode(iommu->cap) || !map)
1604                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1605                                       addr, mask);
1606 }
1607
1608 static void iommu_flush_iova(struct iova_domain *iovad)
1609 {
1610         struct dmar_domain *domain;
1611         int idx;
1612
1613         domain = container_of(iovad, struct dmar_domain, iovad);
1614
1615         for_each_domain_iommu(idx, domain) {
1616                 struct intel_iommu *iommu = g_iommus[idx];
1617                 u16 did = domain->iommu_did[iommu->seq_id];
1618
1619                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1620
1621                 if (!cap_caching_mode(iommu->cap))
1622                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1623                                               0, MAX_AGAW_PFN_WIDTH);
1624         }
1625 }
1626
1627 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1628 {
1629         u32 pmen;
1630         unsigned long flags;
1631
1632         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1633         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1634         pmen &= ~DMA_PMEN_EPM;
1635         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1636
1637         /* wait for the protected region status bit to clear */
1638         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1639                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1640
1641         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1642 }
1643
1644 static void iommu_enable_translation(struct intel_iommu *iommu)
1645 {
1646         u32 sts;
1647         unsigned long flags;
1648
1649         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1650         iommu->gcmd |= DMA_GCMD_TE;
1651         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1652
1653         /* Make sure hardware complete it */
1654         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1655                       readl, (sts & DMA_GSTS_TES), sts);
1656
1657         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1658 }
1659
1660 static void iommu_disable_translation(struct intel_iommu *iommu)
1661 {
1662         u32 sts;
1663         unsigned long flag;
1664
1665         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1666         iommu->gcmd &= ~DMA_GCMD_TE;
1667         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1668
1669         /* Make sure hardware complete it */
1670         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1671                       readl, (!(sts & DMA_GSTS_TES)), sts);
1672
1673         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1674 }
1675
1676
1677 static int iommu_init_domains(struct intel_iommu *iommu)
1678 {
1679         u32 ndomains, nlongs;
1680         size_t size;
1681
1682         ndomains = cap_ndoms(iommu->cap);
1683         pr_debug("%s: Number of Domains supported <%d>\n",
1684                  iommu->name, ndomains);
1685         nlongs = BITS_TO_LONGS(ndomains);
1686
1687         spin_lock_init(&iommu->lock);
1688
1689         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1690         if (!iommu->domain_ids) {
1691                 pr_err("%s: Allocating domain id array failed\n",
1692                        iommu->name);
1693                 return -ENOMEM;
1694         }
1695
1696         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1697         iommu->domains = kzalloc(size, GFP_KERNEL);
1698
1699         if (iommu->domains) {
1700                 size = 256 * sizeof(struct dmar_domain *);
1701                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1702         }
1703
1704         if (!iommu->domains || !iommu->domains[0]) {
1705                 pr_err("%s: Allocating domain array failed\n",
1706                        iommu->name);
1707                 kfree(iommu->domain_ids);
1708                 kfree(iommu->domains);
1709                 iommu->domain_ids = NULL;
1710                 iommu->domains    = NULL;
1711                 return -ENOMEM;
1712         }
1713
1714
1715
1716         /*
1717          * If Caching mode is set, then invalid translations are tagged
1718          * with domain-id 0, hence we need to pre-allocate it. We also
1719          * use domain-id 0 as a marker for non-allocated domain-id, so
1720          * make sure it is not used for a real domain.
1721          */
1722         set_bit(0, iommu->domain_ids);
1723
1724         return 0;
1725 }
1726
1727 static void disable_dmar_iommu(struct intel_iommu *iommu)
1728 {
1729         struct device_domain_info *info, *tmp;
1730         unsigned long flags;
1731
1732         if (!iommu->domains || !iommu->domain_ids)
1733                 return;
1734
1735 again:
1736         spin_lock_irqsave(&device_domain_lock, flags);
1737         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1738                 struct dmar_domain *domain;
1739
1740                 if (info->iommu != iommu)
1741                         continue;
1742
1743                 if (!info->dev || !info->domain)
1744                         continue;
1745
1746                 domain = info->domain;
1747
1748                 __dmar_remove_one_dev_info(info);
1749
1750                 if (!domain_type_is_vm_or_si(domain)) {
1751                         /*
1752                          * The domain_exit() function  can't be called under
1753                          * device_domain_lock, as it takes this lock itself.
1754                          * So release the lock here and re-run the loop
1755                          * afterwards.
1756                          */
1757                         spin_unlock_irqrestore(&device_domain_lock, flags);
1758                         domain_exit(domain);
1759                         goto again;
1760                 }
1761         }
1762         spin_unlock_irqrestore(&device_domain_lock, flags);
1763
1764         if (iommu->gcmd & DMA_GCMD_TE)
1765                 iommu_disable_translation(iommu);
1766 }
1767
1768 static void free_dmar_iommu(struct intel_iommu *iommu)
1769 {
1770         if ((iommu->domains) && (iommu->domain_ids)) {
1771                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1772                 int i;
1773
1774                 for (i = 0; i < elems; i++)
1775                         kfree(iommu->domains[i]);
1776                 kfree(iommu->domains);
1777                 kfree(iommu->domain_ids);
1778                 iommu->domains = NULL;
1779                 iommu->domain_ids = NULL;
1780         }
1781
1782         g_iommus[iommu->seq_id] = NULL;
1783
1784         /* free context mapping */
1785         free_context_table(iommu);
1786
1787 #ifdef CONFIG_INTEL_IOMMU_SVM
1788         if (pasid_enabled(iommu)) {
1789                 if (ecap_prs(iommu->ecap))
1790                         intel_svm_finish_prq(iommu);
1791                 intel_svm_free_pasid_tables(iommu);
1792         }
1793 #endif
1794 }
1795
1796 static struct dmar_domain *alloc_domain(int flags)
1797 {
1798         struct dmar_domain *domain;
1799
1800         domain = alloc_domain_mem();
1801         if (!domain)
1802                 return NULL;
1803
1804         memset(domain, 0, sizeof(*domain));
1805         domain->nid = -1;
1806         domain->flags = flags;
1807         domain->has_iotlb_device = false;
1808         INIT_LIST_HEAD(&domain->devices);
1809
1810         return domain;
1811 }
1812
1813 /* Must be called with iommu->lock */
1814 static int domain_attach_iommu(struct dmar_domain *domain,
1815                                struct intel_iommu *iommu)
1816 {
1817         unsigned long ndomains;
1818         int num;
1819
1820         assert_spin_locked(&device_domain_lock);
1821         assert_spin_locked(&iommu->lock);
1822
1823         domain->iommu_refcnt[iommu->seq_id] += 1;
1824         domain->iommu_count += 1;
1825         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1826                 ndomains = cap_ndoms(iommu->cap);
1827                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1828
1829                 if (num >= ndomains) {
1830                         pr_err("%s: No free domain ids\n", iommu->name);
1831                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1832                         domain->iommu_count -= 1;
1833                         return -ENOSPC;
1834                 }
1835
1836                 set_bit(num, iommu->domain_ids);
1837                 set_iommu_domain(iommu, num, domain);
1838
1839                 domain->iommu_did[iommu->seq_id] = num;
1840                 domain->nid                      = iommu->node;
1841
1842                 domain_update_iommu_cap(domain);
1843         }
1844
1845         return 0;
1846 }
1847
1848 static int domain_detach_iommu(struct dmar_domain *domain,
1849                                struct intel_iommu *iommu)
1850 {
1851         int num, count = INT_MAX;
1852
1853         assert_spin_locked(&device_domain_lock);
1854         assert_spin_locked(&iommu->lock);
1855
1856         domain->iommu_refcnt[iommu->seq_id] -= 1;
1857         count = --domain->iommu_count;
1858         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1859                 num = domain->iommu_did[iommu->seq_id];
1860                 clear_bit(num, iommu->domain_ids);
1861                 set_iommu_domain(iommu, num, NULL);
1862
1863                 domain_update_iommu_cap(domain);
1864                 domain->iommu_did[iommu->seq_id] = 0;
1865         }
1866
1867         return count;
1868 }
1869
1870 static struct iova_domain reserved_iova_list;
1871 static struct lock_class_key reserved_rbtree_key;
1872
1873 static int dmar_init_reserved_ranges(void)
1874 {
1875         struct pci_dev *pdev = NULL;
1876         struct iova *iova;
1877         int i;
1878
1879         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1880
1881         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1882                 &reserved_rbtree_key);
1883
1884         /* IOAPIC ranges shouldn't be accessed by DMA */
1885         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1886                 IOVA_PFN(IOAPIC_RANGE_END));
1887         if (!iova) {
1888                 pr_err("Reserve IOAPIC range failed\n");
1889                 return -ENODEV;
1890         }
1891
1892         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1893         for_each_pci_dev(pdev) {
1894                 struct resource *r;
1895
1896                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1897                         r = &pdev->resource[i];
1898                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1899                                 continue;
1900                         iova = reserve_iova(&reserved_iova_list,
1901                                             IOVA_PFN(r->start),
1902                                             IOVA_PFN(r->end));
1903                         if (!iova) {
1904                                 pr_err("Reserve iova failed\n");
1905                                 return -ENODEV;
1906                         }
1907                 }
1908         }
1909         return 0;
1910 }
1911
1912 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1913 {
1914         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1915 }
1916
1917 static inline int guestwidth_to_adjustwidth(int gaw)
1918 {
1919         int agaw;
1920         int r = (gaw - 12) % 9;
1921
1922         if (r == 0)
1923                 agaw = gaw;
1924         else
1925                 agaw = gaw + 9 - r;
1926         if (agaw > 64)
1927                 agaw = 64;
1928         return agaw;
1929 }
1930
1931 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1932                        int guest_width)
1933 {
1934         int adjust_width, agaw;
1935         unsigned long sagaw;
1936         int err;
1937
1938         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1939
1940         err = init_iova_flush_queue(&domain->iovad,
1941                                     iommu_flush_iova, iova_entry_free);
1942         if (err)
1943                 return err;
1944
1945         domain_reserve_special_ranges(domain);
1946
1947         /* calculate AGAW */
1948         if (guest_width > cap_mgaw(iommu->cap))
1949                 guest_width = cap_mgaw(iommu->cap);
1950         domain->gaw = guest_width;
1951         adjust_width = guestwidth_to_adjustwidth(guest_width);
1952         agaw = width_to_agaw(adjust_width);
1953         sagaw = cap_sagaw(iommu->cap);
1954         if (!test_bit(agaw, &sagaw)) {
1955                 /* hardware doesn't support it, choose a bigger one */
1956                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1957                 agaw = find_next_bit(&sagaw, 5, agaw);
1958                 if (agaw >= 5)
1959                         return -ENODEV;
1960         }
1961         domain->agaw = agaw;
1962
1963         if (ecap_coherent(iommu->ecap))
1964                 domain->iommu_coherency = 1;
1965         else
1966                 domain->iommu_coherency = 0;
1967
1968         if (ecap_sc_support(iommu->ecap))
1969                 domain->iommu_snooping = 1;
1970         else
1971                 domain->iommu_snooping = 0;
1972
1973         if (intel_iommu_superpage)
1974                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1975         else
1976                 domain->iommu_superpage = 0;
1977
1978         domain->nid = iommu->node;
1979
1980         /* always allocate the top pgd */
1981         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1982         if (!domain->pgd)
1983                 return -ENOMEM;
1984         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1985         return 0;
1986 }
1987
1988 static void domain_exit(struct dmar_domain *domain)
1989 {
1990         struct page *freelist = NULL;
1991
1992         /* Domain 0 is reserved, so dont process it */
1993         if (!domain)
1994                 return;
1995
1996         /* Remove associated devices and clear attached or cached domains */
1997         rcu_read_lock();
1998         domain_remove_dev_info(domain);
1999         rcu_read_unlock();
2000
2001         /* destroy iovas */
2002         put_iova_domain(&domain->iovad);
2003
2004         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2005
2006         dma_free_pagelist(freelist);
2007
2008         free_domain_mem(domain);
2009 }
2010
2011 static int domain_context_mapping_one(struct dmar_domain *domain,
2012                                       struct intel_iommu *iommu,
2013                                       u8 bus, u8 devfn)
2014 {
2015         u16 did = domain->iommu_did[iommu->seq_id];
2016         int translation = CONTEXT_TT_MULTI_LEVEL;
2017         struct device_domain_info *info = NULL;
2018         struct context_entry *context;
2019         unsigned long flags;
2020         struct dma_pte *pgd;
2021         int ret, agaw;
2022
2023         WARN_ON(did == 0);
2024
2025         if (hw_pass_through && domain_type_is_si(domain))
2026                 translation = CONTEXT_TT_PASS_THROUGH;
2027
2028         pr_debug("Set context mapping for %02x:%02x.%d\n",
2029                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2030
2031         BUG_ON(!domain->pgd);
2032
2033         spin_lock_irqsave(&device_domain_lock, flags);
2034         spin_lock(&iommu->lock);
2035
2036         ret = -ENOMEM;
2037         context = iommu_context_addr(iommu, bus, devfn, 1);
2038         if (!context)
2039                 goto out_unlock;
2040
2041         ret = 0;
2042         if (context_present(context))
2043                 goto out_unlock;
2044
2045         /*
2046          * For kdump cases, old valid entries may be cached due to the
2047          * in-flight DMA and copied pgtable, but there is no unmapping
2048          * behaviour for them, thus we need an explicit cache flush for
2049          * the newly-mapped device. For kdump, at this point, the device
2050          * is supposed to finish reset at its driver probe stage, so no
2051          * in-flight DMA will exist, and we don't need to worry anymore
2052          * hereafter.
2053          */
2054         if (context_copied(context)) {
2055                 u16 did_old = context_domain_id(context);
2056
2057                 if (did_old < cap_ndoms(iommu->cap)) {
2058                         iommu->flush.flush_context(iommu, did_old,
2059                                                    (((u16)bus) << 8) | devfn,
2060                                                    DMA_CCMD_MASK_NOBIT,
2061                                                    DMA_CCMD_DEVICE_INVL);
2062                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2063                                                  DMA_TLB_DSI_FLUSH);
2064                 }
2065         }
2066
2067         pgd = domain->pgd;
2068
2069         context_clear_entry(context);
2070         context_set_domain_id(context, did);
2071
2072         /*
2073          * Skip top levels of page tables for iommu which has less agaw
2074          * than default.  Unnecessary for PT mode.
2075          */
2076         if (translation != CONTEXT_TT_PASS_THROUGH) {
2077                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2078                         ret = -ENOMEM;
2079                         pgd = phys_to_virt(dma_pte_addr(pgd));
2080                         if (!dma_pte_present(pgd))
2081                                 goto out_unlock;
2082                 }
2083
2084                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2085                 if (info && info->ats_supported)
2086                         translation = CONTEXT_TT_DEV_IOTLB;
2087                 else
2088                         translation = CONTEXT_TT_MULTI_LEVEL;
2089
2090                 context_set_address_root(context, virt_to_phys(pgd));
2091                 context_set_address_width(context, iommu->agaw);
2092         } else {
2093                 /*
2094                  * In pass through mode, AW must be programmed to
2095                  * indicate the largest AGAW value supported by
2096                  * hardware. And ASR is ignored by hardware.
2097                  */
2098                 context_set_address_width(context, iommu->msagaw);
2099         }
2100
2101         context_set_translation_type(context, translation);
2102         context_set_fault_enable(context);
2103         context_set_present(context);
2104         domain_flush_cache(domain, context, sizeof(*context));
2105
2106         /*
2107          * It's a non-present to present mapping. If hardware doesn't cache
2108          * non-present entry we only need to flush the write-buffer. If the
2109          * _does_ cache non-present entries, then it does so in the special
2110          * domain #0, which we have to flush:
2111          */
2112         if (cap_caching_mode(iommu->cap)) {
2113                 iommu->flush.flush_context(iommu, 0,
2114                                            (((u16)bus) << 8) | devfn,
2115                                            DMA_CCMD_MASK_NOBIT,
2116                                            DMA_CCMD_DEVICE_INVL);
2117                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2118         } else {
2119                 iommu_flush_write_buffer(iommu);
2120         }
2121         iommu_enable_dev_iotlb(info);
2122
2123         ret = 0;
2124
2125 out_unlock:
2126         spin_unlock(&iommu->lock);
2127         spin_unlock_irqrestore(&device_domain_lock, flags);
2128
2129         return ret;
2130 }
2131
2132 struct domain_context_mapping_data {
2133         struct dmar_domain *domain;
2134         struct intel_iommu *iommu;
2135 };
2136
2137 static int domain_context_mapping_cb(struct pci_dev *pdev,
2138                                      u16 alias, void *opaque)
2139 {
2140         struct domain_context_mapping_data *data = opaque;
2141
2142         return domain_context_mapping_one(data->domain, data->iommu,
2143                                           PCI_BUS_NUM(alias), alias & 0xff);
2144 }
2145
2146 static int
2147 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2148 {
2149         struct intel_iommu *iommu;
2150         u8 bus, devfn;
2151         struct domain_context_mapping_data data;
2152
2153         iommu = device_to_iommu(dev, &bus, &devfn);
2154         if (!iommu)
2155                 return -ENODEV;
2156
2157         if (!dev_is_pci(dev))
2158                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2159
2160         data.domain = domain;
2161         data.iommu = iommu;
2162
2163         return pci_for_each_dma_alias(to_pci_dev(dev),
2164                                       &domain_context_mapping_cb, &data);
2165 }
2166
2167 static int domain_context_mapped_cb(struct pci_dev *pdev,
2168                                     u16 alias, void *opaque)
2169 {
2170         struct intel_iommu *iommu = opaque;
2171
2172         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2173 }
2174
2175 static int domain_context_mapped(struct device *dev)
2176 {
2177         struct intel_iommu *iommu;
2178         u8 bus, devfn;
2179
2180         iommu = device_to_iommu(dev, &bus, &devfn);
2181         if (!iommu)
2182                 return -ENODEV;
2183
2184         if (!dev_is_pci(dev))
2185                 return device_context_mapped(iommu, bus, devfn);
2186
2187         return !pci_for_each_dma_alias(to_pci_dev(dev),
2188                                        domain_context_mapped_cb, iommu);
2189 }
2190
2191 /* Returns a number of VTD pages, but aligned to MM page size */
2192 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2193                                             size_t size)
2194 {
2195         host_addr &= ~PAGE_MASK;
2196         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2197 }
2198
2199 /* Return largest possible superpage level for a given mapping */
2200 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2201                                           unsigned long iov_pfn,
2202                                           unsigned long phy_pfn,
2203                                           unsigned long pages)
2204 {
2205         int support, level = 1;
2206         unsigned long pfnmerge;
2207
2208         support = domain->iommu_superpage;
2209
2210         /* To use a large page, the virtual *and* physical addresses
2211            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2212            of them will mean we have to use smaller pages. So just
2213            merge them and check both at once. */
2214         pfnmerge = iov_pfn | phy_pfn;
2215
2216         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2217                 pages >>= VTD_STRIDE_SHIFT;
2218                 if (!pages)
2219                         break;
2220                 pfnmerge >>= VTD_STRIDE_SHIFT;
2221                 level++;
2222                 support--;
2223         }
2224         return level;
2225 }
2226
2227 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2228                             struct scatterlist *sg, unsigned long phys_pfn,
2229                             unsigned long nr_pages, int prot)
2230 {
2231         struct dma_pte *first_pte = NULL, *pte = NULL;
2232         phys_addr_t uninitialized_var(pteval);
2233         unsigned long sg_res = 0;
2234         unsigned int largepage_lvl = 0;
2235         unsigned long lvl_pages = 0;
2236
2237         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2238
2239         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2240                 return -EINVAL;
2241
2242         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2243
2244         if (!sg) {
2245                 sg_res = nr_pages;
2246                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2247         }
2248
2249         while (nr_pages > 0) {
2250                 uint64_t tmp;
2251
2252                 if (!sg_res) {
2253                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2254
2255                         sg_res = aligned_nrpages(sg->offset, sg->length);
2256                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2257                         sg->dma_length = sg->length;
2258                         pteval = (sg_phys(sg) - pgoff) | prot;
2259                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2260                 }
2261
2262                 if (!pte) {
2263                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2264
2265                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2266                         if (!pte)
2267                                 return -ENOMEM;
2268                         /* It is large page*/
2269                         if (largepage_lvl > 1) {
2270                                 unsigned long nr_superpages, end_pfn;
2271
2272                                 pteval |= DMA_PTE_LARGE_PAGE;
2273                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2274
2275                                 nr_superpages = sg_res / lvl_pages;
2276                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2277
2278                                 /*
2279                                  * Ensure that old small page tables are
2280                                  * removed to make room for superpage(s).
2281                                  * We're adding new large pages, so make sure
2282                                  * we don't remove their parent tables.
2283                                  */
2284                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2285                                                        largepage_lvl + 1);
2286                         } else {
2287                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2288                         }
2289
2290                 }
2291                 /* We don't need lock here, nobody else
2292                  * touches the iova range
2293                  */
2294                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2295                 if (tmp) {
2296                         static int dumps = 5;
2297                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2298                                 iov_pfn, tmp, (unsigned long long)pteval);
2299                         if (dumps) {
2300                                 dumps--;
2301                                 debug_dma_dump_mappings(NULL);
2302                         }
2303                         WARN_ON(1);
2304                 }
2305
2306                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2307
2308                 BUG_ON(nr_pages < lvl_pages);
2309                 BUG_ON(sg_res < lvl_pages);
2310
2311                 nr_pages -= lvl_pages;
2312                 iov_pfn += lvl_pages;
2313                 phys_pfn += lvl_pages;
2314                 pteval += lvl_pages * VTD_PAGE_SIZE;
2315                 sg_res -= lvl_pages;
2316
2317                 /* If the next PTE would be the first in a new page, then we
2318                    need to flush the cache on the entries we've just written.
2319                    And then we'll need to recalculate 'pte', so clear it and
2320                    let it get set again in the if (!pte) block above.
2321
2322                    If we're done (!nr_pages) we need to flush the cache too.
2323
2324                    Also if we've been setting superpages, we may need to
2325                    recalculate 'pte' and switch back to smaller pages for the
2326                    end of the mapping, if the trailing size is not enough to
2327                    use another superpage (i.e. sg_res < lvl_pages). */
2328                 pte++;
2329                 if (!nr_pages || first_pte_in_page(pte) ||
2330                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2331                         domain_flush_cache(domain, first_pte,
2332                                            (void *)pte - (void *)first_pte);
2333                         pte = NULL;
2334                 }
2335
2336                 if (!sg_res && nr_pages)
2337                         sg = sg_next(sg);
2338         }
2339         return 0;
2340 }
2341
2342 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2343                                     struct scatterlist *sg, unsigned long nr_pages,
2344                                     int prot)
2345 {
2346         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2347 }
2348
2349 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2350                                      unsigned long phys_pfn, unsigned long nr_pages,
2351                                      int prot)
2352 {
2353         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2354 }
2355
2356 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2357 {
2358         unsigned long flags;
2359         struct context_entry *context;
2360         u16 did_old;
2361
2362         if (!iommu)
2363                 return;
2364
2365         spin_lock_irqsave(&iommu->lock, flags);
2366         context = iommu_context_addr(iommu, bus, devfn, 0);
2367         if (!context) {
2368                 spin_unlock_irqrestore(&iommu->lock, flags);
2369                 return;
2370         }
2371         did_old = context_domain_id(context);
2372         context_clear_entry(context);
2373         __iommu_flush_cache(iommu, context, sizeof(*context));
2374         spin_unlock_irqrestore(&iommu->lock, flags);
2375         iommu->flush.flush_context(iommu,
2376                                    did_old,
2377                                    (((u16)bus) << 8) | devfn,
2378                                    DMA_CCMD_MASK_NOBIT,
2379                                    DMA_CCMD_DEVICE_INVL);
2380         iommu->flush.flush_iotlb(iommu,
2381                                  did_old,
2382                                  0,
2383                                  0,
2384                                  DMA_TLB_DSI_FLUSH);
2385 }
2386
2387 static inline void unlink_domain_info(struct device_domain_info *info)
2388 {
2389         assert_spin_locked(&device_domain_lock);
2390         list_del(&info->link);
2391         list_del(&info->global);
2392         if (info->dev)
2393                 info->dev->archdata.iommu = NULL;
2394 }
2395
2396 static void domain_remove_dev_info(struct dmar_domain *domain)
2397 {
2398         struct device_domain_info *info, *tmp;
2399         unsigned long flags;
2400
2401         spin_lock_irqsave(&device_domain_lock, flags);
2402         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2403                 __dmar_remove_one_dev_info(info);
2404         spin_unlock_irqrestore(&device_domain_lock, flags);
2405 }
2406
2407 /*
2408  * find_domain
2409  * Note: we use struct device->archdata.iommu stores the info
2410  */
2411 static struct dmar_domain *find_domain(struct device *dev)
2412 {
2413         struct device_domain_info *info;
2414
2415         /* No lock here, assumes no domain exit in normal case */
2416         info = dev->archdata.iommu;
2417         if (likely(info))
2418                 return info->domain;
2419         return NULL;
2420 }
2421
2422 static inline struct device_domain_info *
2423 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2424 {
2425         struct device_domain_info *info;
2426
2427         list_for_each_entry(info, &device_domain_list, global)
2428                 if (info->iommu->segment == segment && info->bus == bus &&
2429                     info->devfn == devfn)
2430                         return info;
2431
2432         return NULL;
2433 }
2434
2435 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2436                                                     int bus, int devfn,
2437                                                     struct device *dev,
2438                                                     struct dmar_domain *domain)
2439 {
2440         struct dmar_domain *found = NULL;
2441         struct device_domain_info *info;
2442         unsigned long flags;
2443         int ret;
2444
2445         info = alloc_devinfo_mem();
2446         if (!info)
2447                 return NULL;
2448
2449         info->bus = bus;
2450         info->devfn = devfn;
2451         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2452         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2453         info->ats_qdep = 0;
2454         info->dev = dev;
2455         info->domain = domain;
2456         info->iommu = iommu;
2457
2458         if (dev && dev_is_pci(dev)) {
2459                 struct pci_dev *pdev = to_pci_dev(info->dev);
2460
2461                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2462                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2463                     dmar_find_matched_atsr_unit(pdev))
2464                         info->ats_supported = 1;
2465
2466                 if (ecs_enabled(iommu)) {
2467                         if (pasid_enabled(iommu)) {
2468                                 int features = pci_pasid_features(pdev);
2469                                 if (features >= 0)
2470                                         info->pasid_supported = features | 1;
2471                         }
2472
2473                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2474                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2475                                 info->pri_supported = 1;
2476                 }
2477         }
2478
2479         spin_lock_irqsave(&device_domain_lock, flags);
2480         if (dev)
2481                 found = find_domain(dev);
2482
2483         if (!found) {
2484                 struct device_domain_info *info2;
2485                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2486                 if (info2) {
2487                         found      = info2->domain;
2488                         info2->dev = dev;
2489                 }
2490         }
2491
2492         if (found) {
2493                 spin_unlock_irqrestore(&device_domain_lock, flags);
2494                 free_devinfo_mem(info);
2495                 /* Caller must free the original domain */
2496                 return found;
2497         }
2498
2499         spin_lock(&iommu->lock);
2500         ret = domain_attach_iommu(domain, iommu);
2501         spin_unlock(&iommu->lock);
2502
2503         if (ret) {
2504                 spin_unlock_irqrestore(&device_domain_lock, flags);
2505                 free_devinfo_mem(info);
2506                 return NULL;
2507         }
2508
2509         list_add(&info->link, &domain->devices);
2510         list_add(&info->global, &device_domain_list);
2511         if (dev)
2512                 dev->archdata.iommu = info;
2513         spin_unlock_irqrestore(&device_domain_lock, flags);
2514
2515         if (dev && domain_context_mapping(domain, dev)) {
2516                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2517                 dmar_remove_one_dev_info(domain, dev);
2518                 return NULL;
2519         }
2520
2521         return domain;
2522 }
2523
2524 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2525 {
2526         *(u16 *)opaque = alias;
2527         return 0;
2528 }
2529
2530 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2531 {
2532         struct device_domain_info *info = NULL;
2533         struct dmar_domain *domain = NULL;
2534         struct intel_iommu *iommu;
2535         u16 req_id, dma_alias;
2536         unsigned long flags;
2537         u8 bus, devfn;
2538
2539         iommu = device_to_iommu(dev, &bus, &devfn);
2540         if (!iommu)
2541                 return NULL;
2542
2543         req_id = ((u16)bus << 8) | devfn;
2544
2545         if (dev_is_pci(dev)) {
2546                 struct pci_dev *pdev = to_pci_dev(dev);
2547
2548                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2549
2550                 spin_lock_irqsave(&device_domain_lock, flags);
2551                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2552                                                       PCI_BUS_NUM(dma_alias),
2553                                                       dma_alias & 0xff);
2554                 if (info) {
2555                         iommu = info->iommu;
2556                         domain = info->domain;
2557                 }
2558                 spin_unlock_irqrestore(&device_domain_lock, flags);
2559
2560                 /* DMA alias already has a domain, use it */
2561                 if (info)
2562                         goto out;
2563         }
2564
2565         /* Allocate and initialize new domain for the device */
2566         domain = alloc_domain(0);
2567         if (!domain)
2568                 return NULL;
2569         if (domain_init(domain, iommu, gaw)) {
2570                 domain_exit(domain);
2571                 return NULL;
2572         }
2573
2574 out:
2575
2576         return domain;
2577 }
2578
2579 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2580                                               struct dmar_domain *domain)
2581 {
2582         struct intel_iommu *iommu;
2583         struct dmar_domain *tmp;
2584         u16 req_id, dma_alias;
2585         u8 bus, devfn;
2586
2587         iommu = device_to_iommu(dev, &bus, &devfn);
2588         if (!iommu)
2589                 return NULL;
2590
2591         req_id = ((u16)bus << 8) | devfn;
2592
2593         if (dev_is_pci(dev)) {
2594                 struct pci_dev *pdev = to_pci_dev(dev);
2595
2596                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2597
2598                 /* register PCI DMA alias device */
2599                 if (req_id != dma_alias) {
2600                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2601                                         dma_alias & 0xff, NULL, domain);
2602
2603                         if (!tmp || tmp != domain)
2604                                 return tmp;
2605                 }
2606         }
2607
2608         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2609         if (!tmp || tmp != domain)
2610                 return tmp;
2611
2612         return domain;
2613 }
2614
2615 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2616 {
2617         struct dmar_domain *domain, *tmp;
2618
2619         domain = find_domain(dev);
2620         if (domain)
2621                 goto out;
2622
2623         domain = find_or_alloc_domain(dev, gaw);
2624         if (!domain)
2625                 goto out;
2626
2627         tmp = set_domain_for_dev(dev, domain);
2628         if (!tmp || domain != tmp) {
2629                 domain_exit(domain);
2630                 domain = tmp;
2631         }
2632
2633 out:
2634
2635         return domain;
2636 }
2637
2638 static int iommu_domain_identity_map(struct dmar_domain *domain,
2639                                      unsigned long long start,
2640                                      unsigned long long end)
2641 {
2642         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2643         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2644
2645         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2646                           dma_to_mm_pfn(last_vpfn))) {
2647                 pr_err("Reserving iova failed\n");
2648                 return -ENOMEM;
2649         }
2650
2651         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2652         /*
2653          * RMRR range might have overlap with physical memory range,
2654          * clear it first
2655          */
2656         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2657
2658         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2659                                   last_vpfn - first_vpfn + 1,
2660                                   DMA_PTE_READ|DMA_PTE_WRITE);
2661 }
2662
2663 static int domain_prepare_identity_map(struct device *dev,
2664                                        struct dmar_domain *domain,
2665                                        unsigned long long start,
2666                                        unsigned long long end)
2667 {
2668         /* For _hardware_ passthrough, don't bother. But for software
2669            passthrough, we do it anyway -- it may indicate a memory
2670            range which is reserved in E820, so which didn't get set
2671            up to start with in si_domain */
2672         if (domain == si_domain && hw_pass_through) {
2673                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2674                         dev_name(dev), start, end);
2675                 return 0;
2676         }
2677
2678         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2679                 dev_name(dev), start, end);
2680
2681         if (end < start) {
2682                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2683                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2684                         dmi_get_system_info(DMI_BIOS_VENDOR),
2685                         dmi_get_system_info(DMI_BIOS_VERSION),
2686                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2687                 return -EIO;
2688         }
2689
2690         if (end >> agaw_to_width(domain->agaw)) {
2691                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2692                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2693                      agaw_to_width(domain->agaw),
2694                      dmi_get_system_info(DMI_BIOS_VENDOR),
2695                      dmi_get_system_info(DMI_BIOS_VERSION),
2696                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2697                 return -EIO;
2698         }
2699
2700         return iommu_domain_identity_map(domain, start, end);
2701 }
2702
2703 static int iommu_prepare_identity_map(struct device *dev,
2704                                       unsigned long long start,
2705                                       unsigned long long end)
2706 {
2707         struct dmar_domain *domain;
2708         int ret;
2709
2710         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2711         if (!domain)
2712                 return -ENOMEM;
2713
2714         ret = domain_prepare_identity_map(dev, domain, start, end);
2715         if (ret)
2716                 domain_exit(domain);
2717
2718         return ret;
2719 }
2720
2721 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2722                                          struct device *dev)
2723 {
2724         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2725                 return 0;
2726         return iommu_prepare_identity_map(dev, rmrr->base_address,
2727                                           rmrr->end_address);
2728 }
2729
2730 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2731 static inline void iommu_prepare_isa(void)
2732 {
2733         struct pci_dev *pdev;
2734         int ret;
2735
2736         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2737         if (!pdev)
2738                 return;
2739
2740         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2741         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2742
2743         if (ret)
2744                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2745
2746         pci_dev_put(pdev);
2747 }
2748 #else
2749 static inline void iommu_prepare_isa(void)
2750 {
2751         return;
2752 }
2753 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2754
2755 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2756
2757 static int __init si_domain_init(int hw)
2758 {
2759         int nid, ret = 0;
2760
2761         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2762         if (!si_domain)
2763                 return -EFAULT;
2764
2765         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2766                 domain_exit(si_domain);
2767                 return -EFAULT;
2768         }
2769
2770         pr_debug("Identity mapping domain allocated\n");
2771
2772         if (hw)
2773                 return 0;
2774
2775         for_each_online_node(nid) {
2776                 unsigned long start_pfn, end_pfn;
2777                 int i;
2778
2779                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2780                         ret = iommu_domain_identity_map(si_domain,
2781                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2782                         if (ret)
2783                                 return ret;
2784                 }
2785         }
2786
2787         return 0;
2788 }
2789
2790 static int identity_mapping(struct device *dev)
2791 {
2792         struct device_domain_info *info;
2793
2794         if (likely(!iommu_identity_mapping))
2795                 return 0;
2796
2797         info = dev->archdata.iommu;
2798         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2799                 return (info->domain == si_domain);
2800
2801         return 0;
2802 }
2803
2804 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2805 {
2806         struct dmar_domain *ndomain;
2807         struct intel_iommu *iommu;
2808         u8 bus, devfn;
2809
2810         iommu = device_to_iommu(dev, &bus, &devfn);
2811         if (!iommu)
2812                 return -ENODEV;
2813
2814         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2815         if (ndomain != domain)
2816                 return -EBUSY;
2817
2818         return 0;
2819 }
2820
2821 static bool device_has_rmrr(struct device *dev)
2822 {
2823         struct dmar_rmrr_unit *rmrr;
2824         struct device *tmp;
2825         int i;
2826
2827         rcu_read_lock();
2828         for_each_rmrr_units(rmrr) {
2829                 /*
2830                  * Return TRUE if this RMRR contains the device that
2831                  * is passed in.
2832                  */
2833                 for_each_active_dev_scope(rmrr->devices,
2834                                           rmrr->devices_cnt, i, tmp)
2835                         if (tmp == dev) {
2836                                 rcu_read_unlock();
2837                                 return true;
2838                         }
2839         }
2840         rcu_read_unlock();
2841         return false;
2842 }
2843
2844 /*
2845  * There are a couple cases where we need to restrict the functionality of
2846  * devices associated with RMRRs.  The first is when evaluating a device for
2847  * identity mapping because problems exist when devices are moved in and out
2848  * of domains and their respective RMRR information is lost.  This means that
2849  * a device with associated RMRRs will never be in a "passthrough" domain.
2850  * The second is use of the device through the IOMMU API.  This interface
2851  * expects to have full control of the IOVA space for the device.  We cannot
2852  * satisfy both the requirement that RMRR access is maintained and have an
2853  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2854  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2855  * We therefore prevent devices associated with an RMRR from participating in
2856  * the IOMMU API, which eliminates them from device assignment.
2857  *
2858  * In both cases we assume that PCI USB devices with RMRRs have them largely
2859  * for historical reasons and that the RMRR space is not actively used post
2860  * boot.  This exclusion may change if vendors begin to abuse it.
2861  *
2862  * The same exception is made for graphics devices, with the requirement that
2863  * any use of the RMRR regions will be torn down before assigning the device
2864  * to a guest.
2865  */
2866 static bool device_is_rmrr_locked(struct device *dev)
2867 {
2868         if (!device_has_rmrr(dev))
2869                 return false;
2870
2871         if (dev_is_pci(dev)) {
2872                 struct pci_dev *pdev = to_pci_dev(dev);
2873
2874                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2875                         return false;
2876         }
2877
2878         return true;
2879 }
2880
2881 static int iommu_should_identity_map(struct device *dev, int startup)
2882 {
2883
2884         if (dev_is_pci(dev)) {
2885                 struct pci_dev *pdev = to_pci_dev(dev);
2886
2887                 if (device_is_rmrr_locked(dev))
2888                         return 0;
2889
2890                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2891                         return 1;
2892
2893                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2894                         return 1;
2895
2896                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2897                         return 0;
2898
2899                 /*
2900                  * We want to start off with all devices in the 1:1 domain, and
2901                  * take them out later if we find they can't access all of memory.
2902                  *
2903                  * However, we can't do this for PCI devices behind bridges,
2904                  * because all PCI devices behind the same bridge will end up
2905                  * with the same source-id on their transactions.
2906                  *
2907                  * Practically speaking, we can't change things around for these
2908                  * devices at run-time, because we can't be sure there'll be no
2909                  * DMA transactions in flight for any of their siblings.
2910                  *
2911                  * So PCI devices (unless they're on the root bus) as well as
2912                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2913                  * the 1:1 domain, just in _case_ one of their siblings turns out
2914                  * not to be able to map all of memory.
2915                  */
2916                 if (!pci_is_pcie(pdev)) {
2917                         if (!pci_is_root_bus(pdev->bus))
2918                                 return 0;
2919                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2920                                 return 0;
2921                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2922                         return 0;
2923         } else {
2924                 if (device_has_rmrr(dev))
2925                         return 0;
2926         }
2927
2928         /*
2929          * At boot time, we don't yet know if devices will be 64-bit capable.
2930          * Assume that they will — if they turn out not to be, then we can
2931          * take them out of the 1:1 domain later.
2932          */
2933         if (!startup) {
2934                 /*
2935                  * If the device's dma_mask is less than the system's memory
2936                  * size then this is not a candidate for identity mapping.
2937                  */
2938                 u64 dma_mask = *dev->dma_mask;
2939
2940                 if (dev->coherent_dma_mask &&
2941                     dev->coherent_dma_mask < dma_mask)
2942                         dma_mask = dev->coherent_dma_mask;
2943
2944                 return dma_mask >= dma_get_required_mask(dev);
2945         }
2946
2947         return 1;
2948 }
2949
2950 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2951 {
2952         int ret;
2953
2954         if (!iommu_should_identity_map(dev, 1))
2955                 return 0;
2956
2957         ret = domain_add_dev_info(si_domain, dev);
2958         if (!ret)
2959                 pr_info("%s identity mapping for device %s\n",
2960                         hw ? "Hardware" : "Software", dev_name(dev));
2961         else if (ret == -ENODEV)
2962                 /* device not associated with an iommu */
2963                 ret = 0;
2964
2965         return ret;
2966 }
2967
2968
2969 static int __init iommu_prepare_static_identity_mapping(int hw)
2970 {
2971         struct pci_dev *pdev = NULL;
2972         struct dmar_drhd_unit *drhd;
2973         struct intel_iommu *iommu;
2974         struct device *dev;
2975         int i;
2976         int ret = 0;
2977
2978         for_each_pci_dev(pdev) {
2979                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2980                 if (ret)
2981                         return ret;
2982         }
2983
2984         for_each_active_iommu(iommu, drhd)
2985                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2986                         struct acpi_device_physical_node *pn;
2987                         struct acpi_device *adev;
2988
2989                         if (dev->bus != &acpi_bus_type)
2990                                 continue;
2991
2992                         adev= to_acpi_device(dev);
2993                         mutex_lock(&adev->physical_node_lock);
2994                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2995                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2996                                 if (ret)
2997                                         break;
2998                         }
2999                         mutex_unlock(&adev->physical_node_lock);
3000                         if (ret)
3001                                 return ret;
3002                 }
3003
3004         return 0;
3005 }
3006
3007 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3008 {
3009         /*
3010          * Start from the sane iommu hardware state.
3011          * If the queued invalidation is already initialized by us
3012          * (for example, while enabling interrupt-remapping) then
3013          * we got the things already rolling from a sane state.
3014          */
3015         if (!iommu->qi) {
3016                 /*
3017                  * Clear any previous faults.
3018                  */
3019                 dmar_fault(-1, iommu);
3020                 /*
3021                  * Disable queued invalidation if supported and already enabled
3022                  * before OS handover.
3023                  */
3024                 dmar_disable_qi(iommu);
3025         }
3026
3027         if (dmar_enable_qi(iommu)) {
3028                 /*
3029                  * Queued Invalidate not enabled, use Register Based Invalidate
3030                  */
3031                 iommu->flush.flush_context = __iommu_flush_context;
3032                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3033                 pr_info("%s: Using Register based invalidation\n",
3034                         iommu->name);
3035         } else {
3036                 iommu->flush.flush_context = qi_flush_context;
3037                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3038                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3039         }
3040 }
3041
3042 static int copy_context_table(struct intel_iommu *iommu,
3043                               struct root_entry *old_re,
3044                               struct context_entry **tbl,
3045                               int bus, bool ext)
3046 {
3047         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3048         struct context_entry *new_ce = NULL, ce;
3049         struct context_entry *old_ce = NULL;
3050         struct root_entry re;
3051         phys_addr_t old_ce_phys;
3052
3053         tbl_idx = ext ? bus * 2 : bus;
3054         memcpy(&re, old_re, sizeof(re));
3055
3056         for (devfn = 0; devfn < 256; devfn++) {
3057                 /* First calculate the correct index */
3058                 idx = (ext ? devfn * 2 : devfn) % 256;
3059
3060                 if (idx == 0) {
3061                         /* First save what we may have and clean up */
3062                         if (new_ce) {
3063                                 tbl[tbl_idx] = new_ce;
3064                                 __iommu_flush_cache(iommu, new_ce,
3065                                                     VTD_PAGE_SIZE);
3066                                 pos = 1;
3067                         }
3068
3069                         if (old_ce)
3070                                 iounmap(old_ce);
3071
3072                         ret = 0;
3073                         if (devfn < 0x80)
3074                                 old_ce_phys = root_entry_lctp(&re);
3075                         else
3076                                 old_ce_phys = root_entry_uctp(&re);
3077
3078                         if (!old_ce_phys) {
3079                                 if (ext && devfn == 0) {
3080                                         /* No LCTP, try UCTP */
3081                                         devfn = 0x7f;
3082                                         continue;
3083                                 } else {
3084                                         goto out;
3085                                 }
3086                         }
3087
3088                         ret = -ENOMEM;
3089                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3090                                         MEMREMAP_WB);
3091                         if (!old_ce)
3092                                 goto out;
3093
3094                         new_ce = alloc_pgtable_page(iommu->node);
3095                         if (!new_ce)
3096                                 goto out_unmap;
3097
3098                         ret = 0;
3099                 }
3100
3101                 /* Now copy the context entry */
3102                 memcpy(&ce, old_ce + idx, sizeof(ce));
3103
3104                 if (!__context_present(&ce))
3105                         continue;
3106
3107                 did = context_domain_id(&ce);
3108                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3109                         set_bit(did, iommu->domain_ids);
3110
3111                 /*
3112                  * We need a marker for copied context entries. This
3113                  * marker needs to work for the old format as well as
3114                  * for extended context entries.
3115                  *
3116                  * Bit 67 of the context entry is used. In the old
3117                  * format this bit is available to software, in the
3118                  * extended format it is the PGE bit, but PGE is ignored
3119                  * by HW if PASIDs are disabled (and thus still
3120                  * available).
3121                  *
3122                  * So disable PASIDs first and then mark the entry
3123                  * copied. This means that we don't copy PASID
3124                  * translations from the old kernel, but this is fine as
3125                  * faults there are not fatal.
3126                  */
3127                 context_clear_pasid_enable(&ce);
3128                 context_set_copied(&ce);
3129
3130                 new_ce[idx] = ce;
3131         }
3132
3133         tbl[tbl_idx + pos] = new_ce;
3134
3135         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3136
3137 out_unmap:
3138         memunmap(old_ce);
3139
3140 out:
3141         return ret;
3142 }
3143
3144 static int copy_translation_tables(struct intel_iommu *iommu)
3145 {
3146         struct context_entry **ctxt_tbls;
3147         struct root_entry *old_rt;
3148         phys_addr_t old_rt_phys;
3149         int ctxt_table_entries;
3150         unsigned long flags;
3151         u64 rtaddr_reg;
3152         int bus, ret;
3153         bool new_ext, ext;
3154
3155         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3156         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3157         new_ext    = !!ecap_ecs(iommu->ecap);
3158
3159         /*
3160          * The RTT bit can only be changed when translation is disabled,
3161          * but disabling translation means to open a window for data
3162          * corruption. So bail out and don't copy anything if we would
3163          * have to change the bit.
3164          */
3165         if (new_ext != ext)
3166                 return -EINVAL;
3167
3168         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3169         if (!old_rt_phys)
3170                 return -EINVAL;
3171
3172         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3173         if (!old_rt)
3174                 return -ENOMEM;
3175
3176         /* This is too big for the stack - allocate it from slab */
3177         ctxt_table_entries = ext ? 512 : 256;
3178         ret = -ENOMEM;
3179         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3180         if (!ctxt_tbls)
3181                 goto out_unmap;
3182
3183         for (bus = 0; bus < 256; bus++) {
3184                 ret = copy_context_table(iommu, &old_rt[bus],
3185                                          ctxt_tbls, bus, ext);
3186                 if (ret) {
3187                         pr_err("%s: Failed to copy context table for bus %d\n",
3188                                 iommu->name, bus);
3189                         continue;
3190                 }
3191         }
3192
3193         spin_lock_irqsave(&iommu->lock, flags);
3194
3195         /* Context tables are copied, now write them to the root_entry table */
3196         for (bus = 0; bus < 256; bus++) {
3197                 int idx = ext ? bus * 2 : bus;
3198                 u64 val;
3199
3200                 if (ctxt_tbls[idx]) {
3201                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3202                         iommu->root_entry[bus].lo = val;
3203                 }
3204
3205                 if (!ext || !ctxt_tbls[idx + 1])
3206                         continue;
3207
3208                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3209                 iommu->root_entry[bus].hi = val;
3210         }
3211
<