1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
26#include <linux/debugfs.h>
27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
35#include <linux/timer.h>
36#include <linux/iova.h>
37#include <linux/iommu.h>
38#include <linux/intel-iommu.h>
39#include <linux/syscore_ops.h>
40#include <linux/tboot.h>
41#include <linux/dmi.h>
42#include <asm/cacheflush.h>
43#include <asm/iommu.h>
44#include "pci.h"
45
46#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
49#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
51#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
52
53#define IOAPIC_RANGE_START (0xfee00000)
54#define IOAPIC_RANGE_END (0xfeefffff)
55#define IOVA_START_ADDR (0x1000)
56
57#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
59#define MAX_AGAW_WIDTH 64
60
61#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64
65
66#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
69
70#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
71#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
72#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
73
74
75#define LEVEL_STRIDE (9)
76#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
77
78static inline int agaw_to_level(int agaw)
79{
80 return agaw + 2;
81}
82
83static inline int agaw_to_width(int agaw)
84{
85 return 30 + agaw * LEVEL_STRIDE;
86}
87
88static inline int width_to_agaw(int width)
89{
90 return (width - 30) / LEVEL_STRIDE;
91}
92
93static inline unsigned int level_to_offset_bits(int level)
94{
95 return (level - 1) * LEVEL_STRIDE;
96}
97
98static inline int pfn_level_offset(unsigned long pfn, int level)
99{
100 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
101}
102
103static inline unsigned long level_mask(int level)
104{
105 return -1UL << level_to_offset_bits(level);
106}
107
108static inline unsigned long level_size(int level)
109{
110 return 1UL << level_to_offset_bits(level);
111}
112
113static inline unsigned long align_to_level(unsigned long pfn, int level)
114{
115 return (pfn + level_size(level) - 1) & level_mask(level);
116}
117
118
119
120static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
121{
122 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
123}
124
125static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
126{
127 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
128}
129static inline unsigned long page_to_dma_pfn(struct page *pg)
130{
131 return mm_to_dma_pfn(page_to_pfn(pg));
132}
133static inline unsigned long virt_to_dma_pfn(void *p)
134{
135 return page_to_dma_pfn(virt_to_page(p));
136}
137
138
139static struct intel_iommu **g_iommus;
140
141static void __init check_tylersburg_isoch(void);
142static int rwbf_quirk;
143
144
145
146
147
148
149
150struct root_entry {
151 u64 val;
152 u64 rsvd1;
153};
154#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
155static inline bool root_present(struct root_entry *root)
156{
157 return (root->val & 1);
158}
159static inline void set_root_present(struct root_entry *root)
160{
161 root->val |= 1;
162}
163static inline void set_root_value(struct root_entry *root, unsigned long value)
164{
165 root->val |= value & VTD_PAGE_MASK;
166}
167
168static inline struct context_entry *
169get_context_addr_from_root(struct root_entry *root)
170{
171 return (struct context_entry *)
172 (root_present(root)?phys_to_virt(
173 root->val & VTD_PAGE_MASK) :
174 NULL);
175}
176
177
178
179
180
181
182
183
184
185
186
187
188struct context_entry {
189 u64 lo;
190 u64 hi;
191};
192
193static inline bool context_present(struct context_entry *context)
194{
195 return (context->lo & 1);
196}
197static inline void context_set_present(struct context_entry *context)
198{
199 context->lo |= 1;
200}
201
202static inline void context_set_fault_enable(struct context_entry *context)
203{
204 context->lo &= (((u64)-1) << 2) | 1;
205}
206
207static inline void context_set_translation_type(struct context_entry *context,
208 unsigned long value)
209{
210 context->lo &= (((u64)-1) << 4) | 3;
211 context->lo |= (value & 3) << 2;
212}
213
214static inline void context_set_address_root(struct context_entry *context,
215 unsigned long value)
216{
217 context->lo |= value & VTD_PAGE_MASK;
218}
219
220static inline void context_set_address_width(struct context_entry *context,
221 unsigned long value)
222{
223 context->hi |= value & 7;
224}
225
226static inline void context_set_domain_id(struct context_entry *context,
227 unsigned long value)
228{
229 context->hi |= (value & ((1 << 16) - 1)) << 8;
230}
231
232static inline void context_clear_entry(struct context_entry *context)
233{
234 context->lo = 0;
235 context->hi = 0;
236}
237
238
239
240
241
242
243
244
245
246
247struct dma_pte {
248 u64 val;
249};
250
251static inline void dma_clear_pte(struct dma_pte *pte)
252{
253 pte->val = 0;
254}
255
256static inline void dma_set_pte_readable(struct dma_pte *pte)
257{
258 pte->val |= DMA_PTE_READ;
259}
260
261static inline void dma_set_pte_writable(struct dma_pte *pte)
262{
263 pte->val |= DMA_PTE_WRITE;
264}
265
266static inline void dma_set_pte_snp(struct dma_pte *pte)
267{
268 pte->val |= DMA_PTE_SNP;
269}
270
271static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
272{
273 pte->val = (pte->val & ~3) | (prot & 3);
274}
275
276static inline u64 dma_pte_addr(struct dma_pte *pte)
277{
278#ifdef CONFIG_64BIT
279 return pte->val & VTD_PAGE_MASK;
280#else
281
282 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
283#endif
284}
285
286static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
287{
288 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
289}
290
291static inline bool dma_pte_present(struct dma_pte *pte)
292{
293 return (pte->val & 3) != 0;
294}
295
296static inline int first_pte_in_page(struct dma_pte *pte)
297{
298 return !((unsigned long)pte & ~VTD_PAGE_MASK);
299}
300
301
302
303
304
305
306
307static struct dmar_domain *si_domain;
308static int hw_pass_through = 1;
309
310
311#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
312
313
314
315
316#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
317
318
319#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
320
321struct dmar_domain {
322 int id;
323 int nid;
324 unsigned long iommu_bmp;
325
326 struct list_head devices;
327 struct iova_domain iovad;
328
329 struct dma_pte *pgd;
330 int gaw;
331
332
333 int agaw;
334
335 int flags;
336
337 int iommu_coherency;
338 int iommu_snooping;
339 int iommu_count;
340 spinlock_t iommu_lock;
341 u64 max_addr;
342};
343
344
345struct device_domain_info {
346 struct list_head link;
347 struct list_head global;
348 int segment;
349 u8 bus;
350 u8 devfn;
351 struct pci_dev *dev;
352 struct intel_iommu *iommu;
353 struct dmar_domain *domain;
354};
355
356static void flush_unmaps_timeout(unsigned long data);
357
358DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
359
360#define HIGH_WATER_MARK 250
361struct deferred_flush_tables {
362 int next;
363 struct iova *iova[HIGH_WATER_MARK];
364 struct dmar_domain *domain[HIGH_WATER_MARK];
365};
366
367static struct deferred_flush_tables *deferred_flush;
368
369
370static int g_num_of_iommus;
371
372static DEFINE_SPINLOCK(async_umap_flush_lock);
373static LIST_HEAD(unmaps_to_do);
374
375static int timer_on;
376static long list_size;
377
378static void domain_remove_dev_info(struct dmar_domain *domain);
379
380#ifdef CONFIG_DMAR_DEFAULT_ON
381int dmar_disabled = 0;
382#else
383int dmar_disabled = 1;
384#endif
385
386static int dmar_map_gfx = 1;
387static int dmar_forcedac;
388static int intel_iommu_strict;
389
390#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
391static DEFINE_SPINLOCK(device_domain_lock);
392static LIST_HEAD(device_domain_list);
393
394static struct iommu_ops intel_iommu_ops;
395
396static int __init intel_iommu_setup(char *str)
397{
398 if (!str)
399 return -EINVAL;
400 while (*str) {
401 if (!strncmp(str, "on", 2)) {
402 dmar_disabled = 0;
403 printk(KERN_INFO "Intel-IOMMU: enabled\n");
404 } else if (!strncmp(str, "off", 3)) {
405 dmar_disabled = 1;
406 printk(KERN_INFO "Intel-IOMMU: disabled\n");
407 } else if (!strncmp(str, "igfx_off", 8)) {
408 dmar_map_gfx = 0;
409 printk(KERN_INFO
410 "Intel-IOMMU: disable GFX device mapping\n");
411 } else if (!strncmp(str, "forcedac", 8)) {
412 printk(KERN_INFO
413 "Intel-IOMMU: Forcing DAC for PCI devices\n");
414 dmar_forcedac = 1;
415 } else if (!strncmp(str, "strict", 6)) {
416 printk(KERN_INFO
417 "Intel-IOMMU: disable batched IOTLB flush\n");
418 intel_iommu_strict = 1;
419 }
420
421 str += strcspn(str, ",");
422 while (*str == ',')
423 str++;
424 }
425 return 0;
426}
427__setup("intel_iommu=", intel_iommu_setup);
428
429static struct kmem_cache *iommu_domain_cache;
430static struct kmem_cache *iommu_devinfo_cache;
431static struct kmem_cache *iommu_iova_cache;
432
433static inline void *alloc_pgtable_page(int node)
434{
435 struct page *page;
436 void *vaddr = NULL;
437
438 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
439 if (page)
440 vaddr = page_address(page);
441 return vaddr;
442}
443
444static inline void free_pgtable_page(void *vaddr)
445{
446 free_page((unsigned long)vaddr);
447}
448
449static inline void *alloc_domain_mem(void)
450{
451 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
452}
453
454static void free_domain_mem(void *vaddr)
455{
456 kmem_cache_free(iommu_domain_cache, vaddr);
457}
458
459static inline void * alloc_devinfo_mem(void)
460{
461 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
462}
463
464static inline void free_devinfo_mem(void *vaddr)
465{
466 kmem_cache_free(iommu_devinfo_cache, vaddr);
467}
468
469struct iova *alloc_iova_mem(void)
470{
471 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
472}
473
474void free_iova_mem(struct iova *iova)
475{
476 kmem_cache_free(iommu_iova_cache, iova);
477}
478
479
480static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
481{
482 unsigned long sagaw;
483 int agaw = -1;
484
485 sagaw = cap_sagaw(iommu->cap);
486 for (agaw = width_to_agaw(max_gaw);
487 agaw >= 0; agaw--) {
488 if (test_bit(agaw, &sagaw))
489 break;
490 }
491
492 return agaw;
493}
494
495
496
497
498int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
499{
500 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
501}
502
503
504
505
506
507
508int iommu_calculate_agaw(struct intel_iommu *iommu)
509{
510 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
511}
512
513
514static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
515{
516 int iommu_id;
517
518
519 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
520 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
521
522 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
523 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
524 return NULL;
525
526 return g_iommus[iommu_id];
527}
528
529static void domain_update_iommu_coherency(struct dmar_domain *domain)
530{
531 int i;
532
533 domain->iommu_coherency = 1;
534
535 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
536 if (!ecap_coherent(g_iommus[i]->ecap)) {
537 domain->iommu_coherency = 0;
538 break;
539 }
540 }
541}
542
543static void domain_update_iommu_snooping(struct dmar_domain *domain)
544{
545 int i;
546
547 domain->iommu_snooping = 1;
548
549 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
550 if (!ecap_sc_support(g_iommus[i]->ecap)) {
551 domain->iommu_snooping = 0;
552 break;
553 }
554 }
555}
556
557
558static void domain_update_iommu_cap(struct dmar_domain *domain)
559{
560 domain_update_iommu_coherency(domain);
561 domain_update_iommu_snooping(domain);
562}
563
564static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
565{
566 struct dmar_drhd_unit *drhd = NULL;
567 int i;
568
569 for_each_drhd_unit(drhd) {
570 if (drhd->ignored)
571 continue;
572 if (segment != drhd->segment)
573 continue;
574
575 for (i = 0; i < drhd->devices_cnt; i++) {
576 if (drhd->devices[i] &&
577 drhd->devices[i]->bus->number == bus &&
578 drhd->devices[i]->devfn == devfn)
579 return drhd->iommu;
580 if (drhd->devices[i] &&
581 drhd->devices[i]->subordinate &&
582 drhd->devices[i]->subordinate->number <= bus &&
583 drhd->devices[i]->subordinate->subordinate >= bus)
584 return drhd->iommu;
585 }
586
587 if (drhd->include_all)
588 return drhd->iommu;
589 }
590
591 return NULL;
592}
593
594static void domain_flush_cache(struct dmar_domain *domain,
595 void *addr, int size)
596{
597 if (!domain->iommu_coherency)
598 clflush_cache_range(addr, size);
599}
600
601
602static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
603 u8 bus, u8 devfn)
604{
605 struct root_entry *root;
606 struct context_entry *context;
607 unsigned long phy_addr;
608 unsigned long flags;
609
610 spin_lock_irqsave(&iommu->lock, flags);
611 root = &iommu->root_entry[bus];
612 context = get_context_addr_from_root(root);
613 if (!context) {
614 context = (struct context_entry *)
615 alloc_pgtable_page(iommu->node);
616 if (!context) {
617 spin_unlock_irqrestore(&iommu->lock, flags);
618 return NULL;
619 }
620 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
621 phy_addr = virt_to_phys((void *)context);
622 set_root_value(root, phy_addr);
623 set_root_present(root);
624 __iommu_flush_cache(iommu, root, sizeof(*root));
625 }
626 spin_unlock_irqrestore(&iommu->lock, flags);
627 return &context[devfn];
628}
629
630static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
631{
632 struct root_entry *root;
633 struct context_entry *context;
634 int ret;
635 unsigned long flags;
636
637 spin_lock_irqsave(&iommu->lock, flags);
638 root = &iommu->root_entry[bus];
639 context = get_context_addr_from_root(root);
640 if (!context) {
641 ret = 0;
642 goto out;
643 }
644 ret = context_present(&context[devfn]);
645out:
646 spin_unlock_irqrestore(&iommu->lock, flags);
647 return ret;
648}
649
650static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
651{
652 struct root_entry *root;
653 struct context_entry *context;
654 unsigned long flags;
655
656 spin_lock_irqsave(&iommu->lock, flags);
657 root = &iommu->root_entry[bus];
658 context = get_context_addr_from_root(root);
659 if (context) {
660 context_clear_entry(&context[devfn]);
661 __iommu_flush_cache(iommu, &context[devfn], \
662 sizeof(*context));
663 }
664 spin_unlock_irqrestore(&iommu->lock, flags);
665}
666
667static void free_context_table(struct intel_iommu *iommu)
668{
669 struct root_entry *root;
670 int i;
671 unsigned long flags;
672 struct context_entry *context;
673
674 spin_lock_irqsave(&iommu->lock, flags);
675 if (!iommu->root_entry) {
676 goto out;
677 }
678 for (i = 0; i < ROOT_ENTRY_NR; i++) {
679 root = &iommu->root_entry[i];
680 context = get_context_addr_from_root(root);
681 if (context)
682 free_pgtable_page(context);
683 }
684 free_pgtable_page(iommu->root_entry);
685 iommu->root_entry = NULL;
686out:
687 spin_unlock_irqrestore(&iommu->lock, flags);
688}
689
690static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
691 unsigned long pfn)
692{
693 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
694 struct dma_pte *parent, *pte = NULL;
695 int level = agaw_to_level(domain->agaw);
696 int offset;
697
698 BUG_ON(!domain->pgd);
699 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
700 parent = domain->pgd;
701
702 while (level > 0) {
703 void *tmp_page;
704
705 offset = pfn_level_offset(pfn, level);
706 pte = &parent[offset];
707 if (level == 1)
708 break;
709
710 if (!dma_pte_present(pte)) {
711 uint64_t pteval;
712
713 tmp_page = alloc_pgtable_page(domain->nid);
714
715 if (!tmp_page)
716 return NULL;
717
718 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
719 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
720 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
721
722 free_pgtable_page(tmp_page);
723 } else {
724 dma_pte_addr(pte);
725 domain_flush_cache(domain, pte, sizeof(*pte));
726 }
727 }
728 parent = phys_to_virt(dma_pte_addr(pte));
729 level--;
730 }
731
732 return pte;
733}
734
735
736static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
737 unsigned long pfn,
738 int level)
739{
740 struct dma_pte *parent, *pte = NULL;
741 int total = agaw_to_level(domain->agaw);
742 int offset;
743
744 parent = domain->pgd;
745 while (level <= total) {
746 offset = pfn_level_offset(pfn, total);
747 pte = &parent[offset];
748 if (level == total)
749 return pte;
750
751 if (!dma_pte_present(pte))
752 break;
753 parent = phys_to_virt(dma_pte_addr(pte));
754 total--;
755 }
756 return NULL;
757}
758
759
760static void dma_pte_clear_range(struct dmar_domain *domain,
761 unsigned long start_pfn,
762 unsigned long last_pfn)
763{
764 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
765 struct dma_pte *first_pte, *pte;
766
767 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
768 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
769 BUG_ON(start_pfn > last_pfn);
770
771
772 do {
773 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
774 if (!pte) {
775 start_pfn = align_to_level(start_pfn + 1, 2);
776 continue;
777 }
778 do {
779 dma_clear_pte(pte);
780 start_pfn++;
781 pte++;
782 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
783
784 domain_flush_cache(domain, first_pte,
785 (void *)pte - (void *)first_pte);
786
787 } while (start_pfn && start_pfn <= last_pfn);
788}
789
790
791static void dma_pte_free_pagetable(struct dmar_domain *domain,
792 unsigned long start_pfn,
793 unsigned long last_pfn)
794{
795 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
796 struct dma_pte *first_pte, *pte;
797 int total = agaw_to_level(domain->agaw);
798 int level;
799 unsigned long tmp;
800
801 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
802 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
803 BUG_ON(start_pfn > last_pfn);
804
805
806 level = 2;
807 while (level <= total) {
808 tmp = align_to_level(start_pfn, level);
809
810
811 if (tmp + level_size(level) - 1 > last_pfn)
812 return;
813
814 do {
815 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
816 if (!pte) {
817 tmp = align_to_level(tmp + 1, level + 1);
818 continue;
819 }
820 do {
821 if (dma_pte_present(pte)) {
822 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
823 dma_clear_pte(pte);
824 }
825 pte++;
826 tmp += level_size(level);
827 } while (!first_pte_in_page(pte) &&
828 tmp + level_size(level) - 1 <= last_pfn);
829
830 domain_flush_cache(domain, first_pte,
831 (void *)pte - (void *)first_pte);
832
833 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
834 level++;
835 }
836
837 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
838 free_pgtable_page(domain->pgd);
839 domain->pgd = NULL;
840 }
841}
842
843
844static int iommu_alloc_root_entry(struct intel_iommu *iommu)
845{
846 struct root_entry *root;
847 unsigned long flags;
848
849 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
850 if (!root)
851 return -ENOMEM;
852
853 __iommu_flush_cache(iommu, root, ROOT_SIZE);
854
855 spin_lock_irqsave(&iommu->lock, flags);
856 iommu->root_entry = root;
857 spin_unlock_irqrestore(&iommu->lock, flags);
858
859 return 0;
860}
861
862static void iommu_set_root_entry(struct intel_iommu *iommu)
863{
864 void *addr;
865 u32 sts;
866 unsigned long flag;
867
868 addr = iommu->root_entry;
869
870 spin_lock_irqsave(&iommu->register_lock, flag);
871 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
872
873 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
874
875
876 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
877 readl, (sts & DMA_GSTS_RTPS), sts);
878
879 spin_unlock_irqrestore(&iommu->register_lock, flag);
880}
881
882static void iommu_flush_write_buffer(struct intel_iommu *iommu)
883{
884 u32 val;
885 unsigned long flag;
886
887 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
888 return;
889
890 spin_lock_irqsave(&iommu->register_lock, flag);
891 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
892
893
894 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
895 readl, (!(val & DMA_GSTS_WBFS)), val);
896
897 spin_unlock_irqrestore(&iommu->register_lock, flag);
898}
899
900
901static void __iommu_flush_context(struct intel_iommu *iommu,
902 u16 did, u16 source_id, u8 function_mask,
903 u64 type)
904{
905 u64 val = 0;
906 unsigned long flag;
907
908 switch (type) {
909 case DMA_CCMD_GLOBAL_INVL:
910 val = DMA_CCMD_GLOBAL_INVL;
911 break;
912 case DMA_CCMD_DOMAIN_INVL:
913 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
914 break;
915 case DMA_CCMD_DEVICE_INVL:
916 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
917 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
918 break;
919 default:
920 BUG();
921 }
922 val |= DMA_CCMD_ICC;
923
924 spin_lock_irqsave(&iommu->register_lock, flag);
925 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
926
927
928 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
929 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
930
931 spin_unlock_irqrestore(&iommu->register_lock, flag);
932}
933
934
935static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
936 u64 addr, unsigned int size_order, u64 type)
937{
938 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
939 u64 val = 0, val_iva = 0;
940 unsigned long flag;
941
942 switch (type) {
943 case DMA_TLB_GLOBAL_FLUSH:
944
945 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
946 break;
947 case DMA_TLB_DSI_FLUSH:
948 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
949 break;
950 case DMA_TLB_PSI_FLUSH:
951 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
952
953 val_iva = size_order | addr;
954 break;
955 default:
956 BUG();
957 }
958
959#if 0
960
961
962
963
964 if (cap_read_drain(iommu->cap))
965 val |= DMA_TLB_READ_DRAIN;
966#endif
967 if (cap_write_drain(iommu->cap))
968 val |= DMA_TLB_WRITE_DRAIN;
969
970 spin_lock_irqsave(&iommu->register_lock, flag);
971
972 if (val_iva)
973 dmar_writeq(iommu->reg + tlb_offset, val_iva);
974 dmar_writeq(iommu->reg + tlb_offset + 8, val);
975
976
977 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
978 dmar_readq, (!(val & DMA_TLB_IVT)), val);
979
980 spin_unlock_irqrestore(&iommu->register_lock, flag);
981
982
983 if (DMA_TLB_IAIG(val) == 0)
984 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
985 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
986 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
987 (unsigned long long)DMA_TLB_IIRG(type),
988 (unsigned long long)DMA_TLB_IAIG(val));
989}
990
991static struct device_domain_info *iommu_support_dev_iotlb(
992 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
993{
994 int found = 0;
995 unsigned long flags;
996 struct device_domain_info *info;
997 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
998
999 if (!ecap_dev_iotlb_support(iommu->ecap))
1000 return NULL;
1001
1002 if (!iommu->qi)
1003 return NULL;
1004
1005 spin_lock_irqsave(&device_domain_lock, flags);
1006 list_for_each_entry(info, &domain->devices, link)
1007 if (info->bus == bus && info->devfn == devfn) {
1008 found = 1;
1009 break;
1010 }
1011 spin_unlock_irqrestore(&device_domain_lock, flags);
1012
1013 if (!found || !info->dev)
1014 return NULL;
1015
1016 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1017 return NULL;
1018
1019 if (!dmar_find_matched_atsr_unit(info->dev))
1020 return NULL;
1021
1022 info->iommu = iommu;
1023
1024 return info;
1025}
1026
1027static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1028{
1029 if (!info)
1030 return;
1031
1032 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1033}
1034
1035static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1036{
1037 if (!info->dev || !pci_ats_enabled(info->dev))
1038 return;
1039
1040 pci_disable_ats(info->dev);
1041}
1042
1043static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1044 u64 addr, unsigned mask)
1045{
1046 u16 sid, qdep;
1047 unsigned long flags;
1048 struct device_domain_info *info;
1049
1050 spin_lock_irqsave(&device_domain_lock, flags);
1051 list_for_each_entry(info, &domain->devices, link) {
1052 if (!info->dev || !pci_ats_enabled(info->dev))
1053 continue;
1054
1055 sid = info->bus << 8 | info->devfn;
1056 qdep = pci_ats_queue_depth(info->dev);
1057 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1058 }
1059 spin_unlock_irqrestore(&device_domain_lock, flags);
1060}
1061
1062static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1063 unsigned long pfn, unsigned int pages, int map)
1064{
1065 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1066 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1067
1068 BUG_ON(pages == 0);
1069
1070
1071
1072
1073
1074
1075
1076 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1077 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1078 DMA_TLB_DSI_FLUSH);
1079 else
1080 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1081 DMA_TLB_PSI_FLUSH);
1082
1083
1084
1085
1086
1087 if (!cap_caching_mode(iommu->cap) || !map)
1088 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1089}
1090
1091static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1092{
1093 u32 pmen;
1094 unsigned long flags;
1095
1096 spin_lock_irqsave(&iommu->register_lock, flags);
1097 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1098 pmen &= ~DMA_PMEN_EPM;
1099 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1100
1101
1102 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1103 readl, !(pmen & DMA_PMEN_PRS), pmen);
1104
1105 spin_unlock_irqrestore(&iommu->register_lock, flags);
1106}
1107
1108static int iommu_enable_translation(struct intel_iommu *iommu)
1109{
1110 u32 sts;
1111 unsigned long flags;
1112
1113 spin_lock_irqsave(&iommu->register_lock, flags);
1114 iommu->gcmd |= DMA_GCMD_TE;
1115 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1116
1117
1118 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1119 readl, (sts & DMA_GSTS_TES), sts);
1120
1121 spin_unlock_irqrestore(&iommu->register_lock, flags);
1122 return 0;
1123}
1124
1125static int iommu_disable_translation(struct intel_iommu *iommu)
1126{
1127 u32 sts;
1128 unsigned long flag;
1129
1130 spin_lock_irqsave(&iommu->register_lock, flag);
1131 iommu->gcmd &= ~DMA_GCMD_TE;
1132 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1133
1134
1135 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1136 readl, (!(sts & DMA_GSTS_TES)), sts);
1137
1138 spin_unlock_irqrestore(&iommu->register_lock, flag);
1139 return 0;
1140}
1141
1142
1143static int iommu_init_domains(struct intel_iommu *iommu)
1144{
1145 unsigned long ndomains;
1146 unsigned long nlongs;
1147
1148 ndomains = cap_ndoms(iommu->cap);
1149 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1150 ndomains);
1151 nlongs = BITS_TO_LONGS(ndomains);
1152
1153 spin_lock_init(&iommu->lock);
1154
1155
1156
1157
1158 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1159 if (!iommu->domain_ids) {
1160 printk(KERN_ERR "Allocating domain id array failed\n");
1161 return -ENOMEM;
1162 }
1163 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1164 GFP_KERNEL);
1165 if (!iommu->domains) {
1166 printk(KERN_ERR "Allocating domain array failed\n");
1167 return -ENOMEM;
1168 }
1169
1170
1171
1172
1173
1174 if (cap_caching_mode(iommu->cap))
1175 set_bit(0, iommu->domain_ids);
1176 return 0;
1177}
1178
1179
1180static void domain_exit(struct dmar_domain *domain);
1181static void vm_domain_exit(struct dmar_domain *domain);
1182
1183void free_dmar_iommu(struct intel_iommu *iommu)
1184{
1185 struct dmar_domain *domain;
1186 int i;
1187 unsigned long flags;
1188
1189 if ((iommu->domains) && (iommu->domain_ids)) {
1190 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1191 domain = iommu->domains[i];
1192 clear_bit(i, iommu->domain_ids);
1193
1194 spin_lock_irqsave(&domain->iommu_lock, flags);
1195 if (--domain->iommu_count == 0) {
1196 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1197 vm_domain_exit(domain);
1198 else
1199 domain_exit(domain);
1200 }
1201 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1202 }
1203 }
1204
1205 if (iommu->gcmd & DMA_GCMD_TE)
1206 iommu_disable_translation(iommu);
1207
1208 if (iommu->irq) {
1209 irq_set_handler_data(iommu->irq, NULL);
1210
1211 free_irq(iommu->irq, iommu);
1212 destroy_irq(iommu->irq);
1213 }
1214
1215 kfree(iommu->domains);
1216 kfree(iommu->domain_ids);
1217
1218 g_iommus[iommu->seq_id] = NULL;
1219
1220
1221 for (i = 0; i < g_num_of_iommus; i++) {
1222 if (g_iommus[i])
1223 break;
1224 }
1225
1226 if (i == g_num_of_iommus)
1227 kfree(g_iommus);
1228
1229
1230 free_context_table(iommu);
1231}
1232
1233static struct dmar_domain *alloc_domain(void)
1234{
1235 struct dmar_domain *domain;
1236
1237 domain = alloc_domain_mem();
1238 if (!domain)
1239 return NULL;
1240
1241 domain->nid = -1;
1242 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1243 domain->flags = 0;
1244
1245 return domain;
1246}
1247
1248static int iommu_attach_domain(struct dmar_domain *domain,
1249 struct intel_iommu *iommu)
1250{
1251 int num;
1252 unsigned long ndomains;
1253 unsigned long flags;
1254
1255 ndomains = cap_ndoms(iommu->cap);
1256
1257 spin_lock_irqsave(&iommu->lock, flags);
1258
1259 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1260 if (num >= ndomains) {
1261 spin_unlock_irqrestore(&iommu->lock, flags);
1262 printk(KERN_ERR "IOMMU: no free domain ids\n");
1263 return -ENOMEM;
1264 }
1265
1266 domain->id = num;
1267 set_bit(num, iommu->domain_ids);
1268 set_bit(iommu->seq_id, &domain->iommu_bmp);
1269 iommu->domains[num] = domain;
1270 spin_unlock_irqrestore(&iommu->lock, flags);
1271
1272 return 0;
1273}
1274
1275static void iommu_detach_domain(struct dmar_domain *domain,
1276 struct intel_iommu *iommu)
1277{
1278 unsigned long flags;
1279 int num, ndomains;
1280 int found = 0;
1281
1282 spin_lock_irqsave(&iommu->lock, flags);
1283 ndomains = cap_ndoms(iommu->cap);
1284 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1285 if (iommu->domains[num] == domain) {
1286 found = 1;
1287 break;
1288 }
1289 }
1290
1291 if (found) {
1292 clear_bit(num, iommu->domain_ids);
1293 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1294 iommu->domains[num] = NULL;
1295 }
1296 spin_unlock_irqrestore(&iommu->lock, flags);
1297}
1298
1299static struct iova_domain reserved_iova_list;
1300static struct lock_class_key reserved_rbtree_key;
1301
1302static int dmar_init_reserved_ranges(void)
1303{
1304 struct pci_dev *pdev = NULL;
1305 struct iova *iova;
1306 int i;
1307
1308 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1309
1310 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1311 &reserved_rbtree_key);
1312
1313
1314 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1315 IOVA_PFN(IOAPIC_RANGE_END));
1316 if (!iova) {
1317 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1318 return -ENODEV;
1319 }
1320
1321
1322 for_each_pci_dev(pdev) {
1323 struct resource *r;
1324
1325 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1326 r = &pdev->resource[i];
1327 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1328 continue;
1329 iova = reserve_iova(&reserved_iova_list,
1330 IOVA_PFN(r->start),
1331 IOVA_PFN(r->end));
1332 if (!iova) {
1333 printk(KERN_ERR "Reserve iova failed\n");
1334 return -ENODEV;
1335 }
1336 }
1337 }
1338 return 0;
1339}
1340
1341static void domain_reserve_special_ranges(struct dmar_domain *domain)
1342{
1343 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1344}
1345
1346static inline int guestwidth_to_adjustwidth(int gaw)
1347{
1348 int agaw;
1349 int r = (gaw - 12) % 9;
1350
1351 if (r == 0)
1352 agaw = gaw;
1353 else
1354 agaw = gaw + 9 - r;
1355 if (agaw > 64)
1356 agaw = 64;
1357 return agaw;
1358}
1359
1360static int domain_init(struct dmar_domain *domain, int guest_width)
1361{
1362 struct intel_iommu *iommu;
1363 int adjust_width, agaw;
1364 unsigned long sagaw;
1365
1366 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1367 spin_lock_init(&domain->iommu_lock);
1368
1369 domain_reserve_special_ranges(domain);
1370
1371
1372 iommu = domain_get_iommu(domain);
1373 if (guest_width > cap_mgaw(iommu->cap))
1374 guest_width = cap_mgaw(iommu->cap);
1375 domain->gaw = guest_width;
1376 adjust_width = guestwidth_to_adjustwidth(guest_width);
1377 agaw = width_to_agaw(adjust_width);
1378 sagaw = cap_sagaw(iommu->cap);
1379 if (!test_bit(agaw, &sagaw)) {
1380
1381 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1382 agaw = find_next_bit(&sagaw, 5, agaw);
1383 if (agaw >= 5)
1384 return -ENODEV;
1385 }
1386 domain->agaw = agaw;
1387 INIT_LIST_HEAD(&domain->devices);
1388
1389 if (ecap_coherent(iommu->ecap))
1390 domain->iommu_coherency = 1;
1391 else
1392 domain->iommu_coherency = 0;
1393
1394 if (ecap_sc_support(iommu->ecap))
1395 domain->iommu_snooping = 1;
1396 else
1397 domain->iommu_snooping = 0;
1398
1399 domain->iommu_count = 1;
1400 domain->nid = iommu->node;
1401
1402
1403 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1404 if (!domain->pgd)
1405 return -ENOMEM;
1406 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1407 return 0;
1408}
1409
1410static void domain_exit(struct dmar_domain *domain)
1411{
1412 struct dmar_drhd_unit *drhd;
1413 struct intel_iommu *iommu;
1414
1415
1416 if (!domain)
1417 return;
1418
1419 domain_remove_dev_info(domain);
1420
1421 put_iova_domain(&domain->iovad);
1422
1423
1424 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1425
1426
1427 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1428
1429 for_each_active_iommu(iommu, drhd)
1430 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1431 iommu_detach_domain(domain, iommu);
1432
1433 free_domain_mem(domain);
1434}
1435
1436static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1437 u8 bus, u8 devfn, int translation)
1438{
1439 struct context_entry *context;
1440 unsigned long flags;
1441 struct intel_iommu *iommu;
1442 struct dma_pte *pgd;
1443 unsigned long num;
1444 unsigned long ndomains;
1445 int id;
1446 int agaw;
1447 struct device_domain_info *info = NULL;
1448
1449 pr_debug("Set context mapping for %02x:%02x.%d\n",
1450 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1451
1452 BUG_ON(!domain->pgd);
1453 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1454 translation != CONTEXT_TT_MULTI_LEVEL);
1455
1456 iommu = device_to_iommu(segment, bus, devfn);
1457 if (!iommu)
1458 return -ENODEV;
1459
1460 context = device_to_context_entry(iommu, bus, devfn);
1461 if (!context)
1462 return -ENOMEM;
1463 spin_lock_irqsave(&iommu->lock, flags);
1464 if (context_present(context)) {
1465 spin_unlock_irqrestore(&iommu->lock, flags);
1466 return 0;
1467 }
1468
1469 id = domain->id;
1470 pgd = domain->pgd;
1471
1472 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1473 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1474 int found = 0;
1475
1476
1477 ndomains = cap_ndoms(iommu->cap);
1478 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1479 if (iommu->domains[num] == domain) {
1480 id = num;
1481 found = 1;
1482 break;
1483 }
1484 }
1485
1486 if (found == 0) {
1487 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1488 if (num >= ndomains) {
1489 spin_unlock_irqrestore(&iommu->lock, flags);
1490 printk(KERN_ERR "IOMMU: no free domain ids\n");
1491 return -EFAULT;
1492 }
1493
1494 set_bit(num, iommu->domain_ids);
1495 iommu->domains[num] = domain;
1496 id = num;
1497 }
1498
1499
1500
1501
1502
1503 if (translation != CONTEXT_TT_PASS_THROUGH) {
1504 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1505 pgd = phys_to_virt(dma_pte_addr(pgd));
1506 if (!dma_pte_present(pgd)) {
1507 spin_unlock_irqrestore(&iommu->lock, flags);
1508 return -ENOMEM;
1509 }
1510 }
1511 }
1512 }
1513
1514 context_set_domain_id(context, id);
1515
1516 if (translation != CONTEXT_TT_PASS_THROUGH) {
1517 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1518 translation = info ? CONTEXT_TT_DEV_IOTLB :
1519 CONTEXT_TT_MULTI_LEVEL;
1520 }
1521
1522
1523
1524
1525 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1526 context_set_address_width(context, iommu->msagaw);
1527 else {
1528 context_set_address_root(context, virt_to_phys(pgd));
1529 context_set_address_width(context, iommu->agaw);
1530 }
1531
1532 context_set_translation_type(context, translation);
1533 context_set_fault_enable(context);
1534 context_set_present(context);
1535 domain_flush_cache(domain, context, sizeof(*context));
1536
1537
1538
1539
1540
1541
1542
1543 if (cap_caching_mode(iommu->cap)) {
1544 iommu->flush.flush_context(iommu, 0,
1545 (((u16)bus) << 8) | devfn,
1546 DMA_CCMD_MASK_NOBIT,
1547 DMA_CCMD_DEVICE_INVL);
1548 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1549 } else {
1550 iommu_flush_write_buffer(iommu);
1551 }
1552 iommu_enable_dev_iotlb(info);
1553 spin_unlock_irqrestore(&iommu->lock, flags);
1554
1555 spin_lock_irqsave(&domain->iommu_lock, flags);
1556 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1557 domain->iommu_count++;
1558 if (domain->iommu_count == 1)
1559 domain->nid = iommu->node;
1560 domain_update_iommu_cap(domain);
1561 }
1562 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1563 return 0;
1564}
1565
1566static int
1567domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1568 int translation)
1569{
1570 int ret;
1571 struct pci_dev *tmp, *parent;
1572
1573 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1574 pdev->bus->number, pdev->devfn,
1575 translation);
1576 if (ret)
1577 return ret;
1578
1579
1580 tmp = pci_find_upstream_pcie_bridge(pdev);
1581 if (!tmp)
1582 return 0;
1583
1584 parent = pdev->bus->self;
1585 while (parent != tmp) {
1586 ret = domain_context_mapping_one(domain,
1587 pci_domain_nr(parent->bus),
1588 parent->bus->number,
1589 parent->devfn, translation);
1590 if (ret)
1591 return ret;
1592 parent = parent->bus->self;
1593 }
1594 if (pci_is_pcie(tmp))
1595 return domain_context_mapping_one(domain,
1596 pci_domain_nr(tmp->subordinate),
1597 tmp->subordinate->number, 0,
1598 translation);
1599 else
1600 return domain_context_mapping_one(domain,
1601 pci_domain_nr(tmp->bus),
1602 tmp->bus->number,
1603 tmp->devfn,
1604 translation);
1605}
1606
1607static int domain_context_mapped(struct pci_dev *pdev)
1608{
1609 int ret;
1610 struct pci_dev *tmp, *parent;
1611 struct intel_iommu *iommu;
1612
1613 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1614 pdev->devfn);
1615 if (!iommu)
1616 return -ENODEV;
1617
1618 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1619 if (!ret)
1620 return ret;
1621
1622 tmp = pci_find_upstream_pcie_bridge(pdev);
1623 if (!tmp)
1624 return ret;
1625
1626 parent = pdev->bus->self;
1627 while (parent != tmp) {
1628 ret = device_context_mapped(iommu, parent->bus->number,
1629 parent->devfn);
1630 if (!ret)
1631 return ret;
1632 parent = parent->bus->self;
1633 }
1634 if (pci_is_pcie(tmp))
1635 return device_context_mapped(iommu, tmp->subordinate->number,
1636 0);
1637 else
1638 return device_context_mapped(iommu, tmp->bus->number,
1639 tmp->devfn);
1640}
1641
1642
1643static inline unsigned long aligned_nrpages(unsigned long host_addr,
1644 size_t size)
1645{
1646 host_addr &= ~PAGE_MASK;
1647 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1648}
1649
1650static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1651 struct scatterlist *sg, unsigned long phys_pfn,
1652 unsigned long nr_pages, int prot)
1653{
1654 struct dma_pte *first_pte = NULL, *pte = NULL;
1655 phys_addr_t uninitialized_var(pteval);
1656 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1657 unsigned long sg_res;
1658
1659 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1660
1661 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1662 return -EINVAL;
1663
1664 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1665
1666 if (sg)
1667 sg_res = 0;
1668 else {
1669 sg_res = nr_pages + 1;
1670 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1671 }
1672
1673 while (nr_pages--) {
1674 uint64_t tmp;
1675
1676 if (!sg_res) {
1677 sg_res = aligned_nrpages(sg->offset, sg->length);
1678 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1679 sg->dma_length = sg->length;
1680 pteval = page_to_phys(sg_page(sg)) | prot;
1681 }
1682 if (!pte) {
1683 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1684 if (!pte)
1685 return -ENOMEM;
1686 }
1687
1688
1689
1690 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1691 if (tmp) {
1692 static int dumps = 5;
1693 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1694 iov_pfn, tmp, (unsigned long long)pteval);
1695 if (dumps) {
1696 dumps--;
1697 debug_dma_dump_mappings(NULL);
1698 }
1699 WARN_ON(1);
1700 }
1701 pte++;
1702 if (!nr_pages || first_pte_in_page(pte)) {
1703 domain_flush_cache(domain, first_pte,
1704 (void *)pte - (void *)first_pte);
1705 pte = NULL;
1706 }
1707 iov_pfn++;
1708 pteval += VTD_PAGE_SIZE;
1709 sg_res--;
1710 if (!sg_res)
1711 sg = sg_next(sg);
1712 }
1713 return 0;
1714}
1715
1716static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1717 struct scatterlist *sg, unsigned long nr_pages,
1718 int prot)
1719{
1720 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1721}
1722
1723static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1724 unsigned long phys_pfn, unsigned long nr_pages,
1725 int prot)
1726{
1727 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1728}
1729
1730static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1731{
1732 if (!iommu)
1733 return;
1734
1735 clear_context_table(iommu, bus, devfn);
1736 iommu->flush.flush_context(iommu, 0, 0, 0,
1737 DMA_CCMD_GLOBAL_INVL);
1738 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1739}
1740
1741static void domain_remove_dev_info(struct dmar_domain *domain)
1742{
1743 struct device_domain_info *info;
1744 unsigned long flags;
1745 struct intel_iommu *iommu;
1746
1747 spin_lock_irqsave(&device_domain_lock, flags);
1748 while (!list_empty(&domain->devices)) {
1749 info = list_entry(domain->devices.next,
1750 struct device_domain_info, link);
1751 list_del(&info->link);
1752 list_del(&info->global);
1753 if (info->dev)
1754 info->dev->dev.archdata.iommu = NULL;
1755 spin_unlock_irqrestore(&device_domain_lock, flags);
1756
1757 iommu_disable_dev_iotlb(info);
1758 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1759 iommu_detach_dev(iommu, info->bus, info->devfn);
1760 free_devinfo_mem(info);
1761
1762 spin_lock_irqsave(&device_domain_lock, flags);
1763 }
1764 spin_unlock_irqrestore(&device_domain_lock, flags);
1765}
1766
1767
1768
1769
1770
1771static struct dmar_domain *
1772find_domain(struct pci_dev *pdev)
1773{
1774 struct device_domain_info *info;
1775
1776
1777 info = pdev->dev.archdata.iommu;
1778 if (info)
1779 return info->domain;
1780 return NULL;
1781}
1782
1783
1784static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1785{
1786 struct dmar_domain *domain, *found = NULL;
1787 struct intel_iommu *iommu;
1788 struct dmar_drhd_unit *drhd;
1789 struct device_domain_info *info, *tmp;
1790 struct pci_dev *dev_tmp;
1791 unsigned long flags;
1792 int bus = 0, devfn = 0;
1793 int segment;
1794 int ret;
1795
1796 domain = find_domain(pdev);
1797 if (domain)
1798 return domain;
1799
1800 segment = pci_domain_nr(pdev->bus);
1801
1802 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1803 if (dev_tmp) {
1804 if (pci_is_pcie(dev_tmp)) {
1805 bus = dev_tmp->subordinate->number;
1806 devfn = 0;
1807 } else {
1808 bus = dev_tmp->bus->number;
1809 devfn = dev_tmp->devfn;
1810 }
1811 spin_lock_irqsave(&device_domain_lock, flags);
1812 list_for_each_entry(info, &device_domain_list, global) {
1813 if (info->segment == segment &&
1814 info->bus == bus && info->devfn == devfn) {
1815 found = info->domain;
1816 break;
1817 }
1818 }
1819 spin_unlock_irqrestore(&device_domain_lock, flags);
1820
1821 if (found) {
1822 domain = found;
1823 goto found_domain;
1824 }
1825 }
1826
1827 domain = alloc_domain();
1828 if (!domain)
1829 goto error;
1830
1831
1832 drhd = dmar_find_matched_drhd_unit(pdev);
1833 if (!drhd) {
1834 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1835 pci_name(pdev));
1836 return NULL;
1837 }
1838 iommu = drhd->iommu;
1839
1840 ret = iommu_attach_domain(domain, iommu);
1841 if (ret) {
1842 free_domain_mem(domain);
1843 goto error;
1844 }
1845
1846 if (domain_init(domain, gaw)) {
1847 domain_exit(domain);
1848 goto error;
1849 }
1850
1851
1852 if (dev_tmp) {
1853 info = alloc_devinfo_mem();
1854 if (!info) {
1855 domain_exit(domain);
1856 goto error;
1857 }
1858 info->segment = segment;
1859 info->bus = bus;
1860 info->devfn = devfn;
1861 info->dev = NULL;
1862 info->domain = domain;
1863
1864 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1865
1866
1867 found = NULL;
1868 spin_lock_irqsave(&device_domain_lock, flags);
1869 list_for_each_entry(tmp, &device_domain_list, global) {
1870 if (tmp->segment == segment &&
1871 tmp->bus == bus && tmp->devfn == devfn) {
1872 found = tmp->domain;
1873 break;
1874 }
1875 }
1876 if (found) {
1877 spin_unlock_irqrestore(&device_domain_lock, flags);
1878 free_devinfo_mem(info);
1879 domain_exit(domain);
1880 domain = found;
1881 } else {
1882 list_add(&info->link, &domain->devices);
1883 list_add(&info->global, &device_domain_list);
1884 spin_unlock_irqrestore(&device_domain_lock, flags);
1885 }
1886 }
1887
1888found_domain:
1889 info = alloc_devinfo_mem();
1890 if (!info)
1891 goto error;
1892 info->segment = segment;
1893 info->bus = pdev->bus->number;
1894 info->devfn = pdev->devfn;
1895 info->dev = pdev;
1896 info->domain = domain;
1897 spin_lock_irqsave(&device_domain_lock, flags);
1898
1899 found = find_domain(pdev);
1900 if (found != NULL) {
1901 spin_unlock_irqrestore(&device_domain_lock, flags);
1902 if (found != domain) {
1903 domain_exit(domain);
1904 domain = found;
1905 }
1906 free_devinfo_mem(info);
1907 return domain;
1908 }
1909 list_add(&info->link, &domain->devices);
1910 list_add(&info->global, &device_domain_list);
1911 pdev->dev.archdata.iommu = info;
1912 spin_unlock_irqrestore(&device_domain_lock, flags);
1913 return domain;
1914error:
1915
1916 return find_domain(pdev);
1917}
1918
1919static int iommu_identity_mapping;
1920#define IDENTMAP_ALL 1
1921#define IDENTMAP_GFX 2
1922#define IDENTMAP_AZALIA 4
1923
1924static int iommu_domain_identity_map(struct dmar_domain *domain,
1925 unsigned long long start,
1926 unsigned long long end)
1927{
1928 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1929 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1930
1931 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1932 dma_to_mm_pfn(last_vpfn))) {
1933 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1934 return -ENOMEM;
1935 }
1936
1937 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1938 start, end, domain->id);
1939
1940
1941
1942
1943 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1944
1945 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1946 last_vpfn - first_vpfn + 1,
1947 DMA_PTE_READ|DMA_PTE_WRITE);
1948}
1949
1950static int iommu_prepare_identity_map(struct pci_dev *pdev,
1951 unsigned long long start,
1952 unsigned long long end)
1953{
1954 struct dmar_domain *domain;
1955 int ret;
1956
1957 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1958 if (!domain)
1959 return -ENOMEM;
1960
1961
1962
1963
1964
1965 if (domain == si_domain && hw_pass_through) {
1966 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1967 pci_name(pdev), start, end);
1968 return 0;
1969 }
1970
1971 printk(KERN_INFO
1972 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1973 pci_name(pdev), start, end);
1974
1975 if (end < start) {
1976 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1977 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1978 dmi_get_system_info(DMI_BIOS_VENDOR),
1979 dmi_get_system_info(DMI_BIOS_VERSION),
1980 dmi_get_system_info(DMI_PRODUCT_VERSION));
1981 ret = -EIO;
1982 goto error;
1983 }
1984
1985 if (end >> agaw_to_width(domain->agaw)) {
1986 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1987 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1988 agaw_to_width(domain->agaw),
1989 dmi_get_system_info(DMI_BIOS_VENDOR),
1990 dmi_get_system_info(DMI_BIOS_VERSION),
1991 dmi_get_system_info(DMI_PRODUCT_VERSION));
1992 ret = -EIO;
1993 goto error;
1994 }
1995
1996 ret = iommu_domain_identity_map(domain, start, end);
1997 if (ret)
1998 goto error;
1999
2000
2001 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2002 if (ret)
2003 goto error;
2004
2005 return 0;
2006
2007 error:
2008 domain_exit(domain);
2009 return ret;
2010}
2011
2012static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2013 struct pci_dev *pdev)
2014{
2015 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2016 return 0;
2017 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2018 rmrr->end_address + 1);
2019}
2020
2021#ifdef CONFIG_DMAR_FLOPPY_WA
2022static inline void iommu_prepare_isa(void)
2023{
2024 struct pci_dev *pdev;
2025 int ret;
2026
2027 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2028 if (!pdev)
2029 return;
2030
2031 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2032 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2033
2034 if (ret)
2035 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2036 "floppy might not work\n");
2037
2038}
2039#else
2040static inline void iommu_prepare_isa(void)
2041{
2042 return;
2043}
2044#endif
2045
2046static int md_domain_init(struct dmar_domain *domain, int guest_width);
2047
2048static int __init si_domain_work_fn(unsigned long start_pfn,
2049 unsigned long end_pfn, void *datax)
2050{
2051 int *ret = datax;
2052
2053 *ret = iommu_domain_identity_map(si_domain,
2054 (uint64_t)start_pfn << PAGE_SHIFT,
2055 (uint64_t)end_pfn << PAGE_SHIFT);
2056 return *ret;
2057
2058}
2059
2060static int __init si_domain_init(int hw)
2061{
2062 struct dmar_drhd_unit *drhd;
2063 struct intel_iommu *iommu;
2064 int nid, ret = 0;
2065
2066 si_domain = alloc_domain();
2067 if (!si_domain)
2068 return -EFAULT;
2069
2070 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2071
2072 for_each_active_iommu(iommu, drhd) {
2073 ret = iommu_attach_domain(si_domain, iommu);
2074 if (ret) {
2075 domain_exit(si_domain);
2076 return -EFAULT;
2077 }
2078 }
2079
2080 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2081 domain_exit(si_domain);
2082 return -EFAULT;
2083 }
2084
2085 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2086
2087 if (hw)
2088 return 0;
2089
2090 for_each_online_node(nid) {
2091 work_with_active_regions(nid, si_domain_work_fn, &ret);
2092 if (ret)
2093 return ret;
2094 }
2095
2096 return 0;
2097}
2098
2099static void domain_remove_one_dev_info(struct dmar_domain *domain,
2100 struct pci_dev *pdev);
2101static int identity_mapping(struct pci_dev *pdev)
2102{
2103 struct device_domain_info *info;
2104
2105 if (likely(!iommu_identity_mapping))
2106 return 0;
2107
2108
2109 list_for_each_entry(info, &si_domain->devices, link)
2110 if (info->dev == pdev)
2111 return 1;
2112 return 0;
2113}
2114
2115static int domain_add_dev_info(struct dmar_domain *domain,
2116 struct pci_dev *pdev,
2117 int translation)
2118{
2119 struct device_domain_info *info;
2120 unsigned long flags;
2121 int ret;
2122
2123 info = alloc_devinfo_mem();
2124 if (!info)
2125 return -ENOMEM;
2126
2127 ret = domain_context_mapping(domain, pdev, translation);
2128 if (ret) {
2129 free_devinfo_mem(info);
2130 return ret;
2131 }
2132
2133 info->segment = pci_domain_nr(pdev->bus);
2134 info->bus = pdev->bus->number;
2135 info->devfn = pdev->devfn;
2136 info->dev = pdev;
2137 info->domain = domain;
2138
2139 spin_lock_irqsave(&device_domain_lock, flags);
2140 list_add(&info->link, &domain->devices);
2141 list_add(&info->global, &device_domain_list);
2142 pdev->dev.archdata.iommu = info;
2143 spin_unlock_irqrestore(&device_domain_lock, flags);
2144
2145 return 0;
2146}
2147
2148static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2149{
2150 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2151 return 1;
2152
2153 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2154 return 1;
2155
2156 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2157 return 0;
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176 if (!pci_is_pcie(pdev)) {
2177 if (!pci_is_root_bus(pdev->bus))
2178 return 0;
2179 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2180 return 0;
2181 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2182 return 0;
2183
2184
2185
2186
2187
2188
2189 if (!startup)
2190 return pdev->dma_mask > DMA_BIT_MASK(32);
2191
2192 return 1;
2193}
2194
2195static int __init iommu_prepare_static_identity_mapping(int hw)
2196{
2197 struct pci_dev *pdev = NULL;
2198 int ret;
2199
2200 ret = si_domain_init(hw);
2201 if (ret)
2202 return -EFAULT;
2203
2204 for_each_pci_dev(pdev) {
2205 if (iommu_should_identity_map(pdev, 1)) {
2206 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2207 hw ? "hardware" : "software", pci_name(pdev));
2208
2209 ret = domain_add_dev_info(si_domain, pdev,
2210 hw ? CONTEXT_TT_PASS_THROUGH :
2211 CONTEXT_TT_MULTI_LEVEL);
2212 if (ret)
2213 return ret;
2214 }
2215 }
2216
2217 return 0;
2218}
2219
2220static int __init init_dmars(int force_on)
2221{
2222 struct dmar_drhd_unit *drhd;
2223 struct dmar_rmrr_unit *rmrr;
2224 struct pci_dev *pdev;
2225 struct intel_iommu *iommu;
2226 int i, ret;
2227
2228
2229
2230
2231
2232
2233
2234 for_each_drhd_unit(drhd) {
2235 g_num_of_iommus++;
2236
2237
2238
2239
2240
2241 }
2242
2243 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2244 GFP_KERNEL);
2245 if (!g_iommus) {
2246 printk(KERN_ERR "Allocating global iommu array failed\n");
2247 ret = -ENOMEM;
2248 goto error;
2249 }
2250
2251 deferred_flush = kzalloc(g_num_of_iommus *
2252 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2253 if (!deferred_flush) {
2254 ret = -ENOMEM;
2255 goto error;
2256 }
2257
2258 for_each_drhd_unit(drhd) {
2259 if (drhd->ignored)
2260 continue;
2261
2262 iommu = drhd->iommu;
2263 g_iommus[iommu->seq_id] = iommu;
2264
2265 ret = iommu_init_domains(iommu);
2266 if (ret)
2267 goto error;
2268
2269
2270
2271
2272
2273
2274 ret = iommu_alloc_root_entry(iommu);
2275 if (ret) {
2276 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2277 goto error;
2278 }
2279 if (!ecap_pass_through(iommu->ecap))
2280 hw_pass_through = 0;
2281 }
2282
2283
2284
2285
2286 for_each_drhd_unit(drhd) {
2287 if (drhd->ignored)
2288 continue;
2289
2290 iommu = drhd->iommu;
2291
2292
2293
2294
2295
2296
2297 if (iommu->qi)
2298 continue;
2299
2300
2301
2302
2303 dmar_fault(-1, iommu);
2304
2305
2306
2307
2308 dmar_disable_qi(iommu);
2309 }
2310
2311 for_each_drhd_unit(drhd) {
2312 if (drhd->ignored)
2313 continue;
2314
2315 iommu = drhd->iommu;
2316
2317 if (dmar_enable_qi(iommu)) {
2318
2319
2320
2321
2322 iommu->flush.flush_context = __iommu_flush_context;
2323 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2324 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2325 "invalidation\n",
2326 iommu->seq_id,
2327 (unsigned long long)drhd->reg_base_addr);
2328 } else {
2329 iommu->flush.flush_context = qi_flush_context;
2330 iommu->flush.flush_iotlb = qi_flush_iotlb;
2331 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2332 "invalidation\n",
2333 iommu->seq_id,
2334 (unsigned long long)drhd->reg_base_addr);
2335 }
2336 }
2337
2338 if (iommu_pass_through)
2339 iommu_identity_mapping |= IDENTMAP_ALL;
2340
2341#ifdef CONFIG_DMAR_BROKEN_GFX_WA
2342 iommu_identity_mapping |= IDENTMAP_GFX;
2343#endif
2344
2345 check_tylersburg_isoch();
2346
2347
2348
2349
2350
2351
2352 if (iommu_identity_mapping) {
2353 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2354 if (ret) {
2355 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2356 goto error;
2357 }
2358 }
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2374 for_each_rmrr_units(rmrr) {
2375 for (i = 0; i < rmrr->devices_cnt; i++) {
2376 pdev = rmrr->devices[i];
2377
2378
2379
2380
2381 if (!pdev)
2382 continue;
2383 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2384 if (ret)
2385 printk(KERN_ERR
2386 "IOMMU: mapping reserved region failed\n");
2387 }
2388 }
2389
2390 iommu_prepare_isa();
2391
2392
2393
2394
2395
2396
2397
2398
2399 for_each_drhd_unit(drhd) {
2400 if (drhd->ignored) {
2401
2402
2403
2404
2405 if (force_on)
2406 iommu_disable_protect_mem_regions(drhd->iommu);
2407 continue;
2408 }
2409 iommu = drhd->iommu;
2410
2411 iommu_flush_write_buffer(iommu);
2412
2413 ret = dmar_set_interrupt(iommu);
2414 if (ret)
2415 goto error;
2416
2417 iommu_set_root_entry(iommu);
2418
2419 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2420 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2421
2422 ret = iommu_enable_translation(iommu);
2423 if (ret)
2424 goto error;
2425
2426 iommu_disable_protect_mem_regions(iommu);
2427 }
2428
2429 return 0;
2430error:
2431 for_each_drhd_unit(drhd) {
2432 if (drhd->ignored)
2433 continue;
2434 iommu = drhd->iommu;
2435 free_iommu(iommu);
2436 }
2437 kfree(g_iommus);
2438 return ret;
2439}
2440
2441
2442static struct iova *intel_alloc_iova(struct device *dev,
2443 struct dmar_domain *domain,
2444 unsigned long nrpages, uint64_t dma_mask)
2445{
2446 struct pci_dev *pdev = to_pci_dev(dev);
2447 struct iova *iova = NULL;
2448
2449
2450 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2451
2452 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2453
2454
2455
2456
2457
2458 iova = alloc_iova(&domain->iovad, nrpages,
2459 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2460 if (iova)
2461 return iova;
2462 }
2463 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2464 if (unlikely(!iova)) {
2465 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2466 nrpages, pci_name(pdev));
2467 return NULL;
2468 }
2469
2470 return iova;
2471}
2472
2473static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2474{
2475 struct dmar_domain *domain;
2476 int ret;
2477
2478 domain = get_domain_for_dev(pdev,
2479 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2480 if (!domain) {
2481 printk(KERN_ERR
2482 "Allocating domain for %s failed", pci_name(pdev));
2483 return NULL;
2484 }
2485
2486
2487 if (unlikely(!domain_context_mapped(pdev))) {
2488 ret = domain_context_mapping(domain, pdev,
2489 CONTEXT_TT_MULTI_LEVEL);
2490 if (ret) {
2491 printk(KERN_ERR
2492 "Domain context map for %s failed",
2493 pci_name(pdev));
2494 return NULL;
2495 }
2496 }
2497
2498 return domain;
2499}
2500
2501static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2502{
2503 struct device_domain_info *info;
2504
2505
2506 info = dev->dev.archdata.iommu;
2507 if (likely(info))
2508 return info->domain;
2509
2510 return __get_valid_domain_for_dev(dev);
2511}
2512
2513static int iommu_dummy(struct pci_dev *pdev)
2514{
2515 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2516}
2517
2518
2519static int iommu_no_mapping(struct device *dev)
2520{
2521 struct pci_dev *pdev;
2522 int found;
2523
2524 if (unlikely(dev->bus != &pci_bus_type))
2525 return 1;
2526
2527 pdev = to_pci_dev(dev);
2528 if (iommu_dummy(pdev))
2529 return 1;
2530
2531 if (!iommu_identity_mapping)
2532 return 0;
2533
2534 found = identity_mapping(pdev);
2535 if (found) {
2536 if (iommu_should_identity_map(pdev, 0))
2537 return 1;
2538 else {
2539
2540
2541
2542
2543 domain_remove_one_dev_info(si_domain, pdev);
2544 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2545 pci_name(pdev));
2546 return 0;
2547 }
2548 } else {
2549
2550
2551
2552
2553 if (iommu_should_identity_map(pdev, 0)) {
2554 int ret;
2555 ret = domain_add_dev_info(si_domain, pdev,
2556 hw_pass_through ?
2557 CONTEXT_TT_PASS_THROUGH :
2558 CONTEXT_TT_MULTI_LEVEL);
2559 if (!ret) {
2560 printk(KERN_INFO "64bit %s uses identity mapping\n",
2561 pci_name(pdev));
2562 return 1;
2563 }
2564 }
2565 }
2566
2567 return 0;
2568}
2569
2570static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2571 size_t size, int dir, u64 dma_mask)
2572{
2573 struct pci_dev *pdev = to_pci_dev(hwdev);
2574 struct dmar_domain *domain;
2575 phys_addr_t start_paddr;
2576 struct iova *iova;
2577 int prot = 0;
2578 int ret;
2579 struct intel_iommu *iommu;
2580 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2581
2582 BUG_ON(dir == DMA_NONE);
2583
2584 if (iommu_no_mapping(hwdev))
2585 return paddr;
2586
2587 domain = get_valid_domain_for_dev(pdev);
2588 if (!domain)
2589 return 0;
2590
2591 iommu = domain_get_iommu(domain);
2592 size = aligned_nrpages(paddr, size);
2593
2594 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2595 pdev->dma_mask);
2596 if (!iova)
2597 goto error;
2598
2599
2600
2601
2602
2603 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2604 !cap_zlr(iommu->cap))
2605 prot |= DMA_PTE_READ;
2606 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2607 prot |= DMA_PTE_WRITE;
2608
2609
2610
2611
2612
2613
2614 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2615 mm_to_dma_pfn(paddr_pfn), size, prot);
2616 if (ret)
2617 goto error;
2618
2619
2620 if (cap_caching_mode(iommu->cap))
2621 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2622 else
2623 iommu_flush_write_buffer(iommu);
2624
2625 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2626 start_paddr += paddr & ~PAGE_MASK;
2627 return start_paddr;
2628
2629error:
2630 if (iova)
2631 __free_iova(&domain->iovad, iova);
2632 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2633 pci_name(pdev), size, (unsigned long long)paddr, dir);
2634 return 0;
2635}
2636
2637static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2638 unsigned long offset, size_t size,
2639 enum dma_data_direction dir,
2640 struct dma_attrs *attrs)
2641{
2642 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2643 dir, to_pci_dev(dev)->dma_mask);
2644}
2645
2646static void flush_unmaps(void)
2647{
2648 int i, j;
2649
2650 timer_on = 0;
2651
2652
2653 for (i = 0; i < g_num_of_iommus; i++) {
2654 struct intel_iommu *iommu = g_iommus[i];
2655 if (!iommu)
2656 continue;
2657
2658 if (!deferred_flush[i].next)
2659 continue;
2660
2661
2662 if (!cap_caching_mode(iommu->cap))
2663 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2664 DMA_TLB_GLOBAL_FLUSH);
2665 for (j = 0; j < deferred_flush[i].next; j++) {
2666 unsigned long mask;
2667 struct iova *iova = deferred_flush[i].iova[j];
2668 struct dmar_domain *domain = deferred_flush[i].domain[j];
2669
2670
2671 if (cap_caching_mode(iommu->cap))
2672 iommu_flush_iotlb_psi(iommu, domain->id,
2673 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2674 else {
2675 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2676 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2677 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2678 }
2679 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2680 }
2681 deferred_flush[i].next = 0;
2682 }
2683
2684 list_size = 0;
2685}
2686
2687static void flush_unmaps_timeout(unsigned long data)
2688{
2689 unsigned long flags;
2690
2691 spin_lock_irqsave(&async_umap_flush_lock, flags);
2692 flush_unmaps();
2693 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2694}
2695
2696static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2697{
2698 unsigned long flags;
2699 int next, iommu_id;
2700 struct intel_iommu *iommu;
2701
2702 spin_lock_irqsave(&async_umap_flush_lock, flags);
2703 if (list_size == HIGH_WATER_MARK)
2704 flush_unmaps();
2705
2706 iommu = domain_get_iommu(dom);
2707 iommu_id = iommu->seq_id;
2708
2709 next = deferred_flush[iommu_id].next;
2710 deferred_flush[iommu_id].domain[next] = dom;
2711 deferred_flush[iommu_id].iova[next] = iova;
2712 deferred_flush[iommu_id].next++;
2713
2714 if (!timer_on) {
2715 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2716 timer_on = 1;
2717 }
2718 list_size++;
2719 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2720}
2721
2722static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2723 size_t size, enum dma_data_direction dir,
2724 struct dma_attrs *attrs)
2725{
2726 struct pci_dev *pdev = to_pci_dev(dev);
2727 struct dmar_domain *domain;
2728 unsigned long start_pfn, last_pfn;
2729 struct iova *iova;
2730 struct intel_iommu *iommu;
2731
2732 if (iommu_no_mapping(dev))
2733 return;
2734
2735 domain = find_domain(pdev);
2736 BUG_ON(!domain);
2737
2738 iommu = domain_get_iommu(domain);
2739
2740 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2741 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2742 (unsigned long long)dev_addr))
2743 return;
2744
2745 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2746 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2747
2748 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2749 pci_name(pdev), start_pfn, last_pfn);
2750
2751
2752 dma_pte_clear_range(domain, start_pfn, last_pfn);
2753
2754
2755 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2756
2757 if (intel_iommu_strict) {
2758 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2759 last_pfn - start_pfn + 1, 0);
2760
2761 __free_iova(&domain->iovad, iova);
2762 } else {
2763 add_unmap(domain, iova);
2764
2765
2766
2767
2768 }
2769}
2770
2771static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2772 dma_addr_t *dma_handle, gfp_t flags)
2773{
2774 void *vaddr;
2775 int order;
2776
2777 size = PAGE_ALIGN(size);
2778 order = get_order(size);
2779
2780 if (!iommu_no_mapping(hwdev))
2781 flags &= ~(GFP_DMA | GFP_DMA32);
2782 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2783 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2784 flags |= GFP_DMA;
2785 else
2786 flags |= GFP_DMA32;
2787 }
2788
2789 vaddr = (void *)__get_free_pages(flags, order);
2790 if (!vaddr)
2791 return NULL;
2792 memset(vaddr, 0, size);
2793
2794 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2795 DMA_BIDIRECTIONAL,
2796 hwdev->coherent_dma_mask);
2797 if (*dma_handle)
2798 return vaddr;
2799 free_pages((unsigned long)vaddr, order);
2800 return NULL;
2801}
2802
2803static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2804 dma_addr_t dma_handle)
2805{
2806 int order;
2807
2808 size = PAGE_ALIGN(size);
2809 order = get_order(size);
2810
2811 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2812 free_pages((unsigned long)vaddr, order);
2813}
2814
2815static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2816 int nelems, enum dma_data_direction dir,
2817 struct dma_attrs *attrs)
2818{
2819 struct pci_dev *pdev = to_pci_dev(hwdev);
2820 struct dmar_domain *domain;
2821 unsigned long start_pfn, last_pfn;
2822 struct iova *iova;
2823 struct intel_iommu *iommu;
2824
2825 if (iommu_no_mapping(hwdev))
2826 return;
2827
2828 domain = find_domain(pdev);
2829 BUG_ON(!domain);
2830
2831 iommu = domain_get_iommu(domain);
2832
2833 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2834 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2835 (unsigned long long)sglist[0].dma_address))
2836 return;
2837
2838 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2839 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2840
2841
2842 dma_pte_clear_range(domain, start_pfn, last_pfn);
2843
2844
2845 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2846
2847 if (intel_iommu_strict) {
2848 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2849 last_pfn - start_pfn + 1, 0);
2850
2851 __free_iova(&domain->iovad, iova);
2852 } else {
2853 add_unmap(domain, iova);
2854
2855
2856
2857
2858 }
2859}
2860
2861static int intel_nontranslate_map_sg(struct device *hddev,
2862 struct scatterlist *sglist, int nelems, int dir)
2863{
2864 int i;
2865 struct scatterlist *sg;
2866
2867 for_each_sg(sglist, sg, nelems, i) {
2868 BUG_ON(!sg_page(sg));
2869 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2870 sg->dma_length = sg->length;
2871 }
2872 return nelems;
2873}
2874
2875static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2876 enum dma_data_direction dir, struct dma_attrs *attrs)
2877{
2878 int i;
2879 struct pci_dev *pdev = to_pci_dev(hwdev);
2880 struct dmar_domain *domain;
2881 size_t size = 0;
2882 int prot = 0;
2883 struct iova *iova = NULL;
2884 int ret;
2885 struct scatterlist *sg;
2886 unsigned long start_vpfn;
2887 struct intel_iommu *iommu;
2888
2889 BUG_ON(dir == DMA_NONE);
2890 if (iommu_no_mapping(hwdev))
2891 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2892
2893 domain = get_valid_domain_for_dev(pdev);
2894 if (!domain)
2895 return 0;
2896
2897 iommu = domain_get_iommu(domain);
2898
2899 for_each_sg(sglist, sg, nelems, i)
2900 size += aligned_nrpages(sg->offset, sg->length);
2901
2902 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2903 pdev->dma_mask);
2904 if (!iova) {
2905 sglist->dma_length = 0;
2906 return 0;
2907 }
2908
2909
2910
2911
2912
2913 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2914 !cap_zlr(iommu->cap))
2915 prot |= DMA_PTE_READ;
2916 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2917 prot |= DMA_PTE_WRITE;
2918
2919 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2920
2921 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2922 if (unlikely(ret)) {
2923
2924 dma_pte_clear_range(domain, start_vpfn,
2925 start_vpfn + size - 1);
2926
2927 dma_pte_free_pagetable(domain, start_vpfn,
2928 start_vpfn + size - 1);
2929
2930 __free_iova(&domain->iovad, iova);
2931 return 0;
2932 }
2933
2934
2935 if (cap_caching_mode(iommu->cap))
2936 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
2937 else
2938 iommu_flush_write_buffer(iommu);
2939
2940 return nelems;
2941}
2942
2943static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2944{
2945 return !dma_addr;
2946}
2947
2948struct dma_map_ops intel_dma_ops = {
2949 .alloc_coherent = intel_alloc_coherent,
2950 .free_coherent = intel_free_coherent,
2951 .map_sg = intel_map_sg,
2952 .unmap_sg = intel_unmap_sg,
2953 .map_page = intel_map_page,
2954 .unmap_page = intel_unmap_page,
2955 .mapping_error = intel_mapping_error,
2956};
2957
2958static inline int iommu_domain_cache_init(void)
2959{
2960 int ret = 0;
2961
2962 iommu_domain_cache = kmem_cache_create("iommu_domain",
2963 sizeof(struct dmar_domain),
2964 0,
2965 SLAB_HWCACHE_ALIGN,
2966
2967 NULL);
2968 if (!iommu_domain_cache) {
2969 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2970 ret = -ENOMEM;
2971 }
2972
2973 return ret;
2974}
2975
2976static inline int iommu_devinfo_cache_init(void)
2977{
2978 int ret = 0;
2979
2980 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2981 sizeof(struct device_domain_info),
2982 0,
2983 SLAB_HWCACHE_ALIGN,
2984 NULL);
2985 if (!iommu_devinfo_cache) {
2986 printk(KERN_ERR "Couldn't create devinfo cache\n");
2987 ret = -ENOMEM;
2988 }
2989
2990 return ret;
2991}
2992
2993static inline int iommu_iova_cache_init(void)
2994{
2995 int ret = 0;
2996
2997 iommu_iova_cache = kmem_cache_create("iommu_iova",
2998 sizeof(struct iova),
2999 0,
3000 SLAB_HWCACHE_ALIGN,
3001 NULL);
3002 if (!iommu_iova_cache) {
3003 printk(KERN_ERR "Couldn't create iova cache\n");
3004 ret = -ENOMEM;
3005 }
3006
3007 return ret;
3008}
3009
3010static int __init iommu_init_mempool(void)
3011{
3012 int ret;
3013 ret = iommu_iova_cache_init();
3014 if (ret)
3015 return ret;
3016
3017 ret = iommu_domain_cache_init();
3018 if (ret)
3019 goto domain_error;
3020
3021 ret = iommu_devinfo_cache_init();
3022 if (!ret)
3023 return ret;
3024
3025 kmem_cache_destroy(iommu_domain_cache);
3026domain_error:
3027 kmem_cache_destroy(iommu_iova_cache);
3028
3029 return -ENOMEM;
3030}
3031
3032static void __init iommu_exit_mempool(void)
3033{
3034 kmem_cache_destroy(iommu_devinfo_cache);
3035 kmem_cache_destroy(iommu_domain_cache);
3036 kmem_cache_destroy(iommu_iova_cache);
3037
3038}
3039
3040static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3041{
3042 struct dmar_drhd_unit *drhd;
3043 u32 vtbar;
3044 int rc;
3045
3046
3047
3048
3049
3050
3051 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3052 if (rc) {
3053
3054 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3055 return;
3056 }
3057 vtbar &= 0xffff0000;
3058
3059
3060 drhd = dmar_find_matched_drhd_unit(pdev);
3061 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3062 TAINT_FIRMWARE_WORKAROUND,
3063 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3064 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3065}
3066DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3067
3068static void __init init_no_remapping_devices(void)
3069{
3070 struct dmar_drhd_unit *drhd;
3071
3072 for_each_drhd_unit(drhd) {
3073 if (!drhd->include_all) {
3074 int i;
3075 for (i = 0; i < drhd->devices_cnt; i++)
3076 if (drhd->devices[i] != NULL)
3077 break;
3078
3079 if (i == drhd->devices_cnt)
3080 drhd->ignored = 1;
3081 }
3082 }
3083
3084 if (dmar_map_gfx)
3085 return;
3086
3087 for_each_drhd_unit(drhd) {
3088 int i;
3089 if (drhd->ignored || drhd->include_all)
3090 continue;
3091
3092 for (i = 0; i < drhd->devices_cnt; i++)
3093 if (drhd->devices[i] &&
3094 !IS_GFX_DEVICE(drhd->devices[i]))
3095 break;
3096
3097 if (i < drhd->devices_cnt)
3098 continue;
3099
3100
3101 drhd->ignored = 1;
3102 for (i = 0; i < drhd->devices_cnt; i++) {
3103 if (!drhd->devices[i])
3104 continue;
3105 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3106 }
3107 }
3108}
3109
3110#ifdef CONFIG_SUSPEND
3111static int init_iommu_hw(void)
3112{
3113 struct dmar_drhd_unit *drhd;
3114 struct intel_iommu *iommu = NULL;
3115
3116 for_each_active_iommu(iommu, drhd)
3117 if (iommu->qi)
3118 dmar_reenable_qi(iommu);
3119
3120 for_each_active_iommu(iommu, drhd) {
3121 iommu_flush_write_buffer(iommu);
3122
3123 iommu_set_root_entry(iommu);
3124
3125 iommu->flush.flush_context(iommu, 0, 0, 0,
3126 DMA_CCMD_GLOBAL_INVL);
3127 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3128 DMA_TLB_GLOBAL_FLUSH);
3129 iommu_enable_translation(iommu);
3130 iommu_disable_protect_mem_regions(iommu);
3131 }
3132
3133 return 0;
3134}
3135
3136static void iommu_flush_all(void)
3137{
3138 struct dmar_drhd_unit *drhd;
3139 struct intel_iommu *iommu;
3140
3141 for_each_active_iommu(iommu, drhd) {
3142 iommu->flush.flush_context(iommu, 0, 0, 0,
3143 DMA_CCMD_GLOBAL_INVL);
3144 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3145 DMA_TLB_GLOBAL_FLUSH);
3146 }
3147}
3148
3149static int iommu_suspend(void)
3150{
3151 struct dmar_drhd_unit *drhd;
3152 struct intel_iommu *iommu = NULL;
3153 unsigned long flag;
3154
3155 for_each_active_iommu(iommu, drhd) {
3156 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3157 GFP_ATOMIC);
3158 if (!iommu->iommu_state)
3159 goto nomem;
3160 }
3161
3162 iommu_flush_all();
3163
3164 for_each_active_iommu(iommu, drhd) {
3165 iommu_disable_translation(iommu);
3166
3167 spin_lock_irqsave(&iommu->register_lock, flag);
3168
3169 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3170 readl(iommu->reg + DMAR_FECTL_REG);
3171 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3172 readl(iommu->reg + DMAR_FEDATA_REG);
3173 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3174 readl(iommu->reg + DMAR_FEADDR_REG);
3175 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3176 readl(iommu->reg + DMAR_FEUADDR_REG);
3177
3178 spin_unlock_irqrestore(&iommu->register_lock, flag);
3179 }
3180 return 0;
3181
3182nomem:
3183 for_each_active_iommu(iommu, drhd)
3184 kfree(iommu->iommu_state);
3185
3186 return -ENOMEM;
3187}
3188
3189static void iommu_resume(void)
3190{
3191 struct dmar_drhd_unit *drhd;
3192 struct intel_iommu *iommu = NULL;
3193 unsigned long flag;
3194
3195 if (init_iommu_hw()) {
3196 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3197 return;
3198 }
3199
3200 for_each_active_iommu(iommu, drhd) {
3201
3202 spin_lock_irqsave(&iommu->register_lock, flag);
3203
3204 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3205 iommu->reg + DMAR_FECTL_REG);
3206 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3207 iommu->reg + DMAR_FEDATA_REG);
3208 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3209 iommu->reg + DMAR_FEADDR_REG);
3210 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3211 iommu->reg + DMAR_FEUADDR_REG);
3212
3213 spin_unlock_irqrestore(&iommu->register_lock, flag);
3214 }
3215
3216 for_each_active_iommu(iommu, drhd)
3217 kfree(iommu->iommu_state);
3218}
3219
3220static struct syscore_ops iommu_syscore_ops = {
3221 .resume = iommu_resume,
3222 .suspend = iommu_suspend,
3223};
3224
3225static void __init init_iommu_pm_ops(void)
3226{
3227 register_syscore_ops(&iommu_syscore_ops);
3228}
3229
3230#else
3231static inline int init_iommu_pm_ops(void) { }
3232#endif
3233
3234
3235
3236
3237
3238
3239
3240static int device_notifier(struct notifier_block *nb,
3241 unsigned long action, void *data)
3242{
3243 struct device *dev = data;
3244 struct pci_dev *pdev = to_pci_dev(dev);
3245 struct dmar_domain *domain;
3246
3247 if (iommu_no_mapping(dev))
3248 return 0;
3249
3250 domain = find_domain(pdev);
3251 if (!domain)
3252 return 0;
3253
3254 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3255 domain_remove_one_dev_info(domain, pdev);
3256
3257 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3258 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3259 list_empty(&domain->devices))
3260 domain_exit(domain);
3261 }
3262
3263 return 0;
3264}
3265
3266static struct notifier_block device_nb = {
3267 .notifier_call = device_notifier,
3268};
3269
3270int __init intel_iommu_init(void)
3271{
3272 int ret = 0;
3273 int force_on = 0;
3274
3275
3276 force_on = tboot_force_iommu();
3277
3278 if (dmar_table_init()) {
3279 if (force_on)
3280 panic("tboot: Failed to initialize DMAR table\n");
3281 return -ENODEV;
3282 }
3283
3284 if (dmar_dev_scope_init()) {
3285 if (force_on)
3286 panic("tboot: Failed to initialize DMAR device scope\n");
3287 return -ENODEV;
3288 }
3289
3290
3291
3292
3293
3294 if (no_iommu || dmar_disabled)
3295 return -ENODEV;
3296
3297 if (iommu_init_mempool()) {
3298 if (force_on)
3299 panic("tboot: Failed to initialize iommu memory\n");
3300 return -ENODEV;
3301 }
3302
3303 if (dmar_init_reserved_ranges()) {
3304 if (force_on)
3305 panic("tboot: Failed to reserve iommu ranges\n");
3306 return -ENODEV;
3307 }
3308
3309 init_no_remapping_devices();
3310
3311 ret = init_dmars(force_on);
3312 if (ret) {
3313 if (force_on)
3314 panic("tboot: Failed to initialize DMARs\n");
3315 printk(KERN_ERR "IOMMU: dmar init failed\n");
3316 put_iova_domain(&reserved_iova_list);
3317 iommu_exit_mempool();
3318 return ret;
3319 }
3320 printk(KERN_INFO
3321 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3322
3323 init_timer(&unmap_timer);
3324#ifdef CONFIG_SWIOTLB
3325 swiotlb = 0;
3326#endif
3327 dma_ops = &intel_dma_ops;
3328
3329 init_iommu_pm_ops();
3330
3331 register_iommu(&intel_iommu_ops);
3332
3333 bus_register_notifier(&pci_bus_type, &device_nb);
3334
3335 return 0;
3336}
3337
3338static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3339 struct pci_dev *pdev)
3340{
3341 struct pci_dev *tmp, *parent;
3342
3343 if (!iommu || !pdev)
3344 return;
3345
3346
3347 tmp = pci_find_upstream_pcie_bridge(pdev);
3348
3349 if (tmp) {
3350 parent = pdev->bus->self;
3351 while (parent != tmp) {
3352 iommu_detach_dev(iommu, parent->bus->number,
3353 parent->devfn);
3354 parent = parent->bus->self;
3355 }
3356 if (pci_is_pcie(tmp))
3357 iommu_detach_dev(iommu,
3358 tmp->subordinate->number, 0);
3359 else
3360 iommu_detach_dev(iommu, tmp->bus->number,
3361 tmp->devfn);
3362 }
3363}
3364
3365static void domain_remove_one_dev_info(struct dmar_domain *domain,
3366 struct pci_dev *pdev)
3367{
3368 struct device_domain_info *info;
3369 struct intel_iommu *iommu;
3370 unsigned long flags;
3371 int found = 0;
3372 struct list_head *entry, *tmp;
3373
3374 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3375 pdev->devfn);
3376 if (!iommu)
3377 return;
3378
3379 spin_lock_irqsave(&device_domain_lock, flags);
3380 list_for_each_safe(entry, tmp, &domain->devices) {
3381 info = list_entry(entry, struct device_domain_info, link);
3382
3383 if (info->bus == pdev->bus->number &&
3384 info->devfn == pdev->devfn) {
3385 list_del(&info->link);
3386 list_del(&info->global);
3387 if (info->dev)
3388 info->dev->dev.archdata.iommu = NULL;
3389 spin_unlock_irqrestore(&device_domain_lock, flags);
3390
3391 iommu_disable_dev_iotlb(info);
3392 iommu_detach_dev(iommu, info->bus, info->devfn);
3393 iommu_detach_dependent_devices(iommu, pdev);
3394 free_devinfo_mem(info);
3395
3396 spin_lock_irqsave(&device_domain_lock, flags);
3397
3398 if (found)
3399 break;
3400 else
3401 continue;
3402 }
3403
3404
3405
3406
3407
3408 if (iommu == device_to_iommu(info->segment, info->bus,
3409 info->devfn))
3410 found = 1;
3411 }
3412
3413 if (found == 0) {
3414 unsigned long tmp_flags;
3415 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3416 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3417 domain->iommu_count--;
3418 domain_update_iommu_cap(domain);
3419 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3420
3421 spin_lock_irqsave(&iommu->lock, tmp_flags);
3422 clear_bit(domain->id, iommu->domain_ids);
3423 iommu->domains[domain->id] = NULL;
3424 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3425 }
3426
3427 spin_unlock_irqrestore(&device_domain_lock, flags);
3428}
3429
3430static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3431{
3432 struct device_domain_info *info;
3433 struct intel_iommu *iommu;
3434 unsigned long flags1, flags2;
3435
3436 spin_lock_irqsave(&device_domain_lock, flags1);
3437 while (!list_empty(&domain->devices)) {
3438 info = list_entry(domain->devices.next,
3439 struct device_domain_info, link);
3440 list_del(&info->link);
3441 list_del(&info->global);
3442 if (info->dev)
3443 info->dev->dev.archdata.iommu = NULL;
3444
3445 spin_unlock_irqrestore(&device_domain_lock, flags1);
3446
3447 iommu_disable_dev_iotlb(info);
3448 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3449 iommu_detach_dev(iommu, info->bus, info->devfn);
3450 iommu_detach_dependent_devices(iommu, info->dev);
3451
3452
3453
3454
3455 spin_lock_irqsave(&domain->iommu_lock, flags2);
3456 if (test_and_clear_bit(iommu->seq_id,
3457 &domain->iommu_bmp)) {
3458 domain->iommu_count--;
3459 domain_update_iommu_cap(domain);
3460 }
3461 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3462
3463 free_devinfo_mem(info);
3464 spin_lock_irqsave(&device_domain_lock, flags1);
3465 }
3466 spin_unlock_irqrestore(&device_domain_lock, flags1);
3467}
3468
3469
3470static unsigned long vm_domid;
3471
3472static struct dmar_domain *iommu_alloc_vm_domain(void)
3473{
3474 struct dmar_domain *domain;
3475
3476 domain = alloc_domain_mem();
3477 if (!domain)
3478 return NULL;
3479
3480 domain->id = vm_domid++;
3481 domain->nid = -1;
3482 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3483 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3484
3485 return domain;
3486}
3487
3488static int md_domain_init(struct dmar_domain *domain, int guest_width)
3489{
3490 int adjust_width;
3491
3492 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3493 spin_lock_init(&domain->iommu_lock);
3494
3495 domain_reserve_special_ranges(domain);
3496
3497
3498 domain->gaw = guest_width;
3499 adjust_width = guestwidth_to_adjustwidth(guest_width);
3500 domain->agaw = width_to_agaw(adjust_width);
3501
3502 INIT_LIST_HEAD(&domain->devices);
3503
3504 domain->iommu_count = 0;
3505 domain->iommu_coherency = 0;
3506 domain->iommu_snooping = 0;
3507 domain->max_addr = 0;
3508 domain->nid = -1;
3509
3510
3511 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3512 if (!domain->pgd)
3513 return -ENOMEM;
3514 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3515 return 0;
3516}
3517
3518static void iommu_free_vm_domain(struct dmar_domain *domain)
3519{
3520 unsigned long flags;
3521 struct dmar_drhd_unit *drhd;
3522 struct intel_iommu *iommu;
3523 unsigned long i;
3524 unsigned long ndomains;
3525
3526 for_each_drhd_unit(drhd) {
3527 if (drhd->ignored)
3528 continue;
3529 iommu = drhd->iommu;
3530
3531 ndomains = cap_ndoms(iommu->cap);
3532 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3533 if (iommu->domains[i] == domain) {
3534 spin_lock_irqsave(&iommu->lock, flags);
3535 clear_bit(i, iommu->domain_ids);
3536 iommu->domains[i] = NULL;
3537 spin_unlock_irqrestore(&iommu->lock, flags);
3538 break;
3539 }
3540 }
3541 }
3542}
3543
3544static void vm_domain_exit(struct dmar_domain *domain)
3545{
3546
3547 if (!domain)
3548 return;
3549
3550 vm_domain_remove_all_dev_info(domain);
3551
3552 put_iova_domain(&domain->iovad);
3553
3554
3555 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3556
3557
3558 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3559
3560 iommu_free_vm_domain(domain);
3561 free_domain_mem(domain);
3562}
3563
3564static int intel_iommu_domain_init(struct iommu_domain *domain)
3565{
3566 struct dmar_domain *dmar_domain;
3567
3568 dmar_domain = iommu_alloc_vm_domain();
3569 if (!dmar_domain) {
3570 printk(KERN_ERR
3571 "intel_iommu_domain_init: dmar_domain == NULL\n");
3572 return -ENOMEM;
3573 }
3574 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3575 printk(KERN_ERR
3576 "intel_iommu_domain_init() failed\n");
3577 vm_domain_exit(dmar_domain);
3578 return -ENOMEM;
3579 }
3580 domain->priv = dmar_domain;
3581
3582 return 0;
3583}
3584
3585static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3586{
3587 struct dmar_domain *dmar_domain = domain->priv;
3588
3589 domain->priv = NULL;
3590 vm_domain_exit(dmar_domain);
3591}
3592
3593static int intel_iommu_attach_device(struct iommu_domain *domain,
3594 struct device *dev)
3595{
3596 struct dmar_domain *dmar_domain = domain->priv;
3597 struct pci_dev *pdev = to_pci_dev(dev);
3598 struct intel_iommu *iommu;
3599 int addr_width;
3600
3601
3602 if (unlikely(domain_context_mapped(pdev))) {
3603 struct dmar_domain *old_domain;
3604
3605 old_domain = find_domain(pdev);
3606 if (old_domain) {
3607 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3608 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3609 domain_remove_one_dev_info(old_domain, pdev);
3610 else
3611 domain_remove_dev_info(old_domain);
3612 }
3613 }
3614
3615 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3616 pdev->devfn);
3617 if (!iommu)
3618 return -ENODEV;
3619
3620
3621 addr_width = agaw_to_width(iommu->agaw);
3622 if (addr_width > cap_mgaw(iommu->cap))
3623 addr_width = cap_mgaw(iommu->cap);
3624
3625 if (dmar_domain->max_addr > (1LL << addr_width)) {
3626 printk(KERN_ERR "%s: iommu width (%d) is not "
3627 "sufficient for the mapped address (%llx)\n",
3628 __func__, addr_width, dmar_domain->max_addr);
3629 return -EFAULT;
3630 }
3631 dmar_domain->gaw = addr_width;
3632
3633
3634
3635
3636 while (iommu->agaw < dmar_domain->agaw) {
3637 struct dma_pte *pte;
3638
3639 pte = dmar_domain->pgd;
3640 if (dma_pte_present(pte)) {
3641 dmar_domain->pgd = (struct dma_pte *)
3642 phys_to_virt(dma_pte_addr(pte));
3643 free_pgtable_page(pte);
3644 }
3645 dmar_domain->agaw--;
3646 }
3647
3648 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3649}
3650
3651static void intel_iommu_detach_device(struct iommu_domain *domain,
3652 struct device *dev)
3653{
3654 struct dmar_domain *dmar_domain = domain->priv;
3655 struct pci_dev *pdev = to_pci_dev(dev);
3656
3657 domain_remove_one_dev_info(dmar_domain, pdev);
3658}
3659
3660static int intel_iommu_map(struct iommu_domain *domain,
3661 unsigned long iova, phys_addr_t hpa,
3662 int gfp_order, int iommu_prot)
3663{
3664 struct dmar_domain *dmar_domain = domain->priv;
3665 u64 max_addr;
3666 int prot = 0;
3667 size_t size;
3668 int ret;
3669
3670 if (iommu_prot & IOMMU_READ)
3671 prot |= DMA_PTE_READ;
3672 if (iommu_prot & IOMMU_WRITE)
3673 prot |= DMA_PTE_WRITE;
3674 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3675 prot |= DMA_PTE_SNP;
3676
3677 size = PAGE_SIZE << gfp_order;
3678 max_addr = iova + size;
3679 if (dmar_domain->max_addr < max_addr) {
3680 u64 end;
3681
3682
3683 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3684 if (end < max_addr) {
3685 printk(KERN_ERR "%s: iommu width (%d) is not "
3686 "sufficient for the mapped address (%llx)\n",
3687 __func__, dmar_domain->gaw, max_addr);
3688 return -EFAULT;
3689 }
3690 dmar_domain->max_addr = max_addr;
3691 }
3692
3693
3694 size = aligned_nrpages(hpa, size);
3695 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3696 hpa >> VTD_PAGE_SHIFT, size, prot);
3697 return ret;
3698}
3699
3700static int intel_iommu_unmap(struct iommu_domain *domain,
3701 unsigned long iova, int gfp_order)
3702{
3703 struct dmar_domain *dmar_domain = domain->priv;
3704 size_t size = PAGE_SIZE << gfp_order;
3705
3706 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3707 (iova + size - 1) >> VTD_PAGE_SHIFT);
3708
3709 if (dmar_domain->max_addr == iova + size)
3710 dmar_domain->max_addr = iova;
3711
3712 return gfp_order;
3713}
3714
3715static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3716 unsigned long iova)
3717{
3718 struct dmar_domain *dmar_domain = domain->priv;
3719 struct dma_pte *pte;
3720 u64 phys = 0;
3721
3722 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3723 if (pte)
3724 phys = dma_pte_addr(pte);
3725
3726 return phys;
3727}
3728
3729static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3730 unsigned long cap)
3731{
3732 struct dmar_domain *dmar_domain = domain->priv;
3733
3734 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3735 return dmar_domain->iommu_snooping;
3736 if (cap == IOMMU_CAP_INTR_REMAP)
3737 return intr_remapping_enabled;
3738
3739 return 0;
3740}
3741
3742static struct iommu_ops intel_iommu_ops = {
3743 .domain_init = intel_iommu_domain_init,
3744 .domain_destroy = intel_iommu_domain_destroy,
3745 .attach_dev = intel_iommu_attach_device,
3746 .detach_dev = intel_iommu_detach_device,
3747 .map = intel_iommu_map,
3748 .unmap = intel_iommu_unmap,
3749 .iova_to_phys = intel_iommu_iova_to_phys,
3750 .domain_has_cap = intel_iommu_domain_has_cap,
3751};
3752
3753static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3754{
3755
3756
3757
3758
3759 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3760 rwbf_quirk = 1;
3761
3762
3763 if (dev->revision == 0x07) {
3764 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3765 dmar_map_gfx = 0;
3766 }
3767}
3768
3769DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3770
3771#define GGC 0x52
3772#define GGC_MEMORY_SIZE_MASK (0xf << 8)
3773#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3774#define GGC_MEMORY_SIZE_1M (0x1 << 8)
3775#define GGC_MEMORY_SIZE_2M (0x3 << 8)
3776#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3777#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3778#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3779#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3780
3781static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3782{
3783 unsigned short ggc;
3784
3785 if (pci_read_config_word(dev, GGC, &ggc))
3786 return;
3787
3788 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3789 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3790 dmar_map_gfx = 0;
3791 }
3792}
3793DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3794DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3795DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3796DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3797
3798
3799
3800
3801
3802
3803
3804
3805static void __init check_tylersburg_isoch(void)
3806{
3807 struct pci_dev *pdev;
3808 uint32_t vtisochctrl;
3809
3810
3811 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3812 if (!pdev)
3813 return;
3814 pci_dev_put(pdev);
3815
3816
3817
3818
3819 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3820 if (!pdev)
3821 return;
3822
3823 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3824 pci_dev_put(pdev);
3825 return;
3826 }
3827
3828 pci_dev_put(pdev);
3829
3830
3831 if (vtisochctrl & 1)
3832 return;
3833
3834
3835 vtisochctrl &= 0x1c;
3836
3837
3838 if (vtisochctrl == 0x10)
3839 return;
3840
3841
3842 if (!vtisochctrl) {
3843 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3844 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3845 dmi_get_system_info(DMI_BIOS_VENDOR),
3846 dmi_get_system_info(DMI_BIOS_VERSION),
3847 dmi_get_system_info(DMI_PRODUCT_VERSION));
3848 iommu_identity_mapping |= IDENTMAP_AZALIA;
3849 return;
3850 }
3851
3852 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3853 vtisochctrl);
3854}
3855