1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/iova.h>
34#include <linux/intel-iommu.h>
35#include <linux/timer.h>
36#include <linux/irq.h>
37#include <linux/interrupt.h>
38#include <linux/tboot.h>
39#include <linux/dmi.h>
40#include <linux/slab.h>
41#include <asm/irq_remapping.h>
42#include <asm/iommu_table.h>
43
44#include "irq_remapping.h"
45
46
47
48
49
50LIST_HEAD(dmar_drhd_units);
51
52struct acpi_table_header * __initdata dmar_tbl;
53static acpi_size dmar_tbl_size;
54
55static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
56{
57
58
59
60
61 if (drhd->include_all)
62 list_add_tail(&drhd->list, &dmar_drhd_units);
63 else
64 list_add(&drhd->list, &dmar_drhd_units);
65}
66
67static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
68 struct pci_dev **dev, u16 segment)
69{
70 struct pci_bus *bus;
71 struct pci_dev *pdev = NULL;
72 struct acpi_dmar_pci_path *path;
73 int count;
74
75 bus = pci_find_bus(segment, scope->bus);
76 path = (struct acpi_dmar_pci_path *)(scope + 1);
77 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
78 / sizeof(struct acpi_dmar_pci_path);
79
80 while (count) {
81 if (pdev)
82 pci_dev_put(pdev);
83
84
85
86
87 if (!bus) {
88 pr_warn("Device scope bus [%d] not found\n", scope->bus);
89 break;
90 }
91 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
92 if (!pdev) {
93
94 break;
95 }
96 path ++;
97 count --;
98 bus = pdev->subordinate;
99 }
100 if (!pdev) {
101 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
103 *dev = NULL;
104 return 0;
105 }
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
109 pci_dev_put(pdev);
110 pr_warn("Device scope type does not match for %s\n",
111 pci_name(pdev));
112 return -EINVAL;
113 }
114 *dev = pdev;
115 return 0;
116}
117
118int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119 struct pci_dev ***devices, u16 segment)
120{
121 struct acpi_dmar_device_scope *scope;
122 void * tmp = start;
123 int index;
124 int ret;
125
126 *cnt = 0;
127 while (start < end) {
128 scope = start;
129 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
131 (*cnt)++;
132 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
133 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
134 pr_warn("Unsupported device scope\n");
135 }
136 start += scope->length;
137 }
138 if (*cnt == 0)
139 return 0;
140
141 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
142 if (!*devices)
143 return -ENOMEM;
144
145 start = tmp;
146 index = 0;
147 while (start < end) {
148 scope = start;
149 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
150 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
151 ret = dmar_parse_one_dev_scope(scope,
152 &(*devices)[index], segment);
153 if (ret) {
154 kfree(*devices);
155 return ret;
156 }
157 index ++;
158 }
159 start += scope->length;
160 }
161
162 return 0;
163}
164
165
166
167
168
169
170static int __init
171dmar_parse_one_drhd(struct acpi_dmar_header *header)
172{
173 struct acpi_dmar_hardware_unit *drhd;
174 struct dmar_drhd_unit *dmaru;
175 int ret = 0;
176
177 drhd = (struct acpi_dmar_hardware_unit *)header;
178 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
179 if (!dmaru)
180 return -ENOMEM;
181
182 dmaru->hdr = header;
183 dmaru->reg_base_addr = drhd->address;
184 dmaru->segment = drhd->segment;
185 dmaru->include_all = drhd->flags & 0x1;
186
187 ret = alloc_iommu(dmaru);
188 if (ret) {
189 kfree(dmaru);
190 return ret;
191 }
192 dmar_register_drhd_unit(dmaru);
193 return 0;
194}
195
196static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
197{
198 struct acpi_dmar_hardware_unit *drhd;
199 int ret = 0;
200
201 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
202
203 if (dmaru->include_all)
204 return 0;
205
206 ret = dmar_parse_dev_scope((void *)(drhd + 1),
207 ((void *)drhd) + drhd->header.length,
208 &dmaru->devices_cnt, &dmaru->devices,
209 drhd->segment);
210 if (ret) {
211 list_del(&dmaru->list);
212 kfree(dmaru);
213 }
214 return ret;
215}
216
217#ifdef CONFIG_ACPI_NUMA
218static int __init
219dmar_parse_one_rhsa(struct acpi_dmar_header *header)
220{
221 struct acpi_dmar_rhsa *rhsa;
222 struct dmar_drhd_unit *drhd;
223
224 rhsa = (struct acpi_dmar_rhsa *)header;
225 for_each_drhd_unit(drhd) {
226 if (drhd->reg_base_addr == rhsa->base_address) {
227 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
228
229 if (!node_online(node))
230 node = -1;
231 drhd->iommu->node = node;
232 return 0;
233 }
234 }
235 WARN_TAINT(
236 1, TAINT_FIRMWARE_WORKAROUND,
237 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
238 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
239 drhd->reg_base_addr,
240 dmi_get_system_info(DMI_BIOS_VENDOR),
241 dmi_get_system_info(DMI_BIOS_VERSION),
242 dmi_get_system_info(DMI_PRODUCT_VERSION));
243
244 return 0;
245}
246#endif
247
248static void __init
249dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
250{
251 struct acpi_dmar_hardware_unit *drhd;
252 struct acpi_dmar_reserved_memory *rmrr;
253 struct acpi_dmar_atsr *atsr;
254 struct acpi_dmar_rhsa *rhsa;
255
256 switch (header->type) {
257 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
258 drhd = container_of(header, struct acpi_dmar_hardware_unit,
259 header);
260 pr_info("DRHD base: %#016Lx flags: %#x\n",
261 (unsigned long long)drhd->address, drhd->flags);
262 break;
263 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
264 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
265 header);
266 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
267 (unsigned long long)rmrr->base_address,
268 (unsigned long long)rmrr->end_address);
269 break;
270 case ACPI_DMAR_TYPE_ATSR:
271 atsr = container_of(header, struct acpi_dmar_atsr, header);
272 pr_info("ATSR flags: %#x\n", atsr->flags);
273 break;
274 case ACPI_DMAR_HARDWARE_AFFINITY:
275 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
276 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
277 (unsigned long long)rhsa->base_address,
278 rhsa->proximity_domain);
279 break;
280 }
281}
282
283
284
285
286static int __init dmar_table_detect(void)
287{
288 acpi_status status = AE_OK;
289
290
291 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
292 (struct acpi_table_header **)&dmar_tbl,
293 &dmar_tbl_size);
294
295 if (ACPI_SUCCESS(status) && !dmar_tbl) {
296 pr_warn("Unable to map DMAR\n");
297 status = AE_NOT_FOUND;
298 }
299
300 return (ACPI_SUCCESS(status) ? 1 : 0);
301}
302
303
304
305
306static int __init
307parse_dmar_table(void)
308{
309 struct acpi_table_dmar *dmar;
310 struct acpi_dmar_header *entry_header;
311 int ret = 0;
312 int drhd_count = 0;
313
314
315
316
317
318 dmar_table_detect();
319
320
321
322
323
324 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
325
326 dmar = (struct acpi_table_dmar *)dmar_tbl;
327 if (!dmar)
328 return -ENODEV;
329
330 if (dmar->width < PAGE_SHIFT - 1) {
331 pr_warn("Invalid DMAR haw\n");
332 return -EINVAL;
333 }
334
335 pr_info("Host address width %d\n", dmar->width + 1);
336
337 entry_header = (struct acpi_dmar_header *)(dmar + 1);
338 while (((unsigned long)entry_header) <
339 (((unsigned long)dmar) + dmar_tbl->length)) {
340
341 if (entry_header->length == 0) {
342 pr_warn("Invalid 0-length structure\n");
343 ret = -EINVAL;
344 break;
345 }
346
347 dmar_table_print_dmar_entry(entry_header);
348
349 switch (entry_header->type) {
350 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
351 drhd_count++;
352 ret = dmar_parse_one_drhd(entry_header);
353 break;
354 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
355 ret = dmar_parse_one_rmrr(entry_header);
356 break;
357 case ACPI_DMAR_TYPE_ATSR:
358 ret = dmar_parse_one_atsr(entry_header);
359 break;
360 case ACPI_DMAR_HARDWARE_AFFINITY:
361#ifdef CONFIG_ACPI_NUMA
362 ret = dmar_parse_one_rhsa(entry_header);
363#endif
364 break;
365 default:
366 pr_warn("Unknown DMAR structure type %d\n",
367 entry_header->type);
368 ret = 0;
369 break;
370 }
371 if (ret)
372 break;
373
374 entry_header = ((void *)entry_header + entry_header->length);
375 }
376 if (drhd_count == 0)
377 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
378 return ret;
379}
380
381static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
382 struct pci_dev *dev)
383{
384 int index;
385
386 while (dev) {
387 for (index = 0; index < cnt; index++)
388 if (dev == devices[index])
389 return 1;
390
391
392 dev = dev->bus->self;
393 }
394
395 return 0;
396}
397
398struct dmar_drhd_unit *
399dmar_find_matched_drhd_unit(struct pci_dev *dev)
400{
401 struct dmar_drhd_unit *dmaru = NULL;
402 struct acpi_dmar_hardware_unit *drhd;
403
404 dev = pci_physfn(dev);
405
406 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
407 drhd = container_of(dmaru->hdr,
408 struct acpi_dmar_hardware_unit,
409 header);
410
411 if (dmaru->include_all &&
412 drhd->segment == pci_domain_nr(dev->bus))
413 return dmaru;
414
415 if (dmar_pci_device_match(dmaru->devices,
416 dmaru->devices_cnt, dev))
417 return dmaru;
418 }
419
420 return NULL;
421}
422
423int __init dmar_dev_scope_init(void)
424{
425 static int dmar_dev_scope_initialized;
426 struct dmar_drhd_unit *drhd, *drhd_n;
427 int ret = -ENODEV;
428
429 if (dmar_dev_scope_initialized)
430 return dmar_dev_scope_initialized;
431
432 if (list_empty(&dmar_drhd_units))
433 goto fail;
434
435 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
436 ret = dmar_parse_dev(drhd);
437 if (ret)
438 goto fail;
439 }
440
441 ret = dmar_parse_rmrr_atsr_dev();
442 if (ret)
443 goto fail;
444
445 dmar_dev_scope_initialized = 1;
446 return 0;
447
448fail:
449 dmar_dev_scope_initialized = ret;
450 return ret;
451}
452
453
454int __init dmar_table_init(void)
455{
456 static int dmar_table_initialized;
457 int ret;
458
459 if (dmar_table_initialized)
460 return 0;
461
462 dmar_table_initialized = 1;
463
464 ret = parse_dmar_table();
465 if (ret) {
466 if (ret != -ENODEV)
467 pr_info("parse DMAR table failure.\n");
468 return ret;
469 }
470
471 if (list_empty(&dmar_drhd_units)) {
472 pr_info("No DMAR devices found\n");
473 return -ENODEV;
474 }
475
476 return 0;
477}
478
479static void warn_invalid_dmar(u64 addr, const char *message)
480{
481 WARN_TAINT_ONCE(
482 1, TAINT_FIRMWARE_WORKAROUND,
483 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
484 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
485 addr, message,
486 dmi_get_system_info(DMI_BIOS_VENDOR),
487 dmi_get_system_info(DMI_BIOS_VERSION),
488 dmi_get_system_info(DMI_PRODUCT_VERSION));
489}
490
491int __init check_zero_address(void)
492{
493 struct acpi_table_dmar *dmar;
494 struct acpi_dmar_header *entry_header;
495 struct acpi_dmar_hardware_unit *drhd;
496
497 dmar = (struct acpi_table_dmar *)dmar_tbl;
498 entry_header = (struct acpi_dmar_header *)(dmar + 1);
499
500 while (((unsigned long)entry_header) <
501 (((unsigned long)dmar) + dmar_tbl->length)) {
502
503 if (entry_header->length == 0) {
504 pr_warn("Invalid 0-length structure\n");
505 return 0;
506 }
507
508 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
509 void __iomem *addr;
510 u64 cap, ecap;
511
512 drhd = (void *)entry_header;
513 if (!drhd->address) {
514 warn_invalid_dmar(0, "");
515 goto failed;
516 }
517
518 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
519 if (!addr ) {
520 printk("IOMMU: can't validate: %llx\n", drhd->address);
521 goto failed;
522 }
523 cap = dmar_readq(addr + DMAR_CAP_REG);
524 ecap = dmar_readq(addr + DMAR_ECAP_REG);
525 early_iounmap(addr, VTD_PAGE_SIZE);
526 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
527 warn_invalid_dmar(drhd->address,
528 " returns all ones");
529 goto failed;
530 }
531 }
532
533 entry_header = ((void *)entry_header + entry_header->length);
534 }
535 return 1;
536
537failed:
538 return 0;
539}
540
541int __init detect_intel_iommu(void)
542{
543 int ret;
544
545 ret = dmar_table_detect();
546 if (ret)
547 ret = check_zero_address();
548 {
549 struct acpi_table_dmar *dmar;
550
551 dmar = (struct acpi_table_dmar *) dmar_tbl;
552
553 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
554 dmar->flags & 0x1)
555 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
556
557 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
558 iommu_detected = 1;
559
560 pci_request_acs();
561 }
562
563#ifdef CONFIG_X86
564 if (ret)
565 x86_init.iommu.iommu_init = intel_iommu_init;
566#endif
567 }
568 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
569 dmar_tbl = NULL;
570
571 return ret ? 1 : -ENODEV;
572}
573
574
575static void unmap_iommu(struct intel_iommu *iommu)
576{
577 iounmap(iommu->reg);
578 release_mem_region(iommu->reg_phys, iommu->reg_size);
579}
580
581
582
583
584
585
586
587
588
589static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
590{
591 int map_size, err=0;
592
593 iommu->reg_phys = phys_addr;
594 iommu->reg_size = VTD_PAGE_SIZE;
595
596 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
597 pr_err("IOMMU: can't reserve memory\n");
598 err = -EBUSY;
599 goto out;
600 }
601
602 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
603 if (!iommu->reg) {
604 pr_err("IOMMU: can't map the region\n");
605 err = -ENOMEM;
606 goto release;
607 }
608
609 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
610 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
611
612 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
613 err = -EINVAL;
614 warn_invalid_dmar(phys_addr, " returns all ones");
615 goto unmap;
616 }
617
618
619 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
620 cap_max_fault_reg_offset(iommu->cap));
621 map_size = VTD_PAGE_ALIGN(map_size);
622 if (map_size > iommu->reg_size) {
623 iounmap(iommu->reg);
624 release_mem_region(iommu->reg_phys, iommu->reg_size);
625 iommu->reg_size = map_size;
626 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
627 iommu->name)) {
628 pr_err("IOMMU: can't reserve memory\n");
629 err = -EBUSY;
630 goto out;
631 }
632 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
633 if (!iommu->reg) {
634 pr_err("IOMMU: can't map the region\n");
635 err = -ENOMEM;
636 goto release;
637 }
638 }
639 err = 0;
640 goto out;
641
642unmap:
643 iounmap(iommu->reg);
644release:
645 release_mem_region(iommu->reg_phys, iommu->reg_size);
646out:
647 return err;
648}
649
650int alloc_iommu(struct dmar_drhd_unit *drhd)
651{
652 struct intel_iommu *iommu;
653 u32 ver, sts;
654 static int iommu_allocated = 0;
655 int agaw = 0;
656 int msagaw = 0;
657 int err;
658
659 if (!drhd->reg_base_addr) {
660 warn_invalid_dmar(0, "");
661 return -EINVAL;
662 }
663
664 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
665 if (!iommu)
666 return -ENOMEM;
667
668 iommu->seq_id = iommu_allocated++;
669 sprintf (iommu->name, "dmar%d", iommu->seq_id);
670
671 err = map_iommu(iommu, drhd->reg_base_addr);
672 if (err) {
673 pr_err("IOMMU: failed to map %s\n", iommu->name);
674 goto error;
675 }
676
677 err = -EINVAL;
678 agaw = iommu_calculate_agaw(iommu);
679 if (agaw < 0) {
680 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
681 iommu->seq_id);
682 goto err_unmap;
683 }
684 msagaw = iommu_calculate_max_sagaw(iommu);
685 if (msagaw < 0) {
686 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
687 iommu->seq_id);
688 goto err_unmap;
689 }
690 iommu->agaw = agaw;
691 iommu->msagaw = msagaw;
692
693 iommu->node = -1;
694
695 ver = readl(iommu->reg + DMAR_VER_REG);
696 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
697 iommu->seq_id,
698 (unsigned long long)drhd->reg_base_addr,
699 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
700 (unsigned long long)iommu->cap,
701 (unsigned long long)iommu->ecap);
702
703
704 sts = readl(iommu->reg + DMAR_GSTS_REG);
705 if (sts & DMA_GSTS_IRES)
706 iommu->gcmd |= DMA_GCMD_IRE;
707 if (sts & DMA_GSTS_TES)
708 iommu->gcmd |= DMA_GCMD_TE;
709 if (sts & DMA_GSTS_QIES)
710 iommu->gcmd |= DMA_GCMD_QIE;
711
712 raw_spin_lock_init(&iommu->register_lock);
713
714 drhd->iommu = iommu;
715 return 0;
716
717 err_unmap:
718 unmap_iommu(iommu);
719 error:
720 kfree(iommu);
721 return err;
722}
723
724void free_iommu(struct intel_iommu *iommu)
725{
726 if (!iommu)
727 return;
728
729 free_dmar_iommu(iommu);
730
731 if (iommu->reg)
732 unmap_iommu(iommu);
733
734 kfree(iommu);
735}
736
737
738
739
740static inline void reclaim_free_desc(struct q_inval *qi)
741{
742 while (qi->desc_status[qi->free_tail] == QI_DONE ||
743 qi->desc_status[qi->free_tail] == QI_ABORT) {
744 qi->desc_status[qi->free_tail] = QI_FREE;
745 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
746 qi->free_cnt++;
747 }
748}
749
750static int qi_check_fault(struct intel_iommu *iommu, int index)
751{
752 u32 fault;
753 int head, tail;
754 struct q_inval *qi = iommu->qi;
755 int wait_index = (index + 1) % QI_LENGTH;
756
757 if (qi->desc_status[wait_index] == QI_ABORT)
758 return -EAGAIN;
759
760 fault = readl(iommu->reg + DMAR_FSTS_REG);
761
762
763
764
765
766
767 if (fault & DMA_FSTS_IQE) {
768 head = readl(iommu->reg + DMAR_IQH_REG);
769 if ((head >> DMAR_IQ_SHIFT) == index) {
770 pr_err("VT-d detected invalid descriptor: "
771 "low=%llx, high=%llx\n",
772 (unsigned long long)qi->desc[index].low,
773 (unsigned long long)qi->desc[index].high);
774 memcpy(&qi->desc[index], &qi->desc[wait_index],
775 sizeof(struct qi_desc));
776 __iommu_flush_cache(iommu, &qi->desc[index],
777 sizeof(struct qi_desc));
778 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
779 return -EINVAL;
780 }
781 }
782
783
784
785
786
787 if (fault & DMA_FSTS_ITE) {
788 head = readl(iommu->reg + DMAR_IQH_REG);
789 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
790 head |= 1;
791 tail = readl(iommu->reg + DMAR_IQT_REG);
792 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
793
794 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
795
796 do {
797 if (qi->desc_status[head] == QI_IN_USE)
798 qi->desc_status[head] = QI_ABORT;
799 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
800 } while (head != tail);
801
802 if (qi->desc_status[wait_index] == QI_ABORT)
803 return -EAGAIN;
804 }
805
806 if (fault & DMA_FSTS_ICE)
807 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
808
809 return 0;
810}
811
812
813
814
815
816int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
817{
818 int rc;
819 struct q_inval *qi = iommu->qi;
820 struct qi_desc *hw, wait_desc;
821 int wait_index, index;
822 unsigned long flags;
823
824 if (!qi)
825 return 0;
826
827 hw = qi->desc;
828
829restart:
830 rc = 0;
831
832 raw_spin_lock_irqsave(&qi->q_lock, flags);
833 while (qi->free_cnt < 3) {
834 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
835 cpu_relax();
836 raw_spin_lock_irqsave(&qi->q_lock, flags);
837 }
838
839 index = qi->free_head;
840 wait_index = (index + 1) % QI_LENGTH;
841
842 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
843
844 hw[index] = *desc;
845
846 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
847 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
848 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
849
850 hw[wait_index] = wait_desc;
851
852 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
853 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
854
855 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
856 qi->free_cnt -= 2;
857
858
859
860
861
862 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
863
864 while (qi->desc_status[wait_index] != QI_DONE) {
865
866
867
868
869
870
871
872 rc = qi_check_fault(iommu, index);
873 if (rc)
874 break;
875
876 raw_spin_unlock(&qi->q_lock);
877 cpu_relax();
878 raw_spin_lock(&qi->q_lock);
879 }
880
881 qi->desc_status[index] = QI_DONE;
882
883 reclaim_free_desc(qi);
884 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
885
886 if (rc == -EAGAIN)
887 goto restart;
888
889 return rc;
890}
891
892
893
894
895void qi_global_iec(struct intel_iommu *iommu)
896{
897 struct qi_desc desc;
898
899 desc.low = QI_IEC_TYPE;
900 desc.high = 0;
901
902
903 qi_submit_sync(&desc, iommu);
904}
905
906void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
907 u64 type)
908{
909 struct qi_desc desc;
910
911 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
912 | QI_CC_GRAN(type) | QI_CC_TYPE;
913 desc.high = 0;
914
915 qi_submit_sync(&desc, iommu);
916}
917
918void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
919 unsigned int size_order, u64 type)
920{
921 u8 dw = 0, dr = 0;
922
923 struct qi_desc desc;
924 int ih = 0;
925
926 if (cap_write_drain(iommu->cap))
927 dw = 1;
928
929 if (cap_read_drain(iommu->cap))
930 dr = 1;
931
932 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
933 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
934 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
935 | QI_IOTLB_AM(size_order);
936
937 qi_submit_sync(&desc, iommu);
938}
939
940void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
941 u64 addr, unsigned mask)
942{
943 struct qi_desc desc;
944
945 if (mask) {
946 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
947 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
948 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
949 } else
950 desc.high = QI_DEV_IOTLB_ADDR(addr);
951
952 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
953 qdep = 0;
954
955 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
956 QI_DIOTLB_TYPE;
957
958 qi_submit_sync(&desc, iommu);
959}
960
961
962
963
964void dmar_disable_qi(struct intel_iommu *iommu)
965{
966 unsigned long flags;
967 u32 sts;
968 cycles_t start_time = get_cycles();
969
970 if (!ecap_qis(iommu->ecap))
971 return;
972
973 raw_spin_lock_irqsave(&iommu->register_lock, flags);
974
975 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
976 if (!(sts & DMA_GSTS_QIES))
977 goto end;
978
979
980
981
982 while ((readl(iommu->reg + DMAR_IQT_REG) !=
983 readl(iommu->reg + DMAR_IQH_REG)) &&
984 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
985 cpu_relax();
986
987 iommu->gcmd &= ~DMA_GCMD_QIE;
988 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
989
990 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
991 !(sts & DMA_GSTS_QIES), sts);
992end:
993 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
994}
995
996
997
998
999static void __dmar_enable_qi(struct intel_iommu *iommu)
1000{
1001 u32 sts;
1002 unsigned long flags;
1003 struct q_inval *qi = iommu->qi;
1004
1005 qi->free_head = qi->free_tail = 0;
1006 qi->free_cnt = QI_LENGTH;
1007
1008 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1009
1010
1011 writel(0, iommu->reg + DMAR_IQT_REG);
1012
1013 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1014
1015 iommu->gcmd |= DMA_GCMD_QIE;
1016 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1017
1018
1019 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1020
1021 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1022}
1023
1024
1025
1026
1027
1028
1029int dmar_enable_qi(struct intel_iommu *iommu)
1030{
1031 struct q_inval *qi;
1032 struct page *desc_page;
1033
1034 if (!ecap_qis(iommu->ecap))
1035 return -ENOENT;
1036
1037
1038
1039
1040 if (iommu->qi)
1041 return 0;
1042
1043 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1044 if (!iommu->qi)
1045 return -ENOMEM;
1046
1047 qi = iommu->qi;
1048
1049
1050 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1051 if (!desc_page) {
1052 kfree(qi);
1053 iommu->qi = 0;
1054 return -ENOMEM;
1055 }
1056
1057 qi->desc = page_address(desc_page);
1058
1059 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1060 if (!qi->desc_status) {
1061 free_page((unsigned long) qi->desc);
1062 kfree(qi);
1063 iommu->qi = 0;
1064 return -ENOMEM;
1065 }
1066
1067 qi->free_head = qi->free_tail = 0;
1068 qi->free_cnt = QI_LENGTH;
1069
1070 raw_spin_lock_init(&qi->q_lock);
1071
1072 __dmar_enable_qi(iommu);
1073
1074 return 0;
1075}
1076
1077
1078
1079enum faulttype {
1080 DMA_REMAP,
1081 INTR_REMAP,
1082 UNKNOWN,
1083};
1084
1085static const char *dma_remap_fault_reasons[] =
1086{
1087 "Software",
1088 "Present bit in root entry is clear",
1089 "Present bit in context entry is clear",
1090 "Invalid context entry",
1091 "Access beyond MGAW",
1092 "PTE Write access is not set",
1093 "PTE Read access is not set",
1094 "Next page table ptr is invalid",
1095 "Root table address invalid",
1096 "Context table ptr is invalid",
1097 "non-zero reserved fields in RTP",
1098 "non-zero reserved fields in CTP",
1099 "non-zero reserved fields in PTE",
1100 "PCE for translation request specifies blocking",
1101};
1102
1103static const char *irq_remap_fault_reasons[] =
1104{
1105 "Detected reserved fields in the decoded interrupt-remapped request",
1106 "Interrupt index exceeded the interrupt-remapping table size",
1107 "Present field in the IRTE entry is clear",
1108 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1109 "Detected reserved fields in the IRTE entry",
1110 "Blocked a compatibility format interrupt request",
1111 "Blocked an interrupt request due to source-id verification failure",
1112};
1113
1114#define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1115
1116const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1117{
1118 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1119 ARRAY_SIZE(irq_remap_fault_reasons))) {
1120 *fault_type = INTR_REMAP;
1121 return irq_remap_fault_reasons[fault_reason - 0x20];
1122 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1123 *fault_type = DMA_REMAP;
1124 return dma_remap_fault_reasons[fault_reason];
1125 } else {
1126 *fault_type = UNKNOWN;
1127 return "Unknown";
1128 }
1129}
1130
1131void dmar_msi_unmask(struct irq_data *data)
1132{
1133 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1134 unsigned long flag;
1135
1136
1137 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1138 writel(0, iommu->reg + DMAR_FECTL_REG);
1139
1140 readl(iommu->reg + DMAR_FECTL_REG);
1141 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1142}
1143
1144void dmar_msi_mask(struct irq_data *data)
1145{
1146 unsigned long flag;
1147 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1148
1149
1150 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1151 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1152
1153 readl(iommu->reg + DMAR_FECTL_REG);
1154 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1155}
1156
1157void dmar_msi_write(int irq, struct msi_msg *msg)
1158{
1159 struct intel_iommu *iommu = irq_get_handler_data(irq);
1160 unsigned long flag;
1161
1162 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1163 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1164 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1165 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1166 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1167}
1168
1169void dmar_msi_read(int irq, struct msi_msg *msg)
1170{
1171 struct intel_iommu *iommu = irq_get_handler_data(irq);
1172 unsigned long flag;
1173
1174 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1175 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1176 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1177 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1178 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1179}
1180
1181static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1182 u8 fault_reason, u16 source_id, unsigned long long addr)
1183{
1184 const char *reason;
1185 int fault_type;
1186
1187 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1188
1189 if (fault_type == INTR_REMAP)
1190 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1191 "fault index %llx\n"
1192 "INTR-REMAP:[fault reason %02d] %s\n",
1193 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1194 PCI_FUNC(source_id & 0xFF), addr >> 48,
1195 fault_reason, reason);
1196 else
1197 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1198 "fault addr %llx \n"
1199 "DMAR:[fault reason %02d] %s\n",
1200 (type ? "DMA Read" : "DMA Write"),
1201 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1202 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1203 return 0;
1204}
1205
1206#define PRIMARY_FAULT_REG_LEN (16)
1207irqreturn_t dmar_fault(int irq, void *dev_id)
1208{
1209 struct intel_iommu *iommu = dev_id;
1210 int reg, fault_index;
1211 u32 fault_status;
1212 unsigned long flag;
1213
1214 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1215 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1216 if (fault_status)
1217 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1218
1219
1220 if (!(fault_status & DMA_FSTS_PPF))
1221 goto unlock_exit;
1222
1223 fault_index = dma_fsts_fault_record_index(fault_status);
1224 reg = cap_fault_reg_offset(iommu->cap);
1225 while (1) {
1226 u8 fault_reason;
1227 u16 source_id;
1228 u64 guest_addr;
1229 int type;
1230 u32 data;
1231
1232
1233 data = readl(iommu->reg + reg +
1234 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1235 if (!(data & DMA_FRCD_F))
1236 break;
1237
1238 fault_reason = dma_frcd_fault_reason(data);
1239 type = dma_frcd_type(data);
1240
1241 data = readl(iommu->reg + reg +
1242 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1243 source_id = dma_frcd_source_id(data);
1244
1245 guest_addr = dmar_readq(iommu->reg + reg +
1246 fault_index * PRIMARY_FAULT_REG_LEN);
1247 guest_addr = dma_frcd_page_addr(guest_addr);
1248
1249 writel(DMA_FRCD_F, iommu->reg + reg +
1250 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1251
1252 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1253
1254 dmar_fault_do_one(iommu, type, fault_reason,
1255 source_id, guest_addr);
1256
1257 fault_index++;
1258 if (fault_index >= cap_num_fault_regs(iommu->cap))
1259 fault_index = 0;
1260 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1261 }
1262
1263 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1264
1265unlock_exit:
1266 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1267 return IRQ_HANDLED;
1268}
1269
1270int dmar_set_interrupt(struct intel_iommu *iommu)
1271{
1272 int irq, ret;
1273
1274
1275
1276
1277 if (iommu->irq)
1278 return 0;
1279
1280 irq = create_irq();
1281 if (!irq) {
1282 pr_err("IOMMU: no free vectors\n");
1283 return -EINVAL;
1284 }
1285
1286 irq_set_handler_data(irq, iommu);
1287 iommu->irq = irq;
1288
1289 ret = arch_setup_dmar_msi(irq);
1290 if (ret) {
1291 irq_set_handler_data(irq, NULL);
1292 iommu->irq = 0;
1293 destroy_irq(irq);
1294 return ret;
1295 }
1296
1297 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1298 if (ret)
1299 pr_err("IOMMU: can't request irq\n");
1300 return ret;
1301}
1302
1303int __init enable_drhd_fault_handling(void)
1304{
1305 struct dmar_drhd_unit *drhd;
1306
1307
1308
1309
1310 for_each_drhd_unit(drhd) {
1311 int ret;
1312 struct intel_iommu *iommu = drhd->iommu;
1313 u32 fault_status;
1314 ret = dmar_set_interrupt(iommu);
1315
1316 if (ret) {
1317 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1318 (unsigned long long)drhd->reg_base_addr, ret);
1319 return -1;
1320 }
1321
1322
1323
1324
1325 dmar_fault(iommu->irq, iommu);
1326 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1327 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1328 }
1329
1330 return 0;
1331}
1332
1333
1334
1335
1336int dmar_reenable_qi(struct intel_iommu *iommu)
1337{
1338 if (!ecap_qis(iommu->ecap))
1339 return -ENOENT;
1340
1341 if (!iommu->qi)
1342 return -ENOENT;
1343
1344
1345
1346
1347 dmar_disable_qi(iommu);
1348
1349
1350
1351
1352
1353 __dmar_enable_qi(iommu);
1354
1355 return 0;
1356}
1357
1358
1359
1360
1361int __init dmar_ir_support(void)
1362{
1363 struct acpi_table_dmar *dmar;
1364 dmar = (struct acpi_table_dmar *)dmar_tbl;
1365 if (!dmar)
1366 return 0;
1367 return dmar->flags & 0x1;
1368}
1369IOMMU_INIT_POST(detect_intel_iommu);
1370