2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
22 #define dev_fmt(fmt) pr_fmt(fmt)
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/export.h>
28 #include <linux/slab.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/memory.h>
37 #include <linux/cpu.h>
38 #include <linux/timer.h>
40 #include <linux/iova.h>
41 #include <linux/iommu.h>
42 #include <linux/intel-iommu.h>
43 #include <linux/syscore_ops.h>
44 #include <linux/tboot.h>
45 #include <linux/dmi.h>
46 #include <linux/pci-ats.h>
47 #include <linux/memblock.h>
48 #include <linux/dma-contiguous.h>
49 #include <linux/dma-direct.h>
50 #include <linux/crash_dump.h>
51 #include <linux/numa.h>
52 #include <asm/irq_remapping.h>
53 #include <asm/cacheflush.h>
54 #include <asm/iommu.h>
56 #include "irq_remapping.h"
57 #include "intel-pasid.h"
59 #define ROOT_SIZE VTD_PAGE_SIZE
60 #define CONTEXT_SIZE VTD_PAGE_SIZE
62 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
63 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
64 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
65 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
67 #define IOAPIC_RANGE_START (0xfee00000)
68 #define IOAPIC_RANGE_END (0xfeefffff)
69 #define IOVA_START_ADDR (0x1000)
71 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
73 #define MAX_AGAW_WIDTH 64
74 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
76 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
77 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
79 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
80 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
81 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
82 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
83 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
85 /* IO virtual address start page frame number */
86 #define IOVA_START_PFN (1)
88 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
90 /* page table handling */
91 #define LEVEL_STRIDE (9)
92 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
95 * This bitmap is used to advertise the page sizes our hardware support
96 * to the IOMMU core, which will then use this information to split
97 * physically contiguous memory regions it is mapping into page sizes
100 * Traditionally the IOMMU core just handed us the mappings directly,
101 * after making sure the size is an order of a 4KiB page and that the
102 * mapping has natural alignment.
104 * To retain this behavior, we currently advertise that we support
105 * all page sizes that are an order of 4KiB.
107 * If at some point we'd like to utilize the IOMMU core's new behavior,
108 * we could change this to advertise the real page sizes we support.
110 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
112 static inline int agaw_to_level(int agaw)
117 static inline int agaw_to_width(int agaw)
119 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
122 static inline int width_to_agaw(int width)
124 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
127 static inline unsigned int level_to_offset_bits(int level)
129 return (level - 1) * LEVEL_STRIDE;
132 static inline int pfn_level_offset(unsigned long pfn, int level)
134 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
137 static inline unsigned long level_mask(int level)
139 return -1UL << level_to_offset_bits(level);
142 static inline unsigned long level_size(int level)
144 return 1UL << level_to_offset_bits(level);
147 static inline unsigned long align_to_level(unsigned long pfn, int level)
149 return (pfn + level_size(level) - 1) & level_mask(level);
152 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
154 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
157 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
158 are never going to work. */
159 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
161 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
164 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
166 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
168 static inline unsigned long page_to_dma_pfn(struct page *pg)
170 return mm_to_dma_pfn(page_to_pfn(pg));
172 static inline unsigned long virt_to_dma_pfn(void *p)
174 return page_to_dma_pfn(virt_to_page(p));
177 /* global iommu list, set NULL for ignored DMAR units */
178 static struct intel_iommu **g_iommus;
180 static void __init check_tylersburg_isoch(void);
181 static int rwbf_quirk;
184 * set to 1 to panic kernel if can't successfully enable VT-d
185 * (used when kernel is launched w/ TXT)
187 static int force_on = 0;
188 int intel_iommu_tboot_noforce;
189 static int no_platform_optin;
191 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
194 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
197 static phys_addr_t root_entry_lctp(struct root_entry *re)
202 return re->lo & VTD_PAGE_MASK;
206 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
209 static phys_addr_t root_entry_uctp(struct root_entry *re)
214 return re->hi & VTD_PAGE_MASK;
217 static inline void context_clear_pasid_enable(struct context_entry *context)
219 context->lo &= ~(1ULL << 11);
222 static inline bool context_pasid_enabled(struct context_entry *context)
224 return !!(context->lo & (1ULL << 11));
227 static inline void context_set_copied(struct context_entry *context)
229 context->hi |= (1ull << 3);
232 static inline bool context_copied(struct context_entry *context)
234 return !!(context->hi & (1ULL << 3));
237 static inline bool __context_present(struct context_entry *context)
239 return (context->lo & 1);
242 bool context_present(struct context_entry *context)
244 return context_pasid_enabled(context) ?
245 __context_present(context) :
246 __context_present(context) && !context_copied(context);
249 static inline void context_set_present(struct context_entry *context)
254 static inline void context_set_fault_enable(struct context_entry *context)
256 context->lo &= (((u64)-1) << 2) | 1;
259 static inline void context_set_translation_type(struct context_entry *context,
262 context->lo &= (((u64)-1) << 4) | 3;
263 context->lo |= (value & 3) << 2;
266 static inline void context_set_address_root(struct context_entry *context,
269 context->lo &= ~VTD_PAGE_MASK;
270 context->lo |= value & VTD_PAGE_MASK;
273 static inline void context_set_address_width(struct context_entry *context,
276 context->hi |= value & 7;
279 static inline void context_set_domain_id(struct context_entry *context,
282 context->hi |= (value & ((1 << 16) - 1)) << 8;
285 static inline int context_domain_id(struct context_entry *c)
287 return((c->hi >> 8) & 0xffff);
290 static inline void context_clear_entry(struct context_entry *context)
297 * This domain is a statically identity mapping domain.
298 * 1. This domain creats a static 1:1 mapping to all usable memory.
299 * 2. It maps to each iommu if successful.
300 * 3. Each iommu mapps to this domain if successful.
302 static struct dmar_domain *si_domain;
303 static int hw_pass_through = 1;
306 * Domain represents a virtual machine, more than one devices
307 * across iommus may be owned in one domain, e.g. kvm guest.
309 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
311 /* si_domain contains mulitple devices */
312 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
314 #define for_each_domain_iommu(idx, domain) \
315 for (idx = 0; idx < g_num_of_iommus; idx++) \
316 if (domain->iommu_refcnt[idx])
318 struct dmar_rmrr_unit {
319 struct list_head list; /* list of rmrr units */
320 struct acpi_dmar_header *hdr; /* ACPI header */
321 u64 base_address; /* reserved base address*/
322 u64 end_address; /* reserved end address */
323 struct dmar_dev_scope *devices; /* target devices */
324 int devices_cnt; /* target device count */
325 struct iommu_resv_region *resv; /* reserved region handle */
328 struct dmar_atsr_unit {
329 struct list_head list; /* list of ATSR units */
330 struct acpi_dmar_header *hdr; /* ACPI header */
331 struct dmar_dev_scope *devices; /* target devices */
332 int devices_cnt; /* target device count */
333 u8 include_all:1; /* include all ports */
336 static LIST_HEAD(dmar_atsr_units);
337 static LIST_HEAD(dmar_rmrr_units);
339 #define for_each_rmrr_units(rmrr) \
340 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
342 /* bitmap for indexing intel_iommus */
343 static int g_num_of_iommus;
345 static void domain_exit(struct dmar_domain *domain);
346 static void domain_remove_dev_info(struct dmar_domain *domain);
347 static void dmar_remove_one_dev_info(struct device *dev);
348 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
349 static void domain_context_clear(struct intel_iommu *iommu,
351 static int domain_detach_iommu(struct dmar_domain *domain,
352 struct intel_iommu *iommu);
354 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
355 int dmar_disabled = 0;
357 int dmar_disabled = 1;
358 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
361 int intel_iommu_enabled = 0;
362 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
364 static int dmar_map_gfx = 1;
365 static int dmar_forcedac;
366 static int intel_iommu_strict;
367 static int intel_iommu_superpage = 1;
368 static int iommu_identity_mapping;
370 #define IDENTMAP_ALL 1
371 #define IDENTMAP_GFX 2
372 #define IDENTMAP_AZALIA 4
374 int intel_iommu_gfx_mapped;
375 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
377 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
378 static DEFINE_SPINLOCK(device_domain_lock);
379 static LIST_HEAD(device_domain_list);
382 * Iterate over elements in device_domain_list and call the specified
383 * callback @fn against each element.
385 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
386 void *data), void *data)
390 struct device_domain_info *info;
392 spin_lock_irqsave(&device_domain_lock, flags);
393 list_for_each_entry(info, &device_domain_list, global) {
394 ret = fn(info, data);
396 spin_unlock_irqrestore(&device_domain_lock, flags);
400 spin_unlock_irqrestore(&device_domain_lock, flags);
405 const struct iommu_ops intel_iommu_ops;
407 static bool translation_pre_enabled(struct intel_iommu *iommu)
409 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
412 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
414 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
417 static void init_translation_status(struct intel_iommu *iommu)
421 gsts = readl(iommu->reg + DMAR_GSTS_REG);
422 if (gsts & DMA_GSTS_TES)
423 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
426 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
427 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
429 return container_of(dom, struct dmar_domain, domain);
432 static int __init intel_iommu_setup(char *str)
437 if (!strncmp(str, "on", 2)) {
439 pr_info("IOMMU enabled\n");
440 } else if (!strncmp(str, "off", 3)) {
442 no_platform_optin = 1;
443 pr_info("IOMMU disabled\n");
444 } else if (!strncmp(str, "igfx_off", 8)) {
446 pr_info("Disable GFX device mapping\n");
447 } else if (!strncmp(str, "forcedac", 8)) {
448 pr_info("Forcing DAC for PCI devices\n");
450 } else if (!strncmp(str, "strict", 6)) {
451 pr_info("Disable batched IOTLB flush\n");
452 intel_iommu_strict = 1;
453 } else if (!strncmp(str, "sp_off", 6)) {
454 pr_info("Disable supported super page\n");
455 intel_iommu_superpage = 0;
456 } else if (!strncmp(str, "sm_on", 5)) {
457 pr_info("Intel-IOMMU: scalable mode supported\n");
459 } else if (!strncmp(str, "tboot_noforce", 13)) {
461 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
462 intel_iommu_tboot_noforce = 1;
465 str += strcspn(str, ",");
471 __setup("intel_iommu=", intel_iommu_setup);
473 static struct kmem_cache *iommu_domain_cache;
474 static struct kmem_cache *iommu_devinfo_cache;
476 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
478 struct dmar_domain **domains;
481 domains = iommu->domains[idx];
485 return domains[did & 0xff];
488 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
489 struct dmar_domain *domain)
491 struct dmar_domain **domains;
494 if (!iommu->domains[idx]) {
495 size_t size = 256 * sizeof(struct dmar_domain *);
496 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
499 domains = iommu->domains[idx];
500 if (WARN_ON(!domains))
503 domains[did & 0xff] = domain;
506 void *alloc_pgtable_page(int node)
511 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
513 vaddr = page_address(page);
517 void free_pgtable_page(void *vaddr)
519 free_page((unsigned long)vaddr);
522 static inline void *alloc_domain_mem(void)
524 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
527 static void free_domain_mem(void *vaddr)
529 kmem_cache_free(iommu_domain_cache, vaddr);
532 static inline void * alloc_devinfo_mem(void)
534 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
537 static inline void free_devinfo_mem(void *vaddr)
539 kmem_cache_free(iommu_devinfo_cache, vaddr);
542 static inline int domain_type_is_vm(struct dmar_domain *domain)
544 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
547 static inline int domain_type_is_si(struct dmar_domain *domain)
549 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
552 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
554 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
555 DOMAIN_FLAG_STATIC_IDENTITY);
558 static inline int domain_pfn_supported(struct dmar_domain *domain,
561 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
563 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
566 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
571 sagaw = cap_sagaw(iommu->cap);
572 for (agaw = width_to_agaw(max_gaw);
574 if (test_bit(agaw, &sagaw))
582 * Calculate max SAGAW for each iommu.
584 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
586 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
590 * calculate agaw for each iommu.
591 * "SAGAW" may be different across iommus, use a default agaw, and
592 * get a supported less agaw for iommus that don't support the default agaw.
594 int iommu_calculate_agaw(struct intel_iommu *iommu)
596 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
599 /* This functionin only returns single iommu in a domain */
600 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
604 /* si_domain and vm domain should not get here. */
605 BUG_ON(domain_type_is_vm_or_si(domain));
606 for_each_domain_iommu(iommu_id, domain)
609 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
612 return g_iommus[iommu_id];
615 static void domain_update_iommu_coherency(struct dmar_domain *domain)
617 struct dmar_drhd_unit *drhd;
618 struct intel_iommu *iommu;
622 domain->iommu_coherency = 1;
624 for_each_domain_iommu(i, domain) {
626 if (!ecap_coherent(g_iommus[i]->ecap)) {
627 domain->iommu_coherency = 0;
634 /* No hardware attached; use lowest common denominator */
636 for_each_active_iommu(iommu, drhd) {
637 if (!ecap_coherent(iommu->ecap)) {
638 domain->iommu_coherency = 0;
645 static int domain_update_iommu_snooping(struct intel_iommu *skip)
647 struct dmar_drhd_unit *drhd;
648 struct intel_iommu *iommu;
652 for_each_active_iommu(iommu, drhd) {
654 if (!ecap_sc_support(iommu->ecap)) {
665 static int domain_update_iommu_superpage(struct intel_iommu *skip)
667 struct dmar_drhd_unit *drhd;
668 struct intel_iommu *iommu;
671 if (!intel_iommu_superpage) {
675 /* set iommu_superpage to the smallest common denominator */
677 for_each_active_iommu(iommu, drhd) {
679 mask &= cap_super_page_val(iommu->cap);
689 /* Some capabilities may be different across iommus */
690 static void domain_update_iommu_cap(struct dmar_domain *domain)
692 domain_update_iommu_coherency(domain);
693 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
694 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
697 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
700 struct root_entry *root = &iommu->root_entry[bus];
701 struct context_entry *context;
705 if (sm_supported(iommu)) {
713 context = phys_to_virt(*entry & VTD_PAGE_MASK);
715 unsigned long phy_addr;
719 context = alloc_pgtable_page(iommu->node);
723 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
724 phy_addr = virt_to_phys((void *)context);
725 *entry = phy_addr | 1;
726 __iommu_flush_cache(iommu, entry, sizeof(*entry));
728 return &context[devfn];
731 static int iommu_dummy(struct device *dev)
733 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
736 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
738 struct dmar_drhd_unit *drhd = NULL;
739 struct intel_iommu *iommu;
741 struct pci_dev *ptmp, *pdev = NULL;
745 if (iommu_dummy(dev))
748 if (dev_is_pci(dev)) {
749 struct pci_dev *pf_pdev;
751 pdev = to_pci_dev(dev);
754 /* VMD child devices currently cannot be handled individually */
755 if (is_vmd(pdev->bus))
759 /* VFs aren't listed in scope tables; we need to look up
760 * the PF instead to find the IOMMU. */
761 pf_pdev = pci_physfn(pdev);
763 segment = pci_domain_nr(pdev->bus);
764 } else if (has_acpi_companion(dev))
765 dev = &ACPI_COMPANION(dev)->dev;
768 for_each_active_iommu(iommu, drhd) {
769 if (pdev && segment != drhd->segment)
772 for_each_active_dev_scope(drhd->devices,
773 drhd->devices_cnt, i, tmp) {
775 /* For a VF use its original BDF# not that of the PF
776 * which we used for the IOMMU lookup. Strictly speaking
777 * we could do this for all PCI devices; we only need to
778 * get the BDF# from the scope table for ACPI matches. */
779 if (pdev && pdev->is_virtfn)
782 *bus = drhd->devices[i].bus;
783 *devfn = drhd->devices[i].devfn;
787 if (!pdev || !dev_is_pci(tmp))
790 ptmp = to_pci_dev(tmp);
791 if (ptmp->subordinate &&
792 ptmp->subordinate->number <= pdev->bus->number &&
793 ptmp->subordinate->busn_res.end >= pdev->bus->number)
797 if (pdev && drhd->include_all) {
799 *bus = pdev->bus->number;
800 *devfn = pdev->devfn;
811 static void domain_flush_cache(struct dmar_domain *domain,
812 void *addr, int size)
814 if (!domain->iommu_coherency)
815 clflush_cache_range(addr, size);
818 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
820 struct context_entry *context;
824 spin_lock_irqsave(&iommu->lock, flags);
825 context = iommu_context_addr(iommu, bus, devfn, 0);
827 ret = context_present(context);
828 spin_unlock_irqrestore(&iommu->lock, flags);
832 static void free_context_table(struct intel_iommu *iommu)
836 struct context_entry *context;
838 spin_lock_irqsave(&iommu->lock, flags);
839 if (!iommu->root_entry) {
842 for (i = 0; i < ROOT_ENTRY_NR; i++) {
843 context = iommu_context_addr(iommu, i, 0, 0);
845 free_pgtable_page(context);
847 if (!sm_supported(iommu))
850 context = iommu_context_addr(iommu, i, 0x80, 0);
852 free_pgtable_page(context);
855 free_pgtable_page(iommu->root_entry);
856 iommu->root_entry = NULL;
858 spin_unlock_irqrestore(&iommu->lock, flags);
861 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
862 unsigned long pfn, int *target_level)
864 struct dma_pte *parent, *pte;
865 int level = agaw_to_level(domain->agaw);
868 BUG_ON(!domain->pgd);
870 if (!domain_pfn_supported(domain, pfn))
871 /* Address beyond IOMMU's addressing capabilities. */
874 parent = domain->pgd;
879 offset = pfn_level_offset(pfn, level);
880 pte = &parent[offset];
881 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
883 if (level == *target_level)
886 if (!dma_pte_present(pte)) {
889 tmp_page = alloc_pgtable_page(domain->nid);
894 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
895 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
896 if (cmpxchg64(&pte->val, 0ULL, pteval))
897 /* Someone else set it while we were thinking; use theirs. */
898 free_pgtable_page(tmp_page);
900 domain_flush_cache(domain, pte, sizeof(*pte));
905 parent = phys_to_virt(dma_pte_addr(pte));
910 *target_level = level;
916 /* return address's pte at specific level */
917 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
919 int level, int *large_page)
921 struct dma_pte *parent, *pte;
922 int total = agaw_to_level(domain->agaw);
925 parent = domain->pgd;
926 while (level <= total) {
927 offset = pfn_level_offset(pfn, total);
928 pte = &parent[offset];
932 if (!dma_pte_present(pte)) {
937 if (dma_pte_superpage(pte)) {
942 parent = phys_to_virt(dma_pte_addr(pte));
948 /* clear last level pte, a tlb flush should be followed */
949 static void dma_pte_clear_range(struct dmar_domain *domain,
950 unsigned long start_pfn,
951 unsigned long last_pfn)
953 unsigned int large_page;
954 struct dma_pte *first_pte, *pte;
956 BUG_ON(!domain_pfn_supported(domain, start_pfn));
957 BUG_ON(!domain_pfn_supported(domain, last_pfn));
958 BUG_ON(start_pfn > last_pfn);
960 /* we don't need lock here; nobody else touches the iova range */
963 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
965 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
970 start_pfn += lvl_to_nr_pages(large_page);
972 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
974 domain_flush_cache(domain, first_pte,
975 (void *)pte - (void *)first_pte);
977 } while (start_pfn && start_pfn <= last_pfn);
980 static void dma_pte_free_level(struct dmar_domain *domain, int level,
981 int retain_level, struct dma_pte *pte,
982 unsigned long pfn, unsigned long start_pfn,
983 unsigned long last_pfn)
985 pfn = max(start_pfn, pfn);
986 pte = &pte[pfn_level_offset(pfn, level)];
989 unsigned long level_pfn;
990 struct dma_pte *level_pte;
992 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
995 level_pfn = pfn & level_mask(level);
996 level_pte = phys_to_virt(dma_pte_addr(pte));
999 dma_pte_free_level(domain, level - 1, retain_level,
1000 level_pte, level_pfn, start_pfn,
1005 * Free the page table if we're below the level we want to
1006 * retain and the range covers the entire table.
1008 if (level < retain_level && !(start_pfn > level_pfn ||
1009 last_pfn < level_pfn + level_size(level) - 1)) {
1011 domain_flush_cache(domain, pte, sizeof(*pte));
1012 free_pgtable_page(level_pte);
1015 pfn += level_size(level);
1016 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1020 * clear last level (leaf) ptes and free page table pages below the
1021 * level we wish to keep intact.
1023 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1024 unsigned long start_pfn,
1025 unsigned long last_pfn,
1028 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1029 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1030 BUG_ON(start_pfn > last_pfn);
1032 dma_pte_clear_range(domain, start_pfn, last_pfn);
1034 /* We don't need lock here; nobody else touches the iova range */
1035 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1036 domain->pgd, 0, start_pfn, last_pfn);
1039 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1040 free_pgtable_page(domain->pgd);
1045 /* When a page at a given level is being unlinked from its parent, we don't
1046 need to *modify* it at all. All we need to do is make a list of all the
1047 pages which can be freed just as soon as we've flushed the IOTLB and we
1048 know the hardware page-walk will no longer touch them.
1049 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1051 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1052 int level, struct dma_pte *pte,
1053 struct page *freelist)
1057 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1058 pg->freelist = freelist;
1064 pte = page_address(pg);
1066 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1067 freelist = dma_pte_list_pagetables(domain, level - 1,
1070 } while (!first_pte_in_page(pte));
1075 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1076 struct dma_pte *pte, unsigned long pfn,
1077 unsigned long start_pfn,
1078 unsigned long last_pfn,
1079 struct page *freelist)
1081 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1083 pfn = max(start_pfn, pfn);
1084 pte = &pte[pfn_level_offset(pfn, level)];
1087 unsigned long level_pfn;
1089 if (!dma_pte_present(pte))
1092 level_pfn = pfn & level_mask(level);
1094 /* If range covers entire pagetable, free it */
1095 if (start_pfn <= level_pfn &&
1096 last_pfn >= level_pfn + level_size(level) - 1) {
1097 /* These suborbinate page tables are going away entirely. Don't
1098 bother to clear them; we're just going to *free* them. */
1099 if (level > 1 && !dma_pte_superpage(pte))
1100 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1106 } else if (level > 1) {
1107 /* Recurse down into a level that isn't *entirely* obsolete */
1108 freelist = dma_pte_clear_level(domain, level - 1,
1109 phys_to_virt(dma_pte_addr(pte)),
1110 level_pfn, start_pfn, last_pfn,
1114 pfn += level_size(level);
1115 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1118 domain_flush_cache(domain, first_pte,
1119 (void *)++last_pte - (void *)first_pte);
1124 /* We can't just free the pages because the IOMMU may still be walking
1125 the page tables, and may have cached the intermediate levels. The
1126 pages can only be freed after the IOTLB flush has been done. */
1127 static struct page *domain_unmap(struct dmar_domain *domain,
1128 unsigned long start_pfn,
1129 unsigned long last_pfn)
1131 struct page *freelist;
1133 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1134 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1135 BUG_ON(start_pfn > last_pfn);
1137 /* we don't need lock here; nobody else touches the iova range */
1138 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1139 domain->pgd, 0, start_pfn, last_pfn, NULL);
1142 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1143 struct page *pgd_page = virt_to_page(domain->pgd);
1144 pgd_page->freelist = freelist;
1145 freelist = pgd_page;
1153 static void dma_free_pagelist(struct page *freelist)
1157 while ((pg = freelist)) {
1158 freelist = pg->freelist;
1159 free_pgtable_page(page_address(pg));
1163 static void iova_entry_free(unsigned long data)
1165 struct page *freelist = (struct page *)data;
1167 dma_free_pagelist(freelist);
1170 /* iommu handling */
1171 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1173 struct root_entry *root;
1174 unsigned long flags;
1176 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1178 pr_err("Allocating root entry for %s failed\n",
1183 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1185 spin_lock_irqsave(&iommu->lock, flags);
1186 iommu->root_entry = root;
1187 spin_unlock_irqrestore(&iommu->lock, flags);
1192 static void iommu_set_root_entry(struct intel_iommu *iommu)
1198 addr = virt_to_phys(iommu->root_entry);
1199 if (sm_supported(iommu))
1200 addr |= DMA_RTADDR_SMT;
1202 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1203 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1205 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1207 /* Make sure hardware complete it */
1208 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1209 readl, (sts & DMA_GSTS_RTPS), sts);
1211 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1214 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1219 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1222 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1223 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1225 /* Make sure hardware complete it */
1226 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1227 readl, (!(val & DMA_GSTS_WBFS)), val);
1229 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1232 /* return value determine if we need a write buffer flush */
1233 static void __iommu_flush_context(struct intel_iommu *iommu,
1234 u16 did, u16 source_id, u8 function_mask,
1241 case DMA_CCMD_GLOBAL_INVL:
1242 val = DMA_CCMD_GLOBAL_INVL;
1244 case DMA_CCMD_DOMAIN_INVL:
1245 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1247 case DMA_CCMD_DEVICE_INVL:
1248 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1249 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1254 val |= DMA_CCMD_ICC;
1256 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1257 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1259 /* Make sure hardware complete it */
1260 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1261 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1263 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1266 /* return value determine if we need a write buffer flush */
1267 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1268 u64 addr, unsigned int size_order, u64 type)
1270 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1271 u64 val = 0, val_iva = 0;
1275 case DMA_TLB_GLOBAL_FLUSH:
1276 /* global flush doesn't need set IVA_REG */
1277 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1279 case DMA_TLB_DSI_FLUSH:
1280 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1282 case DMA_TLB_PSI_FLUSH:
1283 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1284 /* IH bit is passed in as part of address */
1285 val_iva = size_order | addr;
1290 /* Note: set drain read/write */
1293 * This is probably to be super secure.. Looks like we can
1294 * ignore it without any impact.
1296 if (cap_read_drain(iommu->cap))
1297 val |= DMA_TLB_READ_DRAIN;
1299 if (cap_write_drain(iommu->cap))
1300 val |= DMA_TLB_WRITE_DRAIN;
1302 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1303 /* Note: Only uses first TLB reg currently */
1305 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1306 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1308 /* Make sure hardware complete it */
1309 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1310 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1312 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1314 /* check IOTLB invalidation granularity */
1315 if (DMA_TLB_IAIG(val) == 0)
1316 pr_err("Flush IOTLB failed\n");
1317 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1318 pr_debug("TLB flush request %Lx, actual %Lx\n",
1319 (unsigned long long)DMA_TLB_IIRG(type),
1320 (unsigned long long)DMA_TLB_IAIG(val));
1323 static struct device_domain_info *
1324 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1327 struct device_domain_info *info;
1329 assert_spin_locked(&device_domain_lock);
1334 list_for_each_entry(info, &domain->devices, link)
1335 if (info->iommu == iommu && info->bus == bus &&
1336 info->devfn == devfn) {
1337 if (info->ats_supported && info->dev)
1345 static void domain_update_iotlb(struct dmar_domain *domain)
1347 struct device_domain_info *info;
1348 bool has_iotlb_device = false;
1350 assert_spin_locked(&device_domain_lock);
1352 list_for_each_entry(info, &domain->devices, link) {
1353 struct pci_dev *pdev;
1355 if (!info->dev || !dev_is_pci(info->dev))
1358 pdev = to_pci_dev(info->dev);
1359 if (pdev->ats_enabled) {
1360 has_iotlb_device = true;
1365 domain->has_iotlb_device = has_iotlb_device;
1368 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1370 struct pci_dev *pdev;
1372 assert_spin_locked(&device_domain_lock);
1374 if (!info || !dev_is_pci(info->dev))
1377 pdev = to_pci_dev(info->dev);
1378 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1379 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1380 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1381 * reserved, which should be set to 0.
1383 if (!ecap_dit(info->iommu->ecap))
1386 struct pci_dev *pf_pdev;
1388 /* pdev will be returned if device is not a vf */
1389 pf_pdev = pci_physfn(pdev);
1390 info->pfsid = pci_dev_id(pf_pdev);
1393 #ifdef CONFIG_INTEL_IOMMU_SVM
1394 /* The PCIe spec, in its wisdom, declares that the behaviour of
1395 the device if you enable PASID support after ATS support is
1396 undefined. So always enable PASID support on devices which
1397 have it, even if we can't yet know if we're ever going to
1399 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1400 info->pasid_enabled = 1;
1402 if (info->pri_supported &&
1403 (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
1404 !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1405 info->pri_enabled = 1;
1407 if (!pdev->untrusted && info->ats_supported &&
1408 pci_ats_page_aligned(pdev) &&
1409 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1410 info->ats_enabled = 1;
1411 domain_update_iotlb(info->domain);
1412 info->ats_qdep = pci_ats_queue_depth(pdev);
1416 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1418 struct pci_dev *pdev;
1420 assert_spin_locked(&device_domain_lock);
1422 if (!dev_is_pci(info->dev))
1425 pdev = to_pci_dev(info->dev);
1427 if (info->ats_enabled) {
1428 pci_disable_ats(pdev);
1429 info->ats_enabled = 0;
1430 domain_update_iotlb(info->domain);
1432 #ifdef CONFIG_INTEL_IOMMU_SVM
1433 if (info->pri_enabled) {
1434 pci_disable_pri(pdev);
1435 info->pri_enabled = 0;
1437 if (info->pasid_enabled) {
1438 pci_disable_pasid(pdev);
1439 info->pasid_enabled = 0;
1444 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1445 u64 addr, unsigned mask)
1448 unsigned long flags;
1449 struct device_domain_info *info;
1451 if (!domain->has_iotlb_device)
1454 spin_lock_irqsave(&device_domain_lock, flags);
1455 list_for_each_entry(info, &domain->devices, link) {
1456 if (!info->ats_enabled)
1459 sid = info->bus << 8 | info->devfn;
1460 qdep = info->ats_qdep;
1461 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1464 spin_unlock_irqrestore(&device_domain_lock, flags);
1467 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1468 struct dmar_domain *domain,
1469 unsigned long pfn, unsigned int pages,
1472 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1473 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1474 u16 did = domain->iommu_did[iommu->seq_id];
1481 * Fallback to domain selective flush if no PSI support or the size is
1483 * PSI requires page size to be 2 ^ x, and the base address is naturally
1484 * aligned to the size
1486 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1487 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1490 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1494 * In caching mode, changes of pages from non-present to present require
1495 * flush. However, device IOTLB doesn't need to be flushed in this case.
1497 if (!cap_caching_mode(iommu->cap) || !map)
1498 iommu_flush_dev_iotlb(domain, addr, mask);
1501 /* Notification for newly created mappings */
1502 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1503 struct dmar_domain *domain,
1504 unsigned long pfn, unsigned int pages)
1506 /* It's a non-present to present mapping. Only flush if caching mode */
1507 if (cap_caching_mode(iommu->cap))
1508 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1510 iommu_flush_write_buffer(iommu);
1513 static void iommu_flush_iova(struct iova_domain *iovad)
1515 struct dmar_domain *domain;
1518 domain = container_of(iovad, struct dmar_domain, iovad);
1520 for_each_domain_iommu(idx, domain) {
1521 struct intel_iommu *iommu = g_iommus[idx];
1522 u16 did = domain->iommu_did[iommu->seq_id];
1524 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1526 if (!cap_caching_mode(iommu->cap))
1527 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1528 0, MAX_AGAW_PFN_WIDTH);
1532 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1535 unsigned long flags;
1537 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1540 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1541 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1542 pmen &= ~DMA_PMEN_EPM;
1543 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1545 /* wait for the protected region status bit to clear */
1546 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1547 readl, !(pmen & DMA_PMEN_PRS), pmen);
1549 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1552 static void iommu_enable_translation(struct intel_iommu *iommu)
1555 unsigned long flags;
1557 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1558 iommu->gcmd |= DMA_GCMD_TE;
1559 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1561 /* Make sure hardware complete it */
1562 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1563 readl, (sts & DMA_GSTS_TES), sts);
1565 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1568 static void iommu_disable_translation(struct intel_iommu *iommu)
1573 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1574 iommu->gcmd &= ~DMA_GCMD_TE;
1575 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1577 /* Make sure hardware complete it */
1578 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1579 readl, (!(sts & DMA_GSTS_TES)), sts);
1581 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1585 static int iommu_init_domains(struct intel_iommu *iommu)
1587 u32 ndomains, nlongs;
1590 ndomains = cap_ndoms(iommu->cap);
1591 pr_debug("%s: Number of Domains supported <%d>\n",
1592 iommu->name, ndomains);
1593 nlongs = BITS_TO_LONGS(ndomains);
1595 spin_lock_init(&iommu->lock);
1597 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1598 if (!iommu->domain_ids) {
1599 pr_err("%s: Allocating domain id array failed\n",
1604 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1605 iommu->domains = kzalloc(size, GFP_KERNEL);
1607 if (iommu->domains) {
1608 size = 256 * sizeof(struct dmar_domain *);
1609 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1612 if (!iommu->domains || !iommu->domains[0]) {
1613 pr_err("%s: Allocating domain array failed\n",
1615 kfree(iommu->domain_ids);
1616 kfree(iommu->domains);
1617 iommu->domain_ids = NULL;
1618 iommu->domains = NULL;
1625 * If Caching mode is set, then invalid translations are tagged
1626 * with domain-id 0, hence we need to pre-allocate it. We also
1627 * use domain-id 0 as a marker for non-allocated domain-id, so
1628 * make sure it is not used for a real domain.
1630 set_bit(0, iommu->domain_ids);
1633 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1634 * entry for first-level or pass-through translation modes should
1635 * be programmed with a domain id different from those used for
1636 * second-level or nested translation. We reserve a domain id for
1639 if (sm_supported(iommu))
1640 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1645 static void disable_dmar_iommu(struct intel_iommu *iommu)
1647 struct device_domain_info *info, *tmp;
1648 unsigned long flags;
1650 if (!iommu->domains || !iommu->domain_ids)
1654 spin_lock_irqsave(&device_domain_lock, flags);
1655 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1656 struct dmar_domain *domain;
1658 if (info->iommu != iommu)
1661 if (!info->dev || !info->domain)
1664 domain = info->domain;
1666 __dmar_remove_one_dev_info(info);
1668 if (!domain_type_is_vm_or_si(domain)) {
1670 * The domain_exit() function can't be called under
1671 * device_domain_lock, as it takes this lock itself.
1672 * So release the lock here and re-run the loop
1675 spin_unlock_irqrestore(&device_domain_lock, flags);
1676 domain_exit(domain);
1680 spin_unlock_irqrestore(&device_domain_lock, flags);
1682 if (iommu->gcmd & DMA_GCMD_TE)
1683 iommu_disable_translation(iommu);
1686 static void free_dmar_iommu(struct intel_iommu *iommu)
1688 if ((iommu->domains) && (iommu->domain_ids)) {
1689 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1692 for (i = 0; i < elems; i++)
1693 kfree(iommu->domains[i]);
1694 kfree(iommu->domains);
1695 kfree(iommu->domain_ids);
1696 iommu->domains = NULL;
1697 iommu->domain_ids = NULL;
1700 g_iommus[iommu->seq_id] = NULL;
1702 /* free context mapping */
1703 free_context_table(iommu);
1705 #ifdef CONFIG_INTEL_IOMMU_SVM
1706 if (pasid_supported(iommu)) {
1707 if (ecap_prs(iommu->ecap))
1708 intel_svm_finish_prq(iommu);
1713 static struct dmar_domain *alloc_domain(int flags)
1715 struct dmar_domain *domain;
1717 domain = alloc_domain_mem();
1721 memset(domain, 0, sizeof(*domain));
1722 domain->nid = NUMA_NO_NODE;
1723 domain->flags = flags;
1724 domain->has_iotlb_device = false;
1725 INIT_LIST_HEAD(&domain->devices);
1730 /* Must be called with iommu->lock */
1731 static int domain_attach_iommu(struct dmar_domain *domain,
1732 struct intel_iommu *iommu)
1734 unsigned long ndomains;
1737 assert_spin_locked(&device_domain_lock);
1738 assert_spin_locked(&iommu->lock);
1740 domain->iommu_refcnt[iommu->seq_id] += 1;
1741 domain->iommu_count += 1;
1742 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1743 ndomains = cap_ndoms(iommu->cap);
1744 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1746 if (num >= ndomains) {
1747 pr_err("%s: No free domain ids\n", iommu->name);
1748 domain->iommu_refcnt[iommu->seq_id] -= 1;
1749 domain->iommu_count -= 1;
1753 set_bit(num, iommu->domain_ids);
1754 set_iommu_domain(iommu, num, domain);
1756 domain->iommu_did[iommu->seq_id] = num;
1757 domain->nid = iommu->node;
1759 domain_update_iommu_cap(domain);
1765 static int domain_detach_iommu(struct dmar_domain *domain,
1766 struct intel_iommu *iommu)
1770 assert_spin_locked(&device_domain_lock);
1771 assert_spin_locked(&iommu->lock);
1773 domain->iommu_refcnt[iommu->seq_id] -= 1;
1774 count = --domain->iommu_count;
1775 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1776 num = domain->iommu_did[iommu->seq_id];
1777 clear_bit(num, iommu->domain_ids);
1778 set_iommu_domain(iommu, num, NULL);
1780 domain_update_iommu_cap(domain);
1781 domain->iommu_did[iommu->seq_id] = 0;
1787 static struct iova_domain reserved_iova_list;
1788 static struct lock_class_key reserved_rbtree_key;
1790 static int dmar_init_reserved_ranges(void)
1792 struct pci_dev *pdev = NULL;
1796 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1798 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1799 &reserved_rbtree_key);
1801 /* IOAPIC ranges shouldn't be accessed by DMA */
1802 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1803 IOVA_PFN(IOAPIC_RANGE_END));
1805 pr_err("Reserve IOAPIC range failed\n");
1809 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1810 for_each_pci_dev(pdev) {
1813 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1814 r = &pdev->resource[i];
1815 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1817 iova = reserve_iova(&reserved_iova_list,
1821 pci_err(pdev, "Reserve iova for %pR failed\n", r);
1829 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1831 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1834 static inline int guestwidth_to_adjustwidth(int gaw)
1837 int r = (gaw - 12) % 9;
1848 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1851 int adjust_width, agaw;
1852 unsigned long sagaw;
1855 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1857 err = init_iova_flush_queue(&domain->iovad,
1858 iommu_flush_iova, iova_entry_free);
1862 domain_reserve_special_ranges(domain);
1864 /* calculate AGAW */
1865 if (guest_width > cap_mgaw(iommu->cap))
1866 guest_width = cap_mgaw(iommu->cap);
1867 domain->gaw = guest_width;
1868 adjust_width = guestwidth_to_adjustwidth(guest_width);
1869 agaw = width_to_agaw(adjust_width);
1870 sagaw = cap_sagaw(iommu->cap);
1871 if (!test_bit(agaw, &sagaw)) {
1872 /* hardware doesn't support it, choose a bigger one */
1873 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1874 agaw = find_next_bit(&sagaw, 5, agaw);
1878 domain->agaw = agaw;
1880 if (ecap_coherent(iommu->ecap))
1881 domain->iommu_coherency = 1;
1883 domain->iommu_coherency = 0;
1885 if (ecap_sc_support(iommu->ecap))
1886 domain->iommu_snooping = 1;
1888 domain->iommu_snooping = 0;
1890 if (intel_iommu_superpage)
1891 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1893 domain->iommu_superpage = 0;
1895 domain->nid = iommu->node;
1897 /* always allocate the top pgd */
1898 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1901 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1905 static void domain_exit(struct dmar_domain *domain)
1907 struct page *freelist;
1909 /* Remove associated devices and clear attached or cached domains */
1910 domain_remove_dev_info(domain);
1913 put_iova_domain(&domain->iovad);
1915 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1917 dma_free_pagelist(freelist);
1919 free_domain_mem(domain);
1923 * Get the PASID directory size for scalable mode context entry.
1924 * Value of X in the PDTS field of a scalable mode context entry
1925 * indicates PASID directory with 2^(X + 7) entries.
1927 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1931 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1932 pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1940 * Set the RID_PASID field of a scalable mode context entry. The
1941 * IOMMU hardware will use the PASID value set in this field for
1942 * DMA translations of DMA requests without PASID.
1945 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1947 context->hi |= pasid & ((1 << 20) - 1);
1948 context->hi |= (1 << 20);
1952 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1955 static inline void context_set_sm_dte(struct context_entry *context)
1957 context->lo |= (1 << 2);
1961 * Set the PRE(Page Request Enable) field of a scalable mode context
1964 static inline void context_set_sm_pre(struct context_entry *context)
1966 context->lo |= (1 << 4);
1969 /* Convert value to context PASID directory size field coding. */
1970 #define context_pdts(pds) (((pds) & 0x7) << 9)
1972 static int domain_context_mapping_one(struct dmar_domain *domain,
1973 struct intel_iommu *iommu,
1974 struct pasid_table *table,
1977 u16 did = domain->iommu_did[iommu->seq_id];
1978 int translation = CONTEXT_TT_MULTI_LEVEL;
1979 struct device_domain_info *info = NULL;
1980 struct context_entry *context;
1981 unsigned long flags;
1986 if (hw_pass_through && domain_type_is_si(domain))
1987 translation = CONTEXT_TT_PASS_THROUGH;
1989 pr_debug("Set context mapping for %02x:%02x.%d\n",
1990 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1992 BUG_ON(!domain->pgd);
1994 spin_lock_irqsave(&device_domain_lock, flags);
1995 spin_lock(&iommu->lock);
1998 context = iommu_context_addr(iommu, bus, devfn, 1);
2003 if (context_present(context))
2007 * For kdump cases, old valid entries may be cached due to the
2008 * in-flight DMA and copied pgtable, but there is no unmapping
2009 * behaviour for them, thus we need an explicit cache flush for
2010 * the newly-mapped device. For kdump, at this point, the device
2011 * is supposed to finish reset at its driver probe stage, so no
2012 * in-flight DMA will exist, and we don't need to worry anymore
2015 if (context_copied(context)) {
2016 u16 did_old = context_domain_id(context);
2018 if (did_old < cap_ndoms(iommu->cap)) {
2019 iommu->flush.flush_context(iommu, did_old,
2020 (((u16)bus) << 8) | devfn,
2021 DMA_CCMD_MASK_NOBIT,
2022 DMA_CCMD_DEVICE_INVL);
2023 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2028 context_clear_entry(context);
2030 if (sm_supported(iommu)) {
2035 /* Setup the PASID DIR pointer: */
2036 pds = context_get_sm_pds(table);
2037 context->lo = (u64)virt_to_phys(table->table) |
2040 /* Setup the RID_PASID field: */
2041 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2044 * Setup the Device-TLB enable bit and Page request
2047 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2048 if (info && info->ats_supported)
2049 context_set_sm_dte(context);
2050 if (info && info->pri_supported)
2051 context_set_sm_pre(context);
2053 struct dma_pte *pgd = domain->pgd;
2056 context_set_domain_id(context, did);
2058 if (translation != CONTEXT_TT_PASS_THROUGH) {
2060 * Skip top levels of page tables for iommu which has
2061 * less agaw than default. Unnecessary for PT mode.
2063 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2065 pgd = phys_to_virt(dma_pte_addr(pgd));
2066 if (!dma_pte_present(pgd))
2070 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2071 if (info && info->ats_supported)
2072 translation = CONTEXT_TT_DEV_IOTLB;
2074 translation = CONTEXT_TT_MULTI_LEVEL;
2076 context_set_address_root(context, virt_to_phys(pgd));
2077 context_set_address_width(context, agaw);
2080 * In pass through mode, AW must be programmed to
2081 * indicate the largest AGAW value supported by
2082 * hardware. And ASR is ignored by hardware.
2084 context_set_address_width(context, iommu->msagaw);
2087 context_set_translation_type(context, translation);
2090 context_set_fault_enable(context);
2091 context_set_present(context);
2092 domain_flush_cache(domain, context, sizeof(*context));
2095 * It's a non-present to present mapping. If hardware doesn't cache
2096 * non-present entry we only need to flush the write-buffer. If the
2097 * _does_ cache non-present entries, then it does so in the special
2098 * domain #0, which we have to flush:
2100 if (cap_caching_mode(iommu->cap)) {
2101 iommu->flush.flush_context(iommu, 0,
2102 (((u16)bus) << 8) | devfn,
2103 DMA_CCMD_MASK_NOBIT,
2104 DMA_CCMD_DEVICE_INVL);
2105 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2107 iommu_flush_write_buffer(iommu);
2109 iommu_enable_dev_iotlb(info);
2114 spin_unlock(&iommu->lock);
2115 spin_unlock_irqrestore(&device_domain_lock, flags);
2120 struct domain_context_mapping_data {
2121 struct dmar_domain *domain;
2122 struct intel_iommu *iommu;
2123 struct pasid_table *table;
2126 static int domain_context_mapping_cb(struct pci_dev *pdev,
2127 u16 alias, void *opaque)
2129 struct domain_context_mapping_data *data = opaque;
2131 return domain_context_mapping_one(data->domain, data->iommu,
2132 data->table, PCI_BUS_NUM(alias),
2137 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2139 struct domain_context_mapping_data data;
2140 struct pasid_table *table;
2141 struct intel_iommu *iommu;
2144 iommu = device_to_iommu(dev, &bus, &devfn);
2148 table = intel_pasid_get_table(dev);
2150 if (!dev_is_pci(dev))
2151 return domain_context_mapping_one(domain, iommu, table,
2154 data.domain = domain;
2158 return pci_for_each_dma_alias(to_pci_dev(dev),
2159 &domain_context_mapping_cb, &data);
2162 static int domain_context_mapped_cb(struct pci_dev *pdev,
2163 u16 alias, void *opaque)
2165 struct intel_iommu *iommu = opaque;
2167 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2170 static int domain_context_mapped(struct device *dev)
2172 struct intel_iommu *iommu;
2175 iommu = device_to_iommu(dev, &bus, &devfn);
2179 if (!dev_is_pci(dev))
2180 return device_context_mapped(iommu, bus, devfn);
2182 return !pci_for_each_dma_alias(to_pci_dev(dev),
2183 domain_context_mapped_cb, iommu);
2186 /* Returns a number of VTD pages, but aligned to MM page size */
2187 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2190 host_addr &= ~PAGE_MASK;
2191 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2194 /* Return largest possible superpage level for a given mapping */
2195 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2196 unsigned long iov_pfn,
2197 unsigned long phy_pfn,
2198 unsigned long pages)
2200 int support, level = 1;
2201 unsigned long pfnmerge;
2203 support = domain->iommu_superpage;
2205 /* To use a large page, the virtual *and* physical addresses
2206 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2207 of them will mean we have to use smaller pages. So just
2208 merge them and check both at once. */
2209 pfnmerge = iov_pfn | phy_pfn;
2211 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2212 pages >>= VTD_STRIDE_SHIFT;
2215 pfnmerge >>= VTD_STRIDE_SHIFT;
2222 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2223 struct scatterlist *sg, unsigned long phys_pfn,
2224 unsigned long nr_pages, int prot)
2226 struct dma_pte *first_pte = NULL, *pte = NULL;
2227 phys_addr_t uninitialized_var(pteval);
2228 unsigned long sg_res = 0;
2229 unsigned int largepage_lvl = 0;
2230 unsigned long lvl_pages = 0;
2232 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2234 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2237 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2241 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2244 while (nr_pages > 0) {
2248 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2250 sg_res = aligned_nrpages(sg->offset, sg->length);
2251 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2252 sg->dma_length = sg->length;
2253 pteval = (sg_phys(sg) - pgoff) | prot;
2254 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2258 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2260 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2263 /* It is large page*/
2264 if (largepage_lvl > 1) {
2265 unsigned long nr_superpages, end_pfn;
2267 pteval |= DMA_PTE_LARGE_PAGE;
2268 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2270 nr_superpages = sg_res / lvl_pages;
2271 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2274 * Ensure that old small page tables are
2275 * removed to make room for superpage(s).
2276 * We're adding new large pages, so make sure
2277 * we don't remove their parent tables.
2279 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2282 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2286 /* We don't need lock here, nobody else
2287 * touches the iova range
2289 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2291 static int dumps = 5;
2292 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2293 iov_pfn, tmp, (unsigned long long)pteval);
2296 debug_dma_dump_mappings(NULL);
2301 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2303 BUG_ON(nr_pages < lvl_pages);
2304 BUG_ON(sg_res < lvl_pages);
2306 nr_pages -= lvl_pages;
2307 iov_pfn += lvl_pages;
2308 phys_pfn += lvl_pages;
2309 pteval += lvl_pages * VTD_PAGE_SIZE;
2310 sg_res -= lvl_pages;
2312 /* If the next PTE would be the first in a new page, then we
2313 need to flush the cache on the entries we've just written.
2314 And then we'll need to recalculate 'pte', so clear it and
2315 let it get set again in the if (!pte) block above.
2317 If we're done (!nr_pages) we need to flush the cache too.
2319 Also if we've been setting superpages, we may need to
2320 recalculate 'pte' and switch back to smaller pages for the
2321 end of the mapping, if the trailing size is not enough to
2322 use another superpage (i.e. sg_res < lvl_pages). */
2324 if (!nr_pages || first_pte_in_page(pte) ||
2325 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2326 domain_flush_cache(domain, first_pte,
2327 (void *)pte - (void *)first_pte);
2331 if (!sg_res && nr_pages)
2337 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2338 struct scatterlist *sg, unsigned long phys_pfn,
2339 unsigned long nr_pages, int prot)
2342 struct intel_iommu *iommu;
2344 /* Do the real mapping first */
2345 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2349 /* Notify about the new mapping */
2350 if (domain_type_is_vm(domain)) {
2351 /* VM typed domains can have more than one IOMMUs */
2354 for_each_domain_iommu(iommu_id, domain) {
2355 iommu = g_iommus[iommu_id];
2356 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2359 /* General domains only have one IOMMU */
2360 iommu = domain_get_iommu(domain);
2361 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2367 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2368 struct scatterlist *sg, unsigned long nr_pages,
2371 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2374 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2375 unsigned long phys_pfn, unsigned long nr_pages,
2378 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2381 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2383 unsigned long flags;
2384 struct context_entry *context;
2390 spin_lock_irqsave(&iommu->lock, flags);
2391 context = iommu_context_addr(iommu, bus, devfn, 0);
2393 spin_unlock_irqrestore(&iommu->lock, flags);
2396 did_old = context_domain_id(context);
2397 context_clear_entry(context);
2398 __iommu_flush_cache(iommu, context, sizeof(*context));
2399 spin_unlock_irqrestore(&iommu->lock, flags);
2400 iommu->flush.flush_context(iommu,
2402 (((u16)bus) << 8) | devfn,
2403 DMA_CCMD_MASK_NOBIT,
2404 DMA_CCMD_DEVICE_INVL);
2405 iommu->flush.flush_iotlb(iommu,
2412 static inline void unlink_domain_info(struct device_domain_info *info)
2414 assert_spin_locked(&device_domain_lock);
2415 list_del(&info->link);
2416 list_del(&info->global);
2418 info->dev->archdata.iommu = NULL;
2421 static void domain_remove_dev_info(struct dmar_domain *domain)
2423 struct device_domain_info *info, *tmp;
2424 unsigned long flags;
2426 spin_lock_irqsave(&device_domain_lock, flags);
2427 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2428 __dmar_remove_one_dev_info(info);
2429 spin_unlock_irqrestore(&device_domain_lock, flags);
2434 * Note: we use struct device->archdata.iommu stores the info
2436 static struct dmar_domain *find_domain(struct device *dev)
2438 struct device_domain_info *info;
2440 /* No lock here, assumes no domain exit in normal case */
2441 info = dev->archdata.iommu;
2443 return info->domain;
2447 static inline struct device_domain_info *
2448 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2450 struct device_domain_info *info;
2452 list_for_each_entry(info, &device_domain_list, global)
2453 if (info->iommu->segment == segment && info->bus == bus &&
2454 info->devfn == devfn)
2460 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2463 struct dmar_domain *domain)
2465 struct dmar_domain *found = NULL;
2466 struct device_domain_info *info;
2467 unsigned long flags;
2470 info = alloc_devinfo_mem();
2475 info->devfn = devfn;
2476 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2477 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2480 info->domain = domain;
2481 info->iommu = iommu;
2482 info->pasid_table = NULL;
2483 info->auxd_enabled = 0;
2484 INIT_LIST_HEAD(&info->auxiliary_domains);
2486 if (dev && dev_is_pci(dev)) {
2487 struct pci_dev *pdev = to_pci_dev(info->dev);
2489 if (!pdev->untrusted &&
2490 !pci_ats_disabled() &&
2491 ecap_dev_iotlb_support(iommu->ecap) &&
2492 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2493 dmar_find_matched_atsr_unit(pdev))
2494 info->ats_supported = 1;
2496 if (sm_supported(iommu)) {
2497 if (pasid_supported(iommu)) {
2498 int features = pci_pasid_features(pdev);
2500 info->pasid_supported = features | 1;
2503 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2504 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2505 info->pri_supported = 1;
2509 spin_lock_irqsave(&device_domain_lock, flags);
2511 found = find_domain(dev);
2514 struct device_domain_info *info2;
2515 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2517 found = info2->domain;
2523 spin_unlock_irqrestore(&device_domain_lock, flags);
2524 free_devinfo_mem(info);
2525 /* Caller must free the original domain */
2529 spin_lock(&iommu->lock);
2530 ret = domain_attach_iommu(domain, iommu);
2531 spin_unlock(&iommu->lock);
2534 spin_unlock_irqrestore(&device_domain_lock, flags);
2535 free_devinfo_mem(info);
2539 list_add(&info->link, &domain->devices);
2540 list_add(&info->global, &device_domain_list);
2542 dev->archdata.iommu = info;
2543 spin_unlock_irqrestore(&device_domain_lock, flags);
2545 /* PASID table is mandatory for a PCI device in scalable mode. */
2546 if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2547 ret = intel_pasid_alloc_table(dev);
2549 dev_err(dev, "PASID table allocation failed\n");
2550 dmar_remove_one_dev_info(dev);
2554 /* Setup the PASID entry for requests without PASID: */
2555 spin_lock(&iommu->lock);
2556 if (hw_pass_through && domain_type_is_si(domain))
2557 ret = intel_pasid_setup_pass_through(iommu, domain,
2558 dev, PASID_RID2PASID);
2560 ret = intel_pasid_setup_second_level(iommu, domain,
2561 dev, PASID_RID2PASID);
2562 spin_unlock(&iommu->lock);
2564 dev_err(dev, "Setup RID2PASID failed\n");
2565 dmar_remove_one_dev_info(dev);
2570 if (dev && domain_context_mapping(domain, dev)) {
2571 dev_err(dev, "Domain context map failed\n");
2572 dmar_remove_one_dev_info(dev);
2579 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2581 *(u16 *)opaque = alias;
2585 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2587 struct device_domain_info *info;
2588 struct dmar_domain *domain = NULL;
2589 struct intel_iommu *iommu;
2591 unsigned long flags;
2594 iommu = device_to_iommu(dev, &bus, &devfn);
2598 if (dev_is_pci(dev)) {
2599 struct pci_dev *pdev = to_pci_dev(dev);
2601 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2603 spin_lock_irqsave(&device_domain_lock, flags);
2604 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2605 PCI_BUS_NUM(dma_alias),
2608 iommu = info->iommu;
2609 domain = info->domain;
2611 spin_unlock_irqrestore(&device_domain_lock, flags);
2613 /* DMA alias already has a domain, use it */
2618 /* Allocate and initialize new domain for the device */
2619 domain = alloc_domain(0);
2622 if (domain_init(domain, iommu, gaw)) {
2623 domain_exit(domain);
2632 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2633 struct dmar_domain *domain)
2635 struct intel_iommu *iommu;
2636 struct dmar_domain *tmp;
2637 u16 req_id, dma_alias;
2640 iommu = device_to_iommu(dev, &bus, &devfn);
2644 req_id = ((u16)bus << 8) | devfn;
2646 if (dev_is_pci(dev)) {
2647 struct pci_dev *pdev = to_pci_dev(dev);
2649 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2651 /* register PCI DMA alias device */
2652 if (req_id != dma_alias) {
2653 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2654 dma_alias & 0xff, NULL, domain);
2656 if (!tmp || tmp != domain)
2661 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2662 if (!tmp || tmp != domain)
2668 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2670 struct dmar_domain *domain, *tmp;
2672 domain = find_domain(dev);
2676 domain = find_or_alloc_domain(dev, gaw);
2680 tmp = set_domain_for_dev(dev, domain);
2681 if (!tmp || domain != tmp) {
2682 domain_exit(domain);
2691 static int iommu_domain_identity_map(struct dmar_domain *domain,
2692 unsigned long long start,
2693 unsigned long long end)
2695 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2696 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2698 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2699 dma_to_mm_pfn(last_vpfn))) {
2700 pr_err("Reserving iova failed\n");
2704 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2706 * RMRR range might have overlap with physical memory range,
2709 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2711 return __domain_mapping(domain, first_vpfn, NULL,
2712 first_vpfn, last_vpfn - first_vpfn + 1,
2713 DMA_PTE_READ|DMA_PTE_WRITE);
2716 static int domain_prepare_identity_map(struct device *dev,
2717 struct dmar_domain *domain,
2718 unsigned long long start,
2719 unsigned long long end)
2721 /* For _hardware_ passthrough, don't bother. But for software
2722 passthrough, we do it anyway -- it may indicate a memory
2723 range which is reserved in E820, so which didn't get set
2724 up to start with in si_domain */
2725 if (domain == si_domain && hw_pass_through) {
2726 dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
2731 dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
2734 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2735 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2736 dmi_get_system_info(DMI_BIOS_VENDOR),
2737 dmi_get_system_info(DMI_BIOS_VERSION),
2738 dmi_get_system_info(DMI_PRODUCT_VERSION));
2742 if (end >> agaw_to_width(domain->agaw)) {
2743 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2744 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2745 agaw_to_width(domain->agaw),
2746 dmi_get_system_info(DMI_BIOS_VENDOR),
2747 dmi_get_system_info(DMI_BIOS_VERSION),
2748 dmi_get_system_info(DMI_PRODUCT_VERSION));
2752 return iommu_domain_identity_map(domain, start, end);
2755 static int iommu_prepare_identity_map(struct device *dev,
2756 unsigned long long start,
2757 unsigned long long end)
2759 struct dmar_domain *domain;
2762 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2766 ret = domain_prepare_identity_map(dev, domain, start, end);
2768 domain_exit(domain);
2773 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2776 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2778 return iommu_prepare_identity_map(dev, rmrr->base_address,
2782 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2783 static inline void iommu_prepare_isa(void)
2785 struct pci_dev *pdev;
2788 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2792 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2793 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2796 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2801 static inline void iommu_prepare_isa(void)
2805 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2807 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2809 static int __init si_domain_init(int hw)
2813 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2817 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2818 domain_exit(si_domain);
2822 pr_debug("Identity mapping domain allocated\n");
2827 for_each_online_node(nid) {
2828 unsigned long start_pfn, end_pfn;
2831 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2832 ret = iommu_domain_identity_map(si_domain,
2833 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2842 static int identity_mapping(struct device *dev)
2844 struct device_domain_info *info;
2846 if (likely(!iommu_identity_mapping))
2849 info = dev->archdata.iommu;
2850 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2851 return (info->domain == si_domain);
2856 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2858 struct dmar_domain *ndomain;
2859 struct intel_iommu *iommu;
2862 iommu = device_to_iommu(dev, &bus, &devfn);
2866 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2867 if (ndomain != domain)
2873 static bool device_has_rmrr(struct device *dev)
2875 struct dmar_rmrr_unit *rmrr;
2880 for_each_rmrr_units(rmrr) {
2882 * Return TRUE if this RMRR contains the device that
2885 for_each_active_dev_scope(rmrr->devices,
2886 rmrr->devices_cnt, i, tmp)
2897 * There are a couple cases where we need to restrict the functionality of
2898 * devices associated with RMRRs. The first is when evaluating a device for
2899 * identity mapping because problems exist when devices are moved in and out
2900 * of domains and their respective RMRR information is lost. This means that
2901 * a device with associated RMRRs will never be in a "passthrough" domain.
2902 * The second is use of the device through the IOMMU API. This interface
2903 * expects to have full control of the IOVA space for the device. We cannot
2904 * satisfy both the requirement that RMRR access is maintained and have an
2905 * unencumbered IOVA space. We also have no ability to quiesce the device's
2906 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2907 * We therefore prevent devices associated with an RMRR from participating in
2908 * the IOMMU API, which eliminates them from device assignment.
2910 * In both cases we assume that PCI USB devices with RMRRs have them largely
2911 * for historical reasons and that the RMRR space is not actively used post
2912 * boot. This exclusion may change if vendors begin to abuse it.
2914 * The same exception is made for graphics devices, with the requirement that
2915 * any use of the RMRR regions will be torn down before assigning the device
2918 static bool device_is_rmrr_locked(struct device *dev)
2920 if (!device_has_rmrr(dev))
2923 if (dev_is_pci(dev)) {
2924 struct pci_dev *pdev = to_pci_dev(dev);
2926 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2933 static int iommu_should_identity_map(struct device *dev, int startup)
2935 if (dev_is_pci(dev)) {
2936 struct pci_dev *pdev = to_pci_dev(dev);
2938 if (device_is_rmrr_locked(dev))
2942 * Prevent any device marked as untrusted from getting
2943 * placed into the statically identity mapping domain.
2945 if (pdev->untrusted)
2948 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2951 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2954 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2958 * We want to start off with all devices in the 1:1 domain, and
2959 * take them out later if we find they can't access all of memory.
2961 * However, we can't do this for PCI devices behind bridges,
2962 * because all PCI devices behind the same bridge will end up
2963 * with the same source-id on their transactions.
2965 * Practically speaking, we can't change things around for these
2966 * devices at run-time, because we can't be sure there'll be no
2967 * DMA transactions in flight for any of their siblings.
2969 * So PCI devices (unless they're on the root bus) as well as
2970 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2971 * the 1:1 domain, just in _case_ one of their siblings turns out
2972 * not to be able to map all of memory.
2974 if (!pci_is_pcie(pdev)) {
2975 if (!pci_is_root_bus(pdev->bus))
2977 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2979 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2982 if (device_has_rmrr(dev))
2987 * At boot time, we don't yet know if devices will be 64-bit capable.
2988 * Assume that they will — if they turn out not to be, then we can
2989 * take them out of the 1:1 domain later.
2993 * If the device's dma_mask is less than the system's memory
2994 * size then this is not a candidate for identity mapping.
2996 u64 dma_mask = *dev->dma_mask;
2998 if (dev->coherent_dma_mask &&
2999 dev->coherent_dma_mask < dma_mask)
3000 dma_mask = dev->coherent_dma_mask;
3002 return dma_mask >= dma_get_required_mask(dev);
3008 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
3012 if (!iommu_should_identity_map(dev, 1))
3015 ret = domain_add_dev_info(si_domain, dev);
3017 dev_info(dev, "%s identity mapping\n",
3018 hw ? "Hardware" : "Software");
3019 else if (ret == -ENODEV)
3020 /* device not associated with an iommu */
3027 static int __init iommu_prepare_static_identity_mapping(int hw)
3029 struct pci_dev *pdev = NULL;
3030 struct dmar_drhd_unit *drhd;
3031 struct intel_iommu *iommu;
3036 for_each_pci_dev(pdev) {
3037 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3042 for_each_active_iommu(iommu, drhd)
3043 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3044 struct acpi_device_physical_node *pn;
3045 struct acpi_device *adev;
3047 if (dev->bus != &acpi_bus_type)
3050 adev= to_acpi_device(dev);
3051 mutex_lock(&adev->physical_node_lock);
3052 list_for_each_entry(pn, &adev->physical_node_list, node) {
3053 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3057 mutex_unlock(&adev->physical_node_lock);
3065 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3068 * Start from the sane iommu hardware state.
3069 * If the queued invalidation is already initialized by us
3070 * (for example, while enabling interrupt-remapping) then
3071 * we got the things already rolling from a sane state.
3075 * Clear any previous faults.
3077 dmar_fault(-1, iommu);
3079 * Disable queued invalidation if supported and already enabled
3080 * before OS handover.
3082 dmar_disable_qi(iommu);
3085 if (dmar_enable_qi(iommu)) {
3087 * Queued Invalidate not enabled, use Register Based Invalidate
3089 iommu->flush.flush_context = __iommu_flush_context;
3090 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3091 pr_info("%s: Using Register based invalidation\n",
3094 iommu->flush.flush_context = qi_flush_context;
3095 iommu->flush.flush_iotlb = qi_flush_iotlb;
3096 pr_info("%s: Using Queued invalidation\n", iommu->name);
3100 static int copy_context_table(struct intel_iommu *iommu,
3101 struct root_entry *old_re,
3102 struct context_entry **tbl,
3105 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3106 struct context_entry *new_ce = NULL, ce;
3107 struct context_entry *old_ce = NULL;
3108 struct root_entry re;
3109 phys_addr_t old_ce_phys;
3111 tbl_idx = ext ? bus * 2 : bus;
3112 memcpy(&re, old_re, sizeof(re));
3114 for (devfn = 0; devfn < 256; devfn++) {
3115 /* First calculate the correct index */
3116 idx = (ext ? devfn * 2 : devfn) % 256;
3119 /* First save what we may have and clean up */
3121 tbl[tbl_idx] = new_ce;
3122 __iommu_flush_cache(iommu, new_ce,
3132 old_ce_phys = root_entry_lctp(&re);
3134 old_ce_phys = root_entry_uctp(&re);
3137 if (ext && devfn == 0) {
3138 /* No LCTP, try UCTP */
3147 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3152 new_ce = alloc_pgtable_page(iommu->node);
3159 /* Now copy the context entry */
3160 memcpy(&ce, old_ce + idx, sizeof(ce));
3162 if (!__context_present(&ce))
3165 did = context_domain_id(&ce);
3166 if (did >= 0 && did < cap_ndoms(iommu->cap))
3167 set_bit(did, iommu->domain_ids);
3170 * We need a marker for copied context entries. This
3171 * marker needs to work for the old format as well as
3172 * for extended context entries.
3174 * Bit 67 of the context entry is used. In the old
3175 * format this bit is available to software, in the
3176 * extended format it is the PGE bit, but PGE is ignored
3177 * by HW if PASIDs are disabled (and thus still
3180 * So disable PASIDs first and then mark the entry
3181 * copied. This means that we don't copy PASID
3182 * translations from the old kernel, but this is fine as
3183 * faults there are not fatal.
3185 context_clear_pasid_enable(&ce);
3186 context_set_copied(&ce);
3191 tbl[tbl_idx + pos] = new_ce;
3193 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3202 static int copy_translation_tables(struct intel_iommu *iommu)
3204 struct context_entry **ctxt_tbls;
3205 struct root_entry *old_rt;
3206 phys_addr_t old_rt_phys;
3207 int ctxt_table_entries;
3208 unsigned long flags;
3213 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3214 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3215 new_ext = !!ecap_ecs(iommu->ecap);
3218 * The RTT bit can only be changed when translation is disabled,
3219 * but disabling translation means to open a window for data
3220 * corruption. So bail out and don't copy anything if we would
3221 * have to change the bit.
3226 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3230 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3234 /* This is too big for the stack - allocate it from slab */
3235 ctxt_table_entries = ext ? 512 : 256;
3237 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3241 for (bus = 0; bus < 256; bus++) {
3242 ret = copy_context_table(iommu, &old_rt[bus],
3243 ctxt_tbls, bus, ext);
3245 pr_err("%s: Failed to copy context table for bus %d\n",
3251 spin_lock_irqsave(&iommu->lock, flags);
3253 /* Context tables are copied, now write them to the root_entry table */
3254 for (bus = 0; bus < 256; bus++) {
3255 int idx = ext ? bus * 2 : bus;
3258 if (ctxt_tbls[idx]) {
3259 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3260 iommu->root_entry[bus].lo = val;
3263 if (!ext || !ctxt_tbls[idx + 1])
3266 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3267 iommu->root_entry[bus].hi = val;
3270 spin_unlock_irqrestore(&iommu->lock, flags);
3274 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3284 static int __init init_dmars(void)
3286 struct dmar_drhd_unit *drhd;
3287 struct dmar_rmrr_unit *rmrr;
3288 bool copied_tables = false;
3290 struct intel_iommu *iommu;
3296 * initialize and program root entry to not present
3299 for_each_drhd_unit(drhd) {
3301 * lock not needed as this is only incremented in the single
3302 * threaded kernel __init code path all other access are read
3305 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3309 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3312 /* Preallocate enough resources for IOMMU hot-addition */
3313 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3314 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3316 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3319 pr_err("Allocating global iommu array failed\n");
3324 for_each_active_iommu(iommu, drhd) {
3326 * Find the max pasid size of all IOMMU's in the system.
3327 * We need to ensure the system pasid table is no bigger
3328 * than the smallest supported.
3330 if (pasid_supported(iommu)) {
3331 u32 temp = 2 << ecap_pss(iommu->ecap);
3333 intel_pasid_max_id = min_t(u32, temp,
3334 intel_pasid_max_id);
3337 g_iommus[iommu->seq_id] = iommu;
3339 intel_iommu_init_qi(iommu);
3341 ret = iommu_init_domains(iommu);
3345 init_translation_status(iommu);
3347 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3348 iommu_disable_translation(iommu);
3349 clear_translation_pre_enabled(iommu);
3350 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3356 * we could share the same root & context tables
3357 * among all IOMMU's. Need to Split it later.
3359 ret = iommu_alloc_root_entry(iommu);
3363 if (translation_pre_enabled(iommu)) {
3364 pr_info("Translation already enabled - trying to copy translation structures\n");
3366 ret = copy_translation_tables(iommu);
3369 * We found the IOMMU with translation
3370 * enabled - but failed to copy over the
3371 * old root-entry table. Try to proceed
3372 * by disabling translation now and
3373 * allocating a clean root-entry table.
3374 * This might cause DMAR faults, but
3375 * probably the dump will still succeed.
3377 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3379 iommu_disable_translation(iommu);
3380 clear_translation_pre_enabled(iommu);
3382 pr_info("Copied translation tables from previous kernel for %s\n",
3384 copied_tables = true;
3388 if (!ecap_pass_through(iommu->ecap))
3389 hw_pass_through = 0;
3390 #ifdef CONFIG_INTEL_IOMMU_SVM
3391 if (pasid_supported(iommu))
3392 intel_svm_init(iommu);
3397 * Now that qi is enabled on all iommus, set the root entry and flush
3398 * caches. This is required on some Intel X58 chipsets, otherwise the
3399 * flush_context function will loop forever and the boot hangs.
3401 for_each_active_iommu(iommu, drhd) {
3402 iommu_flush_write_buffer(iommu);
3403 iommu_set_root_entry(iommu);
3404 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3405 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3408 if (iommu_pass_through)
3409 iommu_identity_mapping |= IDENTMAP_ALL;
3411 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3416 iommu_identity_mapping |= IDENTMAP_GFX;
3418 check_tylersburg_isoch();
3420 if (iommu_identity_mapping) {
3421 ret = si_domain_init(hw_pass_through);
3428 * If we copied translations from a previous kernel in the kdump
3429 * case, we can not assign the devices to domains now, as that
3430 * would eliminate the old mappings. So skip this part and defer
3431 * the assignment to device driver initialization time.
3437 * If pass through is not set or not enabled, setup context entries for
3438 * identity mappings for rmrr, gfx, and isa and may fall back to static
3439 * identity mapping if iommu_identity_mapping is set.
3441 if (iommu_identity_mapping) {
3442 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3444 pr_crit("Failed to setup IOMMU pass-through\n");
3450 * for each dev attached to rmrr
3452 * locate drhd for dev, alloc domain for dev
3453 * allocate free domain
3454 * allocate page table entries for rmrr
3455 * if context not allocated for bus
3456 * allocate and init context
3457 * set present in root table for this bus
3458 * init context with domain, translation etc
3462 pr_info("Setting RMRR:\n");
3463 for_each_rmrr_units(rmrr) {
3464 /* some BIOS lists non-exist devices in DMAR table. */
3465 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3467 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3469 pr_err("Mapping reserved region failed\n");
3473 iommu_prepare_isa();
3480 * global invalidate context cache
3481 * global invalidate iotlb
3482 * enable translation
3484 for_each_iommu(iommu, drhd) {
3485 if (drhd->ignored) {
3487 * we always have to disable PMRs or DMA may fail on
3491 iommu_disable_protect_mem_regions(iommu);
3495 iommu_flush_write_buffer(iommu);
3497 #ifdef CONFIG_INTEL_IOMMU_SVM
3498 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3500 * Call dmar_alloc_hwirq() with dmar_global_lock held,
3501 * could cause possible lock race condition.
3503 up_write(&dmar_global_lock);
3504 ret = intel_svm_enable_prq(iommu);
3505 down_write(&dmar_global_lock);
3510 ret = dmar_set_interrupt(iommu);
3514 if (!translation_pre_enabled(iommu))
3515 iommu_enable_translation(iommu);
3517 iommu_disable_protect_mem_regions(iommu);
3523 for_each_active_iommu(iommu, drhd) {
3524 disable_dmar_iommu(iommu);
3525 free_dmar_iommu(iommu);
3534 /* This takes a number of _MM_ pages, not VTD pages */
3535 static unsigned long intel_alloc_iova(struct device *dev,
3536 struct dmar_domain *domain,
3537 unsigned long nrpages, uint64_t dma_mask)
3539 unsigned long iova_pfn;
3541 /* Restrict dma_mask to the width that the iommu can handle */
3542 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3543 /* Ensure we reserve the whole size-aligned region */
3544 nrpages = __roundup_pow_of_two(nrpages);
3546 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3548 * First try to allocate an io virtual address in
3549 * DMA_BIT_MASK(32) and if that fails then try allocating
3552 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3553 IOVA_PFN(DMA_BIT_MASK(32)), false);
3557 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3558 IOVA_PFN(dma_mask), true);
3559 if (unlikely(!iova_pfn)) {
3560 dev_err(dev, "Allocating %ld-page iova failed", nrpages);
3567 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3569 struct dmar_domain *domain, *tmp;
3570 struct dmar_rmrr_unit *rmrr;
3571 struct device *i_dev;
3574 domain = find_domain(dev);
3578 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3582 /* We have a new domain - setup possible RMRRs for the device */
3584 for_each_rmrr_units(rmrr) {
3585 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3590 ret = domain_prepare_identity_map(dev, domain,
3594 dev_err(dev, "Mapping reserved region failed\n");
3599 tmp = set_domain_for_dev(dev, domain);
3600 if (!tmp || domain != tmp) {
3601 domain_exit(domain);
3608 dev_err(dev, "Allocating domain failed\n");
3614 /* Check if the dev needs to go through non-identity map and unmap process.*/
3615 static bool iommu_need_mapping(struct device *dev)
3619 if (iommu_dummy(dev))
3622 if (!iommu_identity_mapping)
3625 found = identity_mapping(dev);
3627 if (iommu_should_identity_map(dev, 0))
3631 * 32 bit DMA is removed from si_domain and fall back to
3632 * non-identity mapping.
3634 dmar_remove_one_dev_info(dev);
3635 dev_info(dev, "32bit DMA uses non-identity mapping\n");
3638 * In case of a detached 64 bit DMA device from vm, the device
3639 * is put into si_domain for identity mapping.
3641 if (iommu_should_identity_map(dev, 0) &&
3642 !domain_add_dev_info(si_domain, dev)) {
3643 dev_info(dev, "64bit DMA uses identity mapping\n");
3651 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3652 size_t size, int dir, u64 dma_mask)
3654 struct dmar_domain *domain;
3655 phys_addr_t start_paddr;
3656 unsigned long iova_pfn;
3659 struct intel_iommu *iommu;
3660 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3662 BUG_ON(dir == DMA_NONE);
3664 domain = get_valid_domain_for_dev(dev);
3666 return DMA_MAPPING_ERROR;
3668 iommu = domain_get_iommu(domain);
3669 size = aligned_nrpages(paddr, size);
3671 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3676 * Check if DMAR supports zero-length reads on write only
3679 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3680 !cap_zlr(iommu->cap))
3681 prot |= DMA_PTE_READ;
3682 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3683 prot |= DMA_PTE_WRITE;
3685 * paddr - (paddr + size) might be partial page, we should map the whole
3686 * page. Note: if two part of one page are separately mapped, we
3687 * might have two guest_addr mapping to the same host paddr, but this
3688 * is not a big problem
3690 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3691 mm_to_dma_pfn(paddr_pfn), size, prot);
3695 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3696 start_paddr += paddr & ~PAGE_MASK;
3701 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3702 dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
3703 size, (unsigned long long)paddr, dir);
3704 return DMA_MAPPING_ERROR;
3707 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3708 unsigned long offset, size_t size,
3709 enum dma_data_direction dir,
3710 unsigned long attrs)
3712 if (iommu_need_mapping(dev))
3713 return __intel_map_single(dev, page_to_phys(page) + offset,
3714 size, dir, *dev->dma_mask);
3715 return dma_direct_map_page(dev, page, offset, size, dir, attrs);
3718 static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
3719 size_t size, enum dma_data_direction dir,
3720 unsigned long attrs)
3722 if (iommu_need_mapping(dev))
3723 return __intel_map_single(dev, phys_addr, size, dir,
3725 return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
3728 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3730 struct dmar_domain *domain;
3731 unsigned long start_pfn, last_pfn;
3732 unsigned long nrpages;
3733 unsigned long iova_pfn;
3734 struct intel_iommu *iommu;
3735 struct page *freelist;
3736 struct pci_dev *pdev = NULL;
3738 domain = find_domain(dev);
3741 iommu = domain_get_iommu(domain);
3743 iova_pfn = IOVA_PFN(dev_addr);
3745 nrpages = aligned_nrpages(dev_addr, size);
3746 start_pfn = mm_to_dma_pfn(iova_pfn);
3747 last_pfn = start_pfn + nrpages - 1;
3749 if (dev_is_pci(dev))
3750 pdev = to_pci_dev(dev);
3752 dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn);
3754 freelist = domain_unmap(domain, start_pfn, last_pfn);
3756 if (intel_iommu_strict || (pdev && pdev->untrusted)) {
3757 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3758 nrpages, !freelist, 0);
3760 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3761 dma_free_pagelist(freelist);
3763 queue_iova(&domain->iovad, iova_pfn, nrpages,
3764 (unsigned long)freelist);
3766 * queue up the release of the unmap to save the 1/6th of the
3767 * cpu used up by the iotlb flush operation...
3772 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3773 size_t size, enum dma_data_direction dir,
3774 unsigned long attrs)
3776 if (iommu_need_mapping(dev))
3777 intel_unmap(dev, dev_addr, size);
3779 dma_direct_unmap_page(dev, dev_addr, size, dir, attrs);
3782 static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
3783 size_t size, enum dma_data_direction dir, unsigned long attrs)
3785 if (iommu_need_mapping(dev))
3786 intel_unmap(dev, dev_addr, size);
3789 static void *intel_alloc_coherent(struct device *dev, size_t size,
3790 dma_addr_t *dma_handle, gfp_t flags,
3791 unsigned long attrs)
3793 struct page *page = NULL;
3796 if (!iommu_need_mapping(dev))
3797 return dma_direct_alloc(dev, size, dma_handle, flags, attrs);
3799 size = PAGE_ALIGN(size);
3800 order = get_order(size);
3802 if (gfpflags_allow_blocking(flags)) {
3803 unsigned int count = size >> PAGE_SHIFT;
3805 page = dma_alloc_from_contiguous(dev, count, order,
3806 flags & __GFP_NOWARN);
3810 page = alloc_pages(flags, order);
3813 memset(page_address(page), 0, size);
3815 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3817 dev->coherent_dma_mask);
3818 if (*dma_handle != DMA_MAPPING_ERROR)
3819 return page_address(page);
3820 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3821 __free_pages(page, order);
3826 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3827 dma_addr_t dma_handle, unsigned long attrs)
3830 struct page *page = virt_to_page(vaddr);
3832 if (!iommu_need_mapping(dev))
3833 return dma_direct_free(dev, size, vaddr, dma_handle, attrs);
3835 size = PAGE_ALIGN(size);
3836 order = get_order(size);
3838 intel_unmap(dev, dma_handle, size);
3839 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3840 __free_pages(page, order);
3843 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3844 int nelems, enum dma_data_direction dir,
3845 unsigned long attrs)
3847 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3848 unsigned long nrpages = 0;
3849 struct scatterlist *sg;
3852 if (!iommu_need_mapping(dev))
3853 return dma_direct_unmap_sg(dev, sglist, nelems, dir, attrs);
3855 for_each_sg(sglist, sg, nelems, i) {
3856 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3859 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3862 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3863 enum dma_data_direction dir, unsigned long attrs)
3866 struct dmar_domain *domain;
3869 unsigned long iova_pfn;
3871 struct scatterlist *sg;
3872 unsigned long start_vpfn;
3873 struct intel_iommu *iommu;
3875 BUG_ON(dir == DMA_NONE);
3876 if (!iommu_need_mapping(dev))
3877 return dma_direct_map_sg(dev, sglist, nelems, dir, attrs);
3879 domain = get_valid_domain_for_dev(dev);
3883 iommu = domain_get_iommu(domain);
3885 for_each_sg(sglist, sg, nelems, i)
3886 size += aligned_nrpages(sg->offset, sg->length);
3888 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3891 sglist->dma_length = 0;
3896 * Check if DMAR supports zero-length reads on write only
3899 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3900 !cap_zlr(iommu->cap))
3901 prot |= DMA_PTE_READ;
3902 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3903 prot |= DMA_PTE_WRITE;
3905 start_vpfn = mm_to_dma_pfn(iova_pfn);
3907 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3908 if (unlikely(ret)) {
3909 dma_pte_free_pagetable(domain, start_vpfn,
3910 start_vpfn + size - 1,
3911 agaw_to_level(domain->agaw) + 1);
3912 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3919 static const struct dma_map_ops intel_dma_ops = {
3920 .alloc = intel_alloc_coherent,
3921 .free = intel_free_coherent,
3922 .map_sg = intel_map_sg,
3923 .unmap_sg = intel_unmap_sg,
3924 .map_page = intel_map_page,
3925 .unmap_page = intel_unmap_page,
3926 .map_resource = intel_map_resource,
3927 .unmap_resource = intel_unmap_resource,
3928 .dma_supported = dma_direct_supported,
3931 static inline int iommu_domain_cache_init(void)
3935 iommu_domain_cache = kmem_cache_create("iommu_domain",
3936 sizeof(struct dmar_domain),
3941 if (!iommu_domain_cache) {
3942 pr_err("Couldn't create iommu_domain cache\n");
3949 static inline int iommu_devinfo_cache_init(void)
3953 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3954 sizeof(struct device_domain_info),
3958 if (!iommu_devinfo_cache) {
3959 pr_err("Couldn't create devinfo cache\n");
3966 static int __init iommu_init_mempool(void)
3969 ret = iova_cache_get();
3973 ret = iommu_domain_cache_init();
3977 ret = iommu_devinfo_cache_init();
3981 kmem_cache_destroy(iommu_domain_cache);
3988 static void __init iommu_exit_mempool(void)
3990 kmem_cache_destroy(iommu_devinfo_cache);
3991 kmem_cache_destroy(iommu_domain_cache);
3995 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3997 struct dmar_drhd_unit *drhd;
4001 /* We know that this device on this chipset has its own IOMMU.
4002 * If we find it under a different IOMMU, then the BIOS is lying
4003 * to us. Hope that the IOMMU for this device is actually
4004 * disabled, and it needs no translation...
4006 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4008 /* "can't" happen */
4009 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4012 vtbar &= 0xffff0000;
4014 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4015 drhd = dmar_find_matched_drhd_unit(pdev);
4016 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4017 TAINT_FIRMWARE_WORKAROUND,
4018 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4019 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4021 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4023 static void __init init_no_remapping_devices(void)
4025 struct dmar_drhd_unit *drhd;
4029 for_each_drhd_unit(drhd) {
4030 if (!drhd->include_all) {
4031 for_each_active_dev_scope(drhd->devices,
4032 drhd->devices_cnt, i, dev)
4034 /* ignore DMAR unit if no devices exist */
4035 if (i == drhd->devices_cnt)
4040 for_each_active_drhd_unit(drhd) {
4041 if (drhd->include_all)
4044 for_each_active_dev_scope(drhd->devices,
4045 drhd->devices_cnt, i, dev)
4046 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4048 if (i < drhd->devices_cnt)
4051 /* This IOMMU has *only* gfx devices. Either bypass it or
4052 set the gfx_mapped flag, as appropriate */
4053 if (!dmar_map_gfx) {
4055 for_each_active_dev_scope(drhd->devices,
4056 drhd->devices_cnt, i, dev)
4057 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4062 #ifdef CONFIG_SUSPEND
4063 static int init_iommu_hw(void)
4065 struct dmar_drhd_unit *drhd;
4066 struct intel_iommu *iommu = NULL;
4068 for_each_active_iommu(iommu, drhd)
4070 dmar_reenable_qi(iommu);
4072 for_each_iommu(iommu, drhd) {
4073 if (drhd->ignored) {
4075 * we always have to disable PMRs or DMA may fail on
4079 iommu_disable_protect_mem_regions(iommu);
4083 iommu_flush_write_buffer(iommu);
4085 iommu_set_root_entry(iommu);
4087 iommu->flush.flush_context(iommu, 0, 0, 0,
4088 DMA_CCMD_GLOBAL_INVL);
4089 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4090 iommu_enable_translation(iommu);
4091 iommu_disable_protect_mem_regions(iommu);
4097 static void iommu_flush_all(void)
4099 struct dmar_drhd_unit *drhd;
4100 struct intel_iommu *iommu;
4102 for_each_active_iommu(iommu, drhd) {
4103 iommu->flush.flush_context(iommu, 0, 0, 0,
4104 DMA_CCMD_GLOBAL_INVL);
4105 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4106 DMA_TLB_GLOBAL_FLUSH);
4110 static int iommu_suspend(void)
4112 struct dmar_drhd_unit *drhd;
4113 struct intel_iommu *iommu = NULL;
4116 for_each_active_iommu(iommu, drhd) {
4117 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4119 if (!iommu->iommu_state)
4125 for_each_active_iommu(iommu, drhd) {
4126 iommu_disable_translation(iommu);
4128 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4130 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4131 readl(iommu->reg + DMAR_FECTL_REG);
4132 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4133 readl(iommu->reg + DMAR_FEDATA_REG);
4134 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4135 readl(iommu->reg + DMAR_FEADDR_REG);
4136 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4137 readl(iommu->reg + DMAR_FEUADDR_REG);
4139 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4144 for_each_active_iommu(iommu, drhd)
4145 kfree(iommu->iommu_state);
4150 static void iommu_resume(void)
4152 struct dmar_drhd_unit *drhd;
4153 struct intel_iommu *iommu = NULL;
4156 if (init_iommu_hw()) {
4158 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4160 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4164 for_each_active_iommu(iommu, drhd) {
4166 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4168 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4169 iommu->reg + DMAR_FECTL_REG);
4170 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4171 iommu->reg + DMAR_FEDATA_REG);
4172 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4173 iommu->reg + DMAR_FEADDR_REG);
4174 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4175 iommu->reg + DMAR_FEUADDR_REG);
4177 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4180 for_each_active_iommu(iommu, drhd)
4181 kfree(iommu->iommu_state);
4184 static struct syscore_ops iommu_syscore_ops = {
4185 .resume = iommu_resume,
4186 .suspend = iommu_suspend,
4189 static void __init init_iommu_pm_ops(void)
4191 register_syscore_ops(&iommu_syscore_ops);
4195 static inline void init_iommu_pm_ops(void) {}
4196 #endif /* CONFIG_PM */
4199 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4201 struct acpi_dmar_reserved_memory *rmrr;
4202 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4203 struct dmar_rmrr_unit *rmrru;
4206 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4210 rmrru->hdr = header;
4211 rmrr = (struct acpi_dmar_reserved_memory *)header;
4212 rmrru->base_address = rmrr->base_address;
4213 rmrru->end_address = rmrr->end_address;
4215 length = rmrr->end_address - rmrr->base_address + 1;
4216 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4221 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4222 ((void *)rmrr) + rmrr->header.length,
4223 &rmrru->devices_cnt);
4224 if (rmrru->devices_cnt && rmrru->devices == NULL)
4227 list_add(&rmrru->list, &dmar_rmrr_units);
4238 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4240 struct dmar_atsr_unit *atsru;
4241 struct acpi_dmar_atsr *tmp;
4243 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4244 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4245 if (atsr->segment != tmp->segment)
4247 if (atsr->header.length != tmp->header.length)
4249 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4256 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4258 struct acpi_dmar_atsr *atsr;
4259 struct dmar_atsr_unit *atsru;
4261 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4264 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4265 atsru = dmar_find_atsr(atsr);
4269 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4274 * If memory is allocated from slab by ACPI _DSM method, we need to
4275 * copy the memory content because the memory buffer will be freed
4278 atsru->hdr = (void *)(atsru + 1);
4279 memcpy(atsru->hdr, hdr, hdr->length);
4280 atsru->include_all = atsr->flags & 0x1;
4281 if (!atsru->include_all) {
4282 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4283 (void *)atsr + atsr->header.length,
4284 &atsru->devices_cnt);
4285 if (atsru->devices_cnt && atsru->devices == NULL) {
4291 list_add_rcu(&atsru->list, &dmar_atsr_units);
4296 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4298 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4302 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4304 struct acpi_dmar_atsr *atsr;
4305 struct dmar_atsr_unit *atsru;
4307 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4308 atsru = dmar_find_atsr(atsr);
4310 list_del_rcu(&atsru->list);
4312 intel_iommu_free_atsr(atsru);
4318 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4322 struct acpi_dmar_atsr *atsr;
4323 struct dmar_atsr_unit *atsru;
4325 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4326 atsru = dmar_find_atsr(atsr);
4330 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4331 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4339 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4342 struct intel_iommu *iommu = dmaru->iommu;
4344 if (g_iommus[iommu->seq_id])
4347 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4348 pr_warn("%s: Doesn't support hardware pass through.\n",
4352 if (!ecap_sc_support(iommu->ecap) &&
4353 domain_update_iommu_snooping(iommu)) {
4354 pr_warn("%s: Doesn't support snooping.\n",
4358 sp = domain_update_iommu_superpage(iommu) - 1;
4359 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4360 pr_warn("%s: Doesn't support large page.\n",
4366 * Disable translation if already enabled prior to OS handover.
4368 if (iommu->gcmd & DMA_GCMD_TE)
4369 iommu_disable_translation(iommu);
4371 g_iommus[iommu->seq_id] = iommu;
4372 ret = iommu_init_domains(iommu);
4374 ret = iommu_alloc_root_entry(iommu);
4378 #ifdef CONFIG_INTEL_IOMMU_SVM
4379 if (pasid_supported(iommu))
4380 intel_svm_init(iommu);
4383 if (dmaru->ignored) {
4385 * we always have to disable PMRs or DMA may fail on this device
4388 iommu_disable_protect_mem_regions(iommu);
4392 intel_iommu_init_qi(iommu);
4393 iommu_flush_write_buffer(iommu);
4395 #ifdef CONFIG_INTEL_IOMMU_SVM
4396 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
4397 ret = intel_svm_enable_prq(iommu);
4402 ret = dmar_set_interrupt(iommu);
4406 iommu_set_root_entry(iommu);
4407 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4408 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4409 iommu_enable_translation(iommu);
4411 iommu_disable_protect_mem_regions(iommu);
4415 disable_dmar_iommu(iommu);
4417 free_dmar_iommu(iommu);
4421 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4424 struct intel_iommu *iommu = dmaru->iommu;
4426 if (!intel_iommu_enabled)
4432 ret = intel_iommu_add(dmaru);
4434 disable_dmar_iommu(iommu);
4435 free_dmar_iommu(iommu);
4441 static void intel_iommu_free_dmars(void)
4443 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4444 struct dmar_atsr_unit *atsru, *atsr_n;
4446 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4447 list_del(&rmrru->list);
4448 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4453 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4454 list_del(&atsru->list);
4455 intel_iommu_free_atsr(atsru);
4459 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4462 struct pci_bus *bus;
4463 struct pci_dev *bridge = NULL;
4465 struct acpi_dmar_atsr *atsr;
4466 struct dmar_atsr_unit *atsru;
4468 dev = pci_physfn(dev);
4469 for (bus = dev->bus; bus; bus = bus->parent) {
4471 /* If it's an integrated device, allow ATS */
4474 /* Connected via non-PCIe: no ATS */
4475 if (!pci_is_pcie(bridge) ||
4476 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4478 /* If we found the root port, look it up in the ATSR */
4479 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4484 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4485 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4486 if (atsr->segment != pci_domain_nr(dev->bus))
4489 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4490 if (tmp == &bridge->dev)
4493 if (atsru->include_all)
4503 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4506 struct dmar_rmrr_unit *rmrru;
4507 struct dmar_atsr_unit *atsru;
4508 struct acpi_dmar_atsr *atsr;
4509 struct acpi_dmar_reserved_memory *rmrr;
4511 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4514 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4515 rmrr = container_of(rmrru->hdr,
4516 struct acpi_dmar_reserved_memory, header);
4517 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4518 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4519 ((void *)rmrr) + rmrr->header.length,
4520 rmrr->segment, rmrru->devices,
4521 rmrru->devices_cnt);
4524 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4525 dmar_remove_dev_scope(info, rmrr->segment,
4526 rmrru->devices, rmrru->devices_cnt);
4530 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4531 if (atsru->include_all)
4534 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4535 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4536 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4537 (void *)atsr + atsr->header.length,
4538 atsr->segment, atsru->devices,
4539 atsru->devices_cnt);
4544 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4545 if (dmar_remove_dev_scope(info, atsr->segment,
4546 atsru->devices, atsru->devices_cnt))
4555 * Here we only respond to action of unbound device from driver.
4557 * Added device is not attached to its DMAR domain here yet. That will happen
4558 * when mapping the device to iova.
4560 static int device_notifier(struct notifier_block *nb,
4561 unsigned long action, void *data)
4563 struct device *dev = data;
4564 struct dmar_domain *domain;
4566 if (iommu_dummy(dev))
4569 if (action == BUS_NOTIFY_REMOVED_DEVICE) {
4570 domain = find_domain(dev);
4574 dmar_remove_one_dev_info(dev);
4575 if (!domain_type_is_vm_or_si(domain) &&
4576 list_empty(&domain->devices))
4577 domain_exit(domain);
4578 } else if (action == BUS_NOTIFY_ADD_DEVICE) {
4579 if (iommu_should_identity_map(dev, 1))
4580 domain_add_dev_info(si_domain, dev);
4586 static struct notifier_block device_nb = {
4587 .notifier_call = device_notifier,
4590 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4591 unsigned long val, void *v)
4593 struct memory_notify *mhp = v;
4594 unsigned long long start, end;
4595 unsigned long start_vpfn, last_vpfn;
4598 case MEM_GOING_ONLINE:
4599 start = mhp->start_pfn << PAGE_SHIFT;
4600 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4601 if (iommu_domain_identity_map(si_domain, start, end)) {
4602 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4609 case MEM_CANCEL_ONLINE:
4610 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4611 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4612 while (start_vpfn <= last_vpfn) {
4614 struct dmar_drhd_unit *drhd;
4615 struct intel_iommu *iommu;
4616 struct page *freelist;
4618 iova = find_iova(&si_domain->iovad, start_vpfn);
4620 pr_debug("Failed get IOVA for PFN %lx\n",
4625 iova = split_and_remove_iova(&si_domain->iovad, iova,
4626 start_vpfn, last_vpfn);
4628 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4629 start_vpfn, last_vpfn);
4633 freelist = domain_unmap(si_domain, iova->pfn_lo,
4637 for_each_active_iommu(iommu, drhd)
4638 iommu_flush_iotlb_psi(iommu, si_domain,
4639 iova->pfn_lo, iova_size(iova),
4642 dma_free_pagelist(freelist);
4644 start_vpfn = iova->pfn_hi + 1;
4645 free_iova_mem(iova);
4653 static struct notifier_block intel_iommu_memory_nb = {
4654 .notifier_call = intel_iommu_memory_notifier,
4658 static void free_all_cpu_cached_iovas(unsigned int cpu)
4662 for (i = 0; i < g_num_of_iommus; i++) {
4663 struct intel_iommu *iommu = g_iommus[i];
4664 struct dmar_domain *domain;
4670 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4671 domain = get_iommu_domain(iommu, (u16)did);
4675 free_cpu_cached_iovas(cpu, &domain->iovad);
4680 static int intel_iommu_cpu_dead(unsigned int cpu)
4682 free_all_cpu_cached_iovas(cpu);
4686 static void intel_disable_iommus(void)
4688 struct intel_iommu *iommu = NULL;
4689 struct dmar_drhd_unit *drhd;
4691 for_each_iommu(iommu, drhd)
4692 iommu_disable_translation(iommu);
4695 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4697 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4699 return container_of(iommu_dev, struct intel_iommu, iommu);
4702 static ssize_t intel_iommu_show_version(struct device *dev,
4703 struct device_attribute *attr,
4706 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4707 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4708 return sprintf(buf, "%d:%d\n",
4709 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4711 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4713 static ssize_t intel_iommu_show_address(struct device *dev,
4714 struct device_attribute *attr,
4717 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4718 return sprintf(buf, "%llx\n", iommu->reg_phys);
4720 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4722 static ssize_t intel_iommu_show_cap(struct device *dev,
4723 struct device_attribute *attr,
4726 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4727 return sprintf(buf, "%llx\n", iommu->cap);
4729 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4731 static ssize_t intel_iommu_show_ecap(struct device *dev,
4732 struct device_attribute *attr,
4735 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4736 return sprintf(buf, "%llx\n", iommu->ecap);
4738 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4740 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4741 struct device_attribute *attr,
4744 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4745 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4747 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4749 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4750 struct device_attribute *attr,
4753 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4754 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4755 cap_ndoms(iommu->cap)));
4757 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4759 static struct attribute *intel_iommu_attrs[] = {
4760 &dev_attr_version.attr,
4761 &dev_attr_address.attr,
4763 &dev_attr_ecap.attr,
4764 &dev_attr_domains_supported.attr,
4765 &dev_attr_domains_used.attr,
4769 static struct attribute_group intel_iommu_group = {
4770 .name = "intel-iommu",
4771 .attrs = intel_iommu_attrs,
4774 const struct attribute_group *intel_iommu_groups[] = {
4779 static int __init platform_optin_force_iommu(void)
4781 struct pci_dev *pdev = NULL;
4782 bool has_untrusted_dev = false;
4784 if (!dmar_platform_optin() || no_platform_optin)
4787 for_each_pci_dev(pdev) {
4788 if (pdev->untrusted) {
4789 has_untrusted_dev = true;
4794 if (!has_untrusted_dev)
4797 if (no_iommu || dmar_disabled)
4798 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
4801 * If Intel-IOMMU is disabled by default, we will apply identity
4802 * map for all devices except those marked as being untrusted.
4805 iommu_identity_mapping |= IDENTMAP_ALL;
4808 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4816 int __init intel_iommu_init(void)
4819 struct dmar_drhd_unit *drhd;
4820 struct intel_iommu *iommu;
4823 * Intel IOMMU is required for a TXT/tboot launch or platform
4824 * opt in, so enforce that.
4826 force_on = tboot_force_iommu() || platform_optin_force_iommu();
4828 if (iommu_init_mempool()) {
4830 panic("tboot: Failed to initialize iommu memory\n");
4834 down_write(&dmar_global_lock);
4835 if (dmar_table_init()) {
4837 panic("tboot: Failed to initialize DMAR table\n");
4841 if (dmar_dev_scope_init() < 0) {
4843 panic("tboot: Failed to initialize DMAR device scope\n");
4847 up_write(&dmar_global_lock);
4850 * The bus notifier takes the dmar_global_lock, so lockdep will
4851 * complain later when we register it under the lock.
4853 dmar_register_bus_notifier();
4855 down_write(&dmar_global_lock);
4857 if (no_iommu || dmar_disabled) {
4859 * We exit the function here to ensure IOMMU's remapping and
4860 * mempool aren't setup, which means that the IOMMU's PMRs
4861 * won't be disabled via the call to init_dmars(). So disable
4862 * it explicitly here. The PMRs were setup by tboot prior to
4863 * calling SENTER, but the kernel is expected to reset/tear
4866 if (intel_iommu_tboot_noforce) {
4867 for_each_iommu(iommu, drhd)
4868 iommu_disable_protect_mem_regions(iommu);
4872 * Make sure the IOMMUs are switched off, even when we
4873 * boot into a kexec kernel and the previous kernel left
4876 intel_disable_iommus();
4880 if (list_empty(&dmar_rmrr_units))
4881 pr_info("No RMRR found\n");
4883 if (list_empty(&dmar_atsr_units))
4884 pr_info("No ATSR found\n");
4886 if (dmar_init_reserved_ranges()) {
4888 panic("tboot: Failed to reserve iommu ranges\n");
4889 goto out_free_reserved_range;
4893 intel_iommu_gfx_mapped = 1;
4895 init_no_remapping_devices();
4900 panic("tboot: Failed to initialize DMARs\n");
4901 pr_err("Initialization failed\n");
4902 goto out_free_reserved_range;
4904 up_write(&dmar_global_lock);
4905 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4907 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4910 dma_ops = &intel_dma_ops;
4912 init_iommu_pm_ops();
4914 for_each_active_iommu(iommu, drhd) {
4915 iommu_device_sysfs_add(&iommu->iommu, NULL,
4918 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4919 iommu_device_register(&iommu->iommu);
4922 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4923 bus_register_notifier(&pci_bus_type, &device_nb);
4924 if (si_domain && !hw_pass_through)
4925 register_memory_notifier(&intel_iommu_memory_nb);
4926 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4927 intel_iommu_cpu_dead);
4928 intel_iommu_enabled = 1;
4929 intel_iommu_debugfs_init();
4933 out_free_reserved_range:
4934 put_iova_domain(&reserved_iova_list);
4936 intel_iommu_free_dmars();
4937 up_write(&dmar_global_lock);
4938 iommu_exit_mempool();
4942 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4944 struct intel_iommu *iommu = opaque;
4946 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4951 * NB - intel-iommu lacks any sort of reference counting for the users of
4952 * dependent devices. If multiple endpoints have intersecting dependent
4953 * devices, unbinding the driver from any one of them will possibly leave
4954 * the others unable to operate.
4956 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4958 if (!iommu || !dev || !dev_is_pci(dev))
4961 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4964 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4966 struct intel_iommu *iommu;
4967 unsigned long flags;
4969 assert_spin_locked(&device_domain_lock);
4974 iommu = info->iommu;
4977 if (dev_is_pci(info->dev) && sm_supported(iommu))
4978 intel_pasid_tear_down_entry(iommu, info->dev,
4981 iommu_disable_dev_iotlb(info);
4982 domain_context_clear(iommu, info->dev);
4983 intel_pasid_free_table(info->dev);
4986 unlink_domain_info(info);
4988 spin_lock_irqsave(&iommu->lock, flags);
4989 domain_detach_iommu(info->domain, iommu);
4990 spin_unlock_irqrestore(&iommu->lock, flags);
4992 free_devinfo_mem(info);
4995 static void dmar_remove_one_dev_info(struct device *dev)
4997 struct device_domain_info *info;
4998 unsigned long flags;
5000 spin_lock_irqsave(&device_domain_lock, flags);
5001 info = dev->archdata.iommu;
5002 __dmar_remove_one_dev_info(info);
5003 spin_unlock_irqrestore(&device_domain_lock, flags);
5006 static int md_domain_init(struct dmar_domain *domain, int guest_width)
5010 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5011 domain_reserve_special_ranges(domain);
5013 /* calculate AGAW */
5014 domain->gaw = guest_width;
5015 adjust_width = guestwidth_to_adjustwidth(guest_width);
5016 domain->agaw = width_to_agaw(adjust_width);
5018 domain->iommu_coherency = 0;
5019 domain->iommu_snooping = 0;
5020 domain->iommu_superpage = 0;
5021 domain->max_addr = 0;
5023 /* always allocate the top pgd */
5024 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5027 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5031 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5033 struct dmar_domain *dmar_domain;
5034 struct iommu_domain *domain;
5036 if (type != IOMMU_DOMAIN_UNMANAGED)
5039 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5041 pr_err("Can't allocate dmar_domain\n");
5044 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5045 pr_err("Domain initialization failed\n");
5046 domain_exit(dmar_domain);
5049 domain_update_iommu_cap(dmar_domain);
5051 domain = &dmar_domain->domain;
5052 domain->geometry.aperture_start = 0;
5053 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5054 domain->geometry.force_aperture = true;
5059 static void intel_iommu_domain_free(struct iommu_domain *domain)
5061 domain_exit(to_dmar_domain(domain));
5065 * Check whether a @domain could be attached to the @dev through the
5066 * aux-domain attach/detach APIs.
5069 is_aux_domain(struct device *dev, struct iommu_domain *domain)
5071 struct device_domain_info *info = dev->archdata.iommu;
5073 return info && info->auxd_enabled &&
5074 domain->type == IOMMU_DOMAIN_UNMANAGED;
5077 static void auxiliary_link_device(struct dmar_domain *domain,
5080 struct device_domain_info *info = dev->archdata.iommu;
5082 assert_spin_locked(&device_domain_lock);
5086 domain->auxd_refcnt++;
5087 list_add(&domain->auxd, &info->auxiliary_domains);
5090 static void auxiliary_unlink_device(struct dmar_domain *domain,
5093 struct device_domain_info *info = dev->archdata.iommu;
5095 assert_spin_locked(&device_domain_lock);
5099 list_del(&domain->auxd);
5100 domain->auxd_refcnt--;
5102 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5103 intel_pasid_free_id(domain->default_pasid);
5106 static int aux_domain_add_dev(struct dmar_domain *domain,
5111 unsigned long flags;
5112 struct intel_iommu *iommu;
5114 iommu = device_to_iommu(dev, &bus, &devfn);
5118 if (domain->default_pasid <= 0) {
5121 pasid = intel_pasid_alloc_id(domain, PASID_MIN,
5122 pci_max_pasids(to_pci_dev(dev)),
5125 pr_err("Can't allocate default pasid\n");
5128 domain->default_pasid = pasid;
5131 spin_lock_irqsave(&device_domain_lock, flags);
5133 * iommu->lock must be held to attach domain to iommu and setup the
5134 * pasid entry for second level translation.
5136 spin_lock(&iommu->lock);
5137 ret = domain_attach_iommu(domain, iommu);
5141 /* Setup the PASID entry for mediated devices: */
5142 ret = intel_pasid_setup_second_level(iommu, domain, dev,
5143 domain->default_pasid);
5146 spin_unlock(&iommu->lock);
5148 auxiliary_link_device(domain, dev);
5150 spin_unlock_irqrestore(&device_domain_lock, flags);
5155 domain_detach_iommu(domain, iommu);
5157 spin_unlock(&iommu->lock);
5158 spin_unlock_irqrestore(&device_domain_lock, flags);
5159 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5160 intel_pasid_free_id(domain->default_pasid);
5165 static void aux_domain_remove_dev(struct dmar_domain *domain,
5168 struct device_domain_info *info;
5169 struct intel_iommu *iommu;
5170 unsigned long flags;
5172 if (!is_aux_domain(dev, &domain->domain))
5175 spin_lock_irqsave(&device_domain_lock, flags);
5176 info = dev->archdata.iommu;
5177 iommu = info->iommu;
5179 auxiliary_unlink_device(domain, dev);
5181 spin_lock(&iommu->lock);
5182 intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid);
5183 domain_detach_iommu(domain, iommu);
5184 spin_unlock(&iommu->lock);
5186 spin_unlock_irqrestore(&device_domain_lock, flags);
5189 static int prepare_domain_attach_device(struct iommu_domain *domain,
5192 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5193 struct intel_iommu *iommu;
5197 iommu = device_to_iommu(dev, &bus, &devfn);
5201 /* check if this iommu agaw is sufficient for max mapped address */
5202 addr_width = agaw_to_width(iommu->agaw);
5203 if (addr_width > cap_mgaw(iommu->cap))
5204 addr_width = cap_mgaw(iommu->cap);
5206 if (dmar_domain->max_addr > (1LL << addr_width)) {
5207 dev_err(dev, "%s: iommu width (%d) is not "
5208 "sufficient for the mapped address (%llx)\n",
5209 __func__, addr_width, dmar_domain->max_addr);
5212 dmar_domain->gaw = addr_width;
5215 * Knock out extra levels of page tables if necessary
5217 while (iommu->agaw < dmar_domain->agaw) {
5218 struct dma_pte *pte;
5220 pte = dmar_domain->pgd;
5221 if (dma_pte_present(pte)) {
5222 dmar_domain->pgd = (struct dma_pte *)
5223 phys_to_virt(dma_pte_addr(pte));
5224 free_pgtable_page(pte);
5226 dmar_domain->agaw--;
5232 static int intel_iommu_attach_device(struct iommu_domain *domain,
5237 if (device_is_rmrr_locked(dev)) {
5238 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5242 if (is_aux_domain(dev, domain))
5245 /* normally dev is not mapped */
5246 if (unlikely(domain_context_mapped(dev))) {
5247 struct dmar_domain *old_domain;
5249 old_domain = find_domain(dev);
5251 dmar_remove_one_dev_info(dev);
5253 if (!domain_type_is_vm_or_si(old_domain) &&
5254 list_empty(&old_domain->devices))
5255 domain_exit(old_domain);
5259 ret = prepare_domain_attach_device(domain, dev);
5263 return domain_add_dev_info(to_dmar_domain(domain), dev);
5266 static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
5271 if (!is_aux_domain(dev, domain))
5274 ret = prepare_domain_attach_device(domain, dev);
5278 return aux_domain_add_dev(to_dmar_domain(domain), dev);
5281 static void intel_iommu_detach_device(struct iommu_domain *domain,
5284 dmar_remove_one_dev_info(dev);
5287 static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
5290 aux_domain_remove_dev(to_dmar_domain(domain), dev);
5293 static int intel_iommu_map(struct iommu_domain *domain,
5294 unsigned long iova, phys_addr_t hpa,
5295 size_t size, int iommu_prot)
5297 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5302 if (iommu_prot & IOMMU_READ)
5303 prot |= DMA_PTE_READ;
5304 if (iommu_prot & IOMMU_WRITE)
5305 prot |= DMA_PTE_WRITE;
5306 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5307 prot |= DMA_PTE_SNP;
5309 max_addr = iova + size;
5310 if (dmar_domain->max_addr < max_addr) {
5313 /* check if minimum agaw is sufficient for mapped address */
5314 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5315 if (end < max_addr) {
5316 pr_err("%s: iommu width (%d) is not "
5317 "sufficient for the mapped address (%llx)\n",
5318 __func__, dmar_domain->gaw, max_addr);
5321 dmar_domain->max_addr = max_addr;
5323 /* Round up size to next multiple of PAGE_SIZE, if it and
5324 the low bits of hpa would take us onto the next page */
5325 size = aligned_nrpages(hpa, size);
5326 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5327 hpa >> VTD_PAGE_SHIFT, size, prot);
5331 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5332 unsigned long iova, size_t size)
5334 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5335 struct page *freelist = NULL;
5336 unsigned long start_pfn, last_pfn;
5337 unsigned int npages;
5338 int iommu_id, level = 0;
5340 /* Cope with horrid API which requires us to unmap more than the
5341 size argument if it happens to be a large-page mapping. */
5342 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5344 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5345 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5347 start_pfn = iova >> VTD_PAGE_SHIFT;
5348 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5350 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5352 npages = last_pfn - start_pfn + 1;
5354 for_each_domain_iommu(iommu_id, dmar_domain)
5355 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5356 start_pfn, npages, !freelist, 0);
5358 dma_free_pagelist(freelist);
5360 if (dmar_domain->max_addr == iova + size)
5361 dmar_domain->max_addr = iova;
5366 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5369 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5370 struct dma_pte *pte;
5374 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5376 phys = dma_pte_addr(pte);
5381 static inline bool scalable_mode_support(void)
5383 struct dmar_drhd_unit *drhd;
5384 struct intel_iommu *iommu;
5388 for_each_active_iommu(iommu, drhd) {
5389 if (!sm_supported(iommu)) {
5399 static inline bool iommu_pasid_support(void)
5401 struct dmar_drhd_unit *drhd;
5402 struct intel_iommu *iommu;
5406 for_each_active_iommu(iommu, drhd) {
5407 if (!pasid_supported(iommu)) {
5417 static bool intel_iommu_capable(enum iommu_cap cap)
5419 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5420 return domain_update_iommu_snooping(NULL) == 1;
5421 if (cap == IOMMU_CAP_INTR_REMAP)
5422 return irq_remapping_enabled == 1;
5427 static int intel_iommu_add_device(struct device *dev)
5429 struct intel_iommu *iommu;
5430 struct iommu_group *group;
5433 iommu = device_to_iommu(dev, &bus, &devfn);
5437 iommu_device_link(&iommu->iommu, dev);
5439 group = iommu_group_get_for_dev(dev);
5442 return PTR_ERR(group);
5444 iommu_group_put(group);
5448 static void intel_iommu_remove_device(struct device *dev)
5450 struct intel_iommu *iommu;
5453 iommu = device_to_iommu(dev, &bus, &devfn);
5457 iommu_group_remove_device(dev);
5459 iommu_device_unlink(&iommu->iommu, dev);
5462 static void intel_iommu_get_resv_regions(struct device *device,
5463 struct list_head *head)
5465 struct iommu_resv_region *reg;
5466 struct dmar_rmrr_unit *rmrr;
5467 struct device *i_dev;
5471 for_each_rmrr_units(rmrr) {
5472 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5474 if (i_dev != device)
5477 list_add_tail(&rmrr->resv->list, head);
5482 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5483 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5487 list_add_tail(®->list, head);
5490 static void intel_iommu_put_resv_regions(struct device *dev,
5491 struct list_head *head)
5493 struct iommu_resv_region *entry, *next;
5495 list_for_each_entry_safe(entry, next, head, list) {
5496 if (entry->type == IOMMU_RESV_MSI)
5501 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
5503 struct device_domain_info *info;
5504 struct context_entry *context;
5505 struct dmar_domain *domain;
5506 unsigned long flags;
5510 domain = get_valid_domain_for_dev(dev);
5514 spin_lock_irqsave(&device_domain_lock, flags);
5515 spin_lock(&iommu->lock);
5518 info = dev->archdata.iommu;
5519 if (!info || !info->pasid_supported)
5522 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5523 if (WARN_ON(!context))
5526 ctx_lo = context[0].lo;
5528 if (!(ctx_lo & CONTEXT_PASIDE)) {
5529 ctx_lo |= CONTEXT_PASIDE;
5530 context[0].lo = ctx_lo;
5532 iommu->flush.flush_context(iommu,
5533 domain->iommu_did[iommu->seq_id],
5534 PCI_DEVID(info->bus, info->devfn),
5535 DMA_CCMD_MASK_NOBIT,
5536 DMA_CCMD_DEVICE_INVL);
5539 /* Enable PASID support in the device, if it wasn't already */
5540 if (!info->pasid_enabled)
5541 iommu_enable_dev_iotlb(info);
5546 spin_unlock(&iommu->lock);
5547 spin_unlock_irqrestore(&device_domain_lock, flags);
5552 #ifdef CONFIG_INTEL_IOMMU_SVM
5553 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5555 struct intel_iommu *iommu;
5558 if (iommu_dummy(dev)) {
5560 "No IOMMU translation for device; cannot enable SVM\n");
5564 iommu = device_to_iommu(dev, &bus, &devfn);
5566 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5572 #endif /* CONFIG_INTEL_IOMMU_SVM */
5574 static int intel_iommu_enable_auxd(struct device *dev)
5576 struct device_domain_info *info;
5577 struct intel_iommu *iommu;
5578 unsigned long flags;
5582 iommu = device_to_iommu(dev, &bus, &devfn);
5583 if (!iommu || dmar_disabled)
5586 if (!sm_supported(iommu) || !pasid_supported(iommu))
5589 ret = intel_iommu_enable_pasid(iommu, dev);
5593 spin_lock_irqsave(&device_domain_lock, flags);
5594 info = dev->archdata.iommu;
5595 info->auxd_enabled = 1;
5596 spin_unlock_irqrestore(&device_domain_lock, flags);
5601 static int intel_iommu_disable_auxd(struct device *dev)
5603 struct device_domain_info *info;
5604 unsigned long flags;
5606 spin_lock_irqsave(&device_domain_lock, flags);
5607 info = dev->archdata.iommu;
5608 if (!WARN_ON(!info))
5609 info->auxd_enabled = 0;
5610 spin_unlock_irqrestore(&device_domain_lock, flags);
5616 * A PCI express designated vendor specific extended capability is defined
5617 * in the section 3.7 of Intel scalable I/O virtualization technical spec
5618 * for system software and tools to detect endpoint devices supporting the
5619 * Intel scalable IO virtualization without host driver dependency.
5621 * Returns the address of the matching extended capability structure within
5622 * the device's PCI configuration space or 0 if the device does not support
5625 static int siov_find_pci_dvsec(struct pci_dev *pdev)
5630 pos = pci_find_next_ext_capability(pdev, 0, 0x23);
5632 pci_read_config_word(pdev, pos + 4, &vendor);
5633 pci_read_config_word(pdev, pos + 8, &id);
5634 if (vendor == PCI_VENDOR_ID_INTEL && id == 5)
5637 pos = pci_find_next_ext_capability(pdev, pos, 0x23);
5644 intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat)
5646 if (feat == IOMMU_DEV_FEAT_AUX) {
5649 if (!dev_is_pci(dev) || dmar_disabled ||
5650 !scalable_mode_support() || !iommu_pasid_support())
5653 ret = pci_pasid_features(to_pci_dev(dev));
5657 return !!siov_find_pci_dvsec(to_pci_dev(dev));
5664 intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
5666 if (feat == IOMMU_DEV_FEAT_AUX)
5667 return intel_iommu_enable_auxd(dev);
5673 intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
5675 if (feat == IOMMU_DEV_FEAT_AUX)
5676 return intel_iommu_disable_auxd(dev);
5682 intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
5684 struct device_domain_info *info = dev->archdata.iommu;
5686 if (feat == IOMMU_DEV_FEAT_AUX)
5687 return scalable_mode_support() && info && info->auxd_enabled;
5693 intel_iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
5695 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5697 return dmar_domain->default_pasid > 0 ?
5698 dmar_domain->default_pasid : -EINVAL;
5701 const struct iommu_ops intel_iommu_ops = {
5702 .capable = intel_iommu_capable,
5703 .domain_alloc = intel_iommu_domain_alloc,
5704 .domain_free = intel_iommu_domain_free,
5705 .attach_dev = intel_iommu_attach_device,
5706 .detach_dev = intel_iommu_detach_device,
5707 .aux_attach_dev = intel_iommu_aux_attach_device,
5708 .aux_detach_dev = intel_iommu_aux_detach_device,
5709 .aux_get_pasid = intel_iommu_aux_get_pasid,
5710 .map = intel_iommu_map,
5711 .unmap = intel_iommu_unmap,
5712 .iova_to_phys = intel_iommu_iova_to_phys,
5713 .add_device = intel_iommu_add_device,
5714 .remove_device = intel_iommu_remove_device,
5715 .get_resv_regions = intel_iommu_get_resv_regions,
5716 .put_resv_regions = intel_iommu_put_resv_regions,
5717 .device_group = pci_device_group,
5718 .dev_has_feat = intel_iommu_dev_has_feat,
5719 .dev_feat_enabled = intel_iommu_dev_feat_enabled,
5720 .dev_enable_feat = intel_iommu_dev_enable_feat,
5721 .dev_disable_feat = intel_iommu_dev_disable_feat,
5722 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5725 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5727 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5728 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
5732 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5733 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5734 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5735 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5736 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5737 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5738 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5740 static void quirk_iommu_rwbf(struct pci_dev *dev)
5743 * Mobile 4 Series Chipset neglects to set RWBF capability,
5744 * but needs it. Same seems to hold for the desktop versions.
5746 pci_info(dev, "Forcing write-buffer flush capability\n");
5750 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5751 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5752 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5753 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5754 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5755 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5756 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5759 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5760 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5761 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5762 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5763 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5764 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5765 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5766 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5768 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5772 if (pci_read_config_word(dev, GGC, &ggc))
5775 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5776 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5778 } else if (dmar_map_gfx) {
5779 /* we have to ensure the gfx device is idle before we flush */
5780 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
5781 intel_iommu_strict = 1;
5784 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5785 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5786 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5787 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5789 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5790 ISOCH DMAR unit for the Azalia sound device, but not give it any
5791 TLB entries, which causes it to deadlock. Check for that. We do
5792 this in a function called from init_dmars(), instead of in a PCI
5793 quirk, because we don't want to print the obnoxious "BIOS broken"
5794 message if VT-d is actually disabled.
5796 static void __init check_tylersburg_isoch(void)
5798 struct pci_dev *pdev;
5799 uint32_t vtisochctrl;
5801 /* If there's no Azalia in the system anyway, forget it. */
5802 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5807 /* System Management Registers. Might be hidden, in which case
5808 we can't do the sanity check. But that's OK, because the
5809 known-broken BIOSes _don't_ actually hide it, so far. */
5810 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5814 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5821 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5822 if (vtisochctrl & 1)
5825 /* Drop all bits other than the number of TLB entries */
5826 vtisochctrl &= 0x1c;
5828 /* If we have the recommended number of TLB entries (16), fine. */
5829 if (vtisochctrl == 0x10)
5832 /* Zero TLB entries? You get to ride the short bus to school. */
5834 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5835 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5836 dmi_get_system_info(DMI_BIOS_VENDOR),
5837 dmi_get_system_info(DMI_BIOS_VERSION),
5838 dmi_get_system_info(DMI_PRODUCT_VERSION));
5839 iommu_identity_mapping |= IDENTMAP_AZALIA;
5843 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",