1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Copyright 2013 Red Hat Inc.
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
8 * Heterogeneous Memory Management (HMM)
10 * See Documentation/vm/hmm.rst for reasons and overview of what HMM is and it
11 * is for. Here we focus on the HMM API description, with some explanation of
12 * the underlying implementation.
14 * Short description: HMM provides a set of helpers to share a virtual address
15 * space between CPU and a device, so that the device can access any valid
16 * address of the process (while still obeying memory protection). HMM also
17 * provides helpers to migrate process memory to device memory, and back. Each
18 * set of functionality (address space mirroring, and migration to and from
19 * device memory) can be used independently of the other.
22 * HMM address space mirroring API:
24 * Use HMM address space mirroring if you want to mirror a range of the CPU
25 * page tables of a process into a device page table. Here, "mirror" means "keep
26 * synchronized". Prerequisites: the device must provide the ability to write-
27 * protect its page tables (at PAGE_SIZE granularity), and must be able to
28 * recover from the resulting potential page faults.
30 * HMM guarantees that at any point in time, a given virtual address points to
31 * either the same memory in both CPU and device page tables (that is: CPU and
32 * device page tables each point to the same pages), or that one page table (CPU
33 * or device) points to no entry, while the other still points to the old page
34 * for the address. The latter case happens when the CPU page table update
35 * happens first, and then the update is mirrored over to the device page table.
36 * This does not cause any issue, because the CPU page table cannot start
37 * pointing to a new page until the device page table is invalidated.
39 * HMM uses mmu_notifiers to monitor the CPU page tables, and forwards any
40 * updates to each device driver that has registered a mirror. It also provides
41 * some API calls to help with taking a snapshot of the CPU page table, and to
42 * synchronize with any updates that might happen concurrently.
45 * HMM migration to and from device memory:
47 * HMM provides a set of helpers to hotplug device memory as ZONE_DEVICE, with
48 * a new MEMORY_DEVICE_PRIVATE type. This provides a struct page for each page
49 * of the device memory, and allows the device driver to manage its memory
50 * using those struct pages. Having struct pages for device memory makes
51 * migration easier. Because that memory is not addressable by the CPU it must
52 * never be pinned to the device; in other words, any CPU page fault can always
53 * cause the device memory to be migrated (copied/moved) back to regular memory.
55 * A new migrate helper (migrate_vma()) has been added (see mm/migrate.c) that
56 * allows use of a device DMA engine to perform the copy operation between
57 * regular system memory and device memory.
62 #include <linux/kconfig.h>
63 #include <asm/pgtable.h>
65 #include <linux/device.h>
66 #include <linux/migrate.h>
67 #include <linux/memremap.h>
68 #include <linux/completion.h>
69 #include <linux/mmu_notifier.h>
72 * hmm_pfn_flag_e - HMM flag enums
75 * HMM_PFN_VALID: pfn is valid. It has, at least, read permission.
76 * HMM_PFN_WRITE: CPU page table has write permission set
77 * HMM_PFN_DEVICE_PRIVATE: private device memory (ZONE_DEVICE)
79 * The driver provides a flags array for mapping page protections to device
80 * PTE bits. If the driver valid bit for an entry is bit 3,
81 * i.e., (entry & (1 << 3)), then the driver must provide
82 * an array in hmm_range.flags with hmm_range.flags[HMM_PFN_VALID] == 1 << 3.
83 * Same logic apply to all flags. This is the same idea as vm_page_prot in vma
84 * except that this is per device driver rather than per architecture.
89 HMM_PFN_DEVICE_PRIVATE,
94 * hmm_pfn_value_e - HMM pfn special value
97 * HMM_PFN_ERROR: corresponding CPU page table entry points to poisoned memory
98 * HMM_PFN_NONE: corresponding CPU page table entry is pte_none()
99 * HMM_PFN_SPECIAL: corresponding CPU page table entry is special; i.e., the
100 * result of vmf_insert_pfn() or vm_insert_page(). Therefore, it should not
101 * be mirrored by a device, because the entry will never have HMM_PFN_VALID
102 * set and the pfn value is undefined.
104 * Driver provides values for none entry, error entry, and special entry.
105 * Driver can alias (i.e., use same value) error and special, but
106 * it should not alias none with error or special.
108 * HMM pfn value returned by hmm_vma_get_pfns() or hmm_vma_fault() will be:
109 * hmm_range.values[HMM_PFN_ERROR] if CPU page table entry is poisonous,
110 * hmm_range.values[HMM_PFN_NONE] if there is no CPU page table entry,
111 * hmm_range.values[HMM_PFN_SPECIAL] if CPU page table entry is a special one
113 enum hmm_pfn_value_e {
121 * struct hmm_range - track invalidation lock on virtual address range
123 * @notifier: a mmu_interval_notifier that includes the start/end
124 * @notifier_seq: result of mmu_interval_read_begin()
125 * @hmm: the core HMM structure this range is active against
126 * @vma: the vm area struct for the range
127 * @list: all range lock are on a list
128 * @start: range virtual start address (inclusive)
129 * @end: range virtual end address (exclusive)
130 * @pfns: array of pfns (big enough for the range)
131 * @flags: pfn flags to match device driver page table
132 * @values: pfn value for some special case (none, special, error, ...)
133 * @default_flags: default flags for the range (write, read, ... see hmm doc)
134 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
135 * @pfn_shifts: pfn shift value (should be <= PAGE_SHIFT)
136 * @valid: pfns array did not change since it has been fill by an HMM function
139 struct mmu_interval_notifier *notifier;
140 unsigned long notifier_seq;
144 const uint64_t *flags;
145 const uint64_t *values;
146 uint64_t default_flags;
147 uint64_t pfn_flags_mask;
152 * hmm_device_entry_to_page() - return struct page pointed to by a device entry
153 * @range: range use to decode device entry value
154 * @entry: device entry value to get corresponding struct page from
155 * Return: struct page pointer if entry is a valid, NULL otherwise
157 * If the device entry is valid (ie valid flag set) then return the struct page
158 * matching the entry value. Otherwise return NULL.
160 static inline struct page *hmm_device_entry_to_page(const struct hmm_range *range,
163 if (entry == range->values[HMM_PFN_NONE])
165 if (entry == range->values[HMM_PFN_ERROR])
167 if (entry == range->values[HMM_PFN_SPECIAL])
169 if (!(entry & range->flags[HMM_PFN_VALID]))
171 return pfn_to_page(entry >> range->pfn_shift);
175 * hmm_device_entry_to_pfn() - return pfn value store in a device entry
176 * @range: range use to decode device entry value
177 * @entry: device entry to extract pfn from
178 * Return: pfn value if device entry is valid, -1UL otherwise
180 static inline unsigned long
181 hmm_device_entry_to_pfn(const struct hmm_range *range, uint64_t pfn)
183 if (pfn == range->values[HMM_PFN_NONE])
185 if (pfn == range->values[HMM_PFN_ERROR])
187 if (pfn == range->values[HMM_PFN_SPECIAL])
189 if (!(pfn & range->flags[HMM_PFN_VALID]))
191 return (pfn >> range->pfn_shift);
195 * hmm_device_entry_from_page() - create a valid device entry for a page
196 * @range: range use to encode HMM pfn value
197 * @page: page for which to create the device entry
198 * Return: valid device entry for the page
200 static inline uint64_t hmm_device_entry_from_page(const struct hmm_range *range,
203 return (page_to_pfn(page) << range->pfn_shift) |
204 range->flags[HMM_PFN_VALID];
208 * hmm_device_entry_from_pfn() - create a valid device entry value from pfn
209 * @range: range use to encode HMM pfn value
210 * @pfn: pfn value for which to create the device entry
211 * Return: valid device entry for the pfn
213 static inline uint64_t hmm_device_entry_from_pfn(const struct hmm_range *range,
216 return (pfn << range->pfn_shift) |
217 range->flags[HMM_PFN_VALID];
221 * Retry fault if non-blocking, drop mmap_sem and return -EAGAIN in that case.
223 #define HMM_FAULT_ALLOW_RETRY (1 << 0)
225 /* Don't fault in missing PTEs, just snapshot the current state. */
226 #define HMM_FAULT_SNAPSHOT (1 << 1)
228 #ifdef CONFIG_HMM_MIRROR
230 * Please see Documentation/vm/hmm.rst for how to use the range API.
232 long hmm_range_fault(struct hmm_range *range, unsigned int flags);
234 static inline long hmm_range_fault(struct hmm_range *range, unsigned int flags)
241 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
243 * When waiting for mmu notifiers we need some kind of time out otherwise we
244 * could potentialy wait for ever, 1000ms ie 1s sounds like a long time to
247 #define HMM_RANGE_DEFAULT_TIMEOUT 1000
249 #endif /* LINUX_HMM_H */