1 #include <linux/init.h>
4 #include <linux/spinlock.h>
6 #include <linux/interrupt.h>
7 #include <linux/export.h>
10 #include <asm/tlbflush.h>
11 #include <asm/mmu_context.h>
12 #include <asm/cache.h>
14 #include <asm/uv/uv.h>
15 #include <linux/debugfs.h>
18 * TLB flushing, formerly SMP-only
21 * These mean you can really definitely utterly forget about
22 * writing to user space from interrupts. (Its not allowed anyway).
24 * Optimizations Manfred Spraul <manfred@colorfullife.com>
26 * More scalable flush, from Andi Kleen
28 * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
31 atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1);
33 void leave_mm(int cpu)
35 struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
38 * It's plausible that we're in lazy TLB mode while our mm is init_mm.
39 * If so, our callers still expect us to flush the TLB, but there
40 * aren't any user TLB entries in init_mm to worry about.
42 * This needs to happen before any other sanity checks due to
43 * intel_idle's shenanigans.
45 if (loaded_mm == &init_mm)
48 /* Warn if we're not lazy. */
49 WARN_ON(cpumask_test_cpu(smp_processor_id(), mm_cpumask(loaded_mm)));
51 switch_mm(NULL, &init_mm, NULL);
54 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
55 struct task_struct *tsk)
59 local_irq_save(flags);
60 switch_mm_irqs_off(prev, next, tsk);
61 local_irq_restore(flags);
64 void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
65 struct task_struct *tsk)
67 struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);
68 unsigned cpu = smp_processor_id();
72 * NB: The scheduler will call us with prev == next when switching
73 * from lazy TLB mode to normal mode if active_mm isn't changing.
74 * When this happens, we don't assume that CR3 (and hence
75 * cpu_tlbstate.loaded_mm) matches next.
77 * NB: leave_mm() calls us with prev == NULL and tsk == NULL.
80 /* We don't want flush_tlb_func_* to run concurrently with us. */
81 if (IS_ENABLED(CONFIG_PROVE_LOCKING))
82 WARN_ON_ONCE(!irqs_disabled());
85 * Verify that CR3 is what we think it is. This will catch
86 * hypothetical buggy code that directly switches to swapper_pg_dir
87 * without going through leave_mm() / switch_mm_irqs_off().
89 VM_BUG_ON(read_cr3_pa() != __pa(real_prev->pgd));
91 if (real_prev == next) {
92 VM_BUG_ON(this_cpu_read(cpu_tlbstate.ctxs[0].ctx_id) !=
93 next->context.ctx_id);
95 if (cpumask_test_cpu(cpu, mm_cpumask(next))) {
97 * There's nothing to do: we weren't lazy, and we
98 * aren't changing our mm. We don't need to flush
99 * anything, nor do we need to update CR3, CR4, or
105 /* Resume remote flushes and then read tlb_gen. */
106 cpumask_set_cpu(cpu, mm_cpumask(next));
107 next_tlb_gen = atomic64_read(&next->context.tlb_gen);
109 if (this_cpu_read(cpu_tlbstate.ctxs[0].tlb_gen) < next_tlb_gen) {
111 * Ideally, we'd have a flush_tlb() variant that
112 * takes the known CR3 value as input. This would
113 * be faster on Xen PV and on hypothetical CPUs
114 * on which INVPCID is fast.
116 this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen,
118 write_cr3(__sme_pa(next->pgd));
119 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH,
124 * We just exited lazy mode, which means that CR4 and/or LDTR
125 * may be stale. (Changes to the required CR4 and LDTR states
126 * are not reflected in tlb_gen.)
129 VM_BUG_ON(this_cpu_read(cpu_tlbstate.ctxs[0].ctx_id) ==
130 next->context.ctx_id);
132 if (IS_ENABLED(CONFIG_VMAP_STACK)) {
134 * If our current stack is in vmalloc space and isn't
135 * mapped in the new pgd, we'll double-fault. Forcibly
138 unsigned int index = pgd_index(current_stack_pointer());
139 pgd_t *pgd = next->pgd + index;
141 if (unlikely(pgd_none(*pgd)))
142 set_pgd(pgd, init_mm.pgd[index]);
145 /* Stop remote flushes for the previous mm */
146 if (cpumask_test_cpu(cpu, mm_cpumask(real_prev)))
147 cpumask_clear_cpu(cpu, mm_cpumask(real_prev));
149 VM_WARN_ON_ONCE(cpumask_test_cpu(cpu, mm_cpumask(next)));
152 * Start remote flushes and then read tlb_gen.
154 cpumask_set_cpu(cpu, mm_cpumask(next));
155 next_tlb_gen = atomic64_read(&next->context.tlb_gen);
157 this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, next->context.ctx_id);
158 this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, next_tlb_gen);
159 this_cpu_write(cpu_tlbstate.loaded_mm, next);
160 write_cr3(__sme_pa(next->pgd));
162 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
166 switch_ldt(real_prev, next);
170 * flush_tlb_func_common()'s memory ordering requirement is that any
171 * TLB fills that happen after we flush the TLB are ordered after we
172 * read active_mm's tlb_gen. We don't need any explicit barriers
173 * because all x86 flush operations are serializing and the
174 * atomic64_read operation won't be reordered by the compiler.
176 static void flush_tlb_func_common(const struct flush_tlb_info *f,
177 bool local, enum tlb_flush_reason reason)
180 * We have three different tlb_gen values in here. They are:
182 * - mm_tlb_gen: the latest generation.
183 * - local_tlb_gen: the generation that this CPU has already caught
185 * - f->new_tlb_gen: the generation that the requester of the flush
186 * wants us to catch up to.
188 struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
189 u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen);
190 u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[0].tlb_gen);
192 /* This code cannot presently handle being reentered. */
193 VM_WARN_ON(!irqs_disabled());
195 VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[0].ctx_id) !=
196 loaded_mm->context.ctx_id);
198 if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(loaded_mm))) {
200 * We're in lazy mode -- don't flush. We can get here on
201 * remote flushes due to races and on local flushes if a
202 * kernel thread coincidentally flushes the mm it's lazily
208 if (unlikely(local_tlb_gen == mm_tlb_gen)) {
210 * There's nothing to do: we're already up to date. This can
211 * happen if two concurrent flushes happen -- the first flush to
212 * be handled can catch us all the way up, leaving no work for
215 trace_tlb_flush(reason, 0);
219 WARN_ON_ONCE(local_tlb_gen > mm_tlb_gen);
220 WARN_ON_ONCE(f->new_tlb_gen > mm_tlb_gen);
223 * If we get to this point, we know that our TLB is out of date.
224 * This does not strictly imply that we need to flush (it's
225 * possible that f->new_tlb_gen <= local_tlb_gen), but we're
226 * going to need to flush in the very near future, so we might
227 * as well get it over with.
229 * The only question is whether to do a full or partial flush.
231 * We do a partial flush if requested and two extra conditions
234 * 1. f->new_tlb_gen == local_tlb_gen + 1. We have an invariant that
235 * we've always done all needed flushes to catch up to
236 * local_tlb_gen. If, for example, local_tlb_gen == 2 and
237 * f->new_tlb_gen == 3, then we know that the flush needed to bring
238 * us up to date for tlb_gen 3 is the partial flush we're
241 * As an example of why this check is needed, suppose that there
242 * are two concurrent flushes. The first is a full flush that
243 * changes context.tlb_gen from 1 to 2. The second is a partial
244 * flush that changes context.tlb_gen from 2 to 3. If they get
245 * processed on this CPU in reverse order, we'll see
246 * local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
247 * If we were to use __flush_tlb_single() and set local_tlb_gen to
248 * 3, we'd be break the invariant: we'd update local_tlb_gen above
249 * 1 without the full flush that's needed for tlb_gen 2.
251 * 2. f->new_tlb_gen == mm_tlb_gen. This is purely an optimiation.
252 * Partial TLB flushes are not all that much cheaper than full TLB
253 * flushes, so it seems unlikely that it would be a performance win
254 * to do a partial flush if that won't bring our TLB fully up to
255 * date. By doing a full flush instead, we can increase
256 * local_tlb_gen all the way to mm_tlb_gen and we can probably
257 * avoid another flush in the very near future.
259 if (f->end != TLB_FLUSH_ALL &&
260 f->new_tlb_gen == local_tlb_gen + 1 &&
261 f->new_tlb_gen == mm_tlb_gen) {
264 unsigned long nr_pages = (f->end - f->start) >> PAGE_SHIFT;
267 while (addr < f->end) {
268 __flush_tlb_single(addr);
272 count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_pages);
273 trace_tlb_flush(reason, nr_pages);
278 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
279 trace_tlb_flush(reason, TLB_FLUSH_ALL);
282 /* Both paths above update our state to mm_tlb_gen. */
283 this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, mm_tlb_gen);
286 static void flush_tlb_func_local(void *info, enum tlb_flush_reason reason)
288 const struct flush_tlb_info *f = info;
290 flush_tlb_func_common(f, true, reason);
293 static void flush_tlb_func_remote(void *info)
295 const struct flush_tlb_info *f = info;
297 inc_irq_stat(irq_tlb_count);
299 if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.loaded_mm))
302 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
303 flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN);
306 void native_flush_tlb_others(const struct cpumask *cpumask,
307 const struct flush_tlb_info *info)
309 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
310 if (info->end == TLB_FLUSH_ALL)
311 trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
313 trace_tlb_flush(TLB_REMOTE_SEND_IPI,
314 (info->end - info->start) >> PAGE_SHIFT);
316 if (is_uv_system()) {
318 * This whole special case is confused. UV has a "Broadcast
319 * Assist Unit", which seems to be a fancy way to send IPIs.
320 * Back when x86 used an explicit TLB flush IPI, UV was
321 * optimized to use its own mechanism. These days, x86 uses
322 * smp_call_function_many(), but UV still uses a manual IPI,
323 * and that IPI's action is out of date -- it does a manual
324 * flush instead of calling flush_tlb_func_remote(). This
325 * means that the percpu tlb_gen variables won't be updated
326 * and we'll do pointless flushes on future context switches.
328 * Rather than hooking native_flush_tlb_others() here, I think
329 * that UV should be updated so that smp_call_function_many(),
330 * etc, are optimal on UV.
334 cpu = smp_processor_id();
335 cpumask = uv_flush_tlb_others(cpumask, info);
337 smp_call_function_many(cpumask, flush_tlb_func_remote,
341 smp_call_function_many(cpumask, flush_tlb_func_remote,
346 * See Documentation/x86/tlb.txt for details. We choose 33
347 * because it is large enough to cover the vast majority (at
348 * least 95%) of allocations, and is small enough that we are
349 * confident it will not cause too much overhead. Each single
350 * flush is about 100 ns, so this caps the maximum overhead at
353 * This is in units of pages.
355 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
357 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
358 unsigned long end, unsigned long vmflag)
362 struct flush_tlb_info info = {
368 /* This is also a barrier that synchronizes with switch_mm(). */
369 info.new_tlb_gen = inc_mm_tlb_gen(mm);
371 /* Should we flush just the requested range? */
372 if ((end != TLB_FLUSH_ALL) &&
373 !(vmflag & VM_HUGETLB) &&
374 ((end - start) >> PAGE_SHIFT) <= tlb_single_page_flush_ceiling) {
379 info.end = TLB_FLUSH_ALL;
382 if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
383 VM_WARN_ON(irqs_disabled());
385 flush_tlb_func_local(&info, TLB_LOCAL_MM_SHOOTDOWN);
389 if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids)
390 flush_tlb_others(mm_cpumask(mm), &info);
396 static void do_flush_tlb_all(void *info)
398 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
402 void flush_tlb_all(void)
404 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
405 on_each_cpu(do_flush_tlb_all, NULL, 1);
408 static void do_kernel_range_flush(void *info)
410 struct flush_tlb_info *f = info;
413 /* flush range by one by one 'invlpg' */
414 for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
415 __flush_tlb_single(addr);
418 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
421 /* Balance as user space task's flush, a bit conservative */
422 if (end == TLB_FLUSH_ALL ||
423 (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
424 on_each_cpu(do_flush_tlb_all, NULL, 1);
426 struct flush_tlb_info info;
429 on_each_cpu(do_kernel_range_flush, &info, 1);
433 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
435 struct flush_tlb_info info = {
438 .end = TLB_FLUSH_ALL,
443 if (cpumask_test_cpu(cpu, &batch->cpumask)) {
444 VM_WARN_ON(irqs_disabled());
446 flush_tlb_func_local(&info, TLB_LOCAL_SHOOTDOWN);
450 if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
451 flush_tlb_others(&batch->cpumask, &info);
453 cpumask_clear(&batch->cpumask);
458 static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
459 size_t count, loff_t *ppos)
464 len = sprintf(buf, "%ld\n", tlb_single_page_flush_ceiling);
465 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
468 static ssize_t tlbflush_write_file(struct file *file,
469 const char __user *user_buf, size_t count, loff_t *ppos)
475 len = min(count, sizeof(buf) - 1);
476 if (copy_from_user(buf, user_buf, len))
480 if (kstrtoint(buf, 0, &ceiling))
486 tlb_single_page_flush_ceiling = ceiling;
490 static const struct file_operations fops_tlbflush = {
491 .read = tlbflush_read_file,
492 .write = tlbflush_write_file,
493 .llseek = default_llseek,
496 static int __init create_tlb_single_page_flush_ceiling(void)
498 debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR | S_IWUSR,
499 arch_debugfs_dir, NULL, &fops_tlbflush);
502 late_initcall(create_tlb_single_page_flush_ceiling);