1 // SPDX-License-Identifier: GPL-2.0
3 * PARISC64 Huge TLB page support.
5 * This parisc implementation is heavily based on the SPARC and x86 code.
7 * Copyright (C) 2015 Helge Deller <deller@gmx.de>
12 #include <linux/sched/mm.h>
13 #include <linux/hugetlb.h>
14 #include <linux/pagemap.h>
15 #include <linux/sysctl.h>
18 #include <asm/pgalloc.h>
20 #include <asm/tlbflush.h>
21 #include <asm/cacheflush.h>
22 #include <asm/mmu_context.h>
26 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
27 unsigned long len, unsigned long pgoff, unsigned long flags)
29 struct hstate *h = hstate_file(file);
31 if (len & ~huge_page_mask(h))
36 if (flags & MAP_FIXED)
37 if (prepare_hugepage_range(file, addr, len))
41 addr = ALIGN(addr, huge_page_size(h));
43 /* we need to make sure the colouring is OK */
44 return arch_get_unmapped_area(file, addr, len, pgoff, flags);
48 pte_t *huge_pte_alloc(struct mm_struct *mm,
49 unsigned long addr, unsigned long sz)
57 /* We must align the address, because our caller will run
58 * set_huge_pte_at() on whatever we return, which writes out
59 * all of the sub-ptes for the hugepage range. So we have
60 * to give it the first such sub-pte.
64 pgd = pgd_offset(mm, addr);
65 p4d = p4d_offset(pgd, addr);
66 pud = pud_alloc(mm, p4d, addr);
68 pmd = pmd_alloc(mm, pud, addr);
70 pte = pte_alloc_map(mm, pmd, addr);
75 pte_t *huge_pte_offset(struct mm_struct *mm,
76 unsigned long addr, unsigned long sz)
86 pgd = pgd_offset(mm, addr);
87 if (!pgd_none(*pgd)) {
88 p4d = p4d_offset(pgd, addr);
89 if (!p4d_none(*p4d)) {
90 pud = pud_offset(p4d, addr);
91 if (!pud_none(*pud)) {
92 pmd = pmd_offset(pud, addr);
94 pte = pte_offset_map(pmd, addr);
101 /* Purge data and instruction TLB entries. Must be called holding
102 * the pa_tlb_lock. The TLB purge instructions are slow on SMP
103 * machines since the purge must be broadcast to all CPUs.
105 static inline void purge_tlb_entries_huge(struct mm_struct *mm, unsigned long addr)
109 /* We may use multiple physical huge pages (e.g. 2x1 MB) to emulate
110 * Linux standard huge pages (e.g. 2 MB) */
111 BUILD_BUG_ON(REAL_HPAGE_SHIFT > HPAGE_SHIFT);
114 addr |= _HUGE_PAGE_SIZE_ENCODING_DEFAULT;
116 for (i = 0; i < (1 << (HPAGE_SHIFT-REAL_HPAGE_SHIFT)); i++) {
117 purge_tlb_entries(mm, addr);
118 addr += (1UL << REAL_HPAGE_SHIFT);
122 /* __set_huge_pte_at() must be called holding the pa_tlb_lock. */
123 static void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
124 pte_t *ptep, pte_t entry)
126 unsigned long addr_start;
132 for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
133 set_pte(ptep, entry);
137 pte_val(entry) += PAGE_SIZE;
140 purge_tlb_entries_huge(mm, addr_start);
143 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
144 pte_t *ptep, pte_t entry)
148 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
149 __set_huge_pte_at(mm, addr, ptep, entry);
150 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
154 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
160 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
162 __set_huge_pte_at(mm, addr, ptep, __pte(0));
163 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
169 void huge_ptep_set_wrprotect(struct mm_struct *mm,
170 unsigned long addr, pte_t *ptep)
175 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
177 __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte));
178 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
181 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
182 unsigned long addr, pte_t *ptep,
183 pte_t pte, int dirty)
187 struct mm_struct *mm = vma->vm_mm;
189 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);
190 changed = !pte_same(*ptep, pte);
192 __set_huge_pte_at(mm, addr, ptep, pte);
194 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);
199 int pmd_huge(pmd_t pmd)
204 int pud_huge(pud_t pud)