arm64: handle 52-bit physical addresses in page table entries

The top 4 bits of a 52-bit physical address are positioned at bits
12..15 of a page table entry. Introduce macros to convert between a
physical address and its placement in a table entry, and change all
macros/functions that access PTEs to use them.

Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Tested-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Tested-by: Bob Picco <bob.picco@oracle.com>
Reviewed-by: Bob Picco <bob.picco@oracle.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
[catalin.marinas@arm.com: some long lines wrapped]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 9810ebf949b31c480141c2ec739b42072432393f..b3f7b68b042dc43e5b062a4ecc38f1ca2006a0d8 100644 (file)
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -287,6 +287,7 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
                                       unsigned long hyp_idmap_start)
 {
        int idmap_idx;
+       u64 pgd_addr;
 
        /*
         * Use the first entry to access the HYP mappings. It is
@@ -294,7 +295,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
         * extended idmap.
         */
        VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
-       merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);
+       pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));
+       merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);
 
        /*
         * Create another extended level entry that points to the boot HYP map,
@@ -304,7 +306,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
         */
        idmap_idx = hyp_idmap_start >> VA_BITS;
        VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
-       merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
+       pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));
+       merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);
 }
 
 static inline unsigned int kvm_get_vmid_bits(void)

diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 5ca6a573a701249d87f080a24ab790b87058bee8..e9d9f1b006efec5708fd0f33c006d2e017711a4d 100644 (file)
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -44,7 +44,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 
 static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
 {
-       set_pud(pud, __pud(pmd | prot));
+       set_pud(pud, __pud(__phys_to_pud_val(pmd) | prot));
 }
 
 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
@@ -73,7 +73,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
 
 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
 {
-       set_pgd(pgdp, __pgd(pud | prot));
+       set_pgd(pgdp, __pgd(__phys_to_pgd_val(pud) | prot));
 }
 
 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
@@ -129,7 +129,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
                                  pmdval_t prot)
 {
-       set_pmd(pmdp, __pmd(pte | prot));
+       set_pmd(pmdp, __pmd(__phys_to_pmd_val(pte) | prot));
 }
 
 /*

diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 5513ccd687f4fe84c8f546fc8bda6d234f6319ad..85069f37ae37e0b73ca7dbf6e470e44c4a8ff363 100644 (file)
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -168,10 +168,12 @@
 #define PTE_UXN                        (_AT(pteval_t, 1) << 54)        /* User XN */
 #define PTE_HYP_XN             (_AT(pteval_t, 1) << 54)        /* HYP XN */
 
-#ifdef CONFIG_ARM64_PA_BITS_52
 #define PTE_ADDR_LOW           (((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
+#ifdef CONFIG_ARM64_PA_BITS_52
 #define PTE_ADDR_HIGH          (_AT(pteval_t, 0xf) << 12)
-#define PTE_ADDR_MASK_52       (PTE_ADDR_LOW | PTE_ADDR_HIGH)
+#define PTE_ADDR_MASK          (PTE_ADDR_LOW | PTE_ADDR_HIGH)
+#else
+#define PTE_ADDR_MASK          PTE_ADDR_LOW
 #endif
 
 /*

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 5d9554fb2692f00ca1b872e97e9ddf7ae5bd75bd..4fd8af303e2c4fef8a714896870c2f03e24aa4ac 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -57,9 +57,22 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 
 #define pte_ERROR(pte)         __pte_error(__FILE__, __LINE__, pte_val(pte))
 
-#define pte_pfn(pte)           ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)
+/*
+ * Macros to convert between a physical address and its placement in a
+ * page table entry, taking care of 52-bit addresses.
+ */
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define __pte_to_phys(pte)     \
+       ((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
+#define __phys_to_pte_val(phys)        (((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
+#else
+#define __pte_to_phys(pte)     (pte_val(pte) & PTE_ADDR_MASK)
+#define __phys_to_pte_val(phys)        (phys)
+#endif
 
-#define pfn_pte(pfn,prot)      (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+#define pte_pfn(pte)           (__pte_to_phys(pte) >> PAGE_SHIFT)
+#define pfn_pte(pfn,prot)      \
+       __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
 #define pte_none(pte)          (!pte_val(pte))
 #define pte_clear(mm,addr,ptep)        set_pte(ptep, __pte(0))
@@ -284,6 +297,11 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)
 
 #define __HAVE_ARCH_PTE_SPECIAL
 
+static inline pte_t pgd_pte(pgd_t pgd)
+{
+       return __pte(pgd_val(pgd));
+}
+
 static inline pte_t pud_pte(pud_t pud)
 {
        return __pte(pud_val(pud));
@@ -349,16 +367,24 @@ static inline int pmd_protnone(pmd_t pmd)
 
 #define pmd_mkhuge(pmd)                (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
 
-#define pmd_pfn(pmd)           (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
-#define pfn_pmd(pfn,prot)      (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+#define __pmd_to_phys(pmd)     __pte_to_phys(pmd_pte(pmd))
+#define __phys_to_pmd_val(phys)        __phys_to_pte_val(phys)
+#define pmd_pfn(pmd)           ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
+#define pfn_pmd(pfn,prot)      __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 #define mk_pmd(page,prot)      pfn_pmd(page_to_pfn(page),prot)
 
 #define pud_write(pud)         pte_write(pud_pte(pud))
-#define pud_pfn(pud)           (((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
-#define pfn_pud(pfn,prot)      (__pud(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+
+#define __pud_to_phys(pud)     __pte_to_phys(pud_pte(pud))
+#define __phys_to_pud_val(phys)        __phys_to_pte_val(phys)
+#define pud_pfn(pud)           ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
+#define pfn_pud(pfn,prot)      __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
 #define set_pmd_at(mm, addr, pmdp, pmd)        set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
 
+#define __pgd_to_phys(pgd)     __pte_to_phys(pgd_pte(pgd))
+#define __phys_to_pgd_val(phys)        __phys_to_pte_val(phys)
+
 #define __pgprot_modify(prot,mask,bits) \
        __pgprot((pgprot_val(prot) & ~(mask)) | (bits))
 
@@ -409,7 +435,7 @@ static inline void pmd_clear(pmd_t *pmdp)
 
 static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 {
-       return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK;
+       return __pmd_to_phys(pmd);
 }
 
 /* Find an entry in the third-level page table. */
@@ -427,7 +453,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 #define pte_set_fixmap_offset(pmd, addr)       pte_set_fixmap(pte_offset_phys(pmd, addr))
 #define pte_clear_fixmap()             clear_fixmap(FIX_PTE)
 
-#define pmd_page(pmd)          pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
+#define pmd_page(pmd)          pfn_to_page(__phys_to_pfn(__pmd_to_phys(pmd)))
 
 /* use ONLY for statically allocated translation tables */
 #define pte_offset_kimg(dir,addr)      ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
@@ -460,7 +486,7 @@ static inline void pud_clear(pud_t *pudp)
 
 static inline phys_addr_t pud_page_paddr(pud_t pud)
 {
-       return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK;
+       return __pud_to_phys(pud);
 }
 
 /* Find an entry in the second-level page table. */
@@ -473,7 +499,7 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
 #define pmd_set_fixmap_offset(pud, addr)       pmd_set_fixmap(pmd_offset_phys(pud, addr))
 #define pmd_clear_fixmap()             clear_fixmap(FIX_PMD)
 
-#define pud_page(pud)          pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
+#define pud_page(pud)          pfn_to_page(__phys_to_pfn(__pud_to_phys(pud)))
 
 /* use ONLY for statically allocated translation tables */
 #define pmd_offset_kimg(dir,addr)      ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
@@ -512,7 +538,7 @@ static inline void pgd_clear(pgd_t *pgdp)
 
 static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
 {
-       return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK;
+       return __pgd_to_phys(pgd);
 }
 
 /* Find an entry in the frst-level page table. */
@@ -525,7 +551,7 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
 #define pud_set_fixmap_offset(pgd, addr)       pud_set_fixmap(pud_offset_phys(pgd, addr))
 #define pud_clear_fixmap()             clear_fixmap(FIX_PUD)
 
-#define pgd_page(pgd)          pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK))
+#define pgd_page(pgd)          pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
 
 /* use ONLY for statically allocated translation tables */
 #define pud_offset_kimg(dir,addr)      ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index bb06223691ba835329e61039291ea0085b15a889..eeec0001e204f7a5cddf5bbafa27de2e32025602 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -161,7 +161,7 @@ ENDPROC(preserve_boot_args)
         * supporting this configuration with 64K pages.
         */
        orr     \pte, \phys, \phys, lsr #36
-       and     \pte, \pte, #PTE_ADDR_MASK_52
+       and     \pte, \pte, #PTE_ADDR_MASK
 #else
        mov     \pte, \phys
 #endif