arm64: Implement pmem API support

Add a clean-to-point-of-persistence cache maintenance helper, and wire
up the basic architectural support for the pmem driver based on it.

Reviewed-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
[catalin.marinas@arm.com: move arch_*_pmem() functions to arch/arm64/mm/flush.c]
[catalin.marinas@arm.com: change dmb(sy) to dmb(osh)]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index dfd908630631a30851a8f573e41dbe8b21af5f19..0b0576a5472451d6b1d368c996fa476da4063225 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -960,6 +960,17 @@ config ARM64_UAO
          regular load/store instructions if the cpu does not implement the
          feature.
 
+config ARM64_PMEM
+       bool "Enable support for persistent memory"
+       select ARCH_HAS_PMEM_API
+       help
+         Say Y to enable support for the persistent memory API based on the
+         ARMv8.2 DCPoP feature.
+
+         The feature is detected at runtime, and the kernel will use DC CVAC
+         operations if DC CVAP is not supported (following the behaviour of
+         DC CVAP itself if the system does not define a point of persistence).
+
 endmenu
 
 config ARM64_MODULE_CMODEL_LARGE

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 1b67c3782d005b5d1872e0ceedcf4250b3b3d8c9..5d8903c450316becec7058627637e3f40bcc2098 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -352,6 +352,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
        dc      \op, \kaddr
 alternative_else
        dc      civac, \kaddr
+alternative_endif
+       .elseif (\op == cvap)
+alternative_if ARM64_HAS_DCPOP
+       sys 3, c7, c12, 1, \kaddr       // dc cvap
+alternative_else
+       dc      cvac, \kaddr
 alternative_endif
        .else
        dc      \op, \kaddr

diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index b4b43a94dffd98ae1f93ee0806595eda5ea8ca19..76d1cc85d5b115915aaa63138121e9ba286d8f4e 100644 (file)
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -69,6 +69,7 @@ extern void flush_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __inval_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);
+extern void __clean_dcache_area_pop(void *addr, size_t len);
 extern void __clean_dcache_area_pou(void *addr, size_t len);
 extern long __flush_cache_user_range(unsigned long start, unsigned long end);
 extern void sync_icache_aliases(void *kaddr, unsigned long len);

diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 8d2272c6822c703b2076eccc04667ecb775c6860..8da621627d7c892a0833d325991e567678b4eced 100644 (file)
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -39,7 +39,8 @@
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003     18
 #define ARM64_WORKAROUND_858921                        19
 #define ARM64_WORKAROUND_CAVIUM_30115          20
+#define ARM64_HAS_DCPOP                                21
 
-#define ARM64_NCAPS                            21
+#define ARM64_NCAPS                            22
 
 #endif /* __ASM_CPUCAPS_H */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index a2542ef3ff25c59140dcbc063cd6fa1be2a257ff..cd52d365d1f01aefae2f63e6a0207e65e0fdcb57 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -889,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .min_field_value = 0,
                .matches = has_no_fpsimd,
        },
+#ifdef CONFIG_ARM64_PMEM
+       {
+               .desc = "Data cache clean to Point of Persistence",
+               .capability = ARM64_HAS_DCPOP,
+               .def_scope = SCOPE_SYSTEM,
+               .matches = has_cpuid_feature,
+               .sys_reg = SYS_ID_AA64ISAR1_EL1,
+               .field_pos = ID_AA64ISAR1_DPB_SHIFT,
+               .min_field_value = 1,
+       },
+#endif
        {},
 };
 

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index ed47fbbb4b05783e15174885d5dc90f2fb43cf98..7f1dbe962cf581c88eb488f01636f78996b4ee43 100644 (file)
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -171,6 +171,20 @@ __dma_clean_area:
 ENDPIPROC(__clean_dcache_area_poc)
 ENDPROC(__dma_clean_area)
 
+/*
+ *     __clean_dcache_area_pop(kaddr, size)
+ *
+ *     Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ *     are cleaned to the PoP.
+ *
+ *     - kaddr   - kernel address
+ *     - size    - size in question
+ */
+ENTRY(__clean_dcache_area_pop)
+       dcache_by_line_op cvap, sy, x0, x1, x2, x3
+       ret
+ENDPIPROC(__clean_dcache_area_pop)
+
 /*
  *     __dma_flush_area(start, size)
  *

diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 21a8d828cbf4f84db380ffdc5060072852da19a7..280f90ff33a2151f97eb8b0906eea88842be1ac6 100644 (file)
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page);
  * Additional functions defined in assembly.
  */
 EXPORT_SYMBOL(flush_icache_range);
+
+#ifdef CONFIG_ARCH_HAS_PMEM_API
+static inline void arch_wb_cache_pmem(void *addr, size_t size)
+{
+       /* Ensure order against any prior non-cacheable writes */
+       dmb(osh);
+       __clean_dcache_area_pop(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
+
+static inline void arch_invalidate_pmem(void *addr, size_t size)
+{
+       __inval_dcache_area(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
+#endif