powerpc/mm: Optimize detection of thread local mm's

Instead of comparing the whole CPU mask every time, let's
keep a counter of how many bits are set in the mask. Thus
testing for a local mm only requires testing if that counter
is 1 and the current CPU bit is set in the mask.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index 1a220cdff92369acdea4e2f9cef1830a6366d47b..c3b00e8ff79103876b3368a5cf3b4d708c579bf8 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -83,6 +83,9 @@ typedef struct {
        mm_context_id_t id;
        u16 user_psize;         /* page size index */
 
+       /* Number of bits in the mm_cpumask */
+       atomic_t active_cpus;
+
        /* NPU NMMU context */
        struct npu_context *npu_context;
 

diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index fb99c27bbf5ed1cc2c45b189099d0203f9303a6c..2338abf6101a416480f86cb89532e73145eaa27d 100644 (file)
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -96,6 +96,14 @@ static inline void switch_mm_pgdir(struct task_struct *tsk,
                                   struct mm_struct *mm) { }
 #endif
 
+#ifdef CONFIG_PPC_BOOK3S_64
+static inline void inc_mm_active_cpus(struct mm_struct *mm)
+{
+       atomic_inc(&mm->context.active_cpus);
+}
+#else
+static inline void inc_mm_active_cpus(struct mm_struct *mm) { }
+#endif
 
 /*
  * switch_mm is the entry point called from the architecture independent
@@ -110,6 +118,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
        /* Mark this context has been used on the new CPU */
        if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
                cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+               inc_mm_active_cpus(next);
 
                /*
                 * This full barrier orders the store to the cpumask above vs

diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index 609557569f6520f13167838ef11d4dfc1401a0c7..a7eabff27a0faa24904485af089f8434eeee246c 100644 (file)
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h
@@ -69,13 +69,22 @@ static inline int mm_is_core_local(struct mm_struct *mm)
                              topology_sibling_cpumask(smp_processor_id()));
 }
 
+#ifdef CONFIG_PPC_BOOK3S_64
+static inline int mm_is_thread_local(struct mm_struct *mm)
+{
+       if (atomic_read(&mm->context.active_cpus) > 1)
+               return false;
+       return cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm));
+}
+#else /* CONFIG_PPC_BOOK3S_64 */
 static inline int mm_is_thread_local(struct mm_struct *mm)
 {
        return cpumask_equal(mm_cpumask(mm),
                              cpumask_of(smp_processor_id()));
 }
+#endif /* !CONFIG_PPC_BOOK3S_64 */
 
-#else
+#else /* CONFIG_SMP */
 static inline int mm_is_core_local(struct mm_struct *mm)
 {
        return 1;

diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index 60188f4b3ecdad7f9d7329acd808e626c4b46d10..05e15386d4cb356da7e5b320be4b92907c99500b 100644 (file)
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -170,6 +170,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 #ifdef CONFIG_SPAPR_TCE_IOMMU
        mm_iommu_init(mm);
 #endif
+       atomic_set(&mm->context.active_cpus, 0);
+
        return 0;
 }