x86/intel_rdt: Modify the intel_pqr_state for better performance

Currently we have pqr_state and rdt_default_state which store the cached
CLOSID/RMIDs and the user configured cpu default values respectively. We
touch both of these during context switch. Put all of them in one
structure so that we can spare a cache line.

Reported-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Vikas Shivappa <vikas.shivappa@linux.intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: ravi.v.shankar@intel.com
Cc: tony.luck@intel.com
Cc: fenghua.yu@intel.com
Cc: peterz@infradead.org
Cc: eranian@google.com
Cc: sai.praneeth.prakhya@intel.com
Cc: ak@linux.intel.com
Cc: davidcc@google.com
Link: http://lkml.kernel.org/r/1502304395-7166-3-git-send-email-vikas.shivappa@linux.intel.com

diff --git a/arch/x86/include/asm/intel_rdt_sched.h b/arch/x86/include/asm/intel_rdt_sched.h
index 3badc0a87ef54b6b80a1872f40db7b837a19dd61..b4bbf8b21512f9423bbe557ad3255c39650565af 100644 (file)
--- a/arch/x86/include/asm/intel_rdt_sched.h
+++ b/arch/x86/include/asm/intel_rdt_sched.h
@@ -10,8 +10,10 @@
 
 /**
  * struct intel_pqr_state - State cache for the PQR MSR
- * @rmid:              The cached Resource Monitoring ID
- * @closid:            The cached Class Of Service ID
+ * @cur_rmid:          The cached Resource Monitoring ID
+ * @cur_closid:        The cached Class Of Service ID
+ * @default_rmid:      The user assigned Resource Monitoring ID
+ * @default_closid:    The user assigned cached Class Of Service ID
  *
  * The upper 32 bits of IA32_PQR_ASSOC contain closid and the
  * lower 10 bits rmid. The update to IA32_PQR_ASSOC always
@@ -22,12 +24,13 @@
  * not change.
  */
 struct intel_pqr_state {
-       u32                     rmid;
-       u32                     closid;
+       u32                     cur_rmid;
+       u32                     cur_closid;
+       u32                     default_rmid;
+       u32                     default_closid;
 };
 
 DECLARE_PER_CPU(struct intel_pqr_state, pqr_state);
-DECLARE_PER_CPU_READ_MOSTLY(struct intel_pqr_state, rdt_cpu_default);
 
 DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
 DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
@@ -49,8 +52,9 @@ DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key);
  */
 static void __intel_rdt_sched_in(void)
 {
-       struct intel_pqr_state newstate = this_cpu_read(rdt_cpu_default);
-       struct intel_pqr_state *curstate = this_cpu_ptr(&pqr_state);
+       struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
+       u32 closid = state->default_closid;
+       u32 rmid = state->default_rmid;
 
        /*
         * If this task has a closid/rmid assigned, use it.
@@ -58,18 +62,18 @@ static void __intel_rdt_sched_in(void)
         */
        if (static_branch_likely(&rdt_alloc_enable_key)) {
                if (current->closid)
-                       newstate.closid = current->closid;
+                       closid = current->closid;
        }
 
        if (static_branch_likely(&rdt_mon_enable_key)) {
                if (current->rmid)
-                       newstate.rmid = current->rmid;
+                       rmid = current->rmid;
        }
 
-       if (newstate.closid != curstate->closid ||
-           newstate.rmid != curstate->rmid) {
-               *curstate = newstate;
-               wrmsr(IA32_PQR_ASSOC, newstate.rmid, newstate.closid);
+       if (closid != state->cur_closid || rmid != state->cur_rmid) {
+               state->cur_closid = closid;
+               state->cur_rmid = rmid;
+               wrmsr(IA32_PQR_ASSOC, rmid, closid);
        }
 }
 

diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 4b9edb2617a51f04ea7bf19db0f8814aedf6247e..97c8d8321e044db900866eab0bb79058c8aad79c 100644 (file)
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -47,8 +47,6 @@ DEFINE_MUTEX(rdtgroup_mutex);
  */
 DEFINE_PER_CPU(struct intel_pqr_state, pqr_state);
 
-DEFINE_PER_CPU_READ_MOSTLY(struct intel_pqr_state, rdt_cpu_default);
-
 /*
  * Used to store the max resource name width and max resource data width
  * to display the schemata in a tabular format
@@ -550,10 +548,10 @@ static void clear_closid_rmid(int cpu)
 {
        struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
 
-       per_cpu(rdt_cpu_default.closid, cpu) = 0;
-       per_cpu(rdt_cpu_default.rmid, cpu) = 0;
-       state->closid = 0;
-       state->rmid = 0;
+       state->default_closid = 0;
+       state->default_rmid = 0;
+       state->cur_closid = 0;
+       state->cur_rmid = 0;
        wrmsr(IA32_PQR_ASSOC, 0, 0);
 }
 

diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index 2621ae3f07fcfb931257694abe36780668eaffa6..86a69794d7e475dc58faf889204c7af883fe29f5 100644 (file)
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -202,8 +202,8 @@ static void update_cpu_closid_rmid(void *info)
        struct rdtgroup *r = info;
 
        if (r) {
-               this_cpu_write(rdt_cpu_default.closid, r->closid);
-               this_cpu_write(rdt_cpu_default.rmid, r->mon.rmid);
+               this_cpu_write(pqr_state.default_closid, r->closid);
+               this_cpu_write(pqr_state.default_rmid, r->mon.rmid);
        }
 
        /*
@@ -1733,7 +1733,7 @@ static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp,
 
        /* Update per cpu rmid of the moved CPUs first */
        for_each_cpu(cpu, &rdtgrp->cpu_mask)
-               per_cpu(rdt_cpu_default.rmid, cpu) = prdtgrp->mon.rmid;
+               per_cpu(pqr_state.default_rmid, cpu) = prdtgrp->mon.rmid;
        /*
         * Update the MSR on moved CPUs and CPUs which have moved
         * task running on them.
@@ -1774,8 +1774,8 @@ static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp,
 
        /* Update per cpu closid and rmid of the moved CPUs first */
        for_each_cpu(cpu, &rdtgrp->cpu_mask) {
-               per_cpu(rdt_cpu_default.closid, cpu) = rdtgroup_default.closid;
-               per_cpu(rdt_cpu_default.rmid, cpu) = rdtgroup_default.mon.rmid;
+               per_cpu(pqr_state.default_closid, cpu) = rdtgroup_default.closid;
+               per_cpu(pqr_state.default_rmid, cpu) = rdtgroup_default.mon.rmid;
        }
 
        /*