Merge tag 'v5.3-rc3' into drm-next-5.4

[linux.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 249671873aa90daae744914305450acc0576d534..cdbb7a84cb6e186242afeebb32f4c5c7fb2d0639 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -695,12 +695,15 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
        if (mem_cgroup_disabled())
                return;
 
-       __this_cpu_add(memcg->vmstats_local->stat[idx], val);
-
        x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
        if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
                struct mem_cgroup *mi;
 
+               /*
+                * Batch local counters to keep them in sync with
+                * the hierarchical ones.
+                */
+               __this_cpu_add(memcg->vmstats_local->stat[idx], x);
                for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
                        atomic_long_add(x, &mi->vmstats[idx]);
                x = 0;
@@ -749,13 +752,15 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
        /* Update memcg */
        __mod_memcg_state(memcg, idx, val);
 
-       /* Update lruvec */
-       __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
-
        x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
        if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
                struct mem_cgroup_per_node *pi;
 
+               /*
+                * Batch local counters to keep them in sync with
+                * the hierarchical ones.
+                */
+               __this_cpu_add(pn->lruvec_stat_local->count[idx], x);
                for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
                        atomic_long_add(x, &pi->lruvec_stat[idx]);
                x = 0;
@@ -777,12 +782,15 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
        if (mem_cgroup_disabled())
                return;
 
-       __this_cpu_add(memcg->vmstats_local->events[idx], count);
-
        x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
        if (unlikely(x > MEMCG_CHARGE_BATCH)) {
                struct mem_cgroup *mi;
 
+               /*
+                * Batch local counters to keep them in sync with
+                * the hierarchical ones.
+                */
+               __this_cpu_add(memcg->vmstats_local->events[idx], x);
                for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
                        atomic_long_add(x, &mi->vmevents[idx]);
                x = 0;