blk-cgroup: pass blkg_rwstat structures by reference

[linux.git] / block / blk-cgroup.c

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b97b479e4f64f6800cf1ca8d0937019fea0f1489..db039a869d95efb158e2c4591457fcb7aac45289 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -79,6 +79,7 @@ static void blkg_free(struct blkcg_gq *blkg)
 
        blkg_rwstat_exit(&blkg->stat_ios);
        blkg_rwstat_exit(&blkg->stat_bytes);
+       percpu_ref_exit(&blkg->refcnt);
        kfree(blkg);
 }
 
@@ -86,8 +87,6 @@ static void __blkg_release(struct rcu_head *rcu)
 {
        struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
 
-       percpu_ref_exit(&blkg->refcnt);
-
        /* release the blkcg and parent blkg refs this blkg has been holding */
        css_put(&blkg->blkcg->css);
        if (blkg->parent)
@@ -132,6 +131,9 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
        if (!blkg)
                return NULL;
 
+       if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
+               goto err_free;
+
        if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
            blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
                goto err_free;
@@ -244,11 +246,6 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
                blkg_get(blkg->parent);
        }
 
-       ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,
-                             GFP_NOWAIT | __GFP_NOWARN);
-       if (ret)
-               goto err_cancel_ref;
-
        /* invoke per-policy init */
        for (i = 0; i < BLKCG_MAX_POLS; i++) {
                struct blkcg_policy *pol = blkcg_policy[i];
@@ -281,8 +278,6 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
        blkg_put(blkg);
        return ERR_PTR(ret);
 
-err_cancel_ref:
-       percpu_ref_exit(&blkg->refcnt);
 err_put_congested:
        wb_congested_put(wb_congested);
 err_put_css:
@@ -602,8 +597,9 @@ EXPORT_SYMBOL_GPL(blkg_prfill_stat);
 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
                       int off)
 {
-       struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
+       struct blkg_rwstat rwstat = { };
 
+       blkg_rwstat_read((void *)pd + off, &rwstat);
        return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
@@ -611,8 +607,9 @@ EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
 static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
                                    struct blkg_policy_data *pd, int off)
 {
-       struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
+       struct blkg_rwstat rwstat = { };
 
+       blkg_rwstat_read((void *)pd->blkg + off, &rwstat);
        return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 
@@ -654,8 +651,9 @@ static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
                                              struct blkg_policy_data *pd,
                                              int off)
 {
-       struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
-                                                             NULL, off);
+       struct blkg_rwstat rwstat;
+
+       blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat);
        return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 
@@ -736,6 +734,7 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
  * @blkg: blkg of interest
  * @pol: blkcg_policy which contains the blkg_rwstat
  * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
+ * @sum: blkg_rwstat structure containing the results
  *
  * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
  * online descendants and their aux counts.  The caller must be holding the
@@ -744,13 +743,12 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
  * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
  * is at @off bytes into @blkg's blkg_policy_data of the policy.
  */
-struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
-                                            struct blkcg_policy *pol, int off)
+void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
+               int off, struct blkg_rwstat *sum)
 {
        struct blkcg_gq *pos_blkg;
        struct cgroup_subsys_state *pos_css;
-       struct blkg_rwstat sum = { };
-       int i;
+       unsigned int i;
 
        lockdep_assert_held(&blkg->q->queue_lock);
 
@@ -767,13 +765,10 @@ struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
                        rwstat = (void *)pos_blkg + off;
 
                for (i = 0; i < BLKG_RWSTAT_NR; i++)
-                       atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
-                               percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
-                               &sum.aux_cnt[i]);
+                       atomic64_set(&sum->aux_cnt[i],
+                               blkg_rwstat_read_counter(rwstat, i));
        }
        rcu_read_unlock();
-
-       return sum;
 }
 EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
 
@@ -881,7 +876,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
                        blkg_free(new_blkg);
                } else {
                        blkg = blkg_create(pos, q, new_blkg);
-                       if (unlikely(IS_ERR(blkg))) {
+                       if (IS_ERR(blkg)) {
                                ret = PTR_ERR(blkg);
                                goto fail_unlock;
                        }
@@ -959,14 +954,14 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
 
                spin_lock_irq(&blkg->q->queue_lock);
 
-               rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
-                                       offsetof(struct blkcg_gq, stat_bytes));
+               blkg_rwstat_recursive_sum(blkg, NULL,
+                               offsetof(struct blkcg_gq, stat_bytes), &rwstat);
                rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
                wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
                dbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
 
-               rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
-                                       offsetof(struct blkcg_gq, stat_ios));
+               blkg_rwstat_recursive_sum(blkg, NULL,
+                                       offsetof(struct blkcg_gq, stat_ios), &rwstat);
                rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
                wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
                dios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
@@ -1006,8 +1001,12 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
                }
 next:
                if (has_stats) {
-                       off += scnprintf(buf+off, size-off, "\n");
-                       seq_commit(sf, off);
+                       if (off < size - 1) {
+                               off += scnprintf(buf+off, size-off, "\n");
+                               seq_commit(sf, off);
+                       } else {
+                               seq_commit(sf, -1);
+                       }
                }
        }
 
@@ -1391,7 +1390,8 @@ int blkcg_activate_policy(struct request_queue *q,
 
        spin_lock_irq(&q->queue_lock);
 
-       list_for_each_entry(blkg, &q->blkg_list, q_node) {
+       /* blkg_list is pushed at the head, reverse walk to init parents first */
+       list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
                struct blkg_policy_data *pd;
 
                if (blkg->pd[pol->plid])