2 * Block stat tracking code
4 * Copyright (C) 2016 Jens Axboe
6 #include <linux/kernel.h>
7 #include <linux/rculist.h>
8 #include <linux/blk-mq.h>
14 struct blk_queue_stats {
15 struct list_head callbacks;
17 bool enable_accounting;
20 static void blk_stat_init(struct blk_rq_stat *stat)
23 stat->max = stat->nr_samples = stat->mean = 0;
27 /* src is a per-cpu stat, mean isn't initialized */
28 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
33 dst->min = min(dst->min, src->min);
34 dst->max = max(dst->max, src->max);
36 dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
37 dst->nr_samples + src->nr_samples);
39 dst->nr_samples += src->nr_samples;
42 static void __blk_stat_add(struct blk_rq_stat *stat, u64 value)
44 stat->min = min(stat->min, value);
45 stat->max = max(stat->max, value);
50 void blk_stat_add(struct request *rq)
52 struct request_queue *q = rq->q;
53 struct blk_stat_callback *cb;
54 struct blk_rq_stat *stat;
59 value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
61 blk_throtl_stat_add(rq, value);
64 list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
65 if (!blk_stat_is_active(cb))
68 bucket = cb->bucket_fn(rq);
72 stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
73 __blk_stat_add(stat, value);
74 put_cpu_ptr(cb->cpu_stat);
79 static void blk_stat_timer_fn(struct timer_list *t)
81 struct blk_stat_callback *cb = from_timer(cb, t, timer);
85 for (bucket = 0; bucket < cb->buckets; bucket++)
86 blk_stat_init(&cb->stat[bucket]);
88 for_each_online_cpu(cpu) {
89 struct blk_rq_stat *cpu_stat;
91 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
92 for (bucket = 0; bucket < cb->buckets; bucket++) {
93 blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
94 blk_stat_init(&cpu_stat[bucket]);
101 struct blk_stat_callback *
102 blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
103 int (*bucket_fn)(const struct request *),
104 unsigned int buckets, void *data)
106 struct blk_stat_callback *cb;
108 cb = kmalloc(sizeof(*cb), GFP_KERNEL);
112 cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
118 cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
119 __alignof__(struct blk_rq_stat));
126 cb->timer_fn = timer_fn;
127 cb->bucket_fn = bucket_fn;
129 cb->buckets = buckets;
130 timer_setup(&cb->timer, blk_stat_timer_fn, 0);
134 EXPORT_SYMBOL_GPL(blk_stat_alloc_callback);
136 void blk_stat_add_callback(struct request_queue *q,
137 struct blk_stat_callback *cb)
142 for_each_possible_cpu(cpu) {
143 struct blk_rq_stat *cpu_stat;
145 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
146 for (bucket = 0; bucket < cb->buckets; bucket++)
147 blk_stat_init(&cpu_stat[bucket]);
150 spin_lock(&q->stats->lock);
151 list_add_tail_rcu(&cb->list, &q->stats->callbacks);
152 blk_queue_flag_set(QUEUE_FLAG_STATS, q);
153 spin_unlock(&q->stats->lock);
155 EXPORT_SYMBOL_GPL(blk_stat_add_callback);
157 void blk_stat_remove_callback(struct request_queue *q,
158 struct blk_stat_callback *cb)
160 spin_lock(&q->stats->lock);
161 list_del_rcu(&cb->list);
162 if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
163 blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
164 spin_unlock(&q->stats->lock);
166 del_timer_sync(&cb->timer);
168 EXPORT_SYMBOL_GPL(blk_stat_remove_callback);
170 static void blk_stat_free_callback_rcu(struct rcu_head *head)
172 struct blk_stat_callback *cb;
174 cb = container_of(head, struct blk_stat_callback, rcu);
175 free_percpu(cb->cpu_stat);
180 void blk_stat_free_callback(struct blk_stat_callback *cb)
183 call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
185 EXPORT_SYMBOL_GPL(blk_stat_free_callback);
187 void blk_stat_enable_accounting(struct request_queue *q)
189 spin_lock(&q->stats->lock);
190 q->stats->enable_accounting = true;
191 blk_queue_flag_set(QUEUE_FLAG_STATS, q);
192 spin_unlock(&q->stats->lock);
195 struct blk_queue_stats *blk_alloc_queue_stats(void)
197 struct blk_queue_stats *stats;
199 stats = kmalloc(sizeof(*stats), GFP_KERNEL);
203 INIT_LIST_HEAD(&stats->callbacks);
204 spin_lock_init(&stats->lock);
205 stats->enable_accounting = false;
210 void blk_free_queue_stats(struct blk_queue_stats *stats)
215 WARN_ON(!list_empty(&stats->callbacks));