summaryrefslogtreecommitdiff
path: root/block/cfq-iosched.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c280
1 files changed, 229 insertions, 51 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 2e13e9e689bd..49913804e8dd 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -15,6 +15,7 @@
#include <linux/ioprio.h>
#include <linux/blktrace_api.h>
#include "blk.h"
+#include "blk-cgroup.h"
static struct blkio_policy_type blkio_policy_cfq;
@@ -365,9 +366,177 @@ CFQ_CFQQ_FNS(deep);
CFQ_CFQQ_FNS(wait_busy);
#undef CFQ_CFQQ_FNS
-#ifdef CONFIG_CFQ_GROUP_IOSCHED
+#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-#include "blk-cgroup.h"
+/* blkg state flags */
+enum blkg_state_flags {
+ BLKG_waiting = 0,
+ BLKG_idling,
+ BLKG_empty,
+};
+
+#define BLKG_FLAG_FNS(name) \
+static inline void blkio_mark_blkg_##name( \
+ struct blkio_group_stats *stats) \
+{ \
+ stats->flags |= (1 << BLKG_##name); \
+} \
+static inline void blkio_clear_blkg_##name( \
+ struct blkio_group_stats *stats) \
+{ \
+ stats->flags &= ~(1 << BLKG_##name); \
+} \
+static inline int blkio_blkg_##name(struct blkio_group_stats *stats) \
+{ \
+ return (stats->flags & (1 << BLKG_##name)) != 0; \
+} \
+
+BLKG_FLAG_FNS(waiting)
+BLKG_FLAG_FNS(idling)
+BLKG_FLAG_FNS(empty)
+#undef BLKG_FLAG_FNS
+
+/* This should be called with the queue_lock held. */
+static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
+{
+ unsigned long long now;
+
+ if (!blkio_blkg_waiting(stats))
+ return;
+
+ now = sched_clock();
+ if (time_after64(now, stats->start_group_wait_time))
+ blkg_stat_add(&stats->group_wait_time,
+ now - stats->start_group_wait_time);
+ blkio_clear_blkg_waiting(stats);
+}
+
+/* This should be called with the queue_lock held. */
+static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
+ struct blkio_policy_type *pol,
+ struct blkio_group *curr_blkg)
+{
+ struct blkg_policy_data *pd = blkg->pd[pol->plid];
+
+ if (blkio_blkg_waiting(&pd->stats))
+ return;
+ if (blkg == curr_blkg)
+ return;
+ pd->stats.start_group_wait_time = sched_clock();
+ blkio_mark_blkg_waiting(&pd->stats);
+}
+
+/* This should be called with the queue_lock held. */
+static void blkio_end_empty_time(struct blkio_group_stats *stats)
+{
+ unsigned long long now;
+
+ if (!blkio_blkg_empty(stats))
+ return;
+
+ now = sched_clock();
+ if (time_after64(now, stats->start_empty_time))
+ blkg_stat_add(&stats->empty_time,
+ now - stats->start_empty_time);
+ blkio_clear_blkg_empty(stats);
+}
+
+static void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol,
+ unsigned long dequeue)
+{
+ struct blkg_policy_data *pd = blkg->pd[pol->plid];
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ blkg_stat_add(&pd->stats.dequeue, dequeue);
+}
+
+static void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
+ struct blkio_policy_type *pol)
+{
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ if (blkg_rwstat_sum(&stats->queued))
+ return;
+
+ /*
+ * group is already marked empty. This can happen if cfqq got new
+ * request in parent group and moved to this group while being added
+ * to service tree. Just ignore the event and move on.
+ */
+ if (blkio_blkg_empty(stats))
+ return;
+
+ stats->start_empty_time = sched_clock();
+ blkio_mark_blkg_empty(stats);
+}
+
+static void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol)
+{
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ if (blkio_blkg_idling(stats)) {
+ unsigned long long now = sched_clock();
+
+ if (time_after64(now, stats->start_idle_time))
+ blkg_stat_add(&stats->idle_time,
+ now - stats->start_idle_time);
+ blkio_clear_blkg_idling(stats);
+ }
+}
+
+static void cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol)
+{
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+
+ lockdep_assert_held(blkg->q->queue_lock);
+ BUG_ON(blkio_blkg_idling(stats));
+
+ stats->start_idle_time = sched_clock();
+ blkio_mark_blkg_idling(stats);
+}
+
+static void cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol)
+{
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ blkg_stat_add(&stats->avg_queue_size_sum,
+ blkg_rwstat_sum(&stats->queued));
+ blkg_stat_add(&stats->avg_queue_size_samples, 1);
+ blkio_update_group_wait_time(stats);
+}
+
+#else /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
+
+static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
+ struct blkio_policy_type *pol,
+ struct blkio_group *curr_blkg) { }
+static void blkio_end_empty_time(struct blkio_group_stats *stats) { }
+static void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol,
+ unsigned long dequeue) { }
+static void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
+ struct blkio_policy_type *pol) { }
+static void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol) { }
+static void cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol) { }
+static void cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
+ struct blkio_policy_type *pol) { }
+
+#endif /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
+
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
static inline struct cfq_group *blkg_to_cfqg(struct blkio_group *blkg)
{
@@ -403,75 +572,98 @@ static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
struct blkio_group *curr_blkg,
bool direction, bool sync)
{
- blkiocg_update_io_add_stats(blkg, pol, curr_blkg, direction, sync);
-}
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, unsigned long dequeue)
-{
- blkiocg_update_dequeue_stats(blkg, pol, dequeue);
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ blkg_rwstat_add(&stats->queued, rw, 1);
+ blkio_end_empty_time(stats);
+ blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
}
static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
struct blkio_policy_type *pol, unsigned long time,
unsigned long unaccounted_time)
{
- blkiocg_update_timeslice_used(blkg, pol, time, unaccounted_time);
-}
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
-static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
-{
- blkiocg_set_start_empty_time(blkg, pol);
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ blkg_stat_add(&stats->time, time);
+#ifdef CONFIG_DEBUG_BLK_CGROUP
+ blkg_stat_add(&stats->unaccounted_time, unaccounted_time);
+#endif
}
static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync)
{
- blkiocg_update_io_remove_stats(blkg, pol, direction, sync);
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ blkg_rwstat_add(&stats->queued, rw, -1);
}
static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync)
{
- blkiocg_update_io_merged_stats(blkg, pol, direction, sync);
-}
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-static inline void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
-{
- blkiocg_update_idle_time_stats(blkg, pol);
-}
+ lockdep_assert_held(blkg->q->queue_lock);
-static inline void
-cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
-{
- blkiocg_update_avg_queue_size_stats(blkg, pol);
-}
-
-static inline void
-cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
-{
- blkiocg_update_set_idle_time_stats(blkg, pol);
+ blkg_rwstat_add(&stats->merged, rw, 1);
}
static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, uint64_t bytes,
bool direction, bool sync)
{
- blkiocg_update_dispatch_stats(blkg, pol, bytes, direction, sync);
+ int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
+ struct blkg_policy_data *pd = blkg->pd[pol->plid];
+ struct blkio_group_stats_cpu *stats_cpu;
+ unsigned long flags;
+
+ /* If per cpu stats are not allocated yet, don't do any accounting. */
+ if (pd->stats_cpu == NULL)
+ return;
+
+ /*
+ * Disabling interrupts to provide mutual exclusion between two
+ * writes on same cpu. It probably is not needed for 64bit. Not
+ * optimizing that case yet.
+ */
+ local_irq_save(flags);
+
+ stats_cpu = this_cpu_ptr(pd->stats_cpu);
+
+ blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
+ blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
+ blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
+
+ local_irq_restore(flags);
}
static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, uint64_t start_time,
uint64_t io_start_time, bool direction, bool sync)
{
- blkiocg_update_completion_stats(blkg, pol, start_time, io_start_time,
- direction, sync);
+ struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ unsigned long long now = sched_clock();
+ int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
+
+ lockdep_assert_held(blkg->q->queue_lock);
+
+ if (time_after64(now, io_start_time))
+ blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
+ if (time_after64(io_start_time, start_time))
+ blkg_rwstat_add(&stats->wait_time, rw,
+ io_start_time - start_time);
}
#else /* CONFIG_CFQ_GROUP_IOSCHED */
@@ -489,29 +681,15 @@ static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg, bool direction,
bool sync) { }
-static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, unsigned long dequeue) { }
static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
struct blkio_policy_type *pol, unsigned long time,
unsigned long unaccounted_time) { }
-static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync) { }
static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync) { }
-static inline void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-static inline void
-cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-
-static inline void
-cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-
static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, uint64_t bytes,
bool direction, bool sync) { }
generated by cgit v1.2.3 (git 2.39.0) at 2024-12-20 07:32:57 +0300