summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fs/btrfs/async-thread.c107
-rw-r--r--fs/btrfs/async-thread.h3
2 files changed, 101 insertions, 9 deletions
diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c
index 193c84964db9..977bce2ec887 100644
--- a/fs/btrfs/async-thread.c
+++ b/fs/btrfs/async-thread.c
@@ -30,6 +30,9 @@
#define WORK_ORDER_DONE_BIT 2
#define WORK_HIGH_PRIO_BIT 3
+#define NO_THRESHOLD (-1)
+#define DFT_THRESHOLD (32)
+
/*
* container for the kthread task pointer and the list of pending work
* One of these is allocated per thread.
@@ -737,6 +740,14 @@ struct __btrfs_workqueue_struct {
/* Spinlock for ordered_list */
spinlock_t list_lock;
+
+ /* Thresholding related variants */
+ atomic_t pending;
+ int max_active;
+ int current_max;
+ int thresh;
+ unsigned int count;
+ spinlock_t thres_lock;
};
struct btrfs_workqueue_struct {
@@ -745,19 +756,34 @@ struct btrfs_workqueue_struct {
};
static inline struct __btrfs_workqueue_struct
-*__btrfs_alloc_workqueue(char *name, int flags, int max_active)
+*__btrfs_alloc_workqueue(char *name, int flags, int max_active, int thresh)
{
struct __btrfs_workqueue_struct *ret = kzalloc(sizeof(*ret), GFP_NOFS);
if (unlikely(!ret))
return NULL;
+ ret->max_active = max_active;
+ atomic_set(&ret->pending, 0);
+ if (thresh == 0)
+ thresh = DFT_THRESHOLD;
+ /* For low threshold, disabling threshold is a better choice */
+ if (thresh < DFT_THRESHOLD) {
+ ret->current_max = max_active;
+ ret->thresh = NO_THRESHOLD;
+ } else {
+ ret->current_max = 1;
+ ret->thresh = thresh;
+ }
+
if (flags & WQ_HIGHPRI)
ret->normal_wq = alloc_workqueue("%s-%s-high", flags,
- max_active, "btrfs", name);
+ ret->max_active,
+ "btrfs", name);
else
ret->normal_wq = alloc_workqueue("%s-%s", flags,
- max_active, "btrfs", name);
+ ret->max_active, "btrfs",
+ name);
if (unlikely(!ret->normal_wq)) {
kfree(ret);
return NULL;
@@ -765,6 +791,7 @@ static inline struct __btrfs_workqueue_struct
INIT_LIST_HEAD(&ret->ordered_list);
spin_lock_init(&ret->list_lock);
+ spin_lock_init(&ret->thres_lock);
return ret;
}
@@ -773,7 +800,8 @@ __btrfs_destroy_workqueue(struct __btrfs_workqueue_struct *wq);
struct btrfs_workqueue_struct *btrfs_alloc_workqueue(char *name,
int flags,
- int max_active)
+ int max_active,
+ int thresh)
{
struct btrfs_workqueue_struct *ret = kzalloc(sizeof(*ret), GFP_NOFS);
@@ -781,14 +809,15 @@ struct btrfs_workqueue_struct *btrfs_alloc_workqueue(char *name,
return NULL;
ret->normal = __btrfs_alloc_workqueue(name, flags & ~WQ_HIGHPRI,
- max_active);
+ max_active, thresh);
if (unlikely(!ret->normal)) {
kfree(ret);
return NULL;
}
if (flags & WQ_HIGHPRI) {
- ret->high = __btrfs_alloc_workqueue(name, flags, max_active);
+ ret->high = __btrfs_alloc_workqueue(name, flags, max_active,
+ thresh);
if (unlikely(!ret->high)) {
__btrfs_destroy_workqueue(ret->normal);
kfree(ret);
@@ -798,6 +827,66 @@ struct btrfs_workqueue_struct *btrfs_alloc_workqueue(char *name,
return ret;
}
+/*
+ * Hook for threshold which will be called in btrfs_queue_work.
+ * This hook WILL be called in IRQ handler context,
+ * so workqueue_set_max_active MUST NOT be called in this hook
+ */
+static inline void thresh_queue_hook(struct __btrfs_workqueue_struct *wq)
+{
+ if (wq->thresh == NO_THRESHOLD)
+ return;
+ atomic_inc(&wq->pending);
+}
+
+/*
+ * Hook for threshold which will be called before executing the work,
+ * This hook is called in kthread content.
+ * So workqueue_set_max_active is called here.
+ */
+static inline void thresh_exec_hook(struct __btrfs_workqueue_struct *wq)
+{
+ int new_max_active;
+ long pending;
+ int need_change = 0;
+
+ if (wq->thresh == NO_THRESHOLD)
+ return;
+
+ atomic_dec(&wq->pending);
+ spin_lock(&wq->thres_lock);
+ /*
+ * Use wq->count to limit the calling frequency of
+ * workqueue_set_max_active.
+ */
+ wq->count++;
+ wq->count %= (wq->thresh / 4);
+ if (!wq->count)
+ goto out;
+ new_max_active = wq->current_max;
+
+ /*
+ * pending may be changed later, but it's OK since we really
+ * don't need it so accurate to calculate new_max_active.
+ */
+ pending = atomic_read(&wq->pending);
+ if (pending > wq->thresh)
+ new_max_active++;
+ if (pending < wq->thresh / 2)
+ new_max_active--;
+ new_max_active = clamp_val(new_max_active, 1, wq->max_active);
+ if (new_max_active != wq->current_max) {
+ need_change = 1;
+ wq->current_max = new_max_active;
+ }
+out:
+ spin_unlock(&wq->thres_lock);
+
+ if (need_change) {
+ workqueue_set_max_active(wq->normal_wq, wq->current_max);
+ }
+}
+
static void run_ordered_work(struct __btrfs_workqueue_struct *wq)
{
struct list_head *list = &wq->ordered_list;
@@ -858,6 +947,7 @@ static void normal_work_helper(struct work_struct *arg)
need_order = 1;
wq = work->wq;
+ thresh_exec_hook(wq);
work->func(work);
if (need_order) {
set_bit(WORK_DONE_BIT, &work->flags);
@@ -884,6 +974,7 @@ static inline void __btrfs_queue_work(struct __btrfs_workqueue_struct *wq,
unsigned long flags;
work->wq = wq;
+ thresh_queue_hook(wq);
if (work->ordered_func) {
spin_lock_irqsave(&wq->list_lock, flags);
list_add_tail(&work->ordered_list, &wq->ordered_list);
@@ -922,9 +1013,9 @@ void btrfs_destroy_workqueue(struct btrfs_workqueue_struct *wq)
void btrfs_workqueue_set_max(struct btrfs_workqueue_struct *wq, int max)
{
- workqueue_set_max_active(wq->normal->normal_wq, max);
+ wq->normal->max_active = max;
if (wq->high)
- workqueue_set_max_active(wq->high->normal_wq, max);
+ wq->high->max_active = max;
}
void btrfs_set_work_high_priority(struct btrfs_work_struct *work)
diff --git a/fs/btrfs/async-thread.h b/fs/btrfs/async-thread.h
index fce623cfe3da..3129d8a6128b 100644
--- a/fs/btrfs/async-thread.h
+++ b/fs/btrfs/async-thread.h
@@ -138,7 +138,8 @@ struct btrfs_work_struct {
struct btrfs_workqueue_struct *btrfs_alloc_workqueue(char *name,
int flags,
- int max_active);
+ int max_active,
+ int thresh);
void btrfs_init_work(struct btrfs_work_struct *work,
void (*func)(struct btrfs_work_struct *),
void (*ordered_func)(struct btrfs_work_struct *),
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-18 08:46:24 +0300