summaryrefslogtreecommitdiff
path: root/drivers/powercap/idle_inject.c
blob: bafc59904ed354f3c14ecbbdef17f34114218577 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2018 Linaro Limited
 *
 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
 *
 * The idle injection framework provides a way to force CPUs to enter idle
 * states for a specified fraction of time over a specified period.
 *
 * It relies on the smpboot kthreads feature providing common code for CPU
 * hotplug and thread [un]parking.
 *
 * All of the kthreads used for idle injection are created at init time.
 *
 * Next, the users of the idle injection framework provide a cpumask via
 * its register function. The kthreads will be synchronized with respect to
 * this cpumask.
 *
 * The idle + run duration is specified via separate helpers and that allows
 * idle injection to be started.
 *
 * The idle injection kthreads will call play_idle_precise() with the idle
 * duration and max allowed latency specified as per the above.
 *
 * After all of them have been woken up, a timer is set to start the next idle
 * injection cycle.
 *
 * The timer interrupt handler will wake up the idle injection kthreads for
 * all of the CPUs in the cpumask provided by the user.
 *
 * Idle injection is stopped synchronously and no leftover idle injection
 * kthread activity after its completion is guaranteed.
 *
 * It is up to the user of this framework to provide a lock for higher-level
 * synchronization to prevent race conditions like starting idle injection
 * while unregistering from the framework.
 */
#define pr_fmt(fmt) "ii_dev: " fmt

#include <linux/cpu.h>
#include <linux/hrtimer.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/smpboot.h>
#include <linux/idle_inject.h>

#include <uapi/linux/sched/types.h>

/**
 * struct idle_inject_thread - task on/off switch structure
 * @tsk: task injecting the idle cycles
 * @should_run: whether or not to run the task (for the smpboot kthread API)
 */
struct idle_inject_thread {
	struct task_struct *tsk;
	int should_run;
};

/**
 * struct idle_inject_device - idle injection data
 * @timer: idle injection period timer
 * @idle_duration_us: duration of CPU idle time to inject
 * @run_duration_us: duration of CPU run time to allow
 * @latency_us: max allowed latency
 * @update: Optional callback deciding whether or not to skip idle
 *		injection in the given cycle.
 * @cpumask: mask of CPUs affected by idle injection
 *
 * This structure is used to define per instance idle inject device data. Each
 * instance has an idle duration, a run duration and mask of CPUs to inject
 * idle.
 *
 * Actual CPU idle time is injected by calling kernel scheduler interface
 * play_idle_precise(). There is one optional callback that can be registered
 * by calling idle_inject_register_full():
 *
 * update() - This callback is invoked just before waking up CPUs to inject
 * idle. If it returns false, CPUs are not woken up to inject idle in the given
 * cycle. It also allows the caller to readjust the idle and run duration by
 * calling idle_inject_set_duration() for the next cycle.
 */
struct idle_inject_device {
	struct hrtimer timer;
	unsigned int idle_duration_us;
	unsigned int run_duration_us;
	unsigned int latency_us;
	bool (*update)(void);
	unsigned long cpumask[];
};

static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);

/**
 * idle_inject_wakeup - Wake up idle injection threads
 * @ii_dev: target idle injection device
 *
 * Every idle injection task associated with the given idle injection device
 * and running on an online CPU will be woken up.
 */
static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
		iit = per_cpu_ptr(&idle_inject_thread, cpu);
		iit->should_run = 1;
		wake_up_process(iit->tsk);
	}
}

/**
 * idle_inject_timer_fn - idle injection timer function
 * @timer: idle injection hrtimer
 *
 * This function is called when the idle injection timer expires.  It wakes up
 * idle injection tasks associated with the timer and they, in turn, invoke
 * play_idle_precise() to inject a specified amount of CPU idle time.
 *
 * Return: HRTIMER_RESTART.
 */
static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
{
	unsigned int duration_us;
	struct idle_inject_device *ii_dev =
		container_of(timer, struct idle_inject_device, timer);

	if (!ii_dev->update || ii_dev->update())
		idle_inject_wakeup(ii_dev);

	duration_us = READ_ONCE(ii_dev->run_duration_us);
	duration_us += READ_ONCE(ii_dev->idle_duration_us);

	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));

	return HRTIMER_RESTART;
}

/**
 * idle_inject_fn - idle injection work function
 * @cpu: the CPU owning the task
 *
 * This function calls play_idle_precise() to inject a specified amount of CPU
 * idle time.
 */
static void idle_inject_fn(unsigned int cpu)
{
	struct idle_inject_device *ii_dev;
	struct idle_inject_thread *iit;

	ii_dev = per_cpu(idle_inject_device, cpu);
	iit = per_cpu_ptr(&idle_inject_thread, cpu);

	/*
	 * Let the smpboot main loop know that the task should not run again.
	 */
	iit->should_run = 0;

	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
}

/**
 * idle_inject_set_duration - idle and run duration update helper
 * @ii_dev: idle injection control device structure
 * @run_duration_us: CPU run time to allow in microseconds
 * @idle_duration_us: CPU idle time to inject in microseconds
 */
void idle_inject_set_duration(struct idle_inject_device *ii_dev,
			      unsigned int run_duration_us,
			      unsigned int idle_duration_us)
{
	if (run_duration_us + idle_duration_us) {
		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
	}
	if (!run_duration_us)
		pr_debug("CPU is forced to 100 percent idle\n");
}
EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);

/**
 * idle_inject_get_duration - idle and run duration retrieval helper
 * @ii_dev: idle injection control device structure
 * @run_duration_us: memory location to store the current CPU run time
 * @idle_duration_us: memory location to store the current CPU idle time
 */
void idle_inject_get_duration(struct idle_inject_device *ii_dev,
			      unsigned int *run_duration_us,
			      unsigned int *idle_duration_us)
{
	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);

/**
 * idle_inject_set_latency - set the maximum latency allowed
 * @ii_dev: idle injection control device structure
 * @latency_us: set the latency requirement for the idle state
 */
void idle_inject_set_latency(struct idle_inject_device *ii_dev,
			     unsigned int latency_us)
{
	WRITE_ONCE(ii_dev->latency_us, latency_us);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);

/**
 * idle_inject_start - start idle injections
 * @ii_dev: idle injection control device structure
 *
 * The function starts idle injection by first waking up all of the idle
 * injection kthreads associated with @ii_dev to let them inject CPU idle time
 * sets up a timer to start the next idle injection period.
 *
 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
 */
int idle_inject_start(struct idle_inject_device *ii_dev)
{
	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);

	if (!(idle_duration_us + run_duration_us))
		return -EINVAL;

	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	idle_inject_wakeup(ii_dev);

	hrtimer_start(&ii_dev->timer,
		      ns_to_ktime((idle_duration_us + run_duration_us) *
				  NSEC_PER_USEC),
		      HRTIMER_MODE_REL);

	return 0;
}
EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);

/**
 * idle_inject_stop - stops idle injections
 * @ii_dev: idle injection control device structure
 *
 * The function stops idle injection and waits for the threads to finish work.
 * If CPU idle time is being injected when this function runs, then it will
 * wait until the end of the cycle.
 *
 * When it returns, there is no more idle injection kthread activity.  The
 * kthreads are scheduled out and the periodic timer is off.
 */
void idle_inject_stop(struct idle_inject_device *ii_dev)
{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	hrtimer_cancel(&ii_dev->timer);

	/*
	 * Stopping idle injection requires all of the idle injection kthreads
	 * associated with the given cpumask to be parked and stay that way, so
	 * prevent CPUs from going online at this point.  Any CPUs going online
	 * after the loop below will be covered by clearing the should_run flag
	 * that will cause the smpboot main loop to schedule them out.
	 */
	cpu_hotplug_disable();

	/*
	 * Iterate over all (online + offline) CPUs here in case one of them
	 * goes offline with the should_run flag set so as to prevent its idle
	 * injection kthread from running when the CPU goes online again after
	 * the ii_dev has been freed.
	 */
	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
		iit = per_cpu_ptr(&idle_inject_thread, cpu);
		iit->should_run = 0;

		wait_task_inactive(iit->tsk, TASK_ANY);
	}

	cpu_hotplug_enable();
}
EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);

/**
 * idle_inject_setup - prepare the current task for idle injection
 * @cpu: not used
 *
 * Called once, this function is in charge of setting the current task's
 * scheduler parameters to make it an RT task.
 */
static void idle_inject_setup(unsigned int cpu)
{
	sched_set_fifo(current);
}

/**
 * idle_inject_should_run - function helper for the smpboot API
 * @cpu: CPU the kthread is running on
 *
 * Return: whether or not the thread can run.
 */
static int idle_inject_should_run(unsigned int cpu)
{
	struct idle_inject_thread *iit =
		per_cpu_ptr(&idle_inject_thread, cpu);

	return iit->should_run;
}

/**
 * idle_inject_register_full - initialize idle injection on a set of CPUs
 * @cpumask: CPUs to be affected by idle injection
 * @update: This callback is called just before waking up CPUs to inject
 * idle
 *
 * This function creates an idle injection control device structure for the
 * given set of CPUs and initializes the timer associated with it. This
 * function also allows to register update()callback.
 * It does not start any injection cycles.
 *
 * Return: NULL if memory allocation fails, idle injection control device
 * pointer on success.
 */

struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
						     bool (*update)(void))
{
	struct idle_inject_device *ii_dev;
	int cpu, cpu_rb;

	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
	if (!ii_dev)
		return NULL;

	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	ii_dev->timer.function = idle_inject_timer_fn;
	ii_dev->latency_us = UINT_MAX;
	ii_dev->update = update;

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {

		if (per_cpu(idle_inject_device, cpu)) {
			pr_err("cpu%d is already registered\n", cpu);
			goto out_rollback;
		}

		per_cpu(idle_inject_device, cpu) = ii_dev;
	}

	return ii_dev;

out_rollback:
	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
		if (cpu == cpu_rb)
			break;
		per_cpu(idle_inject_device, cpu_rb) = NULL;
	}

	kfree(ii_dev);

	return NULL;
}
EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);

/**
 * idle_inject_register - initialize idle injection on a set of CPUs
 * @cpumask: CPUs to be affected by idle injection
 *
 * This function creates an idle injection control device structure for the
 * given set of CPUs and initializes the timer associated with it.  It does not
 * start any injection cycles.
 *
 * Return: NULL if memory allocation fails, idle injection control device
 * pointer on success.
 */
struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
{
	return idle_inject_register_full(cpumask, NULL);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);

/**
 * idle_inject_unregister - unregister idle injection control device
 * @ii_dev: idle injection control device to unregister
 *
 * The function stops idle injection for the given control device,
 * unregisters its kthreads and frees memory allocated when that device was
 * created.
 */
void idle_inject_unregister(struct idle_inject_device *ii_dev)
{
	unsigned int cpu;

	idle_inject_stop(ii_dev);

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
		per_cpu(idle_inject_device, cpu) = NULL;

	kfree(ii_dev);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);

static struct smp_hotplug_thread idle_inject_threads = {
	.store = &idle_inject_thread.tsk,
	.setup = idle_inject_setup,
	.thread_fn = idle_inject_fn,
	.thread_comm = "idle_inject/%u",
	.thread_should_run = idle_inject_should_run,
};

static int __init idle_inject_init(void)
{
	return smpboot_register_percpu_thread(&idle_inject_threads);
}
early_initcall(idle_inject_init);