xref: /linux/kernel/power/qos.c (revision af1f459233d4edeef634f559539e7f4b64cb1d25)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Power Management Quality of Service (PM QoS) support base.
4  *
5  * Copyright (C) 2020 Intel Corporation
6  *
7  * Authors:
8  *	Mark Gross <mgross@linux.intel.com>
9  *	Rafael J. Wysocki <rafael.j.wysocki@intel.com>
10  *
11  * Provided here is an interface for specifying PM QoS dependencies.  It allows
12  * entities depending on QoS constraints to register their requests which are
13  * aggregated as appropriate to produce effective constraints (target values)
14  * that can be monitored by entities needing to respect them, either by polling
15  * or through a built-in notification mechanism.
16  *
17  * In addition to the basic functionality, more specific interfaces for managing
18  * global CPU latency QoS requests and frequency QoS requests are provided.
19  */
20 
21 /*#define DEBUG*/
22 
23 #include <linux/pm_qos.h>
24 #include <linux/sched.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/time.h>
28 #include <linux/fs.h>
29 #include <linux/device.h>
30 #include <linux/miscdevice.h>
31 #include <linux/string.h>
32 #include <linux/platform_device.h>
33 #include <linux/init.h>
34 #include <linux/kernel.h>
35 #include <linux/debugfs.h>
36 #include <linux/seq_file.h>
37 
38 #include <linux/uaccess.h>
39 #include <linux/export.h>
40 #include <trace/events/power.h>
41 
42 /*
43  * locking rule: all changes to constraints or notifiers lists
44  * or pm_qos_object list and pm_qos_objects need to happen with pm_qos_lock
45  * held, taken with _irqsave.  One lock to rule them all
46  */
47 static DEFINE_SPINLOCK(pm_qos_lock);
48 
49 /**
50  * pm_qos_read_value - Return the current effective constraint value.
51  * @c: List of PM QoS constraint requests.
52  */
53 s32 pm_qos_read_value(struct pm_qos_constraints *c)
54 {
55 	return READ_ONCE(c->target_value);
56 }
57 
58 static int pm_qos_get_value(struct pm_qos_constraints *c)
59 {
60 	if (plist_head_empty(&c->list))
61 		return c->no_constraint_value;
62 
63 	switch (c->type) {
64 	case PM_QOS_MIN:
65 		return plist_first(&c->list)->prio;
66 
67 	case PM_QOS_MAX:
68 		return plist_last(&c->list)->prio;
69 
70 	default:
71 		WARN(1, "Unknown PM QoS type in %s\n", __func__);
72 		return PM_QOS_DEFAULT_VALUE;
73 	}
74 }
75 
76 static void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
77 {
78 	WRITE_ONCE(c->target_value, value);
79 }
80 
81 /**
82  * pm_qos_update_target - Update a list of PM QoS constraint requests.
83  * @c: List of PM QoS requests.
84  * @node: Target list entry.
85  * @action: Action to carry out (add, update or remove).
86  * @value: New request value for the target list entry.
87  *
88  * Update the given list of PM QoS constraint requests, @c, by carrying an
89  * @action involving the @node list entry and @value on it.
90  *
91  * The recognized values of @action are PM_QOS_ADD_REQ (store @value in @node
92  * and add it to the list), PM_QOS_UPDATE_REQ (remove @node from the list, store
93  * @value in it and add it to the list again), and PM_QOS_REMOVE_REQ (remove
94  * @node from the list, ignore @value).
95  *
96  * Return: 1 if the aggregate constraint value has changed, 0  otherwise.
97  */
98 int pm_qos_update_target(struct pm_qos_constraints *c, struct plist_node *node,
99 			 enum pm_qos_req_action action, int value)
100 {
101 	int prev_value, curr_value, new_value;
102 	unsigned long flags;
103 
104 	spin_lock_irqsave(&pm_qos_lock, flags);
105 
106 	prev_value = pm_qos_get_value(c);
107 	if (value == PM_QOS_DEFAULT_VALUE)
108 		new_value = c->default_value;
109 	else
110 		new_value = value;
111 
112 	switch (action) {
113 	case PM_QOS_REMOVE_REQ:
114 		plist_del(node, &c->list);
115 		break;
116 	case PM_QOS_UPDATE_REQ:
117 		/*
118 		 * To change the list, atomically remove, reinit with new value
119 		 * and add, then see if the aggregate has changed.
120 		 */
121 		plist_del(node, &c->list);
122 		fallthrough;
123 	case PM_QOS_ADD_REQ:
124 		plist_node_init(node, new_value);
125 		plist_add(node, &c->list);
126 		break;
127 	default:
128 		/* no action */
129 		;
130 	}
131 
132 	curr_value = pm_qos_get_value(c);
133 	pm_qos_set_value(c, curr_value);
134 
135 	spin_unlock_irqrestore(&pm_qos_lock, flags);
136 
137 	trace_pm_qos_update_target(action, prev_value, curr_value);
138 
139 	if (prev_value == curr_value)
140 		return 0;
141 
142 	if (c->notifiers)
143 		blocking_notifier_call_chain(c->notifiers, curr_value, NULL);
144 
145 	return 1;
146 }
147 
148 /**
149  * pm_qos_flags_remove_req - Remove device PM QoS flags request.
150  * @pqf: Device PM QoS flags set to remove the request from.
151  * @req: Request to remove from the set.
152  */
153 static void pm_qos_flags_remove_req(struct pm_qos_flags *pqf,
154 				    struct pm_qos_flags_request *req)
155 {
156 	s32 val = 0;
157 
158 	list_del(&req->node);
159 	list_for_each_entry(req, &pqf->list, node)
160 		val |= req->flags;
161 
162 	pqf->effective_flags = val;
163 }
164 
165 /**
166  * pm_qos_update_flags - Update a set of PM QoS flags.
167  * @pqf: Set of PM QoS flags to update.
168  * @req: Request to add to the set, to modify, or to remove from the set.
169  * @action: Action to take on the set.
170  * @val: Value of the request to add or modify.
171  *
172  * Return: 1 if the aggregate constraint value has changed, 0 otherwise.
173  */
174 bool pm_qos_update_flags(struct pm_qos_flags *pqf,
175 			 struct pm_qos_flags_request *req,
176 			 enum pm_qos_req_action action, s32 val)
177 {
178 	unsigned long irqflags;
179 	s32 prev_value, curr_value;
180 
181 	spin_lock_irqsave(&pm_qos_lock, irqflags);
182 
183 	prev_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags;
184 
185 	switch (action) {
186 	case PM_QOS_REMOVE_REQ:
187 		pm_qos_flags_remove_req(pqf, req);
188 		break;
189 	case PM_QOS_UPDATE_REQ:
190 		pm_qos_flags_remove_req(pqf, req);
191 		fallthrough;
192 	case PM_QOS_ADD_REQ:
193 		req->flags = val;
194 		INIT_LIST_HEAD(&req->node);
195 		list_add_tail(&req->node, &pqf->list);
196 		pqf->effective_flags |= val;
197 		break;
198 	default:
199 		/* no action */
200 		;
201 	}
202 
203 	curr_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags;
204 
205 	spin_unlock_irqrestore(&pm_qos_lock, irqflags);
206 
207 	trace_pm_qos_update_flags(action, prev_value, curr_value);
208 
209 	return prev_value != curr_value;
210 }
211 
212 #ifdef CONFIG_CPU_IDLE
213 /* Definitions related to the CPU latency QoS. */
214 
215 static struct pm_qos_constraints cpu_latency_constraints = {
216 	.list = PLIST_HEAD_INIT(cpu_latency_constraints.list),
217 	.target_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
218 	.default_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
219 	.no_constraint_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
220 	.type = PM_QOS_MIN,
221 };
222 
223 static inline bool cpu_latency_qos_value_invalid(s32 value)
224 {
225 	return value < 0 && value != PM_QOS_DEFAULT_VALUE;
226 }
227 
228 /**
229  * cpu_latency_qos_limit - Return current system-wide CPU latency QoS limit.
230  */
231 s32 cpu_latency_qos_limit(void)
232 {
233 	return pm_qos_read_value(&cpu_latency_constraints);
234 }
235 
236 /**
237  * cpu_latency_qos_request_active - Check the given PM QoS request.
238  * @req: PM QoS request to check.
239  *
240  * Return: 'true' if @req has been added to the CPU latency QoS list, 'false'
241  * otherwise.
242  */
243 bool cpu_latency_qos_request_active(struct pm_qos_request *req)
244 {
245 	return req->qos == &cpu_latency_constraints;
246 }
247 EXPORT_SYMBOL_GPL(cpu_latency_qos_request_active);
248 
249 static void cpu_latency_qos_apply(struct pm_qos_request *req,
250 				  enum pm_qos_req_action action, s32 value)
251 {
252 	int ret = pm_qos_update_target(req->qos, &req->node, action, value);
253 	if (ret > 0)
254 		wake_up_all_idle_cpus();
255 }
256 
257 /**
258  * cpu_latency_qos_add_request - Add new CPU latency QoS request.
259  * @req: Pointer to a preallocated handle.
260  * @value: Requested constraint value.
261  *
262  * Use @value to initialize the request handle pointed to by @req, insert it as
263  * a new entry to the CPU latency QoS list and recompute the effective QoS
264  * constraint for that list.
265  *
266  * Callers need to save the handle for later use in updates and removal of the
267  * QoS request represented by it.
268  */
269 void cpu_latency_qos_add_request(struct pm_qos_request *req, s32 value)
270 {
271 	if (!req || cpu_latency_qos_value_invalid(value))
272 		return;
273 
274 	if (cpu_latency_qos_request_active(req)) {
275 		WARN(1, KERN_ERR "%s called for already added request\n", __func__);
276 		return;
277 	}
278 
279 	trace_pm_qos_add_request(value);
280 
281 	req->qos = &cpu_latency_constraints;
282 	cpu_latency_qos_apply(req, PM_QOS_ADD_REQ, value);
283 }
284 EXPORT_SYMBOL_GPL(cpu_latency_qos_add_request);
285 
286 /**
287  * cpu_latency_qos_update_request - Modify existing CPU latency QoS request.
288  * @req : QoS request to update.
289  * @new_value: New requested constraint value.
290  *
291  * Use @new_value to update the QoS request represented by @req in the CPU
292  * latency QoS list along with updating the effective constraint value for that
293  * list.
294  */
295 void cpu_latency_qos_update_request(struct pm_qos_request *req, s32 new_value)
296 {
297 	if (!req || cpu_latency_qos_value_invalid(new_value))
298 		return;
299 
300 	if (!cpu_latency_qos_request_active(req)) {
301 		WARN(1, KERN_ERR "%s called for unknown object\n", __func__);
302 		return;
303 	}
304 
305 	trace_pm_qos_update_request(new_value);
306 
307 	if (new_value == req->node.prio)
308 		return;
309 
310 	cpu_latency_qos_apply(req, PM_QOS_UPDATE_REQ, new_value);
311 }
312 EXPORT_SYMBOL_GPL(cpu_latency_qos_update_request);
313 
314 /**
315  * cpu_latency_qos_remove_request - Remove existing CPU latency QoS request.
316  * @req: QoS request to remove.
317  *
318  * Remove the CPU latency QoS request represented by @req from the CPU latency
319  * QoS list along with updating the effective constraint value for that list.
320  */
321 void cpu_latency_qos_remove_request(struct pm_qos_request *req)
322 {
323 	if (!req)
324 		return;
325 
326 	if (!cpu_latency_qos_request_active(req)) {
327 		WARN(1, KERN_ERR "%s called for unknown object\n", __func__);
328 		return;
329 	}
330 
331 	trace_pm_qos_remove_request(PM_QOS_DEFAULT_VALUE);
332 
333 	cpu_latency_qos_apply(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
334 	memset(req, 0, sizeof(*req));
335 }
336 EXPORT_SYMBOL_GPL(cpu_latency_qos_remove_request);
337 
338 /* User space interface to the CPU latency QoS via misc device. */
339 
340 static int cpu_latency_qos_open(struct inode *inode, struct file *filp)
341 {
342 	struct pm_qos_request *req;
343 
344 	req = kzalloc(sizeof(*req), GFP_KERNEL);
345 	if (!req)
346 		return -ENOMEM;
347 
348 	cpu_latency_qos_add_request(req, PM_QOS_DEFAULT_VALUE);
349 	filp->private_data = req;
350 
351 	return 0;
352 }
353 
354 static int cpu_latency_qos_release(struct inode *inode, struct file *filp)
355 {
356 	struct pm_qos_request *req = filp->private_data;
357 
358 	filp->private_data = NULL;
359 
360 	cpu_latency_qos_remove_request(req);
361 	kfree(req);
362 
363 	return 0;
364 }
365 
366 static ssize_t cpu_latency_qos_read(struct file *filp, char __user *buf,
367 				    size_t count, loff_t *f_pos)
368 {
369 	struct pm_qos_request *req = filp->private_data;
370 	unsigned long flags;
371 	s32 value;
372 
373 	if (!req || !cpu_latency_qos_request_active(req))
374 		return -EINVAL;
375 
376 	spin_lock_irqsave(&pm_qos_lock, flags);
377 	value = pm_qos_get_value(&cpu_latency_constraints);
378 	spin_unlock_irqrestore(&pm_qos_lock, flags);
379 
380 	return simple_read_from_buffer(buf, count, f_pos, &value, sizeof(s32));
381 }
382 
383 static ssize_t cpu_latency_qos_write(struct file *filp, const char __user *buf,
384 				     size_t count, loff_t *f_pos)
385 {
386 	s32 value;
387 
388 	if (count == sizeof(s32)) {
389 		if (copy_from_user(&value, buf, sizeof(s32)))
390 			return -EFAULT;
391 	} else {
392 		int ret;
393 
394 		ret = kstrtos32_from_user(buf, count, 16, &value);
395 		if (ret)
396 			return ret;
397 	}
398 
399 	cpu_latency_qos_update_request(filp->private_data, value);
400 
401 	return count;
402 }
403 
404 static const struct file_operations cpu_latency_qos_fops = {
405 	.write = cpu_latency_qos_write,
406 	.read = cpu_latency_qos_read,
407 	.open = cpu_latency_qos_open,
408 	.release = cpu_latency_qos_release,
409 	.llseek = noop_llseek,
410 };
411 
412 static struct miscdevice cpu_latency_qos_miscdev = {
413 	.minor = MISC_DYNAMIC_MINOR,
414 	.name = "cpu_dma_latency",
415 	.fops = &cpu_latency_qos_fops,
416 };
417 
418 static int __init cpu_latency_qos_init(void)
419 {
420 	int ret;
421 
422 	ret = misc_register(&cpu_latency_qos_miscdev);
423 	if (ret < 0)
424 		pr_err("%s: %s setup failed\n", __func__,
425 		       cpu_latency_qos_miscdev.name);
426 
427 	return ret;
428 }
429 late_initcall(cpu_latency_qos_init);
430 #endif /* CONFIG_CPU_IDLE */
431 
432 /* Definitions related to the frequency QoS below. */
433 
434 static inline bool freq_qos_value_invalid(s32 value)
435 {
436 	return value < 0 && value != PM_QOS_DEFAULT_VALUE;
437 }
438 
439 /**
440  * freq_constraints_init - Initialize frequency QoS constraints.
441  * @qos: Frequency QoS constraints to initialize.
442  */
443 void freq_constraints_init(struct freq_constraints *qos)
444 {
445 	struct pm_qos_constraints *c;
446 
447 	c = &qos->min_freq;
448 	plist_head_init(&c->list);
449 	c->target_value = FREQ_QOS_MIN_DEFAULT_VALUE;
450 	c->default_value = FREQ_QOS_MIN_DEFAULT_VALUE;
451 	c->no_constraint_value = FREQ_QOS_MIN_DEFAULT_VALUE;
452 	c->type = PM_QOS_MAX;
453 	c->notifiers = &qos->min_freq_notifiers;
454 	BLOCKING_INIT_NOTIFIER_HEAD(c->notifiers);
455 
456 	c = &qos->max_freq;
457 	plist_head_init(&c->list);
458 	c->target_value = FREQ_QOS_MAX_DEFAULT_VALUE;
459 	c->default_value = FREQ_QOS_MAX_DEFAULT_VALUE;
460 	c->no_constraint_value = FREQ_QOS_MAX_DEFAULT_VALUE;
461 	c->type = PM_QOS_MIN;
462 	c->notifiers = &qos->max_freq_notifiers;
463 	BLOCKING_INIT_NOTIFIER_HEAD(c->notifiers);
464 }
465 
466 /**
467  * freq_qos_read_value - Get frequency QoS constraint for a given list.
468  * @qos: Constraints to evaluate.
469  * @type: QoS request type.
470  */
471 s32 freq_qos_read_value(struct freq_constraints *qos,
472 			enum freq_qos_req_type type)
473 {
474 	s32 ret;
475 
476 	switch (type) {
477 	case FREQ_QOS_MIN:
478 		ret = IS_ERR_OR_NULL(qos) ?
479 			FREQ_QOS_MIN_DEFAULT_VALUE :
480 			pm_qos_read_value(&qos->min_freq);
481 		break;
482 	case FREQ_QOS_MAX:
483 		ret = IS_ERR_OR_NULL(qos) ?
484 			FREQ_QOS_MAX_DEFAULT_VALUE :
485 			pm_qos_read_value(&qos->max_freq);
486 		break;
487 	default:
488 		WARN_ON(1);
489 		ret = 0;
490 	}
491 
492 	return ret;
493 }
494 
495 /**
496  * freq_qos_apply - Add/modify/remove frequency QoS request.
497  * @req: Constraint request to apply.
498  * @action: Action to perform (add/update/remove).
499  * @value: Value to assign to the QoS request.
500  *
501  * This is only meant to be called from inside pm_qos, not drivers.
502  */
503 int freq_qos_apply(struct freq_qos_request *req,
504 			  enum pm_qos_req_action action, s32 value)
505 {
506 	int ret;
507 
508 	switch(req->type) {
509 	case FREQ_QOS_MIN:
510 		ret = pm_qos_update_target(&req->qos->min_freq, &req->pnode,
511 					   action, value);
512 		break;
513 	case FREQ_QOS_MAX:
514 		ret = pm_qos_update_target(&req->qos->max_freq, &req->pnode,
515 					   action, value);
516 		break;
517 	default:
518 		ret = -EINVAL;
519 	}
520 
521 	return ret;
522 }
523 
524 /**
525  * freq_qos_add_request - Insert new frequency QoS request into a given list.
526  * @qos: Constraints to update.
527  * @req: Preallocated request object.
528  * @type: Request type.
529  * @value: Request value.
530  *
531  * Insert a new entry into the @qos list of requests, recompute the effective
532  * QoS constraint value for that list and initialize the @req object.  The
533  * caller needs to save that object for later use in updates and removal.
534  *
535  * Return 1 if the effective constraint value has changed, 0 if the effective
536  * constraint value has not changed, or a negative error code on failures.
537  */
538 int freq_qos_add_request(struct freq_constraints *qos,
539 			 struct freq_qos_request *req,
540 			 enum freq_qos_req_type type, s32 value)
541 {
542 	int ret;
543 
544 	if (IS_ERR_OR_NULL(qos) || !req || freq_qos_value_invalid(value))
545 		return -EINVAL;
546 
547 	if (WARN(freq_qos_request_active(req),
548 		 "%s() called for active request\n", __func__))
549 		return -EINVAL;
550 
551 	req->qos = qos;
552 	req->type = type;
553 	ret = freq_qos_apply(req, PM_QOS_ADD_REQ, value);
554 	if (ret < 0) {
555 		req->qos = NULL;
556 		req->type = 0;
557 	}
558 
559 	return ret;
560 }
561 EXPORT_SYMBOL_GPL(freq_qos_add_request);
562 
563 /**
564  * freq_qos_update_request - Modify existing frequency QoS request.
565  * @req: Request to modify.
566  * @new_value: New request value.
567  *
568  * Update an existing frequency QoS request along with the effective constraint
569  * value for the list of requests it belongs to.
570  *
571  * Return 1 if the effective constraint value has changed, 0 if the effective
572  * constraint value has not changed, or a negative error code on failures.
573  */
574 int freq_qos_update_request(struct freq_qos_request *req, s32 new_value)
575 {
576 	if (!req || freq_qos_value_invalid(new_value))
577 		return -EINVAL;
578 
579 	if (WARN(!freq_qos_request_active(req),
580 		 "%s() called for unknown object\n", __func__))
581 		return -EINVAL;
582 
583 	if (req->pnode.prio == new_value)
584 		return 0;
585 
586 	return freq_qos_apply(req, PM_QOS_UPDATE_REQ, new_value);
587 }
588 EXPORT_SYMBOL_GPL(freq_qos_update_request);
589 
590 /**
591  * freq_qos_remove_request - Remove frequency QoS request from its list.
592  * @req: Request to remove.
593  *
594  * Remove the given frequency QoS request from the list of constraints it
595  * belongs to and recompute the effective constraint value for that list.
596  *
597  * Return 1 if the effective constraint value has changed, 0 if the effective
598  * constraint value has not changed, or a negative error code on failures.
599  */
600 int freq_qos_remove_request(struct freq_qos_request *req)
601 {
602 	int ret;
603 
604 	if (!req)
605 		return -EINVAL;
606 
607 	if (WARN(!freq_qos_request_active(req),
608 		 "%s() called for unknown object\n", __func__))
609 		return -EINVAL;
610 
611 	ret = freq_qos_apply(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
612 	req->qos = NULL;
613 	req->type = 0;
614 
615 	return ret;
616 }
617 EXPORT_SYMBOL_GPL(freq_qos_remove_request);
618 
619 /**
620  * freq_qos_add_notifier - Add frequency QoS change notifier.
621  * @qos: List of requests to add the notifier to.
622  * @type: Request type.
623  * @notifier: Notifier block to add.
624  */
625 int freq_qos_add_notifier(struct freq_constraints *qos,
626 			  enum freq_qos_req_type type,
627 			  struct notifier_block *notifier)
628 {
629 	int ret;
630 
631 	if (IS_ERR_OR_NULL(qos) || !notifier)
632 		return -EINVAL;
633 
634 	switch (type) {
635 	case FREQ_QOS_MIN:
636 		ret = blocking_notifier_chain_register(qos->min_freq.notifiers,
637 						       notifier);
638 		break;
639 	case FREQ_QOS_MAX:
640 		ret = blocking_notifier_chain_register(qos->max_freq.notifiers,
641 						       notifier);
642 		break;
643 	default:
644 		WARN_ON(1);
645 		ret = -EINVAL;
646 	}
647 
648 	return ret;
649 }
650 EXPORT_SYMBOL_GPL(freq_qos_add_notifier);
651 
652 /**
653  * freq_qos_remove_notifier - Remove frequency QoS change notifier.
654  * @qos: List of requests to remove the notifier from.
655  * @type: Request type.
656  * @notifier: Notifier block to remove.
657  */
658 int freq_qos_remove_notifier(struct freq_constraints *qos,
659 			     enum freq_qos_req_type type,
660 			     struct notifier_block *notifier)
661 {
662 	int ret;
663 
664 	if (IS_ERR_OR_NULL(qos) || !notifier)
665 		return -EINVAL;
666 
667 	switch (type) {
668 	case FREQ_QOS_MIN:
669 		ret = blocking_notifier_chain_unregister(qos->min_freq.notifiers,
670 							 notifier);
671 		break;
672 	case FREQ_QOS_MAX:
673 		ret = blocking_notifier_chain_unregister(qos->max_freq.notifiers,
674 							 notifier);
675 		break;
676 	default:
677 		WARN_ON(1);
678 		ret = -EINVAL;
679 	}
680 
681 	return ret;
682 }
683 EXPORT_SYMBOL_GPL(freq_qos_remove_notifier);
684