1 // SPDX-License-Identifier: GPL-2.0-only
6 #include "funnel-workqueue.h"
15 #include "funnel-queue.h"
17 #include "memory-alloc.h"
20 #include "string-utils.h"
23 #include "status-codes.h"
30  * There are two types of work queues: simple, with one worker thread, and round-robin, which uses
31  * a group of the former to do the work, and assigns work to them in round-robin fashion (roughly).
32  * Externally, both are represented via the same common sub-structure, though there's actually not
51 	 * caching -- and if the max priority is 2, just fit in one x86-64 cache line if aligned.
92  * condition where a high-priority completion can be enqueued followed by a lower-priority one, and
93  * we'll grab the latter (but we'll catch the high-priority item on the next call). If strict
100 	for (i = queue->common.type->max_priority; i >= 0; i--) {  in poll_for_completion()
101 		struct funnel_queue_entry *link = vdo_funnel_queue_poll(queue->priority_lists[i]);  in poll_for_completion()
113 	VDO_ASSERT_LOG_ONLY(completion->my_queue == NULL,  in enqueue_work_queue_completion()
115 			    completion, completion->callback, queue, completion->my_queue);  in enqueue_work_queue_completion()
116 	if (completion->priority == VDO_WORK_Q_DEFAULT_PRIORITY)  in enqueue_work_queue_completion()
117 		completion->priority = queue->common.type->default_priority;  in enqueue_work_queue_completion()
119 	if (VDO_ASSERT(completion->priority <= queue->common.type->max_priority,  in enqueue_work_queue_completion()
121 		completion->priority = 0;  in enqueue_work_queue_completion()
123 	completion->my_queue = &queue->common;  in enqueue_work_queue_completion()
126 	vdo_funnel_queue_put(queue->priority_lists[completion->priority],  in enqueue_work_queue_completion()
127 			     &completion->work_queue_entry_link);  in enqueue_work_queue_completion()
131 	 * simplest safe implementation here would be to wake-up any waiting threads after  in enqueue_work_queue_completion()
146 	if ((atomic_read(&queue->idle) != 1) || (atomic_cmpxchg(&queue->idle, 1, 0) != 1))  in enqueue_work_queue_completion()
150 	wake_up(&queue->waiting_worker_threads);  in enqueue_work_queue_completion()
155 	if (queue->common.type->start != NULL)  in run_start_hook()
156 		queue->common.type->start(queue->private);  in run_start_hook()
161 	if (queue->common.type->finish != NULL)  in run_finish_hook()
162 		queue->common.type->finish(queue->private);  in run_finish_hook()
180 		prepare_to_wait(&queue->waiting_worker_threads, &wait,  in wait_for_next_completion()
190 		atomic_set(&queue->idle, 1);  in wait_for_next_completion()
191 		smp_mb(); /* store-load barrier between "idle" and funnel queue */  in wait_for_next_completion()
198 		 * We need to check for thread-stop after setting TASK_INTERRUPTIBLE state up  in wait_for_next_completion()
216 	finish_wait(&queue->waiting_worker_threads, &wait);  in wait_for_next_completion()
217 	atomic_set(&queue->idle, 0);  in wait_for_next_completion()
225 	if (VDO_ASSERT(completion->my_queue == &queue->common,  in process_completion()
227 		       completion, queue, completion->my_queue) == VDO_SUCCESS)  in process_completion()
228 		completion->my_queue = NULL;  in process_completion()
265 	complete(queue->started);  in work_queue_runner()
277 		vdo_free_funnel_queue(queue->priority_lists[i]);  in free_simple_work_queue()
278 	vdo_free(queue->common.name);  in free_simple_work_queue()
284 	struct simple_work_queue **queue_table = queue->service_queues;  in free_round_robin_work_queue()
285 	unsigned int count = queue->num_service_queues;  in free_round_robin_work_queue()
288 	queue->service_queues = NULL;  in free_round_robin_work_queue()
293 	vdo_free(queue->common.name);  in free_round_robin_work_queue()
304 	if (queue->round_robin_mode)  in vdo_free_work_queue()
321 	VDO_ASSERT_LOG_ONLY((type->max_priority <= VDO_WORK_Q_MAX_PRIORITY),  in make_simple_work_queue()
322 			    "queue priority count %u within limit %u", type->max_priority,  in make_simple_work_queue()
329 	queue->private = private;  in make_simple_work_queue()
330 	queue->started = &started;  in make_simple_work_queue()
331 	queue->common.type = type;  in make_simple_work_queue()
332 	queue->common.owner = owner;  in make_simple_work_queue()
333 	init_waitqueue_head(&queue->waiting_worker_threads);  in make_simple_work_queue()
335 	result = vdo_duplicate_string(name, "queue name", &queue->common.name);  in make_simple_work_queue()
338 		return -ENOMEM;  in make_simple_work_queue()
341 	for (i = 0; i <= type->max_priority; i++) {  in make_simple_work_queue()
342 		result = vdo_make_funnel_queue(&queue->priority_lists[i]);  in make_simple_work_queue()
350 			     queue->common.name);  in make_simple_work_queue()
356 	queue->thread = thread;  in make_simple_work_queue()
373  * vdo_make_work_queue() - Create a work queue; if multiple threads are requested, completions will
374  *                         be distributed to them in round-robin fashion.
397 			*queue_ptr = &simple_queue->common;  in vdo_make_work_queue()
401 	result = vdo_allocate(1, struct round_robin_work_queue, "round-robin work queue",  in vdo_make_work_queue()
407 			      "subordinate work queues", &queue->service_queues);  in vdo_make_work_queue()
413 	queue->num_service_queues = thread_count;  in vdo_make_work_queue()
414 	queue->common.round_robin_mode = true;  in vdo_make_work_queue()
415 	queue->common.owner = owner;  in vdo_make_work_queue()
417 	result = vdo_duplicate_string(name, "queue name", &queue->common.name);  in vdo_make_work_queue()
419 		vdo_free(queue->service_queues);  in vdo_make_work_queue()
421 		return -ENOMEM;  in vdo_make_work_queue()
424 	*queue_ptr = &queue->common;  in vdo_make_work_queue()
431 						context, type, &queue->service_queues[i]);  in vdo_make_work_queue()
433 			queue->num_service_queues = i;  in vdo_make_work_queue()
445 	if (queue->thread == NULL)  in finish_simple_work_queue()
449 	kthread_stop(queue->thread);  in finish_simple_work_queue()
450 	queue->thread = NULL;  in finish_simple_work_queue()
455 	struct simple_work_queue **queue_table = queue->service_queues;  in finish_round_robin_work_queue()
456 	unsigned int count = queue->num_service_queues;  in finish_round_robin_work_queue()
469 	if (queue->round_robin_mode)  in vdo_finish_work_queue()
480 	char task_state_report = '-';  in dump_simple_work_queue()
482 	if (queue->thread != NULL) {  in dump_simple_work_queue()
483 		task_state_report = task_state_to_char(queue->thread);  in dump_simple_work_queue()
484 		thread_status = atomic_read(&queue->idle) ? "idle" : "running";  in dump_simple_work_queue()
487 	vdo_log_info("workQ %px (%s) %s (%c)", &queue->common, queue->common.name,  in dump_simple_work_queue()
490 	/* ->waiting_worker_threads wait queue status? anyone waiting? */  in dump_simple_work_queue()
500 	if (queue->round_robin_mode) {  in vdo_dump_work_queue()
504 		for (i = 0; i < round_robin->num_service_queues; i++)  in vdo_dump_work_queue()
505 			dump_simple_work_queue(round_robin->service_queues[i]);  in vdo_dump_work_queue()
515 		 * Format "%ps" logs a null pointer as "(null)" with a bunch of leading spaces. We  in get_function_name()
518 		strscpy(buffer, "-", buffer_length);  in get_function_name()
527 #pragma GCC diagnostic ignored "-Wformat"  in get_function_name()
528 		snprintf(buffer, buffer_length, "%.*ps", buffer_length - 1, pointer);  in get_function_name()
542 			  (completion->my_queue == NULL ? "-" : completion->my_queue->name));  in vdo_dump_completion_to_buffer()
544 	if (current_length < length - 1) {  in vdo_dump_completion_to_buffer()
545 		get_function_name((void *) completion->callback, buffer + current_length,  in vdo_dump_completion_to_buffer()
546 				  length - current_length);  in vdo_dump_completion_to_buffer()
564 	if (!queue->round_robin_mode) {  in vdo_enqueue_work_queue()
577 		unsigned int index = rotor % round_robin->num_service_queues;  in vdo_enqueue_work_queue()
579 		simple_queue = round_robin->service_queues[index];  in vdo_enqueue_work_queue()
588  * Return the work queue pointer recorded at initialization time in the work-queue stack handle
613 	return (queue == NULL) ? NULL : &queue->common;  in vdo_get_current_work_queue()
618 	return queue->owner;  in vdo_get_work_queue_owner()
622  * vdo_get_work_queue_private_data() - Returns the private data for the current thread's work
630 	return (queue != NULL) ? queue->private : NULL;  in vdo_get_work_queue_private_data()
636 	return (queue->type == type);  in vdo_work_queue_type_is()