1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2023 Red Hat 4 */ 5 6 #include "completion.h" 7 8 #include <linux/kernel.h> 9 10 #include "logger.h" 11 #include "permassert.h" 12 13 #include "status-codes.h" 14 #include "types.h" 15 #include "vio.h" 16 #include "vdo.h" 17 18 /** 19 * DOC: vdo completions. 20 * 21 * Most of vdo's data structures are lock free, each either belonging to a single "zone," or 22 * divided into a number of zones whose accesses to the structure do not overlap. During normal 23 * operation, at most one thread will be operating in any given zone. Each zone has a 24 * vdo_work_queue which holds vdo_completions that are to be run in that zone. A completion may 25 * only be enqueued on one queue or operating in a single zone at a time. 26 * 27 * At each step of a multi-threaded operation, the completion performing the operation is given a 28 * callback, error handler, and thread id for the next step. A completion is "run" when it is 29 * operating on the correct thread (as specified by its callback_thread_id). If the value of its 30 * "result" field is an error (i.e. not VDO_SUCCESS), the function in its "error_handler" will be 31 * invoked. If the error_handler is NULL, or there is no error, the function set as its "callback" 32 * will be invoked. Generally, a completion will not be run directly, but rather will be 33 * "launched." In this case, it will check whether it is operating on the correct thread. If it is, 34 * it will run immediately. Otherwise, it will be enqueue on the vdo_work_queue associated with the 35 * completion's "callback_thread_id". When it is dequeued, it will be on the correct thread, and 36 * will get run. In some cases, the completion should get queued instead of running immediately, 37 * even if it is being launched from the correct thread. This is usually in cases where there is a 38 * long chain of callbacks, all on the same thread, which could overflow the stack. In such cases, 39 * the completion's "requeue" field should be set to true. Doing so will skip the current thread 40 * check and simply enqueue the completion. 41 * 42 * A completion may be "finished," in which case its "complete" field will be set to true before it 43 * is next run. It is a bug to attempt to set the result or re-finish a finished completion. 44 * Because a completion's fields are not safe to examine from any thread other than the one on 45 * which the completion is currently operating, this field is used only to aid in detecting 46 * programming errors. It can not be used for cross-thread checking on the status of an operation. 47 * A completion must be "reset" before it can be reused after it has been finished. Resetting will 48 * also clear any error from the result field. 49 **/ 50 51 void vdo_initialize_completion(struct vdo_completion *completion, 52 struct vdo *vdo, 53 enum vdo_completion_type type) 54 { 55 memset(completion, 0, sizeof(*completion)); 56 completion->vdo = vdo; 57 completion->type = type; 58 vdo_reset_completion(completion); 59 } 60 61 static inline void assert_incomplete(struct vdo_completion *completion) 62 { 63 VDO_ASSERT_LOG_ONLY(!completion->complete, "completion is not complete"); 64 } 65 66 /** 67 * vdo_set_completion_result() - Set the result of a completion. 68 * 69 * Older errors will not be masked. 70 */ 71 void vdo_set_completion_result(struct vdo_completion *completion, int result) 72 { 73 assert_incomplete(completion); 74 if (completion->result == VDO_SUCCESS) 75 completion->result = result; 76 } 77 78 /** 79 * vdo_launch_completion_with_priority() - Run or enqueue a completion. 80 * @priority: The priority at which to enqueue the completion. 81 * 82 * If called on the correct thread (i.e. the one specified in the completion's callback_thread_id 83 * field) and not marked for requeue, the completion will be run immediately. Otherwise, the 84 * completion will be enqueued on the specified thread. 85 */ 86 void vdo_launch_completion_with_priority(struct vdo_completion *completion, 87 enum vdo_completion_priority priority) 88 { 89 thread_id_t callback_thread = completion->callback_thread_id; 90 91 if (completion->requeue || (callback_thread != vdo_get_callback_thread_id())) { 92 vdo_enqueue_completion(completion, priority); 93 return; 94 } 95 96 vdo_run_completion(completion); 97 } 98 99 /** vdo_finish_completion() - Mark a completion as complete and then launch it. */ 100 void vdo_finish_completion(struct vdo_completion *completion) 101 { 102 assert_incomplete(completion); 103 completion->complete = true; 104 if (completion->callback != NULL) 105 vdo_launch_completion(completion); 106 } 107 108 void vdo_enqueue_completion(struct vdo_completion *completion, 109 enum vdo_completion_priority priority) 110 { 111 struct vdo *vdo = completion->vdo; 112 thread_id_t thread_id = completion->callback_thread_id; 113 114 if (VDO_ASSERT(thread_id < vdo->thread_config.thread_count, 115 "thread_id %u (completion type %d) is less than thread count %u", 116 thread_id, completion->type, 117 vdo->thread_config.thread_count) != VDO_SUCCESS) 118 BUG(); 119 120 completion->requeue = false; 121 completion->priority = priority; 122 completion->my_queue = NULL; 123 vdo_enqueue_work_queue(vdo->threads[thread_id].queue, completion); 124 } 125 126 /** 127 * vdo_requeue_completion_if_needed() - Requeue a completion if not called on the specified thread. 128 * 129 * Return: True if the completion was requeued; callers may not access the completion in this case. 130 */ 131 bool vdo_requeue_completion_if_needed(struct vdo_completion *completion, 132 thread_id_t callback_thread_id) 133 { 134 if (vdo_get_callback_thread_id() == callback_thread_id) 135 return false; 136 137 completion->callback_thread_id = callback_thread_id; 138 vdo_enqueue_completion(completion, VDO_WORK_Q_DEFAULT_PRIORITY); 139 return true; 140 } 141