1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "clock.h" 4 5 #include <linux/freezer.h> 6 #include <linux/kthread.h> 7 #include <linux/preempt.h> 8 9 static inline bool io_timer_cmp(const void *l, const void *r, void __always_unused *args) 10 { 11 struct io_timer **_l = (struct io_timer **)l; 12 struct io_timer **_r = (struct io_timer **)r; 13 14 return (*_l)->expire < (*_r)->expire; 15 } 16 17 static inline void io_timer_swp(void *l, void *r, void __always_unused *args) 18 { 19 struct io_timer **_l = (struct io_timer **)l; 20 struct io_timer **_r = (struct io_timer **)r; 21 22 swap(*_l, *_r); 23 } 24 25 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer) 26 { 27 const struct min_heap_callbacks callbacks = { 28 .less = io_timer_cmp, 29 .swp = io_timer_swp, 30 }; 31 32 spin_lock(&clock->timer_lock); 33 34 if (time_after_eq64((u64) atomic64_read(&clock->now), timer->expire)) { 35 spin_unlock(&clock->timer_lock); 36 timer->fn(timer); 37 return; 38 } 39 40 for (size_t i = 0; i < clock->timers.nr; i++) 41 if (clock->timers.data[i] == timer) 42 goto out; 43 44 BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL)); 45 out: 46 spin_unlock(&clock->timer_lock); 47 } 48 49 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer) 50 { 51 const struct min_heap_callbacks callbacks = { 52 .less = io_timer_cmp, 53 .swp = io_timer_swp, 54 }; 55 56 spin_lock(&clock->timer_lock); 57 58 for (size_t i = 0; i < clock->timers.nr; i++) 59 if (clock->timers.data[i] == timer) { 60 min_heap_del(&clock->timers, i, &callbacks, NULL); 61 break; 62 } 63 64 spin_unlock(&clock->timer_lock); 65 } 66 67 struct io_clock_wait { 68 struct io_timer io_timer; 69 struct timer_list cpu_timer; 70 struct task_struct *task; 71 int expired; 72 }; 73 74 static void io_clock_wait_fn(struct io_timer *timer) 75 { 76 struct io_clock_wait *wait = container_of(timer, 77 struct io_clock_wait, io_timer); 78 79 wait->expired = 1; 80 wake_up_process(wait->task); 81 } 82 83 static void io_clock_cpu_timeout(struct timer_list *timer) 84 { 85 struct io_clock_wait *wait = container_of(timer, 86 struct io_clock_wait, cpu_timer); 87 88 wait->expired = 1; 89 wake_up_process(wait->task); 90 } 91 92 void bch2_io_clock_schedule_timeout(struct io_clock *clock, u64 until) 93 { 94 struct io_clock_wait wait = { 95 .io_timer.expire = until, 96 .io_timer.fn = io_clock_wait_fn, 97 .io_timer.fn2 = (void *) _RET_IP_, 98 .task = current, 99 }; 100 101 bch2_io_timer_add(clock, &wait.io_timer); 102 schedule(); 103 bch2_io_timer_del(clock, &wait.io_timer); 104 } 105 106 void bch2_kthread_io_clock_wait(struct io_clock *clock, 107 u64 io_until, unsigned long cpu_timeout) 108 { 109 bool kthread = (current->flags & PF_KTHREAD) != 0; 110 struct io_clock_wait wait = { 111 .io_timer.expire = io_until, 112 .io_timer.fn = io_clock_wait_fn, 113 .io_timer.fn2 = (void *) _RET_IP_, 114 .task = current, 115 }; 116 117 bch2_io_timer_add(clock, &wait.io_timer); 118 119 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0); 120 121 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT) 122 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies); 123 124 do { 125 set_current_state(TASK_INTERRUPTIBLE); 126 if (kthread && kthread_should_stop()) 127 break; 128 129 if (wait.expired) 130 break; 131 132 schedule(); 133 try_to_freeze(); 134 } while (0); 135 136 __set_current_state(TASK_RUNNING); 137 del_timer_sync(&wait.cpu_timer); 138 destroy_timer_on_stack(&wait.cpu_timer); 139 bch2_io_timer_del(clock, &wait.io_timer); 140 } 141 142 static struct io_timer *get_expired_timer(struct io_clock *clock, u64 now) 143 { 144 struct io_timer *ret = NULL; 145 const struct min_heap_callbacks callbacks = { 146 .less = io_timer_cmp, 147 .swp = io_timer_swp, 148 }; 149 150 if (clock->timers.nr && 151 time_after_eq64(now, clock->timers.data[0]->expire)) { 152 ret = *min_heap_peek(&clock->timers); 153 min_heap_pop(&clock->timers, &callbacks, NULL); 154 } 155 156 return ret; 157 } 158 159 void __bch2_increment_clock(struct io_clock *clock, u64 sectors) 160 { 161 struct io_timer *timer; 162 u64 now = atomic64_add_return(sectors, &clock->now); 163 164 spin_lock(&clock->timer_lock); 165 while ((timer = get_expired_timer(clock, now))) 166 timer->fn(timer); 167 spin_unlock(&clock->timer_lock); 168 } 169 170 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock) 171 { 172 out->atomic++; 173 spin_lock(&clock->timer_lock); 174 u64 now = atomic64_read(&clock->now); 175 176 printbuf_tabstop_push(out, 40); 177 prt_printf(out, "current time:\t%llu\n", now); 178 179 for (unsigned i = 0; i < clock->timers.nr; i++) 180 prt_printf(out, "%ps %ps:\t%llu\n", 181 clock->timers.data[i]->fn, 182 clock->timers.data[i]->fn2, 183 clock->timers.data[i]->expire); 184 spin_unlock(&clock->timer_lock); 185 --out->atomic; 186 } 187 188 void bch2_io_clock_exit(struct io_clock *clock) 189 { 190 free_heap(&clock->timers); 191 free_percpu(clock->pcpu_buf); 192 } 193 194 int bch2_io_clock_init(struct io_clock *clock) 195 { 196 atomic64_set(&clock->now, 0); 197 spin_lock_init(&clock->timer_lock); 198 199 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus(); 200 201 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf); 202 if (!clock->pcpu_buf) 203 return -BCH_ERR_ENOMEM_io_clock_init; 204 205 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL)) 206 return -BCH_ERR_ENOMEM_io_clock_init; 207 208 return 0; 209 } 210