1 // SPDX-License-Identifier: GPL-2.0
2 #include "bcachefs.h"
3 #include "clock.h"
4
5 #include <linux/freezer.h>
6 #include <linux/kthread.h>
7 #include <linux/preempt.h>
8
io_timer_cmp(const void * l,const void * r,void __always_unused * args)9 static inline bool io_timer_cmp(const void *l, const void *r, void __always_unused *args)
10 {
11 struct io_timer **_l = (struct io_timer **)l;
12 struct io_timer **_r = (struct io_timer **)r;
13
14 return (*_l)->expire < (*_r)->expire;
15 }
16
17 static const struct min_heap_callbacks callbacks = {
18 .less = io_timer_cmp,
19 .swp = NULL,
20 };
21
bch2_io_timer_add(struct io_clock * clock,struct io_timer * timer)22 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
23 {
24 spin_lock(&clock->timer_lock);
25
26 if (time_after_eq64((u64) atomic64_read(&clock->now), timer->expire)) {
27 spin_unlock(&clock->timer_lock);
28 timer->fn(timer);
29 return;
30 }
31
32 for (size_t i = 0; i < clock->timers.nr; i++)
33 if (clock->timers.data[i] == timer)
34 goto out;
35
36 BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL));
37 out:
38 spin_unlock(&clock->timer_lock);
39 }
40
bch2_io_timer_del(struct io_clock * clock,struct io_timer * timer)41 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer)
42 {
43 spin_lock(&clock->timer_lock);
44
45 for (size_t i = 0; i < clock->timers.nr; i++)
46 if (clock->timers.data[i] == timer) {
47 min_heap_del(&clock->timers, i, &callbacks, NULL);
48 break;
49 }
50
51 spin_unlock(&clock->timer_lock);
52 }
53
54 struct io_clock_wait {
55 struct io_timer io_timer;
56 struct timer_list cpu_timer;
57 struct task_struct *task;
58 int expired;
59 };
60
io_clock_wait_fn(struct io_timer * timer)61 static void io_clock_wait_fn(struct io_timer *timer)
62 {
63 struct io_clock_wait *wait = container_of(timer,
64 struct io_clock_wait, io_timer);
65
66 wait->expired = 1;
67 wake_up_process(wait->task);
68 }
69
io_clock_cpu_timeout(struct timer_list * timer)70 static void io_clock_cpu_timeout(struct timer_list *timer)
71 {
72 struct io_clock_wait *wait = container_of(timer,
73 struct io_clock_wait, cpu_timer);
74
75 wait->expired = 1;
76 wake_up_process(wait->task);
77 }
78
bch2_io_clock_schedule_timeout(struct io_clock * clock,u64 until)79 void bch2_io_clock_schedule_timeout(struct io_clock *clock, u64 until)
80 {
81 struct io_clock_wait wait = {
82 .io_timer.expire = until,
83 .io_timer.fn = io_clock_wait_fn,
84 .io_timer.fn2 = (void *) _RET_IP_,
85 .task = current,
86 };
87
88 bch2_io_timer_add(clock, &wait.io_timer);
89 schedule();
90 bch2_io_timer_del(clock, &wait.io_timer);
91 }
92
bch2_kthread_io_clock_wait(struct io_clock * clock,u64 io_until,unsigned long cpu_timeout)93 void bch2_kthread_io_clock_wait(struct io_clock *clock,
94 u64 io_until, unsigned long cpu_timeout)
95 {
96 bool kthread = (current->flags & PF_KTHREAD) != 0;
97 struct io_clock_wait wait = {
98 .io_timer.expire = io_until,
99 .io_timer.fn = io_clock_wait_fn,
100 .io_timer.fn2 = (void *) _RET_IP_,
101 .task = current,
102 };
103
104 bch2_io_timer_add(clock, &wait.io_timer);
105
106 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0);
107
108 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT)
109 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies);
110
111 do {
112 set_current_state(TASK_INTERRUPTIBLE);
113 if (kthread && kthread_should_stop())
114 break;
115
116 if (wait.expired)
117 break;
118
119 schedule();
120 try_to_freeze();
121 } while (0);
122
123 __set_current_state(TASK_RUNNING);
124 del_timer_sync(&wait.cpu_timer);
125 destroy_timer_on_stack(&wait.cpu_timer);
126 bch2_io_timer_del(clock, &wait.io_timer);
127 }
128
get_expired_timer(struct io_clock * clock,u64 now)129 static struct io_timer *get_expired_timer(struct io_clock *clock, u64 now)
130 {
131 struct io_timer *ret = NULL;
132
133 if (clock->timers.nr &&
134 time_after_eq64(now, clock->timers.data[0]->expire)) {
135 ret = *min_heap_peek(&clock->timers);
136 min_heap_pop(&clock->timers, &callbacks, NULL);
137 }
138
139 return ret;
140 }
141
__bch2_increment_clock(struct io_clock * clock,u64 sectors)142 void __bch2_increment_clock(struct io_clock *clock, u64 sectors)
143 {
144 struct io_timer *timer;
145 u64 now = atomic64_add_return(sectors, &clock->now);
146
147 spin_lock(&clock->timer_lock);
148 while ((timer = get_expired_timer(clock, now)))
149 timer->fn(timer);
150 spin_unlock(&clock->timer_lock);
151 }
152
bch2_io_timers_to_text(struct printbuf * out,struct io_clock * clock)153 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock)
154 {
155 out->atomic++;
156 spin_lock(&clock->timer_lock);
157 u64 now = atomic64_read(&clock->now);
158
159 printbuf_tabstop_push(out, 40);
160 prt_printf(out, "current time:\t%llu\n", now);
161
162 for (unsigned i = 0; i < clock->timers.nr; i++)
163 prt_printf(out, "%ps %ps:\t%llu\n",
164 clock->timers.data[i]->fn,
165 clock->timers.data[i]->fn2,
166 clock->timers.data[i]->expire);
167 spin_unlock(&clock->timer_lock);
168 --out->atomic;
169 }
170
bch2_io_clock_exit(struct io_clock * clock)171 void bch2_io_clock_exit(struct io_clock *clock)
172 {
173 free_heap(&clock->timers);
174 free_percpu(clock->pcpu_buf);
175 }
176
bch2_io_clock_init(struct io_clock * clock)177 int bch2_io_clock_init(struct io_clock *clock)
178 {
179 atomic64_set(&clock->now, 0);
180 spin_lock_init(&clock->timer_lock);
181
182 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
183
184 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
185 if (!clock->pcpu_buf)
186 return -BCH_ERR_ENOMEM_io_clock_init;
187
188 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL))
189 return -BCH_ERR_ENOMEM_io_clock_init;
190
191 return 0;
192 }
193