1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2025 Google LLC
4 * Author: Vincent Donnefort <vdonnefort@google.com>
5 */
6
7 #include <linux/cpumask.h>
8 #include <linux/trace_remote.h>
9 #include <linux/tracefs.h>
10 #include <linux/simple_ring_buffer.h>
11
12 #include <asm/arch_timer.h>
13 #include <asm/kvm_host.h>
14 #include <asm/kvm_hyptrace.h>
15 #include <asm/kvm_mmu.h>
16
17 #include "hyp_trace.h"
18
19 /* Same 10min used by clocksource when width is more than 32-bits */
20 #define CLOCK_MAX_CONVERSION_S 600
21 /*
22 * Time to give for the clock init. Long enough to get a good mult/shift
23 * estimation. Short enough to not delay the tracing start too much.
24 */
25 #define CLOCK_INIT_MS 100
26 /*
27 * Time between clock checks. Must be small enough to catch clock deviation when
28 * it is still tiny.
29 */
30 #define CLOCK_UPDATE_MS 500
31
32 static struct hyp_trace_clock {
33 u64 cycles;
34 u64 cyc_overflow64;
35 u64 boot;
36 u32 mult;
37 u32 shift;
38 struct delayed_work work;
39 struct completion ready;
40 } hyp_clock;
41
__hyp_clock_work(struct work_struct * work)42 static void __hyp_clock_work(struct work_struct *work)
43 {
44 struct delayed_work *dwork = to_delayed_work(work);
45 struct hyp_trace_clock *hyp_clock;
46 struct system_time_snapshot snap;
47 u64 rate, delta_cycles;
48 u64 boot, delta_boot;
49
50 hyp_clock = container_of(dwork, struct hyp_trace_clock, work);
51
52 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap);
53 boot = ktime_to_ns(snap.systime);
54
55 delta_boot = boot - hyp_clock->boot;
56 delta_cycles = snap.cycles - hyp_clock->cycles;
57
58 /* Compare hyp clock with the kernel boot clock */
59 if (hyp_clock->mult) {
60 u64 err, cur = delta_cycles;
61
62 if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) {
63 __uint128_t tmp = (__uint128_t)cur * hyp_clock->mult;
64
65 cur = tmp >> hyp_clock->shift;
66 } else {
67 cur *= hyp_clock->mult;
68 cur >>= hyp_clock->shift;
69 }
70 cur += hyp_clock->boot;
71
72 err = abs_diff(cur, boot);
73 /* No deviation, only update epoch if necessary */
74 if (!err) {
75 if (delta_cycles >= (hyp_clock->cyc_overflow64 >> 1))
76 goto fast_forward;
77
78 goto resched;
79 }
80
81 /* Warn if the error is above tracing precision (1us) */
82 if (err > NSEC_PER_USEC)
83 pr_warn_ratelimited("hyp trace clock off by %lluus\n",
84 err / NSEC_PER_USEC);
85 }
86
87 rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot);
88
89 clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift,
90 rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S);
91
92 /* Add a comfortable 50% margin */
93 hyp_clock->cyc_overflow64 = (U64_MAX / hyp_clock->mult) >> 1;
94
95 fast_forward:
96 hyp_clock->cycles = snap.cycles;
97 hyp_clock->boot = boot;
98 kvm_call_hyp_nvhe(__tracing_update_clock, hyp_clock->mult,
99 hyp_clock->shift, hyp_clock->boot, hyp_clock->cycles);
100 complete(&hyp_clock->ready);
101
102 resched:
103 schedule_delayed_work(&hyp_clock->work,
104 msecs_to_jiffies(CLOCK_UPDATE_MS));
105 }
106
hyp_trace_clock_enable(struct hyp_trace_clock * hyp_clock,bool enable)107 static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enable)
108 {
109 struct system_time_snapshot snap;
110
111 if (!enable) {
112 cancel_delayed_work_sync(&hyp_clock->work);
113 return;
114 }
115
116 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap);
117
118 hyp_clock->boot = ktime_to_ns(snap.systime);
119 hyp_clock->cycles = snap.cycles;
120 hyp_clock->mult = 0;
121
122 init_completion(&hyp_clock->ready);
123 INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work);
124 schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS));
125 wait_for_completion(&hyp_clock->ready);
126 }
127
128 /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */
129 static struct hyp_trace_buffer {
130 struct hyp_trace_desc *desc;
131 size_t desc_size;
132 } trace_buffer;
133
__map_hyp(void * start,size_t size)134 static int __map_hyp(void *start, size_t size)
135 {
136 if (is_protected_kvm_enabled())
137 return 0;
138
139 return create_hyp_mappings(start, start + size, PAGE_HYP);
140 }
141
__share_page(unsigned long va)142 static int __share_page(unsigned long va)
143 {
144 return kvm_share_hyp((void *)va, (void *)va + 1);
145 }
146
__unshare_page(unsigned long va)147 static void __unshare_page(unsigned long va)
148 {
149 kvm_unshare_hyp((void *)va, (void *)va + 1);
150 }
151
hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer * trace_buffer,size_t size)152 static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size)
153 {
154 int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1;
155 size_t backing_size;
156 void *start;
157 int ret;
158
159 backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages *
160 num_possible_cpus());
161
162 start = alloc_pages_exact(backing_size, GFP_KERNEL_ACCOUNT);
163 if (!start)
164 return -ENOMEM;
165
166 ret = __map_hyp(start, backing_size);
167 if (ret) {
168 free_pages_exact(start, backing_size);
169 return ret;
170 }
171
172 trace_buffer->desc->bpages_backing_start = (unsigned long)start;
173 trace_buffer->desc->bpages_backing_size = backing_size;
174
175 return ret;
176 }
177
hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer * trace_buffer)178 static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer)
179 {
180 free_pages_exact((void *)trace_buffer->desc->bpages_backing_start,
181 trace_buffer->desc->bpages_backing_size);
182 }
183
hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer * trace_buffer,int last_cpu)184 static void hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer *trace_buffer, int last_cpu)
185 {
186 struct ring_buffer_desc *rb_desc;
187 int cpu, p;
188
189 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) {
190 if (cpu > last_cpu)
191 break;
192
193 __unshare_page(rb_desc->meta_va);
194 for (p = 0; p < rb_desc->nr_page_va; p++)
195 __unshare_page(rb_desc->page_va[p]);
196 }
197 }
198
hyp_trace_buffer_share_hyp(struct hyp_trace_buffer * trace_buffer)199 static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer)
200 {
201 struct ring_buffer_desc *rb_desc;
202 int cpu, p, ret = 0;
203
204 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) {
205 ret = __share_page(rb_desc->meta_va);
206 if (ret)
207 break;
208
209 for (p = 0; p < rb_desc->nr_page_va; p++) {
210 ret = __share_page(rb_desc->page_va[p]);
211 if (ret)
212 break;
213 }
214
215 if (ret) {
216 while (--p >= 0)
217 __unshare_page(rb_desc->page_va[p]);
218 __unshare_page(rb_desc->meta_va);
219 break;
220 }
221 }
222
223 if (ret)
224 hyp_trace_buffer_unshare_hyp(trace_buffer, --cpu);
225
226 return ret;
227 }
228
hyp_trace_load(unsigned long size,void * priv)229 static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv)
230 {
231 struct hyp_trace_buffer *trace_buffer = priv;
232 size_t desc_size, tb_desc_size;
233 struct hyp_trace_desc *desc;
234 int ret;
235
236 if (WARN_ON(trace_buffer->desc))
237 return ERR_PTR(-EINVAL);
238
239 tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus());
240 desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc));
241 if (desc_size == SIZE_MAX)
242 return ERR_PTR(-E2BIG);
243
244 desc_size = PAGE_ALIGN(desc_size);
245 if (!desc_size)
246 return ERR_PTR(-E2BIG);
247
248 desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL);
249 if (!desc)
250 return ERR_PTR(-ENOMEM);
251
252 ret = __map_hyp(desc, desc_size);
253 if (ret)
254 goto err_free_desc;
255
256 trace_buffer->desc = desc;
257 trace_buffer->desc_size = desc_size;
258
259 ret = hyp_trace_buffer_alloc_bpages_backing(trace_buffer, size);
260 if (ret)
261 goto err_free_desc;
262
263 ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size,
264 cpu_possible_mask);
265 if (ret)
266 goto err_free_backing;
267
268 ret = hyp_trace_buffer_share_hyp(trace_buffer);
269 if (ret)
270 goto err_free_buffer;
271
272 ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size);
273 if (ret)
274 goto err_unload_pages;
275
276 return &desc->trace_buffer_desc;
277
278 err_unload_pages:
279 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX);
280
281 err_free_buffer:
282 trace_remote_free_buffer(&desc->trace_buffer_desc);
283
284 err_free_backing:
285 hyp_trace_buffer_free_bpages_backing(trace_buffer);
286
287 err_free_desc:
288 free_pages_exact(desc, desc_size);
289 trace_buffer->desc = NULL;
290
291 return ERR_PTR(ret);
292 }
293
hyp_trace_unload(struct trace_buffer_desc * desc,void * priv)294 static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv)
295 {
296 struct hyp_trace_buffer *trace_buffer = priv;
297
298 if (WARN_ON(desc != &trace_buffer->desc->trace_buffer_desc))
299 return;
300
301 kvm_call_hyp_nvhe(__tracing_unload);
302 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX);
303 trace_remote_free_buffer(desc);
304 hyp_trace_buffer_free_bpages_backing(trace_buffer);
305 free_pages_exact(trace_buffer->desc, trace_buffer->desc_size);
306 trace_buffer->desc = NULL;
307 trace_buffer->desc_size = 0;
308 }
309
hyp_trace_enable_tracing(bool enable,void * priv)310 static int hyp_trace_enable_tracing(bool enable, void *priv)
311 {
312 int ret;
313
314 hyp_trace_clock_enable(&hyp_clock, enable);
315
316 ret = kvm_call_hyp_nvhe(__tracing_enable, enable);
317 if (ret)
318 hyp_trace_clock_enable(&hyp_clock, !enable);
319
320 return ret;
321 }
322
hyp_trace_swap_reader_page(unsigned int cpu,void * priv)323 static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv)
324 {
325 return kvm_call_hyp_nvhe(__tracing_swap_reader, cpu);
326 }
327
hyp_trace_reset(unsigned int cpu,void * priv)328 static int hyp_trace_reset(unsigned int cpu, void *priv)
329 {
330 return kvm_call_hyp_nvhe(__tracing_reset, cpu);
331 }
332
hyp_trace_enable_event(unsigned short id,bool enable,void * priv)333 static int hyp_trace_enable_event(unsigned short id, bool enable, void *priv)
334 {
335 struct hyp_event_id *event_id = lm_alias(&__hyp_event_ids_start[id]);
336 struct page *page;
337 atomic_t *enabled;
338 void *map;
339
340 if (is_protected_kvm_enabled())
341 return kvm_call_hyp_nvhe(__tracing_enable_event, id, enable);
342
343 enabled = &event_id->enabled;
344 page = virt_to_page(enabled);
345 map = vmap(&page, 1, VM_MAP, PAGE_KERNEL);
346 if (!map)
347 return -ENOMEM;
348
349 enabled = map + offset_in_page(enabled);
350 atomic_set(enabled, enable);
351
352 vunmap(map);
353
354 return 0;
355 }
356
hyp_trace_clock_show(struct seq_file * m,void * v)357 static int hyp_trace_clock_show(struct seq_file *m, void *v)
358 {
359 seq_puts(m, "[boot]\n");
360
361 return 0;
362 }
363 DEFINE_SHOW_ATTRIBUTE(hyp_trace_clock);
364
hyp_trace_write_event_write(struct file * f,const char __user * ubuf,size_t cnt,loff_t * pos)365 static ssize_t hyp_trace_write_event_write(struct file *f, const char __user *ubuf,
366 size_t cnt, loff_t *pos)
367 {
368 unsigned long val;
369 int ret;
370
371 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
372 if (ret)
373 return ret;
374
375 kvm_call_hyp_nvhe(__tracing_write_event, val);
376
377 return cnt;
378 }
379
380 static const struct file_operations hyp_trace_write_event_fops = {
381 .write = hyp_trace_write_event_write,
382 };
383
hyp_trace_init_tracefs(struct dentry * d,void * priv)384 static int hyp_trace_init_tracefs(struct dentry *d, void *priv)
385 {
386 if (!tracefs_create_file("write_event", 0200, d, NULL, &hyp_trace_write_event_fops))
387 return -ENOMEM;
388
389 return tracefs_create_file("trace_clock", 0440, d, NULL, &hyp_trace_clock_fops) ?
390 0 : -ENOMEM;
391 }
392
393 static struct trace_remote_callbacks trace_remote_callbacks = {
394 .init = hyp_trace_init_tracefs,
395 .load_trace_buffer = hyp_trace_load,
396 .unload_trace_buffer = hyp_trace_unload,
397 .enable_tracing = hyp_trace_enable_tracing,
398 .swap_reader_page = hyp_trace_swap_reader_page,
399 .reset = hyp_trace_reset,
400 .enable_event = hyp_trace_enable_event,
401 };
402
403 static const char *__hyp_enter_exit_reason_str(u8 reason);
404
405 #include <asm/kvm_define_hypevents.h>
406
__hyp_enter_exit_reason_str(u8 reason)407 static const char *__hyp_enter_exit_reason_str(u8 reason)
408 {
409 static const char strs[][12] = {
410 "smc",
411 "hvc",
412 "sys",
413 "psci",
414 "host_abort",
415 "guest_exit",
416 "eret_host",
417 "eret_guest",
418 "unknown",
419 };
420
421 return strs[min(reason, HYP_REASON_UNKNOWN)];
422 }
423
hyp_trace_init_events(void)424 static void __init hyp_trace_init_events(void)
425 {
426 struct hyp_event_id *hyp_event_id = __hyp_event_ids_start;
427 struct remote_event *event = __hyp_events_start;
428 int id = 0;
429
430 /* Events on both sides hypervisor are sorted */
431 for (; event < __hyp_events_end; event++, hyp_event_id++, id++)
432 event->id = hyp_event_id->id = id;
433 }
434
kvm_hyp_trace_init(void)435 int __init kvm_hyp_trace_init(void)
436 {
437 int cpu;
438
439 if (is_kernel_in_hyp_mode())
440 return 0;
441
442 for_each_possible_cpu(cpu) {
443 const struct arch_timer_erratum_workaround *wa =
444 per_cpu(timer_unstable_counter_workaround, cpu);
445
446 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) &&
447 wa && wa->read_cntvct_el0) {
448 pr_warn("hyp trace can't handle CNTVCT workaround '%s'\n", wa->desc);
449 return -EOPNOTSUPP;
450 }
451 }
452
453 hyp_trace_init_events();
454
455 return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer,
456 __hyp_events_start, __hyp_events_end - __hyp_events_start);
457 }
458