1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2025 Google LLC 4 * Author: Vincent Donnefort <vdonnefort@google.com> 5 */ 6 7 #include <linux/cpumask.h> 8 #include <linux/trace_remote.h> 9 #include <linux/tracefs.h> 10 #include <linux/simple_ring_buffer.h> 11 12 #include <asm/arch_timer.h> 13 #include <asm/kvm_host.h> 14 #include <asm/kvm_hyptrace.h> 15 #include <asm/kvm_mmu.h> 16 17 #include "hyp_trace.h" 18 19 /* Same 10min used by clocksource when width is more than 32-bits */ 20 #define CLOCK_MAX_CONVERSION_S 600 21 /* 22 * Time to give for the clock init. Long enough to get a good mult/shift 23 * estimation. Short enough to not delay the tracing start too much. 24 */ 25 #define CLOCK_INIT_MS 100 26 /* 27 * Time between clock checks. Must be small enough to catch clock deviation when 28 * it is still tiny. 29 */ 30 #define CLOCK_UPDATE_MS 500 31 32 static struct hyp_trace_clock { 33 u64 cycles; 34 u64 cyc_overflow64; 35 u64 boot; 36 u32 mult; 37 u32 shift; 38 struct delayed_work work; 39 struct completion ready; 40 } hyp_clock; 41 42 static void __hyp_clock_work(struct work_struct *work) 43 { 44 struct delayed_work *dwork = to_delayed_work(work); 45 struct hyp_trace_clock *hyp_clock; 46 struct system_time_snapshot snap; 47 u64 rate, delta_cycles; 48 u64 boot, delta_boot; 49 50 hyp_clock = container_of(dwork, struct hyp_trace_clock, work); 51 52 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); 53 boot = ktime_to_ns(snap.systime); 54 55 delta_boot = boot - hyp_clock->boot; 56 delta_cycles = snap.cycles - hyp_clock->cycles; 57 58 /* Compare hyp clock with the kernel boot clock */ 59 if (hyp_clock->mult) { 60 u64 err, cur = delta_cycles; 61 62 if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) { 63 __uint128_t tmp = (__uint128_t)cur * hyp_clock->mult; 64 65 cur = tmp >> hyp_clock->shift; 66 } else { 67 cur *= hyp_clock->mult; 68 cur >>= hyp_clock->shift; 69 } 70 cur += hyp_clock->boot; 71 72 err = abs_diff(cur, boot); 73 /* No deviation, only update epoch if necessary */ 74 if (!err) { 75 if (delta_cycles >= (hyp_clock->cyc_overflow64 >> 1)) 76 goto fast_forward; 77 78 goto resched; 79 } 80 81 /* Warn if the error is above tracing precision (1us) */ 82 if (err > NSEC_PER_USEC) 83 pr_warn_ratelimited("hyp trace clock off by %lluus\n", 84 err / NSEC_PER_USEC); 85 } 86 87 rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot); 88 89 clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift, 90 rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S); 91 92 /* Add a comfortable 50% margin */ 93 hyp_clock->cyc_overflow64 = (U64_MAX / hyp_clock->mult) >> 1; 94 95 fast_forward: 96 hyp_clock->cycles = snap.cycles; 97 hyp_clock->boot = boot; 98 kvm_call_hyp_nvhe(__tracing_update_clock, hyp_clock->mult, 99 hyp_clock->shift, hyp_clock->boot, hyp_clock->cycles); 100 complete(&hyp_clock->ready); 101 102 resched: 103 schedule_delayed_work(&hyp_clock->work, 104 msecs_to_jiffies(CLOCK_UPDATE_MS)); 105 } 106 107 static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enable) 108 { 109 struct system_time_snapshot snap; 110 111 if (!enable) { 112 cancel_delayed_work_sync(&hyp_clock->work); 113 return; 114 } 115 116 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); 117 118 hyp_clock->boot = ktime_to_ns(snap.systime); 119 hyp_clock->cycles = snap.cycles; 120 hyp_clock->mult = 0; 121 122 init_completion(&hyp_clock->ready); 123 INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work); 124 schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS)); 125 wait_for_completion(&hyp_clock->ready); 126 } 127 128 /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */ 129 static struct hyp_trace_buffer { 130 struct hyp_trace_desc *desc; 131 size_t desc_size; 132 } trace_buffer; 133 134 static int __map_hyp(void *start, size_t size) 135 { 136 if (is_protected_kvm_enabled()) 137 return 0; 138 139 return create_hyp_mappings(start, start + size, PAGE_HYP); 140 } 141 142 static int __share_page(unsigned long va) 143 { 144 return kvm_share_hyp((void *)va, (void *)va + 1); 145 } 146 147 static void __unshare_page(unsigned long va) 148 { 149 kvm_unshare_hyp((void *)va, (void *)va + 1); 150 } 151 152 static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) 153 { 154 int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; 155 size_t backing_size; 156 void *start; 157 int ret; 158 159 backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * 160 num_possible_cpus()); 161 162 start = alloc_pages_exact(backing_size, GFP_KERNEL_ACCOUNT); 163 if (!start) 164 return -ENOMEM; 165 166 ret = __map_hyp(start, backing_size); 167 if (ret) { 168 free_pages_exact(start, backing_size); 169 return ret; 170 } 171 172 trace_buffer->desc->bpages_backing_start = (unsigned long)start; 173 trace_buffer->desc->bpages_backing_size = backing_size; 174 175 return ret; 176 } 177 178 static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) 179 { 180 free_pages_exact((void *)trace_buffer->desc->bpages_backing_start, 181 trace_buffer->desc->bpages_backing_size); 182 } 183 184 static void hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer *trace_buffer, int last_cpu) 185 { 186 struct ring_buffer_desc *rb_desc; 187 int cpu, p; 188 189 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) { 190 if (cpu > last_cpu) 191 break; 192 193 __unshare_page(rb_desc->meta_va); 194 for (p = 0; p < rb_desc->nr_page_va; p++) 195 __unshare_page(rb_desc->page_va[p]); 196 } 197 } 198 199 static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer) 200 { 201 struct ring_buffer_desc *rb_desc; 202 int cpu, p, ret = 0; 203 204 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) { 205 ret = __share_page(rb_desc->meta_va); 206 if (ret) 207 break; 208 209 for (p = 0; p < rb_desc->nr_page_va; p++) { 210 ret = __share_page(rb_desc->page_va[p]); 211 if (ret) 212 break; 213 } 214 215 if (ret) { 216 while (--p >= 0) 217 __unshare_page(rb_desc->page_va[p]); 218 __unshare_page(rb_desc->meta_va); 219 break; 220 } 221 } 222 223 if (ret) 224 hyp_trace_buffer_unshare_hyp(trace_buffer, --cpu); 225 226 return ret; 227 } 228 229 static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) 230 { 231 struct hyp_trace_buffer *trace_buffer = priv; 232 size_t desc_size, tb_desc_size; 233 struct hyp_trace_desc *desc; 234 int ret; 235 236 if (WARN_ON(trace_buffer->desc)) 237 return ERR_PTR(-EINVAL); 238 239 tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus()); 240 desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc)); 241 if (desc_size == SIZE_MAX) 242 return ERR_PTR(-E2BIG); 243 244 desc_size = PAGE_ALIGN(desc_size); 245 if (!desc_size) 246 return ERR_PTR(-E2BIG); 247 248 desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL); 249 if (!desc) 250 return ERR_PTR(-ENOMEM); 251 252 ret = __map_hyp(desc, desc_size); 253 if (ret) 254 goto err_free_desc; 255 256 trace_buffer->desc = desc; 257 trace_buffer->desc_size = desc_size; 258 259 ret = hyp_trace_buffer_alloc_bpages_backing(trace_buffer, size); 260 if (ret) 261 goto err_free_desc; 262 263 ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size, 264 cpu_possible_mask); 265 if (ret) 266 goto err_free_backing; 267 268 ret = hyp_trace_buffer_share_hyp(trace_buffer); 269 if (ret) 270 goto err_free_buffer; 271 272 ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size); 273 if (ret) 274 goto err_unload_pages; 275 276 return &desc->trace_buffer_desc; 277 278 err_unload_pages: 279 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX); 280 281 err_free_buffer: 282 trace_remote_free_buffer(&desc->trace_buffer_desc); 283 284 err_free_backing: 285 hyp_trace_buffer_free_bpages_backing(trace_buffer); 286 287 err_free_desc: 288 free_pages_exact(desc, desc_size); 289 trace_buffer->desc = NULL; 290 291 return ERR_PTR(ret); 292 } 293 294 static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv) 295 { 296 struct hyp_trace_buffer *trace_buffer = priv; 297 298 if (WARN_ON(desc != &trace_buffer->desc->trace_buffer_desc)) 299 return; 300 301 kvm_call_hyp_nvhe(__tracing_unload); 302 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX); 303 trace_remote_free_buffer(desc); 304 hyp_trace_buffer_free_bpages_backing(trace_buffer); 305 free_pages_exact(trace_buffer->desc, trace_buffer->desc_size); 306 trace_buffer->desc = NULL; 307 trace_buffer->desc_size = 0; 308 } 309 310 static int hyp_trace_enable_tracing(bool enable, void *priv) 311 { 312 int ret; 313 314 hyp_trace_clock_enable(&hyp_clock, enable); 315 316 ret = kvm_call_hyp_nvhe(__tracing_enable, enable); 317 if (ret) 318 hyp_trace_clock_enable(&hyp_clock, !enable); 319 320 return ret; 321 } 322 323 static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv) 324 { 325 return kvm_call_hyp_nvhe(__tracing_swap_reader, cpu); 326 } 327 328 static int hyp_trace_reset(unsigned int cpu, void *priv) 329 { 330 return kvm_call_hyp_nvhe(__tracing_reset, cpu); 331 } 332 333 static int hyp_trace_enable_event(unsigned short id, bool enable, void *priv) 334 { 335 struct hyp_event_id *event_id = lm_alias(&__hyp_event_ids_start[id]); 336 struct page *page; 337 atomic_t *enabled; 338 void *map; 339 340 if (is_protected_kvm_enabled()) 341 return kvm_call_hyp_nvhe(__tracing_enable_event, id, enable); 342 343 enabled = &event_id->enabled; 344 page = virt_to_page(enabled); 345 map = vmap(&page, 1, VM_MAP, PAGE_KERNEL); 346 if (!map) 347 return -ENOMEM; 348 349 enabled = map + offset_in_page(enabled); 350 atomic_set(enabled, enable); 351 352 vunmap(map); 353 354 return 0; 355 } 356 357 static int hyp_trace_clock_show(struct seq_file *m, void *v) 358 { 359 seq_puts(m, "[boot]\n"); 360 361 return 0; 362 } 363 DEFINE_SHOW_ATTRIBUTE(hyp_trace_clock); 364 365 static ssize_t hyp_trace_write_event_write(struct file *f, const char __user *ubuf, 366 size_t cnt, loff_t *pos) 367 { 368 unsigned long val; 369 int ret; 370 371 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 372 if (ret) 373 return ret; 374 375 kvm_call_hyp_nvhe(__tracing_write_event, val); 376 377 return cnt; 378 } 379 380 static const struct file_operations hyp_trace_write_event_fops = { 381 .write = hyp_trace_write_event_write, 382 }; 383 384 static int hyp_trace_init_tracefs(struct dentry *d, void *priv) 385 { 386 if (!tracefs_create_file("write_event", 0200, d, NULL, &hyp_trace_write_event_fops)) 387 return -ENOMEM; 388 389 return tracefs_create_file("trace_clock", 0440, d, NULL, &hyp_trace_clock_fops) ? 390 0 : -ENOMEM; 391 } 392 393 static struct trace_remote_callbacks trace_remote_callbacks = { 394 .init = hyp_trace_init_tracefs, 395 .load_trace_buffer = hyp_trace_load, 396 .unload_trace_buffer = hyp_trace_unload, 397 .enable_tracing = hyp_trace_enable_tracing, 398 .swap_reader_page = hyp_trace_swap_reader_page, 399 .reset = hyp_trace_reset, 400 .enable_event = hyp_trace_enable_event, 401 }; 402 403 static const char *__hyp_enter_exit_reason_str(u8 reason); 404 405 #include <asm/kvm_define_hypevents.h> 406 407 static const char *__hyp_enter_exit_reason_str(u8 reason) 408 { 409 static const char strs[][12] = { 410 "smc", 411 "hvc", 412 "sys", 413 "psci", 414 "host_abort", 415 "guest_exit", 416 "eret_host", 417 "eret_guest", 418 "unknown", 419 }; 420 421 return strs[min(reason, HYP_REASON_UNKNOWN)]; 422 } 423 424 static void __init hyp_trace_init_events(void) 425 { 426 struct hyp_event_id *hyp_event_id = __hyp_event_ids_start; 427 struct remote_event *event = __hyp_events_start; 428 int id = 0; 429 430 /* Events on both sides hypervisor are sorted */ 431 for (; event < __hyp_events_end; event++, hyp_event_id++, id++) 432 event->id = hyp_event_id->id = id; 433 } 434 435 int __init kvm_hyp_trace_init(void) 436 { 437 int cpu; 438 439 if (is_kernel_in_hyp_mode()) 440 return 0; 441 442 for_each_possible_cpu(cpu) { 443 const struct arch_timer_erratum_workaround *wa = 444 per_cpu(timer_unstable_counter_workaround, cpu); 445 446 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) && 447 wa && wa->read_cntvct_el0) { 448 pr_warn("hyp trace can't handle CNTVCT workaround '%s'\n", wa->desc); 449 return -EOPNOTSUPP; 450 } 451 } 452 453 hyp_trace_init_events(); 454 455 return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer, 456 __hyp_events_start, __hyp_events_end - __hyp_events_start); 457 } 458