1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2025 Google LLC 4 * Author: Vincent Donnefort <vdonnefort@google.com> 5 */ 6 7 #include <linux/cpumask.h> 8 #include <linux/trace_remote.h> 9 #include <linux/tracefs.h> 10 #include <linux/simple_ring_buffer.h> 11 12 #include <asm/arch_timer.h> 13 #include <asm/kvm_host.h> 14 #include <asm/kvm_hyptrace.h> 15 #include <asm/kvm_mmu.h> 16 #include <asm/kvm_pkvm.h> 17 18 #include "hyp_trace.h" 19 20 /* Same 10min used by clocksource when width is more than 32-bits */ 21 #define CLOCK_MAX_CONVERSION_S 600 22 /* 23 * Time to give for the clock init. Long enough to get a good mult/shift 24 * estimation. Short enough to not delay the tracing start too much. 25 */ 26 #define CLOCK_INIT_MS 100 27 /* 28 * Time between clock checks. Must be small enough to catch clock deviation when 29 * it is still tiny. 30 */ 31 #define CLOCK_UPDATE_MS 500 32 33 static struct hyp_trace_clock { 34 u64 cycles; 35 u64 cyc_overflow64; 36 u64 boot; 37 u32 mult; 38 u32 shift; 39 struct delayed_work work; 40 struct completion ready; 41 } hyp_clock; 42 43 static void __hyp_clock_work(struct work_struct *work) 44 { 45 struct delayed_work *dwork = to_delayed_work(work); 46 struct hyp_trace_clock *hyp_clock; 47 struct system_time_snapshot snap; 48 u64 rate, delta_cycles; 49 u64 boot, delta_boot; 50 51 hyp_clock = container_of(dwork, struct hyp_trace_clock, work); 52 53 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); 54 boot = ktime_to_ns(snap.systime); 55 56 delta_boot = boot - hyp_clock->boot; 57 delta_cycles = snap.cycles - hyp_clock->cycles; 58 59 /* Compare hyp clock with the kernel boot clock */ 60 if (hyp_clock->mult) { 61 u64 err, cur = delta_cycles; 62 63 WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64); 64 cur = mul_u64_u32_shr(cur, hyp_clock->mult, hyp_clock->shift); 65 cur += hyp_clock->boot; 66 67 err = abs_diff(cur, boot); 68 /* No deviation, only update epoch if necessary */ 69 if (!err) { 70 if (delta_cycles >= (hyp_clock->cyc_overflow64 >> 1)) 71 goto fast_forward; 72 73 goto resched; 74 } 75 76 /* Warn if the error is above tracing precision (1us) */ 77 if (err > NSEC_PER_USEC) 78 pr_warn_ratelimited("hyp trace clock off by %lluus\n", 79 err / NSEC_PER_USEC); 80 } 81 82 rate = mul_u64_u64_div_u64(delta_cycles, NSEC_PER_SEC, delta_boot); 83 84 clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift, 85 rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S); 86 87 /* Add a comfortable 50% margin */ 88 hyp_clock->cyc_overflow64 = (U64_MAX / hyp_clock->mult) >> 1; 89 90 fast_forward: 91 hyp_clock->cycles = snap.cycles; 92 hyp_clock->boot = boot; 93 kvm_call_hyp_nvhe(__tracing_update_clock, hyp_clock->mult, 94 hyp_clock->shift, hyp_clock->boot, hyp_clock->cycles); 95 complete(&hyp_clock->ready); 96 97 resched: 98 schedule_delayed_work(&hyp_clock->work, 99 msecs_to_jiffies(CLOCK_UPDATE_MS)); 100 } 101 102 static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enable) 103 { 104 struct system_time_snapshot snap; 105 106 if (!enable) { 107 cancel_delayed_work_sync(&hyp_clock->work); 108 return; 109 } 110 111 ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); 112 113 hyp_clock->boot = ktime_to_ns(snap.systime); 114 hyp_clock->cycles = snap.cycles; 115 hyp_clock->mult = 0; 116 117 init_completion(&hyp_clock->ready); 118 INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work); 119 schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS)); 120 wait_for_completion(&hyp_clock->ready); 121 } 122 123 /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */ 124 static struct hyp_trace_buffer { 125 struct hyp_trace_desc *desc; 126 size_t desc_size; 127 } trace_buffer; 128 129 static int __map_hyp(void *start, size_t size) 130 { 131 if (is_protected_kvm_enabled()) 132 return 0; 133 134 return create_hyp_mappings(start, start + size, PAGE_HYP); 135 } 136 137 static int __share_page(unsigned long va) 138 { 139 return kvm_share_hyp((void *)va, (void *)va + 1); 140 } 141 142 static void __unshare_page(unsigned long va) 143 { 144 kvm_unshare_hyp((void *)va, (void *)va + 1); 145 } 146 147 static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) 148 { 149 unsigned long nr_bpages; 150 size_t backing_size; 151 void *start; 152 int ret; 153 154 if (is_protected_kvm_enabled()) { 155 trace_buffer->desc->bpages_backing_start = 0; 156 trace_buffer->desc->bpages_backing_size = 0; 157 return 0; 158 } 159 160 nr_bpages = __calc_nr_pages_ring_buffer_desc(size); 161 backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * 162 num_possible_cpus()); 163 164 start = alloc_pages_exact(backing_size, GFP_KERNEL_ACCOUNT); 165 if (!start) 166 return -ENOMEM; 167 168 ret = __map_hyp(start, backing_size); 169 if (ret) { 170 free_pages_exact(start, backing_size); 171 return ret; 172 } 173 174 trace_buffer->desc->bpages_backing_start = (unsigned long)start; 175 trace_buffer->desc->bpages_backing_size = backing_size; 176 177 return ret; 178 } 179 180 static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) 181 { 182 if (!trace_buffer->desc->bpages_backing_start) 183 return; 184 185 free_pages_exact((void *)trace_buffer->desc->bpages_backing_start, 186 trace_buffer->desc->bpages_backing_size); 187 } 188 189 static void hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer *trace_buffer, int last_cpu) 190 { 191 struct ring_buffer_desc *rb_desc; 192 int cpu, p; 193 194 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) { 195 if (cpu > last_cpu) 196 break; 197 198 __unshare_page(rb_desc->meta_va); 199 for (p = 0; p < rb_desc->nr_page_va; p++) 200 __unshare_page(rb_desc->page_va[p]); 201 } 202 } 203 204 static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer) 205 { 206 struct ring_buffer_desc *rb_desc; 207 int cpu, p, ret = 0; 208 209 for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) { 210 ret = __share_page(rb_desc->meta_va); 211 if (ret) 212 break; 213 214 for (p = 0; p < rb_desc->nr_page_va; p++) { 215 ret = __share_page(rb_desc->page_va[p]); 216 if (ret) 217 break; 218 } 219 220 if (ret) { 221 while (--p >= 0) 222 __unshare_page(rb_desc->page_va[p]); 223 __unshare_page(rb_desc->meta_va); 224 break; 225 } 226 } 227 228 if (ret) 229 hyp_trace_buffer_unshare_hyp(trace_buffer, --cpu); 230 231 return ret; 232 } 233 234 static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) 235 { 236 struct hyp_trace_buffer *trace_buffer = priv; 237 size_t desc_size, tb_desc_size; 238 struct hyp_trace_desc *desc; 239 int ret; 240 241 if (WARN_ON(trace_buffer->desc)) 242 return ERR_PTR(-EINVAL); 243 244 tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus()); 245 desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc)); 246 if (desc_size == SIZE_MAX) 247 return ERR_PTR(-E2BIG); 248 249 desc_size = PAGE_ALIGN(desc_size); 250 if (!desc_size) 251 return ERR_PTR(-E2BIG); 252 253 desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL); 254 if (!desc) 255 return ERR_PTR(-ENOMEM); 256 257 ret = __map_hyp(desc, desc_size); 258 if (ret) 259 goto err_free_desc; 260 261 trace_buffer->desc = desc; 262 trace_buffer->desc_size = desc_size; 263 264 ret = hyp_trace_buffer_alloc_bpages_backing(trace_buffer, size); 265 if (ret) 266 goto err_free_desc; 267 268 ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size, 269 cpu_possible_mask); 270 if (ret) 271 goto err_free_backing; 272 273 ret = hyp_trace_buffer_share_hyp(trace_buffer); 274 if (ret) 275 goto err_free_buffer; 276 277 ret = pkvm_call_hyp_req(__tracing_load, desc, desc_size); 278 if (ret) 279 goto err_unload_pages; 280 281 return &desc->trace_buffer_desc; 282 283 err_unload_pages: 284 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX); 285 286 err_free_buffer: 287 trace_remote_free_buffer(&desc->trace_buffer_desc); 288 289 err_free_backing: 290 hyp_trace_buffer_free_bpages_backing(trace_buffer); 291 292 err_free_desc: 293 free_pages_exact(desc, desc_size); 294 trace_buffer->desc = NULL; 295 296 return ERR_PTR(ret); 297 } 298 299 static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv) 300 { 301 struct hyp_trace_buffer *trace_buffer = priv; 302 303 if (WARN_ON(desc != &trace_buffer->desc->trace_buffer_desc)) 304 return; 305 306 kvm_call_hyp_nvhe(__tracing_unload); 307 hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX); 308 trace_remote_free_buffer(desc); 309 hyp_trace_buffer_free_bpages_backing(trace_buffer); 310 free_pages_exact(trace_buffer->desc, trace_buffer->desc_size); 311 trace_buffer->desc = NULL; 312 trace_buffer->desc_size = 0; 313 } 314 315 static int hyp_trace_enable_tracing(bool enable, void *priv) 316 { 317 int ret; 318 319 hyp_trace_clock_enable(&hyp_clock, enable); 320 321 ret = kvm_call_hyp_nvhe(__tracing_enable, enable); 322 if (ret) 323 hyp_trace_clock_enable(&hyp_clock, !enable); 324 325 return ret; 326 } 327 328 static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv) 329 { 330 return kvm_call_hyp_nvhe(__tracing_swap_reader, cpu); 331 } 332 333 static int hyp_trace_reset(unsigned int cpu, void *priv) 334 { 335 return kvm_call_hyp_nvhe(__tracing_reset, cpu); 336 } 337 338 static int hyp_trace_enable_event(unsigned short id, bool enable, void *priv) 339 { 340 struct hyp_event_id *event_id = lm_alias(&__hyp_event_ids_start[id]); 341 struct page *page; 342 atomic_t *enabled; 343 void *map; 344 345 if (is_protected_kvm_enabled()) 346 return kvm_call_hyp_nvhe(__tracing_enable_event, id, enable); 347 348 enabled = &event_id->enabled; 349 page = virt_to_page(enabled); 350 map = vmap(&page, 1, VM_MAP, PAGE_KERNEL); 351 if (!map) 352 return -ENOMEM; 353 354 enabled = map + offset_in_page(enabled); 355 atomic_set(enabled, enable); 356 357 vunmap(map); 358 359 return 0; 360 } 361 362 static int hyp_trace_clock_show(struct seq_file *m, void *v) 363 { 364 seq_puts(m, "[boot]\n"); 365 366 return 0; 367 } 368 DEFINE_SHOW_ATTRIBUTE(hyp_trace_clock); 369 370 static ssize_t hyp_trace_write_event_write(struct file *f, const char __user *ubuf, 371 size_t cnt, loff_t *pos) 372 { 373 unsigned long val; 374 int ret; 375 376 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 377 if (ret) 378 return ret; 379 380 kvm_call_hyp_nvhe(__tracing_write_event, val); 381 382 return cnt; 383 } 384 385 static const struct file_operations hyp_trace_write_event_fops = { 386 .write = hyp_trace_write_event_write, 387 }; 388 389 static int hyp_trace_init_tracefs(struct dentry *d, void *priv) 390 { 391 if (!tracefs_create_file("write_event", 0200, d, NULL, &hyp_trace_write_event_fops)) 392 return -ENOMEM; 393 394 return tracefs_create_file("trace_clock", 0440, d, NULL, &hyp_trace_clock_fops) ? 395 0 : -ENOMEM; 396 } 397 398 static struct trace_remote_callbacks trace_remote_callbacks = { 399 .init = hyp_trace_init_tracefs, 400 .load_trace_buffer = hyp_trace_load, 401 .unload_trace_buffer = hyp_trace_unload, 402 .enable_tracing = hyp_trace_enable_tracing, 403 .swap_reader_page = hyp_trace_swap_reader_page, 404 .reset = hyp_trace_reset, 405 .enable_event = hyp_trace_enable_event, 406 }; 407 408 static const char *__hyp_enter_exit_reason_str(u8 reason); 409 410 struct remote_event_format_hyp_printk; 411 static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq); 412 413 #include "define_hypevents.h" 414 415 static const char *__hyp_enter_exit_reason_str(u8 reason) 416 { 417 static const char strs[][12] = { 418 "smc", 419 "hvc", 420 "sys", 421 "psci", 422 "host_abort", 423 "guest_exit", 424 "eret_host", 425 "eret_guest", 426 "unknown", 427 }; 428 429 return strs[min(reason, HYP_REASON_UNKNOWN)]; 430 } 431 432 static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq) 433 { 434 const struct hyp_string_fmt *hyp_str_fmt; 435 const char *fmt; 436 437 trace_seq_putc(seq, ' '); 438 439 if (entry->fmt_id >= (__hyp_string_fmts_end - __hyp_string_fmts_start)) { 440 trace_seq_printf(seq, "Unknown hyp_string_fmt ID %u\n", entry->fmt_id); 441 return; 442 } 443 444 hyp_str_fmt = &__hyp_string_fmts_start[entry->fmt_id]; 445 fmt = hyp_str_fmt->fmt; 446 447 switch (hyp_str_fmt->nr_args) { 448 case 0: 449 trace_seq_puts(seq, fmt); 450 break; 451 case 1: 452 trace_seq_printf(seq, fmt, entry->args[0]); 453 break; 454 case 2: 455 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1]); 456 break; 457 case 3: 458 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2]); 459 break; 460 case 4: 461 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], 462 entry->args[3]); 463 break; 464 case 5: 465 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], 466 entry->args[3], entry->args[4]); 467 break; 468 case 6: 469 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], 470 entry->args[3], entry->args[4], entry->args[5]); 471 break; 472 case 7: 473 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], 474 entry->args[3], entry->args[4], entry->args[5], entry->args[6]); 475 break; 476 default: 477 trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], 478 entry->args[2], entry->args[3], entry->args[4], entry->args[5], 479 entry->args[6], entry->args[7]); 480 break; 481 } 482 483 if (trace_seq_used(seq) && seq->buffer[trace_seq_used(seq) - 1] != '\n') 484 trace_seq_putc(seq, '\n'); 485 } 486 487 static void __init hyp_trace_init_events(void) 488 { 489 struct hyp_event_id *hyp_event_id = __hyp_event_ids_start; 490 struct remote_event *event = __hyp_events_start; 491 int id = 0; 492 493 /* Events on both sides hypervisor are sorted */ 494 for (; event < __hyp_events_end; event++, hyp_event_id++, id++) 495 event->id = hyp_event_id->id = id; 496 497 WARN(__hyp_string_fmts_end - __hyp_string_fmts_start > U16_MAX + 1, 498 "Too many trace_hyp_printk() callsites\n"); 499 } 500 501 int __init kvm_hyp_trace_init(void) 502 { 503 int cpu; 504 505 if (is_kernel_in_hyp_mode()) 506 return 0; 507 508 for_each_possible_cpu(cpu) { 509 const struct arch_timer_erratum_workaround *wa = 510 per_cpu(timer_unstable_counter_workaround, cpu); 511 512 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) && 513 wa && wa->read_cntvct_el0) { 514 pr_warn("hyp trace can't handle CNTVCT workaround '%s'\n", wa->desc); 515 return -EOPNOTSUPP; 516 } 517 } 518 519 hyp_trace_init_events(); 520 521 return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer, 522 __hyp_events_start, __hyp_events_end - __hyp_events_start); 523 } 524