1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2025 Google LLC 4 * Author: Vincent Donnefort <vdonnefort@google.com> 5 */ 6 7 #include <nvhe/clock.h> 8 #include <nvhe/mem_protect.h> 9 #include <nvhe/mm.h> 10 #include <nvhe/trace.h> 11 12 #include <asm/percpu.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/local.h> 15 16 #include "simple_ring_buffer.c" 17 18 static DEFINE_PER_CPU(struct simple_rb_per_cpu, __simple_rbs); 19 20 static struct hyp_trace_buffer { 21 struct simple_rb_per_cpu __percpu *simple_rbs; 22 void *bpages_backing_start; 23 size_t bpages_backing_size; 24 hyp_spinlock_t lock; 25 } trace_buffer = { 26 .simple_rbs = &__simple_rbs, 27 .lock = __HYP_SPIN_LOCK_UNLOCKED, 28 }; 29 30 static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer) 31 { 32 return trace_buffer->bpages_backing_size > 0; 33 } 34 35 void *tracing_reserve_entry(unsigned long length) 36 { 37 return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length, 38 trace_clock()); 39 } 40 41 void tracing_commit_entry(void) 42 { 43 simple_ring_buffer_commit(this_cpu_ptr(trace_buffer.simple_rbs)); 44 } 45 46 static int __admit_host_mem(void *start, u64 size) 47 { 48 if (!PAGE_ALIGNED(start) || !PAGE_ALIGNED(size) || !size) 49 return -EINVAL; 50 51 if (!is_protected_kvm_enabled()) 52 return 0; 53 54 return __pkvm_host_donate_hyp(hyp_virt_to_pfn(start), size >> PAGE_SHIFT); 55 } 56 57 static void __release_host_mem(void *start, u64 size) 58 { 59 if (!is_protected_kvm_enabled()) 60 return; 61 62 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); 63 } 64 65 static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer, 66 struct hyp_trace_desc *desc) 67 { 68 void *start = (void *)kern_hyp_va(desc->bpages_backing_start); 69 size_t size = desc->bpages_backing_size; 70 int ret; 71 72 ret = __admit_host_mem(start, size); 73 if (ret) 74 return ret; 75 76 memset(start, 0, size); 77 78 trace_buffer->bpages_backing_start = start; 79 trace_buffer->bpages_backing_size = size; 80 81 return 0; 82 } 83 84 static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer) 85 { 86 void *start = trace_buffer->bpages_backing_start; 87 size_t size = trace_buffer->bpages_backing_size; 88 89 if (!size) 90 return; 91 92 memset(start, 0, size); 93 94 __release_host_mem(start, size); 95 96 trace_buffer->bpages_backing_start = 0; 97 trace_buffer->bpages_backing_size = 0; 98 } 99 100 static void *__pin_shared_page(unsigned long kern_va) 101 { 102 void *va = kern_hyp_va((void *)kern_va); 103 104 if (!is_protected_kvm_enabled()) 105 return va; 106 107 return hyp_pin_shared_mem(va, va + PAGE_SIZE) ? NULL : va; 108 } 109 110 static void __unpin_shared_page(void *va) 111 { 112 if (!is_protected_kvm_enabled()) 113 return; 114 115 hyp_unpin_shared_mem(va, va + PAGE_SIZE); 116 } 117 118 static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer) 119 { 120 int cpu; 121 122 hyp_assert_lock_held(&trace_buffer->lock); 123 124 if (!hyp_trace_buffer_loaded(trace_buffer)) 125 return; 126 127 for (cpu = 0; cpu < hyp_nr_cpus; cpu++) 128 simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu), 129 __unpin_shared_page); 130 131 hyp_trace_buffer_unload_bpage_backing(trace_buffer); 132 } 133 134 static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, 135 struct hyp_trace_desc *desc) 136 { 137 struct simple_buffer_page *bpages; 138 struct ring_buffer_desc *rb_desc; 139 int ret, cpu; 140 141 hyp_assert_lock_held(&trace_buffer->lock); 142 143 if (hyp_trace_buffer_loaded(trace_buffer)) 144 return -EINVAL; 145 146 ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc); 147 if (ret) 148 return ret; 149 150 bpages = trace_buffer->bpages_backing_start; 151 for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) { 152 ret = simple_ring_buffer_init_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu), 153 bpages, rb_desc, __pin_shared_page, 154 __unpin_shared_page); 155 if (ret) 156 break; 157 158 bpages += rb_desc->nr_page_va; 159 } 160 161 if (ret) 162 hyp_trace_buffer_unload(trace_buffer); 163 164 return ret; 165 } 166 167 static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_size) 168 { 169 struct ring_buffer_desc *rb_desc; 170 unsigned int cpu; 171 size_t nr_bpages; 172 void *desc_end; 173 174 if (!is_protected_kvm_enabled()) 175 return true; 176 177 /* 178 * Both desc_size and bpages_backing_size are untrusted host-provided 179 * values. We rely on __pkvm_host_donate_hyp() to enforce their validity. 180 */ 181 desc_end = (void *)desc + desc_size; 182 nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page); 183 184 for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) { 185 /* Can we read nr_page_va? */ 186 if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end) 187 return false; 188 189 /* Overflow desc? */ 190 if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end) 191 return false; 192 193 /* Overflow bpages backing memory? */ 194 if (nr_bpages < rb_desc->nr_page_va) 195 return false; 196 197 if (cpu >= hyp_nr_cpus) 198 return false; 199 200 if (cpu != rb_desc->cpu) 201 return false; 202 203 nr_bpages -= rb_desc->nr_page_va; 204 } 205 206 return true; 207 } 208 209 int __tracing_load(unsigned long desc_hva, size_t desc_size) 210 { 211 struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva); 212 int ret; 213 214 ret = __admit_host_mem(desc, desc_size); 215 if (ret) 216 return ret; 217 218 if (!hyp_trace_desc_is_valid(desc, desc_size)) { 219 ret = -EINVAL; 220 goto err_release_desc; 221 } 222 223 hyp_spin_lock(&trace_buffer.lock); 224 225 ret = hyp_trace_buffer_load(&trace_buffer, desc); 226 227 hyp_spin_unlock(&trace_buffer.lock); 228 229 err_release_desc: 230 __release_host_mem(desc, desc_size); 231 return ret; 232 } 233 234 void __tracing_unload(void) 235 { 236 hyp_spin_lock(&trace_buffer.lock); 237 hyp_trace_buffer_unload(&trace_buffer); 238 hyp_spin_unlock(&trace_buffer.lock); 239 } 240 241 int __tracing_enable(bool enable) 242 { 243 int cpu, ret = enable ? -EINVAL : 0; 244 245 hyp_spin_lock(&trace_buffer.lock); 246 247 if (!hyp_trace_buffer_loaded(&trace_buffer)) 248 goto unlock; 249 250 for (cpu = 0; cpu < hyp_nr_cpus; cpu++) 251 simple_ring_buffer_enable_tracing(per_cpu_ptr(trace_buffer.simple_rbs, cpu), 252 enable); 253 254 ret = 0; 255 256 unlock: 257 hyp_spin_unlock(&trace_buffer.lock); 258 259 return ret; 260 } 261 262 int __tracing_swap_reader(unsigned int cpu) 263 { 264 int ret = -ENODEV; 265 266 if (cpu >= hyp_nr_cpus) 267 return -EINVAL; 268 269 hyp_spin_lock(&trace_buffer.lock); 270 271 if (hyp_trace_buffer_loaded(&trace_buffer)) 272 ret = simple_ring_buffer_swap_reader_page( 273 per_cpu_ptr(trace_buffer.simple_rbs, cpu)); 274 275 hyp_spin_unlock(&trace_buffer.lock); 276 277 return ret; 278 } 279 280 void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) 281 { 282 int cpu; 283 284 /* After this loop, all CPUs are observing the new bank... */ 285 for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { 286 struct simple_rb_per_cpu *simple_rb = per_cpu_ptr(trace_buffer.simple_rbs, cpu); 287 288 while (READ_ONCE(simple_rb->status) == SIMPLE_RB_WRITING) 289 ; 290 } 291 292 /* ...we can now override the old one and swap. */ 293 trace_clock_update(mult, shift, epoch_ns, epoch_cyc); 294 } 295 296 int __tracing_reset(unsigned int cpu) 297 { 298 int ret = -ENODEV; 299 300 if (cpu >= hyp_nr_cpus) 301 return -EINVAL; 302 303 hyp_spin_lock(&trace_buffer.lock); 304 305 if (hyp_trace_buffer_loaded(&trace_buffer)) 306 ret = simple_ring_buffer_reset(per_cpu_ptr(trace_buffer.simple_rbs, cpu)); 307 308 hyp_spin_unlock(&trace_buffer.lock); 309 310 return ret; 311 } 312