1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Atish Patra <atish.patra@wdc.com> 7 */ 8 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/kvm_host.h> 12 #include <linux/uaccess.h> 13 #include <clocksource/timer-riscv.h> 14 #include <asm/delay.h> 15 #include <asm/kvm_isa.h> 16 #include <asm/kvm_nacl.h> 17 #include <asm/kvm_vcpu_timer.h> 18 19 static u64 kvm_riscv_current_cycles(struct kvm_guest_timer *gt) 20 { 21 return get_cycles64() + gt->time_delta; 22 } 23 24 static u64 kvm_riscv_delta_cycles2ns(u64 cycles, 25 struct kvm_guest_timer *gt, 26 struct kvm_vcpu_timer *t) 27 { 28 unsigned long flags; 29 u64 cycles_now, cycles_delta, delta_ns; 30 31 local_irq_save(flags); 32 cycles_now = kvm_riscv_current_cycles(gt); 33 if (cycles_now < cycles) 34 cycles_delta = cycles - cycles_now; 35 else 36 cycles_delta = 0; 37 delta_ns = (cycles_delta * gt->nsec_mult) >> gt->nsec_shift; 38 local_irq_restore(flags); 39 40 return delta_ns; 41 } 42 43 static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h) 44 { 45 u64 delta_ns; 46 struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt); 47 struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer); 48 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 49 50 if (kvm_riscv_current_cycles(gt) < t->next_cycles) { 51 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t); 52 hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns)); 53 return HRTIMER_RESTART; 54 } 55 56 t->next_set = false; 57 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_TIMER); 58 59 return HRTIMER_NORESTART; 60 } 61 62 static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) 63 { 64 if (!t->init_done) 65 return -EINVAL; 66 67 hrtimer_cancel(&t->hrt); 68 69 if (!t->next_set) 70 return -EINVAL; 71 t->next_set = false; 72 73 return 0; 74 } 75 76 static int kvm_riscv_vcpu_update_vstimecmp(struct kvm_vcpu *vcpu, u64 ncycles) 77 { 78 #if defined(CONFIG_32BIT) 79 ncsr_write(CSR_VSTIMECMP, ULONG_MAX); 80 ncsr_write(CSR_VSTIMECMPH, ncycles >> 32); 81 ncsr_write(CSR_VSTIMECMP, (u32)ncycles); 82 #else 83 ncsr_write(CSR_VSTIMECMP, ncycles); 84 #endif 85 return 0; 86 } 87 88 static int kvm_riscv_vcpu_update_hrtimer(struct kvm_vcpu *vcpu, u64 ncycles) 89 { 90 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 91 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 92 u64 delta_ns; 93 94 if (!t->init_done) 95 return -EINVAL; 96 97 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_TIMER); 98 99 delta_ns = kvm_riscv_delta_cycles2ns(ncycles, gt, t); 100 t->next_cycles = ncycles; 101 hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL); 102 t->next_set = true; 103 104 return 0; 105 } 106 107 int kvm_riscv_vcpu_timer_next_event(struct kvm_vcpu *vcpu, u64 ncycles) 108 { 109 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 110 111 return t->timer_next_event(vcpu, ncycles); 112 } 113 114 static enum hrtimer_restart kvm_riscv_vcpu_vstimer_expired(struct hrtimer *h) 115 { 116 u64 delta_ns; 117 struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt); 118 struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer); 119 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 120 121 if (kvm_riscv_current_cycles(gt) < t->next_cycles) { 122 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t); 123 hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns)); 124 return HRTIMER_RESTART; 125 } 126 127 t->next_set = false; 128 kvm_vcpu_kick(vcpu); 129 130 return HRTIMER_NORESTART; 131 } 132 133 bool kvm_riscv_vcpu_timer_pending(struct kvm_vcpu *vcpu) 134 { 135 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 136 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 137 138 if (!kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t) || 139 kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER)) 140 return true; 141 else 142 return false; 143 } 144 145 static void kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu *vcpu) 146 { 147 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 148 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 149 u64 delta_ns; 150 151 if (!t->init_done) 152 return; 153 154 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t); 155 hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL); 156 t->next_set = true; 157 } 158 159 static void kvm_riscv_vcpu_timer_unblocking(struct kvm_vcpu *vcpu) 160 { 161 kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer); 162 } 163 164 int kvm_riscv_vcpu_get_reg_timer(struct kvm_vcpu *vcpu, 165 const struct kvm_one_reg *reg) 166 { 167 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 168 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 169 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr; 170 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 171 KVM_REG_SIZE_MASK | 172 KVM_REG_RISCV_TIMER); 173 u64 reg_val; 174 175 if (KVM_REG_SIZE(reg->id) != sizeof(u64)) 176 return -EINVAL; 177 if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64)) 178 return -ENOENT; 179 180 switch (reg_num) { 181 case KVM_REG_RISCV_TIMER_REG(frequency): 182 reg_val = riscv_timebase; 183 break; 184 case KVM_REG_RISCV_TIMER_REG(time): 185 reg_val = kvm_riscv_current_cycles(gt); 186 break; 187 case KVM_REG_RISCV_TIMER_REG(compare): 188 reg_val = t->next_cycles; 189 break; 190 case KVM_REG_RISCV_TIMER_REG(state): 191 reg_val = (t->next_set) ? KVM_RISCV_TIMER_STATE_ON : 192 KVM_RISCV_TIMER_STATE_OFF; 193 break; 194 default: 195 return -ENOENT; 196 } 197 198 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 199 return -EFAULT; 200 201 return 0; 202 } 203 204 int kvm_riscv_vcpu_set_reg_timer(struct kvm_vcpu *vcpu, 205 const struct kvm_one_reg *reg) 206 { 207 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 208 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 209 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr; 210 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 211 KVM_REG_SIZE_MASK | 212 KVM_REG_RISCV_TIMER); 213 u64 reg_val; 214 int ret = 0; 215 216 if (KVM_REG_SIZE(reg->id) != sizeof(u64)) 217 return -EINVAL; 218 if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64)) 219 return -ENOENT; 220 221 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 222 return -EFAULT; 223 224 switch (reg_num) { 225 case KVM_REG_RISCV_TIMER_REG(frequency): 226 if (reg_val != riscv_timebase) 227 return -EINVAL; 228 break; 229 case KVM_REG_RISCV_TIMER_REG(time): 230 gt->time_delta = reg_val - get_cycles64(); 231 break; 232 case KVM_REG_RISCV_TIMER_REG(compare): 233 t->next_cycles = reg_val; 234 break; 235 case KVM_REG_RISCV_TIMER_REG(state): 236 if (reg_val == KVM_RISCV_TIMER_STATE_ON) 237 ret = kvm_riscv_vcpu_timer_next_event(vcpu, t->next_cycles); 238 else 239 ret = kvm_riscv_vcpu_timer_cancel(t); 240 break; 241 default: 242 ret = -ENOENT; 243 break; 244 } 245 246 return ret; 247 } 248 249 int kvm_riscv_vcpu_timer_init(struct kvm_vcpu *vcpu) 250 { 251 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 252 253 if (t->init_done) 254 return -EINVAL; 255 256 t->init_done = true; 257 t->next_set = false; 258 259 /* Enable sstc for every vcpu if available in hardware */ 260 if (!kvm_riscv_isa_check_host(SSTC)) { 261 t->sstc_enabled = true; 262 hrtimer_setup(&t->hrt, kvm_riscv_vcpu_vstimer_expired, CLOCK_MONOTONIC, 263 HRTIMER_MODE_REL); 264 t->timer_next_event = kvm_riscv_vcpu_update_vstimecmp; 265 } else { 266 t->sstc_enabled = false; 267 hrtimer_setup(&t->hrt, kvm_riscv_vcpu_hrtimer_expired, CLOCK_MONOTONIC, 268 HRTIMER_MODE_REL); 269 t->timer_next_event = kvm_riscv_vcpu_update_hrtimer; 270 } 271 272 return 0; 273 } 274 275 int kvm_riscv_vcpu_timer_deinit(struct kvm_vcpu *vcpu) 276 { 277 int ret; 278 279 ret = kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer); 280 vcpu->arch.timer.init_done = false; 281 282 return ret; 283 } 284 285 int kvm_riscv_vcpu_timer_reset(struct kvm_vcpu *vcpu) 286 { 287 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 288 289 t->next_cycles = -1ULL; 290 return kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer); 291 } 292 293 static void kvm_riscv_vcpu_update_timedelta(struct kvm_vcpu *vcpu) 294 { 295 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; 296 297 #if defined(CONFIG_32BIT) 298 ncsr_write(CSR_HTIMEDELTA, (u32)(gt->time_delta)); 299 ncsr_write(CSR_HTIMEDELTAH, (u32)(gt->time_delta >> 32)); 300 #else 301 ncsr_write(CSR_HTIMEDELTA, gt->time_delta); 302 #endif 303 } 304 305 void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu) 306 { 307 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 308 309 kvm_riscv_vcpu_update_timedelta(vcpu); 310 311 if (!t->sstc_enabled) 312 return; 313 314 #if defined(CONFIG_32BIT) 315 ncsr_write(CSR_VSTIMECMP, ULONG_MAX); 316 ncsr_write(CSR_VSTIMECMPH, (u32)(t->next_cycles >> 32)); 317 ncsr_write(CSR_VSTIMECMP, (u32)(t->next_cycles)); 318 #else 319 ncsr_write(CSR_VSTIMECMP, t->next_cycles); 320 #endif 321 322 /* timer should be enabled for the remaining operations */ 323 if (unlikely(!t->init_done)) 324 return; 325 326 kvm_riscv_vcpu_timer_unblocking(vcpu); 327 } 328 329 void kvm_riscv_vcpu_timer_sync(struct kvm_vcpu *vcpu) 330 { 331 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 332 333 if (!t->sstc_enabled) 334 return; 335 336 #if defined(CONFIG_32BIT) 337 t->next_cycles = ncsr_read(CSR_VSTIMECMP); 338 t->next_cycles |= (u64)ncsr_read(CSR_VSTIMECMPH) << 32; 339 #else 340 t->next_cycles = ncsr_read(CSR_VSTIMECMP); 341 #endif 342 } 343 344 void kvm_riscv_vcpu_timer_save(struct kvm_vcpu *vcpu) 345 { 346 struct kvm_vcpu_timer *t = &vcpu->arch.timer; 347 348 if (!t->sstc_enabled) 349 return; 350 351 /* 352 * The vstimecmp CSRs are saved by kvm_riscv_vcpu_timer_sync() 353 * upon every VM exit so no need to save here. 354 * 355 * If VS-timer expires when no VCPU running on a host CPU then 356 * WFI executed by such host CPU will be effective NOP resulting 357 * in no power savings. This is because as-per RISC-V Privileged 358 * specificaiton: "WFI is also required to resume execution for 359 * locally enabled interrupts pending at any privilege level, 360 * regardless of the global interrupt enable at each privilege 361 * level." 362 * 363 * To address the above issue, vstimecmp CSR must be set to -1UL 364 * over here when VCPU is scheduled-out or exits to user space. 365 */ 366 367 csr_write(CSR_VSTIMECMP, -1UL); 368 #if defined(CONFIG_32BIT) 369 csr_write(CSR_VSTIMECMPH, -1UL); 370 #endif 371 372 /* timer should be enabled for the remaining operations */ 373 if (unlikely(!t->init_done)) 374 return; 375 376 if (kvm_vcpu_is_blocking(vcpu)) 377 kvm_riscv_vcpu_timer_blocking(vcpu); 378 } 379 380 void kvm_riscv_guest_timer_init(struct kvm *kvm) 381 { 382 struct kvm_guest_timer *gt = &kvm->arch.timer; 383 384 riscv_cs_get_mult_shift(>->nsec_mult, >->nsec_shift); 385 gt->time_delta = -get_cycles64(); 386 } 387