1 /* 2 * KVM paravirt_ops implementation 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 * 18 * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 19 * Copyright IBM Corporation, 2007 20 * Authors: Anthony Liguori <aliguori@us.ibm.com> 21 */ 22 23 #include <linux/module.h> 24 #include <linux/kernel.h> 25 #include <linux/kvm_para.h> 26 #include <linux/cpu.h> 27 #include <linux/mm.h> 28 #include <linux/highmem.h> 29 #include <linux/hardirq.h> 30 #include <linux/notifier.h> 31 #include <linux/reboot.h> 32 #include <linux/hash.h> 33 #include <linux/sched.h> 34 #include <linux/slab.h> 35 #include <linux/kprobes.h> 36 #include <asm/timer.h> 37 #include <asm/cpu.h> 38 #include <asm/traps.h> 39 #include <asm/desc.h> 40 #include <asm/tlbflush.h> 41 42 #define MMU_QUEUE_SIZE 1024 43 44 static int kvmapf = 1; 45 46 static int parse_no_kvmapf(char *arg) 47 { 48 kvmapf = 0; 49 return 0; 50 } 51 52 early_param("no-kvmapf", parse_no_kvmapf); 53 54 static int steal_acc = 1; 55 static int parse_no_stealacc(char *arg) 56 { 57 steal_acc = 0; 58 return 0; 59 } 60 61 early_param("no-steal-acc", parse_no_stealacc); 62 63 struct kvm_para_state { 64 u8 mmu_queue[MMU_QUEUE_SIZE]; 65 int mmu_queue_len; 66 }; 67 68 static DEFINE_PER_CPU(struct kvm_para_state, para_state); 69 static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); 70 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); 71 static int has_steal_clock = 0; 72 73 static struct kvm_para_state *kvm_para_state(void) 74 { 75 return &per_cpu(para_state, raw_smp_processor_id()); 76 } 77 78 /* 79 * No need for any "IO delay" on KVM 80 */ 81 static void kvm_io_delay(void) 82 { 83 } 84 85 #define KVM_TASK_SLEEP_HASHBITS 8 86 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS) 87 88 struct kvm_task_sleep_node { 89 struct hlist_node link; 90 wait_queue_head_t wq; 91 u32 token; 92 int cpu; 93 bool halted; 94 struct mm_struct *mm; 95 }; 96 97 static struct kvm_task_sleep_head { 98 spinlock_t lock; 99 struct hlist_head list; 100 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE]; 101 102 static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, 103 u32 token) 104 { 105 struct hlist_node *p; 106 107 hlist_for_each(p, &b->list) { 108 struct kvm_task_sleep_node *n = 109 hlist_entry(p, typeof(*n), link); 110 if (n->token == token) 111 return n; 112 } 113 114 return NULL; 115 } 116 117 void kvm_async_pf_task_wait(u32 token) 118 { 119 u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); 120 struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; 121 struct kvm_task_sleep_node n, *e; 122 DEFINE_WAIT(wait); 123 int cpu, idle; 124 125 cpu = get_cpu(); 126 idle = idle_cpu(cpu); 127 put_cpu(); 128 129 spin_lock(&b->lock); 130 e = _find_apf_task(b, token); 131 if (e) { 132 /* dummy entry exist -> wake up was delivered ahead of PF */ 133 hlist_del(&e->link); 134 kfree(e); 135 spin_unlock(&b->lock); 136 return; 137 } 138 139 n.token = token; 140 n.cpu = smp_processor_id(); 141 n.mm = current->active_mm; 142 n.halted = idle || preempt_count() > 1; 143 atomic_inc(&n.mm->mm_count); 144 init_waitqueue_head(&n.wq); 145 hlist_add_head(&n.link, &b->list); 146 spin_unlock(&b->lock); 147 148 for (;;) { 149 if (!n.halted) 150 prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE); 151 if (hlist_unhashed(&n.link)) 152 break; 153 154 if (!n.halted) { 155 local_irq_enable(); 156 schedule(); 157 local_irq_disable(); 158 } else { 159 /* 160 * We cannot reschedule. So halt. 161 */ 162 native_safe_halt(); 163 local_irq_disable(); 164 } 165 } 166 if (!n.halted) 167 finish_wait(&n.wq, &wait); 168 169 return; 170 } 171 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait); 172 173 static void apf_task_wake_one(struct kvm_task_sleep_node *n) 174 { 175 hlist_del_init(&n->link); 176 if (!n->mm) 177 return; 178 mmdrop(n->mm); 179 if (n->halted) 180 smp_send_reschedule(n->cpu); 181 else if (waitqueue_active(&n->wq)) 182 wake_up(&n->wq); 183 } 184 185 static void apf_task_wake_all(void) 186 { 187 int i; 188 189 for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { 190 struct hlist_node *p, *next; 191 struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; 192 spin_lock(&b->lock); 193 hlist_for_each_safe(p, next, &b->list) { 194 struct kvm_task_sleep_node *n = 195 hlist_entry(p, typeof(*n), link); 196 if (n->cpu == smp_processor_id()) 197 apf_task_wake_one(n); 198 } 199 spin_unlock(&b->lock); 200 } 201 } 202 203 void kvm_async_pf_task_wake(u32 token) 204 { 205 u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); 206 struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; 207 struct kvm_task_sleep_node *n; 208 209 if (token == ~0) { 210 apf_task_wake_all(); 211 return; 212 } 213 214 again: 215 spin_lock(&b->lock); 216 n = _find_apf_task(b, token); 217 if (!n) { 218 /* 219 * async PF was not yet handled. 220 * Add dummy entry for the token. 221 */ 222 n = kmalloc(sizeof(*n), GFP_ATOMIC); 223 if (!n) { 224 /* 225 * Allocation failed! Busy wait while other cpu 226 * handles async PF. 227 */ 228 spin_unlock(&b->lock); 229 cpu_relax(); 230 goto again; 231 } 232 n->token = token; 233 n->cpu = smp_processor_id(); 234 n->mm = NULL; 235 init_waitqueue_head(&n->wq); 236 hlist_add_head(&n->link, &b->list); 237 } else 238 apf_task_wake_one(n); 239 spin_unlock(&b->lock); 240 return; 241 } 242 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); 243 244 u32 kvm_read_and_reset_pf_reason(void) 245 { 246 u32 reason = 0; 247 248 if (__get_cpu_var(apf_reason).enabled) { 249 reason = __get_cpu_var(apf_reason).reason; 250 __get_cpu_var(apf_reason).reason = 0; 251 } 252 253 return reason; 254 } 255 EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); 256 257 dotraplinkage void __kprobes 258 do_async_page_fault(struct pt_regs *regs, unsigned long error_code) 259 { 260 switch (kvm_read_and_reset_pf_reason()) { 261 default: 262 do_page_fault(regs, error_code); 263 break; 264 case KVM_PV_REASON_PAGE_NOT_PRESENT: 265 /* page is swapped out by the host. */ 266 kvm_async_pf_task_wait((u32)read_cr2()); 267 break; 268 case KVM_PV_REASON_PAGE_READY: 269 kvm_async_pf_task_wake((u32)read_cr2()); 270 break; 271 } 272 } 273 274 static void kvm_mmu_op(void *buffer, unsigned len) 275 { 276 int r; 277 unsigned long a1, a2; 278 279 do { 280 a1 = __pa(buffer); 281 a2 = 0; /* on i386 __pa() always returns <4G */ 282 r = kvm_hypercall3(KVM_HC_MMU_OP, len, a1, a2); 283 buffer += r; 284 len -= r; 285 } while (len); 286 } 287 288 static void mmu_queue_flush(struct kvm_para_state *state) 289 { 290 if (state->mmu_queue_len) { 291 kvm_mmu_op(state->mmu_queue, state->mmu_queue_len); 292 state->mmu_queue_len = 0; 293 } 294 } 295 296 static void kvm_deferred_mmu_op(void *buffer, int len) 297 { 298 struct kvm_para_state *state = kvm_para_state(); 299 300 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU) { 301 kvm_mmu_op(buffer, len); 302 return; 303 } 304 if (state->mmu_queue_len + len > sizeof state->mmu_queue) 305 mmu_queue_flush(state); 306 memcpy(state->mmu_queue + state->mmu_queue_len, buffer, len); 307 state->mmu_queue_len += len; 308 } 309 310 static void kvm_mmu_write(void *dest, u64 val) 311 { 312 __u64 pte_phys; 313 struct kvm_mmu_op_write_pte wpte; 314 315 #ifdef CONFIG_HIGHPTE 316 struct page *page; 317 unsigned long dst = (unsigned long) dest; 318 319 page = kmap_atomic_to_page(dest); 320 pte_phys = page_to_pfn(page); 321 pte_phys <<= PAGE_SHIFT; 322 pte_phys += (dst & ~(PAGE_MASK)); 323 #else 324 pte_phys = (unsigned long)__pa(dest); 325 #endif 326 wpte.header.op = KVM_MMU_OP_WRITE_PTE; 327 wpte.pte_val = val; 328 wpte.pte_phys = pte_phys; 329 330 kvm_deferred_mmu_op(&wpte, sizeof wpte); 331 } 332 333 /* 334 * We only need to hook operations that are MMU writes. We hook these so that 335 * we can use lazy MMU mode to batch these operations. We could probably 336 * improve the performance of the host code if we used some of the information 337 * here to simplify processing of batched writes. 338 */ 339 static void kvm_set_pte(pte_t *ptep, pte_t pte) 340 { 341 kvm_mmu_write(ptep, pte_val(pte)); 342 } 343 344 static void kvm_set_pte_at(struct mm_struct *mm, unsigned long addr, 345 pte_t *ptep, pte_t pte) 346 { 347 kvm_mmu_write(ptep, pte_val(pte)); 348 } 349 350 static void kvm_set_pmd(pmd_t *pmdp, pmd_t pmd) 351 { 352 kvm_mmu_write(pmdp, pmd_val(pmd)); 353 } 354 355 #if PAGETABLE_LEVELS >= 3 356 #ifdef CONFIG_X86_PAE 357 static void kvm_set_pte_atomic(pte_t *ptep, pte_t pte) 358 { 359 kvm_mmu_write(ptep, pte_val(pte)); 360 } 361 362 static void kvm_pte_clear(struct mm_struct *mm, 363 unsigned long addr, pte_t *ptep) 364 { 365 kvm_mmu_write(ptep, 0); 366 } 367 368 static void kvm_pmd_clear(pmd_t *pmdp) 369 { 370 kvm_mmu_write(pmdp, 0); 371 } 372 #endif 373 374 static void kvm_set_pud(pud_t *pudp, pud_t pud) 375 { 376 kvm_mmu_write(pudp, pud_val(pud)); 377 } 378 379 #if PAGETABLE_LEVELS == 4 380 static void kvm_set_pgd(pgd_t *pgdp, pgd_t pgd) 381 { 382 kvm_mmu_write(pgdp, pgd_val(pgd)); 383 } 384 #endif 385 #endif /* PAGETABLE_LEVELS >= 3 */ 386 387 static void kvm_flush_tlb(void) 388 { 389 struct kvm_mmu_op_flush_tlb ftlb = { 390 .header.op = KVM_MMU_OP_FLUSH_TLB, 391 }; 392 393 kvm_deferred_mmu_op(&ftlb, sizeof ftlb); 394 } 395 396 static void kvm_release_pt(unsigned long pfn) 397 { 398 struct kvm_mmu_op_release_pt rpt = { 399 .header.op = KVM_MMU_OP_RELEASE_PT, 400 .pt_phys = (u64)pfn << PAGE_SHIFT, 401 }; 402 403 kvm_mmu_op(&rpt, sizeof rpt); 404 } 405 406 static void kvm_enter_lazy_mmu(void) 407 { 408 paravirt_enter_lazy_mmu(); 409 } 410 411 static void kvm_leave_lazy_mmu(void) 412 { 413 struct kvm_para_state *state = kvm_para_state(); 414 415 mmu_queue_flush(state); 416 paravirt_leave_lazy_mmu(); 417 } 418 419 static void __init paravirt_ops_setup(void) 420 { 421 pv_info.name = "KVM"; 422 pv_info.paravirt_enabled = 1; 423 424 if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY)) 425 pv_cpu_ops.io_delay = kvm_io_delay; 426 427 if (kvm_para_has_feature(KVM_FEATURE_MMU_OP)) { 428 pv_mmu_ops.set_pte = kvm_set_pte; 429 pv_mmu_ops.set_pte_at = kvm_set_pte_at; 430 pv_mmu_ops.set_pmd = kvm_set_pmd; 431 #if PAGETABLE_LEVELS >= 3 432 #ifdef CONFIG_X86_PAE 433 pv_mmu_ops.set_pte_atomic = kvm_set_pte_atomic; 434 pv_mmu_ops.pte_clear = kvm_pte_clear; 435 pv_mmu_ops.pmd_clear = kvm_pmd_clear; 436 #endif 437 pv_mmu_ops.set_pud = kvm_set_pud; 438 #if PAGETABLE_LEVELS == 4 439 pv_mmu_ops.set_pgd = kvm_set_pgd; 440 #endif 441 #endif 442 pv_mmu_ops.flush_tlb_user = kvm_flush_tlb; 443 pv_mmu_ops.release_pte = kvm_release_pt; 444 pv_mmu_ops.release_pmd = kvm_release_pt; 445 pv_mmu_ops.release_pud = kvm_release_pt; 446 447 pv_mmu_ops.lazy_mode.enter = kvm_enter_lazy_mmu; 448 pv_mmu_ops.lazy_mode.leave = kvm_leave_lazy_mmu; 449 } 450 #ifdef CONFIG_X86_IO_APIC 451 no_timer_check = 1; 452 #endif 453 } 454 455 static void kvm_register_steal_time(void) 456 { 457 int cpu = smp_processor_id(); 458 struct kvm_steal_time *st = &per_cpu(steal_time, cpu); 459 460 if (!has_steal_clock) 461 return; 462 463 memset(st, 0, sizeof(*st)); 464 465 wrmsrl(MSR_KVM_STEAL_TIME, (__pa(st) | KVM_MSR_ENABLED)); 466 printk(KERN_INFO "kvm-stealtime: cpu %d, msr %lx\n", 467 cpu, __pa(st)); 468 } 469 470 void __cpuinit kvm_guest_cpu_init(void) 471 { 472 if (!kvm_para_available()) 473 return; 474 475 if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) { 476 u64 pa = __pa(&__get_cpu_var(apf_reason)); 477 478 #ifdef CONFIG_PREEMPT 479 pa |= KVM_ASYNC_PF_SEND_ALWAYS; 480 #endif 481 wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED); 482 __get_cpu_var(apf_reason).enabled = 1; 483 printk(KERN_INFO"KVM setup async PF for cpu %d\n", 484 smp_processor_id()); 485 } 486 487 if (has_steal_clock) 488 kvm_register_steal_time(); 489 } 490 491 static void kvm_pv_disable_apf(void *unused) 492 { 493 if (!__get_cpu_var(apf_reason).enabled) 494 return; 495 496 wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); 497 __get_cpu_var(apf_reason).enabled = 0; 498 499 printk(KERN_INFO"Unregister pv shared memory for cpu %d\n", 500 smp_processor_id()); 501 } 502 503 static int kvm_pv_reboot_notify(struct notifier_block *nb, 504 unsigned long code, void *unused) 505 { 506 if (code == SYS_RESTART) 507 on_each_cpu(kvm_pv_disable_apf, NULL, 1); 508 return NOTIFY_DONE; 509 } 510 511 static struct notifier_block kvm_pv_reboot_nb = { 512 .notifier_call = kvm_pv_reboot_notify, 513 }; 514 515 static u64 kvm_steal_clock(int cpu) 516 { 517 u64 steal; 518 struct kvm_steal_time *src; 519 int version; 520 521 src = &per_cpu(steal_time, cpu); 522 do { 523 version = src->version; 524 rmb(); 525 steal = src->steal; 526 rmb(); 527 } while ((version & 1) || (version != src->version)); 528 529 return steal; 530 } 531 532 void kvm_disable_steal_time(void) 533 { 534 if (!has_steal_clock) 535 return; 536 537 wrmsr(MSR_KVM_STEAL_TIME, 0, 0); 538 } 539 540 #ifdef CONFIG_SMP 541 static void __init kvm_smp_prepare_boot_cpu(void) 542 { 543 #ifdef CONFIG_KVM_CLOCK 544 WARN_ON(kvm_register_clock("primary cpu clock")); 545 #endif 546 kvm_guest_cpu_init(); 547 native_smp_prepare_boot_cpu(); 548 } 549 550 static void __cpuinit kvm_guest_cpu_online(void *dummy) 551 { 552 kvm_guest_cpu_init(); 553 } 554 555 static void kvm_guest_cpu_offline(void *dummy) 556 { 557 kvm_disable_steal_time(); 558 kvm_pv_disable_apf(NULL); 559 apf_task_wake_all(); 560 } 561 562 static int __cpuinit kvm_cpu_notify(struct notifier_block *self, 563 unsigned long action, void *hcpu) 564 { 565 int cpu = (unsigned long)hcpu; 566 switch (action) { 567 case CPU_ONLINE: 568 case CPU_DOWN_FAILED: 569 case CPU_ONLINE_FROZEN: 570 smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0); 571 break; 572 case CPU_DOWN_PREPARE: 573 case CPU_DOWN_PREPARE_FROZEN: 574 smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1); 575 break; 576 default: 577 break; 578 } 579 return NOTIFY_OK; 580 } 581 582 static struct notifier_block __cpuinitdata kvm_cpu_notifier = { 583 .notifier_call = kvm_cpu_notify, 584 }; 585 #endif 586 587 static void __init kvm_apf_trap_init(void) 588 { 589 set_intr_gate(14, &async_page_fault); 590 } 591 592 void __init kvm_guest_init(void) 593 { 594 int i; 595 596 if (!kvm_para_available()) 597 return; 598 599 paravirt_ops_setup(); 600 register_reboot_notifier(&kvm_pv_reboot_nb); 601 for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) 602 spin_lock_init(&async_pf_sleepers[i].lock); 603 if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF)) 604 x86_init.irqs.trap_init = kvm_apf_trap_init; 605 606 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { 607 has_steal_clock = 1; 608 pv_time_ops.steal_clock = kvm_steal_clock; 609 } 610 611 #ifdef CONFIG_SMP 612 smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; 613 register_cpu_notifier(&kvm_cpu_notifier); 614 #else 615 kvm_guest_cpu_init(); 616 #endif 617 } 618 619 static __init int activate_jump_labels(void) 620 { 621 if (has_steal_clock) { 622 jump_label_inc(¶virt_steal_enabled); 623 if (steal_acc) 624 jump_label_inc(¶virt_steal_rq_enabled); 625 } 626 627 return 0; 628 } 629 arch_initcall(activate_jump_labels); 630