1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 5 * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com> 6 * Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/string.h> 11 #include <linux/kvm.h> 12 #include <linux/kvm_host.h> 13 #include <linux/highmem.h> 14 #include <linux/gfp.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/hugetlb.h> 18 #include <linux/list.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/iommu.h> 21 #include <linux/file.h> 22 #include <linux/mm.h> 23 #include <linux/rcupdate_wait.h> 24 25 #include <asm/kvm_ppc.h> 26 #include <asm/kvm_book3s.h> 27 #include <asm/book3s/64/mmu-hash.h> 28 #include <asm/hvcall.h> 29 #include <asm/synch.h> 30 #include <asm/ppc-opcode.h> 31 #include <asm/udbg.h> 32 #include <asm/iommu.h> 33 #include <asm/tce.h> 34 #include <asm/mmu_context.h> 35 36 static struct kvmppc_spapr_tce_table *kvmppc_find_table(struct kvm *kvm, 37 unsigned long liobn) 38 { 39 struct kvmppc_spapr_tce_table *stt; 40 41 list_for_each_entry_lockless(stt, &kvm->arch.spapr_tce_tables, list) 42 if (stt->liobn == liobn) 43 return stt; 44 45 return NULL; 46 } 47 48 static unsigned long kvmppc_tce_pages(unsigned long iommu_pages) 49 { 50 return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE; 51 } 52 53 static unsigned long kvmppc_stt_pages(unsigned long tce_pages) 54 { 55 unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) + 56 (tce_pages * sizeof(struct page *)); 57 58 return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE; 59 } 60 61 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head) 62 { 63 struct kvmppc_spapr_tce_iommu_table *stit = container_of(head, 64 struct kvmppc_spapr_tce_iommu_table, rcu); 65 66 iommu_tce_table_put(stit->tbl); 67 68 kfree(stit); 69 } 70 71 static void kvm_spapr_tce_liobn_put(struct kref *kref) 72 { 73 struct kvmppc_spapr_tce_iommu_table *stit = container_of(kref, 74 struct kvmppc_spapr_tce_iommu_table, kref); 75 76 list_del_rcu(&stit->next); 77 78 call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free); 79 } 80 81 void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, 82 struct iommu_group *grp) 83 { 84 int i; 85 struct kvmppc_spapr_tce_table *stt; 86 struct kvmppc_spapr_tce_iommu_table *stit, *tmp; 87 struct iommu_table_group *table_group = NULL; 88 89 rcu_read_lock(); 90 list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { 91 92 table_group = iommu_group_get_iommudata(grp); 93 if (WARN_ON(!table_group)) 94 continue; 95 96 list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) { 97 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 98 if (table_group->tables[i] != stit->tbl) 99 continue; 100 101 kref_put(&stit->kref, kvm_spapr_tce_liobn_put); 102 } 103 } 104 cond_resched_rcu(); 105 } 106 rcu_read_unlock(); 107 } 108 109 long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, 110 struct iommu_group *grp) 111 { 112 struct kvmppc_spapr_tce_table *stt = NULL; 113 bool found = false; 114 struct iommu_table *tbl = NULL; 115 struct iommu_table_group *table_group; 116 long i; 117 struct kvmppc_spapr_tce_iommu_table *stit; 118 struct fd f; 119 120 f = fdget(tablefd); 121 if (!fd_file(f)) 122 return -EBADF; 123 124 rcu_read_lock(); 125 list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { 126 if (stt == fd_file(f)->private_data) { 127 found = true; 128 break; 129 } 130 } 131 rcu_read_unlock(); 132 133 if (!found) { 134 fdput(f); 135 return -EINVAL; 136 } 137 138 table_group = iommu_group_get_iommudata(grp); 139 if (WARN_ON(!table_group)) { 140 fdput(f); 141 return -EFAULT; 142 } 143 144 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 145 struct iommu_table *tbltmp = table_group->tables[i]; 146 147 if (!tbltmp) 148 continue; 149 /* Make sure hardware table parameters are compatible */ 150 if ((tbltmp->it_page_shift <= stt->page_shift) && 151 (tbltmp->it_offset << tbltmp->it_page_shift == 152 stt->offset << stt->page_shift) && 153 (tbltmp->it_size << tbltmp->it_page_shift >= 154 stt->size << stt->page_shift)) { 155 /* 156 * Reference the table to avoid races with 157 * add/remove DMA windows. 158 */ 159 tbl = iommu_tce_table_get(tbltmp); 160 break; 161 } 162 } 163 if (!tbl) { 164 fdput(f); 165 return -EINVAL; 166 } 167 168 rcu_read_lock(); 169 list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { 170 if (tbl != stit->tbl) 171 continue; 172 173 if (!kref_get_unless_zero(&stit->kref)) { 174 /* stit is being destroyed */ 175 iommu_tce_table_put(tbl); 176 rcu_read_unlock(); 177 fdput(f); 178 return -ENOTTY; 179 } 180 /* 181 * The table is already known to this KVM, we just increased 182 * its KVM reference counter and can return. 183 */ 184 rcu_read_unlock(); 185 fdput(f); 186 return 0; 187 } 188 rcu_read_unlock(); 189 190 stit = kzalloc(sizeof(*stit), GFP_KERNEL); 191 if (!stit) { 192 iommu_tce_table_put(tbl); 193 fdput(f); 194 return -ENOMEM; 195 } 196 197 stit->tbl = tbl; 198 kref_init(&stit->kref); 199 200 list_add_rcu(&stit->next, &stt->iommu_tables); 201 202 fdput(f); 203 return 0; 204 } 205 206 static void release_spapr_tce_table(struct rcu_head *head) 207 { 208 struct kvmppc_spapr_tce_table *stt = container_of(head, 209 struct kvmppc_spapr_tce_table, rcu); 210 unsigned long i, npages = kvmppc_tce_pages(stt->size); 211 212 for (i = 0; i < npages; i++) 213 if (stt->pages[i]) 214 __free_page(stt->pages[i]); 215 216 kfree(stt); 217 } 218 219 static struct page *kvm_spapr_get_tce_page(struct kvmppc_spapr_tce_table *stt, 220 unsigned long sttpage) 221 { 222 struct page *page = stt->pages[sttpage]; 223 224 if (page) 225 return page; 226 227 mutex_lock(&stt->alloc_lock); 228 page = stt->pages[sttpage]; 229 if (!page) { 230 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 231 WARN_ON_ONCE(!page); 232 if (page) 233 stt->pages[sttpage] = page; 234 } 235 mutex_unlock(&stt->alloc_lock); 236 237 return page; 238 } 239 240 static vm_fault_t kvm_spapr_tce_fault(struct vm_fault *vmf) 241 { 242 struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data; 243 struct page *page; 244 245 if (vmf->pgoff >= kvmppc_tce_pages(stt->size)) 246 return VM_FAULT_SIGBUS; 247 248 page = kvm_spapr_get_tce_page(stt, vmf->pgoff); 249 if (!page) 250 return VM_FAULT_OOM; 251 252 get_page(page); 253 vmf->page = page; 254 return 0; 255 } 256 257 static const struct vm_operations_struct kvm_spapr_tce_vm_ops = { 258 .fault = kvm_spapr_tce_fault, 259 }; 260 261 static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma) 262 { 263 vma->vm_ops = &kvm_spapr_tce_vm_ops; 264 return 0; 265 } 266 267 static int kvm_spapr_tce_release(struct inode *inode, struct file *filp) 268 { 269 struct kvmppc_spapr_tce_table *stt = filp->private_data; 270 struct kvmppc_spapr_tce_iommu_table *stit, *tmp; 271 struct kvm *kvm = stt->kvm; 272 273 mutex_lock(&kvm->lock); 274 list_del_rcu(&stt->list); 275 mutex_unlock(&kvm->lock); 276 277 list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) { 278 WARN_ON(!kref_read(&stit->kref)); 279 while (1) { 280 if (kref_put(&stit->kref, kvm_spapr_tce_liobn_put)) 281 break; 282 } 283 } 284 285 account_locked_vm(kvm->mm, 286 kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false); 287 288 kvm_put_kvm(stt->kvm); 289 290 call_rcu(&stt->rcu, release_spapr_tce_table); 291 292 return 0; 293 } 294 295 static const struct file_operations kvm_spapr_tce_fops = { 296 .mmap = kvm_spapr_tce_mmap, 297 .release = kvm_spapr_tce_release, 298 }; 299 300 int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, 301 struct kvm_create_spapr_tce_64 *args) 302 { 303 struct kvmppc_spapr_tce_table *stt = NULL; 304 struct kvmppc_spapr_tce_table *siter; 305 struct mm_struct *mm = kvm->mm; 306 unsigned long npages; 307 int ret; 308 309 if (!args->size || args->page_shift < 12 || args->page_shift > 34 || 310 (args->offset + args->size > (ULLONG_MAX >> args->page_shift))) 311 return -EINVAL; 312 313 npages = kvmppc_tce_pages(args->size); 314 ret = account_locked_vm(mm, kvmppc_stt_pages(npages), true); 315 if (ret) 316 return ret; 317 318 ret = -ENOMEM; 319 stt = kzalloc(struct_size(stt, pages, npages), GFP_KERNEL | __GFP_NOWARN); 320 if (!stt) 321 goto fail_acct; 322 323 stt->liobn = args->liobn; 324 stt->page_shift = args->page_shift; 325 stt->offset = args->offset; 326 stt->size = args->size; 327 stt->kvm = kvm; 328 mutex_init(&stt->alloc_lock); 329 INIT_LIST_HEAD_RCU(&stt->iommu_tables); 330 331 mutex_lock(&kvm->lock); 332 333 /* Check this LIOBN hasn't been previously allocated */ 334 ret = 0; 335 list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) { 336 if (siter->liobn == args->liobn) { 337 ret = -EBUSY; 338 break; 339 } 340 } 341 342 kvm_get_kvm(kvm); 343 if (!ret) 344 ret = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops, 345 stt, O_RDWR | O_CLOEXEC); 346 347 if (ret >= 0) 348 list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables); 349 else 350 kvm_put_kvm_no_destroy(kvm); 351 352 mutex_unlock(&kvm->lock); 353 354 if (ret >= 0) 355 return ret; 356 357 kfree(stt); 358 fail_acct: 359 account_locked_vm(mm, kvmppc_stt_pages(npages), false); 360 return ret; 361 } 362 363 static long kvmppc_tce_to_ua(struct kvm *kvm, unsigned long tce, 364 unsigned long *ua) 365 { 366 unsigned long gfn = tce >> PAGE_SHIFT; 367 struct kvm_memory_slot *memslot; 368 369 memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn); 370 if (!memslot) 371 return -EINVAL; 372 373 *ua = __gfn_to_hva_memslot(memslot, gfn) | 374 (tce & ~(PAGE_MASK | TCE_PCI_READ | TCE_PCI_WRITE)); 375 376 return 0; 377 } 378 379 static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt, 380 unsigned long tce) 381 { 382 unsigned long gpa = tce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 383 enum dma_data_direction dir = iommu_tce_direction(tce); 384 struct kvmppc_spapr_tce_iommu_table *stit; 385 unsigned long ua = 0; 386 387 /* Allow userspace to poison TCE table */ 388 if (dir == DMA_NONE) 389 return H_SUCCESS; 390 391 if (iommu_tce_check_gpa(stt->page_shift, gpa)) 392 return H_TOO_HARD; 393 394 if (kvmppc_tce_to_ua(stt->kvm, tce, &ua)) 395 return H_TOO_HARD; 396 397 rcu_read_lock(); 398 list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { 399 unsigned long hpa = 0; 400 struct mm_iommu_table_group_mem_t *mem; 401 long shift = stit->tbl->it_page_shift; 402 403 mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift); 404 if (!mem || mm_iommu_ua_to_hpa(mem, ua, shift, &hpa)) { 405 rcu_read_unlock(); 406 return H_TOO_HARD; 407 } 408 } 409 rcu_read_unlock(); 410 411 return H_SUCCESS; 412 } 413 414 /* 415 * Handles TCE requests for emulated devices. 416 * Puts guest TCE values to the table and expects user space to convert them. 417 * Cannot fail so kvmppc_tce_validate must be called before it. 418 */ 419 static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt, 420 unsigned long idx, unsigned long tce) 421 { 422 struct page *page; 423 u64 *tbl; 424 unsigned long sttpage; 425 426 idx -= stt->offset; 427 sttpage = idx / TCES_PER_PAGE; 428 page = stt->pages[sttpage]; 429 430 if (!page) { 431 /* We allow any TCE, not just with read|write permissions */ 432 if (!tce) 433 return; 434 435 page = kvm_spapr_get_tce_page(stt, sttpage); 436 if (!page) 437 return; 438 } 439 tbl = page_to_virt(page); 440 441 tbl[idx % TCES_PER_PAGE] = tce; 442 } 443 444 static void kvmppc_clear_tce(struct mm_struct *mm, struct kvmppc_spapr_tce_table *stt, 445 struct iommu_table *tbl, unsigned long entry) 446 { 447 unsigned long i; 448 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); 449 unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift); 450 451 for (i = 0; i < subpages; ++i) { 452 unsigned long hpa = 0; 453 enum dma_data_direction dir = DMA_NONE; 454 455 iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir); 456 } 457 } 458 459 static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm, 460 struct iommu_table *tbl, unsigned long entry) 461 { 462 struct mm_iommu_table_group_mem_t *mem = NULL; 463 const unsigned long pgsize = 1ULL << tbl->it_page_shift; 464 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry); 465 466 if (!pua) 467 return H_SUCCESS; 468 469 mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize); 470 if (!mem) 471 return H_TOO_HARD; 472 473 mm_iommu_mapped_dec(mem); 474 475 *pua = cpu_to_be64(0); 476 477 return H_SUCCESS; 478 } 479 480 static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm, 481 struct iommu_table *tbl, unsigned long entry) 482 { 483 enum dma_data_direction dir = DMA_NONE; 484 unsigned long hpa = 0; 485 long ret; 486 487 if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, 488 &dir))) 489 return H_TOO_HARD; 490 491 if (dir == DMA_NONE) 492 return H_SUCCESS; 493 494 ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); 495 if (ret != H_SUCCESS) 496 iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir); 497 498 return ret; 499 } 500 501 static long kvmppc_tce_iommu_unmap(struct kvm *kvm, 502 struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl, 503 unsigned long entry) 504 { 505 unsigned long i, ret = H_SUCCESS; 506 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); 507 unsigned long io_entry = entry * subpages; 508 509 for (i = 0; i < subpages; ++i) { 510 ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i); 511 if (ret != H_SUCCESS) 512 break; 513 } 514 515 iommu_tce_kill(tbl, io_entry, subpages); 516 517 return ret; 518 } 519 520 static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, 521 unsigned long entry, unsigned long ua, 522 enum dma_data_direction dir) 523 { 524 long ret; 525 unsigned long hpa; 526 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); 527 struct mm_iommu_table_group_mem_t *mem; 528 529 if (!pua) 530 /* it_userspace allocation might be delayed */ 531 return H_TOO_HARD; 532 533 mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift); 534 if (!mem) 535 /* This only handles v2 IOMMU type, v1 is handled via ioctl() */ 536 return H_TOO_HARD; 537 538 if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa))) 539 return H_TOO_HARD; 540 541 if (mm_iommu_mapped_inc(mem)) 542 return H_TOO_HARD; 543 544 ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir); 545 if (WARN_ON_ONCE(ret)) { 546 mm_iommu_mapped_dec(mem); 547 return H_TOO_HARD; 548 } 549 550 if (dir != DMA_NONE) 551 kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); 552 553 *pua = cpu_to_be64(ua); 554 555 return 0; 556 } 557 558 static long kvmppc_tce_iommu_map(struct kvm *kvm, 559 struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl, 560 unsigned long entry, unsigned long ua, 561 enum dma_data_direction dir) 562 { 563 unsigned long i, pgoff, ret = H_SUCCESS; 564 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); 565 unsigned long io_entry = entry * subpages; 566 567 for (i = 0, pgoff = 0; i < subpages; 568 ++i, pgoff += IOMMU_PAGE_SIZE(tbl)) { 569 570 ret = kvmppc_tce_iommu_do_map(kvm, tbl, 571 io_entry + i, ua + pgoff, dir); 572 if (ret != H_SUCCESS) 573 break; 574 } 575 576 iommu_tce_kill(tbl, io_entry, subpages); 577 578 return ret; 579 } 580 581 long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, 582 unsigned long ioba, unsigned long tce) 583 { 584 struct kvmppc_spapr_tce_table *stt; 585 long ret, idx; 586 struct kvmppc_spapr_tce_iommu_table *stit; 587 unsigned long entry, ua = 0; 588 enum dma_data_direction dir; 589 590 /* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */ 591 /* liobn, ioba, tce); */ 592 593 stt = kvmppc_find_table(vcpu->kvm, liobn); 594 if (!stt) 595 return H_TOO_HARD; 596 597 ret = kvmppc_ioba_validate(stt, ioba, 1); 598 if (ret != H_SUCCESS) 599 return ret; 600 601 idx = srcu_read_lock(&vcpu->kvm->srcu); 602 603 ret = kvmppc_tce_validate(stt, tce); 604 if (ret != H_SUCCESS) 605 goto unlock_exit; 606 607 dir = iommu_tce_direction(tce); 608 609 if ((dir != DMA_NONE) && kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) { 610 ret = H_PARAMETER; 611 goto unlock_exit; 612 } 613 614 entry = ioba >> stt->page_shift; 615 616 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { 617 if (dir == DMA_NONE) 618 ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt, 619 stit->tbl, entry); 620 else 621 ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl, 622 entry, ua, dir); 623 624 625 if (ret != H_SUCCESS) { 626 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry); 627 goto unlock_exit; 628 } 629 } 630 631 kvmppc_tce_put(stt, entry, tce); 632 633 unlock_exit: 634 srcu_read_unlock(&vcpu->kvm->srcu, idx); 635 636 return ret; 637 } 638 EXPORT_SYMBOL_GPL(kvmppc_h_put_tce); 639 640 long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu, 641 unsigned long liobn, unsigned long ioba, 642 unsigned long tce_list, unsigned long npages) 643 { 644 struct kvmppc_spapr_tce_table *stt; 645 long i, ret = H_SUCCESS, idx; 646 unsigned long entry, ua = 0; 647 u64 __user *tces; 648 u64 tce; 649 struct kvmppc_spapr_tce_iommu_table *stit; 650 651 stt = kvmppc_find_table(vcpu->kvm, liobn); 652 if (!stt) 653 return H_TOO_HARD; 654 655 entry = ioba >> stt->page_shift; 656 /* 657 * SPAPR spec says that the maximum size of the list is 512 TCEs 658 * so the whole table fits in 4K page 659 */ 660 if (npages > 512) 661 return H_PARAMETER; 662 663 if (tce_list & (SZ_4K - 1)) 664 return H_PARAMETER; 665 666 ret = kvmppc_ioba_validate(stt, ioba, npages); 667 if (ret != H_SUCCESS) 668 return ret; 669 670 idx = srcu_read_lock(&vcpu->kvm->srcu); 671 if (kvmppc_tce_to_ua(vcpu->kvm, tce_list, &ua)) { 672 ret = H_TOO_HARD; 673 goto unlock_exit; 674 } 675 tces = (u64 __user *) ua; 676 677 for (i = 0; i < npages; ++i) { 678 if (get_user(tce, tces + i)) { 679 ret = H_TOO_HARD; 680 goto unlock_exit; 681 } 682 tce = be64_to_cpu(tce); 683 684 ret = kvmppc_tce_validate(stt, tce); 685 if (ret != H_SUCCESS) 686 goto unlock_exit; 687 } 688 689 for (i = 0; i < npages; ++i) { 690 /* 691 * This looks unsafe, because we validate, then regrab 692 * the TCE from userspace which could have been changed by 693 * another thread. 694 * 695 * But it actually is safe, because the relevant checks will be 696 * re-executed in the following code. If userspace tries to 697 * change this dodgily it will result in a messier failure mode 698 * but won't threaten the host. 699 */ 700 if (get_user(tce, tces + i)) { 701 ret = H_TOO_HARD; 702 goto unlock_exit; 703 } 704 tce = be64_to_cpu(tce); 705 706 if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) { 707 ret = H_PARAMETER; 708 goto unlock_exit; 709 } 710 711 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { 712 ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, 713 stit->tbl, entry + i, ua, 714 iommu_tce_direction(tce)); 715 716 if (ret != H_SUCCESS) { 717 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, 718 entry + i); 719 goto unlock_exit; 720 } 721 } 722 723 kvmppc_tce_put(stt, entry + i, tce); 724 } 725 726 unlock_exit: 727 srcu_read_unlock(&vcpu->kvm->srcu, idx); 728 729 return ret; 730 } 731 EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect); 732 733 long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu, 734 unsigned long liobn, unsigned long ioba, 735 unsigned long tce_value, unsigned long npages) 736 { 737 struct kvmppc_spapr_tce_table *stt; 738 long i, ret; 739 struct kvmppc_spapr_tce_iommu_table *stit; 740 741 stt = kvmppc_find_table(vcpu->kvm, liobn); 742 if (!stt) 743 return H_TOO_HARD; 744 745 ret = kvmppc_ioba_validate(stt, ioba, npages); 746 if (ret != H_SUCCESS) 747 return ret; 748 749 /* Check permission bits only to allow userspace poison TCE for debug */ 750 if (tce_value & (TCE_PCI_WRITE | TCE_PCI_READ)) 751 return H_PARAMETER; 752 753 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { 754 unsigned long entry = ioba >> stt->page_shift; 755 756 for (i = 0; i < npages; ++i) { 757 ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt, 758 stit->tbl, entry + i); 759 760 if (ret == H_SUCCESS) 761 continue; 762 763 if (ret == H_TOO_HARD) 764 return ret; 765 766 WARN_ON_ONCE(1); 767 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i); 768 } 769 } 770 771 for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift)) 772 kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value); 773 774 return ret; 775 } 776 EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce); 777 778 long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn, 779 unsigned long ioba) 780 { 781 struct kvmppc_spapr_tce_table *stt; 782 long ret; 783 unsigned long idx; 784 struct page *page; 785 u64 *tbl; 786 787 stt = kvmppc_find_table(vcpu->kvm, liobn); 788 if (!stt) 789 return H_TOO_HARD; 790 791 ret = kvmppc_ioba_validate(stt, ioba, 1); 792 if (ret != H_SUCCESS) 793 return ret; 794 795 idx = (ioba >> stt->page_shift) - stt->offset; 796 page = stt->pages[idx / TCES_PER_PAGE]; 797 if (!page) { 798 kvmppc_set_gpr(vcpu, 4, 0); 799 return H_SUCCESS; 800 } 801 tbl = (u64 *)page_address(page); 802 803 kvmppc_set_gpr(vcpu, 4, tbl[idx % TCES_PER_PAGE]); 804 805 return H_SUCCESS; 806 } 807 EXPORT_SYMBOL_GPL(kvmppc_h_get_tce); 808