1 // SPDX-License-Identifier: GPL-2.0-only 2 /************************************************************************** 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 **************************************************************************/ 6 7 #include <linux/highmem.h> 8 #include <linux/vmalloc.h> 9 10 #include "mmu.h" 11 #include "psb_drv.h" 12 #include "psb_reg.h" 13 14 /* 15 * Code for the SGX MMU: 16 */ 17 18 /* 19 * clflush on one processor only: 20 * clflush should apparently flush the cache line on all processors in an 21 * SMP system. 22 */ 23 24 /* 25 * kmap atomic: 26 * The usage of the slots must be completely encapsulated within a spinlock, and 27 * no other functions that may be using the locks for other purposed may be 28 * called from within the locked region. 29 * Since the slots are per processor, this will guarantee that we are the only 30 * user. 31 */ 32 33 /* 34 * TODO: Inserting ptes from an interrupt handler: 35 * This may be desirable for some SGX functionality where the GPU can fault in 36 * needed pages. For that, we need to make an atomic insert_pages function, that 37 * may fail. 38 * If it fails, the caller need to insert the page using a workqueue function, 39 * but on average it should be fast. 40 */ 41 42 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 43 { 44 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 45 } 46 47 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 48 { 49 return offset >> PSB_PDE_SHIFT; 50 } 51 52 static inline void psb_clflush(void *addr) 53 { 54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 55 } 56 57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 58 { 59 if (!driver->has_clflush) 60 return; 61 62 mb(); 63 psb_clflush(addr); 64 mb(); 65 } 66 67 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force) 68 { 69 struct drm_device *dev = driver->dev; 70 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 71 72 if (atomic_read(&driver->needs_tlbflush) || force) { 73 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL); 74 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 75 76 /* Make sure data cache is turned off before enabling it */ 77 wmb(); 78 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 79 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 80 if (driver->msvdx_mmu_invaldc) 81 atomic_set(driver->msvdx_mmu_invaldc, 1); 82 } 83 atomic_set(&driver->needs_tlbflush, 0); 84 } 85 86 #if 0 87 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 88 { 89 down_write(&driver->sem); 90 psb_mmu_flush_pd_locked(driver, force); 91 up_write(&driver->sem); 92 } 93 #endif 94 95 void psb_mmu_flush(struct psb_mmu_driver *driver) 96 { 97 struct drm_device *dev = driver->dev; 98 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 99 uint32_t val; 100 101 down_write(&driver->sem); 102 val = PSB_RSGX32(PSB_CR_BIF_CTRL); 103 if (atomic_read(&driver->needs_tlbflush)) 104 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 105 else 106 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL); 107 108 /* Make sure data cache is turned off and MMU is flushed before 109 restoring bank interface control register */ 110 wmb(); 111 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC), 112 PSB_CR_BIF_CTRL); 113 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 114 115 atomic_set(&driver->needs_tlbflush, 0); 116 if (driver->msvdx_mmu_invaldc) 117 atomic_set(driver->msvdx_mmu_invaldc, 1); 118 up_write(&driver->sem); 119 } 120 121 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 122 { 123 struct drm_device *dev = pd->driver->dev; 124 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 125 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 : 126 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4; 127 128 down_write(&pd->driver->sem); 129 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset); 130 wmb(); 131 psb_mmu_flush_pd_locked(pd->driver, 1); 132 pd->hw_context = hw_context; 133 up_write(&pd->driver->sem); 134 135 } 136 137 static inline unsigned long psb_pd_addr_end(unsigned long addr, 138 unsigned long end) 139 { 140 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 141 return (addr < end) ? addr : end; 142 } 143 144 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 145 { 146 uint32_t mask = PSB_PTE_VALID; 147 148 if (type & PSB_MMU_CACHED_MEMORY) 149 mask |= PSB_PTE_CACHED; 150 if (type & PSB_MMU_RO_MEMORY) 151 mask |= PSB_PTE_RO; 152 if (type & PSB_MMU_WO_MEMORY) 153 mask |= PSB_PTE_WO; 154 155 return (pfn << PAGE_SHIFT) | mask; 156 } 157 158 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 159 int trap_pagefaults, int invalid_type) 160 { 161 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 162 uint32_t *v; 163 int i; 164 165 if (!pd) 166 return NULL; 167 168 pd->p = alloc_page(GFP_DMA32); 169 if (!pd->p) 170 goto out_err1; 171 pd->dummy_pt = alloc_page(GFP_DMA32); 172 if (!pd->dummy_pt) 173 goto out_err2; 174 pd->dummy_page = alloc_page(GFP_DMA32); 175 if (!pd->dummy_page) 176 goto out_err3; 177 178 if (!trap_pagefaults) { 179 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 180 invalid_type); 181 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 182 invalid_type); 183 } else { 184 pd->invalid_pde = 0; 185 pd->invalid_pte = 0; 186 } 187 188 v = kmap_local_page(pd->dummy_pt); 189 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 190 v[i] = pd->invalid_pte; 191 192 kunmap_local(v); 193 194 v = kmap_local_page(pd->p); 195 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 196 v[i] = pd->invalid_pde; 197 198 kunmap_local(v); 199 200 clear_page(kmap(pd->dummy_page)); 201 kunmap(pd->dummy_page); 202 203 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 204 if (!pd->tables) 205 goto out_err4; 206 207 pd->hw_context = -1; 208 pd->pd_mask = PSB_PTE_VALID; 209 pd->driver = driver; 210 211 return pd; 212 213 out_err4: 214 __free_page(pd->dummy_page); 215 out_err3: 216 __free_page(pd->dummy_pt); 217 out_err2: 218 __free_page(pd->p); 219 out_err1: 220 kfree(pd); 221 return NULL; 222 } 223 224 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 225 { 226 __free_page(pt->p); 227 kfree(pt); 228 } 229 230 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 231 { 232 struct psb_mmu_driver *driver = pd->driver; 233 struct drm_device *dev = driver->dev; 234 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 235 struct psb_mmu_pt *pt; 236 int i; 237 238 down_write(&driver->sem); 239 if (pd->hw_context != -1) { 240 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4); 241 psb_mmu_flush_pd_locked(driver, 1); 242 } 243 244 /* Should take the spinlock here, but we don't need to do that 245 since we have the semaphore in write mode. */ 246 247 for (i = 0; i < 1024; ++i) { 248 pt = pd->tables[i]; 249 if (pt) 250 psb_mmu_free_pt(pt); 251 } 252 253 vfree(pd->tables); 254 __free_page(pd->dummy_page); 255 __free_page(pd->dummy_pt); 256 __free_page(pd->p); 257 kfree(pd); 258 up_write(&driver->sem); 259 } 260 261 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 262 { 263 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 264 void *v; 265 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 266 uint32_t clflush_count = PAGE_SIZE / clflush_add; 267 spinlock_t *lock = &pd->driver->lock; 268 uint8_t *clf; 269 uint32_t *ptes; 270 int i; 271 272 if (!pt) 273 return NULL; 274 275 pt->p = alloc_page(GFP_DMA32); 276 if (!pt->p) { 277 kfree(pt); 278 return NULL; 279 } 280 281 spin_lock(lock); 282 283 v = kmap_atomic(pt->p); 284 clf = (uint8_t *) v; 285 ptes = (uint32_t *) v; 286 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 287 *ptes++ = pd->invalid_pte; 288 289 if (pd->driver->has_clflush && pd->hw_context != -1) { 290 mb(); 291 for (i = 0; i < clflush_count; ++i) { 292 psb_clflush(clf); 293 clf += clflush_add; 294 } 295 mb(); 296 } 297 kunmap_atomic(v); 298 spin_unlock(lock); 299 300 pt->count = 0; 301 pt->pd = pd; 302 pt->index = 0; 303 304 return pt; 305 } 306 307 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 308 unsigned long addr) 309 { 310 uint32_t index = psb_mmu_pd_index(addr); 311 struct psb_mmu_pt *pt; 312 uint32_t *v; 313 spinlock_t *lock = &pd->driver->lock; 314 315 spin_lock(lock); 316 pt = pd->tables[index]; 317 while (!pt) { 318 spin_unlock(lock); 319 pt = psb_mmu_alloc_pt(pd); 320 if (!pt) 321 return NULL; 322 spin_lock(lock); 323 324 if (pd->tables[index]) { 325 spin_unlock(lock); 326 psb_mmu_free_pt(pt); 327 spin_lock(lock); 328 pt = pd->tables[index]; 329 continue; 330 } 331 332 v = kmap_atomic(pd->p); 333 pd->tables[index] = pt; 334 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 335 pt->index = index; 336 kunmap_atomic((void *) v); 337 338 if (pd->hw_context != -1) { 339 psb_mmu_clflush(pd->driver, (void *)&v[index]); 340 atomic_set(&pd->driver->needs_tlbflush, 1); 341 } 342 } 343 pt->v = kmap_atomic(pt->p); 344 return pt; 345 } 346 347 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 348 unsigned long addr) 349 { 350 uint32_t index = psb_mmu_pd_index(addr); 351 struct psb_mmu_pt *pt; 352 spinlock_t *lock = &pd->driver->lock; 353 354 spin_lock(lock); 355 pt = pd->tables[index]; 356 if (!pt) { 357 spin_unlock(lock); 358 return NULL; 359 } 360 pt->v = kmap_atomic(pt->p); 361 return pt; 362 } 363 364 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 365 { 366 struct psb_mmu_pd *pd = pt->pd; 367 uint32_t *v; 368 369 kunmap_atomic(pt->v); 370 if (pt->count == 0) { 371 v = kmap_atomic(pd->p); 372 v[pt->index] = pd->invalid_pde; 373 pd->tables[pt->index] = NULL; 374 375 if (pd->hw_context != -1) { 376 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]); 377 atomic_set(&pd->driver->needs_tlbflush, 1); 378 } 379 kunmap_atomic(v); 380 spin_unlock(&pd->driver->lock); 381 psb_mmu_free_pt(pt); 382 return; 383 } 384 spin_unlock(&pd->driver->lock); 385 } 386 387 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr, 388 uint32_t pte) 389 { 390 pt->v[psb_mmu_pt_index(addr)] = pte; 391 } 392 393 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 394 unsigned long addr) 395 { 396 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 397 } 398 399 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 400 { 401 struct psb_mmu_pd *pd; 402 403 down_read(&driver->sem); 404 pd = driver->default_pd; 405 up_read(&driver->sem); 406 407 return pd; 408 } 409 410 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 411 { 412 struct drm_device *dev = driver->dev; 413 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 414 415 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL); 416 psb_mmu_free_pagedir(driver->default_pd); 417 kfree(driver); 418 } 419 420 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev, 421 int trap_pagefaults, 422 int invalid_type, 423 atomic_t *msvdx_mmu_invaldc) 424 { 425 struct psb_mmu_driver *driver; 426 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 427 428 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 429 430 if (!driver) 431 return NULL; 432 433 driver->dev = dev; 434 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 435 invalid_type); 436 if (!driver->default_pd) 437 goto out_err1; 438 439 spin_lock_init(&driver->lock); 440 init_rwsem(&driver->sem); 441 down_write(&driver->sem); 442 atomic_set(&driver->needs_tlbflush, 1); 443 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc; 444 445 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL); 446 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT, 447 PSB_CR_BIF_CTRL); 448 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT, 449 PSB_CR_BIF_CTRL); 450 451 driver->has_clflush = 0; 452 453 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) { 454 uint32_t tfms, misc, cap0, cap4, clflush_size; 455 456 /* 457 * clflush size is determined at kernel setup for x86_64 but not 458 * for i386. We have to do it here. 459 */ 460 461 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 462 clflush_size = ((misc >> 8) & 0xff) * 8; 463 driver->has_clflush = 1; 464 driver->clflush_add = 465 PAGE_SIZE * clflush_size / sizeof(uint32_t); 466 driver->clflush_mask = driver->clflush_add - 1; 467 driver->clflush_mask = ~driver->clflush_mask; 468 } 469 470 up_write(&driver->sem); 471 return driver; 472 473 out_err1: 474 kfree(driver); 475 return NULL; 476 } 477 478 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 479 uint32_t num_pages, uint32_t desired_tile_stride, 480 uint32_t hw_tile_stride) 481 { 482 struct psb_mmu_pt *pt; 483 uint32_t rows = 1; 484 uint32_t i; 485 unsigned long addr; 486 unsigned long end; 487 unsigned long next; 488 unsigned long add; 489 unsigned long row_add; 490 unsigned long clflush_add = pd->driver->clflush_add; 491 unsigned long clflush_mask = pd->driver->clflush_mask; 492 493 if (!pd->driver->has_clflush) 494 return; 495 496 if (hw_tile_stride) 497 rows = num_pages / desired_tile_stride; 498 else 499 desired_tile_stride = num_pages; 500 501 add = desired_tile_stride << PAGE_SHIFT; 502 row_add = hw_tile_stride << PAGE_SHIFT; 503 mb(); 504 for (i = 0; i < rows; ++i) { 505 506 addr = address; 507 end = addr + add; 508 509 do { 510 next = psb_pd_addr_end(addr, end); 511 pt = psb_mmu_pt_map_lock(pd, addr); 512 if (!pt) 513 continue; 514 do { 515 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]); 516 } while (addr += clflush_add, 517 (addr & clflush_mask) < next); 518 519 psb_mmu_pt_unmap_unlock(pt); 520 } while (addr = next, next != end); 521 address += row_add; 522 } 523 mb(); 524 } 525 526 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 527 unsigned long address, uint32_t num_pages) 528 { 529 struct psb_mmu_pt *pt; 530 unsigned long addr; 531 unsigned long end; 532 unsigned long next; 533 unsigned long f_address = address; 534 535 down_read(&pd->driver->sem); 536 537 addr = address; 538 end = addr + (num_pages << PAGE_SHIFT); 539 540 do { 541 next = psb_pd_addr_end(addr, end); 542 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 543 if (!pt) 544 goto out; 545 do { 546 psb_mmu_invalidate_pte(pt, addr); 547 --pt->count; 548 } while (addr += PAGE_SIZE, addr < next); 549 psb_mmu_pt_unmap_unlock(pt); 550 551 } while (addr = next, next != end); 552 553 out: 554 if (pd->hw_context != -1) 555 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 556 557 up_read(&pd->driver->sem); 558 559 if (pd->hw_context != -1) 560 psb_mmu_flush(pd->driver); 561 562 return; 563 } 564 565 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 566 uint32_t num_pages, uint32_t desired_tile_stride, 567 uint32_t hw_tile_stride) 568 { 569 struct psb_mmu_pt *pt; 570 uint32_t rows = 1; 571 uint32_t i; 572 unsigned long addr; 573 unsigned long end; 574 unsigned long next; 575 unsigned long add; 576 unsigned long row_add; 577 unsigned long f_address = address; 578 579 if (hw_tile_stride) 580 rows = num_pages / desired_tile_stride; 581 else 582 desired_tile_stride = num_pages; 583 584 add = desired_tile_stride << PAGE_SHIFT; 585 row_add = hw_tile_stride << PAGE_SHIFT; 586 587 down_read(&pd->driver->sem); 588 589 /* Make sure we only need to flush this processor's cache */ 590 591 for (i = 0; i < rows; ++i) { 592 593 addr = address; 594 end = addr + add; 595 596 do { 597 next = psb_pd_addr_end(addr, end); 598 pt = psb_mmu_pt_map_lock(pd, addr); 599 if (!pt) 600 continue; 601 do { 602 psb_mmu_invalidate_pte(pt, addr); 603 --pt->count; 604 605 } while (addr += PAGE_SIZE, addr < next); 606 psb_mmu_pt_unmap_unlock(pt); 607 608 } while (addr = next, next != end); 609 address += row_add; 610 } 611 if (pd->hw_context != -1) 612 psb_mmu_flush_ptes(pd, f_address, num_pages, 613 desired_tile_stride, hw_tile_stride); 614 615 up_read(&pd->driver->sem); 616 617 if (pd->hw_context != -1) 618 psb_mmu_flush(pd->driver); 619 } 620 621 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 622 unsigned long address, uint32_t num_pages, 623 int type) 624 { 625 struct psb_mmu_pt *pt; 626 uint32_t pte; 627 unsigned long addr; 628 unsigned long end; 629 unsigned long next; 630 unsigned long f_address = address; 631 int ret = -ENOMEM; 632 633 down_read(&pd->driver->sem); 634 635 addr = address; 636 end = addr + (num_pages << PAGE_SHIFT); 637 638 do { 639 next = psb_pd_addr_end(addr, end); 640 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 641 if (!pt) { 642 ret = -ENOMEM; 643 goto out; 644 } 645 do { 646 pte = psb_mmu_mask_pte(start_pfn++, type); 647 psb_mmu_set_pte(pt, addr, pte); 648 pt->count++; 649 } while (addr += PAGE_SIZE, addr < next); 650 psb_mmu_pt_unmap_unlock(pt); 651 652 } while (addr = next, next != end); 653 ret = 0; 654 655 out: 656 if (pd->hw_context != -1) 657 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 658 659 up_read(&pd->driver->sem); 660 661 if (pd->hw_context != -1) 662 psb_mmu_flush(pd->driver); 663 664 return ret; 665 } 666 667 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 668 unsigned long address, uint32_t num_pages, 669 uint32_t desired_tile_stride, uint32_t hw_tile_stride, 670 int type) 671 { 672 struct psb_mmu_pt *pt; 673 uint32_t rows = 1; 674 uint32_t i; 675 uint32_t pte; 676 unsigned long addr; 677 unsigned long end; 678 unsigned long next; 679 unsigned long add; 680 unsigned long row_add; 681 unsigned long f_address = address; 682 int ret = -ENOMEM; 683 684 if (hw_tile_stride) { 685 if (num_pages % desired_tile_stride != 0) 686 return -EINVAL; 687 rows = num_pages / desired_tile_stride; 688 } else { 689 desired_tile_stride = num_pages; 690 } 691 692 add = desired_tile_stride << PAGE_SHIFT; 693 row_add = hw_tile_stride << PAGE_SHIFT; 694 695 down_read(&pd->driver->sem); 696 697 for (i = 0; i < rows; ++i) { 698 699 addr = address; 700 end = addr + add; 701 702 do { 703 next = psb_pd_addr_end(addr, end); 704 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 705 if (!pt) 706 goto out; 707 do { 708 pte = psb_mmu_mask_pte(page_to_pfn(*pages++), 709 type); 710 psb_mmu_set_pte(pt, addr, pte); 711 pt->count++; 712 } while (addr += PAGE_SIZE, addr < next); 713 psb_mmu_pt_unmap_unlock(pt); 714 715 } while (addr = next, next != end); 716 717 address += row_add; 718 } 719 720 ret = 0; 721 out: 722 if (pd->hw_context != -1) 723 psb_mmu_flush_ptes(pd, f_address, num_pages, 724 desired_tile_stride, hw_tile_stride); 725 726 up_read(&pd->driver->sem); 727 728 if (pd->hw_context != -1) 729 psb_mmu_flush(pd->driver); 730 731 return ret; 732 } 733 734 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 735 unsigned long *pfn) 736 { 737 int ret; 738 struct psb_mmu_pt *pt; 739 uint32_t tmp; 740 spinlock_t *lock = &pd->driver->lock; 741 742 down_read(&pd->driver->sem); 743 pt = psb_mmu_pt_map_lock(pd, virtual); 744 if (!pt) { 745 uint32_t *v; 746 747 spin_lock(lock); 748 v = kmap_atomic(pd->p); 749 tmp = v[psb_mmu_pd_index(virtual)]; 750 kunmap_atomic(v); 751 spin_unlock(lock); 752 753 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 754 !(pd->invalid_pte & PSB_PTE_VALID)) { 755 ret = -EINVAL; 756 goto out; 757 } 758 ret = 0; 759 *pfn = pd->invalid_pte >> PAGE_SHIFT; 760 goto out; 761 } 762 tmp = pt->v[psb_mmu_pt_index(virtual)]; 763 if (!(tmp & PSB_PTE_VALID)) { 764 ret = -EINVAL; 765 } else { 766 ret = 0; 767 *pfn = tmp >> PAGE_SHIFT; 768 } 769 psb_mmu_pt_unmap_unlock(pt); 770 out: 771 up_read(&pd->driver->sem); 772 return ret; 773 } 774