1 // SPDX-License-Identifier: GPL-2.0-only 2 /************************************************************************** 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 **************************************************************************/ 6 7 #include <linux/highmem.h> 8 #include <linux/vmalloc.h> 9 10 #include <asm/cpuid/api.h> 11 12 #include "mmu.h" 13 #include "psb_drv.h" 14 #include "psb_reg.h" 15 16 /* 17 * Code for the SGX MMU: 18 */ 19 20 /* 21 * clflush on one processor only: 22 * clflush should apparently flush the cache line on all processors in an 23 * SMP system. 24 */ 25 26 /* 27 * kmap atomic: 28 * The usage of the slots must be completely encapsulated within a spinlock, and 29 * no other functions that may be using the locks for other purposed may be 30 * called from within the locked region. 31 * Since the slots are per processor, this will guarantee that we are the only 32 * user. 33 */ 34 35 /* 36 * TODO: Inserting ptes from an interrupt handler: 37 * This may be desirable for some SGX functionality where the GPU can fault in 38 * needed pages. For that, we need to make an atomic insert_pages function, that 39 * may fail. 40 * If it fails, the caller need to insert the page using a workqueue function, 41 * but on average it should be fast. 42 */ 43 44 static inline uint32_t psb_mmu_pt_index(uint32_t offset) 45 { 46 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 47 } 48 49 static inline uint32_t psb_mmu_pd_index(uint32_t offset) 50 { 51 return offset >> PSB_PDE_SHIFT; 52 } 53 54 static inline void psb_clflush(void *addr) 55 { 56 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 57 } 58 59 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr) 60 { 61 if (!driver->has_clflush) 62 return; 63 64 mb(); 65 psb_clflush(addr); 66 mb(); 67 } 68 69 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force) 70 { 71 struct drm_device *dev = driver->dev; 72 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 73 74 if (atomic_read(&driver->needs_tlbflush) || force) { 75 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL); 76 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 77 78 /* Make sure data cache is turned off before enabling it */ 79 wmb(); 80 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 81 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 82 if (driver->msvdx_mmu_invaldc) 83 atomic_set(driver->msvdx_mmu_invaldc, 1); 84 } 85 atomic_set(&driver->needs_tlbflush, 0); 86 } 87 88 #if 0 89 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 90 { 91 down_write(&driver->sem); 92 psb_mmu_flush_pd_locked(driver, force); 93 up_write(&driver->sem); 94 } 95 #endif 96 97 void psb_mmu_flush(struct psb_mmu_driver *driver) 98 { 99 struct drm_device *dev = driver->dev; 100 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 101 uint32_t val; 102 103 down_write(&driver->sem); 104 val = PSB_RSGX32(PSB_CR_BIF_CTRL); 105 if (atomic_read(&driver->needs_tlbflush)) 106 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL); 107 else 108 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL); 109 110 /* Make sure data cache is turned off and MMU is flushed before 111 restoring bank interface control register */ 112 wmb(); 113 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC), 114 PSB_CR_BIF_CTRL); 115 (void)PSB_RSGX32(PSB_CR_BIF_CTRL); 116 117 atomic_set(&driver->needs_tlbflush, 0); 118 if (driver->msvdx_mmu_invaldc) 119 atomic_set(driver->msvdx_mmu_invaldc, 1); 120 up_write(&driver->sem); 121 } 122 123 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 124 { 125 struct drm_device *dev = pd->driver->dev; 126 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 127 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 : 128 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4; 129 130 down_write(&pd->driver->sem); 131 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset); 132 wmb(); 133 psb_mmu_flush_pd_locked(pd->driver, 1); 134 pd->hw_context = hw_context; 135 up_write(&pd->driver->sem); 136 137 } 138 139 static inline unsigned long psb_pd_addr_end(unsigned long addr, 140 unsigned long end) 141 { 142 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 143 return (addr < end) ? addr : end; 144 } 145 146 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 147 { 148 uint32_t mask = PSB_PTE_VALID; 149 150 if (type & PSB_MMU_CACHED_MEMORY) 151 mask |= PSB_PTE_CACHED; 152 if (type & PSB_MMU_RO_MEMORY) 153 mask |= PSB_PTE_RO; 154 if (type & PSB_MMU_WO_MEMORY) 155 mask |= PSB_PTE_WO; 156 157 return (pfn << PAGE_SHIFT) | mask; 158 } 159 160 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 161 int trap_pagefaults, int invalid_type) 162 { 163 struct psb_mmu_pd *pd = kmalloc_obj(*pd); 164 uint32_t *v; 165 int i; 166 167 if (!pd) 168 return NULL; 169 170 pd->p = alloc_page(GFP_DMA32); 171 if (!pd->p) 172 goto out_err1; 173 pd->dummy_pt = alloc_page(GFP_DMA32); 174 if (!pd->dummy_pt) 175 goto out_err2; 176 pd->dummy_page = alloc_page(GFP_DMA32); 177 if (!pd->dummy_page) 178 goto out_err3; 179 180 if (!trap_pagefaults) { 181 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 182 invalid_type); 183 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 184 invalid_type); 185 } else { 186 pd->invalid_pde = 0; 187 pd->invalid_pte = 0; 188 } 189 190 v = kmap_local_page(pd->dummy_pt); 191 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 192 v[i] = pd->invalid_pte; 193 194 kunmap_local(v); 195 196 v = kmap_local_page(pd->p); 197 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 198 v[i] = pd->invalid_pde; 199 200 kunmap_local(v); 201 202 clear_page(kmap(pd->dummy_page)); 203 kunmap(pd->dummy_page); 204 205 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 206 if (!pd->tables) 207 goto out_err4; 208 209 pd->hw_context = -1; 210 pd->pd_mask = PSB_PTE_VALID; 211 pd->driver = driver; 212 213 return pd; 214 215 out_err4: 216 __free_page(pd->dummy_page); 217 out_err3: 218 __free_page(pd->dummy_pt); 219 out_err2: 220 __free_page(pd->p); 221 out_err1: 222 kfree(pd); 223 return NULL; 224 } 225 226 static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 227 { 228 __free_page(pt->p); 229 kfree(pt); 230 } 231 232 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 233 { 234 struct psb_mmu_driver *driver = pd->driver; 235 struct drm_device *dev = driver->dev; 236 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 237 struct psb_mmu_pt *pt; 238 int i; 239 240 down_write(&driver->sem); 241 if (pd->hw_context != -1) { 242 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4); 243 psb_mmu_flush_pd_locked(driver, 1); 244 } 245 246 /* Should take the spinlock here, but we don't need to do that 247 since we have the semaphore in write mode. */ 248 249 for (i = 0; i < 1024; ++i) { 250 pt = pd->tables[i]; 251 if (pt) 252 psb_mmu_free_pt(pt); 253 } 254 255 vfree(pd->tables); 256 __free_page(pd->dummy_page); 257 __free_page(pd->dummy_pt); 258 __free_page(pd->p); 259 kfree(pd); 260 up_write(&driver->sem); 261 } 262 263 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 264 { 265 struct psb_mmu_pt *pt = kmalloc_obj(*pt); 266 void *v; 267 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 268 uint32_t clflush_count = PAGE_SIZE / clflush_add; 269 spinlock_t *lock = &pd->driver->lock; 270 uint8_t *clf; 271 uint32_t *ptes; 272 int i; 273 274 if (!pt) 275 return NULL; 276 277 pt->p = alloc_page(GFP_DMA32); 278 if (!pt->p) { 279 kfree(pt); 280 return NULL; 281 } 282 283 spin_lock(lock); 284 285 v = kmap_atomic(pt->p); 286 clf = (uint8_t *) v; 287 ptes = (uint32_t *) v; 288 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 289 *ptes++ = pd->invalid_pte; 290 291 if (pd->driver->has_clflush && pd->hw_context != -1) { 292 mb(); 293 for (i = 0; i < clflush_count; ++i) { 294 psb_clflush(clf); 295 clf += clflush_add; 296 } 297 mb(); 298 } 299 kunmap_atomic(v); 300 spin_unlock(lock); 301 302 pt->count = 0; 303 pt->pd = pd; 304 pt->index = 0; 305 306 return pt; 307 } 308 309 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 310 unsigned long addr) 311 { 312 uint32_t index = psb_mmu_pd_index(addr); 313 struct psb_mmu_pt *pt; 314 uint32_t *v; 315 spinlock_t *lock = &pd->driver->lock; 316 317 spin_lock(lock); 318 pt = pd->tables[index]; 319 while (!pt) { 320 spin_unlock(lock); 321 pt = psb_mmu_alloc_pt(pd); 322 if (!pt) 323 return NULL; 324 spin_lock(lock); 325 326 if (pd->tables[index]) { 327 spin_unlock(lock); 328 psb_mmu_free_pt(pt); 329 spin_lock(lock); 330 pt = pd->tables[index]; 331 continue; 332 } 333 334 v = kmap_atomic(pd->p); 335 pd->tables[index] = pt; 336 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 337 pt->index = index; 338 kunmap_atomic((void *) v); 339 340 if (pd->hw_context != -1) { 341 psb_mmu_clflush(pd->driver, (void *)&v[index]); 342 atomic_set(&pd->driver->needs_tlbflush, 1); 343 } 344 } 345 pt->v = kmap_atomic(pt->p); 346 return pt; 347 } 348 349 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 350 unsigned long addr) 351 { 352 uint32_t index = psb_mmu_pd_index(addr); 353 struct psb_mmu_pt *pt; 354 spinlock_t *lock = &pd->driver->lock; 355 356 spin_lock(lock); 357 pt = pd->tables[index]; 358 if (!pt) { 359 spin_unlock(lock); 360 return NULL; 361 } 362 pt->v = kmap_atomic(pt->p); 363 return pt; 364 } 365 366 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 367 { 368 struct psb_mmu_pd *pd = pt->pd; 369 uint32_t *v; 370 371 kunmap_atomic(pt->v); 372 if (pt->count == 0) { 373 v = kmap_atomic(pd->p); 374 v[pt->index] = pd->invalid_pde; 375 pd->tables[pt->index] = NULL; 376 377 if (pd->hw_context != -1) { 378 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]); 379 atomic_set(&pd->driver->needs_tlbflush, 1); 380 } 381 kunmap_atomic(v); 382 spin_unlock(&pd->driver->lock); 383 psb_mmu_free_pt(pt); 384 return; 385 } 386 spin_unlock(&pd->driver->lock); 387 } 388 389 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr, 390 uint32_t pte) 391 { 392 pt->v[psb_mmu_pt_index(addr)] = pte; 393 } 394 395 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 396 unsigned long addr) 397 { 398 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 399 } 400 401 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 402 { 403 struct psb_mmu_pd *pd; 404 405 down_read(&driver->sem); 406 pd = driver->default_pd; 407 up_read(&driver->sem); 408 409 return pd; 410 } 411 412 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 413 { 414 struct drm_device *dev = driver->dev; 415 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 416 417 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL); 418 psb_mmu_free_pagedir(driver->default_pd); 419 kfree(driver); 420 } 421 422 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev, 423 int trap_pagefaults, 424 int invalid_type, 425 atomic_t *msvdx_mmu_invaldc) 426 { 427 struct psb_mmu_driver *driver; 428 struct drm_psb_private *dev_priv = to_drm_psb_private(dev); 429 430 driver = kmalloc_obj(*driver); 431 432 if (!driver) 433 return NULL; 434 435 driver->dev = dev; 436 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 437 invalid_type); 438 if (!driver->default_pd) 439 goto out_err1; 440 441 spin_lock_init(&driver->lock); 442 init_rwsem(&driver->sem); 443 down_write(&driver->sem); 444 atomic_set(&driver->needs_tlbflush, 1); 445 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc; 446 447 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL); 448 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT, 449 PSB_CR_BIF_CTRL); 450 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT, 451 PSB_CR_BIF_CTRL); 452 453 driver->has_clflush = 0; 454 455 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) { 456 uint32_t tfms, misc, cap0, cap4, clflush_size; 457 458 /* 459 * clflush size is determined at kernel setup for x86_64 but not 460 * for i386. We have to do it here. 461 */ 462 463 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 464 clflush_size = ((misc >> 8) & 0xff) * 8; 465 driver->has_clflush = 1; 466 driver->clflush_add = 467 PAGE_SIZE * clflush_size / sizeof(uint32_t); 468 driver->clflush_mask = driver->clflush_add - 1; 469 driver->clflush_mask = ~driver->clflush_mask; 470 } 471 472 up_write(&driver->sem); 473 return driver; 474 475 out_err1: 476 kfree(driver); 477 return NULL; 478 } 479 480 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address, 481 uint32_t num_pages, uint32_t desired_tile_stride, 482 uint32_t hw_tile_stride) 483 { 484 struct psb_mmu_pt *pt; 485 uint32_t rows = 1; 486 uint32_t i; 487 unsigned long addr; 488 unsigned long end; 489 unsigned long next; 490 unsigned long add; 491 unsigned long row_add; 492 unsigned long clflush_add = pd->driver->clflush_add; 493 unsigned long clflush_mask = pd->driver->clflush_mask; 494 495 if (!pd->driver->has_clflush) 496 return; 497 498 if (hw_tile_stride) 499 rows = num_pages / desired_tile_stride; 500 else 501 desired_tile_stride = num_pages; 502 503 add = desired_tile_stride << PAGE_SHIFT; 504 row_add = hw_tile_stride << PAGE_SHIFT; 505 mb(); 506 for (i = 0; i < rows; ++i) { 507 508 addr = address; 509 end = addr + add; 510 511 do { 512 next = psb_pd_addr_end(addr, end); 513 pt = psb_mmu_pt_map_lock(pd, addr); 514 if (!pt) 515 continue; 516 do { 517 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]); 518 } while (addr += clflush_add, 519 (addr & clflush_mask) < next); 520 521 psb_mmu_pt_unmap_unlock(pt); 522 } while (addr = next, next != end); 523 address += row_add; 524 } 525 mb(); 526 } 527 528 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 529 unsigned long address, uint32_t num_pages) 530 { 531 struct psb_mmu_pt *pt; 532 unsigned long addr; 533 unsigned long end; 534 unsigned long next; 535 unsigned long f_address = address; 536 537 down_read(&pd->driver->sem); 538 539 addr = address; 540 end = addr + (num_pages << PAGE_SHIFT); 541 542 do { 543 next = psb_pd_addr_end(addr, end); 544 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 545 if (!pt) 546 goto out; 547 do { 548 psb_mmu_invalidate_pte(pt, addr); 549 --pt->count; 550 } while (addr += PAGE_SIZE, addr < next); 551 psb_mmu_pt_unmap_unlock(pt); 552 553 } while (addr = next, next != end); 554 555 out: 556 if (pd->hw_context != -1) 557 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 558 559 up_read(&pd->driver->sem); 560 561 if (pd->hw_context != -1) 562 psb_mmu_flush(pd->driver); 563 564 return; 565 } 566 567 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 568 uint32_t num_pages, uint32_t desired_tile_stride, 569 uint32_t hw_tile_stride) 570 { 571 struct psb_mmu_pt *pt; 572 uint32_t rows = 1; 573 uint32_t i; 574 unsigned long addr; 575 unsigned long end; 576 unsigned long next; 577 unsigned long add; 578 unsigned long row_add; 579 unsigned long f_address = address; 580 581 if (hw_tile_stride) 582 rows = num_pages / desired_tile_stride; 583 else 584 desired_tile_stride = num_pages; 585 586 add = desired_tile_stride << PAGE_SHIFT; 587 row_add = hw_tile_stride << PAGE_SHIFT; 588 589 down_read(&pd->driver->sem); 590 591 /* Make sure we only need to flush this processor's cache */ 592 593 for (i = 0; i < rows; ++i) { 594 595 addr = address; 596 end = addr + add; 597 598 do { 599 next = psb_pd_addr_end(addr, end); 600 pt = psb_mmu_pt_map_lock(pd, addr); 601 if (!pt) 602 continue; 603 do { 604 psb_mmu_invalidate_pte(pt, addr); 605 --pt->count; 606 607 } while (addr += PAGE_SIZE, addr < next); 608 psb_mmu_pt_unmap_unlock(pt); 609 610 } while (addr = next, next != end); 611 address += row_add; 612 } 613 if (pd->hw_context != -1) 614 psb_mmu_flush_ptes(pd, f_address, num_pages, 615 desired_tile_stride, hw_tile_stride); 616 617 up_read(&pd->driver->sem); 618 619 if (pd->hw_context != -1) 620 psb_mmu_flush(pd->driver); 621 } 622 623 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 624 unsigned long address, uint32_t num_pages, 625 int type) 626 { 627 struct psb_mmu_pt *pt; 628 uint32_t pte; 629 unsigned long addr; 630 unsigned long end; 631 unsigned long next; 632 unsigned long f_address = address; 633 int ret = -ENOMEM; 634 635 down_read(&pd->driver->sem); 636 637 addr = address; 638 end = addr + (num_pages << PAGE_SHIFT); 639 640 do { 641 next = psb_pd_addr_end(addr, end); 642 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 643 if (!pt) { 644 ret = -ENOMEM; 645 goto out; 646 } 647 do { 648 pte = psb_mmu_mask_pte(start_pfn++, type); 649 psb_mmu_set_pte(pt, addr, pte); 650 pt->count++; 651 } while (addr += PAGE_SIZE, addr < next); 652 psb_mmu_pt_unmap_unlock(pt); 653 654 } while (addr = next, next != end); 655 ret = 0; 656 657 out: 658 if (pd->hw_context != -1) 659 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 660 661 up_read(&pd->driver->sem); 662 663 if (pd->hw_context != -1) 664 psb_mmu_flush(pd->driver); 665 666 return ret; 667 } 668 669 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 670 unsigned long address, uint32_t num_pages, 671 uint32_t desired_tile_stride, uint32_t hw_tile_stride, 672 int type) 673 { 674 struct psb_mmu_pt *pt; 675 uint32_t rows = 1; 676 uint32_t i; 677 uint32_t pte; 678 unsigned long addr; 679 unsigned long end; 680 unsigned long next; 681 unsigned long add; 682 unsigned long row_add; 683 unsigned long f_address = address; 684 int ret = -ENOMEM; 685 686 if (hw_tile_stride) { 687 if (num_pages % desired_tile_stride != 0) 688 return -EINVAL; 689 rows = num_pages / desired_tile_stride; 690 } else { 691 desired_tile_stride = num_pages; 692 } 693 694 add = desired_tile_stride << PAGE_SHIFT; 695 row_add = hw_tile_stride << PAGE_SHIFT; 696 697 down_read(&pd->driver->sem); 698 699 for (i = 0; i < rows; ++i) { 700 701 addr = address; 702 end = addr + add; 703 704 do { 705 next = psb_pd_addr_end(addr, end); 706 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 707 if (!pt) 708 goto out; 709 do { 710 pte = psb_mmu_mask_pte(page_to_pfn(*pages++), 711 type); 712 psb_mmu_set_pte(pt, addr, pte); 713 pt->count++; 714 } while (addr += PAGE_SIZE, addr < next); 715 psb_mmu_pt_unmap_unlock(pt); 716 717 } while (addr = next, next != end); 718 719 address += row_add; 720 } 721 722 ret = 0; 723 out: 724 if (pd->hw_context != -1) 725 psb_mmu_flush_ptes(pd, f_address, num_pages, 726 desired_tile_stride, hw_tile_stride); 727 728 up_read(&pd->driver->sem); 729 730 if (pd->hw_context != -1) 731 psb_mmu_flush(pd->driver); 732 733 return ret; 734 } 735