1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #include <linux/vmalloc.h> 32 #include <linux/sched.h> 33 #include <linux/highmem.h> 34 #include <linux/pagemap.h> 35 #include <linux/file.h> 36 #include <linux/swap.h> 37 #include "ttm/ttm_module.h" 38 #include "ttm/ttm_bo_driver.h" 39 #include "ttm/ttm_placement.h" 40 41 static int ttm_tt_swapin(struct ttm_tt *ttm); 42 43 #if defined(CONFIG_X86) 44 static void ttm_tt_clflush_page(struct page *page) 45 { 46 uint8_t *page_virtual; 47 unsigned int i; 48 49 if (unlikely(page == NULL)) 50 return; 51 52 page_virtual = kmap_atomic(page, KM_USER0); 53 54 for (i = 0; i < PAGE_SIZE; i += boot_cpu_data.x86_clflush_size) 55 clflush(page_virtual + i); 56 57 kunmap_atomic(page_virtual, KM_USER0); 58 } 59 60 static void ttm_tt_cache_flush_clflush(struct page *pages[], 61 unsigned long num_pages) 62 { 63 unsigned long i; 64 65 mb(); 66 for (i = 0; i < num_pages; ++i) 67 ttm_tt_clflush_page(*pages++); 68 mb(); 69 } 70 #elif !defined(__powerpc__) 71 static void ttm_tt_ipi_handler(void *null) 72 { 73 ; 74 } 75 #endif 76 77 void ttm_tt_cache_flush(struct page *pages[], unsigned long num_pages) 78 { 79 80 #if defined(CONFIG_X86) 81 if (cpu_has_clflush) { 82 ttm_tt_cache_flush_clflush(pages, num_pages); 83 return; 84 } 85 #elif defined(__powerpc__) 86 unsigned long i; 87 88 for (i = 0; i < num_pages; ++i) { 89 if (pages[i]) { 90 unsigned long start = (unsigned long)page_address(pages[i]); 91 flush_dcache_range(start, start + PAGE_SIZE); 92 } 93 } 94 #else 95 if (on_each_cpu(ttm_tt_ipi_handler, NULL, 1) != 0) 96 printk(KERN_ERR TTM_PFX 97 "Timed out waiting for drm cache flush.\n"); 98 #endif 99 } 100 101 /** 102 * Allocates storage for pointers to the pages that back the ttm. 103 * 104 * Uses kmalloc if possible. Otherwise falls back to vmalloc. 105 */ 106 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 107 { 108 unsigned long size = ttm->num_pages * sizeof(*ttm->pages); 109 ttm->pages = NULL; 110 111 if (size <= PAGE_SIZE) 112 ttm->pages = kzalloc(size, GFP_KERNEL); 113 114 if (!ttm->pages) { 115 ttm->pages = vmalloc_user(size); 116 if (ttm->pages) 117 ttm->page_flags |= TTM_PAGE_FLAG_VMALLOC; 118 } 119 } 120 121 static void ttm_tt_free_page_directory(struct ttm_tt *ttm) 122 { 123 if (ttm->page_flags & TTM_PAGE_FLAG_VMALLOC) { 124 vfree(ttm->pages); 125 ttm->page_flags &= ~TTM_PAGE_FLAG_VMALLOC; 126 } else { 127 kfree(ttm->pages); 128 } 129 ttm->pages = NULL; 130 } 131 132 static struct page *ttm_tt_alloc_page(unsigned page_flags) 133 { 134 if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) 135 return alloc_page(GFP_HIGHUSER | __GFP_ZERO); 136 137 return alloc_page(GFP_HIGHUSER); 138 } 139 140 static void ttm_tt_free_user_pages(struct ttm_tt *ttm) 141 { 142 int write; 143 int dirty; 144 struct page *page; 145 int i; 146 struct ttm_backend *be = ttm->be; 147 148 BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER)); 149 write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0); 150 dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0); 151 152 if (be) 153 be->func->clear(be); 154 155 for (i = 0; i < ttm->num_pages; ++i) { 156 page = ttm->pages[i]; 157 if (page == NULL) 158 continue; 159 160 if (page == ttm->dummy_read_page) { 161 BUG_ON(write); 162 continue; 163 } 164 165 if (write && dirty && !PageReserved(page)) 166 set_page_dirty_lock(page); 167 168 ttm->pages[i] = NULL; 169 ttm_mem_global_free(ttm->bdev->mem_glob, PAGE_SIZE, false); 170 put_page(page); 171 } 172 ttm->state = tt_unpopulated; 173 ttm->first_himem_page = ttm->num_pages; 174 ttm->last_lomem_page = -1; 175 } 176 177 static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index) 178 { 179 struct page *p; 180 struct ttm_bo_device *bdev = ttm->bdev; 181 struct ttm_mem_global *mem_glob = bdev->mem_glob; 182 int ret; 183 184 while (NULL == (p = ttm->pages[index])) { 185 p = ttm_tt_alloc_page(ttm->page_flags); 186 187 if (!p) 188 return NULL; 189 190 if (PageHighMem(p)) { 191 ret = 192 ttm_mem_global_alloc(mem_glob, PAGE_SIZE, 193 false, false, true); 194 if (unlikely(ret != 0)) 195 goto out_err; 196 ttm->pages[--ttm->first_himem_page] = p; 197 } else { 198 ret = 199 ttm_mem_global_alloc(mem_glob, PAGE_SIZE, 200 false, false, false); 201 if (unlikely(ret != 0)) 202 goto out_err; 203 ttm->pages[++ttm->last_lomem_page] = p; 204 } 205 } 206 return p; 207 out_err: 208 put_page(p); 209 return NULL; 210 } 211 212 struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index) 213 { 214 int ret; 215 216 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 217 ret = ttm_tt_swapin(ttm); 218 if (unlikely(ret != 0)) 219 return NULL; 220 } 221 return __ttm_tt_get_page(ttm, index); 222 } 223 224 int ttm_tt_populate(struct ttm_tt *ttm) 225 { 226 struct page *page; 227 unsigned long i; 228 struct ttm_backend *be; 229 int ret; 230 231 if (ttm->state != tt_unpopulated) 232 return 0; 233 234 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 235 ret = ttm_tt_swapin(ttm); 236 if (unlikely(ret != 0)) 237 return ret; 238 } 239 240 be = ttm->be; 241 242 for (i = 0; i < ttm->num_pages; ++i) { 243 page = __ttm_tt_get_page(ttm, i); 244 if (!page) 245 return -ENOMEM; 246 } 247 248 be->func->populate(be, ttm->num_pages, ttm->pages, 249 ttm->dummy_read_page); 250 ttm->state = tt_unbound; 251 return 0; 252 } 253 254 #ifdef CONFIG_X86 255 static inline int ttm_tt_set_page_caching(struct page *p, 256 enum ttm_caching_state c_state) 257 { 258 if (PageHighMem(p)) 259 return 0; 260 261 switch (c_state) { 262 case tt_cached: 263 return set_pages_wb(p, 1); 264 case tt_wc: 265 return set_memory_wc((unsigned long) page_address(p), 1); 266 default: 267 return set_pages_uc(p, 1); 268 } 269 } 270 #else /* CONFIG_X86 */ 271 static inline int ttm_tt_set_page_caching(struct page *p, 272 enum ttm_caching_state c_state) 273 { 274 return 0; 275 } 276 #endif /* CONFIG_X86 */ 277 278 /* 279 * Change caching policy for the linear kernel map 280 * for range of pages in a ttm. 281 */ 282 283 static int ttm_tt_set_caching(struct ttm_tt *ttm, 284 enum ttm_caching_state c_state) 285 { 286 int i, j; 287 struct page *cur_page; 288 int ret; 289 290 if (ttm->caching_state == c_state) 291 return 0; 292 293 if (c_state != tt_cached) { 294 ret = ttm_tt_populate(ttm); 295 if (unlikely(ret != 0)) 296 return ret; 297 } 298 299 if (ttm->caching_state == tt_cached) 300 ttm_tt_cache_flush(ttm->pages, ttm->num_pages); 301 302 for (i = 0; i < ttm->num_pages; ++i) { 303 cur_page = ttm->pages[i]; 304 if (likely(cur_page != NULL)) { 305 ret = ttm_tt_set_page_caching(cur_page, c_state); 306 if (unlikely(ret != 0)) 307 goto out_err; 308 } 309 } 310 311 ttm->caching_state = c_state; 312 313 return 0; 314 315 out_err: 316 for (j = 0; j < i; ++j) { 317 cur_page = ttm->pages[j]; 318 if (likely(cur_page != NULL)) { 319 (void)ttm_tt_set_page_caching(cur_page, 320 ttm->caching_state); 321 } 322 } 323 324 return ret; 325 } 326 327 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 328 { 329 enum ttm_caching_state state; 330 331 if (placement & TTM_PL_FLAG_WC) 332 state = tt_wc; 333 else if (placement & TTM_PL_FLAG_UNCACHED) 334 state = tt_uncached; 335 else 336 state = tt_cached; 337 338 return ttm_tt_set_caching(ttm, state); 339 } 340 341 static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) 342 { 343 int i; 344 struct page *cur_page; 345 struct ttm_backend *be = ttm->be; 346 347 if (be) 348 be->func->clear(be); 349 (void)ttm_tt_set_caching(ttm, tt_cached); 350 for (i = 0; i < ttm->num_pages; ++i) { 351 cur_page = ttm->pages[i]; 352 ttm->pages[i] = NULL; 353 if (cur_page) { 354 if (page_count(cur_page) != 1) 355 printk(KERN_ERR TTM_PFX 356 "Erroneous page count. " 357 "Leaking pages.\n"); 358 ttm_mem_global_free(ttm->bdev->mem_glob, PAGE_SIZE, 359 PageHighMem(cur_page)); 360 __free_page(cur_page); 361 } 362 } 363 ttm->state = tt_unpopulated; 364 ttm->first_himem_page = ttm->num_pages; 365 ttm->last_lomem_page = -1; 366 } 367 368 void ttm_tt_destroy(struct ttm_tt *ttm) 369 { 370 struct ttm_backend *be; 371 372 if (unlikely(ttm == NULL)) 373 return; 374 375 be = ttm->be; 376 if (likely(be != NULL)) { 377 be->func->destroy(be); 378 ttm->be = NULL; 379 } 380 381 if (likely(ttm->pages != NULL)) { 382 if (ttm->page_flags & TTM_PAGE_FLAG_USER) 383 ttm_tt_free_user_pages(ttm); 384 else 385 ttm_tt_free_alloced_pages(ttm); 386 387 ttm_tt_free_page_directory(ttm); 388 } 389 390 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) && 391 ttm->swap_storage) 392 fput(ttm->swap_storage); 393 394 kfree(ttm); 395 } 396 397 int ttm_tt_set_user(struct ttm_tt *ttm, 398 struct task_struct *tsk, 399 unsigned long start, unsigned long num_pages) 400 { 401 struct mm_struct *mm = tsk->mm; 402 int ret; 403 int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0; 404 struct ttm_mem_global *mem_glob = ttm->bdev->mem_glob; 405 406 BUG_ON(num_pages != ttm->num_pages); 407 BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0); 408 409 /** 410 * Account user pages as lowmem pages for now. 411 */ 412 413 ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE, 414 false, false, false); 415 if (unlikely(ret != 0)) 416 return ret; 417 418 down_read(&mm->mmap_sem); 419 ret = get_user_pages(tsk, mm, start, num_pages, 420 write, 0, ttm->pages, NULL); 421 up_read(&mm->mmap_sem); 422 423 if (ret != num_pages && write) { 424 ttm_tt_free_user_pages(ttm); 425 ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE, false); 426 return -ENOMEM; 427 } 428 429 ttm->tsk = tsk; 430 ttm->start = start; 431 ttm->state = tt_unbound; 432 433 return 0; 434 } 435 436 struct ttm_tt *ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size, 437 uint32_t page_flags, struct page *dummy_read_page) 438 { 439 struct ttm_bo_driver *bo_driver = bdev->driver; 440 struct ttm_tt *ttm; 441 442 if (!bo_driver) 443 return NULL; 444 445 ttm = kzalloc(sizeof(*ttm), GFP_KERNEL); 446 if (!ttm) 447 return NULL; 448 449 ttm->bdev = bdev; 450 451 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 452 ttm->first_himem_page = ttm->num_pages; 453 ttm->last_lomem_page = -1; 454 ttm->caching_state = tt_cached; 455 ttm->page_flags = page_flags; 456 457 ttm->dummy_read_page = dummy_read_page; 458 459 ttm_tt_alloc_page_directory(ttm); 460 if (!ttm->pages) { 461 ttm_tt_destroy(ttm); 462 printk(KERN_ERR TTM_PFX "Failed allocating page table\n"); 463 return NULL; 464 } 465 ttm->be = bo_driver->create_ttm_backend_entry(bdev); 466 if (!ttm->be) { 467 ttm_tt_destroy(ttm); 468 printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n"); 469 return NULL; 470 } 471 ttm->state = tt_unpopulated; 472 return ttm; 473 } 474 475 void ttm_tt_unbind(struct ttm_tt *ttm) 476 { 477 int ret; 478 struct ttm_backend *be = ttm->be; 479 480 if (ttm->state == tt_bound) { 481 ret = be->func->unbind(be); 482 BUG_ON(ret); 483 ttm->state = tt_unbound; 484 } 485 } 486 487 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 488 { 489 int ret = 0; 490 struct ttm_backend *be; 491 492 if (!ttm) 493 return -EINVAL; 494 495 if (ttm->state == tt_bound) 496 return 0; 497 498 be = ttm->be; 499 500 ret = ttm_tt_populate(ttm); 501 if (ret) 502 return ret; 503 504 ret = be->func->bind(be, bo_mem); 505 if (ret) { 506 printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n"); 507 return ret; 508 } 509 510 ttm->state = tt_bound; 511 512 if (ttm->page_flags & TTM_PAGE_FLAG_USER) 513 ttm->page_flags |= TTM_PAGE_FLAG_USER_DIRTY; 514 return 0; 515 } 516 EXPORT_SYMBOL(ttm_tt_bind); 517 518 static int ttm_tt_swapin(struct ttm_tt *ttm) 519 { 520 struct address_space *swap_space; 521 struct file *swap_storage; 522 struct page *from_page; 523 struct page *to_page; 524 void *from_virtual; 525 void *to_virtual; 526 int i; 527 int ret; 528 529 if (ttm->page_flags & TTM_PAGE_FLAG_USER) { 530 ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start, 531 ttm->num_pages); 532 if (unlikely(ret != 0)) 533 return ret; 534 535 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 536 return 0; 537 } 538 539 swap_storage = ttm->swap_storage; 540 BUG_ON(swap_storage == NULL); 541 542 swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; 543 544 for (i = 0; i < ttm->num_pages; ++i) { 545 from_page = read_mapping_page(swap_space, i, NULL); 546 if (IS_ERR(from_page)) 547 goto out_err; 548 to_page = __ttm_tt_get_page(ttm, i); 549 if (unlikely(to_page == NULL)) 550 goto out_err; 551 552 preempt_disable(); 553 from_virtual = kmap_atomic(from_page, KM_USER0); 554 to_virtual = kmap_atomic(to_page, KM_USER1); 555 memcpy(to_virtual, from_virtual, PAGE_SIZE); 556 kunmap_atomic(to_virtual, KM_USER1); 557 kunmap_atomic(from_virtual, KM_USER0); 558 preempt_enable(); 559 page_cache_release(from_page); 560 } 561 562 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP)) 563 fput(swap_storage); 564 ttm->swap_storage = NULL; 565 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 566 567 return 0; 568 out_err: 569 ttm_tt_free_alloced_pages(ttm); 570 return -ENOMEM; 571 } 572 573 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage) 574 { 575 struct address_space *swap_space; 576 struct file *swap_storage; 577 struct page *from_page; 578 struct page *to_page; 579 void *from_virtual; 580 void *to_virtual; 581 int i; 582 583 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); 584 BUG_ON(ttm->caching_state != tt_cached); 585 586 /* 587 * For user buffers, just unpin the pages, as there should be 588 * vma references. 589 */ 590 591 if (ttm->page_flags & TTM_PAGE_FLAG_USER) { 592 ttm_tt_free_user_pages(ttm); 593 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 594 ttm->swap_storage = NULL; 595 return 0; 596 } 597 598 if (!persistant_swap_storage) { 599 swap_storage = shmem_file_setup("ttm swap", 600 ttm->num_pages << PAGE_SHIFT, 601 0); 602 if (unlikely(IS_ERR(swap_storage))) { 603 printk(KERN_ERR "Failed allocating swap storage.\n"); 604 return -ENOMEM; 605 } 606 } else 607 swap_storage = persistant_swap_storage; 608 609 swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; 610 611 for (i = 0; i < ttm->num_pages; ++i) { 612 from_page = ttm->pages[i]; 613 if (unlikely(from_page == NULL)) 614 continue; 615 to_page = read_mapping_page(swap_space, i, NULL); 616 if (unlikely(to_page == NULL)) 617 goto out_err; 618 619 preempt_disable(); 620 from_virtual = kmap_atomic(from_page, KM_USER0); 621 to_virtual = kmap_atomic(to_page, KM_USER1); 622 memcpy(to_virtual, from_virtual, PAGE_SIZE); 623 kunmap_atomic(to_virtual, KM_USER1); 624 kunmap_atomic(from_virtual, KM_USER0); 625 preempt_enable(); 626 set_page_dirty(to_page); 627 mark_page_accessed(to_page); 628 page_cache_release(to_page); 629 } 630 631 ttm_tt_free_alloced_pages(ttm); 632 ttm->swap_storage = swap_storage; 633 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 634 if (persistant_swap_storage) 635 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP; 636 637 return 0; 638 out_err: 639 if (!persistant_swap_storage) 640 fput(swap_storage); 641 642 return -ENOMEM; 643 } 644