1 /* BEGIN CSTYLED */ 2 3 /* 4 * Copyright (c) 2009, Intel Corporation. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 * IN THE SOFTWARE. 25 * 26 * Authors: 27 * Eric Anholt <eric@anholt.net> 28 * 29 */ 30 31 /* 32 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 33 * Use is subject to license terms. 34 */ 35 36 #include <sys/x86_archext.h> 37 #include <sys/vfs_opreg.h> 38 #include "drmP.h" 39 #include "drm.h" 40 #include "i915_drm.h" 41 #include "i915_drv.h" 42 43 #ifndef roundup 44 #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) 45 #endif /* !roundup */ 46 47 #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 48 49 static timeout_id_t worktimer_id = NULL; 50 51 extern int drm_mm_init(struct drm_mm *mm, 52 unsigned long start, unsigned long size); 53 extern void drm_mm_put_block(struct drm_mm_node *cur); 54 extern int choose_addr(struct as *as, caddr_t *addrp, size_t len, offset_t off, 55 int vacalign, uint_t flags); 56 57 static void 58 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj, 59 uint32_t read_domains, 60 uint32_t write_domain); 61 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 62 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 63 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 64 static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, 65 int write); 66 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 67 int write); 68 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 69 uint64_t offset, 70 uint64_t size); 71 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 72 static void i915_gem_object_free_page_list(struct drm_gem_object *obj); 73 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); 74 static int i915_gem_object_get_page_list(struct drm_gem_object *obj); 75 76 static void 77 i915_gem_cleanup_ringbuffer(struct drm_device *dev); 78 79 /*ARGSUSED*/ 80 int 81 i915_gem_init_ioctl(DRM_IOCTL_ARGS) 82 { 83 DRM_DEVICE; 84 drm_i915_private_t *dev_priv = dev->dev_private; 85 struct drm_i915_gem_init args; 86 87 if (dev->driver->use_gem != 1) 88 return ENODEV; 89 90 DRM_COPYFROM_WITH_RETURN(&args, 91 (struct drm_i915_gem_init *) data, sizeof(args)); 92 93 spin_lock(&dev->struct_mutex); 94 95 if ((args.gtt_start >= args.gtt_end) || 96 ((args.gtt_start & (PAGE_SIZE - 1)) != 0) || 97 ((args.gtt_end & (PAGE_SIZE - 1)) != 0)) { 98 spin_unlock(&dev->struct_mutex); 99 DRM_ERROR("i915_gem_init_ioctel invalid arg 0x%lx args.start 0x%lx end 0x%lx", &args, args.gtt_start, args.gtt_end); 100 return EINVAL; 101 } 102 103 dev->gtt_total = (uint32_t) (args.gtt_end - args.gtt_start); 104 105 (void) drm_mm_init(&dev_priv->mm.gtt_space, 106 (unsigned long) args.gtt_start, dev->gtt_total); 107 DRM_DEBUG("i915_gem_init_ioctl dev->gtt_total %x, dev_priv->mm.gtt_space 0x%x gtt_start 0x%lx", dev->gtt_total, dev_priv->mm.gtt_space, args.gtt_start); 108 ASSERT(dev->gtt_total != 0); 109 110 spin_unlock(&dev->struct_mutex); 111 112 113 return 0; 114 } 115 116 /*ARGSUSED*/ 117 int 118 i915_gem_get_aperture_ioctl(DRM_IOCTL_ARGS) 119 { 120 DRM_DEVICE; 121 struct drm_i915_gem_get_aperture args; 122 int ret; 123 124 if (dev->driver->use_gem != 1) 125 return ENODEV; 126 127 args.aper_size = (uint64_t)dev->gtt_total; 128 args.aper_available_size = (args.aper_size - 129 atomic_read(&dev->pin_memory)); 130 131 ret = DRM_COPY_TO_USER((struct drm_i915_gem_get_aperture __user *) data, &args, sizeof(args)); 132 133 if ( ret != 0) 134 DRM_ERROR(" i915_gem_get_aperture_ioctl error! %d", ret); 135 136 DRM_DEBUG("i915_gem_get_aaperture_ioctl called sizeof %d, aper_size 0x%x, aper_available_size 0x%x\n", sizeof(args), dev->gtt_total, args.aper_available_size); 137 138 return 0; 139 } 140 141 /** 142 * Creates a new mm object and returns a handle to it. 143 */ 144 /*ARGSUSED*/ 145 int 146 i915_gem_create_ioctl(DRM_IOCTL_ARGS) 147 { 148 DRM_DEVICE; 149 struct drm_i915_gem_create args; 150 struct drm_gem_object *obj; 151 int handlep; 152 int ret; 153 154 if (dev->driver->use_gem != 1) 155 return ENODEV; 156 157 DRM_COPYFROM_WITH_RETURN(&args, 158 (struct drm_i915_gem_create *) data, sizeof(args)); 159 160 161 args.size = (uint64_t) roundup(args.size, PAGE_SIZE); 162 163 if (args.size == 0) { 164 DRM_ERROR("Invalid obj size %d", args.size); 165 return EINVAL; 166 } 167 /* Allocate the new object */ 168 obj = drm_gem_object_alloc(dev, args.size); 169 if (obj == NULL) { 170 DRM_ERROR("Failed to alloc obj"); 171 return ENOMEM; 172 } 173 174 ret = drm_gem_handle_create(fpriv, obj, &handlep); 175 spin_lock(&dev->struct_mutex); 176 drm_gem_object_handle_unreference(obj); 177 spin_unlock(&dev->struct_mutex); 178 if (ret) 179 return ret; 180 181 args.handle = handlep; 182 183 ret = DRM_COPY_TO_USER((struct drm_i915_gem_create *) data, &args, sizeof(args)); 184 185 if ( ret != 0) 186 DRM_ERROR(" gem create error! %d", ret); 187 188 DRM_DEBUG("i915_gem_create_ioctl object name %d, size 0x%lx, list 0x%lx, obj 0x%lx",handlep, args.size, &fpriv->object_idr, obj); 189 190 return 0; 191 } 192 193 /** 194 * Reads data from the object referenced by handle. 195 * 196 * On error, the contents of *data are undefined. 197 */ 198 /*ARGSUSED*/ 199 int 200 i915_gem_pread_ioctl(DRM_IOCTL_ARGS) 201 { 202 DRM_DEVICE; 203 struct drm_i915_gem_pread args; 204 struct drm_gem_object *obj; 205 int ret; 206 207 if (dev->driver->use_gem != 1) 208 return ENODEV; 209 210 DRM_COPYFROM_WITH_RETURN(&args, 211 (struct drm_i915_gem_pread __user *) data, sizeof(args)); 212 213 obj = drm_gem_object_lookup(fpriv, args.handle); 214 if (obj == NULL) 215 return EBADF; 216 217 /* Bounds check source. 218 * 219 * XXX: This could use review for overflow issues... 220 */ 221 if (args.offset > obj->size || args.size > obj->size || 222 args.offset + args.size > obj->size) { 223 drm_gem_object_unreference(obj); 224 DRM_ERROR("i915_gem_pread_ioctl invalid args"); 225 return EINVAL; 226 } 227 228 spin_lock(&dev->struct_mutex); 229 230 ret = i915_gem_object_set_cpu_read_domain_range(obj, args.offset, args.size); 231 if (ret != 0) { 232 drm_gem_object_unreference(obj); 233 spin_unlock(&dev->struct_mutex); 234 DRM_ERROR("pread failed to read domain range ret %d!!!", ret); 235 return EFAULT; 236 } 237 238 unsigned long unwritten = 0; 239 uint32_t *user_data; 240 user_data = (uint32_t *) (uintptr_t) args.data_ptr; 241 242 unwritten = DRM_COPY_TO_USER(user_data, obj->kaddr + args.offset, args.size); 243 if (unwritten) { 244 ret = EFAULT; 245 DRM_ERROR("i915_gem_pread error!!! unwritten %d", unwritten); 246 } 247 248 drm_gem_object_unreference(obj); 249 spin_unlock(&dev->struct_mutex); 250 251 return ret; 252 } 253 254 /*ARGSUSED*/ 255 static int 256 i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 257 struct drm_i915_gem_pwrite *args, 258 struct drm_file *file_priv) 259 { 260 uint32_t *user_data; 261 int ret = 0; 262 unsigned long unwritten = 0; 263 264 user_data = (uint32_t *) (uintptr_t) args->data_ptr; 265 spin_lock(&dev->struct_mutex); 266 ret = i915_gem_object_pin(obj, 0); 267 if (ret) { 268 spin_unlock(&dev->struct_mutex); 269 DRM_ERROR("i915_gem_gtt_pwrite failed to pin ret %d", ret); 270 return ret; 271 } 272 273 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 274 if (ret) 275 goto err; 276 277 DRM_DEBUG("obj %d write domain 0x%x read domain 0x%x", obj->name, obj->write_domain, obj->read_domains); 278 279 unwritten = DRM_COPY_FROM_USER(obj->kaddr + args->offset, user_data, args->size); 280 if (unwritten) { 281 ret = EFAULT; 282 DRM_ERROR("i915_gem_gtt_pwrite error!!! unwritten %d", unwritten); 283 goto err; 284 } 285 286 err: 287 i915_gem_object_unpin(obj); 288 spin_unlock(&dev->struct_mutex); 289 if (ret) 290 DRM_ERROR("i915_gem_gtt_pwrite error %d", ret); 291 return ret; 292 } 293 294 /*ARGSUSED*/ 295 int 296 i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 297 struct drm_i915_gem_pwrite *args, 298 struct drm_file *file_priv) 299 { 300 DRM_ERROR(" i915_gem_shmem_pwrite Not support"); 301 return -1; 302 } 303 304 /** 305 * Writes data to the object referenced by handle. 306 * 307 * On error, the contents of the buffer that were to be modified are undefined. 308 */ 309 /*ARGSUSED*/ 310 int 311 i915_gem_pwrite_ioctl(DRM_IOCTL_ARGS) 312 { 313 DRM_DEVICE; 314 struct drm_i915_gem_pwrite args; 315 struct drm_gem_object *obj; 316 struct drm_i915_gem_object *obj_priv; 317 int ret = 0; 318 319 if (dev->driver->use_gem != 1) 320 return ENODEV; 321 322 ret = DRM_COPY_FROM_USER(&args, 323 (struct drm_i915_gem_pwrite __user *) data, sizeof(args)); 324 if (ret) 325 DRM_ERROR("i915_gem_pwrite_ioctl failed to copy from user"); 326 obj = drm_gem_object_lookup(fpriv, args.handle); 327 if (obj == NULL) 328 return EBADF; 329 obj_priv = obj->driver_private; 330 DRM_DEBUG("i915_gem_pwrite_ioctl, obj->name %d",obj->name); 331 332 /* Bounds check destination. 333 * 334 * XXX: This could use review for overflow issues... 335 */ 336 if (args.offset > obj->size || args.size > obj->size || 337 args.offset + args.size > obj->size) { 338 drm_gem_object_unreference(obj); 339 DRM_ERROR("i915_gem_pwrite_ioctl invalid arg"); 340 return EINVAL; 341 } 342 343 /* We can only do the GTT pwrite on untiled buffers, as otherwise 344 * it would end up going through the fenced access, and we'll get 345 * different detiling behavior between reading and writing. 346 * pread/pwrite currently are reading and writing from the CPU 347 * perspective, requiring manual detiling by the client. 348 */ 349 if (obj_priv->tiling_mode == I915_TILING_NONE && 350 dev->gtt_total != 0) 351 ret = i915_gem_gtt_pwrite(dev, obj, &args, fpriv); 352 else 353 ret = i915_gem_shmem_pwrite(dev, obj, &args, fpriv); 354 355 if (ret) 356 DRM_ERROR("pwrite failed %d\n", ret); 357 358 drm_gem_object_unreference(obj); 359 360 return ret; 361 } 362 363 /** 364 * Called when user space prepares to use an object with the CPU, either 365 * through the mmap ioctl's mapping or a GTT mapping. 366 */ 367 /*ARGSUSED*/ 368 int 369 i915_gem_set_domain_ioctl(DRM_IOCTL_ARGS) 370 { 371 DRM_DEVICE; 372 struct drm_i915_gem_set_domain args; 373 struct drm_gem_object *obj; 374 int ret = 0; 375 376 if (dev->driver->use_gem != 1) 377 return ENODEV; 378 379 DRM_COPYFROM_WITH_RETURN(&args, 380 (struct drm_i915_gem_set_domain __user *) data, sizeof(args)); 381 382 uint32_t read_domains = args.read_domains; 383 uint32_t write_domain = args.write_domain; 384 385 /* Only handle setting domains to types used by the CPU. */ 386 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 387 ret = EINVAL; 388 389 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 390 ret = EINVAL; 391 392 /* Having something in the write domain implies it's in the read 393 * domain, and only that read domain. Enforce that in the request. 394 */ 395 if (write_domain != 0 && read_domains != write_domain) 396 ret = EINVAL; 397 if (ret) { 398 DRM_ERROR("set_domain invalid read or write"); 399 return EINVAL; 400 } 401 402 obj = drm_gem_object_lookup(fpriv, args.handle); 403 if (obj == NULL) 404 return EBADF; 405 406 spin_lock(&dev->struct_mutex); 407 DRM_DEBUG("set_domain_ioctl %p(name %d size 0x%x), %08x %08x\n", 408 obj, obj->name, obj->size, args.read_domains, args.write_domain); 409 410 if (read_domains & I915_GEM_DOMAIN_GTT) { 411 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 412 413 /* Silently promote "you're not bound, there was nothing to do" 414 * to success, since the client was just asking us to 415 * make sure everything was done. 416 */ 417 if (ret == EINVAL) 418 ret = 0; 419 } else { 420 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 421 } 422 423 drm_gem_object_unreference(obj); 424 spin_unlock(&dev->struct_mutex); 425 if (ret) 426 DRM_ERROR("i915_set_domain_ioctl ret %d", ret); 427 return ret; 428 } 429 430 /** 431 * Called when user space has done writes to this buffer 432 */ 433 /*ARGSUSED*/ 434 int 435 i915_gem_sw_finish_ioctl(DRM_IOCTL_ARGS) 436 { 437 DRM_DEVICE; 438 struct drm_i915_gem_sw_finish args; 439 struct drm_gem_object *obj; 440 struct drm_i915_gem_object *obj_priv; 441 int ret = 0; 442 443 if (dev->driver->use_gem != 1) 444 return ENODEV; 445 446 DRM_COPYFROM_WITH_RETURN(&args, 447 (struct drm_i915_gem_sw_finish __user *) data, sizeof(args)); 448 449 spin_lock(&dev->struct_mutex); 450 obj = drm_gem_object_lookup(fpriv, args.handle); 451 if (obj == NULL) { 452 spin_unlock(&dev->struct_mutex); 453 return EBADF; 454 } 455 456 DRM_DEBUG("%s: sw_finish %d (%p name %d size 0x%x)\n", 457 __func__, args.handle, obj, obj->name, obj->size); 458 459 obj_priv = obj->driver_private; 460 /* Pinned buffers may be scanout, so flush the cache */ 461 if (obj_priv->pin_count) 462 { 463 i915_gem_object_flush_cpu_write_domain(obj); 464 } 465 466 drm_gem_object_unreference(obj); 467 spin_unlock(&dev->struct_mutex); 468 return ret; 469 } 470 471 /** 472 * Maps the contents of an object, returning the address it is mapped 473 * into. 474 * 475 * While the mapping holds a reference on the contents of the object, it doesn't 476 * imply a ref on the object itself. 477 */ 478 /*ARGSUSED*/ 479 int 480 i915_gem_mmap_ioctl(DRM_IOCTL_ARGS) 481 { 482 DRM_DEVICE; 483 struct drm_i915_gem_mmap args; 484 struct drm_gem_object *obj; 485 caddr_t vvaddr = NULL; 486 int ret; 487 488 if (dev->driver->use_gem != 1) 489 return ENODEV; 490 491 DRM_COPYFROM_WITH_RETURN( 492 &args, (struct drm_i915_gem_mmap __user *)data, 493 sizeof (struct drm_i915_gem_mmap)); 494 495 obj = drm_gem_object_lookup(fpriv, args.handle); 496 if (obj == NULL) 497 return EBADF; 498 499 ret = ddi_devmap_segmap(fpriv->dev, (off_t)obj->map->handle, 500 ttoproc(curthread)->p_as, &vvaddr, obj->map->size, 501 PROT_ALL, PROT_ALL, MAP_SHARED, fpriv->credp); 502 if (ret) 503 return ret; 504 505 spin_lock(&dev->struct_mutex); 506 drm_gem_object_unreference(obj); 507 spin_unlock(&dev->struct_mutex); 508 509 args.addr_ptr = (uint64_t)(uintptr_t)vvaddr; 510 511 DRM_COPYTO_WITH_RETURN( 512 (struct drm_i915_gem_mmap __user *)data, 513 &args, sizeof (struct drm_i915_gem_mmap)); 514 515 return 0; 516 } 517 518 static void 519 i915_gem_object_free_page_list(struct drm_gem_object *obj) 520 { 521 struct drm_i915_gem_object *obj_priv = obj->driver_private; 522 if (obj_priv->page_list == NULL) 523 return; 524 525 kmem_free(obj_priv->page_list, 526 btop(obj->size) * sizeof(caddr_t)); 527 528 obj_priv->page_list = NULL; 529 } 530 531 static void 532 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno) 533 { 534 struct drm_device *dev = obj->dev; 535 drm_i915_private_t *dev_priv = dev->dev_private; 536 struct drm_i915_gem_object *obj_priv = obj->driver_private; 537 538 /* Add a reference if we're newly entering the active list. */ 539 if (!obj_priv->active) { 540 drm_gem_object_reference(obj); 541 obj_priv->active = 1; 542 } 543 /* Move from whatever list we were on to the tail of execution. */ 544 list_move_tail(&obj_priv->list, 545 &dev_priv->mm.active_list, (caddr_t)obj_priv); 546 obj_priv->last_rendering_seqno = seqno; 547 } 548 549 static void 550 i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 551 { 552 struct drm_device *dev = obj->dev; 553 drm_i915_private_t *dev_priv = dev->dev_private; 554 struct drm_i915_gem_object *obj_priv = obj->driver_private; 555 556 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list, (caddr_t)obj_priv); 557 obj_priv->last_rendering_seqno = 0; 558 } 559 560 static void 561 i915_gem_object_move_to_inactive(struct drm_gem_object *obj) 562 { 563 struct drm_device *dev = obj->dev; 564 drm_i915_private_t *dev_priv = dev->dev_private; 565 struct drm_i915_gem_object *obj_priv = obj->driver_private; 566 567 if (obj_priv->pin_count != 0) 568 { 569 list_del_init(&obj_priv->list); 570 } else { 571 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list, (caddr_t)obj_priv); 572 } 573 obj_priv->last_rendering_seqno = 0; 574 if (obj_priv->active) { 575 obj_priv->active = 0; 576 drm_gem_object_unreference(obj); 577 } 578 } 579 580 /** 581 * Creates a new sequence number, emitting a write of it to the status page 582 * plus an interrupt, which will trigger i915_user_interrupt_handler. 583 * 584 * Must be called with struct_lock held. 585 * 586 * Returned sequence numbers are nonzero on success. 587 */ 588 static uint32_t 589 i915_add_request(struct drm_device *dev, uint32_t flush_domains) 590 { 591 drm_i915_private_t *dev_priv = dev->dev_private; 592 struct drm_i915_gem_request *request; 593 uint32_t seqno; 594 int was_empty; 595 RING_LOCALS; 596 597 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER); 598 if (request == NULL) { 599 DRM_ERROR("Failed to alloc request"); 600 return 0; 601 } 602 /* Grab the seqno we're going to make this request be, and bump the 603 * next (skipping 0 so it can be the reserved no-seqno value). 604 */ 605 seqno = dev_priv->mm.next_gem_seqno; 606 dev_priv->mm.next_gem_seqno++; 607 if (dev_priv->mm.next_gem_seqno == 0) 608 dev_priv->mm.next_gem_seqno++; 609 610 DRM_DEBUG("add_request seqno = %d dev 0x%lx", seqno, dev); 611 612 BEGIN_LP_RING(4); 613 OUT_RING(MI_STORE_DWORD_INDEX); 614 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 615 OUT_RING(seqno); 616 OUT_RING(0); 617 ADVANCE_LP_RING(); 618 619 BEGIN_LP_RING(2); 620 OUT_RING(0); 621 OUT_RING(MI_USER_INTERRUPT); 622 ADVANCE_LP_RING(); 623 624 request->seqno = seqno; 625 request->emitted_jiffies = jiffies; 626 was_empty = list_empty(&dev_priv->mm.request_list); 627 list_add_tail(&request->list, &dev_priv->mm.request_list, (caddr_t)request); 628 629 /* Associate any objects on the flushing list matching the write 630 * domain we're flushing with our flush. 631 */ 632 if (flush_domains != 0) { 633 struct drm_i915_gem_object *obj_priv, *next; 634 635 obj_priv = list_entry(dev_priv->mm.flushing_list.next, struct drm_i915_gem_object, list), 636 next = list_entry(obj_priv->list.next, struct drm_i915_gem_object, list); 637 for(; &obj_priv->list != &dev_priv->mm.flushing_list; 638 obj_priv = next, 639 next = list_entry(next->list.next, struct drm_i915_gem_object, list)) { 640 struct drm_gem_object *obj = obj_priv->obj; 641 642 if ((obj->write_domain & flush_domains) == 643 obj->write_domain) { 644 obj->write_domain = 0; 645 i915_gem_object_move_to_active(obj, seqno); 646 } 647 } 648 649 } 650 651 if (was_empty && !dev_priv->mm.suspended) 652 { 653 /* change to delay HZ and then run work (not insert to workqueue of Linux) */ 654 worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ); 655 DRM_DEBUG("i915_gem: schedule_delayed_work"); 656 } 657 return seqno; 658 } 659 660 /** 661 * Command execution barrier 662 * 663 * Ensures that all commands in the ring are finished 664 * before signalling the CPU 665 */ 666 uint32_t 667 i915_retire_commands(struct drm_device *dev) 668 { 669 drm_i915_private_t *dev_priv = dev->dev_private; 670 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 671 uint32_t flush_domains = 0; 672 RING_LOCALS; 673 674 /* The sampler always gets flushed on i965 (sigh) */ 675 if (IS_I965G(dev)) 676 flush_domains |= I915_GEM_DOMAIN_SAMPLER; 677 BEGIN_LP_RING(2); 678 OUT_RING(cmd); 679 OUT_RING(0); /* noop */ 680 ADVANCE_LP_RING(); 681 682 return flush_domains; 683 } 684 685 /** 686 * Moves buffers associated only with the given active seqno from the active 687 * to inactive list, potentially freeing them. 688 */ 689 static void 690 i915_gem_retire_request(struct drm_device *dev, 691 struct drm_i915_gem_request *request) 692 { 693 drm_i915_private_t *dev_priv = dev->dev_private; 694 /* Move any buffers on the active list that are no longer referenced 695 * by the ringbuffer to the flushing/inactive lists as appropriate. 696 */ 697 while (!list_empty(&dev_priv->mm.active_list)) { 698 struct drm_gem_object *obj; 699 struct drm_i915_gem_object *obj_priv; 700 701 obj_priv = list_entry(dev_priv->mm.active_list.next, 702 struct drm_i915_gem_object, 703 list); 704 obj = obj_priv->obj; 705 706 /* If the seqno being retired doesn't match the oldest in the 707 * list, then the oldest in the list must still be newer than 708 * this seqno. 709 */ 710 if (obj_priv->last_rendering_seqno != request->seqno) 711 return; 712 713 DRM_DEBUG("%s: retire %d moves to inactive list %p\n", 714 __func__, request->seqno, obj); 715 716 if (obj->write_domain != 0) { 717 i915_gem_object_move_to_flushing(obj); 718 } else { 719 i915_gem_object_move_to_inactive(obj); 720 } 721 } 722 } 723 724 /** 725 * Returns true if seq1 is later than seq2. 726 */ 727 static int 728 i915_seqno_passed(uint32_t seq1, uint32_t seq2) 729 { 730 return (int32_t)(seq1 - seq2) >= 0; 731 } 732 733 uint32_t 734 i915_get_gem_seqno(struct drm_device *dev) 735 { 736 drm_i915_private_t *dev_priv = dev->dev_private; 737 738 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX); 739 } 740 741 /** 742 * This function clears the request list as sequence numbers are passed. 743 */ 744 void 745 i915_gem_retire_requests(struct drm_device *dev) 746 { 747 drm_i915_private_t *dev_priv = dev->dev_private; 748 uint32_t seqno; 749 750 seqno = i915_get_gem_seqno(dev); 751 752 while (!list_empty(&dev_priv->mm.request_list)) { 753 struct drm_i915_gem_request *request; 754 uint32_t retiring_seqno; 755 request = (struct drm_i915_gem_request *)(uintptr_t)(dev_priv->mm.request_list.next->contain_ptr); 756 retiring_seqno = request->seqno; 757 758 if (i915_seqno_passed(seqno, retiring_seqno) || 759 dev_priv->mm.wedged) { 760 i915_gem_retire_request(dev, request); 761 762 list_del(&request->list); 763 drm_free(request, sizeof(*request), DRM_MEM_DRIVER); 764 } else 765 break; 766 } 767 } 768 769 void 770 i915_gem_retire_work_handler(void *device) 771 { 772 struct drm_device *dev = (struct drm_device *)device; 773 drm_i915_private_t *dev_priv = dev->dev_private; 774 775 spin_lock(&dev->struct_mutex); 776 777 /* Return if gem idle */ 778 if (worktimer_id == NULL) { 779 spin_unlock(&dev->struct_mutex); 780 return; 781 } 782 783 i915_gem_retire_requests(dev); 784 if (!dev_priv->mm.suspended && !list_empty(&dev_priv->mm.request_list)) 785 { 786 DRM_DEBUG("i915_gem: schedule_delayed_work"); 787 worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ); 788 } 789 spin_unlock(&dev->struct_mutex); 790 } 791 792 /** 793 * i965_reset - reset chip after a hang 794 * @dev: drm device to reset 795 * @flags: reset domains 796 * 797 * Reset the chip. Useful if a hang is detected. 798 * 799 * Procedure is fairly simple: 800 * - reset the chip using the reset reg 801 * - re-init context state 802 * - re-init hardware status page 803 * - re-init ring buffer 804 * - re-init interrupt state 805 * - re-init display 806 */ 807 void i965_reset(struct drm_device *dev, u8 flags) 808 { 809 ddi_acc_handle_t conf_hdl; 810 drm_i915_private_t *dev_priv = dev->dev_private; 811 int timeout = 0; 812 uint8_t gdrst; 813 814 if (flags & GDRST_FULL) 815 i915_save_display(dev); 816 817 if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) { 818 DRM_ERROR(("i915_reset: pci_config_setup fail")); 819 return; 820 } 821 822 /* 823 * Set the reset bit, wait for reset, then clear it. Hardware 824 * will clear the status bit (bit 1) when it's actually ready 825 * for action again. 826 */ 827 gdrst = pci_config_get8(conf_hdl, GDRST); 828 pci_config_put8(conf_hdl, GDRST, gdrst | flags); 829 drv_usecwait(50); 830 pci_config_put8(conf_hdl, GDRST, gdrst | 0xfe); 831 832 /* ...we don't want to loop forever though, 500ms should be plenty */ 833 do { 834 drv_usecwait(100); 835 gdrst = pci_config_get8(conf_hdl, GDRST); 836 } while ((gdrst & 2) && (timeout++ < 5)); 837 838 /* Ok now get things going again... */ 839 840 /* 841 * Everything depends on having the GTT running, so we need to start 842 * there. Fortunately we don't need to do this unless we reset the 843 * chip at a PCI level. 844 * 845 * Next we need to restore the context, but we don't use those 846 * yet either... 847 * 848 * Ring buffer needs to be re-initialized in the KMS case, or if X 849 * was running at the time of the reset (i.e. we weren't VT 850 * switched away). 851 */ 852 if (!dev_priv->mm.suspended) { 853 drm_i915_ring_buffer_t *ring = &dev_priv->ring; 854 struct drm_gem_object *obj = ring->ring_obj; 855 struct drm_i915_gem_object *obj_priv = obj->driver_private; 856 dev_priv->mm.suspended = 0; 857 858 /* Stop the ring if it's running. */ 859 I915_WRITE(PRB0_CTL, 0); 860 I915_WRITE(PRB0_TAIL, 0); 861 I915_WRITE(PRB0_HEAD, 0); 862 863 /* Initialize the ring. */ 864 I915_WRITE(PRB0_START, obj_priv->gtt_offset); 865 I915_WRITE(PRB0_CTL, 866 ((obj->size - 4096) & RING_NR_PAGES) | 867 RING_NO_REPORT | 868 RING_VALID); 869 i915_kernel_lost_context(dev); 870 871 (void) drm_irq_install(dev); 872 } 873 874 /* 875 * Display needs restore too... 876 */ 877 if (flags & GDRST_FULL) 878 i915_restore_display(dev); 879 } 880 881 /** 882 * Waits for a sequence number to be signaled, and cleans up the 883 * request and object lists appropriately for that event. 884 */ 885 int 886 i915_wait_request(struct drm_device *dev, uint32_t seqno) 887 { 888 drm_i915_private_t *dev_priv = dev->dev_private; 889 u32 ier; 890 int ret = 0; 891 892 ASSERT(seqno != 0); 893 894 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) { 895 if (IS_IGDNG(dev)) 896 ier = I915_READ(DEIER) | I915_READ(GTIER); 897 else 898 ier = I915_READ(IER); 899 if (!ier) { 900 DRM_ERROR("something (likely vbetool) disabled " 901 "interrupts, re-enabling\n"); 902 (void) i915_driver_irq_preinstall(dev); 903 i915_driver_irq_postinstall(dev); 904 } 905 906 dev_priv->mm.waiting_gem_seqno = seqno; 907 i915_user_irq_on(dev); 908 DRM_WAIT(ret, &dev_priv->irq_queue, 909 (i915_seqno_passed(i915_get_gem_seqno(dev), seqno) || 910 dev_priv->mm.wedged)); 911 i915_user_irq_off(dev); 912 dev_priv->mm.waiting_gem_seqno = 0; 913 } 914 if (dev_priv->mm.wedged) { 915 ret = EIO; 916 } 917 918 /* GPU maybe hang, reset needed*/ 919 if (ret == -2 && (seqno > i915_get_gem_seqno(dev))) { 920 if (IS_I965G(dev)) { 921 DRM_ERROR("GPU hang detected try to reset ... wait for irq_queue seqno %d, now seqno %d", seqno, i915_get_gem_seqno(dev)); 922 dev_priv->mm.wedged = 1; 923 i965_reset(dev, GDRST_RENDER); 924 i915_gem_retire_requests(dev); 925 dev_priv->mm.wedged = 0; 926 } 927 else 928 DRM_ERROR("GPU hang detected.... reboot required"); 929 return 0; 930 } 931 /* Directly dispatch request retiring. While we have the work queue 932 * to handle this, the waiter on a request often wants an associated 933 * buffer to have made it to the inactive list, and we would need 934 * a separate wait queue to handle that. 935 */ 936 if (ret == 0) 937 i915_gem_retire_requests(dev); 938 939 return ret; 940 } 941 942 static void 943 i915_gem_flush(struct drm_device *dev, 944 uint32_t invalidate_domains, 945 uint32_t flush_domains) 946 { 947 drm_i915_private_t *dev_priv = dev->dev_private; 948 uint32_t cmd; 949 RING_LOCALS; 950 951 DRM_DEBUG("%s: invalidate %08x flush %08x\n", __func__, 952 invalidate_domains, flush_domains); 953 954 if (flush_domains & I915_GEM_DOMAIN_CPU) 955 drm_agp_chipset_flush(dev); 956 957 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU | 958 I915_GEM_DOMAIN_GTT)) { 959 /* 960 * read/write caches: 961 * 962 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 963 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 964 * also flushed at 2d versus 3d pipeline switches. 965 * 966 * read-only caches: 967 * 968 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 969 * MI_READ_FLUSH is set, and is always flushed on 965. 970 * 971 * I915_GEM_DOMAIN_COMMAND may not exist? 972 * 973 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 974 * invalidated when MI_EXE_FLUSH is set. 975 * 976 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 977 * invalidated with every MI_FLUSH. 978 * 979 * TLBs: 980 * 981 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 982 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 983 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 984 * are flushed at any MI_FLUSH. 985 */ 986 987 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 988 if ((invalidate_domains|flush_domains) & 989 I915_GEM_DOMAIN_RENDER) 990 cmd &= ~MI_NO_WRITE_FLUSH; 991 if (!IS_I965G(dev)) { 992 /* 993 * On the 965, the sampler cache always gets flushed 994 * and this bit is reserved. 995 */ 996 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) 997 cmd |= MI_READ_FLUSH; 998 } 999 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 1000 cmd |= MI_EXE_FLUSH; 1001 1002 DRM_DEBUG("%s: queue flush %08x to ring\n", __func__, cmd); 1003 1004 BEGIN_LP_RING(2); 1005 OUT_RING(cmd); 1006 OUT_RING(0); /* noop */ 1007 ADVANCE_LP_RING(); 1008 } 1009 } 1010 1011 /** 1012 * Ensures that all rendering to the object has completed and the object is 1013 * safe to unbind from the GTT or access from the CPU. 1014 */ 1015 static int 1016 i915_gem_object_wait_rendering(struct drm_gem_object *obj) 1017 { 1018 struct drm_device *dev = obj->dev; 1019 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1020 int ret, seqno; 1021 1022 /* This function only exists to support waiting for existing rendering, 1023 * not for emitting required flushes. 1024 */ 1025 1026 if((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) { 1027 DRM_ERROR("write domain should not be GPU DOMAIN %d", obj_priv->active); 1028 return 0; 1029 } 1030 1031 /* If there is rendering queued on the buffer being evicted, wait for 1032 * it. 1033 */ 1034 if (obj_priv->active) { 1035 DRM_DEBUG("%s: object %d %p wait for seqno %08x\n", 1036 __func__, obj->name, obj, obj_priv->last_rendering_seqno); 1037 1038 seqno = obj_priv->last_rendering_seqno; 1039 if (seqno == 0) { 1040 DRM_DEBUG("last rendering maybe finished"); 1041 return 0; 1042 } 1043 ret = i915_wait_request(dev, seqno); 1044 if (ret != 0) { 1045 DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev)); 1046 return ret; 1047 } 1048 } 1049 1050 return 0; 1051 } 1052 1053 /** 1054 * Unbinds an object from the GTT aperture. 1055 */ 1056 int 1057 i915_gem_object_unbind(struct drm_gem_object *obj, uint32_t type) 1058 { 1059 struct drm_device *dev = obj->dev; 1060 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1061 int ret = 0; 1062 1063 if (obj_priv->gtt_space == NULL) 1064 return 0; 1065 1066 if (obj_priv->pin_count != 0) { 1067 DRM_ERROR("Attempting to unbind pinned buffer\n"); 1068 return EINVAL; 1069 } 1070 1071 /* Wait for any rendering to complete 1072 */ 1073 ret = i915_gem_object_wait_rendering(obj); 1074 if (ret) { 1075 DRM_ERROR("wait_rendering failed: %d\n", ret); 1076 return ret; 1077 } 1078 1079 /* Move the object to the CPU domain to ensure that 1080 * any possible CPU writes while it's not in the GTT 1081 * are flushed when we go to remap it. This will 1082 * also ensure that all pending GPU writes are finished 1083 * before we unbind. 1084 */ 1085 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 1086 if (ret) { 1087 DRM_ERROR("set_domain failed: %d\n", ret); 1088 return ret; 1089 } 1090 1091 if (!obj_priv->agp_mem) { 1092 (void) drm_agp_unbind_pages(dev, obj->size / PAGE_SIZE, 1093 obj_priv->gtt_offset, type); 1094 obj_priv->agp_mem = -1; 1095 } 1096 1097 ASSERT(!obj_priv->active); 1098 1099 i915_gem_object_free_page_list(obj); 1100 1101 if (obj_priv->gtt_space) { 1102 atomic_dec(&dev->gtt_count); 1103 atomic_sub(obj->size, &dev->gtt_memory); 1104 drm_mm_put_block(obj_priv->gtt_space); 1105 obj_priv->gtt_space = NULL; 1106 } 1107 1108 /* Remove ourselves from the LRU list if present. */ 1109 if (!list_empty(&obj_priv->list)) 1110 list_del_init(&obj_priv->list); 1111 1112 return 0; 1113 } 1114 1115 static int 1116 i915_gem_evict_something(struct drm_device *dev) 1117 { 1118 drm_i915_private_t *dev_priv = dev->dev_private; 1119 struct drm_gem_object *obj; 1120 struct drm_i915_gem_object *obj_priv; 1121 int ret = 0; 1122 1123 for (;;) { 1124 /* If there's an inactive buffer available now, grab it 1125 * and be done. 1126 */ 1127 if (!list_empty(&dev_priv->mm.inactive_list)) { 1128 obj_priv = list_entry(dev_priv->mm.inactive_list.next, 1129 struct drm_i915_gem_object, 1130 list); 1131 obj = obj_priv->obj; 1132 ASSERT(!(obj_priv->pin_count != 0)); 1133 DRM_DEBUG("%s: evicting %d\n", __func__, obj->name); 1134 ASSERT(!(obj_priv->active)); 1135 /* Wait on the rendering and unbind the buffer. */ 1136 ret = i915_gem_object_unbind(obj, 1); 1137 break; 1138 } 1139 /* If we didn't get anything, but the ring is still processing 1140 * things, wait for one of those things to finish and hopefully 1141 * leave us a buffer to evict. 1142 */ 1143 if (!list_empty(&dev_priv->mm.request_list)) { 1144 struct drm_i915_gem_request *request; 1145 1146 request = list_entry(dev_priv->mm.request_list.next, 1147 struct drm_i915_gem_request, 1148 list); 1149 1150 ret = i915_wait_request(dev, request->seqno); 1151 if (ret) { 1152 break; 1153 } 1154 /* if waiting caused an object to become inactive, 1155 * then loop around and wait for it. Otherwise, we 1156 * assume that waiting freed and unbound something, 1157 * so there should now be some space in the GTT 1158 */ 1159 if (!list_empty(&dev_priv->mm.inactive_list)) 1160 continue; 1161 break; 1162 } 1163 1164 /* If we didn't have anything on the request list but there 1165 * are buffers awaiting a flush, emit one and try again. 1166 * When we wait on it, those buffers waiting for that flush 1167 * will get moved to inactive. 1168 */ 1169 if (!list_empty(&dev_priv->mm.flushing_list)) { 1170 obj_priv = list_entry(dev_priv->mm.flushing_list.next, 1171 struct drm_i915_gem_object, 1172 list); 1173 obj = obj_priv->obj; 1174 1175 i915_gem_flush(dev, 1176 obj->write_domain, 1177 obj->write_domain); 1178 (void) i915_add_request(dev, obj->write_domain); 1179 1180 obj = NULL; 1181 continue; 1182 } 1183 1184 DRM_ERROR("inactive empty %d request empty %d " 1185 "flushing empty %d\n", 1186 list_empty(&dev_priv->mm.inactive_list), 1187 list_empty(&dev_priv->mm.request_list), 1188 list_empty(&dev_priv->mm.flushing_list)); 1189 /* If we didn't do any of the above, there's nothing to be done 1190 * and we just can't fit it in. 1191 */ 1192 return ENOMEM; 1193 } 1194 return ret; 1195 } 1196 1197 static int 1198 i915_gem_evict_everything(struct drm_device *dev) 1199 { 1200 int ret; 1201 1202 for (;;) { 1203 ret = i915_gem_evict_something(dev); 1204 if (ret != 0) 1205 break; 1206 } 1207 if (ret == ENOMEM) 1208 return 0; 1209 else 1210 DRM_ERROR("evict_everything ret %d", ret); 1211 return ret; 1212 } 1213 1214 /** 1215 * Finds free space in the GTT aperture and binds the object there. 1216 */ 1217 static int 1218 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, uint32_t alignment) 1219 { 1220 struct drm_device *dev = obj->dev; 1221 drm_i915_private_t *dev_priv = dev->dev_private; 1222 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1223 struct drm_mm_node *free_space; 1224 int page_count, ret; 1225 1226 if (dev_priv->mm.suspended) 1227 return EBUSY; 1228 if (alignment == 0) 1229 alignment = PAGE_SIZE; 1230 if (alignment & (PAGE_SIZE - 1)) { 1231 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 1232 return EINVAL; 1233 } 1234 1235 if (obj_priv->gtt_space) { 1236 DRM_ERROR("Already bind!!"); 1237 return 0; 1238 } 1239 search_free: 1240 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 1241 (unsigned long) obj->size, alignment, 0); 1242 if (free_space != NULL) { 1243 obj_priv->gtt_space = drm_mm_get_block(free_space, (unsigned long) obj->size, 1244 alignment); 1245 if (obj_priv->gtt_space != NULL) { 1246 obj_priv->gtt_space->private = obj; 1247 obj_priv->gtt_offset = obj_priv->gtt_space->start; 1248 } 1249 } 1250 if (obj_priv->gtt_space == NULL) { 1251 /* If the gtt is empty and we're still having trouble 1252 * fitting our object in, we're out of memory. 1253 */ 1254 if (list_empty(&dev_priv->mm.inactive_list) && 1255 list_empty(&dev_priv->mm.flushing_list) && 1256 list_empty(&dev_priv->mm.active_list)) { 1257 DRM_ERROR("GTT full, but LRU list empty\n"); 1258 return ENOMEM; 1259 } 1260 1261 ret = i915_gem_evict_something(dev); 1262 if (ret != 0) { 1263 DRM_ERROR("Failed to evict a buffer %d\n", ret); 1264 return ret; 1265 } 1266 goto search_free; 1267 } 1268 1269 ret = i915_gem_object_get_page_list(obj); 1270 if (ret) { 1271 drm_mm_put_block(obj_priv->gtt_space); 1272 obj_priv->gtt_space = NULL; 1273 DRM_ERROR("bind to gtt failed to get page list"); 1274 return ret; 1275 } 1276 1277 page_count = obj->size / PAGE_SIZE; 1278 /* Create an AGP memory structure pointing at our pages, and bind it 1279 * into the GTT. 1280 */ 1281 DRM_DEBUG("Binding object %d of page_count %d at gtt_offset 0x%x obj->pfnarray = 0x%lx", 1282 obj->name, page_count, obj_priv->gtt_offset, obj->pfnarray); 1283 1284 obj_priv->agp_mem = drm_agp_bind_pages(dev, 1285 obj->pfnarray, 1286 page_count, 1287 obj_priv->gtt_offset); 1288 if (obj_priv->agp_mem) { 1289 i915_gem_object_free_page_list(obj); 1290 drm_mm_put_block(obj_priv->gtt_space); 1291 obj_priv->gtt_space = NULL; 1292 DRM_ERROR("Failed to bind pages obj %d, obj 0x%lx", obj->name, obj); 1293 return ENOMEM; 1294 } 1295 atomic_inc(&dev->gtt_count); 1296 atomic_add(obj->size, &dev->gtt_memory); 1297 1298 /* Assert that the object is not currently in any GPU domain. As it 1299 * wasn't in the GTT, there shouldn't be any way it could have been in 1300 * a GPU cache 1301 */ 1302 ASSERT(!(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))); 1303 ASSERT(!(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))); 1304 1305 return 0; 1306 } 1307 1308 void 1309 i915_gem_clflush_object(struct drm_gem_object *obj) 1310 { 1311 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1312 1313 /* If we don't have a page list set up, then we're not pinned 1314 * to GPU, and we can ignore the cache flush because it'll happen 1315 * again at bind time. 1316 */ 1317 1318 if (obj_priv->page_list == NULL) 1319 return; 1320 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE); 1321 } 1322 1323 /** Flushes any GPU write domain for the object if it's dirty. */ 1324 static void 1325 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) 1326 { 1327 struct drm_device *dev = obj->dev; 1328 uint32_t seqno; 1329 1330 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) 1331 return; 1332 1333 /* Queue the GPU write cache flushing we need. */ 1334 i915_gem_flush(dev, 0, obj->write_domain); 1335 seqno = i915_add_request(dev, obj->write_domain); 1336 DRM_DEBUG("flush_gpu_write_domain seqno = %d", seqno); 1337 obj->write_domain = 0; 1338 i915_gem_object_move_to_active(obj, seqno); 1339 } 1340 1341 /** Flushes the GTT write domain for the object if it's dirty. */ 1342 static void 1343 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 1344 { 1345 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 1346 return; 1347 1348 /* No actual flushing is required for the GTT write domain. Writes 1349 * to it immediately go to main memory as far as we know, so there's 1350 * no chipset flush. It also doesn't land in render cache. 1351 */ 1352 obj->write_domain = 0; 1353 } 1354 1355 /** Flushes the CPU write domain for the object if it's dirty. */ 1356 static void 1357 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 1358 { 1359 struct drm_device *dev = obj->dev; 1360 1361 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 1362 return; 1363 1364 i915_gem_clflush_object(obj); 1365 drm_agp_chipset_flush(dev); 1366 obj->write_domain = 0; 1367 } 1368 1369 /** 1370 * Moves a single object to the GTT read, and possibly write domain. 1371 * 1372 * This function returns when the move is complete, including waiting on 1373 * flushes to occur. 1374 */ 1375 static int 1376 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 1377 { 1378 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1379 int ret; 1380 1381 /* Not valid to be called on unbound objects. */ 1382 if (obj_priv->gtt_space == NULL) 1383 return EINVAL; 1384 1385 i915_gem_object_flush_gpu_write_domain(obj); 1386 /* Wait on any GPU rendering and flushing to occur. */ 1387 ret = i915_gem_object_wait_rendering(obj); 1388 if (ret != 0) { 1389 DRM_ERROR("set_to_gtt_domain wait_rendering ret %d", ret); 1390 return ret; 1391 } 1392 /* If we're writing through the GTT domain, then CPU and GPU caches 1393 * will need to be invalidated at next use. 1394 */ 1395 if (write) 1396 obj->read_domains &= I915_GEM_DOMAIN_GTT; 1397 i915_gem_object_flush_cpu_write_domain(obj); 1398 1399 DRM_DEBUG("i915_gem_object_set_to_gtt_domain obj->read_domains %x ", obj->read_domains); 1400 /* It should now be out of any other write domains, and we can update 1401 * the domain values for our changes. 1402 */ 1403 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0)); 1404 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1405 if (write) { 1406 obj->write_domain = I915_GEM_DOMAIN_GTT; 1407 obj_priv->dirty = 1; 1408 } 1409 1410 return 0; 1411 } 1412 1413 /** 1414 * Moves a single object to the CPU read, and possibly write domain. 1415 * 1416 * This function returns when the move is complete, including waiting on 1417 * flushes to occur. 1418 */ 1419 static int 1420 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 1421 { 1422 struct drm_device *dev = obj->dev; 1423 int ret; 1424 1425 1426 i915_gem_object_flush_gpu_write_domain(obj); 1427 /* Wait on any GPU rendering and flushing to occur. */ 1428 1429 ret = i915_gem_object_wait_rendering(obj); 1430 if (ret != 0) 1431 return ret; 1432 1433 i915_gem_object_flush_gtt_write_domain(obj); 1434 1435 /* If we have a partially-valid cache of the object in the CPU, 1436 * finish invalidating it and free the per-page flags. 1437 */ 1438 i915_gem_object_set_to_full_cpu_read_domain(obj); 1439 1440 /* Flush the CPU cache if it's still invalid. */ 1441 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1442 i915_gem_clflush_object(obj); 1443 drm_agp_chipset_flush(dev); 1444 obj->read_domains |= I915_GEM_DOMAIN_CPU; 1445 } 1446 1447 /* It should now be out of any other write domains, and we can update 1448 * the domain values for our changes. 1449 */ 1450 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0)); 1451 1452 /* If we're writing through the CPU, then the GPU read domains will 1453 * need to be invalidated at next use. 1454 */ 1455 if (write) { 1456 obj->read_domains &= I915_GEM_DOMAIN_CPU; 1457 obj->write_domain = I915_GEM_DOMAIN_CPU; 1458 } 1459 1460 return 0; 1461 } 1462 1463 /* 1464 * Set the next domain for the specified object. This 1465 * may not actually perform the necessary flushing/invaliding though, 1466 * as that may want to be batched with other set_domain operations 1467 * 1468 * This is (we hope) the only really tricky part of gem. The goal 1469 * is fairly simple -- track which caches hold bits of the object 1470 * and make sure they remain coherent. A few concrete examples may 1471 * help to explain how it works. For shorthand, we use the notation 1472 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the 1473 * a pair of read and write domain masks. 1474 * 1475 * Case 1: the batch buffer 1476 * 1477 * 1. Allocated 1478 * 2. Written by CPU 1479 * 3. Mapped to GTT 1480 * 4. Read by GPU 1481 * 5. Unmapped from GTT 1482 * 6. Freed 1483 * 1484 * Let's take these a step at a time 1485 * 1486 * 1. Allocated 1487 * Pages allocated from the kernel may still have 1488 * cache contents, so we set them to (CPU, CPU) always. 1489 * 2. Written by CPU (using pwrite) 1490 * The pwrite function calls set_domain (CPU, CPU) and 1491 * this function does nothing (as nothing changes) 1492 * 3. Mapped by GTT 1493 * This function asserts that the object is not 1494 * currently in any GPU-based read or write domains 1495 * 4. Read by GPU 1496 * i915_gem_execbuffer calls set_domain (COMMAND, 0). 1497 * As write_domain is zero, this function adds in the 1498 * current read domains (CPU+COMMAND, 0). 1499 * flush_domains is set to CPU. 1500 * invalidate_domains is set to COMMAND 1501 * clflush is run to get data out of the CPU caches 1502 * then i915_dev_set_domain calls i915_gem_flush to 1503 * emit an MI_FLUSH and drm_agp_chipset_flush 1504 * 5. Unmapped from GTT 1505 * i915_gem_object_unbind calls set_domain (CPU, CPU) 1506 * flush_domains and invalidate_domains end up both zero 1507 * so no flushing/invalidating happens 1508 * 6. Freed 1509 * yay, done 1510 * 1511 * Case 2: The shared render buffer 1512 * 1513 * 1. Allocated 1514 * 2. Mapped to GTT 1515 * 3. Read/written by GPU 1516 * 4. set_domain to (CPU,CPU) 1517 * 5. Read/written by CPU 1518 * 6. Read/written by GPU 1519 * 1520 * 1. Allocated 1521 * Same as last example, (CPU, CPU) 1522 * 2. Mapped to GTT 1523 * Nothing changes (assertions find that it is not in the GPU) 1524 * 3. Read/written by GPU 1525 * execbuffer calls set_domain (RENDER, RENDER) 1526 * flush_domains gets CPU 1527 * invalidate_domains gets GPU 1528 * clflush (obj) 1529 * MI_FLUSH and drm_agp_chipset_flush 1530 * 4. set_domain (CPU, CPU) 1531 * flush_domains gets GPU 1532 * invalidate_domains gets CPU 1533 * wait_rendering (obj) to make sure all drawing is complete. 1534 * This will include an MI_FLUSH to get the data from GPU 1535 * to memory 1536 * clflush (obj) to invalidate the CPU cache 1537 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) 1538 * 5. Read/written by CPU 1539 * cache lines are loaded and dirtied 1540 * 6. Read written by GPU 1541 * Same as last GPU access 1542 * 1543 * Case 3: The constant buffer 1544 * 1545 * 1. Allocated 1546 * 2. Written by CPU 1547 * 3. Read by GPU 1548 * 4. Updated (written) by CPU again 1549 * 5. Read by GPU 1550 * 1551 * 1. Allocated 1552 * (CPU, CPU) 1553 * 2. Written by CPU 1554 * (CPU, CPU) 1555 * 3. Read by GPU 1556 * (CPU+RENDER, 0) 1557 * flush_domains = CPU 1558 * invalidate_domains = RENDER 1559 * clflush (obj) 1560 * MI_FLUSH 1561 * drm_agp_chipset_flush 1562 * 4. Updated (written) by CPU again 1563 * (CPU, CPU) 1564 * flush_domains = 0 (no previous write domain) 1565 * invalidate_domains = 0 (no new read domains) 1566 * 5. Read by GPU 1567 * (CPU+RENDER, 0) 1568 * flush_domains = CPU 1569 * invalidate_domains = RENDER 1570 * clflush (obj) 1571 * MI_FLUSH 1572 * drm_agp_chipset_flush 1573 */ 1574 static void 1575 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj, 1576 uint32_t read_domains, 1577 uint32_t write_domain) 1578 { 1579 struct drm_device *dev = obj->dev; 1580 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1581 uint32_t invalidate_domains = 0; 1582 uint32_t flush_domains = 0; 1583 1584 DRM_DEBUG("%s: object %p read %08x -> %08x write %08x -> %08x\n", 1585 __func__, obj, 1586 obj->read_domains, read_domains, 1587 obj->write_domain, write_domain); 1588 /* 1589 * If the object isn't moving to a new write domain, 1590 * let the object stay in multiple read domains 1591 */ 1592 if (write_domain == 0) 1593 read_domains |= obj->read_domains; 1594 else 1595 obj_priv->dirty = 1; 1596 1597 /* 1598 * Flush the current write domain if 1599 * the new read domains don't match. Invalidate 1600 * any read domains which differ from the old 1601 * write domain 1602 */ 1603 if (obj->write_domain && obj->write_domain != read_domains) { 1604 flush_domains |= obj->write_domain; 1605 invalidate_domains |= read_domains & ~obj->write_domain; 1606 } 1607 /* 1608 * Invalidate any read caches which may have 1609 * stale data. That is, any new read domains. 1610 */ 1611 invalidate_domains |= read_domains & ~obj->read_domains; 1612 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) { 1613 DRM_DEBUG("%s: CPU domain flush %08x invalidate %08x\n", 1614 __func__, flush_domains, invalidate_domains); 1615 i915_gem_clflush_object(obj); 1616 } 1617 1618 if ((write_domain | flush_domains) != 0) 1619 obj->write_domain = write_domain; 1620 obj->read_domains = read_domains; 1621 1622 dev->invalidate_domains |= invalidate_domains; 1623 dev->flush_domains |= flush_domains; 1624 1625 DRM_DEBUG("%s: read %08x write %08x invalidate %08x flush %08x\n", 1626 __func__, 1627 obj->read_domains, obj->write_domain, 1628 dev->invalidate_domains, dev->flush_domains); 1629 1630 } 1631 1632 /** 1633 * Moves the object from a partially CPU read to a full one. 1634 * 1635 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(), 1636 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 1637 */ 1638 static void 1639 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 1640 { 1641 struct drm_device *dev = obj->dev; 1642 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1643 1644 if (!obj_priv->page_cpu_valid) 1645 return; 1646 1647 /* If we're partially in the CPU read domain, finish moving it in. 1648 */ 1649 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 1650 int i; 1651 1652 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 1653 if (obj_priv->page_cpu_valid[i]) 1654 continue; 1655 drm_clflush_pages(obj_priv->page_list + i, 1); 1656 } 1657 drm_agp_chipset_flush(dev); 1658 } 1659 1660 /* Free the page_cpu_valid mappings which are now stale, whether 1661 * or not we've got I915_GEM_DOMAIN_CPU. 1662 */ 1663 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE, 1664 DRM_MEM_DRIVER); 1665 obj_priv->page_cpu_valid = NULL; 1666 } 1667 1668 /** 1669 * Set the CPU read domain on a range of the object. 1670 * 1671 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's 1672 * not entirely valid. The page_cpu_valid member of the object flags which 1673 * pages have been flushed, and will be respected by 1674 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping 1675 * of the whole object. 1676 * 1677 * This function returns when the move is complete, including waiting on 1678 * flushes to occur. 1679 */ 1680 static int 1681 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 1682 uint64_t offset, uint64_t size) 1683 { 1684 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1685 int i, ret; 1686 1687 if (offset == 0 && size == obj->size) 1688 return i915_gem_object_set_to_cpu_domain(obj, 0); 1689 1690 i915_gem_object_flush_gpu_write_domain(obj); 1691 /* Wait on any GPU rendering and flushing to occur. */ 1692 ret = i915_gem_object_wait_rendering(obj); 1693 if (ret != 0) 1694 return ret; 1695 i915_gem_object_flush_gtt_write_domain(obj); 1696 1697 /* If we're already fully in the CPU read domain, we're done. */ 1698 if (obj_priv->page_cpu_valid == NULL && 1699 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 1700 return 0; 1701 1702 /* Otherwise, create/clear the per-page CPU read domain flag if we're 1703 * newly adding I915_GEM_DOMAIN_CPU 1704 */ 1705 if (obj_priv->page_cpu_valid == NULL) { 1706 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE, 1707 DRM_MEM_DRIVER); 1708 if (obj_priv->page_cpu_valid == NULL) 1709 return ENOMEM; 1710 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 1711 (void) memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 1712 1713 /* Flush the cache on any pages that are still invalid from the CPU's 1714 * perspective. 1715 */ 1716 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1717 i++) { 1718 if (obj_priv->page_cpu_valid[i]) 1719 continue; 1720 1721 drm_clflush_pages(obj_priv->page_list + i, 1); 1722 obj_priv->page_cpu_valid[i] = 1; 1723 } 1724 1725 /* It should now be out of any other write domains, and we can update 1726 * the domain values for our changes. 1727 */ 1728 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0)); 1729 1730 obj->read_domains |= I915_GEM_DOMAIN_CPU; 1731 1732 return 0; 1733 } 1734 1735 /** 1736 * Pin an object to the GTT and evaluate the relocations landing in it. 1737 */ 1738 static int 1739 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, 1740 struct drm_file *file_priv, 1741 struct drm_i915_gem_exec_object *entry) 1742 { 1743 struct drm_i915_gem_relocation_entry reloc; 1744 struct drm_i915_gem_relocation_entry __user *relocs; 1745 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1746 int i, ret; 1747 1748 /* Choose the GTT offset for our buffer and put it there. */ 1749 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment); 1750 if (ret) { 1751 DRM_ERROR("failed to pin"); 1752 return ret; 1753 } 1754 entry->offset = obj_priv->gtt_offset; 1755 1756 relocs = (struct drm_i915_gem_relocation_entry __user *) 1757 (uintptr_t) entry->relocs_ptr; 1758 /* Apply the relocations, using the GTT aperture to avoid cache 1759 * flushing requirements. 1760 */ 1761 for (i = 0; i < entry->relocation_count; i++) { 1762 struct drm_gem_object *target_obj; 1763 struct drm_i915_gem_object *target_obj_priv; 1764 uint32_t reloc_val, reloc_offset, *reloc_entry; 1765 1766 ret = DRM_COPY_FROM_USER(&reloc, relocs + i, sizeof(reloc)); 1767 if (ret != 0) { 1768 i915_gem_object_unpin(obj); 1769 DRM_ERROR("failed to copy from user"); 1770 return ret; 1771 } 1772 1773 target_obj = drm_gem_object_lookup(file_priv, 1774 reloc.target_handle); 1775 if (target_obj == NULL) { 1776 i915_gem_object_unpin(obj); 1777 return EBADF; 1778 } 1779 target_obj_priv = target_obj->driver_private; 1780 1781 /* The target buffer should have appeared before us in the 1782 * exec_object list, so it should have a GTT space bound by now. 1783 */ 1784 if (target_obj_priv->gtt_space == NULL) { 1785 DRM_ERROR("No GTT space found for object %d\n", 1786 reloc.target_handle); 1787 drm_gem_object_unreference(target_obj); 1788 i915_gem_object_unpin(obj); 1789 return EINVAL; 1790 } 1791 1792 if (reloc.offset > obj->size - 4) { 1793 DRM_ERROR("Relocation beyond object bounds: " 1794 "obj %p target %d offset %d size %d.\n", 1795 obj, reloc.target_handle, 1796 (int) reloc.offset, (int) obj->size); 1797 drm_gem_object_unreference(target_obj); 1798 i915_gem_object_unpin(obj); 1799 return EINVAL; 1800 } 1801 if (reloc.offset & 3) { 1802 DRM_ERROR("Relocation not 4-byte aligned: " 1803 "obj %p target %d offset %d.\n", 1804 obj, reloc.target_handle, 1805 (int) reloc.offset); 1806 drm_gem_object_unreference(target_obj); 1807 i915_gem_object_unpin(obj); 1808 return EINVAL; 1809 } 1810 1811 if (reloc.write_domain & I915_GEM_DOMAIN_CPU || 1812 reloc.read_domains & I915_GEM_DOMAIN_CPU) { 1813 DRM_ERROR("reloc with read/write CPU domains: " 1814 "obj %p target %d offset %d " 1815 "read %08x write %08x", 1816 obj, reloc.target_handle, 1817 (int) reloc.offset, 1818 reloc.read_domains, 1819 reloc.write_domain); 1820 drm_gem_object_unreference(target_obj); 1821 i915_gem_object_unpin(obj); 1822 return EINVAL; 1823 } 1824 1825 if (reloc.write_domain && target_obj->pending_write_domain && 1826 reloc.write_domain != target_obj->pending_write_domain) { 1827 DRM_ERROR("Write domain conflict: " 1828 "obj %p target %d offset %d " 1829 "new %08x old %08x\n", 1830 obj, reloc.target_handle, 1831 (int) reloc.offset, 1832 reloc.write_domain, 1833 target_obj->pending_write_domain); 1834 drm_gem_object_unreference(target_obj); 1835 i915_gem_object_unpin(obj); 1836 return EINVAL; 1837 } 1838 DRM_DEBUG("%s: obj %p offset %08x target %d " 1839 "read %08x write %08x gtt %08x " 1840 "presumed %08x delta %08x\n", 1841 __func__, 1842 obj, 1843 (int) reloc.offset, 1844 (int) reloc.target_handle, 1845 (int) reloc.read_domains, 1846 (int) reloc.write_domain, 1847 (int) target_obj_priv->gtt_offset, 1848 (int) reloc.presumed_offset, 1849 reloc.delta); 1850 1851 target_obj->pending_read_domains |= reloc.read_domains; 1852 target_obj->pending_write_domain |= reloc.write_domain; 1853 1854 /* If the relocation already has the right value in it, no 1855 * more work needs to be done. 1856 */ 1857 if (target_obj_priv->gtt_offset == reloc.presumed_offset) { 1858 drm_gem_object_unreference(target_obj); 1859 continue; 1860 } 1861 1862 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 1863 if (ret != 0) { 1864 drm_gem_object_unreference(target_obj); 1865 i915_gem_object_unpin(obj); 1866 return EINVAL; 1867 } 1868 1869 /* Map the page containing the relocation we're going to 1870 * perform. 1871 */ 1872 1873 int reloc_base = (reloc.offset & ~(PAGE_SIZE-1)); 1874 reloc_offset = reloc.offset & (PAGE_SIZE-1); 1875 reloc_entry = (uint32_t *)(uintptr_t)(obj_priv->page_list[reloc_base/PAGE_SIZE] + reloc_offset); 1876 reloc_val = target_obj_priv->gtt_offset + reloc.delta; 1877 *reloc_entry = reloc_val; 1878 1879 /* Write the updated presumed offset for this entry back out 1880 * to the user. 1881 */ 1882 reloc.presumed_offset = target_obj_priv->gtt_offset; 1883 ret = DRM_COPY_TO_USER(relocs + i, &reloc, sizeof(reloc)); 1884 if (ret != 0) { 1885 drm_gem_object_unreference(target_obj); 1886 i915_gem_object_unpin(obj); 1887 DRM_ERROR("%s: Failed to copy to user ret %d", __func__, ret); 1888 return ret; 1889 } 1890 1891 drm_gem_object_unreference(target_obj); 1892 } 1893 1894 return 0; 1895 } 1896 1897 /** Dispatch a batchbuffer to the ring 1898 */ 1899 static int 1900 i915_dispatch_gem_execbuffer(struct drm_device *dev, 1901 struct drm_i915_gem_execbuffer *exec, 1902 uint64_t exec_offset) 1903 { 1904 drm_i915_private_t *dev_priv = dev->dev_private; 1905 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *) 1906 (uintptr_t) exec->cliprects_ptr; 1907 int nbox = exec->num_cliprects; 1908 int i = 0, count; 1909 uint64_t exec_start, exec_len; 1910 RING_LOCALS; 1911 1912 exec_start = exec_offset + exec->batch_start_offset; 1913 exec_len = exec->batch_len; 1914 1915 if ((exec_start | exec_len) & 0x7) { 1916 DRM_ERROR("alignment\n"); 1917 return EINVAL; 1918 } 1919 1920 if (!exec_start) { 1921 DRM_ERROR("wrong arg"); 1922 return EINVAL; 1923 } 1924 1925 count = nbox ? nbox : 1; 1926 1927 for (i = 0; i < count; i++) { 1928 if (i < nbox) { 1929 int ret = i915_emit_box(dev, boxes, i, 1930 exec->DR1, exec->DR4); 1931 if (ret) { 1932 DRM_ERROR("i915_emit_box %d DR1 0x%lx DRI2 0x%lx", ret, exec->DR1, exec->DR4); 1933 return ret; 1934 } 1935 } 1936 if (IS_I830(dev) || IS_845G(dev)) { 1937 BEGIN_LP_RING(4); 1938 OUT_RING(MI_BATCH_BUFFER); 1939 OUT_RING(exec_start | MI_BATCH_NON_SECURE); 1940 OUT_RING(exec_start + exec_len - 4); 1941 OUT_RING(0); 1942 ADVANCE_LP_RING(); 1943 } else { 1944 BEGIN_LP_RING(2); 1945 if (IS_I965G(dev)) { 1946 OUT_RING(MI_BATCH_BUFFER_START | 1947 (2 << 6) | 1948 (3 << 9) | 1949 MI_BATCH_NON_SECURE_I965); 1950 OUT_RING(exec_start); 1951 1952 } else { 1953 OUT_RING(MI_BATCH_BUFFER_START | 1954 (2 << 6)); 1955 OUT_RING(exec_start | MI_BATCH_NON_SECURE); 1956 } 1957 ADVANCE_LP_RING(); 1958 } 1959 } 1960 /* XXX breadcrumb */ 1961 return 0; 1962 } 1963 1964 /* Throttle our rendering by waiting until the ring has completed our requests 1965 * emitted over 20 msec ago. 1966 * 1967 * This should get us reasonable parallelism between CPU and GPU but also 1968 * relatively low latency when blocking on a particular request to finish. 1969 */ 1970 static int 1971 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) 1972 { 1973 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; 1974 int ret = 0; 1975 uint32_t seqno; 1976 1977 spin_lock(&dev->struct_mutex); 1978 seqno = i915_file_priv->mm.last_gem_throttle_seqno; 1979 i915_file_priv->mm.last_gem_throttle_seqno = 1980 i915_file_priv->mm.last_gem_seqno; 1981 if (seqno) { 1982 ret = i915_wait_request(dev, seqno); 1983 if (ret != 0) 1984 DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev)); 1985 } 1986 spin_unlock(&dev->struct_mutex); 1987 return ret; 1988 } 1989 1990 /*ARGSUSED*/ 1991 int 1992 i915_gem_execbuffer(DRM_IOCTL_ARGS) 1993 { 1994 DRM_DEVICE; 1995 drm_i915_private_t *dev_priv = dev->dev_private; 1996 struct drm_i915_file_private *i915_file_priv = fpriv->driver_priv; 1997 struct drm_i915_gem_execbuffer args; 1998 struct drm_i915_gem_exec_object *exec_list = NULL; 1999 struct drm_gem_object **object_list = NULL; 2000 struct drm_gem_object *batch_obj; 2001 struct drm_i915_gem_object *obj_priv; 2002 int ret = 0, i, pinned = 0; 2003 uint64_t exec_offset; 2004 uint32_t seqno, flush_domains; 2005 int pin_tries; 2006 2007 if (dev->driver->use_gem != 1) 2008 return ENODEV; 2009 2010 DRM_COPYFROM_WITH_RETURN(&args, 2011 (struct drm_i915_gem_execbuffer __user *) data, sizeof(args)); 2012 2013 DRM_DEBUG("buffer_count %d len %x\n", args.buffer_count, args.batch_len); 2014 2015 if (args.buffer_count < 1) { 2016 DRM_ERROR("execbuf with %d buffers\n", args.buffer_count); 2017 return EINVAL; 2018 } 2019 /* Copy in the exec list from userland */ 2020 exec_list = drm_calloc(sizeof(*exec_list), args.buffer_count, 2021 DRM_MEM_DRIVER); 2022 object_list = drm_calloc(sizeof(*object_list), args.buffer_count, 2023 DRM_MEM_DRIVER); 2024 if (exec_list == NULL || object_list == NULL) { 2025 DRM_ERROR("Failed to allocate exec or object list " 2026 "for %d buffers\n", 2027 args.buffer_count); 2028 ret = ENOMEM; 2029 goto pre_mutex_err; 2030 } 2031 2032 ret = DRM_COPY_FROM_USER(exec_list, 2033 (struct drm_i915_gem_exec_object __user *) 2034 (uintptr_t) args.buffers_ptr, 2035 sizeof(*exec_list) * args.buffer_count); 2036 if (ret != 0) { 2037 DRM_ERROR("copy %d exec entries failed %d\n", 2038 args.buffer_count, ret); 2039 goto pre_mutex_err; 2040 } 2041 spin_lock(&dev->struct_mutex); 2042 2043 if (dev_priv->mm.wedged) { 2044 DRM_ERROR("Execbuf while wedged\n"); 2045 spin_unlock(&dev->struct_mutex); 2046 return EIO; 2047 } 2048 2049 if (dev_priv->mm.suspended) { 2050 DRM_ERROR("Execbuf while VT-switched.\n"); 2051 spin_unlock(&dev->struct_mutex); 2052 return EBUSY; 2053 } 2054 2055 /* Look up object handles */ 2056 for (i = 0; i < args.buffer_count; i++) { 2057 object_list[i] = drm_gem_object_lookup(fpriv, 2058 exec_list[i].handle); 2059 if (object_list[i] == NULL) { 2060 DRM_ERROR("Invalid object handle %d at index %d\n", 2061 exec_list[i].handle, i); 2062 ret = EBADF; 2063 goto err; 2064 } 2065 obj_priv = object_list[i]->driver_private; 2066 if (obj_priv->in_execbuffer) { 2067 DRM_ERROR("Object[%d] (%d) %p appears more than once in object list in args.buffer_count %d \n", 2068 i, object_list[i]->name, object_list[i], args.buffer_count); 2069 2070 ret = EBADF; 2071 goto err; 2072 } 2073 2074 obj_priv->in_execbuffer = 1; 2075 } 2076 2077 /* Pin and relocate */ 2078 for (pin_tries = 0; ; pin_tries++) { 2079 ret = 0; 2080 for (i = 0; i < args.buffer_count; i++) { 2081 object_list[i]->pending_read_domains = 0; 2082 object_list[i]->pending_write_domain = 0; 2083 ret = i915_gem_object_pin_and_relocate(object_list[i], 2084 fpriv, 2085 &exec_list[i]); 2086 if (ret) { 2087 DRM_ERROR("Not all object pinned"); 2088 break; 2089 } 2090 pinned = i + 1; 2091 } 2092 /* success */ 2093 if (ret == 0) 2094 { 2095 DRM_DEBUG("gem_execbuffer pin_relocate success"); 2096 break; 2097 } 2098 /* error other than GTT full, or we've already tried again */ 2099 if (ret != ENOMEM || pin_tries >= 1) { 2100 if (ret != ERESTART) 2101 DRM_ERROR("Failed to pin buffers %d\n", ret); 2102 goto err; 2103 } 2104 2105 /* unpin all of our buffers */ 2106 for (i = 0; i < pinned; i++) 2107 i915_gem_object_unpin(object_list[i]); 2108 pinned = 0; 2109 2110 /* evict everyone we can from the aperture */ 2111 ret = i915_gem_evict_everything(dev); 2112 if (ret) 2113 goto err; 2114 } 2115 2116 /* Set the pending read domains for the batch buffer to COMMAND */ 2117 batch_obj = object_list[args.buffer_count-1]; 2118 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND; 2119 batch_obj->pending_write_domain = 0; 2120 2121 /* Zero the gloabl flush/invalidate flags. These 2122 * will be modified as each object is bound to the 2123 * gtt 2124 */ 2125 dev->invalidate_domains = 0; 2126 dev->flush_domains = 0; 2127 2128 for (i = 0; i < args.buffer_count; i++) { 2129 struct drm_gem_object *obj = object_list[i]; 2130 2131 /* Compute new gpu domains and update invalidate/flush */ 2132 i915_gem_object_set_to_gpu_domain(obj, 2133 obj->pending_read_domains, 2134 obj->pending_write_domain); 2135 } 2136 2137 if (dev->invalidate_domains | dev->flush_domains) { 2138 2139 DRM_DEBUG("%s: invalidate_domains %08x flush_domains %08x Then flush\n", 2140 __func__, 2141 dev->invalidate_domains, 2142 dev->flush_domains); 2143 i915_gem_flush(dev, 2144 dev->invalidate_domains, 2145 dev->flush_domains); 2146 if (dev->flush_domains) { 2147 (void) i915_add_request(dev, dev->flush_domains); 2148 2149 } 2150 } 2151 2152 for (i = 0; i < args.buffer_count; i++) { 2153 struct drm_gem_object *obj = object_list[i]; 2154 2155 obj->write_domain = obj->pending_write_domain; 2156 } 2157 2158 exec_offset = exec_list[args.buffer_count - 1].offset; 2159 2160 /* Exec the batchbuffer */ 2161 ret = i915_dispatch_gem_execbuffer(dev, &args, exec_offset); 2162 if (ret) { 2163 DRM_ERROR("dispatch failed %d\n", ret); 2164 goto err; 2165 } 2166 2167 /* 2168 * Ensure that the commands in the batch buffer are 2169 * finished before the interrupt fires 2170 */ 2171 flush_domains = i915_retire_commands(dev); 2172 2173 /* 2174 * Get a seqno representing the execution of the current buffer, 2175 * which we can wait on. We would like to mitigate these interrupts, 2176 * likely by only creating seqnos occasionally (so that we have 2177 * *some* interrupts representing completion of buffers that we can 2178 * wait on when trying to clear up gtt space). 2179 */ 2180 seqno = i915_add_request(dev, flush_domains); 2181 ASSERT(!(seqno == 0)); 2182 i915_file_priv->mm.last_gem_seqno = seqno; 2183 for (i = 0; i < args.buffer_count; i++) { 2184 struct drm_gem_object *obj = object_list[i]; 2185 i915_gem_object_move_to_active(obj, seqno); 2186 DRM_DEBUG("%s: move to exec list %p\n", __func__, obj); 2187 } 2188 2189 err: 2190 if (object_list != NULL) { 2191 for (i = 0; i < pinned; i++) 2192 i915_gem_object_unpin(object_list[i]); 2193 2194 for (i = 0; i < args.buffer_count; i++) { 2195 if (object_list[i]) { 2196 obj_priv = object_list[i]->driver_private; 2197 obj_priv->in_execbuffer = 0; 2198 } 2199 drm_gem_object_unreference(object_list[i]); 2200 } 2201 } 2202 spin_unlock(&dev->struct_mutex); 2203 2204 if (!ret) { 2205 /* Copy the new buffer offsets back to the user's exec list. */ 2206 ret = DRM_COPY_TO_USER((struct drm_i915_relocation_entry __user *) 2207 (uintptr_t) args.buffers_ptr, 2208 exec_list, 2209 sizeof(*exec_list) * args.buffer_count); 2210 if (ret) 2211 DRM_ERROR("failed to copy %d exec entries " 2212 "back to user (%d)\n", 2213 args.buffer_count, ret); 2214 } 2215 2216 pre_mutex_err: 2217 drm_free(object_list, sizeof(*object_list) * args.buffer_count, 2218 DRM_MEM_DRIVER); 2219 drm_free(exec_list, sizeof(*exec_list) * args.buffer_count, 2220 DRM_MEM_DRIVER); 2221 2222 return ret; 2223 } 2224 2225 int 2226 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 2227 { 2228 struct drm_device *dev = obj->dev; 2229 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2230 int ret; 2231 2232 if (obj_priv->gtt_space == NULL) { 2233 ret = i915_gem_object_bind_to_gtt(obj, alignment); 2234 if (ret != 0) { 2235 DRM_ERROR("Failure to bind: %d", ret); 2236 return ret; 2237 } 2238 } 2239 obj_priv->pin_count++; 2240 2241 /* If the object is not active and not pending a flush, 2242 * remove it from the inactive list 2243 */ 2244 if (obj_priv->pin_count == 1) { 2245 atomic_inc(&dev->pin_count); 2246 atomic_add(obj->size, &dev->pin_memory); 2247 if (!obj_priv->active && 2248 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | 2249 I915_GEM_DOMAIN_GTT)) == 0 && 2250 !list_empty(&obj_priv->list)) 2251 list_del_init(&obj_priv->list); 2252 } 2253 return 0; 2254 } 2255 2256 void 2257 i915_gem_object_unpin(struct drm_gem_object *obj) 2258 { 2259 struct drm_device *dev = obj->dev; 2260 drm_i915_private_t *dev_priv = dev->dev_private; 2261 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2262 obj_priv->pin_count--; 2263 ASSERT(!(obj_priv->pin_count < 0)); 2264 ASSERT(!(obj_priv->gtt_space == NULL)); 2265 2266 /* If the object is no longer pinned, and is 2267 * neither active nor being flushed, then stick it on 2268 * the inactive list 2269 */ 2270 if (obj_priv->pin_count == 0) { 2271 if (!obj_priv->active && 2272 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | 2273 I915_GEM_DOMAIN_GTT)) == 0) 2274 list_move_tail(&obj_priv->list, 2275 &dev_priv->mm.inactive_list, (caddr_t)obj_priv); 2276 atomic_dec(&dev->pin_count); 2277 atomic_sub(obj->size, &dev->pin_memory); 2278 } 2279 } 2280 2281 /*ARGSUSED*/ 2282 int 2283 i915_gem_pin_ioctl(DRM_IOCTL_ARGS) 2284 { 2285 DRM_DEVICE; 2286 struct drm_i915_gem_pin args; 2287 struct drm_gem_object *obj; 2288 struct drm_i915_gem_object *obj_priv; 2289 int ret; 2290 2291 if (dev->driver->use_gem != 1) 2292 return ENODEV; 2293 2294 DRM_COPYFROM_WITH_RETURN(&args, 2295 (struct drm_i915_gem_pin __user *) data, sizeof(args)); 2296 2297 spin_lock(&dev->struct_mutex); 2298 2299 obj = drm_gem_object_lookup(fpriv, args.handle); 2300 if (obj == NULL) { 2301 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", 2302 args.handle); 2303 spin_unlock(&dev->struct_mutex); 2304 return EBADF; 2305 } 2306 DRM_DEBUG("i915_gem_pin_ioctl obj->name %d", obj->name); 2307 obj_priv = obj->driver_private; 2308 2309 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != fpriv) { 2310 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 2311 args.handle); 2312 drm_gem_object_unreference(obj); 2313 spin_unlock(&dev->struct_mutex); 2314 return EINVAL; 2315 } 2316 2317 obj_priv->user_pin_count++; 2318 obj_priv->pin_filp = fpriv; 2319 if (obj_priv->user_pin_count == 1) { 2320 ret = i915_gem_object_pin(obj, args.alignment); 2321 if (ret != 0) { 2322 drm_gem_object_unreference(obj); 2323 spin_unlock(&dev->struct_mutex); 2324 return ret; 2325 } 2326 } 2327 2328 /* XXX - flush the CPU caches for pinned objects 2329 * as the X server doesn't manage domains yet 2330 */ 2331 i915_gem_object_flush_cpu_write_domain(obj); 2332 args.offset = obj_priv->gtt_offset; 2333 2334 ret = DRM_COPY_TO_USER((struct drm_i915_gem_pin __user *) data, &args, sizeof(args)); 2335 if ( ret != 0) 2336 DRM_ERROR(" gem pin ioctl error! %d", ret); 2337 2338 drm_gem_object_unreference(obj); 2339 spin_unlock(&dev->struct_mutex); 2340 2341 return 0; 2342 } 2343 2344 /*ARGSUSED*/ 2345 int 2346 i915_gem_unpin_ioctl(DRM_IOCTL_ARGS) 2347 { 2348 DRM_DEVICE; 2349 struct drm_i915_gem_pin args; 2350 struct drm_gem_object *obj; 2351 struct drm_i915_gem_object *obj_priv; 2352 2353 if (dev->driver->use_gem != 1) 2354 return ENODEV; 2355 2356 DRM_COPYFROM_WITH_RETURN(&args, 2357 (struct drm_i915_gem_pin __user *) data, sizeof(args)); 2358 2359 spin_lock(&dev->struct_mutex); 2360 2361 obj = drm_gem_object_lookup(fpriv, args.handle); 2362 if (obj == NULL) { 2363 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", 2364 args.handle); 2365 spin_unlock(&dev->struct_mutex); 2366 return EBADF; 2367 } 2368 obj_priv = obj->driver_private; 2369 DRM_DEBUG("i915_gem_unpin_ioctl, obj->name %d", obj->name); 2370 if (obj_priv->pin_filp != fpriv) { 2371 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 2372 args.handle); 2373 drm_gem_object_unreference(obj); 2374 spin_unlock(&dev->struct_mutex); 2375 return EINVAL; 2376 } 2377 obj_priv->user_pin_count--; 2378 if (obj_priv->user_pin_count == 0) { 2379 obj_priv->pin_filp = NULL; 2380 i915_gem_object_unpin(obj); 2381 } 2382 drm_gem_object_unreference(obj); 2383 spin_unlock(&dev->struct_mutex); 2384 return 0; 2385 } 2386 2387 /*ARGSUSED*/ 2388 int 2389 i915_gem_busy_ioctl(DRM_IOCTL_ARGS) 2390 { 2391 DRM_DEVICE; 2392 struct drm_i915_gem_busy args; 2393 struct drm_gem_object *obj; 2394 struct drm_i915_gem_object *obj_priv; 2395 int ret; 2396 2397 if (dev->driver->use_gem != 1) 2398 return ENODEV; 2399 2400 DRM_COPYFROM_WITH_RETURN(&args, 2401 (struct drm_i915_gem_busy __user *) data, sizeof(args)); 2402 2403 spin_lock(&dev->struct_mutex); 2404 obj = drm_gem_object_lookup(fpriv, args.handle); 2405 if (obj == NULL) { 2406 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n", 2407 args.handle); 2408 spin_unlock(&dev->struct_mutex); 2409 return EBADF; 2410 } 2411 2412 obj_priv = obj->driver_private; 2413 /* Don't count being on the flushing list against the object being 2414 * done. Otherwise, a buffer left on the flushing list but not getting 2415 * flushed (because nobody's flushing that domain) won't ever return 2416 * unbusy and get reused by libdrm's bo cache. The other expected 2417 * consumer of this interface, OpenGL's occlusion queries, also specs 2418 * that the objects get unbusy "eventually" without any interference. 2419 */ 2420 args.busy = obj_priv->active && obj_priv->last_rendering_seqno != 0; 2421 DRM_DEBUG("i915_gem_busy_ioctl call obj->name %d busy %d", obj->name, args.busy); 2422 2423 ret = DRM_COPY_TO_USER((struct drm_i915_gem_busy __user *) data, &args, sizeof(args)); 2424 if ( ret != 0) 2425 DRM_ERROR(" gem busy error! %d", ret); 2426 2427 drm_gem_object_unreference(obj); 2428 spin_unlock(&dev->struct_mutex); 2429 return 0; 2430 } 2431 2432 /*ARGSUSED*/ 2433 int 2434 i915_gem_throttle_ioctl(DRM_IOCTL_ARGS) 2435 { 2436 DRM_DEVICE; 2437 2438 if (dev->driver->use_gem != 1) 2439 return ENODEV; 2440 2441 return i915_gem_ring_throttle(dev, fpriv); 2442 } 2443 2444 static int 2445 i915_gem_object_get_page_list(struct drm_gem_object *obj) 2446 { 2447 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2448 caddr_t va; 2449 long i; 2450 2451 if (obj_priv->page_list) 2452 return 0; 2453 pgcnt_t np = btop(obj->size); 2454 2455 obj_priv->page_list = kmem_zalloc(np * sizeof(caddr_t), KM_SLEEP); 2456 if (obj_priv->page_list == NULL) { 2457 DRM_ERROR("Faled to allocate page list\n"); 2458 return ENOMEM; 2459 } 2460 2461 for (i = 0, va = obj->kaddr; i < np; i++, va += PAGESIZE) { 2462 obj_priv->page_list[i] = va; 2463 } 2464 return 0; 2465 } 2466 2467 2468 int i915_gem_init_object(struct drm_gem_object *obj) 2469 { 2470 struct drm_i915_gem_object *obj_priv; 2471 2472 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER); 2473 if (obj_priv == NULL) 2474 return ENOMEM; 2475 2476 /* 2477 * We've just allocated pages from the kernel, 2478 * so they've just been written by the CPU with 2479 * zeros. They'll need to be clflushed before we 2480 * use them with the GPU. 2481 */ 2482 obj->write_domain = I915_GEM_DOMAIN_CPU; 2483 obj->read_domains = I915_GEM_DOMAIN_CPU; 2484 2485 obj->driver_private = obj_priv; 2486 obj_priv->obj = obj; 2487 INIT_LIST_HEAD(&obj_priv->list); 2488 return 0; 2489 } 2490 2491 void i915_gem_free_object(struct drm_gem_object *obj) 2492 { 2493 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2494 2495 while (obj_priv->pin_count > 0) 2496 i915_gem_object_unpin(obj); 2497 2498 DRM_DEBUG("%s: obj %d",__func__, obj->name); 2499 2500 (void) i915_gem_object_unbind(obj, 1); 2501 if (obj_priv->page_cpu_valid != NULL) 2502 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE, DRM_MEM_DRIVER); 2503 drm_free(obj->driver_private, sizeof(*obj_priv), DRM_MEM_DRIVER); 2504 } 2505 2506 /** Unbinds all objects that are on the given buffer list. */ 2507 static int 2508 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head, uint32_t type) 2509 { 2510 struct drm_gem_object *obj; 2511 struct drm_i915_gem_object *obj_priv; 2512 int ret; 2513 2514 while (!list_empty(head)) { 2515 obj_priv = list_entry(head->next, 2516 struct drm_i915_gem_object, 2517 list); 2518 obj = obj_priv->obj; 2519 2520 if (obj_priv->pin_count != 0) { 2521 DRM_ERROR("Pinned object in unbind list\n"); 2522 spin_unlock(&dev->struct_mutex); 2523 return EINVAL; 2524 } 2525 DRM_DEBUG("%s: obj %d type %d",__func__, obj->name, type); 2526 ret = i915_gem_object_unbind(obj, type); 2527 if (ret != 0) { 2528 DRM_ERROR("Error unbinding object in LeaveVT: %d\n", 2529 ret); 2530 spin_unlock(&dev->struct_mutex); 2531 return ret; 2532 } 2533 } 2534 2535 2536 return 0; 2537 } 2538 2539 static int 2540 i915_gem_idle(struct drm_device *dev, uint32_t type) 2541 { 2542 drm_i915_private_t *dev_priv = dev->dev_private; 2543 uint32_t seqno, cur_seqno, last_seqno; 2544 int stuck, ret; 2545 2546 spin_lock(&dev->struct_mutex); 2547 2548 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) { 2549 spin_unlock(&dev->struct_mutex); 2550 return 0; 2551 } 2552 2553 /* Hack! Don't let anybody do execbuf while we don't control the chip. 2554 * We need to replace this with a semaphore, or something. 2555 */ 2556 dev_priv->mm.suspended = 1; 2557 2558 /* Cancel the retire work handler, wait for it to finish if running 2559 */ 2560 if (worktimer_id != NULL) { 2561 (void) untimeout(worktimer_id); 2562 worktimer_id = NULL; 2563 } 2564 2565 i915_kernel_lost_context(dev); 2566 2567 /* Flush the GPU along with all non-CPU write domains 2568 */ 2569 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT), 2570 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); 2571 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU | 2572 I915_GEM_DOMAIN_GTT)); 2573 if (seqno == 0) { 2574 spin_unlock(&dev->struct_mutex); 2575 return ENOMEM; 2576 } 2577 2578 dev_priv->mm.waiting_gem_seqno = seqno; 2579 last_seqno = 0; 2580 stuck = 0; 2581 for (;;) { 2582 cur_seqno = i915_get_gem_seqno(dev); 2583 if (i915_seqno_passed(cur_seqno, seqno)) 2584 break; 2585 if (last_seqno == cur_seqno) { 2586 if (stuck++ > 100) { 2587 DRM_ERROR("hardware wedged\n"); 2588 dev_priv->mm.wedged = 1; 2589 DRM_WAKEUP(&dev_priv->irq_queue); 2590 break; 2591 } 2592 } 2593 DRM_UDELAY(10); 2594 last_seqno = cur_seqno; 2595 } 2596 dev_priv->mm.waiting_gem_seqno = 0; 2597 2598 i915_gem_retire_requests(dev); 2599 2600 /* Empty the active and flushing lists to inactive. If there's 2601 * anything left at this point, it means that we're wedged and 2602 * nothing good's going to happen by leaving them there. So strip 2603 * the GPU domains and just stuff them onto inactive. 2604 */ 2605 while (!list_empty(&dev_priv->mm.active_list)) { 2606 struct drm_i915_gem_object *obj_priv; 2607 2608 obj_priv = list_entry(dev_priv->mm.active_list.next, 2609 struct drm_i915_gem_object, 2610 list); 2611 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS; 2612 i915_gem_object_move_to_inactive(obj_priv->obj); 2613 } 2614 2615 while (!list_empty(&dev_priv->mm.flushing_list)) { 2616 struct drm_i915_gem_object *obj_priv; 2617 2618 obj_priv = list_entry(dev_priv->mm.flushing_list.next, 2619 struct drm_i915_gem_object, 2620 list); 2621 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS; 2622 i915_gem_object_move_to_inactive(obj_priv->obj); 2623 } 2624 2625 /* Move all inactive buffers out of the GTT. */ 2626 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list, type); 2627 ASSERT(list_empty(&dev_priv->mm.inactive_list)); 2628 if (ret) { 2629 spin_unlock(&dev->struct_mutex); 2630 return ret; 2631 } 2632 2633 i915_gem_cleanup_ringbuffer(dev); 2634 spin_unlock(&dev->struct_mutex); 2635 2636 return 0; 2637 } 2638 2639 static int 2640 i915_gem_init_hws(struct drm_device *dev) 2641 { 2642 drm_i915_private_t *dev_priv = dev->dev_private; 2643 struct drm_gem_object *obj; 2644 struct drm_i915_gem_object *obj_priv; 2645 int ret; 2646 2647 /* If we need a physical address for the status page, it's already 2648 * initialized at driver load time. 2649 */ 2650 if (!I915_NEED_GFX_HWS(dev)) 2651 return 0; 2652 2653 2654 obj = drm_gem_object_alloc(dev, 4096); 2655 if (obj == NULL) { 2656 DRM_ERROR("Failed to allocate status page\n"); 2657 return ENOMEM; 2658 } 2659 2660 obj_priv = obj->driver_private; 2661 2662 ret = i915_gem_object_pin(obj, 4096); 2663 if (ret != 0) { 2664 drm_gem_object_unreference(obj); 2665 return ret; 2666 } 2667 2668 dev_priv->status_gfx_addr = obj_priv->gtt_offset; 2669 dev_priv->hws_map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset; 2670 dev_priv->hws_map.size = 4096; 2671 dev_priv->hws_map.type = 0; 2672 dev_priv->hws_map.flags = 0; 2673 dev_priv->hws_map.mtrr = 0; 2674 2675 drm_core_ioremap(&dev_priv->hws_map, dev); 2676 if (dev_priv->hws_map.handle == NULL) { 2677 DRM_ERROR("Failed to map status page.\n"); 2678 (void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 2679 drm_gem_object_unreference(obj); 2680 return EINVAL; 2681 } 2682 2683 dev_priv->hws_obj = obj; 2684 2685 dev_priv->hw_status_page = dev_priv->hws_map.handle; 2686 2687 (void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE); 2688 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr); 2689 (void) I915_READ(HWS_PGA); /* posting read */ 2690 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr); 2691 2692 return 0; 2693 } 2694 2695 static void 2696 i915_gem_cleanup_hws(struct drm_device *dev) 2697 { 2698 drm_i915_private_t *dev_priv = dev->dev_private; 2699 struct drm_gem_object *obj; 2700 2701 if (dev_priv->hws_obj == NULL) 2702 return; 2703 2704 obj = dev_priv->hws_obj; 2705 2706 drm_core_ioremapfree(&dev_priv->hws_map, dev); 2707 i915_gem_object_unpin(obj); 2708 drm_gem_object_unreference(obj); 2709 dev_priv->hws_obj = NULL; 2710 2711 (void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 2712 dev_priv->hw_status_page = NULL; 2713 2714 /* Write high address into HWS_PGA when disabling. */ 2715 I915_WRITE(HWS_PGA, 0x1ffff000); 2716 } 2717 2718 int 2719 i915_gem_init_ringbuffer(struct drm_device *dev) 2720 { 2721 drm_i915_private_t *dev_priv = dev->dev_private; 2722 struct drm_gem_object *obj; 2723 struct drm_i915_gem_object *obj_priv; 2724 int ret; 2725 u32 head; 2726 2727 ret = i915_gem_init_hws(dev); 2728 if (ret != 0) 2729 return ret; 2730 obj = drm_gem_object_alloc(dev, 128 * 1024); 2731 if (obj == NULL) { 2732 DRM_ERROR("Failed to allocate ringbuffer\n"); 2733 i915_gem_cleanup_hws(dev); 2734 return ENOMEM; 2735 } 2736 2737 obj_priv = obj->driver_private; 2738 ret = i915_gem_object_pin(obj, 4096); 2739 if (ret != 0) { 2740 drm_gem_object_unreference(obj); 2741 i915_gem_cleanup_hws(dev); 2742 return ret; 2743 } 2744 2745 /* Set up the kernel mapping for the ring. */ 2746 dev_priv->ring.Size = obj->size; 2747 dev_priv->ring.tail_mask = obj->size - 1; 2748 2749 dev_priv->ring.map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset; 2750 dev_priv->ring.map.size = obj->size; 2751 dev_priv->ring.map.type = 0; 2752 dev_priv->ring.map.flags = 0; 2753 dev_priv->ring.map.mtrr = 0; 2754 2755 drm_core_ioremap(&dev_priv->ring.map, dev); 2756 if (dev_priv->ring.map.handle == NULL) { 2757 DRM_ERROR("Failed to map ringbuffer.\n"); 2758 (void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); 2759 drm_gem_object_unreference(obj); 2760 i915_gem_cleanup_hws(dev); 2761 return EINVAL; 2762 } 2763 2764 dev_priv->ring.ring_obj = obj; 2765 2766 dev_priv->ring.virtual_start = (u8 *) dev_priv->ring.map.handle; 2767 2768 /* Stop the ring if it's running. */ 2769 I915_WRITE(PRB0_CTL, 0); 2770 I915_WRITE(PRB0_HEAD, 0); 2771 I915_WRITE(PRB0_TAIL, 0); 2772 2773 2774 /* Initialize the ring. */ 2775 I915_WRITE(PRB0_START, obj_priv->gtt_offset); 2776 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 2777 2778 /* G45 ring initialization fails to reset head to zero */ 2779 if (head != 0) { 2780 DRM_ERROR("Ring head not reset to zero " 2781 "ctl %08x head %08x tail %08x start %08x\n", 2782 I915_READ(PRB0_CTL), 2783 I915_READ(PRB0_HEAD), 2784 I915_READ(PRB0_TAIL), 2785 I915_READ(PRB0_START)); 2786 I915_WRITE(PRB0_HEAD, 0); 2787 2788 DRM_ERROR("Ring head forced to zero " 2789 "ctl %08x head %08x tail %08x start %08x\n", 2790 I915_READ(PRB0_CTL), 2791 I915_READ(PRB0_HEAD), 2792 I915_READ(PRB0_TAIL), 2793 I915_READ(PRB0_START)); 2794 } 2795 2796 I915_WRITE(PRB0_CTL, 2797 ((obj->size - 4096) & RING_NR_PAGES) | 2798 RING_NO_REPORT | 2799 RING_VALID); 2800 2801 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 2802 2803 /* If the head is still not zero, the ring is dead */ 2804 if (head != 0) { 2805 DRM_ERROR("Ring initialization failed " 2806 "ctl %08x head %08x tail %08x start %08x\n", 2807 I915_READ(PRB0_CTL), 2808 I915_READ(PRB0_HEAD), 2809 I915_READ(PRB0_TAIL), 2810 I915_READ(PRB0_START)); 2811 return EIO; 2812 } 2813 2814 /* Update our cache of the ring state */ 2815 i915_kernel_lost_context(dev); 2816 2817 return 0; 2818 } 2819 2820 static void 2821 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 2822 { 2823 drm_i915_private_t *dev_priv = dev->dev_private; 2824 2825 if (dev_priv->ring.ring_obj == NULL) 2826 return; 2827 2828 drm_core_ioremapfree(&dev_priv->ring.map, dev); 2829 2830 i915_gem_object_unpin(dev_priv->ring.ring_obj); 2831 drm_gem_object_unreference(dev_priv->ring.ring_obj); 2832 dev_priv->ring.ring_obj = NULL; 2833 (void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); 2834 i915_gem_cleanup_hws(dev); 2835 } 2836 2837 /*ARGSUSED*/ 2838 int 2839 i915_gem_entervt_ioctl(DRM_IOCTL_ARGS) 2840 { 2841 DRM_DEVICE; 2842 drm_i915_private_t *dev_priv = dev->dev_private; 2843 int ret; 2844 2845 if (dev->driver->use_gem != 1) 2846 return ENODEV; 2847 2848 if (dev_priv->mm.wedged) { 2849 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 2850 dev_priv->mm.wedged = 0; 2851 } 2852 /* Set up the kernel mapping for the ring. */ 2853 dev_priv->mm.gtt_mapping.offset = dev->agp->agp_info.agpi_aperbase; 2854 dev_priv->mm.gtt_mapping.size = dev->agp->agp_info.agpi_apersize; 2855 dev_priv->mm.gtt_mapping.type = 0; 2856 dev_priv->mm.gtt_mapping.flags = 0; 2857 dev_priv->mm.gtt_mapping.mtrr = 0; 2858 2859 drm_core_ioremap(&dev_priv->mm.gtt_mapping, dev); 2860 2861 spin_lock(&dev->struct_mutex); 2862 dev_priv->mm.suspended = 0; 2863 ret = i915_gem_init_ringbuffer(dev); 2864 if (ret != 0) 2865 return ret; 2866 2867 spin_unlock(&dev->struct_mutex); 2868 2869 (void) drm_irq_install(dev); 2870 2871 return 0; 2872 } 2873 2874 /*ARGSUSED*/ 2875 int 2876 i915_gem_leavevt_ioctl(DRM_IOCTL_ARGS) 2877 { 2878 DRM_DEVICE; 2879 drm_i915_private_t *dev_priv = dev->dev_private; 2880 int ret; 2881 2882 if (dev->driver->use_gem != 1) 2883 return ENODEV; 2884 2885 ret = i915_gem_idle(dev, 0); 2886 (void) drm_irq_uninstall(dev); 2887 2888 drm_core_ioremapfree(&dev_priv->mm.gtt_mapping, dev); 2889 return ret; 2890 } 2891 2892 void 2893 i915_gem_lastclose(struct drm_device *dev) 2894 { 2895 drm_i915_private_t *dev_priv = dev->dev_private; 2896 int ret; 2897 2898 ret = i915_gem_idle(dev, 1); 2899 if (ret) 2900 DRM_ERROR("failed to idle hardware: %d\n", ret); 2901 2902 drm_mm_clean_ml(&dev_priv->mm.gtt_space); 2903 } 2904 2905 void 2906 i915_gem_load(struct drm_device *dev) 2907 { 2908 drm_i915_private_t *dev_priv = dev->dev_private; 2909 2910 INIT_LIST_HEAD(&dev_priv->mm.active_list); 2911 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 2912 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 2913 INIT_LIST_HEAD(&dev_priv->mm.request_list); 2914 dev_priv->mm.next_gem_seqno = 1; 2915 2916 i915_gem_detect_bit_6_swizzle(dev); 2917 2918 } 2919 2920