xref: /titanic_50/usr/src/uts/intel/io/drm/i915_dma.c (revision 8c4f9701439555b41fbfe7848508f53b52166007)
1 /* BEGIN CSTYLED */
2 
3 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
4  */
5 /*
6  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
7  * Copyright (c) 2009, Intel Corporation.
8  * All Rights Reserved.
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the
12  * "Software"), to deal in the Software without restriction, including
13  * without limitation the rights to use, copy, modify, merge, publish,
14  * distribute, sub license, and/or sell copies of the Software, and to
15  * permit persons to whom the Software is furnished to do so, subject to
16  * the following conditions:
17  *
18  * The above copyright notice and this permission notice (including the
19  * next paragraph) shall be included in all copies or substantial portions
20  * of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
25  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
26  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
27  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
28  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29  *
30  */
31 
32 /*
33  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
34  * Use is subject to license terms.
35  */
36 
37 #include "drmP.h"
38 #include "drm.h"
39 #include "i915_drm.h"
40 #include "i915_drv.h"
41 
42 
43 
44 /* Really want an OS-independent resettable timer.  Would like to have
45  * this loop run for (eg) 3 sec, but have the timer reset every time
46  * the head pointer changes, so that EBUSY only happens if the ring
47  * actually stalls for (eg) 3 seconds.
48  */
49 /*ARGSUSED*/
50 int i915_wait_ring(drm_device_t * dev, int n, const char *caller)
51 {
52 	drm_i915_private_t *dev_priv = dev->dev_private;
53 	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
54 	u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
55 	u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
56 	u32 last_acthd = I915_READ(acthd_reg);
57 	u32 acthd;
58 	int i;
59 
60 	for (i = 0; i < 100000; i++) {
61 		ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
62 		acthd = I915_READ(acthd_reg);
63 		ring->space = ring->head - (ring->tail + 8);
64 		if (ring->space < 0)
65 			ring->space += ring->Size;
66 		if (ring->space >= n)
67 			return 0;
68 
69 		if (ring->head != last_head)
70 			i = 0;
71 
72 		if (acthd != last_acthd)
73 			i = 0;
74 
75 		last_head = ring->head;
76 		last_acthd = acthd;
77 		DRM_UDELAY(10);
78 	}
79 
80 	return (EBUSY);
81 }
82 
83 int i915_init_hardware_status(drm_device_t *dev)
84 {
85        drm_i915_private_t *dev_priv = dev->dev_private;
86        drm_dma_handle_t *dmah;
87 
88        /* Program Hardware Status Page */
89        dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff,1);
90 
91        if (!dmah) {
92                DRM_ERROR("Can not allocate hardware status page\n");
93                return -ENOMEM;
94        }
95 
96        dev_priv->status_page_dmah = dmah;
97        dev_priv->hw_status_page = (void *)dmah->vaddr;
98        dev_priv->dma_status_page = dmah->paddr;
99 
100        (void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
101 
102        I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
103        (void) I915_READ(HWS_PGA);
104 
105        DRM_DEBUG("Enabled hardware status page add 0x%lx read GEM HWS 0x%x\n",dev_priv->hw_status_page, READ_HWSP(dev_priv, 0x20));
106        return 0;
107 }
108 
109 void i915_free_hardware_status(drm_device_t *dev)
110 {
111        drm_i915_private_t *dev_priv = dev->dev_private;
112 	if (!I915_NEED_GFX_HWS(dev)) {
113 		if (dev_priv->status_page_dmah) {
114 			DRM_DEBUG("free status_page_dmal %x", dev_priv->status_page_dmah);
115 			drm_pci_free(dev, dev_priv->status_page_dmah);
116 			dev_priv->status_page_dmah = NULL;
117 			/* Need to rewrite hardware status page */
118 			I915_WRITE(HWS_PGA, 0x1ffff000);
119 		}
120        	} else {
121 		if (dev_priv->status_gfx_addr) {
122 			DRM_DEBUG("free status_gfx_addr %x", dev_priv->status_gfx_addr);
123 			dev_priv->status_gfx_addr = 0;
124 			drm_core_ioremapfree(&dev_priv->hws_map, dev);
125 			I915_WRITE(HWS_PGA, 0x1ffff000);
126 		}
127 	}
128 
129 }
130 
131 void i915_kernel_lost_context(drm_device_t * dev)
132 {
133 	drm_i915_private_t *dev_priv = dev->dev_private;
134 	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
135 
136        ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
137        ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
138 	ring->space = ring->head - (ring->tail + 8);
139 	if (ring->space < 0)
140 		ring->space += ring->Size;
141 
142 }
143 
144 static int i915_dma_cleanup(drm_device_t * dev)
145 {
146 	drm_i915_private_t *dev_priv =
147 		    (drm_i915_private_t *) dev->dev_private;
148 
149 	/* Make sure interrupts are disabled here because the uninstall ioctl
150 	 * may not have been called from userspace and after dev_private
151 	 * is freed, it's too late.
152 	 */
153 	if (dev->irq_enabled)
154 		(void) drm_irq_uninstall(dev);
155 
156 	if (dev_priv->ring.virtual_start) {
157 		drm_core_ioremapfree(&dev_priv->ring.map, dev);
158 		dev_priv->ring.virtual_start = 0;
159 		dev_priv->ring.map.handle = 0;
160 		dev_priv->ring.map.size = 0;
161 	}
162 
163 #ifdef I915_HAVE_GEM
164 	if (I915_NEED_GFX_HWS(dev))
165 #endif
166 	i915_free_hardware_status(dev);
167 
168 	dev_priv->sarea = NULL;
169 	dev_priv->sarea_priv = NULL;
170 
171 	return 0;
172 }
173 
174 static int i915_initialize(drm_device_t * dev,
175 			   drm_i915_init_t * init)
176 {
177 	drm_i915_private_t *dev_priv =
178 	    (drm_i915_private_t *)dev->dev_private;
179 
180 	DRM_GETSAREA();
181 	if (!dev_priv->sarea) {
182 		DRM_ERROR("can not find sarea!\n");
183 		dev->dev_private = (void *)dev_priv;
184 		(void) i915_dma_cleanup(dev);
185 		return (EINVAL);
186 	}
187 
188 	dev_priv->sarea_priv = (drm_i915_sarea_t *)(uintptr_t)
189 			((u8 *) dev_priv->sarea->handle +
190 			 init->sarea_priv_offset);
191 
192 	if (init->ring_size != 0) {
193 		if (dev_priv->ring.ring_obj != NULL) {
194 			(void) i915_dma_cleanup(dev);
195 			DRM_ERROR("Client tried to initialize ringbuffer in "
196 				  "GEM mode\n");
197 			return -EINVAL;
198 		}
199 
200 		dev_priv->ring.Size = init->ring_size;
201 		dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
202 
203 		dev_priv->ring.map.offset = (u_offset_t)init->ring_start;
204 		dev_priv->ring.map.size = init->ring_size;
205 		dev_priv->ring.map.type = 0;
206 		dev_priv->ring.map.flags = 0;
207 		dev_priv->ring.map.mtrr = 0;
208 
209 		drm_core_ioremap(&dev_priv->ring.map, dev);
210 
211 		if (dev_priv->ring.map.handle == NULL) {
212 			(void) i915_dma_cleanup(dev);
213 			DRM_ERROR("can not ioremap virtual address for"
214 			  " ring buffer\n");
215 			return (ENOMEM);
216 		}
217 	}
218 
219 	dev_priv->ring.virtual_start = (u8 *)dev_priv->ring.map.dev_addr;
220 	dev_priv->cpp = init->cpp;
221 	dev_priv->back_offset = init->back_offset;
222 	dev_priv->front_offset = init->front_offset;
223 	dev_priv->current_page = 0;
224 	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
225 
226 	/* Allow hardware batchbuffers unless told otherwise.
227 	 */
228 	dev_priv->allow_batchbuffer = 1;
229 	return 0;
230 }
231 
232 static int i915_dma_resume(drm_device_t * dev)
233 {
234 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
235 
236 	if (!dev_priv->sarea) {
237 		DRM_ERROR("can not find sarea!\n");
238 		return (EINVAL);
239 	}
240 
241 	if (dev_priv->ring.map.handle == NULL) {
242 		DRM_ERROR("can not ioremap virtual address for"
243 			  " ring buffer\n");
244 		return (ENOMEM);
245 	}
246 
247 	/* Program Hardware Status Page */
248 	if (!dev_priv->hw_status_page) {
249 		DRM_ERROR("Can not find hardware status page\n");
250 		return (EINVAL);
251 	}
252 	DRM_DEBUG("i915_dma_resume hw status page @ %p\n", dev_priv->hw_status_page);
253 
254 	if (!I915_NEED_GFX_HWS(dev))
255 		I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
256 	else
257 		I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
258 	DRM_DEBUG("Enabled hardware status page\n");
259 
260 	return 0;
261 }
262 
263 /*ARGSUSED*/
264 static int i915_dma_init(DRM_IOCTL_ARGS)
265 {
266 	DRM_DEVICE;
267 	drm_i915_init_t init;
268 	int retcode = 0;
269 
270 	DRM_COPYFROM_WITH_RETURN(&init, (drm_i915_init_t *)data, sizeof(init));
271 
272 	switch (init.func) {
273 	case I915_INIT_DMA:
274 		retcode = i915_initialize(dev, &init);
275 		break;
276 	case I915_CLEANUP_DMA:
277 		retcode = i915_dma_cleanup(dev);
278 		break;
279 	case I915_RESUME_DMA:
280 		retcode = i915_dma_resume(dev);
281 		break;
282 	default:
283 		retcode = EINVAL;
284 		break;
285 	}
286 
287 	return retcode;
288 }
289 
290 /* Implement basically the same security restrictions as hardware does
291  * for MI_BATCH_NON_SECURE.  These can be made stricter at any time.
292  *
293  * Most of the calculations below involve calculating the size of a
294  * particular instruction.  It's important to get the size right as
295  * that tells us where the next instruction to check is.  Any illegal
296  * instruction detected will be given a size of zero, which is a
297  * signal to abort the rest of the buffer.
298  */
299 static int do_validate_cmd(int cmd)
300 {
301 	switch (((cmd >> 29) & 0x7)) {
302 	case 0x0:
303 		switch ((cmd >> 23) & 0x3f) {
304 		case 0x0:
305 			return 1;	/* MI_NOOP */
306 		case 0x4:
307 			return 1;	/* MI_FLUSH */
308 		default:
309 			return 0;	/* disallow everything else */
310 		}
311 #ifndef __SUNPRO_C
312 		break;
313 #endif
314 	case 0x1:
315 		return 0;	/* reserved */
316 	case 0x2:
317 		return (cmd & 0xff) + 2;	/* 2d commands */
318 	case 0x3:
319 		if (((cmd >> 24) & 0x1f) <= 0x18)
320 			return 1;
321 
322 		switch ((cmd >> 24) & 0x1f) {
323 		case 0x1c:
324 			return 1;
325 		case 0x1d:
326 			switch ((cmd >> 16) & 0xff) {
327 			case 0x3:
328 				return (cmd & 0x1f) + 2;
329 			case 0x4:
330 				return (cmd & 0xf) + 2;
331 			default:
332 				return (cmd & 0xffff) + 2;
333 			}
334 		case 0x1e:
335 			if (cmd & (1 << 23))
336 				return (cmd & 0xffff) + 1;
337 			else
338 				return 1;
339 		case 0x1f:
340 			if ((cmd & (1 << 23)) == 0)	/* inline vertices */
341 				return (cmd & 0x1ffff) + 2;
342 			else if (cmd & (1 << 17))	/* indirect random */
343 				if ((cmd & 0xffff) == 0)
344 					return 0;	/* unknown length, too hard */
345 				else
346 					return (((cmd & 0xffff) + 1) / 2) + 1;
347 			else
348 				return 2;	/* indirect sequential */
349 		default:
350 			return 0;
351 		}
352 	default:
353 		return 0;
354 	}
355 
356 #ifndef __SUNPRO_C
357 	return 0;
358 #endif
359 }
360 
361 static int validate_cmd(int cmd)
362 {
363 	int ret = do_validate_cmd(cmd);
364 
365 /* 	printk("validate_cmd( %x ): %d\n", cmd, ret); */
366 
367 	return ret;
368 }
369 
370 static int i915_emit_cmds(drm_device_t * dev, int __user * buffer, int dwords)
371 {
372 	drm_i915_private_t *dev_priv = dev->dev_private;
373 	int i;
374 	RING_LOCALS;
375 
376 	if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8) {
377 		DRM_ERROR(" emit cmds invalid arg");
378 		return (EINVAL);
379 	}
380 	BEGIN_LP_RING((dwords+1)&~1);
381 
382 	for (i = 0; i < dwords;) {
383 		int cmd, sz;
384 
385 		if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) {
386 			DRM_ERROR("emit cmds failed to get cmd from user");
387 			return (EINVAL);
388 		}
389 
390 		if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords) {
391 			DRM_ERROR("emit cmds invalid");
392 			return (EINVAL);
393 		}
394 		OUT_RING(cmd);
395 
396 		while (++i, --sz) {
397 			if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i],
398 							 sizeof(cmd))) {
399 				DRM_ERROR("emit cmds failed get cmds");
400 				return (EINVAL);
401 			}
402 			OUT_RING(cmd);
403 		}
404 	}
405 
406 	if (dwords & 1)
407 		OUT_RING(0);
408 
409 	ADVANCE_LP_RING();
410 
411 	return 0;
412 }
413 
414 int i915_emit_box(drm_device_t * dev,
415 			 drm_clip_rect_t __user * boxes,
416 			 int i, int DR1, int DR4)
417 {
418 	drm_i915_private_t *dev_priv = dev->dev_private;
419 	drm_clip_rect_t box;
420 	RING_LOCALS;
421 
422 	if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) {
423 		DRM_ERROR("emit box failed to copy from user");
424 		return (EFAULT);
425 	}
426 
427 	if (box.y2 <= box.y1 || box.x2 <= box.x1) {
428 		DRM_ERROR("Bad box %d,%d..%d,%d\n",
429 			  box.x1, box.y1, box.x2, box.y2);
430 		return (EINVAL);
431 	}
432 
433 	if (IS_I965G(dev)) {
434 		BEGIN_LP_RING(4);
435 		OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
436 		OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
437 		OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
438 		OUT_RING(DR4);
439 		ADVANCE_LP_RING();
440 	} else {
441 		BEGIN_LP_RING(6);
442 		OUT_RING(GFX_OP_DRAWRECT_INFO);
443 		OUT_RING(DR1);
444 		OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
445 		OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
446 		OUT_RING(DR4);
447 		OUT_RING(0);
448 		ADVANCE_LP_RING();
449 	}
450 
451 	return 0;
452 }
453 
454 /* XXX: Emitting the counter should really be moved to part of the IRQ
455  * emit.  For now, do it in both places:
456  */
457 
458 void i915_emit_breadcrumb(drm_device_t *dev)
459 {
460 	drm_i915_private_t *dev_priv = dev->dev_private;
461 	RING_LOCALS;
462 
463 	dev_priv->counter++;
464 	if (dev_priv->counter > 0x7FFFFFFFUL)
465 		dev_priv->counter = 0;
466 	if (dev_priv->sarea_priv)
467 		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
468 
469 
470 	BEGIN_LP_RING(4);
471 	OUT_RING(MI_STORE_DWORD_INDEX);
472 	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
473 	OUT_RING(dev_priv->counter);
474 	OUT_RING(0);
475 	ADVANCE_LP_RING();
476 
477 }
478 
479 static int i915_dispatch_cmdbuffer(drm_device_t * dev,
480 				   drm_i915_cmdbuffer_t * cmd)
481 {
482 	int nbox = cmd->num_cliprects;
483 	int i = 0, count, ret;
484 
485 	if (cmd->sz & 0x3) {
486 		DRM_ERROR("alignment");
487 		return (EINVAL);
488 	}
489 
490 	i915_kernel_lost_context(dev);
491 
492 	count = nbox ? nbox : 1;
493 
494 	for (i = 0; i < count; i++) {
495 		if (i < nbox) {
496 			ret = i915_emit_box(dev, cmd->cliprects, i,
497 					    cmd->DR1, cmd->DR4);
498 			if (ret)
499 				return ret;
500 		}
501 
502 		ret = i915_emit_cmds(dev, (int __user *)(void *)cmd->buf, cmd->sz / 4);
503 		if (ret)
504 			return ret;
505 	}
506 
507 	i915_emit_breadcrumb( dev );
508 	return 0;
509 }
510 
511 static int i915_dispatch_batchbuffer(drm_device_t * dev,
512 				     drm_i915_batchbuffer_t * batch)
513 {
514 	drm_i915_private_t *dev_priv = dev->dev_private;
515 	drm_clip_rect_t __user *boxes = batch->cliprects;
516 	int nbox = batch->num_cliprects;
517 	int i = 0, count;
518 	RING_LOCALS;
519 
520 	if ((batch->start | batch->used) & 0x7) {
521 		DRM_ERROR("alignment");
522 		return (EINVAL);
523 	}
524 
525 	i915_kernel_lost_context(dev);
526 
527 	count = nbox ? nbox : 1;
528 
529 	for (i = 0; i < count; i++) {
530 		if (i < nbox) {
531 			int ret = i915_emit_box(dev, boxes, i,
532 						batch->DR1, batch->DR4);
533 			if (ret)
534 				return ret;
535 		}
536 
537 		if (IS_I830(dev) || IS_845G(dev)) {
538 			BEGIN_LP_RING(4);
539 			OUT_RING(MI_BATCH_BUFFER);
540 			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
541 			OUT_RING(batch->start + batch->used - 4);
542 			OUT_RING(0);
543 			ADVANCE_LP_RING();
544 		} else {
545 			BEGIN_LP_RING(2);
546 			if (IS_I965G(dev)) {
547 				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
548 				OUT_RING(batch->start);
549 			} else {
550 				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
551 				OUT_RING(batch->start | MI_BATCH_NON_SECURE);
552 			}
553 			ADVANCE_LP_RING();
554 		}
555 	}
556 
557 	i915_emit_breadcrumb( dev );
558 
559 	return 0;
560 }
561 
562 static int i915_dispatch_flip(struct drm_device * dev, int planes)
563 {
564 	drm_i915_private_t *dev_priv = dev->dev_private;
565 	RING_LOCALS;
566 
567 	if (!dev_priv->sarea_priv)
568 		return -EINVAL;
569 
570 	DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
571 		planes, dev_priv->sarea_priv->pf_current_page);
572 
573 	i915_kernel_lost_context(dev);
574 
575 	BEGIN_LP_RING(2);
576 	OUT_RING(MI_FLUSH | MI_READ_FLUSH);
577 	OUT_RING(0);
578 	ADVANCE_LP_RING();
579 
580 	BEGIN_LP_RING(6);
581 	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
582 	OUT_RING(0);
583 	if (dev_priv->current_page == 0) {
584 		OUT_RING(dev_priv->back_offset);
585 		dev_priv->current_page = 1;
586 	} else {
587 		OUT_RING(dev_priv->front_offset);
588 		dev_priv->current_page = 0;
589 	}
590 	OUT_RING(0);
591 	ADVANCE_LP_RING();
592 
593 	BEGIN_LP_RING(2);
594 	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
595 	OUT_RING(0);
596 	ADVANCE_LP_RING();
597 
598 	dev_priv->sarea_priv->last_enqueue = dev_priv->counter++;
599 
600 	BEGIN_LP_RING(4);
601 	OUT_RING(MI_STORE_DWORD_INDEX);
602 	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
603 	OUT_RING(dev_priv->counter);
604 	OUT_RING(0);
605 	ADVANCE_LP_RING();
606 
607 	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
608 	return 0;
609 }
610 
611 static int i915_quiescent(drm_device_t * dev)
612 {
613 	drm_i915_private_t *dev_priv = dev->dev_private;
614 	int ret;
615 	i915_kernel_lost_context(dev);
616 	ret = i915_wait_ring(dev, dev_priv->ring.Size - 8, __FUNCTION__);
617 
618 	if (ret)
619 	{
620 		i915_kernel_lost_context (dev);
621 		DRM_ERROR ("not quiescent head %08x tail %08x space %08x\n",
622 			   dev_priv->ring.head,
623 			   dev_priv->ring.tail,
624 			   dev_priv->ring.space);
625 	}
626 	return ret;
627 }
628 
629 /*ARGSUSED*/
630 static int i915_flush_ioctl(DRM_IOCTL_ARGS)
631 {
632 	int ret;
633 	DRM_DEVICE;
634 
635 	LOCK_TEST_WITH_RETURN(dev, fpriv);
636 
637 	spin_lock(&dev->struct_mutex);
638 	ret = i915_quiescent(dev);
639 	spin_unlock(&dev->struct_mutex);
640 
641 	return ret;
642 }
643 
644 /*ARGSUSED*/
645 static int i915_batchbuffer(DRM_IOCTL_ARGS)
646 {
647 	DRM_DEVICE;
648 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
649 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
650 	    dev_priv->sarea_priv;
651 	drm_i915_batchbuffer_t batch;
652 	int ret;
653 
654 	if (!dev_priv->allow_batchbuffer) {
655 		DRM_ERROR("Batchbuffer ioctl disabled\n");
656 		return (EINVAL);
657 	}
658 
659 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
660 		drm_i915_batchbuffer32_t batchbuffer32_t;
661 
662 		DRM_COPYFROM_WITH_RETURN(&batchbuffer32_t,
663 			(void *) data, sizeof (batchbuffer32_t));
664 
665 		batch.start = batchbuffer32_t.start;
666 		batch.used = batchbuffer32_t.used;
667 		batch.DR1 = batchbuffer32_t.DR1;
668 		batch.DR4 = batchbuffer32_t.DR4;
669 		batch.num_cliprects = batchbuffer32_t.num_cliprects;
670 		batch.cliprects = (drm_clip_rect_t __user *)
671 			(uintptr_t)batchbuffer32_t.cliprects;
672 	} else
673 		DRM_COPYFROM_WITH_RETURN(&batch, (void *) data,
674 			sizeof(batch));
675 
676 	DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d, counter %d\n",
677 		  batch.start, batch.used, batch.num_cliprects, dev_priv->counter);
678 
679 	LOCK_TEST_WITH_RETURN(dev, fpriv);
680 
681 /*
682 	if (batch.num_cliprects && DRM_VERIFYAREA_READ(batch.cliprects,
683 						       batch.num_cliprects *
684 						       sizeof(drm_clip_rect_t)))
685 		return (EFAULT);
686 
687 */
688 
689 	spin_lock(&dev->struct_mutex);
690 	ret = i915_dispatch_batchbuffer(dev, &batch);
691 	spin_unlock(&dev->struct_mutex);
692 	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
693 
694 	return ret;
695 }
696 
697 /*ARGSUSED*/
698 static int i915_cmdbuffer(DRM_IOCTL_ARGS)
699 {
700 	DRM_DEVICE;
701 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
702 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
703 	    dev_priv->sarea_priv;
704 	drm_i915_cmdbuffer_t cmdbuf;
705 	int ret;
706 
707 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
708 		drm_i915_cmdbuffer32_t cmdbuffer32_t;
709 
710 		DRM_COPYFROM_WITH_RETURN(&cmdbuffer32_t,
711 			(drm_i915_cmdbuffer32_t __user *) data,
712 			sizeof (drm_i915_cmdbuffer32_t));
713 
714 		cmdbuf.buf = (char __user *)(uintptr_t)cmdbuffer32_t.buf;
715 		cmdbuf.sz = cmdbuffer32_t.sz;
716 		cmdbuf.DR1 = cmdbuffer32_t.DR1;
717 		cmdbuf.DR4 = cmdbuffer32_t.DR4;
718 		cmdbuf.num_cliprects = cmdbuffer32_t.num_cliprects;
719 		cmdbuf.cliprects = (drm_clip_rect_t __user *)
720 			(uintptr_t)cmdbuffer32_t.cliprects;
721 	} else
722 		DRM_COPYFROM_WITH_RETURN(&cmdbuf, (void *) data,
723 			sizeof(cmdbuf));
724 
725 	DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
726 		  cmdbuf.buf, cmdbuf.sz, cmdbuf.num_cliprects);
727 
728 	LOCK_TEST_WITH_RETURN(dev, fpriv);
729 
730 /*
731 	if (cmdbuf.num_cliprects &&
732 	    DRM_VERIFYAREA_READ(cmdbuf.cliprects,
733 				cmdbuf.num_cliprects *
734 				sizeof(drm_clip_rect_t))) {
735 		DRM_ERROR("Fault accessing cliprects\n");
736 		return (EFAULT);
737 	}
738 */
739 
740 	spin_lock(&dev->struct_mutex);
741 	ret = i915_dispatch_cmdbuffer(dev, &cmdbuf);
742 	spin_unlock(&dev->struct_mutex);
743 	if (ret) {
744 		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
745 		return ret;
746 	}
747 
748 	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
749 	return 0;
750 }
751 
752 /*ARGSUSED*/
753 static int i915_flip_bufs(DRM_IOCTL_ARGS)
754 {
755 	DRM_DEVICE;
756 	drm_i915_flip_t param;
757 	int ret;
758         DRM_COPYFROM_WITH_RETURN(&param, (drm_i915_flip_t *) data,
759                                  sizeof(param));
760 
761 	DRM_DEBUG("i915_flip_bufs\n");
762 
763 	LOCK_TEST_WITH_RETURN(dev, fpriv);
764 
765 	spin_lock(&dev->struct_mutex);
766 	ret = i915_dispatch_flip(dev, param.pipes);
767 	spin_unlock(&dev->struct_mutex);
768 	return ret;
769 }
770 
771 /*ARGSUSED*/
772 static int i915_getparam(DRM_IOCTL_ARGS)
773 {
774 	DRM_DEVICE;
775 	drm_i915_private_t *dev_priv = dev->dev_private;
776 	drm_i915_getparam_t param;
777 	int value;
778 
779 	if (!dev_priv) {
780 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
781 		return (EINVAL);
782 	}
783 
784 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
785 		drm_i915_getparam32_t getparam32_t;
786 
787 		DRM_COPYFROM_WITH_RETURN(&getparam32_t,
788 			(drm_i915_getparam32_t __user *) data,
789 			sizeof (drm_i915_getparam32_t));
790 
791 		param.param = getparam32_t.param;
792 		param.value = (int __user *)(uintptr_t)getparam32_t.value;
793 	} else
794 		DRM_COPYFROM_WITH_RETURN(&param,
795 		    (drm_i915_getparam_t *) data, sizeof(param));
796 
797 	switch (param.param) {
798 	case I915_PARAM_IRQ_ACTIVE:
799 		value = dev->irq_enabled ? 1 : 0;
800 		break;
801 	case I915_PARAM_ALLOW_BATCHBUFFER:
802 		value = dev_priv->allow_batchbuffer ? 1 : 0;
803 		break;
804 	case I915_PARAM_LAST_DISPATCH:
805 		value = READ_BREADCRUMB(dev_priv);
806 		break;
807 	case I915_PARAM_CHIPSET_ID:
808 		value = dev->pci_device;
809 		break;
810 	case I915_PARAM_HAS_GEM:
811 		value = dev->driver->use_gem;
812 		break;
813 	default:
814 		DRM_ERROR("Unknown get parameter %d\n", param.param);
815 		return (EINVAL);
816 	}
817 
818 	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
819 		DRM_ERROR("i915_getparam failed\n");
820 		return (EFAULT);
821 	}
822 	return 0;
823 }
824 
825 /*ARGSUSED*/
826 static int i915_setparam(DRM_IOCTL_ARGS)
827 {
828 	DRM_DEVICE;
829 	drm_i915_private_t *dev_priv = dev->dev_private;
830 	drm_i915_setparam_t param;
831 
832 	if (!dev_priv) {
833 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
834 		return (EINVAL);
835 	}
836 
837 	DRM_COPYFROM_WITH_RETURN(&param, (drm_i915_setparam_t *) data,
838 				 sizeof(param));
839 
840 	switch (param.param) {
841 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
842 		break;
843 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
844 		dev_priv->tex_lru_log_granularity = param.value;
845 		break;
846 	case I915_SETPARAM_ALLOW_BATCHBUFFER:
847 		dev_priv->allow_batchbuffer = param.value;
848 		break;
849 	default:
850 		DRM_ERROR("unknown set parameter %d\n", param.param);
851 		return (EINVAL);
852 	}
853 
854 	return 0;
855 }
856 
857 /*ARGSUSED*/
858 static int i915_set_status_page(DRM_IOCTL_ARGS)
859 {
860 	DRM_DEVICE;
861 	drm_i915_private_t *dev_priv = dev->dev_private;
862 	drm_i915_hws_addr_t hws;
863 
864 	if (!I915_NEED_GFX_HWS(dev))
865 		return (EINVAL);
866 
867 	if (!dev_priv) {
868 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
869 		return (EINVAL);
870 	}
871 	DRM_COPYFROM_WITH_RETURN(&hws, (drm_i915_hws_addr_t __user *) data,
872 			sizeof(hws));
873 DRM_ERROR("i915_set_status_page set status page addr 0x%08x\n", (u32)hws.addr);
874 
875 	dev_priv->status_gfx_addr = hws.addr & (0x1ffff<<12);
876 	DRM_DEBUG("set gfx_addr 0x%08x\n", dev_priv->status_gfx_addr);
877 
878 	dev_priv->hws_map.offset =
879 	    (u_offset_t)dev->agp->agp_info.agpi_aperbase + hws.addr;
880 	dev_priv->hws_map.size = 4 * 1024; /* 4K pages */
881 	dev_priv->hws_map.type = 0;
882 	dev_priv->hws_map.flags = 0;
883 	dev_priv->hws_map.mtrr = 0;
884 
885 	DRM_DEBUG("set status page: i915_set_status_page: mapoffset 0x%llx\n",
886 	    dev_priv->hws_map.offset);
887 	drm_core_ioremap(&dev_priv->hws_map, dev);
888 	if (dev_priv->hws_map.handle == NULL) {
889 		dev->dev_private = (void *)dev_priv;
890 		(void) i915_dma_cleanup(dev);
891 		dev_priv->status_gfx_addr = 0;
892 		DRM_ERROR("can not ioremap virtual address for"
893 				" G33 hw status page\n");
894 		return (ENOMEM);
895 	}
896 	dev_priv->hw_status_page = dev_priv->hws_map.dev_addr;
897 
898 	(void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
899 	I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
900 	DRM_DEBUG("load hws 0x2080 with gfx mem 0x%x\n",
901 			dev_priv->status_gfx_addr);
902 	DRM_DEBUG("load hws at %p\n", dev_priv->hw_status_page);
903 	return 0;
904 }
905 
906 /*ARGSUSED*/
907 int i915_driver_load(drm_device_t *dev, unsigned long flags)
908 {
909 	struct drm_i915_private *dev_priv;
910 	unsigned long base, size;
911 	int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
912 
913 	/* i915 has 4 more counters */
914 	dev->counters += 4;
915 	dev->types[6] = _DRM_STAT_IRQ;
916 	dev->types[7] = _DRM_STAT_PRIMARY;
917 	dev->types[8] = _DRM_STAT_SECONDARY;
918 	dev->types[9] = _DRM_STAT_DMA;
919 
920 	dev_priv = drm_alloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER);
921 	if (dev_priv == NULL)
922 		return ENOMEM;
923 
924 	(void) memset(dev_priv, 0, sizeof(drm_i915_private_t));
925 	dev->dev_private = (void *)dev_priv;
926 	dev_priv->dev = dev;
927 
928 	/* Add register map (needed for suspend/resume) */
929 
930 	base = drm_get_resource_start(dev, mmio_bar);
931 	size = drm_get_resource_len(dev, mmio_bar);
932 	dev_priv->mmio_map = drm_alloc(sizeof (drm_local_map_t), DRM_MEM_MAPS);
933 	dev_priv->mmio_map->offset = base;
934 	dev_priv->mmio_map->size = size;
935 	dev_priv->mmio_map->type = _DRM_REGISTERS;
936 	dev_priv->mmio_map->flags = _DRM_REMOVABLE;
937 	(void) drm_ioremap(dev, dev_priv->mmio_map);
938 
939 	DRM_DEBUG("i915_driverload mmio %p mmio_map->dev_addr %x", dev_priv->mmio_map, dev_priv->mmio_map->dev_addr);
940 
941 #if defined(__i386)
942 	dev->driver->use_gem = 0;
943 #else
944 	if (IS_I965G(dev)) {
945 		dev->driver->use_gem = 1;
946 	} else {
947 		dev->driver->use_gem = 0;
948 	}
949 #endif	/* __i386 */
950 
951 	dev->driver->get_vblank_counter = i915_get_vblank_counter;
952 	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
953 #if defined(__i386)
954 	if (IS_G4X(dev) || IS_IGDNG(dev) || IS_GM45(dev))
955 #else
956 	if (IS_G4X(dev) || IS_IGDNG(dev))
957 #endif
958 	{
959 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
960 		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
961 	}
962 
963 
964 #ifdef I915_HAVE_GEM
965 	i915_gem_load(dev);
966 #endif
967 
968 	if (!I915_NEED_GFX_HWS(dev)) {
969 		ret = i915_init_hardware_status(dev);
970 		if(ret)
971 			return ret;
972 	}
973 
974 	mutex_init(&dev_priv->user_irq_lock, "userirq", MUTEX_DRIVER, NULL);
975 	mutex_init(&dev_priv->error_lock, "error_lock", MUTEX_DRIVER, NULL);
976 
977 	ret = drm_vblank_init(dev, I915_NUM_PIPE);
978 	if (ret) {
979 		(void) i915_driver_unload(dev);
980 		return ret;
981 	}
982 
983 	return ret;
984 }
985 
986 int i915_driver_unload(struct drm_device *dev)
987 {
988        drm_i915_private_t *dev_priv = dev->dev_private;
989 
990        i915_free_hardware_status(dev);
991 
992 	drm_rmmap(dev, dev_priv->mmio_map);
993 
994         mutex_destroy(&dev_priv->user_irq_lock);
995 
996 	drm_free(dev->dev_private, sizeof(drm_i915_private_t),
997 	    DRM_MEM_DRIVER);
998 	dev->dev_private = NULL;
999 
1000 	return 0;
1001 }
1002 
1003 /*ARGSUSED*/
1004 int i915_driver_open(drm_device_t * dev, struct drm_file *file_priv)
1005 {
1006         struct drm_i915_file_private *i915_file_priv;
1007 
1008         DRM_DEBUG("\n");
1009         i915_file_priv = (struct drm_i915_file_private *)
1010             drm_alloc(sizeof(*i915_file_priv), DRM_MEM_FILES);
1011 
1012         if (!i915_file_priv)
1013                 return -ENOMEM;
1014 
1015         file_priv->driver_priv = i915_file_priv;
1016 
1017         i915_file_priv->mm.last_gem_seqno = 0;
1018         i915_file_priv->mm.last_gem_throttle_seqno = 0;
1019 
1020         return 0;
1021 }
1022 
1023 void i915_driver_lastclose(drm_device_t * dev)
1024 {
1025 	drm_i915_private_t *dev_priv = dev->dev_private;
1026 
1027 	/* agp off can use this to get called before dev_priv */
1028 	if (!dev_priv)
1029 		return;
1030 
1031 #ifdef I915_HAVE_GEM
1032 	i915_gem_lastclose(dev);
1033 #endif
1034 
1035 	DRM_GETSAREA();
1036 	if (dev_priv->agp_heap)
1037 		i915_mem_takedown(&(dev_priv->agp_heap));
1038 	(void) i915_dma_cleanup(dev);
1039 }
1040 
1041 void i915_driver_preclose(drm_device_t * dev, drm_file_t *fpriv)
1042 {
1043 	drm_i915_private_t *dev_priv = dev->dev_private;
1044 	i915_mem_release(dev, fpriv, dev_priv->agp_heap);
1045 }
1046 
1047 /*ARGSUSED*/
1048 void i915_driver_postclose(drm_device_t * dev, struct drm_file *file_priv)
1049 {
1050 	struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1051 
1052 	drm_free(i915_file_priv, sizeof(*i915_file_priv), DRM_MEM_FILES);
1053 }
1054 
1055 drm_ioctl_desc_t i915_ioctls[] = {
1056 	[DRM_IOCTL_NR(DRM_I915_INIT)] =
1057 	    {i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1058 	[DRM_IOCTL_NR(DRM_I915_FLUSH)] =
1059 	    {i915_flush_ioctl, DRM_AUTH},
1060 	[DRM_IOCTL_NR(DRM_I915_FLIP)] =
1061 	    {i915_flip_bufs, DRM_AUTH},
1062 	[DRM_IOCTL_NR(DRM_I915_BATCHBUFFER)] =
1063 	    {i915_batchbuffer, DRM_AUTH},
1064 	[DRM_IOCTL_NR(DRM_I915_IRQ_EMIT)] =
1065 	    {i915_irq_emit, DRM_AUTH},
1066 	[DRM_IOCTL_NR(DRM_I915_IRQ_WAIT)] =
1067 	    {i915_irq_wait, DRM_AUTH},
1068 	[DRM_IOCTL_NR(DRM_I915_GETPARAM)] =
1069 	    {i915_getparam, DRM_AUTH},
1070 	[DRM_IOCTL_NR(DRM_I915_SETPARAM)] =
1071 	    {i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1072 	[DRM_IOCTL_NR(DRM_I915_ALLOC)] =
1073 	    {i915_mem_alloc, DRM_AUTH},
1074 	[DRM_IOCTL_NR(DRM_I915_FREE)] =
1075 	    {i915_mem_free, DRM_AUTH},
1076 	[DRM_IOCTL_NR(DRM_I915_INIT_HEAP)] =
1077 	    {i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1078 	[DRM_IOCTL_NR(DRM_I915_CMDBUFFER)] =
1079 	    {i915_cmdbuffer, DRM_AUTH},
1080 	[DRM_IOCTL_NR(DRM_I915_DESTROY_HEAP)] =
1081 	    {i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1082 	[DRM_IOCTL_NR(DRM_I915_SET_VBLANK_PIPE)] =
1083 	    {i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1084 	[DRM_IOCTL_NR(DRM_I915_GET_VBLANK_PIPE)] =
1085 	    {i915_vblank_pipe_get, DRM_AUTH},
1086 	[DRM_IOCTL_NR(DRM_I915_VBLANK_SWAP)] =
1087 	    {i915_vblank_swap, DRM_AUTH},
1088 	[DRM_IOCTL_NR(DRM_I915_HWS_ADDR)] =
1089 	    {i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
1090 #ifdef I915_HAVE_GEM
1091         [DRM_IOCTL_NR(DRM_I915_GEM_INIT)] =
1092             {i915_gem_init_ioctl, DRM_AUTH},
1093         [DRM_IOCTL_NR(DRM_I915_GEM_EXECBUFFER)] =
1094             {i915_gem_execbuffer, DRM_AUTH},
1095         [DRM_IOCTL_NR(DRM_I915_GEM_PIN)] =
1096             {i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY},
1097         [DRM_IOCTL_NR(DRM_I915_GEM_UNPIN)] =
1098             {i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY},
1099         [DRM_IOCTL_NR(DRM_I915_GEM_BUSY)] =
1100             {i915_gem_busy_ioctl, DRM_AUTH},
1101         [DRM_IOCTL_NR(DRM_I915_GEM_THROTTLE)] =
1102             {i915_gem_throttle_ioctl, DRM_AUTH},
1103         [DRM_IOCTL_NR(DRM_I915_GEM_ENTERVT)] =
1104             {i915_gem_entervt_ioctl, DRM_AUTH},
1105         [DRM_IOCTL_NR(DRM_I915_GEM_LEAVEVT)] =
1106             {i915_gem_leavevt_ioctl, DRM_AUTH},
1107         [DRM_IOCTL_NR(DRM_I915_GEM_CREATE)] =
1108             {i915_gem_create_ioctl, 0},
1109         [DRM_IOCTL_NR(DRM_I915_GEM_PREAD)] =
1110             {i915_gem_pread_ioctl, 0},
1111         [DRM_IOCTL_NR(DRM_I915_GEM_PWRITE)] =
1112             {i915_gem_pwrite_ioctl, 0},
1113         [DRM_IOCTL_NR(DRM_I915_GEM_MMAP)] =
1114             {i915_gem_mmap_ioctl, 0},
1115         [DRM_IOCTL_NR(DRM_I915_GEM_SET_DOMAIN)] =
1116             {i915_gem_set_domain_ioctl, 0},
1117         [DRM_IOCTL_NR(DRM_I915_GEM_SW_FINISH)] =
1118             {i915_gem_sw_finish_ioctl, 0},
1119         [DRM_IOCTL_NR(DRM_I915_GEM_SET_TILING)] =
1120             {i915_gem_set_tiling, 0},
1121         [DRM_IOCTL_NR(DRM_I915_GEM_GET_TILING)] =
1122             {i915_gem_get_tiling, 0},
1123         [DRM_IOCTL_NR(DRM_I915_GEM_GET_APERTURE)] =
1124             {i915_gem_get_aperture_ioctl, 0},
1125 #endif
1126 };
1127 
1128 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
1129 
1130 /**
1131  * Determine if the device really is AGP or not.
1132  *
1133  * All Intel graphics chipsets are treated as AGP, even if they are really
1134  * PCI-e.
1135  *
1136  * \param dev   The device to be tested.
1137  *
1138  * \returns
1139  * A value of 1 is always retured to indictate every i9x5 is AGP.
1140  */
1141 /*ARGSUSED*/
1142 int i915_driver_device_is_agp(drm_device_t * dev)
1143 {
1144 	return 1;
1145 }
1146 
1147