xref: /titanic_41/usr/src/uts/intel/io/drm/i915_irq.c (revision 2e6e901d9406e1a048063c872149376a6bf7cb63)

/* BEGIN CSTYLED */

/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
 */
/*
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * Copyright (c) 2009, Intel Corporation.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"


#define MAX_NOPID ((u32)~0)

/**
 * Interrupts that are always left unmasked.
 *
 * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
 * we leave them always unmasked in IMR and then control enabling them through
 * PIPESTAT alone.
 */

#define I915_INTERRUPT_ENABLE_FIX (I915_ASLE_INTERRUPT |		 \
				   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
				   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \
				   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)

/** Interrupts that we mask and unmask at runtime. */
#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)

/** These are all of the interrupts used by the driver */
#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
				    I915_INTERRUPT_ENABLE_VAR)

void
igdng_enable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
{
	if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != 0)) {
		dev_priv->gt_irq_mask_reg &= ~mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
		(void) I915_READ(GTIMR);
	} else if ((dev_priv->irq_mask_reg & mask) != 0) {
		dev_priv->irq_mask_reg &= ~mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
		(void) I915_READ(DEIMR);

	}
}

static inline void
igdng_disable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
{
	if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != mask)) {
		dev_priv->gt_irq_mask_reg |= mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
		(void) I915_READ(GTIMR);
	} else if ((dev_priv->irq_mask_reg & mask) != mask) {
		dev_priv->irq_mask_reg |= mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
		(void) I915_READ(DEIMR);
	}
}

/* For display hotplug interrupt */
void
igdng_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
{
       if ((dev_priv->irq_mask_reg & mask) != 0) {
               dev_priv->irq_mask_reg &= ~mask;
               I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
               (void) I915_READ(DEIMR);
       }
}

#if 0
static inline void
igdng_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
{
       if ((dev_priv->irq_mask_reg & mask) != mask) {
               dev_priv->irq_mask_reg |= mask;
               I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
               (void) I915_READ(DEIMR);
       }
}
#endif

static inline void
i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
{
        if ((dev_priv->irq_mask_reg & mask) != 0) {
                dev_priv->irq_mask_reg &= ~mask;
                I915_WRITE(IMR, dev_priv->irq_mask_reg);
                (void) I915_READ(IMR);
        }
}

static inline void
i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
{
	if ((dev_priv->irq_mask_reg & mask) != mask) {
                dev_priv->irq_mask_reg |= mask;
                I915_WRITE(IMR, dev_priv->irq_mask_reg);
                (void) I915_READ(IMR);
        }
}

static inline uint32_t
i915_pipestat(int pipe)
{
	if (pipe == 0)
		return PIPEASTAT;
	if (pipe == 1)
		return PIPEBSTAT;
	return 0;
}

void
i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, uint32_t mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != mask) {
		u32 reg = i915_pipestat(pipe);

		dev_priv->pipestat[pipe] |= mask;
		/* Enable the interrupt, clear any pending status */
		I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
		(void) I915_READ(reg);
	}
}

void
i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != 0) {
		u32 reg = i915_pipestat(pipe);

		dev_priv->pipestat[pipe] &= ~mask;
		I915_WRITE(reg, dev_priv->pipestat[pipe]);
		(void) I915_READ(reg);
	}
}

/**
 * i915_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
static int
i915_pipe_enabled(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;

	if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
		return 1;

	return 0;
}

u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long high_frame;
	unsigned long low_frame;
	u32 high1, high2, low, count;

	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;

	if (!i915_pipe_enabled(dev, pipe)) {
	    DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
	    return 0;
	}

	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
		high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
			 PIPE_FRAME_HIGH_SHIFT);
		low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
			PIPE_FRAME_LOW_SHIFT);
		high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
			 PIPE_FRAME_HIGH_SHIFT);
	} while (high1 != high2);

	count = (high1 << 8) | low;

	return count;
}

/**
 * i915_capture_error_state - capture an error record for later analysis
 * @dev: drm device
 *
 * Should be called when an error is detected (either a hang or an error
 * interrupt) to capture error state from the time of the error.  Fills
 * out a structure which becomes available in debugfs for user level tools
 * to pick up.
 */
static void i915_capture_error_state(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_error_state *error;

	spin_lock_irqsave(&dev_priv->error_lock, flags);
#if 0
	if (dev_priv->first_error)
		goto out;
#endif
	error = drm_alloc(sizeof(*error), DRM_MEM_DRIVER);
	if (!error) {
		DRM_DEBUG("out ot memory, not capturing error state\n");
		goto out;
	}

	error->eir = I915_READ(EIR);
	error->pgtbl_er = I915_READ(PGTBL_ER);
	error->pipeastat = I915_READ(PIPEASTAT);
	error->pipebstat = I915_READ(PIPEBSTAT);
	error->instpm = I915_READ(INSTPM);
	if (!IS_I965G(dev)) {
		error->ipeir = I915_READ(IPEIR);
		error->ipehr = I915_READ(IPEHR);
		error->instdone = I915_READ(INSTDONE);
		error->acthd = I915_READ(ACTHD);
	} else {
		error->ipeir = I915_READ(IPEIR_I965);
		error->ipehr = I915_READ(IPEHR_I965);
		error->instdone = I915_READ(INSTDONE_I965);
		error->instps = I915_READ(INSTPS);
		error->instdone1 = I915_READ(INSTDONE1);
		error->acthd = I915_READ(ACTHD_I965);
	}

	(void) uniqtime(&error->time);

	dev_priv->first_error = error;

	DRM_DEBUG("Time: %ld s %ld us\n", error->time.tv_sec,
		   error->time.tv_usec);
	DRM_DEBUG("EIR: 0x%08x\n", error->eir);
	DRM_DEBUG("  PGTBL_ER: 0x%08x\n", error->pgtbl_er);
	DRM_DEBUG("  INSTPM: 0x%08x\n", error->instpm);
	DRM_DEBUG("  IPEIR: 0x%08x\n", error->ipeir);
	DRM_DEBUG("  IPEHR: 0x%08x\n", error->ipehr);
	DRM_DEBUG("  INSTDONE: 0x%08x\n", error->instdone);
	DRM_DEBUG("  ACTHD: 0x%08x\n", error->acthd);
	DRM_DEBUG("  DMA_FADD_P: 0x%08x\n", I915_READ(0x2078));
	if (IS_I965G(dev)) {
		DRM_DEBUG("  INSTPS: 0x%08x\n", error->instps);
		DRM_DEBUG("  INSTDONE1: 0x%08x\n", error->instdone1);
	}
	drm_free(error, sizeof(*error), DRM_MEM_DRIVER);
out:
	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
}

/**
 * i915_handle_error - handle an error interrupt
 * @dev: drm device
 *
 * Do some basic checking of regsiter state at error interrupt time and
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
void i915_handle_error(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 eir = I915_READ(EIR);
	u32 pipea_stats = I915_READ(PIPEASTAT);
	u32 pipeb_stats = I915_READ(PIPEBSTAT);

	i915_capture_error_state(dev);

	DRM_DEBUG("render error detected, EIR: 0x%08x\n",
	       eir);

	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

			DRM_DEBUG("  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR_I965));
			DRM_DEBUG("  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR_I965));
			DRM_DEBUG("  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE_I965));
			DRM_DEBUG("  INSTPS: 0x%08x\n",
			       I915_READ(INSTPS));
			DRM_DEBUG("  INSTDONE1: 0x%08x\n",
			       I915_READ(INSTDONE1));
			DRM_DEBUG("  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD_I965));
			I915_WRITE(IPEIR_I965, ipeir);
			(void)I915_READ(IPEIR_I965);
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
			DRM_DEBUG("page table error\n");
			DRM_DEBUG("  PGTBL_ER: 0x%08x\n",
			       pgtbl_err);
			I915_WRITE(PGTBL_ER, pgtbl_err);
			(void)I915_READ(PGTBL_ER);
		}
	}

	if (IS_I9XX(dev)) {
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
			DRM_DEBUG("page table error\n");
			DRM_DEBUG("PGTBL_ER: 0x%08x\n",
			       pgtbl_err);
			I915_WRITE(PGTBL_ER, pgtbl_err);
			(void)I915_READ(PGTBL_ER);
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
		DRM_DEBUG("memory refresh error\n");
		DRM_DEBUG("PIPEASTAT: 0x%08x\n",
		       pipea_stats);
		DRM_DEBUG("PIPEBSTAT: 0x%08x\n",
		       pipeb_stats);
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
		DRM_DEBUG("instruction error\n");
		DRM_DEBUG("  INSTPM: 0x%08x\n",
		       I915_READ(INSTPM));
		if (!IS_I965G(dev)) {
			u32 ipeir = I915_READ(IPEIR);

			DRM_DEBUG("  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR));
			DRM_DEBUG("  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR));
			DRM_DEBUG("  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE));
			DRM_DEBUG("  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD));
			I915_WRITE(IPEIR, ipeir);
			(void)I915_READ(IPEIR);
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

			DRM_DEBUG("  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR_I965));
			DRM_DEBUG("  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR_I965));
			DRM_DEBUG("  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE_I965));
			DRM_DEBUG("  INSTPS: 0x%08x\n",
			       I915_READ(INSTPS));
			DRM_DEBUG("  INSTDONE1: 0x%08x\n",
			       I915_READ(INSTDONE1));
			DRM_DEBUG("  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD_I965));
			I915_WRITE(IPEIR_I965, ipeir);
			(void)I915_READ(IPEIR_I965);
		}
	}

	I915_WRITE(EIR, eir);
	(void)I915_READ(EIR);
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_DEBUG("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}

}

u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
{
       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
       int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;

       if (!i915_pipe_enabled(dev, pipe)) {
		DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
               return 0;
       }

       return I915_READ(reg);
}

irqreturn_t igdng_irq_handler(struct drm_device *dev)
{
       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
       int ret = IRQ_NONE;
       u32 de_iir, gt_iir, de_ier;
       u32 new_de_iir, new_gt_iir;
       int vblank = 0;

	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
	(void)I915_READ(DEIER);

       de_iir = I915_READ(DEIIR);
       gt_iir = I915_READ(GTIIR);

       for (;;) {
               if (de_iir == 0 && gt_iir == 0)
                       break;

               ret = IRQ_HANDLED;

               I915_WRITE(DEIIR, de_iir);
               new_de_iir = I915_READ(DEIIR);
               I915_WRITE(GTIIR, gt_iir);
               new_gt_iir = I915_READ(GTIIR);

        if (dev_priv->sarea_priv) {
            dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);

	}

               if (gt_iir & GT_USER_INTERRUPT) {
                       dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
                       DRM_WAKEUP(&dev_priv->irq_queue);
               }
               if (de_iir & DE_PIPEA_VBLANK) {
                       vblank++;
                       drm_handle_vblank(dev, 0);
               }

               if (de_iir & DE_PIPEB_VBLANK) {
                       vblank++;
                       drm_handle_vblank(dev, 1);
               }

               de_iir = new_de_iir;
               gt_iir = new_gt_iir;
       }

	I915_WRITE(DEIER, de_ier);
	(void)I915_READ(DEIER);

       return ret;
}

irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
{
        drm_device_t *dev = (drm_device_t *) (void *) arg;
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
        u32 iir;
        u32 pipea_stats = 0, pipeb_stats = 0;
	int vblank = 0;

	if (IS_IGDNG(dev))
		return igdng_irq_handler(dev);

	iir = I915_READ(IIR);

	if (iir == 0) {
		return IRQ_NONE;
	}
start:

	if (dev_priv->sarea_priv) {
		if (dev_priv->hw_status_page)
	    		dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
	}

	I915_WRITE(IIR, iir);

	(void) I915_READ(IIR); /* Flush posted writes */


	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
		i915_handle_error(dev);

        if (iir & I915_USER_INTERRUPT) {
		dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
                DRM_WAKEUP(&dev_priv->irq_queue);
        }

        if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
                pipea_stats = I915_READ(PIPEASTAT);

                /* The vblank interrupt gets enabled even if we didn't ask for
                   it, so make sure it's shut down again */
                if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A))
                        pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
                                         PIPE_VBLANK_INTERRUPT_ENABLE);
                else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
                                        PIPE_VBLANK_INTERRUPT_STATUS))
                {
                        vblank++;
                        drm_handle_vblank(dev, 0);
                }

                I915_WRITE(PIPEASTAT, pipea_stats);
        }
        if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
                pipeb_stats = I915_READ(PIPEBSTAT);

                /* The vblank interrupt gets enabled even if we didn't ask for
                   it, so make sure it's shut down again */
                if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
                        pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
                                         PIPE_VBLANK_INTERRUPT_ENABLE);
                else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
                                        PIPE_VBLANK_INTERRUPT_STATUS))
                {
                        vblank++;
                        drm_handle_vblank(dev, 1);
                }

                I915_WRITE(PIPEBSTAT, pipeb_stats);
        }
       return IRQ_HANDLED;

}

int i915_emit_irq(drm_device_t * dev)
{

	drm_i915_private_t *dev_priv = dev->dev_private;
	RING_LOCALS;

	i915_kernel_lost_context(dev);

	dev_priv->counter++;
	if (dev_priv->counter > 0x7FFFFFFFUL)
		dev_priv->counter = 1;
	if (dev_priv->sarea_priv)
		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;

#if defined(__i386)
	if (IS_GM45(dev)) {
		BEGIN_LP_RING(3);
		OUT_RING(MI_STORE_DWORD_INDEX);
		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
		OUT_RING(dev_priv->counter);
		ADVANCE_LP_RING();

		(void) READ_BREADCRUMB(dev_priv);
		BEGIN_LP_RING(2);
		OUT_RING(0);
		OUT_RING(MI_USER_INTERRUPT);
		ADVANCE_LP_RING();
	} else {
#endif  /* __i386 */
	BEGIN_LP_RING(4);
	OUT_RING(MI_STORE_DWORD_INDEX);
	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
	OUT_RING(dev_priv->counter);
	OUT_RING(MI_USER_INTERRUPT);
	ADVANCE_LP_RING();
#if defined(__i386)
	}
#endif  /* __i386 */

#if defined(__i386)
	if (IS_I965GM(dev) || IS_IGDNG(dev) || IS_GM45(dev))
#else
	if (IS_I965GM(dev) || IS_IGDNG(dev))
#endif  /* __i386 */
	{
		(void) READ_BREADCRUMB(dev_priv);
		BEGIN_LP_RING(2);
		OUT_RING(0);
		OUT_RING(0);
		ADVANCE_LP_RING();
		(void) READ_BREADCRUMB(dev_priv);
	}

	return dev_priv->counter;
}

void i915_user_irq_on(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	spin_lock(&dev_priv->user_irq_lock);
	if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
               if (IS_IGDNG(dev))
                       igdng_enable_irq(dev_priv, GT_USER_INTERRUPT, 1);
               else
                       i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
	}
	spin_unlock(&dev_priv->user_irq_lock);

}

void i915_user_irq_off(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	spin_lock(&dev_priv->user_irq_lock);
	if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
               if (IS_IGDNG(dev))
                       igdng_disable_irq(dev_priv, GT_USER_INTERRUPT, 1);
               else
                       i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
	}
	spin_unlock(&dev_priv->user_irq_lock);
}


static int i915_wait_irq(drm_device_t * dev, int irq_nr)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int ret = 0;
	int wait_time = 0;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

waitmore:
	wait_time++;
	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
		if (dev_priv->sarea_priv) {
			dev_priv->sarea_priv->last_dispatch =
				READ_BREADCRUMB(dev_priv);
		}
		return 0;
	}
	DRM_DEBUG("i915_wait_irq: irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv));
	i915_user_irq_on(dev);
	DRM_WAIT_ON(ret, &dev_priv->irq_queue, 3 * DRM_HZ,
		    READ_BREADCRUMB(dev_priv) >= irq_nr);
	i915_user_irq_off(dev);

	if (ret == EBUSY) {
		if (wait_time > 5) {
		DRM_DEBUG("%d: EBUSY -- rec: %d emitted: %d\n",
			  ret,
			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
			return ret;
		}
		goto waitmore;
	}

	if (dev_priv->sarea_priv)
		dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);

	if (ret == EINTR) {
		if (wait_time > 5) {
			DRM_DEBUG("EINTR wait %d now %d", dev_priv->counter, READ_BREADCRUMB(dev_priv));
			return ret;
		}
		goto waitmore;
	}

	return ret;
}


/* Needs the lock as it touches the ring.
 */
/*ARGSUSED*/
int i915_irq_emit(DRM_IOCTL_ARGS)
{
	DRM_DEVICE;
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_irq_emit_t emit;
	int result;

	LOCK_TEST_WITH_RETURN(dev, fpriv);

	if (!dev_priv) {
		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
		return (EINVAL);
	}


	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
		drm_i915_irq_emit32_t irq_emit32;

		DRM_COPYFROM_WITH_RETURN(&irq_emit32,
			(drm_i915_irq_emit32_t __user *) data,
			sizeof (drm_i915_irq_emit32_t));
		emit.irq_seq = (int __user *)(uintptr_t)irq_emit32.irq_seq;
	} else
		DRM_COPYFROM_WITH_RETURN(&emit,
		    (drm_i915_irq_emit_t __user *) data, sizeof(emit));

	spin_lock(&dev->struct_mutex);
	result = i915_emit_irq(dev);
	spin_unlock(&dev->struct_mutex);

	if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
		DRM_ERROR("copy_to_user\n");
		return (EFAULT);
	}

	return 0;
}

/* Doesn't need the hardware lock.
 */
/*ARGSUSED*/
int i915_irq_wait(DRM_IOCTL_ARGS)
{
	DRM_DEVICE;
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_irq_wait_t irqwait;

	if (!dev_priv) {
		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
		return (EINVAL);
	}

	DRM_COPYFROM_WITH_RETURN(&irqwait,
	    (drm_i915_irq_wait_t __user *) data, sizeof(irqwait));

	return i915_wait_irq(dev, irqwait.irq_seq);
}

static void igdng_enable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	u32 vblank;

	if (pipe == 0)
		vblank = DE_PIPEA_VBLANK;
	else
		vblank = DE_PIPEB_VBLANK;

	if ((dev_priv->de_irq_enable_reg & vblank) == 0) {
		igdng_enable_irq(dev_priv, vblank, 0);
		dev_priv->de_irq_enable_reg |= vblank;
		I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
		(void) I915_READ(DEIER);
	}
}

static void igdng_disable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	u32 vblank;

	if (pipe == 0)
		vblank = DE_PIPEA_VBLANK;
	else
		vblank = DE_PIPEB_VBLANK;

	if ((dev_priv->de_irq_enable_reg & vblank) != 0) {
		igdng_disable_irq(dev_priv, vblank, 0);
		dev_priv->de_irq_enable_reg &= ~vblank;
		I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
		(void) I915_READ(DEIER);
	}
}

int i915_enable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
	u32 pipeconf;

	pipeconf = I915_READ(pipeconf_reg);
	if (!(pipeconf & PIPEACONF_ENABLE))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
	if (IS_IGDNG(dev))
		igdng_enable_vblank(dev, pipe);
	else if (IS_I965G(dev))
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
	else
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_VBLANK_INTERRUPT_ENABLE);
	spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);

	return 0;
}

void i915_disable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
	if (IS_IGDNG(dev))
		igdng_disable_vblank(dev, pipe);
	else
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_VBLANK_INTERRUPT_ENABLE |
			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
	spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
}

/* Set the vblank monitor pipe
 */
/*ARGSUSED*/
int i915_vblank_pipe_set(DRM_IOCTL_ARGS)
{
	DRM_DEVICE;
	drm_i915_private_t *dev_priv = dev->dev_private;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return (-EINVAL);
	}

	return (0);
}

/*ARGSUSED*/
int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
{
	DRM_DEVICE;
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_vblank_pipe_t pipe;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

	DRM_COPYFROM_WITH_RETURN(&pipe, (drm_i915_vblank_pipe_t __user *)data, sizeof (pipe));

	pipe.pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;

	return 0;
}

/**
 * Schedule buffer swap at given vertical blank.
 */
/*ARGSUSED*/
int i915_vblank_swap(DRM_IOCTL_ARGS)
{
        /* The delayed swap mechanism was fundamentally racy, and has been
        * removed.  The model was that the client requested a delayed flip/swap
        * from the kernel, then waited for vblank before continuing to perform
        * rendering.  The problem was that the kernel might wake the client
        * up before it dispatched the vblank swap (since the lock has to be
        * held while touching the ringbuffer), in which case the client would
        * clear and start the next frame before the swap occurred, and
        * flicker would occur in addition to likely missing the vblank.
        *
        * In the absence of this ioctl, userland falls back to a correct path
        * of waiting for a vblank, then dispatching the swap on its own.
        * Context switching to userland and back is plenty fast enough for
        * meeting the requirements of vblank swapping.
        */
	return -EINVAL;

}

/* drm_dma.h hooks
*/

static void igdng_irq_preinstall(struct drm_device *dev)
{
       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

       I915_WRITE(HWSTAM, 0xeffe);

      /* XXX hotplug from PCH */

       I915_WRITE(DEIMR, 0xffffffff);
       I915_WRITE(DEIER, 0x0);
       (void) I915_READ(DEIER);

       /* and GT */
       I915_WRITE(GTIMR, 0xffffffff);
       I915_WRITE(GTIER, 0x0);
       (void) I915_READ(GTIER);
}

static int igdng_irq_postinstall(struct drm_device *dev)
{
       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
       /* enable kind of interrupts always enabled */
       u32 display_mask = DE_MASTER_IRQ_CONTROL /*| DE_PCH_EVENT */;
       u32 render_mask = GT_USER_INTERRUPT;

       dev_priv->irq_mask_reg = ~display_mask;
       dev_priv->de_irq_enable_reg = display_mask;

       /* should always can generate irq */
       I915_WRITE(DEIIR, I915_READ(DEIIR));
       (void) I915_READ(DEIIR);
       I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
       I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
       (void) I915_READ(DEIER);

       /* user interrupt should be enabled, but masked initial */
       dev_priv->gt_irq_mask_reg = 0xffffffff;
       dev_priv->gt_irq_enable_reg = render_mask;

       I915_WRITE(GTIIR, I915_READ(GTIIR));
       (void) I915_READ(GTIIR);
       I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
       I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg);
       (void) I915_READ(GTIER);

       return 0;
}

static void igdng_irq_uninstall(struct drm_device *dev)
{
       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
       I915_WRITE(HWSTAM, 0xffffffff);

       I915_WRITE(DEIMR, 0xffffffff);
       I915_WRITE(DEIER, 0x0);
       I915_WRITE(DEIIR, I915_READ(DEIIR));

       I915_WRITE(GTIMR, 0xffffffff);
       I915_WRITE(GTIER, 0x0);
       I915_WRITE(GTIIR, I915_READ(GTIIR));
}

int i915_driver_irq_preinstall(drm_device_t * dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	if (!dev_priv->mmio_map)
		return -EINVAL;

	if (IS_IGDNG(dev)) {
               igdng_irq_preinstall(dev);
               return 0;
	}

	I915_WRITE16(HWSTAM, 0xeffe);
	I915_WRITE(PIPEASTAT, 0);
	I915_WRITE(PIPEBSTAT, 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE16(IER, 0x0);
	(void) I915_READ(IER);

	return 0;
}

void i915_driver_irq_postinstall(drm_device_t * dev)
{
	int error_mask;
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;

	if (IS_IGDNG(dev)) {
		(void) igdng_irq_postinstall(dev);
		DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
		return;
	}

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;

	dev_priv->pipestat[0] = 0;
	dev_priv->pipestat[1] = 0;

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	/* Disable pipe interrupt enables, clear pending pipe status */
	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
	(void) I915_READ(PIPEASTAT);
        (void) I915_READ(PIPEBSTAT);
	/* Clear pending interrupt status */
	I915_WRITE(IIR, I915_READ(IIR));

	(void) I915_READ(IIR);
	I915_WRITE(IMR, dev_priv->irq_mask_reg);
	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
	(void) I915_READ(IER);

	DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));

	return;
}

void i915_driver_irq_uninstall(drm_device_t * dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	if ((!dev_priv) || (dev->irq_enabled == 0))
		return;

	dev_priv->vblank_pipe = 0;

	if (IS_IGDNG(dev)) {
		igdng_irq_uninstall(dev);
		DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
		return;
	}

	I915_WRITE(HWSTAM, 0xffffffff);
	I915_WRITE(PIPEASTAT, 0);
	I915_WRITE(PIPEBSTAT, 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));

	DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
}