158471f63SJani Nikula /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*- 258471f63SJani Nikula */ 358471f63SJani Nikula /* 458471f63SJani Nikula * 558471f63SJani Nikula * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 658471f63SJani Nikula * All Rights Reserved. 758471f63SJani Nikula * 858471f63SJani Nikula * Permission is hereby granted, free of charge, to any person obtaining a 958471f63SJani Nikula * copy of this software and associated documentation files (the 1058471f63SJani Nikula * "Software"), to deal in the Software without restriction, including 1158471f63SJani Nikula * without limitation the rights to use, copy, modify, merge, publish, 1258471f63SJani Nikula * distribute, sub license, and/or sell copies of the Software, and to 1358471f63SJani Nikula * permit persons to whom the Software is furnished to do so, subject to 1458471f63SJani Nikula * the following conditions: 1558471f63SJani Nikula * 1658471f63SJani Nikula * The above copyright notice and this permission notice (including the 1758471f63SJani Nikula * next paragraph) shall be included in all copies or substantial portions 1858471f63SJani Nikula * of the Software. 1958471f63SJani Nikula * 2058471f63SJani Nikula * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 2158471f63SJani Nikula * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2258471f63SJani Nikula * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 2358471f63SJani Nikula * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 2458471f63SJani Nikula * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 2558471f63SJani Nikula * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 2658471f63SJani Nikula * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 2758471f63SJani Nikula * 2858471f63SJani Nikula */ 2958471f63SJani Nikula 3058471f63SJani Nikula #include <linux/acpi.h> 3158471f63SJani Nikula #include <linux/device.h> 3258471f63SJani Nikula #include <linux/module.h> 3358471f63SJani Nikula #include <linux/oom.h> 3458471f63SJani Nikula #include <linux/pci.h> 3558471f63SJani Nikula #include <linux/pm.h> 3658471f63SJani Nikula #include <linux/pm_runtime.h> 3758471f63SJani Nikula #include <linux/pnp.h> 3858471f63SJani Nikula #include <linux/slab.h> 3958471f63SJani Nikula #include <linux/vga_switcheroo.h> 4058471f63SJani Nikula #include <linux/vt.h> 4158471f63SJani Nikula 4258471f63SJani Nikula #include <drm/drm_aperture.h> 4358471f63SJani Nikula #include <drm/drm_atomic_helper.h> 4458471f63SJani Nikula #include <drm/drm_ioctl.h> 4558471f63SJani Nikula #include <drm/drm_managed.h> 4658471f63SJani Nikula #include <drm/drm_probe_helper.h> 4758471f63SJani Nikula 4858471f63SJani Nikula #include "display/intel_acpi.h" 4958471f63SJani Nikula #include "display/intel_bw.h" 5058471f63SJani Nikula #include "display/intel_cdclk.h" 5158471f63SJani Nikula #include "display/intel_display_types.h" 5258471f63SJani Nikula #include "display/intel_dmc.h" 5358471f63SJani Nikula #include "display/intel_dp.h" 5458471f63SJani Nikula #include "display/intel_dpt.h" 5558471f63SJani Nikula #include "display/intel_fbdev.h" 5658471f63SJani Nikula #include "display/intel_hotplug.h" 5758471f63SJani Nikula #include "display/intel_overlay.h" 5858471f63SJani Nikula #include "display/intel_pch_refclk.h" 5958471f63SJani Nikula #include "display/intel_pipe_crc.h" 6058471f63SJani Nikula #include "display/intel_pps.h" 6158471f63SJani Nikula #include "display/intel_sprite.h" 6258471f63SJani Nikula #include "display/intel_vga.h" 6358471f63SJani Nikula 6458471f63SJani Nikula #include "gem/i915_gem_context.h" 6558471f63SJani Nikula #include "gem/i915_gem_ioctls.h" 6658471f63SJani Nikula #include "gem/i915_gem_mman.h" 6758471f63SJani Nikula #include "gem/i915_gem_pm.h" 6858471f63SJani Nikula #include "gt/intel_gt.h" 6958471f63SJani Nikula #include "gt/intel_gt_pm.h" 7058471f63SJani Nikula #include "gt/intel_rc6.h" 7158471f63SJani Nikula 7258471f63SJani Nikula #include "pxp/intel_pxp_pm.h" 7358471f63SJani Nikula 7458471f63SJani Nikula #include "i915_debugfs.h" 7558471f63SJani Nikula #include "i915_driver.h" 7658471f63SJani Nikula #include "i915_drv.h" 7758471f63SJani Nikula #include "i915_ioc32.h" 7858471f63SJani Nikula #include "i915_irq.h" 7958471f63SJani Nikula #include "i915_memcpy.h" 8058471f63SJani Nikula #include "i915_perf.h" 8158471f63SJani Nikula #include "i915_query.h" 8258471f63SJani Nikula #include "i915_suspend.h" 8358471f63SJani Nikula #include "i915_switcheroo.h" 8458471f63SJani Nikula #include "i915_sysfs.h" 8558471f63SJani Nikula #include "i915_vgpu.h" 8658471f63SJani Nikula #include "intel_dram.h" 8758471f63SJani Nikula #include "intel_gvt.h" 8858471f63SJani Nikula #include "intel_memory_region.h" 8958471f63SJani Nikula #include "intel_pcode.h" 9058471f63SJani Nikula #include "intel_pm.h" 9158471f63SJani Nikula #include "intel_region_ttm.h" 9258471f63SJani Nikula #include "vlv_suspend.h" 9358471f63SJani Nikula 944588d7ebSJani Nikula static const struct drm_driver i915_drm_driver; 9558471f63SJani Nikula 9658471f63SJani Nikula static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) 9758471f63SJani Nikula { 9858471f63SJani Nikula int domain = pci_domain_nr(to_pci_dev(dev_priv->drm.dev)->bus); 9958471f63SJani Nikula 10058471f63SJani Nikula dev_priv->bridge_dev = 10158471f63SJani Nikula pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0)); 10258471f63SJani Nikula if (!dev_priv->bridge_dev) { 10358471f63SJani Nikula drm_err(&dev_priv->drm, "bridge device not found\n"); 10458471f63SJani Nikula return -EIO; 10558471f63SJani Nikula } 10658471f63SJani Nikula return 0; 10758471f63SJani Nikula } 10858471f63SJani Nikula 10958471f63SJani Nikula /* Allocate space for the MCH regs if needed, return nonzero on error */ 11058471f63SJani Nikula static int 11158471f63SJani Nikula intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) 11258471f63SJani Nikula { 11358471f63SJani Nikula int reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 11458471f63SJani Nikula u32 temp_lo, temp_hi = 0; 11558471f63SJani Nikula u64 mchbar_addr; 11658471f63SJani Nikula int ret; 11758471f63SJani Nikula 11858471f63SJani Nikula if (GRAPHICS_VER(dev_priv) >= 4) 11958471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); 12058471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); 12158471f63SJani Nikula mchbar_addr = ((u64)temp_hi << 32) | temp_lo; 12258471f63SJani Nikula 12358471f63SJani Nikula /* If ACPI doesn't have it, assume we need to allocate it ourselves */ 12458471f63SJani Nikula #ifdef CONFIG_PNP 12558471f63SJani Nikula if (mchbar_addr && 12658471f63SJani Nikula pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) 12758471f63SJani Nikula return 0; 12858471f63SJani Nikula #endif 12958471f63SJani Nikula 13058471f63SJani Nikula /* Get some space for it */ 13158471f63SJani Nikula dev_priv->mch_res.name = "i915 MCHBAR"; 13258471f63SJani Nikula dev_priv->mch_res.flags = IORESOURCE_MEM; 13358471f63SJani Nikula ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus, 13458471f63SJani Nikula &dev_priv->mch_res, 13558471f63SJani Nikula MCHBAR_SIZE, MCHBAR_SIZE, 13658471f63SJani Nikula PCIBIOS_MIN_MEM, 13758471f63SJani Nikula 0, pcibios_align_resource, 13858471f63SJani Nikula dev_priv->bridge_dev); 13958471f63SJani Nikula if (ret) { 14058471f63SJani Nikula drm_dbg(&dev_priv->drm, "failed bus alloc: %d\n", ret); 14158471f63SJani Nikula dev_priv->mch_res.start = 0; 14258471f63SJani Nikula return ret; 14358471f63SJani Nikula } 14458471f63SJani Nikula 14558471f63SJani Nikula if (GRAPHICS_VER(dev_priv) >= 4) 14658471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, reg + 4, 14758471f63SJani Nikula upper_32_bits(dev_priv->mch_res.start)); 14858471f63SJani Nikula 14958471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, reg, 15058471f63SJani Nikula lower_32_bits(dev_priv->mch_res.start)); 15158471f63SJani Nikula return 0; 15258471f63SJani Nikula } 15358471f63SJani Nikula 15458471f63SJani Nikula /* Setup MCHBAR if possible, return true if we should disable it again */ 15558471f63SJani Nikula static void 15658471f63SJani Nikula intel_setup_mchbar(struct drm_i915_private *dev_priv) 15758471f63SJani Nikula { 15858471f63SJani Nikula int mchbar_reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 15958471f63SJani Nikula u32 temp; 16058471f63SJani Nikula bool enabled; 16158471f63SJani Nikula 16258471f63SJani Nikula if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 16358471f63SJani Nikula return; 16458471f63SJani Nikula 16558471f63SJani Nikula dev_priv->mchbar_need_disable = false; 16658471f63SJani Nikula 16758471f63SJani Nikula if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 16858471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); 16958471f63SJani Nikula enabled = !!(temp & DEVEN_MCHBAR_EN); 17058471f63SJani Nikula } else { 17158471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 17258471f63SJani Nikula enabled = temp & 1; 17358471f63SJani Nikula } 17458471f63SJani Nikula 17558471f63SJani Nikula /* If it's already enabled, don't have to do anything */ 17658471f63SJani Nikula if (enabled) 17758471f63SJani Nikula return; 17858471f63SJani Nikula 17958471f63SJani Nikula if (intel_alloc_mchbar_resource(dev_priv)) 18058471f63SJani Nikula return; 18158471f63SJani Nikula 18258471f63SJani Nikula dev_priv->mchbar_need_disable = true; 18358471f63SJani Nikula 18458471f63SJani Nikula /* Space is allocated or reserved, so enable it. */ 18558471f63SJani Nikula if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 18658471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 18758471f63SJani Nikula temp | DEVEN_MCHBAR_EN); 18858471f63SJani Nikula } else { 18958471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 19058471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); 19158471f63SJani Nikula } 19258471f63SJani Nikula } 19358471f63SJani Nikula 19458471f63SJani Nikula static void 19558471f63SJani Nikula intel_teardown_mchbar(struct drm_i915_private *dev_priv) 19658471f63SJani Nikula { 19758471f63SJani Nikula int mchbar_reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 19858471f63SJani Nikula 19958471f63SJani Nikula if (dev_priv->mchbar_need_disable) { 20058471f63SJani Nikula if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 20158471f63SJani Nikula u32 deven_val; 20258471f63SJani Nikula 20358471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, DEVEN, 20458471f63SJani Nikula &deven_val); 20558471f63SJani Nikula deven_val &= ~DEVEN_MCHBAR_EN; 20658471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 20758471f63SJani Nikula deven_val); 20858471f63SJani Nikula } else { 20958471f63SJani Nikula u32 mchbar_val; 21058471f63SJani Nikula 21158471f63SJani Nikula pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, 21258471f63SJani Nikula &mchbar_val); 21358471f63SJani Nikula mchbar_val &= ~1; 21458471f63SJani Nikula pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, 21558471f63SJani Nikula mchbar_val); 21658471f63SJani Nikula } 21758471f63SJani Nikula } 21858471f63SJani Nikula 21958471f63SJani Nikula if (dev_priv->mch_res.start) 22058471f63SJani Nikula release_resource(&dev_priv->mch_res); 22158471f63SJani Nikula } 22258471f63SJani Nikula 22358471f63SJani Nikula static int i915_workqueues_init(struct drm_i915_private *dev_priv) 22458471f63SJani Nikula { 22558471f63SJani Nikula /* 22658471f63SJani Nikula * The i915 workqueue is primarily used for batched retirement of 22758471f63SJani Nikula * requests (and thus managing bo) once the task has been completed 22858471f63SJani Nikula * by the GPU. i915_retire_requests() is called directly when we 22958471f63SJani Nikula * need high-priority retirement, such as waiting for an explicit 23058471f63SJani Nikula * bo. 23158471f63SJani Nikula * 23258471f63SJani Nikula * It is also used for periodic low-priority events, such as 23358471f63SJani Nikula * idle-timers and recording error state. 23458471f63SJani Nikula * 23558471f63SJani Nikula * All tasks on the workqueue are expected to acquire the dev mutex 23658471f63SJani Nikula * so there is no point in running more than one instance of the 23758471f63SJani Nikula * workqueue at any time. Use an ordered one. 23858471f63SJani Nikula */ 23958471f63SJani Nikula dev_priv->wq = alloc_ordered_workqueue("i915", 0); 24058471f63SJani Nikula if (dev_priv->wq == NULL) 24158471f63SJani Nikula goto out_err; 24258471f63SJani Nikula 24358471f63SJani Nikula dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); 24458471f63SJani Nikula if (dev_priv->hotplug.dp_wq == NULL) 24558471f63SJani Nikula goto out_free_wq; 24658471f63SJani Nikula 24758471f63SJani Nikula return 0; 24858471f63SJani Nikula 24958471f63SJani Nikula out_free_wq: 25058471f63SJani Nikula destroy_workqueue(dev_priv->wq); 25158471f63SJani Nikula out_err: 25258471f63SJani Nikula drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n"); 25358471f63SJani Nikula 25458471f63SJani Nikula return -ENOMEM; 25558471f63SJani Nikula } 25658471f63SJani Nikula 25758471f63SJani Nikula static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) 25858471f63SJani Nikula { 25958471f63SJani Nikula destroy_workqueue(dev_priv->hotplug.dp_wq); 26058471f63SJani Nikula destroy_workqueue(dev_priv->wq); 26158471f63SJani Nikula } 26258471f63SJani Nikula 26358471f63SJani Nikula /* 26458471f63SJani Nikula * We don't keep the workarounds for pre-production hardware, so we expect our 26558471f63SJani Nikula * driver to fail on these machines in one way or another. A little warning on 26658471f63SJani Nikula * dmesg may help both the user and the bug triagers. 26758471f63SJani Nikula * 26858471f63SJani Nikula * Our policy for removing pre-production workarounds is to keep the 26958471f63SJani Nikula * current gen workarounds as a guide to the bring-up of the next gen 27058471f63SJani Nikula * (workarounds have a habit of persisting!). Anything older than that 27158471f63SJani Nikula * should be removed along with the complications they introduce. 27258471f63SJani Nikula */ 27358471f63SJani Nikula static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) 27458471f63SJani Nikula { 27558471f63SJani Nikula bool pre = false; 27658471f63SJani Nikula 27758471f63SJani Nikula pre |= IS_HSW_EARLY_SDV(dev_priv); 27858471f63SJani Nikula pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6; 27958471f63SJani Nikula pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA; 28058471f63SJani Nikula pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1; 28158471f63SJani Nikula pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3; 28258471f63SJani Nikula pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7; 28358471f63SJani Nikula 28458471f63SJani Nikula if (pre) { 28558471f63SJani Nikula drm_err(&dev_priv->drm, "This is a pre-production stepping. " 28658471f63SJani Nikula "It may not be fully functional.\n"); 28758471f63SJani Nikula add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 28858471f63SJani Nikula } 28958471f63SJani Nikula } 29058471f63SJani Nikula 29158471f63SJani Nikula static void sanitize_gpu(struct drm_i915_private *i915) 29258471f63SJani Nikula { 29358471f63SJani Nikula if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) 2944817c37dSDave Airlie __intel_gt_reset(to_gt(i915), ALL_ENGINES); 29558471f63SJani Nikula } 29658471f63SJani Nikula 29758471f63SJani Nikula /** 29858471f63SJani Nikula * i915_driver_early_probe - setup state not requiring device access 29958471f63SJani Nikula * @dev_priv: device private 30058471f63SJani Nikula * 30158471f63SJani Nikula * Initialize everything that is a "SW-only" state, that is state not 30258471f63SJani Nikula * requiring accessing the device or exposing the driver via kernel internal 30358471f63SJani Nikula * or userspace interfaces. Example steps belonging here: lock initialization, 30458471f63SJani Nikula * system memory allocation, setting up device specific attributes and 30558471f63SJani Nikula * function hooks not requiring accessing the device. 30658471f63SJani Nikula */ 30758471f63SJani Nikula static int i915_driver_early_probe(struct drm_i915_private *dev_priv) 30858471f63SJani Nikula { 30958471f63SJani Nikula int ret = 0; 31058471f63SJani Nikula 31158471f63SJani Nikula if (i915_inject_probe_failure(dev_priv)) 31258471f63SJani Nikula return -ENODEV; 31358471f63SJani Nikula 31458471f63SJani Nikula intel_device_info_subplatform_init(dev_priv); 31558471f63SJani Nikula intel_step_init(dev_priv); 31658471f63SJani Nikula 3174817c37dSDave Airlie intel_gt_init_early(to_gt(dev_priv), dev_priv); 31858471f63SJani Nikula intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug); 3194817c37dSDave Airlie intel_uncore_init_early(&dev_priv->uncore, to_gt(dev_priv)); 32058471f63SJani Nikula 32158471f63SJani Nikula spin_lock_init(&dev_priv->irq_lock); 32258471f63SJani Nikula spin_lock_init(&dev_priv->gpu_error.lock); 32358471f63SJani Nikula mutex_init(&dev_priv->backlight_lock); 32458471f63SJani Nikula 32558471f63SJani Nikula mutex_init(&dev_priv->sb_lock); 32658471f63SJani Nikula cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE); 32758471f63SJani Nikula 32858471f63SJani Nikula mutex_init(&dev_priv->audio.mutex); 32958471f63SJani Nikula mutex_init(&dev_priv->wm.wm_mutex); 33058471f63SJani Nikula mutex_init(&dev_priv->pps_mutex); 33158471f63SJani Nikula mutex_init(&dev_priv->hdcp_comp_mutex); 33258471f63SJani Nikula 33358471f63SJani Nikula i915_memcpy_init_early(dev_priv); 33458471f63SJani Nikula intel_runtime_pm_init_early(&dev_priv->runtime_pm); 33558471f63SJani Nikula 33658471f63SJani Nikula ret = i915_workqueues_init(dev_priv); 33758471f63SJani Nikula if (ret < 0) 33858471f63SJani Nikula return ret; 33958471f63SJani Nikula 34058471f63SJani Nikula ret = vlv_suspend_init(dev_priv); 34158471f63SJani Nikula if (ret < 0) 34258471f63SJani Nikula goto err_workqueues; 34358471f63SJani Nikula 34458471f63SJani Nikula ret = intel_region_ttm_device_init(dev_priv); 34558471f63SJani Nikula if (ret) 34658471f63SJani Nikula goto err_ttm; 34758471f63SJani Nikula 34858471f63SJani Nikula intel_wopcm_init_early(&dev_priv->wopcm); 34958471f63SJani Nikula 3504817c37dSDave Airlie __intel_gt_init_early(to_gt(dev_priv), dev_priv); 35158471f63SJani Nikula 35258471f63SJani Nikula i915_gem_init_early(dev_priv); 35358471f63SJani Nikula 35458471f63SJani Nikula /* This must be called before any calls to HAS_PCH_* */ 35558471f63SJani Nikula intel_detect_pch(dev_priv); 35658471f63SJani Nikula 35758471f63SJani Nikula intel_pm_setup(dev_priv); 35858471f63SJani Nikula ret = intel_power_domains_init(dev_priv); 35958471f63SJani Nikula if (ret < 0) 36058471f63SJani Nikula goto err_gem; 36158471f63SJani Nikula intel_irq_init(dev_priv); 36258471f63SJani Nikula intel_init_display_hooks(dev_priv); 36358471f63SJani Nikula intel_init_clock_gating_hooks(dev_priv); 36458471f63SJani Nikula 36558471f63SJani Nikula intel_detect_preproduction_hw(dev_priv); 36658471f63SJani Nikula 36758471f63SJani Nikula return 0; 36858471f63SJani Nikula 36958471f63SJani Nikula err_gem: 37058471f63SJani Nikula i915_gem_cleanup_early(dev_priv); 3714817c37dSDave Airlie intel_gt_driver_late_release(to_gt(dev_priv)); 37258471f63SJani Nikula intel_region_ttm_device_fini(dev_priv); 37358471f63SJani Nikula err_ttm: 37458471f63SJani Nikula vlv_suspend_cleanup(dev_priv); 37558471f63SJani Nikula err_workqueues: 37658471f63SJani Nikula i915_workqueues_cleanup(dev_priv); 37758471f63SJani Nikula return ret; 37858471f63SJani Nikula } 37958471f63SJani Nikula 38058471f63SJani Nikula /** 38158471f63SJani Nikula * i915_driver_late_release - cleanup the setup done in 38258471f63SJani Nikula * i915_driver_early_probe() 38358471f63SJani Nikula * @dev_priv: device private 38458471f63SJani Nikula */ 38558471f63SJani Nikula static void i915_driver_late_release(struct drm_i915_private *dev_priv) 38658471f63SJani Nikula { 38758471f63SJani Nikula intel_irq_fini(dev_priv); 38858471f63SJani Nikula intel_power_domains_cleanup(dev_priv); 38958471f63SJani Nikula i915_gem_cleanup_early(dev_priv); 3904817c37dSDave Airlie intel_gt_driver_late_release(to_gt(dev_priv)); 39158471f63SJani Nikula intel_region_ttm_device_fini(dev_priv); 39258471f63SJani Nikula vlv_suspend_cleanup(dev_priv); 39358471f63SJani Nikula i915_workqueues_cleanup(dev_priv); 39458471f63SJani Nikula 39558471f63SJani Nikula cpu_latency_qos_remove_request(&dev_priv->sb_qos); 39658471f63SJani Nikula mutex_destroy(&dev_priv->sb_lock); 39758471f63SJani Nikula 39858471f63SJani Nikula i915_params_free(&dev_priv->params); 39958471f63SJani Nikula } 40058471f63SJani Nikula 40158471f63SJani Nikula /** 40258471f63SJani Nikula * i915_driver_mmio_probe - setup device MMIO 40358471f63SJani Nikula * @dev_priv: device private 40458471f63SJani Nikula * 40558471f63SJani Nikula * Setup minimal device state necessary for MMIO accesses later in the 40658471f63SJani Nikula * initialization sequence. The setup here should avoid any other device-wide 40758471f63SJani Nikula * side effects or exposing the driver via kernel internal or user space 40858471f63SJani Nikula * interfaces. 40958471f63SJani Nikula */ 41058471f63SJani Nikula static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) 41158471f63SJani Nikula { 41258471f63SJani Nikula int ret; 41358471f63SJani Nikula 41458471f63SJani Nikula if (i915_inject_probe_failure(dev_priv)) 41558471f63SJani Nikula return -ENODEV; 41658471f63SJani Nikula 41758471f63SJani Nikula ret = i915_get_bridge_dev(dev_priv); 41858471f63SJani Nikula if (ret < 0) 41958471f63SJani Nikula return ret; 42058471f63SJani Nikula 421211b4dbcSDave Airlie ret = intel_uncore_setup_mmio(&dev_priv->uncore); 42258471f63SJani Nikula if (ret < 0) 42358471f63SJani Nikula goto err_bridge; 42458471f63SJani Nikula 425211b4dbcSDave Airlie ret = intel_uncore_init_mmio(&dev_priv->uncore); 426211b4dbcSDave Airlie if (ret) 427211b4dbcSDave Airlie goto err_mmio; 428211b4dbcSDave Airlie 42958471f63SJani Nikula /* Try to make sure MCHBAR is enabled before poking at it */ 43058471f63SJani Nikula intel_setup_mchbar(dev_priv); 43158471f63SJani Nikula intel_device_info_runtime_init(dev_priv); 43258471f63SJani Nikula 4334817c37dSDave Airlie ret = intel_gt_init_mmio(to_gt(dev_priv)); 43458471f63SJani Nikula if (ret) 43558471f63SJani Nikula goto err_uncore; 43658471f63SJani Nikula 43758471f63SJani Nikula /* As early as possible, scrub existing GPU state before clobbering */ 43858471f63SJani Nikula sanitize_gpu(dev_priv); 43958471f63SJani Nikula 44058471f63SJani Nikula return 0; 44158471f63SJani Nikula 44258471f63SJani Nikula err_uncore: 44358471f63SJani Nikula intel_teardown_mchbar(dev_priv); 44458471f63SJani Nikula intel_uncore_fini_mmio(&dev_priv->uncore); 445211b4dbcSDave Airlie err_mmio: 446211b4dbcSDave Airlie intel_uncore_cleanup_mmio(&dev_priv->uncore); 44758471f63SJani Nikula err_bridge: 44858471f63SJani Nikula pci_dev_put(dev_priv->bridge_dev); 44958471f63SJani Nikula 45058471f63SJani Nikula return ret; 45158471f63SJani Nikula } 45258471f63SJani Nikula 45358471f63SJani Nikula /** 45458471f63SJani Nikula * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe() 45558471f63SJani Nikula * @dev_priv: device private 45658471f63SJani Nikula */ 45758471f63SJani Nikula static void i915_driver_mmio_release(struct drm_i915_private *dev_priv) 45858471f63SJani Nikula { 45958471f63SJani Nikula intel_teardown_mchbar(dev_priv); 46058471f63SJani Nikula intel_uncore_fini_mmio(&dev_priv->uncore); 461211b4dbcSDave Airlie intel_uncore_cleanup_mmio(&dev_priv->uncore); 46258471f63SJani Nikula pci_dev_put(dev_priv->bridge_dev); 46358471f63SJani Nikula } 46458471f63SJani Nikula 46558471f63SJani Nikula static void intel_sanitize_options(struct drm_i915_private *dev_priv) 46658471f63SJani Nikula { 46758471f63SJani Nikula intel_gvt_sanitize_options(dev_priv); 46858471f63SJani Nikula } 46958471f63SJani Nikula 47058471f63SJani Nikula /** 47158471f63SJani Nikula * i915_set_dma_info - set all relevant PCI dma info as configured for the 47258471f63SJani Nikula * platform 47358471f63SJani Nikula * @i915: valid i915 instance 47458471f63SJani Nikula * 47558471f63SJani Nikula * Set the dma max segment size, device and coherent masks. The dma mask set 47658471f63SJani Nikula * needs to occur before i915_ggtt_probe_hw. 47758471f63SJani Nikula * 47858471f63SJani Nikula * A couple of platforms have special needs. Address them as well. 47958471f63SJani Nikula * 48058471f63SJani Nikula */ 48158471f63SJani Nikula static int i915_set_dma_info(struct drm_i915_private *i915) 48258471f63SJani Nikula { 48358471f63SJani Nikula unsigned int mask_size = INTEL_INFO(i915)->dma_mask_size; 48458471f63SJani Nikula int ret; 48558471f63SJani Nikula 48658471f63SJani Nikula GEM_BUG_ON(!mask_size); 48758471f63SJani Nikula 48858471f63SJani Nikula /* 48958471f63SJani Nikula * We don't have a max segment size, so set it to the max so sg's 49058471f63SJani Nikula * debugging layer doesn't complain 49158471f63SJani Nikula */ 49258471f63SJani Nikula dma_set_max_seg_size(i915->drm.dev, UINT_MAX); 49358471f63SJani Nikula 49458471f63SJani Nikula ret = dma_set_mask(i915->drm.dev, DMA_BIT_MASK(mask_size)); 49558471f63SJani Nikula if (ret) 49658471f63SJani Nikula goto mask_err; 49758471f63SJani Nikula 49858471f63SJani Nikula /* overlay on gen2 is broken and can't address above 1G */ 49958471f63SJani Nikula if (GRAPHICS_VER(i915) == 2) 50058471f63SJani Nikula mask_size = 30; 50158471f63SJani Nikula 50258471f63SJani Nikula /* 50358471f63SJani Nikula * 965GM sometimes incorrectly writes to hardware status page (HWS) 50458471f63SJani Nikula * using 32bit addressing, overwriting memory if HWS is located 50558471f63SJani Nikula * above 4GB. 50658471f63SJani Nikula * 50758471f63SJani Nikula * The documentation also mentions an issue with undefined 50858471f63SJani Nikula * behaviour if any general state is accessed within a page above 4GB, 50958471f63SJani Nikula * which also needs to be handled carefully. 51058471f63SJani Nikula */ 51158471f63SJani Nikula if (IS_I965G(i915) || IS_I965GM(i915)) 51258471f63SJani Nikula mask_size = 32; 51358471f63SJani Nikula 51458471f63SJani Nikula ret = dma_set_coherent_mask(i915->drm.dev, DMA_BIT_MASK(mask_size)); 51558471f63SJani Nikula if (ret) 51658471f63SJani Nikula goto mask_err; 51758471f63SJani Nikula 51858471f63SJani Nikula return 0; 51958471f63SJani Nikula 52058471f63SJani Nikula mask_err: 52158471f63SJani Nikula drm_err(&i915->drm, "Can't set DMA mask/consistent mask (%d)\n", ret); 52258471f63SJani Nikula return ret; 52358471f63SJani Nikula } 52458471f63SJani Nikula 52558471f63SJani Nikula /** 52658471f63SJani Nikula * i915_driver_hw_probe - setup state requiring device access 52758471f63SJani Nikula * @dev_priv: device private 52858471f63SJani Nikula * 52958471f63SJani Nikula * Setup state that requires accessing the device, but doesn't require 53058471f63SJani Nikula * exposing the driver via kernel internal or userspace interfaces. 53158471f63SJani Nikula */ 53258471f63SJani Nikula static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) 53358471f63SJani Nikula { 53458471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 53558471f63SJani Nikula int ret; 53658471f63SJani Nikula 53758471f63SJani Nikula if (i915_inject_probe_failure(dev_priv)) 53858471f63SJani Nikula return -ENODEV; 53958471f63SJani Nikula 54058471f63SJani Nikula if (HAS_PPGTT(dev_priv)) { 54158471f63SJani Nikula if (intel_vgpu_active(dev_priv) && 54258471f63SJani Nikula !intel_vgpu_has_full_ppgtt(dev_priv)) { 54358471f63SJani Nikula i915_report_error(dev_priv, 54458471f63SJani Nikula "incompatible vGPU found, support for isolated ppGTT required\n"); 54558471f63SJani Nikula return -ENXIO; 54658471f63SJani Nikula } 54758471f63SJani Nikula } 54858471f63SJani Nikula 54958471f63SJani Nikula if (HAS_EXECLISTS(dev_priv)) { 55058471f63SJani Nikula /* 55158471f63SJani Nikula * Older GVT emulation depends upon intercepting CSB mmio, 55258471f63SJani Nikula * which we no longer use, preferring to use the HWSP cache 55358471f63SJani Nikula * instead. 55458471f63SJani Nikula */ 55558471f63SJani Nikula if (intel_vgpu_active(dev_priv) && 55658471f63SJani Nikula !intel_vgpu_has_hwsp_emulation(dev_priv)) { 55758471f63SJani Nikula i915_report_error(dev_priv, 55858471f63SJani Nikula "old vGPU host found, support for HWSP emulation required\n"); 55958471f63SJani Nikula return -ENXIO; 56058471f63SJani Nikula } 56158471f63SJani Nikula } 56258471f63SJani Nikula 56358471f63SJani Nikula intel_sanitize_options(dev_priv); 56458471f63SJani Nikula 56558471f63SJani Nikula /* needs to be done before ggtt probe */ 56658471f63SJani Nikula intel_dram_edram_detect(dev_priv); 56758471f63SJani Nikula 56858471f63SJani Nikula ret = i915_set_dma_info(dev_priv); 56958471f63SJani Nikula if (ret) 57058471f63SJani Nikula return ret; 57158471f63SJani Nikula 57258471f63SJani Nikula i915_perf_init(dev_priv); 57358471f63SJani Nikula 574*647bfd26STvrtko Ursulin ret = intel_gt_assign_ggtt(to_gt(dev_priv)); 575*647bfd26STvrtko Ursulin if (ret) 576*647bfd26STvrtko Ursulin goto err_perf; 577*647bfd26STvrtko Ursulin 57858471f63SJani Nikula ret = i915_ggtt_probe_hw(dev_priv); 57958471f63SJani Nikula if (ret) 58058471f63SJani Nikula goto err_perf; 58158471f63SJani Nikula 58258471f63SJani Nikula ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, dev_priv->drm.driver); 58358471f63SJani Nikula if (ret) 58458471f63SJani Nikula goto err_ggtt; 58558471f63SJani Nikula 58658471f63SJani Nikula ret = i915_ggtt_init_hw(dev_priv); 58758471f63SJani Nikula if (ret) 58858471f63SJani Nikula goto err_ggtt; 58958471f63SJani Nikula 59058471f63SJani Nikula ret = intel_memory_regions_hw_probe(dev_priv); 59158471f63SJani Nikula if (ret) 59258471f63SJani Nikula goto err_ggtt; 59358471f63SJani Nikula 5944817c37dSDave Airlie ret = intel_gt_probe_lmem(to_gt(dev_priv)); 59558471f63SJani Nikula if (ret) 59658471f63SJani Nikula goto err_mem_regions; 59758471f63SJani Nikula 59858471f63SJani Nikula ret = i915_ggtt_enable_hw(dev_priv); 59958471f63SJani Nikula if (ret) { 60058471f63SJani Nikula drm_err(&dev_priv->drm, "failed to enable GGTT\n"); 60158471f63SJani Nikula goto err_mem_regions; 60258471f63SJani Nikula } 60358471f63SJani Nikula 60458471f63SJani Nikula pci_set_master(pdev); 60558471f63SJani Nikula 60658471f63SJani Nikula /* On the 945G/GM, the chipset reports the MSI capability on the 60758471f63SJani Nikula * integrated graphics even though the support isn't actually there 60858471f63SJani Nikula * according to the published specs. It doesn't appear to function 60958471f63SJani Nikula * correctly in testing on 945G. 61058471f63SJani Nikula * This may be a side effect of MSI having been made available for PEG 61158471f63SJani Nikula * and the registers being closely associated. 61258471f63SJani Nikula * 61358471f63SJani Nikula * According to chipset errata, on the 965GM, MSI interrupts may 61458471f63SJani Nikula * be lost or delayed, and was defeatured. MSI interrupts seem to 61558471f63SJani Nikula * get lost on g4x as well, and interrupt delivery seems to stay 61658471f63SJani Nikula * properly dead afterwards. So we'll just disable them for all 61758471f63SJani Nikula * pre-gen5 chipsets. 61858471f63SJani Nikula * 61958471f63SJani Nikula * dp aux and gmbus irq on gen4 seems to be able to generate legacy 62058471f63SJani Nikula * interrupts even when in MSI mode. This results in spurious 62158471f63SJani Nikula * interrupt warnings if the legacy irq no. is shared with another 62258471f63SJani Nikula * device. The kernel then disables that interrupt source and so 62358471f63SJani Nikula * prevents the other device from working properly. 62458471f63SJani Nikula */ 62558471f63SJani Nikula if (GRAPHICS_VER(dev_priv) >= 5) { 62658471f63SJani Nikula if (pci_enable_msi(pdev) < 0) 62758471f63SJani Nikula drm_dbg(&dev_priv->drm, "can't enable MSI"); 62858471f63SJani Nikula } 62958471f63SJani Nikula 63058471f63SJani Nikula ret = intel_gvt_init(dev_priv); 63158471f63SJani Nikula if (ret) 63258471f63SJani Nikula goto err_msi; 63358471f63SJani Nikula 63458471f63SJani Nikula intel_opregion_setup(dev_priv); 63558471f63SJani Nikula 63658471f63SJani Nikula ret = intel_pcode_init(dev_priv); 63758471f63SJani Nikula if (ret) 63858471f63SJani Nikula goto err_msi; 63958471f63SJani Nikula 64058471f63SJani Nikula /* 64158471f63SJani Nikula * Fill the dram structure to get the system dram info. This will be 64258471f63SJani Nikula * used for memory latency calculation. 64358471f63SJani Nikula */ 64458471f63SJani Nikula intel_dram_detect(dev_priv); 64558471f63SJani Nikula 64658471f63SJani Nikula intel_bw_init_hw(dev_priv); 64758471f63SJani Nikula 64858471f63SJani Nikula return 0; 64958471f63SJani Nikula 65058471f63SJani Nikula err_msi: 65158471f63SJani Nikula if (pdev->msi_enabled) 65258471f63SJani Nikula pci_disable_msi(pdev); 65358471f63SJani Nikula err_mem_regions: 65458471f63SJani Nikula intel_memory_regions_driver_release(dev_priv); 65558471f63SJani Nikula err_ggtt: 65658471f63SJani Nikula i915_ggtt_driver_release(dev_priv); 65758471f63SJani Nikula i915_gem_drain_freed_objects(dev_priv); 65858471f63SJani Nikula i915_ggtt_driver_late_release(dev_priv); 65958471f63SJani Nikula err_perf: 66058471f63SJani Nikula i915_perf_fini(dev_priv); 66158471f63SJani Nikula return ret; 66258471f63SJani Nikula } 66358471f63SJani Nikula 66458471f63SJani Nikula /** 66558471f63SJani Nikula * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe() 66658471f63SJani Nikula * @dev_priv: device private 66758471f63SJani Nikula */ 66858471f63SJani Nikula static void i915_driver_hw_remove(struct drm_i915_private *dev_priv) 66958471f63SJani Nikula { 67058471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 67158471f63SJani Nikula 67258471f63SJani Nikula i915_perf_fini(dev_priv); 67358471f63SJani Nikula 67458471f63SJani Nikula if (pdev->msi_enabled) 67558471f63SJani Nikula pci_disable_msi(pdev); 67658471f63SJani Nikula } 67758471f63SJani Nikula 67858471f63SJani Nikula /** 67958471f63SJani Nikula * i915_driver_register - register the driver with the rest of the system 68058471f63SJani Nikula * @dev_priv: device private 68158471f63SJani Nikula * 68258471f63SJani Nikula * Perform any steps necessary to make the driver available via kernel 68358471f63SJani Nikula * internal or userspace interfaces. 68458471f63SJani Nikula */ 68558471f63SJani Nikula static void i915_driver_register(struct drm_i915_private *dev_priv) 68658471f63SJani Nikula { 68758471f63SJani Nikula struct drm_device *dev = &dev_priv->drm; 68858471f63SJani Nikula 68958471f63SJani Nikula i915_gem_driver_register(dev_priv); 69058471f63SJani Nikula i915_pmu_register(dev_priv); 69158471f63SJani Nikula 69258471f63SJani Nikula intel_vgpu_register(dev_priv); 69358471f63SJani Nikula 69458471f63SJani Nikula /* Reveal our presence to userspace */ 69558471f63SJani Nikula if (drm_dev_register(dev, 0)) { 69658471f63SJani Nikula drm_err(&dev_priv->drm, 69758471f63SJani Nikula "Failed to register driver for userspace access!\n"); 69858471f63SJani Nikula return; 69958471f63SJani Nikula } 70058471f63SJani Nikula 70158471f63SJani Nikula i915_debugfs_register(dev_priv); 70258471f63SJani Nikula i915_setup_sysfs(dev_priv); 70358471f63SJani Nikula 70458471f63SJani Nikula /* Depends on sysfs having been initialized */ 70558471f63SJani Nikula i915_perf_register(dev_priv); 70658471f63SJani Nikula 7074817c37dSDave Airlie intel_gt_driver_register(to_gt(dev_priv)); 70858471f63SJani Nikula 70958471f63SJani Nikula intel_display_driver_register(dev_priv); 71058471f63SJani Nikula 71158471f63SJani Nikula intel_power_domains_enable(dev_priv); 71258471f63SJani Nikula intel_runtime_pm_enable(&dev_priv->runtime_pm); 71358471f63SJani Nikula 71458471f63SJani Nikula intel_register_dsm_handler(); 71558471f63SJani Nikula 71658471f63SJani Nikula if (i915_switcheroo_register(dev_priv)) 71758471f63SJani Nikula drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n"); 71858471f63SJani Nikula } 71958471f63SJani Nikula 72058471f63SJani Nikula /** 72158471f63SJani Nikula * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() 72258471f63SJani Nikula * @dev_priv: device private 72358471f63SJani Nikula */ 72458471f63SJani Nikula static void i915_driver_unregister(struct drm_i915_private *dev_priv) 72558471f63SJani Nikula { 72658471f63SJani Nikula i915_switcheroo_unregister(dev_priv); 72758471f63SJani Nikula 72858471f63SJani Nikula intel_unregister_dsm_handler(); 72958471f63SJani Nikula 73058471f63SJani Nikula intel_runtime_pm_disable(&dev_priv->runtime_pm); 73158471f63SJani Nikula intel_power_domains_disable(dev_priv); 73258471f63SJani Nikula 73358471f63SJani Nikula intel_display_driver_unregister(dev_priv); 73458471f63SJani Nikula 7354817c37dSDave Airlie intel_gt_driver_unregister(to_gt(dev_priv)); 73658471f63SJani Nikula 73758471f63SJani Nikula i915_perf_unregister(dev_priv); 73858471f63SJani Nikula i915_pmu_unregister(dev_priv); 73958471f63SJani Nikula 74058471f63SJani Nikula i915_teardown_sysfs(dev_priv); 74158471f63SJani Nikula drm_dev_unplug(&dev_priv->drm); 74258471f63SJani Nikula 74358471f63SJani Nikula i915_gem_driver_unregister(dev_priv); 74458471f63SJani Nikula } 74558471f63SJani Nikula 746211b4dbcSDave Airlie void 747211b4dbcSDave Airlie i915_print_iommu_status(struct drm_i915_private *i915, struct drm_printer *p) 748211b4dbcSDave Airlie { 749211b4dbcSDave Airlie drm_printf(p, "iommu: %s\n", enableddisabled(intel_vtd_active(i915))); 750211b4dbcSDave Airlie } 751211b4dbcSDave Airlie 75258471f63SJani Nikula static void i915_welcome_messages(struct drm_i915_private *dev_priv) 75358471f63SJani Nikula { 75458471f63SJani Nikula if (drm_debug_enabled(DRM_UT_DRIVER)) { 75558471f63SJani Nikula struct drm_printer p = drm_debug_printer("i915 device info:"); 75658471f63SJani Nikula 75758471f63SJani Nikula drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n", 75858471f63SJani Nikula INTEL_DEVID(dev_priv), 75958471f63SJani Nikula INTEL_REVID(dev_priv), 76058471f63SJani Nikula intel_platform_name(INTEL_INFO(dev_priv)->platform), 76158471f63SJani Nikula intel_subplatform(RUNTIME_INFO(dev_priv), 76258471f63SJani Nikula INTEL_INFO(dev_priv)->platform), 76358471f63SJani Nikula GRAPHICS_VER(dev_priv)); 76458471f63SJani Nikula 76558471f63SJani Nikula intel_device_info_print_static(INTEL_INFO(dev_priv), &p); 76658471f63SJani Nikula intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p); 767211b4dbcSDave Airlie i915_print_iommu_status(dev_priv, &p); 7684817c37dSDave Airlie intel_gt_info_print(&to_gt(dev_priv)->info, &p); 76958471f63SJani Nikula } 77058471f63SJani Nikula 77158471f63SJani Nikula if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) 77258471f63SJani Nikula drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n"); 77358471f63SJani Nikula if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 77458471f63SJani Nikula drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n"); 77558471f63SJani Nikula if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) 77658471f63SJani Nikula drm_info(&dev_priv->drm, 77758471f63SJani Nikula "DRM_I915_DEBUG_RUNTIME_PM enabled\n"); 77858471f63SJani Nikula } 77958471f63SJani Nikula 78058471f63SJani Nikula static struct drm_i915_private * 78158471f63SJani Nikula i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) 78258471f63SJani Nikula { 78358471f63SJani Nikula const struct intel_device_info *match_info = 78458471f63SJani Nikula (struct intel_device_info *)ent->driver_data; 78558471f63SJani Nikula struct intel_device_info *device_info; 78658471f63SJani Nikula struct drm_i915_private *i915; 78758471f63SJani Nikula 7884588d7ebSJani Nikula i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver, 78958471f63SJani Nikula struct drm_i915_private, drm); 79058471f63SJani Nikula if (IS_ERR(i915)) 79158471f63SJani Nikula return i915; 79258471f63SJani Nikula 79358471f63SJani Nikula pci_set_drvdata(pdev, i915); 79458471f63SJani Nikula 79558471f63SJani Nikula /* Device parameters start as a copy of module parameters. */ 79658471f63SJani Nikula i915_params_copy(&i915->params, &i915_modparams); 79758471f63SJani Nikula 79858471f63SJani Nikula /* Setup the write-once "constant" device info */ 79958471f63SJani Nikula device_info = mkwrite_device_info(i915); 80058471f63SJani Nikula memcpy(device_info, match_info, sizeof(*device_info)); 80158471f63SJani Nikula RUNTIME_INFO(i915)->device_id = pdev->device; 80258471f63SJani Nikula 80358471f63SJani Nikula return i915; 80458471f63SJani Nikula } 80558471f63SJani Nikula 80658471f63SJani Nikula /** 80758471f63SJani Nikula * i915_driver_probe - setup chip and create an initial config 80858471f63SJani Nikula * @pdev: PCI device 80958471f63SJani Nikula * @ent: matching PCI ID entry 81058471f63SJani Nikula * 81158471f63SJani Nikula * The driver probe routine has to do several things: 81258471f63SJani Nikula * - drive output discovery via intel_modeset_init() 81358471f63SJani Nikula * - initialize the memory manager 81458471f63SJani Nikula * - allocate initial config memory 81558471f63SJani Nikula * - setup the DRM framebuffer with the allocated memory 81658471f63SJani Nikula */ 81758471f63SJani Nikula int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 81858471f63SJani Nikula { 81958471f63SJani Nikula const struct intel_device_info *match_info = 82058471f63SJani Nikula (struct intel_device_info *)ent->driver_data; 82158471f63SJani Nikula struct drm_i915_private *i915; 82258471f63SJani Nikula int ret; 82358471f63SJani Nikula 82458471f63SJani Nikula i915 = i915_driver_create(pdev, ent); 82558471f63SJani Nikula if (IS_ERR(i915)) 82658471f63SJani Nikula return PTR_ERR(i915); 82758471f63SJani Nikula 82858471f63SJani Nikula /* Disable nuclear pageflip by default on pre-ILK */ 829211b4dbcSDave Airlie if (!i915->params.nuclear_pageflip && match_info->graphics.ver < 5) 83058471f63SJani Nikula i915->drm.driver_features &= ~DRIVER_ATOMIC; 83158471f63SJani Nikula 83258471f63SJani Nikula /* 83358471f63SJani Nikula * Check if we support fake LMEM -- for now we only unleash this for 83458471f63SJani Nikula * the live selftests(test-and-exit). 83558471f63SJani Nikula */ 83658471f63SJani Nikula #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 83758471f63SJani Nikula if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) { 83858471f63SJani Nikula if (GRAPHICS_VER(i915) >= 9 && i915_selftest.live < 0 && 83958471f63SJani Nikula i915->params.fake_lmem_start) { 84058471f63SJani Nikula mkwrite_device_info(i915)->memory_regions = 84158471f63SJani Nikula REGION_SMEM | REGION_LMEM | REGION_STOLEN_SMEM; 84258471f63SJani Nikula GEM_BUG_ON(!HAS_LMEM(i915)); 84358471f63SJani Nikula } 84458471f63SJani Nikula } 84558471f63SJani Nikula #endif 84658471f63SJani Nikula 84758471f63SJani Nikula ret = pci_enable_device(pdev); 84858471f63SJani Nikula if (ret) 84958471f63SJani Nikula goto out_fini; 85058471f63SJani Nikula 85158471f63SJani Nikula ret = i915_driver_early_probe(i915); 85258471f63SJani Nikula if (ret < 0) 85358471f63SJani Nikula goto out_pci_disable; 85458471f63SJani Nikula 85558471f63SJani Nikula disable_rpm_wakeref_asserts(&i915->runtime_pm); 85658471f63SJani Nikula 85758471f63SJani Nikula intel_vgpu_detect(i915); 85858471f63SJani Nikula 85958471f63SJani Nikula ret = i915_driver_mmio_probe(i915); 86058471f63SJani Nikula if (ret < 0) 86158471f63SJani Nikula goto out_runtime_pm_put; 86258471f63SJani Nikula 86358471f63SJani Nikula ret = i915_driver_hw_probe(i915); 86458471f63SJani Nikula if (ret < 0) 86558471f63SJani Nikula goto out_cleanup_mmio; 86658471f63SJani Nikula 86758471f63SJani Nikula ret = intel_modeset_init_noirq(i915); 86858471f63SJani Nikula if (ret < 0) 86958471f63SJani Nikula goto out_cleanup_hw; 87058471f63SJani Nikula 87158471f63SJani Nikula ret = intel_irq_install(i915); 87258471f63SJani Nikula if (ret) 87358471f63SJani Nikula goto out_cleanup_modeset; 87458471f63SJani Nikula 87558471f63SJani Nikula ret = intel_modeset_init_nogem(i915); 87658471f63SJani Nikula if (ret) 87758471f63SJani Nikula goto out_cleanup_irq; 87858471f63SJani Nikula 87958471f63SJani Nikula ret = i915_gem_init(i915); 88058471f63SJani Nikula if (ret) 88158471f63SJani Nikula goto out_cleanup_modeset2; 88258471f63SJani Nikula 88358471f63SJani Nikula ret = intel_modeset_init(i915); 88458471f63SJani Nikula if (ret) 88558471f63SJani Nikula goto out_cleanup_gem; 88658471f63SJani Nikula 88758471f63SJani Nikula i915_driver_register(i915); 88858471f63SJani Nikula 88958471f63SJani Nikula enable_rpm_wakeref_asserts(&i915->runtime_pm); 89058471f63SJani Nikula 89158471f63SJani Nikula i915_welcome_messages(i915); 89258471f63SJani Nikula 89358471f63SJani Nikula i915->do_release = true; 89458471f63SJani Nikula 89558471f63SJani Nikula return 0; 89658471f63SJani Nikula 89758471f63SJani Nikula out_cleanup_gem: 89858471f63SJani Nikula i915_gem_suspend(i915); 89958471f63SJani Nikula i915_gem_driver_remove(i915); 90058471f63SJani Nikula i915_gem_driver_release(i915); 90158471f63SJani Nikula out_cleanup_modeset2: 90258471f63SJani Nikula /* FIXME clean up the error path */ 90358471f63SJani Nikula intel_modeset_driver_remove(i915); 90458471f63SJani Nikula intel_irq_uninstall(i915); 90558471f63SJani Nikula intel_modeset_driver_remove_noirq(i915); 90658471f63SJani Nikula goto out_cleanup_modeset; 90758471f63SJani Nikula out_cleanup_irq: 90858471f63SJani Nikula intel_irq_uninstall(i915); 90958471f63SJani Nikula out_cleanup_modeset: 91058471f63SJani Nikula intel_modeset_driver_remove_nogem(i915); 91158471f63SJani Nikula out_cleanup_hw: 91258471f63SJani Nikula i915_driver_hw_remove(i915); 91358471f63SJani Nikula intel_memory_regions_driver_release(i915); 91458471f63SJani Nikula i915_ggtt_driver_release(i915); 91558471f63SJani Nikula i915_gem_drain_freed_objects(i915); 91658471f63SJani Nikula i915_ggtt_driver_late_release(i915); 91758471f63SJani Nikula out_cleanup_mmio: 91858471f63SJani Nikula i915_driver_mmio_release(i915); 91958471f63SJani Nikula out_runtime_pm_put: 92058471f63SJani Nikula enable_rpm_wakeref_asserts(&i915->runtime_pm); 92158471f63SJani Nikula i915_driver_late_release(i915); 92258471f63SJani Nikula out_pci_disable: 92358471f63SJani Nikula pci_disable_device(pdev); 92458471f63SJani Nikula out_fini: 92558471f63SJani Nikula i915_probe_error(i915, "Device initialization failed (%d)\n", ret); 92658471f63SJani Nikula return ret; 92758471f63SJani Nikula } 92858471f63SJani Nikula 92958471f63SJani Nikula void i915_driver_remove(struct drm_i915_private *i915) 93058471f63SJani Nikula { 93158471f63SJani Nikula disable_rpm_wakeref_asserts(&i915->runtime_pm); 93258471f63SJani Nikula 93358471f63SJani Nikula i915_driver_unregister(i915); 93458471f63SJani Nikula 93558471f63SJani Nikula /* Flush any external code that still may be under the RCU lock */ 93658471f63SJani Nikula synchronize_rcu(); 93758471f63SJani Nikula 93858471f63SJani Nikula i915_gem_suspend(i915); 93958471f63SJani Nikula 94058471f63SJani Nikula intel_gvt_driver_remove(i915); 94158471f63SJani Nikula 94258471f63SJani Nikula intel_modeset_driver_remove(i915); 94358471f63SJani Nikula 94458471f63SJani Nikula intel_irq_uninstall(i915); 94558471f63SJani Nikula 94658471f63SJani Nikula intel_modeset_driver_remove_noirq(i915); 94758471f63SJani Nikula 94858471f63SJani Nikula i915_reset_error_state(i915); 94958471f63SJani Nikula i915_gem_driver_remove(i915); 95058471f63SJani Nikula 95158471f63SJani Nikula intel_modeset_driver_remove_nogem(i915); 95258471f63SJani Nikula 95358471f63SJani Nikula i915_driver_hw_remove(i915); 95458471f63SJani Nikula 95558471f63SJani Nikula enable_rpm_wakeref_asserts(&i915->runtime_pm); 95658471f63SJani Nikula } 95758471f63SJani Nikula 95858471f63SJani Nikula static void i915_driver_release(struct drm_device *dev) 95958471f63SJani Nikula { 96058471f63SJani Nikula struct drm_i915_private *dev_priv = to_i915(dev); 96158471f63SJani Nikula struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; 96258471f63SJani Nikula 96358471f63SJani Nikula if (!dev_priv->do_release) 96458471f63SJani Nikula return; 96558471f63SJani Nikula 96658471f63SJani Nikula disable_rpm_wakeref_asserts(rpm); 96758471f63SJani Nikula 96858471f63SJani Nikula i915_gem_driver_release(dev_priv); 96958471f63SJani Nikula 97058471f63SJani Nikula intel_memory_regions_driver_release(dev_priv); 97158471f63SJani Nikula i915_ggtt_driver_release(dev_priv); 97258471f63SJani Nikula i915_gem_drain_freed_objects(dev_priv); 97358471f63SJani Nikula i915_ggtt_driver_late_release(dev_priv); 97458471f63SJani Nikula 97558471f63SJani Nikula i915_driver_mmio_release(dev_priv); 97658471f63SJani Nikula 97758471f63SJani Nikula enable_rpm_wakeref_asserts(rpm); 97858471f63SJani Nikula intel_runtime_pm_driver_release(rpm); 97958471f63SJani Nikula 98058471f63SJani Nikula i915_driver_late_release(dev_priv); 98158471f63SJani Nikula } 98258471f63SJani Nikula 98358471f63SJani Nikula static int i915_driver_open(struct drm_device *dev, struct drm_file *file) 98458471f63SJani Nikula { 98558471f63SJani Nikula struct drm_i915_private *i915 = to_i915(dev); 98658471f63SJani Nikula int ret; 98758471f63SJani Nikula 98858471f63SJani Nikula ret = i915_gem_open(i915, file); 98958471f63SJani Nikula if (ret) 99058471f63SJani Nikula return ret; 99158471f63SJani Nikula 99258471f63SJani Nikula return 0; 99358471f63SJani Nikula } 99458471f63SJani Nikula 99558471f63SJani Nikula /** 99658471f63SJani Nikula * i915_driver_lastclose - clean up after all DRM clients have exited 99758471f63SJani Nikula * @dev: DRM device 99858471f63SJani Nikula * 99958471f63SJani Nikula * Take care of cleaning up after all DRM clients have exited. In the 100058471f63SJani Nikula * mode setting case, we want to restore the kernel's initial mode (just 100158471f63SJani Nikula * in case the last client left us in a bad state). 100258471f63SJani Nikula * 100358471f63SJani Nikula * Additionally, in the non-mode setting case, we'll tear down the GTT 100458471f63SJani Nikula * and DMA structures, since the kernel won't be using them, and clea 100558471f63SJani Nikula * up any GEM state. 100658471f63SJani Nikula */ 100758471f63SJani Nikula static void i915_driver_lastclose(struct drm_device *dev) 100858471f63SJani Nikula { 100958471f63SJani Nikula struct drm_i915_private *i915 = to_i915(dev); 101058471f63SJani Nikula 101158471f63SJani Nikula intel_fbdev_restore_mode(dev); 101258471f63SJani Nikula 101358471f63SJani Nikula if (HAS_DISPLAY(i915)) 101458471f63SJani Nikula vga_switcheroo_process_delayed_switch(); 101558471f63SJani Nikula } 101658471f63SJani Nikula 101758471f63SJani Nikula static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) 101858471f63SJani Nikula { 101958471f63SJani Nikula struct drm_i915_file_private *file_priv = file->driver_priv; 102058471f63SJani Nikula 102158471f63SJani Nikula i915_gem_context_close(file); 102258471f63SJani Nikula 102358471f63SJani Nikula kfree_rcu(file_priv, rcu); 102458471f63SJani Nikula 102558471f63SJani Nikula /* Catch up with all the deferred frees from "this" client */ 102658471f63SJani Nikula i915_gem_flush_free_objects(to_i915(dev)); 102758471f63SJani Nikula } 102858471f63SJani Nikula 102958471f63SJani Nikula static void intel_suspend_encoders(struct drm_i915_private *dev_priv) 103058471f63SJani Nikula { 103158471f63SJani Nikula struct drm_device *dev = &dev_priv->drm; 103258471f63SJani Nikula struct intel_encoder *encoder; 103358471f63SJani Nikula 103458471f63SJani Nikula if (!HAS_DISPLAY(dev_priv)) 103558471f63SJani Nikula return; 103658471f63SJani Nikula 103758471f63SJani Nikula drm_modeset_lock_all(dev); 103858471f63SJani Nikula for_each_intel_encoder(dev, encoder) 103958471f63SJani Nikula if (encoder->suspend) 104058471f63SJani Nikula encoder->suspend(encoder); 104158471f63SJani Nikula drm_modeset_unlock_all(dev); 104258471f63SJani Nikula } 104358471f63SJani Nikula 104458471f63SJani Nikula static void intel_shutdown_encoders(struct drm_i915_private *dev_priv) 104558471f63SJani Nikula { 104658471f63SJani Nikula struct drm_device *dev = &dev_priv->drm; 104758471f63SJani Nikula struct intel_encoder *encoder; 104858471f63SJani Nikula 104958471f63SJani Nikula if (!HAS_DISPLAY(dev_priv)) 105058471f63SJani Nikula return; 105158471f63SJani Nikula 105258471f63SJani Nikula drm_modeset_lock_all(dev); 105358471f63SJani Nikula for_each_intel_encoder(dev, encoder) 105458471f63SJani Nikula if (encoder->shutdown) 105558471f63SJani Nikula encoder->shutdown(encoder); 105658471f63SJani Nikula drm_modeset_unlock_all(dev); 105758471f63SJani Nikula } 105858471f63SJani Nikula 105958471f63SJani Nikula void i915_driver_shutdown(struct drm_i915_private *i915) 106058471f63SJani Nikula { 106158471f63SJani Nikula disable_rpm_wakeref_asserts(&i915->runtime_pm); 106258471f63SJani Nikula intel_runtime_pm_disable(&i915->runtime_pm); 106358471f63SJani Nikula intel_power_domains_disable(i915); 106458471f63SJani Nikula 106558471f63SJani Nikula i915_gem_suspend(i915); 106658471f63SJani Nikula 106758471f63SJani Nikula if (HAS_DISPLAY(i915)) { 106858471f63SJani Nikula drm_kms_helper_poll_disable(&i915->drm); 106958471f63SJani Nikula 107058471f63SJani Nikula drm_atomic_helper_shutdown(&i915->drm); 107158471f63SJani Nikula } 107258471f63SJani Nikula 107358471f63SJani Nikula intel_dp_mst_suspend(i915); 107458471f63SJani Nikula 107558471f63SJani Nikula intel_runtime_pm_disable_interrupts(i915); 107658471f63SJani Nikula intel_hpd_cancel_work(i915); 107758471f63SJani Nikula 107858471f63SJani Nikula intel_suspend_encoders(i915); 107958471f63SJani Nikula intel_shutdown_encoders(i915); 108058471f63SJani Nikula 108158471f63SJani Nikula intel_dmc_ucode_suspend(i915); 108258471f63SJani Nikula 108358471f63SJani Nikula /* 108458471f63SJani Nikula * The only requirement is to reboot with display DC states disabled, 108558471f63SJani Nikula * for now leaving all display power wells in the INIT power domain 108658471f63SJani Nikula * enabled. 108758471f63SJani Nikula * 108858471f63SJani Nikula * TODO: 108958471f63SJani Nikula * - unify the pci_driver::shutdown sequence here with the 109058471f63SJani Nikula * pci_driver.driver.pm.poweroff,poweroff_late sequence. 109158471f63SJani Nikula * - unify the driver remove and system/runtime suspend sequences with 109258471f63SJani Nikula * the above unified shutdown/poweroff sequence. 109358471f63SJani Nikula */ 109458471f63SJani Nikula intel_power_domains_driver_remove(i915); 109558471f63SJani Nikula enable_rpm_wakeref_asserts(&i915->runtime_pm); 109658471f63SJani Nikula 109758471f63SJani Nikula intel_runtime_pm_driver_release(&i915->runtime_pm); 109858471f63SJani Nikula } 109958471f63SJani Nikula 110058471f63SJani Nikula static bool suspend_to_idle(struct drm_i915_private *dev_priv) 110158471f63SJani Nikula { 110258471f63SJani Nikula #if IS_ENABLED(CONFIG_ACPI_SLEEP) 110358471f63SJani Nikula if (acpi_target_system_state() < ACPI_STATE_S3) 110458471f63SJani Nikula return true; 110558471f63SJani Nikula #endif 110658471f63SJani Nikula return false; 110758471f63SJani Nikula } 110858471f63SJani Nikula 110958471f63SJani Nikula static int i915_drm_prepare(struct drm_device *dev) 111058471f63SJani Nikula { 111158471f63SJani Nikula struct drm_i915_private *i915 = to_i915(dev); 111258471f63SJani Nikula 111358471f63SJani Nikula /* 111458471f63SJani Nikula * NB intel_display_suspend() may issue new requests after we've 111558471f63SJani Nikula * ostensibly marked the GPU as ready-to-sleep here. We need to 111658471f63SJani Nikula * split out that work and pull it forward so that after point, 111758471f63SJani Nikula * the GPU is not woken again. 111858471f63SJani Nikula */ 111958471f63SJani Nikula return i915_gem_backup_suspend(i915); 112058471f63SJani Nikula } 112158471f63SJani Nikula 112258471f63SJani Nikula static int i915_drm_suspend(struct drm_device *dev) 112358471f63SJani Nikula { 112458471f63SJani Nikula struct drm_i915_private *dev_priv = to_i915(dev); 112558471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 112658471f63SJani Nikula pci_power_t opregion_target_state; 112758471f63SJani Nikula 112858471f63SJani Nikula disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 112958471f63SJani Nikula 113058471f63SJani Nikula /* We do a lot of poking in a lot of registers, make sure they work 113158471f63SJani Nikula * properly. */ 113258471f63SJani Nikula intel_power_domains_disable(dev_priv); 113358471f63SJani Nikula if (HAS_DISPLAY(dev_priv)) 113458471f63SJani Nikula drm_kms_helper_poll_disable(dev); 113558471f63SJani Nikula 113658471f63SJani Nikula pci_save_state(pdev); 113758471f63SJani Nikula 113858471f63SJani Nikula intel_display_suspend(dev); 113958471f63SJani Nikula 114058471f63SJani Nikula intel_dp_mst_suspend(dev_priv); 114158471f63SJani Nikula 114258471f63SJani Nikula intel_runtime_pm_disable_interrupts(dev_priv); 114358471f63SJani Nikula intel_hpd_cancel_work(dev_priv); 114458471f63SJani Nikula 114558471f63SJani Nikula intel_suspend_encoders(dev_priv); 114658471f63SJani Nikula 114758471f63SJani Nikula intel_suspend_hw(dev_priv); 114858471f63SJani Nikula 114958471f63SJani Nikula /* Must be called before GGTT is suspended. */ 115058471f63SJani Nikula intel_dpt_suspend(dev_priv); 1151*647bfd26STvrtko Ursulin i915_ggtt_suspend(to_gt(dev_priv)->ggtt); 115258471f63SJani Nikula 115358471f63SJani Nikula i915_save_display(dev_priv); 115458471f63SJani Nikula 115558471f63SJani Nikula opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; 115658471f63SJani Nikula intel_opregion_suspend(dev_priv, opregion_target_state); 115758471f63SJani Nikula 115858471f63SJani Nikula intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); 115958471f63SJani Nikula 116058471f63SJani Nikula dev_priv->suspend_count++; 116158471f63SJani Nikula 116258471f63SJani Nikula intel_dmc_ucode_suspend(dev_priv); 116358471f63SJani Nikula 116458471f63SJani Nikula enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 116558471f63SJani Nikula 116658471f63SJani Nikula return 0; 116758471f63SJani Nikula } 116858471f63SJani Nikula 116958471f63SJani Nikula static enum i915_drm_suspend_mode 117058471f63SJani Nikula get_suspend_mode(struct drm_i915_private *dev_priv, bool hibernate) 117158471f63SJani Nikula { 117258471f63SJani Nikula if (hibernate) 117358471f63SJani Nikula return I915_DRM_SUSPEND_HIBERNATE; 117458471f63SJani Nikula 117558471f63SJani Nikula if (suspend_to_idle(dev_priv)) 117658471f63SJani Nikula return I915_DRM_SUSPEND_IDLE; 117758471f63SJani Nikula 117858471f63SJani Nikula return I915_DRM_SUSPEND_MEM; 117958471f63SJani Nikula } 118058471f63SJani Nikula 118158471f63SJani Nikula static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) 118258471f63SJani Nikula { 118358471f63SJani Nikula struct drm_i915_private *dev_priv = to_i915(dev); 118458471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 118558471f63SJani Nikula struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; 118658471f63SJani Nikula int ret; 118758471f63SJani Nikula 118858471f63SJani Nikula disable_rpm_wakeref_asserts(rpm); 118958471f63SJani Nikula 119058471f63SJani Nikula i915_gem_suspend_late(dev_priv); 119158471f63SJani Nikula 119258471f63SJani Nikula intel_uncore_suspend(&dev_priv->uncore); 119358471f63SJani Nikula 119458471f63SJani Nikula intel_power_domains_suspend(dev_priv, 119558471f63SJani Nikula get_suspend_mode(dev_priv, hibernation)); 119658471f63SJani Nikula 119758471f63SJani Nikula intel_display_power_suspend_late(dev_priv); 119858471f63SJani Nikula 119958471f63SJani Nikula ret = vlv_suspend_complete(dev_priv); 120058471f63SJani Nikula if (ret) { 120158471f63SJani Nikula drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret); 120258471f63SJani Nikula intel_power_domains_resume(dev_priv); 120358471f63SJani Nikula 120458471f63SJani Nikula goto out; 120558471f63SJani Nikula } 120658471f63SJani Nikula 120758471f63SJani Nikula /* 120858471f63SJani Nikula * FIXME: Temporary hammer to avoid freezing the machine on our DGFX 120958471f63SJani Nikula * This should be totally removed when we handle the pci states properly 121058471f63SJani Nikula * on runtime PM and on s2idle cases. 121158471f63SJani Nikula */ 121258471f63SJani Nikula if (suspend_to_idle(dev_priv)) 121358471f63SJani Nikula pci_d3cold_disable(pdev); 121458471f63SJani Nikula 121558471f63SJani Nikula pci_disable_device(pdev); 121658471f63SJani Nikula /* 121758471f63SJani Nikula * During hibernation on some platforms the BIOS may try to access 121858471f63SJani Nikula * the device even though it's already in D3 and hang the machine. So 121958471f63SJani Nikula * leave the device in D0 on those platforms and hope the BIOS will 122058471f63SJani Nikula * power down the device properly. The issue was seen on multiple old 122158471f63SJani Nikula * GENs with different BIOS vendors, so having an explicit blacklist 122258471f63SJani Nikula * is inpractical; apply the workaround on everything pre GEN6. The 122358471f63SJani Nikula * platforms where the issue was seen: 122458471f63SJani Nikula * Lenovo Thinkpad X301, X61s, X60, T60, X41 122558471f63SJani Nikula * Fujitsu FSC S7110 122658471f63SJani Nikula * Acer Aspire 1830T 122758471f63SJani Nikula */ 122858471f63SJani Nikula if (!(hibernation && GRAPHICS_VER(dev_priv) < 6)) 122958471f63SJani Nikula pci_set_power_state(pdev, PCI_D3hot); 123058471f63SJani Nikula 123158471f63SJani Nikula out: 123258471f63SJani Nikula enable_rpm_wakeref_asserts(rpm); 123358471f63SJani Nikula if (!dev_priv->uncore.user_forcewake_count) 123458471f63SJani Nikula intel_runtime_pm_driver_release(rpm); 123558471f63SJani Nikula 123658471f63SJani Nikula return ret; 123758471f63SJani Nikula } 123858471f63SJani Nikula 1239b8d65b8aSJani Nikula int i915_driver_suspend_switcheroo(struct drm_i915_private *i915, 1240b8d65b8aSJani Nikula pm_message_t state) 124158471f63SJani Nikula { 124258471f63SJani Nikula int error; 124358471f63SJani Nikula 124458471f63SJani Nikula if (drm_WARN_ON_ONCE(&i915->drm, state.event != PM_EVENT_SUSPEND && 124558471f63SJani Nikula state.event != PM_EVENT_FREEZE)) 124658471f63SJani Nikula return -EINVAL; 124758471f63SJani Nikula 124858471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 124958471f63SJani Nikula return 0; 125058471f63SJani Nikula 125158471f63SJani Nikula error = i915_drm_suspend(&i915->drm); 125258471f63SJani Nikula if (error) 125358471f63SJani Nikula return error; 125458471f63SJani Nikula 125558471f63SJani Nikula return i915_drm_suspend_late(&i915->drm, false); 125658471f63SJani Nikula } 125758471f63SJani Nikula 125858471f63SJani Nikula static int i915_drm_resume(struct drm_device *dev) 125958471f63SJani Nikula { 126058471f63SJani Nikula struct drm_i915_private *dev_priv = to_i915(dev); 126158471f63SJani Nikula int ret; 126258471f63SJani Nikula 126358471f63SJani Nikula disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 126458471f63SJani Nikula 126558471f63SJani Nikula ret = intel_pcode_init(dev_priv); 126658471f63SJani Nikula if (ret) 126758471f63SJani Nikula return ret; 126858471f63SJani Nikula 126958471f63SJani Nikula sanitize_gpu(dev_priv); 127058471f63SJani Nikula 127158471f63SJani Nikula ret = i915_ggtt_enable_hw(dev_priv); 127258471f63SJani Nikula if (ret) 127358471f63SJani Nikula drm_err(&dev_priv->drm, "failed to re-enable GGTT\n"); 127458471f63SJani Nikula 1275*647bfd26STvrtko Ursulin i915_ggtt_resume(to_gt(dev_priv)->ggtt); 127658471f63SJani Nikula /* Must be called after GGTT is resumed. */ 127758471f63SJani Nikula intel_dpt_resume(dev_priv); 127858471f63SJani Nikula 127958471f63SJani Nikula intel_dmc_ucode_resume(dev_priv); 128058471f63SJani Nikula 128158471f63SJani Nikula i915_restore_display(dev_priv); 128258471f63SJani Nikula intel_pps_unlock_regs_wa(dev_priv); 128358471f63SJani Nikula 128458471f63SJani Nikula intel_init_pch_refclk(dev_priv); 128558471f63SJani Nikula 128658471f63SJani Nikula /* 128758471f63SJani Nikula * Interrupts have to be enabled before any batches are run. If not the 128858471f63SJani Nikula * GPU will hang. i915_gem_init_hw() will initiate batches to 128958471f63SJani Nikula * update/restore the context. 129058471f63SJani Nikula * 129158471f63SJani Nikula * drm_mode_config_reset() needs AUX interrupts. 129258471f63SJani Nikula * 129358471f63SJani Nikula * Modeset enabling in intel_modeset_init_hw() also needs working 129458471f63SJani Nikula * interrupts. 129558471f63SJani Nikula */ 129658471f63SJani Nikula intel_runtime_pm_enable_interrupts(dev_priv); 129758471f63SJani Nikula 129858471f63SJani Nikula if (HAS_DISPLAY(dev_priv)) 129958471f63SJani Nikula drm_mode_config_reset(dev); 130058471f63SJani Nikula 130158471f63SJani Nikula i915_gem_resume(dev_priv); 130258471f63SJani Nikula 130358471f63SJani Nikula intel_modeset_init_hw(dev_priv); 130458471f63SJani Nikula intel_init_clock_gating(dev_priv); 130558471f63SJani Nikula intel_hpd_init(dev_priv); 130658471f63SJani Nikula 130758471f63SJani Nikula /* MST sideband requires HPD interrupts enabled */ 130858471f63SJani Nikula intel_dp_mst_resume(dev_priv); 130958471f63SJani Nikula intel_display_resume(dev); 131058471f63SJani Nikula 131158471f63SJani Nikula intel_hpd_poll_disable(dev_priv); 131258471f63SJani Nikula if (HAS_DISPLAY(dev_priv)) 131358471f63SJani Nikula drm_kms_helper_poll_enable(dev); 131458471f63SJani Nikula 131558471f63SJani Nikula intel_opregion_resume(dev_priv); 131658471f63SJani Nikula 131758471f63SJani Nikula intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); 131858471f63SJani Nikula 131958471f63SJani Nikula intel_power_domains_enable(dev_priv); 132058471f63SJani Nikula 132158471f63SJani Nikula intel_gvt_resume(dev_priv); 132258471f63SJani Nikula 132358471f63SJani Nikula enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 132458471f63SJani Nikula 132558471f63SJani Nikula return 0; 132658471f63SJani Nikula } 132758471f63SJani Nikula 132858471f63SJani Nikula static int i915_drm_resume_early(struct drm_device *dev) 132958471f63SJani Nikula { 133058471f63SJani Nikula struct drm_i915_private *dev_priv = to_i915(dev); 133158471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 133258471f63SJani Nikula int ret; 133358471f63SJani Nikula 133458471f63SJani Nikula /* 133558471f63SJani Nikula * We have a resume ordering issue with the snd-hda driver also 133658471f63SJani Nikula * requiring our device to be power up. Due to the lack of a 133758471f63SJani Nikula * parent/child relationship we currently solve this with an early 133858471f63SJani Nikula * resume hook. 133958471f63SJani Nikula * 134058471f63SJani Nikula * FIXME: This should be solved with a special hdmi sink device or 134158471f63SJani Nikula * similar so that power domains can be employed. 134258471f63SJani Nikula */ 134358471f63SJani Nikula 134458471f63SJani Nikula /* 134558471f63SJani Nikula * Note that we need to set the power state explicitly, since we 134658471f63SJani Nikula * powered off the device during freeze and the PCI core won't power 134758471f63SJani Nikula * it back up for us during thaw. Powering off the device during 134858471f63SJani Nikula * freeze is not a hard requirement though, and during the 134958471f63SJani Nikula * suspend/resume phases the PCI core makes sure we get here with the 135058471f63SJani Nikula * device powered on. So in case we change our freeze logic and keep 135158471f63SJani Nikula * the device powered we can also remove the following set power state 135258471f63SJani Nikula * call. 135358471f63SJani Nikula */ 135458471f63SJani Nikula ret = pci_set_power_state(pdev, PCI_D0); 135558471f63SJani Nikula if (ret) { 135658471f63SJani Nikula drm_err(&dev_priv->drm, 135758471f63SJani Nikula "failed to set PCI D0 power state (%d)\n", ret); 135858471f63SJani Nikula return ret; 135958471f63SJani Nikula } 136058471f63SJani Nikula 136158471f63SJani Nikula /* 136258471f63SJani Nikula * Note that pci_enable_device() first enables any parent bridge 136358471f63SJani Nikula * device and only then sets the power state for this device. The 136458471f63SJani Nikula * bridge enabling is a nop though, since bridge devices are resumed 136558471f63SJani Nikula * first. The order of enabling power and enabling the device is 136658471f63SJani Nikula * imposed by the PCI core as described above, so here we preserve the 136758471f63SJani Nikula * same order for the freeze/thaw phases. 136858471f63SJani Nikula * 136958471f63SJani Nikula * TODO: eventually we should remove pci_disable_device() / 137058471f63SJani Nikula * pci_enable_enable_device() from suspend/resume. Due to how they 137158471f63SJani Nikula * depend on the device enable refcount we can't anyway depend on them 137258471f63SJani Nikula * disabling/enabling the device. 137358471f63SJani Nikula */ 137458471f63SJani Nikula if (pci_enable_device(pdev)) 137558471f63SJani Nikula return -EIO; 137658471f63SJani Nikula 137758471f63SJani Nikula pci_set_master(pdev); 137858471f63SJani Nikula 137958471f63SJani Nikula pci_d3cold_enable(pdev); 138058471f63SJani Nikula 138158471f63SJani Nikula disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 138258471f63SJani Nikula 138358471f63SJani Nikula ret = vlv_resume_prepare(dev_priv, false); 138458471f63SJani Nikula if (ret) 138558471f63SJani Nikula drm_err(&dev_priv->drm, 138658471f63SJani Nikula "Resume prepare failed: %d, continuing anyway\n", ret); 138758471f63SJani Nikula 138858471f63SJani Nikula intel_uncore_resume_early(&dev_priv->uncore); 138958471f63SJani Nikula 13904817c37dSDave Airlie intel_gt_check_and_clear_faults(to_gt(dev_priv)); 139158471f63SJani Nikula 139258471f63SJani Nikula intel_display_power_resume_early(dev_priv); 139358471f63SJani Nikula 139458471f63SJani Nikula intel_power_domains_resume(dev_priv); 139558471f63SJani Nikula 139658471f63SJani Nikula enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 139758471f63SJani Nikula 139858471f63SJani Nikula return ret; 139958471f63SJani Nikula } 140058471f63SJani Nikula 1401b8d65b8aSJani Nikula int i915_driver_resume_switcheroo(struct drm_i915_private *i915) 140258471f63SJani Nikula { 140358471f63SJani Nikula int ret; 140458471f63SJani Nikula 140558471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 140658471f63SJani Nikula return 0; 140758471f63SJani Nikula 140858471f63SJani Nikula ret = i915_drm_resume_early(&i915->drm); 140958471f63SJani Nikula if (ret) 141058471f63SJani Nikula return ret; 141158471f63SJani Nikula 141258471f63SJani Nikula return i915_drm_resume(&i915->drm); 141358471f63SJani Nikula } 141458471f63SJani Nikula 141558471f63SJani Nikula static int i915_pm_prepare(struct device *kdev) 141658471f63SJani Nikula { 141758471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 141858471f63SJani Nikula 141958471f63SJani Nikula if (!i915) { 142058471f63SJani Nikula dev_err(kdev, "DRM not initialized, aborting suspend.\n"); 142158471f63SJani Nikula return -ENODEV; 142258471f63SJani Nikula } 142358471f63SJani Nikula 142458471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 142558471f63SJani Nikula return 0; 142658471f63SJani Nikula 142758471f63SJani Nikula return i915_drm_prepare(&i915->drm); 142858471f63SJani Nikula } 142958471f63SJani Nikula 143058471f63SJani Nikula static int i915_pm_suspend(struct device *kdev) 143158471f63SJani Nikula { 143258471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 143358471f63SJani Nikula 143458471f63SJani Nikula if (!i915) { 143558471f63SJani Nikula dev_err(kdev, "DRM not initialized, aborting suspend.\n"); 143658471f63SJani Nikula return -ENODEV; 143758471f63SJani Nikula } 143858471f63SJani Nikula 143958471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 144058471f63SJani Nikula return 0; 144158471f63SJani Nikula 144258471f63SJani Nikula return i915_drm_suspend(&i915->drm); 144358471f63SJani Nikula } 144458471f63SJani Nikula 144558471f63SJani Nikula static int i915_pm_suspend_late(struct device *kdev) 144658471f63SJani Nikula { 144758471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 144858471f63SJani Nikula 144958471f63SJani Nikula /* 145058471f63SJani Nikula * We have a suspend ordering issue with the snd-hda driver also 145158471f63SJani Nikula * requiring our device to be power up. Due to the lack of a 145258471f63SJani Nikula * parent/child relationship we currently solve this with an late 145358471f63SJani Nikula * suspend hook. 145458471f63SJani Nikula * 145558471f63SJani Nikula * FIXME: This should be solved with a special hdmi sink device or 145658471f63SJani Nikula * similar so that power domains can be employed. 145758471f63SJani Nikula */ 145858471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 145958471f63SJani Nikula return 0; 146058471f63SJani Nikula 146158471f63SJani Nikula return i915_drm_suspend_late(&i915->drm, false); 146258471f63SJani Nikula } 146358471f63SJani Nikula 146458471f63SJani Nikula static int i915_pm_poweroff_late(struct device *kdev) 146558471f63SJani Nikula { 146658471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 146758471f63SJani Nikula 146858471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 146958471f63SJani Nikula return 0; 147058471f63SJani Nikula 147158471f63SJani Nikula return i915_drm_suspend_late(&i915->drm, true); 147258471f63SJani Nikula } 147358471f63SJani Nikula 147458471f63SJani Nikula static int i915_pm_resume_early(struct device *kdev) 147558471f63SJani Nikula { 147658471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 147758471f63SJani Nikula 147858471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 147958471f63SJani Nikula return 0; 148058471f63SJani Nikula 148158471f63SJani Nikula return i915_drm_resume_early(&i915->drm); 148258471f63SJani Nikula } 148358471f63SJani Nikula 148458471f63SJani Nikula static int i915_pm_resume(struct device *kdev) 148558471f63SJani Nikula { 148658471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 148758471f63SJani Nikula 148858471f63SJani Nikula if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) 148958471f63SJani Nikula return 0; 149058471f63SJani Nikula 149158471f63SJani Nikula return i915_drm_resume(&i915->drm); 149258471f63SJani Nikula } 149358471f63SJani Nikula 149458471f63SJani Nikula /* freeze: before creating the hibernation_image */ 149558471f63SJani Nikula static int i915_pm_freeze(struct device *kdev) 149658471f63SJani Nikula { 149758471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 149858471f63SJani Nikula int ret; 149958471f63SJani Nikula 150058471f63SJani Nikula if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) { 150158471f63SJani Nikula ret = i915_drm_suspend(&i915->drm); 150258471f63SJani Nikula if (ret) 150358471f63SJani Nikula return ret; 150458471f63SJani Nikula } 150558471f63SJani Nikula 150658471f63SJani Nikula ret = i915_gem_freeze(i915); 150758471f63SJani Nikula if (ret) 150858471f63SJani Nikula return ret; 150958471f63SJani Nikula 151058471f63SJani Nikula return 0; 151158471f63SJani Nikula } 151258471f63SJani Nikula 151358471f63SJani Nikula static int i915_pm_freeze_late(struct device *kdev) 151458471f63SJani Nikula { 151558471f63SJani Nikula struct drm_i915_private *i915 = kdev_to_i915(kdev); 151658471f63SJani Nikula int ret; 151758471f63SJani Nikula 151858471f63SJani Nikula if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) { 151958471f63SJani Nikula ret = i915_drm_suspend_late(&i915->drm, true); 152058471f63SJani Nikula if (ret) 152158471f63SJani Nikula return ret; 152258471f63SJani Nikula } 152358471f63SJani Nikula 152458471f63SJani Nikula ret = i915_gem_freeze_late(i915); 152558471f63SJani Nikula if (ret) 152658471f63SJani Nikula return ret; 152758471f63SJani Nikula 152858471f63SJani Nikula return 0; 152958471f63SJani Nikula } 153058471f63SJani Nikula 153158471f63SJani Nikula /* thaw: called after creating the hibernation image, but before turning off. */ 153258471f63SJani Nikula static int i915_pm_thaw_early(struct device *kdev) 153358471f63SJani Nikula { 153458471f63SJani Nikula return i915_pm_resume_early(kdev); 153558471f63SJani Nikula } 153658471f63SJani Nikula 153758471f63SJani Nikula static int i915_pm_thaw(struct device *kdev) 153858471f63SJani Nikula { 153958471f63SJani Nikula return i915_pm_resume(kdev); 154058471f63SJani Nikula } 154158471f63SJani Nikula 154258471f63SJani Nikula /* restore: called after loading the hibernation image. */ 154358471f63SJani Nikula static int i915_pm_restore_early(struct device *kdev) 154458471f63SJani Nikula { 154558471f63SJani Nikula return i915_pm_resume_early(kdev); 154658471f63SJani Nikula } 154758471f63SJani Nikula 154858471f63SJani Nikula static int i915_pm_restore(struct device *kdev) 154958471f63SJani Nikula { 155058471f63SJani Nikula return i915_pm_resume(kdev); 155158471f63SJani Nikula } 155258471f63SJani Nikula 155358471f63SJani Nikula static int intel_runtime_suspend(struct device *kdev) 155458471f63SJani Nikula { 155558471f63SJani Nikula struct drm_i915_private *dev_priv = kdev_to_i915(kdev); 155658471f63SJani Nikula struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; 155758471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 155858471f63SJani Nikula int ret; 155958471f63SJani Nikula 156058471f63SJani Nikula if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) 156158471f63SJani Nikula return -ENODEV; 156258471f63SJani Nikula 156358471f63SJani Nikula drm_dbg_kms(&dev_priv->drm, "Suspending device\n"); 156458471f63SJani Nikula 156558471f63SJani Nikula disable_rpm_wakeref_asserts(rpm); 156658471f63SJani Nikula 156758471f63SJani Nikula /* 156858471f63SJani Nikula * We are safe here against re-faults, since the fault handler takes 156958471f63SJani Nikula * an RPM reference. 157058471f63SJani Nikula */ 157158471f63SJani Nikula i915_gem_runtime_suspend(dev_priv); 157258471f63SJani Nikula 15734817c37dSDave Airlie intel_gt_runtime_suspend(to_gt(dev_priv)); 157458471f63SJani Nikula 157558471f63SJani Nikula intel_runtime_pm_disable_interrupts(dev_priv); 157658471f63SJani Nikula 157758471f63SJani Nikula intel_uncore_suspend(&dev_priv->uncore); 157858471f63SJani Nikula 157958471f63SJani Nikula intel_display_power_suspend(dev_priv); 158058471f63SJani Nikula 158158471f63SJani Nikula ret = vlv_suspend_complete(dev_priv); 158258471f63SJani Nikula if (ret) { 158358471f63SJani Nikula drm_err(&dev_priv->drm, 158458471f63SJani Nikula "Runtime suspend failed, disabling it (%d)\n", ret); 158558471f63SJani Nikula intel_uncore_runtime_resume(&dev_priv->uncore); 158658471f63SJani Nikula 158758471f63SJani Nikula intel_runtime_pm_enable_interrupts(dev_priv); 158858471f63SJani Nikula 15894817c37dSDave Airlie intel_gt_runtime_resume(to_gt(dev_priv)); 159058471f63SJani Nikula 159158471f63SJani Nikula enable_rpm_wakeref_asserts(rpm); 159258471f63SJani Nikula 159358471f63SJani Nikula return ret; 159458471f63SJani Nikula } 159558471f63SJani Nikula 159658471f63SJani Nikula enable_rpm_wakeref_asserts(rpm); 159758471f63SJani Nikula intel_runtime_pm_driver_release(rpm); 159858471f63SJani Nikula 159958471f63SJani Nikula if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore)) 160058471f63SJani Nikula drm_err(&dev_priv->drm, 160158471f63SJani Nikula "Unclaimed access detected prior to suspending\n"); 160258471f63SJani Nikula 160358471f63SJani Nikula /* 160458471f63SJani Nikula * FIXME: Temporary hammer to avoid freezing the machine on our DGFX 160558471f63SJani Nikula * This should be totally removed when we handle the pci states properly 160658471f63SJani Nikula * on runtime PM and on s2idle cases. 160758471f63SJani Nikula */ 160858471f63SJani Nikula pci_d3cold_disable(pdev); 160958471f63SJani Nikula rpm->suspended = true; 161058471f63SJani Nikula 161158471f63SJani Nikula /* 161258471f63SJani Nikula * FIXME: We really should find a document that references the arguments 161358471f63SJani Nikula * used below! 161458471f63SJani Nikula */ 161558471f63SJani Nikula if (IS_BROADWELL(dev_priv)) { 161658471f63SJani Nikula /* 161758471f63SJani Nikula * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop 161858471f63SJani Nikula * being detected, and the call we do at intel_runtime_resume() 161958471f63SJani Nikula * won't be able to restore them. Since PCI_D3hot matches the 162058471f63SJani Nikula * actual specification and appears to be working, use it. 162158471f63SJani Nikula */ 162258471f63SJani Nikula intel_opregion_notify_adapter(dev_priv, PCI_D3hot); 162358471f63SJani Nikula } else { 162458471f63SJani Nikula /* 162558471f63SJani Nikula * current versions of firmware which depend on this opregion 162658471f63SJani Nikula * notification have repurposed the D1 definition to mean 162758471f63SJani Nikula * "runtime suspended" vs. what you would normally expect (D3) 162858471f63SJani Nikula * to distinguish it from notifications that might be sent via 162958471f63SJani Nikula * the suspend path. 163058471f63SJani Nikula */ 163158471f63SJani Nikula intel_opregion_notify_adapter(dev_priv, PCI_D1); 163258471f63SJani Nikula } 163358471f63SJani Nikula 163458471f63SJani Nikula assert_forcewakes_inactive(&dev_priv->uncore); 163558471f63SJani Nikula 163658471f63SJani Nikula if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 163758471f63SJani Nikula intel_hpd_poll_enable(dev_priv); 163858471f63SJani Nikula 163958471f63SJani Nikula drm_dbg_kms(&dev_priv->drm, "Device suspended\n"); 164058471f63SJani Nikula return 0; 164158471f63SJani Nikula } 164258471f63SJani Nikula 164358471f63SJani Nikula static int intel_runtime_resume(struct device *kdev) 164458471f63SJani Nikula { 164558471f63SJani Nikula struct drm_i915_private *dev_priv = kdev_to_i915(kdev); 164658471f63SJani Nikula struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; 164758471f63SJani Nikula struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); 164858471f63SJani Nikula int ret; 164958471f63SJani Nikula 165058471f63SJani Nikula if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) 165158471f63SJani Nikula return -ENODEV; 165258471f63SJani Nikula 165358471f63SJani Nikula drm_dbg_kms(&dev_priv->drm, "Resuming device\n"); 165458471f63SJani Nikula 165558471f63SJani Nikula drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count)); 165658471f63SJani Nikula disable_rpm_wakeref_asserts(rpm); 165758471f63SJani Nikula 165858471f63SJani Nikula intel_opregion_notify_adapter(dev_priv, PCI_D0); 165958471f63SJani Nikula rpm->suspended = false; 166058471f63SJani Nikula pci_d3cold_enable(pdev); 166158471f63SJani Nikula if (intel_uncore_unclaimed_mmio(&dev_priv->uncore)) 166258471f63SJani Nikula drm_dbg(&dev_priv->drm, 166358471f63SJani Nikula "Unclaimed access during suspend, bios?\n"); 166458471f63SJani Nikula 166558471f63SJani Nikula intel_display_power_resume(dev_priv); 166658471f63SJani Nikula 166758471f63SJani Nikula ret = vlv_resume_prepare(dev_priv, true); 166858471f63SJani Nikula 166958471f63SJani Nikula intel_uncore_runtime_resume(&dev_priv->uncore); 167058471f63SJani Nikula 167158471f63SJani Nikula intel_runtime_pm_enable_interrupts(dev_priv); 167258471f63SJani Nikula 167358471f63SJani Nikula /* 167458471f63SJani Nikula * No point of rolling back things in case of an error, as the best 167558471f63SJani Nikula * we can do is to hope that things will still work (and disable RPM). 167658471f63SJani Nikula */ 16774817c37dSDave Airlie intel_gt_runtime_resume(to_gt(dev_priv)); 167858471f63SJani Nikula 167958471f63SJani Nikula /* 168058471f63SJani Nikula * On VLV/CHV display interrupts are part of the display 168158471f63SJani Nikula * power well, so hpd is reinitialized from there. For 168258471f63SJani Nikula * everyone else do it here. 168358471f63SJani Nikula */ 168458471f63SJani Nikula if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) { 168558471f63SJani Nikula intel_hpd_init(dev_priv); 168658471f63SJani Nikula intel_hpd_poll_disable(dev_priv); 168758471f63SJani Nikula } 168858471f63SJani Nikula 168958471f63SJani Nikula intel_enable_ipc(dev_priv); 169058471f63SJani Nikula 169158471f63SJani Nikula enable_rpm_wakeref_asserts(rpm); 169258471f63SJani Nikula 169358471f63SJani Nikula if (ret) 169458471f63SJani Nikula drm_err(&dev_priv->drm, 169558471f63SJani Nikula "Runtime resume failed, disabling it (%d)\n", ret); 169658471f63SJani Nikula else 169758471f63SJani Nikula drm_dbg_kms(&dev_priv->drm, "Device resumed\n"); 169858471f63SJani Nikula 169958471f63SJani Nikula return ret; 170058471f63SJani Nikula } 170158471f63SJani Nikula 170258471f63SJani Nikula const struct dev_pm_ops i915_pm_ops = { 170358471f63SJani Nikula /* 170458471f63SJani Nikula * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, 170558471f63SJani Nikula * PMSG_RESUME] 170658471f63SJani Nikula */ 170758471f63SJani Nikula .prepare = i915_pm_prepare, 170858471f63SJani Nikula .suspend = i915_pm_suspend, 170958471f63SJani Nikula .suspend_late = i915_pm_suspend_late, 171058471f63SJani Nikula .resume_early = i915_pm_resume_early, 171158471f63SJani Nikula .resume = i915_pm_resume, 171258471f63SJani Nikula 171358471f63SJani Nikula /* 171458471f63SJani Nikula * S4 event handlers 171558471f63SJani Nikula * @freeze, @freeze_late : called (1) before creating the 171658471f63SJani Nikula * hibernation image [PMSG_FREEZE] and 171758471f63SJani Nikula * (2) after rebooting, before restoring 171858471f63SJani Nikula * the image [PMSG_QUIESCE] 171958471f63SJani Nikula * @thaw, @thaw_early : called (1) after creating the hibernation 172058471f63SJani Nikula * image, before writing it [PMSG_THAW] 172158471f63SJani Nikula * and (2) after failing to create or 172258471f63SJani Nikula * restore the image [PMSG_RECOVER] 172358471f63SJani Nikula * @poweroff, @poweroff_late: called after writing the hibernation 172458471f63SJani Nikula * image, before rebooting [PMSG_HIBERNATE] 172558471f63SJani Nikula * @restore, @restore_early : called after rebooting and restoring the 172658471f63SJani Nikula * hibernation image [PMSG_RESTORE] 172758471f63SJani Nikula */ 172858471f63SJani Nikula .freeze = i915_pm_freeze, 172958471f63SJani Nikula .freeze_late = i915_pm_freeze_late, 173058471f63SJani Nikula .thaw_early = i915_pm_thaw_early, 173158471f63SJani Nikula .thaw = i915_pm_thaw, 173258471f63SJani Nikula .poweroff = i915_pm_suspend, 173358471f63SJani Nikula .poweroff_late = i915_pm_poweroff_late, 173458471f63SJani Nikula .restore_early = i915_pm_restore_early, 173558471f63SJani Nikula .restore = i915_pm_restore, 173658471f63SJani Nikula 173758471f63SJani Nikula /* S0ix (via runtime suspend) event handlers */ 173858471f63SJani Nikula .runtime_suspend = intel_runtime_suspend, 173958471f63SJani Nikula .runtime_resume = intel_runtime_resume, 174058471f63SJani Nikula }; 174158471f63SJani Nikula 174258471f63SJani Nikula static const struct file_operations i915_driver_fops = { 174358471f63SJani Nikula .owner = THIS_MODULE, 174458471f63SJani Nikula .open = drm_open, 174558471f63SJani Nikula .release = drm_release_noglobal, 174658471f63SJani Nikula .unlocked_ioctl = drm_ioctl, 174758471f63SJani Nikula .mmap = i915_gem_mmap, 174858471f63SJani Nikula .poll = drm_poll, 174958471f63SJani Nikula .read = drm_read, 175058471f63SJani Nikula .compat_ioctl = i915_ioc32_compat_ioctl, 175158471f63SJani Nikula .llseek = noop_llseek, 175258471f63SJani Nikula }; 175358471f63SJani Nikula 175458471f63SJani Nikula static int 175558471f63SJani Nikula i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, 175658471f63SJani Nikula struct drm_file *file) 175758471f63SJani Nikula { 175858471f63SJani Nikula return -ENODEV; 175958471f63SJani Nikula } 176058471f63SJani Nikula 176158471f63SJani Nikula static const struct drm_ioctl_desc i915_ioctls[] = { 176258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 176358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), 176458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), 176558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), 176658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), 176758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), 176858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW), 176958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 177058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), 177158471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), 177258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 177358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), 177458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 177558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 177658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), 177758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), 177858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 177958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 178058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, drm_invalid_op, DRM_AUTH), 178158471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW), 178258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 178358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 178458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW), 178558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), 178658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), 178758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW), 178858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 178958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 179058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), 179158471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CREATE_EXT, i915_gem_create_ext_ioctl, DRM_RENDER_ALLOW), 179258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), 179358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), 179458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), 179558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_OFFSET, i915_gem_mmap_offset_ioctl, DRM_RENDER_ALLOW), 179658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), 179758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), 179858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW), 179958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW), 180058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), 180158471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id_ioctl, 0), 180258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), 180358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER), 180458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER), 180558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER), 180658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER), 180758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW), 180858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), 180958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), 181058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), 181158471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), 181258471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), 181358471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), 181458471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), 181558471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW), 181658471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW), 181758471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW), 181858471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW), 181958471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW), 182058471f63SJani Nikula DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW), 182158471f63SJani Nikula }; 182258471f63SJani Nikula 18234588d7ebSJani Nikula static const struct drm_driver i915_drm_driver = { 182458471f63SJani Nikula /* Don't use MTRRs here; the Xserver or userspace app should 182558471f63SJani Nikula * deal with them for Intel hardware. 182658471f63SJani Nikula */ 182758471f63SJani Nikula .driver_features = 182858471f63SJani Nikula DRIVER_GEM | 182958471f63SJani Nikula DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ | 183058471f63SJani Nikula DRIVER_SYNCOBJ_TIMELINE, 183158471f63SJani Nikula .release = i915_driver_release, 183258471f63SJani Nikula .open = i915_driver_open, 183358471f63SJani Nikula .lastclose = i915_driver_lastclose, 183458471f63SJani Nikula .postclose = i915_driver_postclose, 183558471f63SJani Nikula 183658471f63SJani Nikula .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 183758471f63SJani Nikula .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 183858471f63SJani Nikula .gem_prime_import = i915_gem_prime_import, 183958471f63SJani Nikula 184058471f63SJani Nikula .dumb_create = i915_gem_dumb_create, 184158471f63SJani Nikula .dumb_map_offset = i915_gem_dumb_mmap_offset, 184258471f63SJani Nikula 184358471f63SJani Nikula .ioctls = i915_ioctls, 184458471f63SJani Nikula .num_ioctls = ARRAY_SIZE(i915_ioctls), 184558471f63SJani Nikula .fops = &i915_driver_fops, 184658471f63SJani Nikula .name = DRIVER_NAME, 184758471f63SJani Nikula .desc = DRIVER_DESC, 184858471f63SJani Nikula .date = DRIVER_DATE, 184958471f63SJani Nikula .major = DRIVER_MAJOR, 185058471f63SJani Nikula .minor = DRIVER_MINOR, 185158471f63SJani Nikula .patchlevel = DRIVER_PATCHLEVEL, 185258471f63SJani Nikula }; 1853