drm/xe/xe_pm.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2022 Intel Corporation
 */

#include "xe_pm.h"

#include <linux/fault-inject.h>
#include <linux/pm_runtime.h>
#include <linux/suspend.h>
#include <linux/dmi.h>

#include <drm/drm_managed.h>
#include <drm/ttm/ttm_placement.h>

#include "display/xe_display.h"
#include "xe_bo.h"
#include "xe_bo_evict.h"
#include "xe_device.h"
#include "xe_ggtt.h"
#include "xe_gt.h"
#include "xe_gt_idle.h"
#include "xe_i2c.h"
#include "xe_irq.h"
#include "xe_late_bind_fw.h"
#include "xe_pcode.h"
#include "xe_pxp.h"
#include "xe_sriov_vf_ccs.h"
#include "xe_sysctrl.h"
#include "xe_trace.h"
#include "xe_vm.h"
#include "xe_wa.h"

/**
 * DOC: Xe Power Management
 *
 * Xe PM implements the main routines for both system level suspend states and
 * for the opportunistic runtime suspend states.
 *
 * System Level Suspend (S-States) - In general this is OS initiated suspend
 * driven by ACPI for achieving S0ix (a.k.a. S2idle, freeze), S3 (suspend to ram),
 * S4 (disk). The main functions here are `xe_pm_suspend` and `xe_pm_resume`. They
 * are the main point for the suspend to and resume from these states.
 *
 * PCI Device Suspend (D-States) - This is the opportunistic PCIe device low power
 * state D3, controlled by the PCI subsystem and ACPI with the help from the
 * runtime_pm infrastructure.
 * PCI D3 is special and can mean D3hot, where Vcc power is on for keeping memory
 * alive and quicker low latency resume or D3Cold where Vcc power is off for
 * better power savings.
 * The Vcc control of PCI hierarchy can only be controlled at the PCI root port
 * level, while the device driver can be behind multiple bridges/switches and
 * paired with other devices. For this reason, the PCI subsystem cannot perform
 * the transition towards D3Cold. The lowest runtime PM possible from the PCI
 * subsystem is D3hot. Then, if all these paired devices in the same root port
 * are in D3hot, ACPI will assist here and run its own methods (_PR3 and _OFF)
 * to perform the transition from D3hot to D3cold. Xe may disallow this
 * transition by calling pci_d3cold_disable(root_pdev) before going to runtime
 * suspend. It will be based on runtime conditions such as VRAM usage for a
 * quick and low latency resume for instance.
 *
 * Runtime PM - This infrastructure provided by the Linux kernel allows the
 * device drivers to indicate when the can be runtime suspended, so the device
 * could be put at D3 (if supported), or allow deeper package sleep states
 * (PC-states), and/or other low level power states. Xe PM component provides
 * `xe_pm_runtime_suspend` and `xe_pm_runtime_resume` functions that PCI
 * subsystem will call before transition to/from runtime suspend.
 *
 * Also, Xe PM provides get and put functions that Xe driver will use to
 * indicate activity. In order to avoid locking complications with the memory
 * management, whenever possible, these get and put functions needs to be called
 * from the higher/outer levels.
 * The main cases that need to be protected from the outer levels are: IOCTL,
 * sysfs, debugfs, dma-buf sharing, GPU execution.
 *
 * This component is not responsible for GT idleness (RC6) nor GT frequency
 * management (RPS).
 */

#ifdef CONFIG_LOCKDEP
static struct lockdep_map xe_pm_runtime_d3cold_map = {
	.name = "xe_rpm_d3cold_map"
};

static struct lockdep_map xe_pm_runtime_nod3cold_map = {
	.name = "xe_rpm_nod3cold_map"
};

static struct lockdep_map xe_pm_block_lockdep_map = {
	.name = "xe_pm_block_map",
};
#endif

static void xe_pm_block_begin_signalling(void)
{
	lock_acquire_shared_recursive(&xe_pm_block_lockdep_map, 0, 1, NULL, _RET_IP_);
}

static void xe_pm_block_end_signalling(void)
{
	lock_release(&xe_pm_block_lockdep_map, _RET_IP_);
}

/**
 * xe_pm_might_block_on_suspend() - Annotate that the code might block on suspend
 *
 * Annotation to use where the code might block or seize to make
 * progress pending resume completion.
 */
void xe_pm_might_block_on_suspend(void)
{
	lock_map_acquire(&xe_pm_block_lockdep_map);
	lock_map_release(&xe_pm_block_lockdep_map);
}

/**
 * xe_pm_block_on_suspend() - Block pending suspend.
 * @xe: The xe device about to be suspended.
 *
 * Block if the pm notifier has start evicting bos, to avoid
 * racing and validating those bos back. The function is
 * annotated to ensure no locks are held that are also grabbed
 * in the pm notifier or the device suspend / resume.
 * This is intended to be used by freezable tasks only.
 * (Not freezable workqueues), with the intention that the function
 * returns %-ERESTARTSYS when tasks are frozen during suspend,
 * and allows the task to freeze. The caller must be able to
 * handle the %-ERESTARTSYS.
 *
 * Return: %0 on success, %-ERESTARTSYS on signal pending or
 * if freezing requested.
 */
int xe_pm_block_on_suspend(struct xe_device *xe)
{
	xe_pm_might_block_on_suspend();

	return wait_for_completion_interruptible(&xe->pm_block);
}

/**
 * xe_rpm_reclaim_safe() - Whether runtime resume can be done from reclaim context
 * @xe: The xe device.
 *
 * Return: true if it is safe to runtime resume from reclaim context.
 * false otherwise.
 */
bool xe_rpm_reclaim_safe(const struct xe_device *xe)
{
	return !xe->d3cold.capable;
}

static void xe_rpm_lockmap_acquire(const struct xe_device *xe)
{
	lock_map_acquire(xe_rpm_reclaim_safe(xe) ?
			 &xe_pm_runtime_nod3cold_map :
			 &xe_pm_runtime_d3cold_map);
}

static void xe_rpm_lockmap_release(const struct xe_device *xe)
{
	lock_map_release(xe_rpm_reclaim_safe(xe) ?
			 &xe_pm_runtime_nod3cold_map :
			 &xe_pm_runtime_d3cold_map);
}

/**
 * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle
 * @xe: xe device instance
 *
 * Return: 0 on success
 */
int xe_pm_suspend(struct xe_device *xe)
{
	struct xe_gt *gt;
	u8 id;
	int err;

	drm_dbg(&xe->drm, "Suspending device\n");
	xe_pm_block_begin_signalling();
	trace_xe_pm_suspend(xe, __builtin_return_address(0));

	err = xe_pxp_pm_suspend(xe->pxp);
	if (err)
		goto err;

	xe_late_bind_wait_for_worker_completion(&xe->late_bind);

	for_each_gt(gt, xe, id)
		xe_gt_suspend_prepare(gt);

	xe_display_pm_suspend(xe);

	/* FIXME: Super racey... */
	err = xe_bo_evict_all(xe);
	if (err)
		goto err_display;

	for_each_gt(gt, xe, id) {
		err = xe_gt_suspend(gt);
		if (err)
			goto err_display;
	}

	xe_irq_suspend(xe);

	xe_display_pm_suspend_late(xe);

	xe_i2c_pm_suspend(xe);

	drm_dbg(&xe->drm, "Device suspended\n");
	xe_pm_block_end_signalling();

	return 0;

err_display:
	xe_display_pm_resume(xe);
	xe_pxp_pm_resume(xe->pxp);
err:
	drm_dbg(&xe->drm, "Device suspend failed %d\n", err);
	xe_pm_block_end_signalling();
	return err;
}

/**
 * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0
 * @xe: xe device instance
 *
 * Return: 0 on success
 */
int xe_pm_resume(struct xe_device *xe)
{
	struct xe_tile *tile;
	struct xe_gt *gt;
	u8 id;
	int err;

	xe_pm_block_begin_signalling();
	drm_dbg(&xe->drm, "Resuming device\n");
	trace_xe_pm_resume(xe, __builtin_return_address(0));

	for_each_gt(gt, xe, id)
		xe_gt_idle_disable_c6(gt);

	for_each_tile(tile, xe, id)
		xe_wa_apply_tile_workarounds(tile);

	err = xe_pcode_ready(xe, true);
	if (err)
		return err;

	xe_display_pm_resume_early(xe);

	/*
	 * This only restores pinned memory which is the memory required for the
	 * GT(s) to resume.
	 */
	err = xe_bo_restore_early(xe);
	if (err)
		goto err;

	xe_i2c_pm_resume(xe, true);

	xe_sysctrl_pm_resume(xe);

	xe_irq_resume(xe);

	for_each_gt(gt, xe, id) {
		err = xe_gt_resume(gt);
		if (err)
			break;
	}

	/*
	 * Try to bring up display before bailing from GT resume failure,
	 * so we don't leave the user clueless with a blank screen.
	 */
	xe_display_pm_resume(xe);
	if (err)
		goto err;

	err = xe_bo_restore_late(xe);
	if (err)
		goto err;

	xe_pxp_pm_resume(xe->pxp);

	if (IS_VF_CCS_READY(xe))
		xe_sriov_vf_ccs_register_context(xe);

	xe_late_bind_fw_load(&xe->late_bind);

	drm_dbg(&xe->drm, "Device resumed\n");
	xe_pm_block_end_signalling();
	return 0;
err:
	drm_dbg(&xe->drm, "Device resume failed %d\n", err);
	xe_pm_block_end_signalling();
	return err;
}

static bool xe_pm_pci_d3cold_capable(struct xe_device *xe)
{
	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
	struct pci_dev *root_pdev;

	root_pdev = pcie_find_root_port(pdev);
	if (!root_pdev)
		return false;

	/* D3Cold requires PME capability */
	if (!pci_pme_capable(root_pdev, PCI_D3cold)) {
		drm_dbg(&xe->drm, "d3cold: PME# not supported\n");
		return false;
	}

	/* D3Cold requires _PR3 power resource */
	if (!pci_pr3_present(root_pdev)) {
		drm_dbg(&xe->drm, "d3cold: ACPI _PR3 not present\n");
		return false;
	}

	return true;
}

static void xe_pm_runtime_init(struct xe_device *xe)
{
	struct device *dev = xe->drm.dev;

	/* Our current VFs do not support RPM. so, disable it */
	if (IS_SRIOV_VF(xe))
		return;

	/*
	 * Disable the system suspend direct complete optimization.
	 * We need to ensure that the regular device suspend/resume functions
	 * are called since our runtime_pm cannot guarantee local memory
	 * eviction for d3cold.
	 * TODO: Check HDA audio dependencies claimed by i915, and then enforce
	 *       this option to integrated graphics as well.
	 */
	if (IS_DGFX(xe))
		dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);

	pm_runtime_use_autosuspend(dev);
	pm_runtime_set_autosuspend_delay(dev, 1000);
	pm_runtime_set_active(dev);
	pm_runtime_allow(dev);
	pm_runtime_mark_last_busy(dev);
	pm_runtime_put(dev);
}

int xe_pm_init_early(struct xe_device *xe)
{
	int err;

	INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list);

	err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock);
	if (err)
		return err;

	err = drmm_mutex_init(&xe->drm, &xe->d3cold.lock);
	if (err)
		return err;

	xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe);
	return 0;
}
ALLOW_ERROR_INJECTION(xe_pm_init_early, ERRNO); /* See xe_pci_probe() */

static u32 vram_threshold_value(struct xe_device *xe)
{
	if (xe->info.platform == XE_BATTLEMAGE) {
		const char *product_name;

		product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
		if (product_name && strstr(product_name, "NUC13RNG")) {
			drm_warn(&xe->drm, "BMG + D3Cold not supported on this platform\n");
			return 0;
		}
	}

	return DEFAULT_VRAM_THRESHOLD;
}

static void xe_pm_wake_rebind_workers(struct xe_device *xe)
{
	struct xe_vm *vm, *next;

	mutex_lock(&xe->rebind_resume_lock);
	list_for_each_entry_safe(vm, next, &xe->rebind_resume_list,
				 preempt.pm_activate_link) {
		list_del_init(&vm->preempt.pm_activate_link);
		xe_vm_resume_rebind_worker(vm);
	}
	mutex_unlock(&xe->rebind_resume_lock);
}

static int xe_pm_notifier_callback(struct notifier_block *nb,
				   unsigned long action, void *data)
{
	struct xe_device *xe = container_of(nb, struct xe_device, pm_notifier);
	int err = 0;

	switch (action) {
	case PM_HIBERNATION_PREPARE:
	case PM_SUSPEND_PREPARE:
	{
		struct xe_validation_ctx ctx;

		reinit_completion(&xe->pm_block);
		xe_pm_block_begin_signalling();
		xe_pm_runtime_get(xe);
		(void)xe_validation_ctx_init(&ctx, &xe->val, NULL,
					     (struct xe_val_flags) {.exclusive = true});
		err = xe_bo_evict_all_user(xe);
		xe_validation_ctx_fini(&ctx);
		if (err)
			drm_dbg(&xe->drm, "Notifier evict user failed (%d)\n", err);

		err = xe_bo_notifier_prepare_all_pinned(xe);
		if (err)
			drm_dbg(&xe->drm, "Notifier prepare pin failed (%d)\n", err);
		/*
		 * Keep the runtime pm reference until post hibernation / post suspend to
		 * avoid a runtime suspend interfering with evicted objects or backup
		 * allocations.
		 */
		xe_pm_block_end_signalling();
		break;
	}
	case PM_POST_HIBERNATION:
	case PM_POST_SUSPEND:
		complete_all(&xe->pm_block);
		xe_pm_wake_rebind_workers(xe);
		xe_bo_notifier_unprepare_all_pinned(xe);
		xe_pm_runtime_put(xe);
		break;
	}

	return NOTIFY_DONE;
}

/**
 * xe_pm_init - Initialize Xe Power Management
 * @xe: xe device instance
 *
 * This component is responsible for System and Device sleep states.
 *
 * Returns 0 for success, negative error code otherwise.
 */
int xe_pm_init(struct xe_device *xe)
{
	u32 vram_threshold;
	int err;

	xe->pm_notifier.notifier_call = xe_pm_notifier_callback;
	err = register_pm_notifier(&xe->pm_notifier);
	if (err)
		return err;

	err = drmm_mutex_init(&xe->drm, &xe->rebind_resume_lock);
	if (err)
		goto err_unregister;

	init_completion(&xe->pm_block);
	complete_all(&xe->pm_block);
	INIT_LIST_HEAD(&xe->rebind_resume_list);

	/* For now suspend/resume is only allowed with GuC */
	if (!xe_device_uc_enabled(xe))
		return 0;

	if (xe->d3cold.capable) {
		vram_threshold = vram_threshold_value(xe);
		err = xe_pm_set_vram_threshold(xe, vram_threshold);
		if (err)
			goto err_unregister;
	}

	xe_pm_runtime_init(xe);
	return 0;

err_unregister:
	unregister_pm_notifier(&xe->pm_notifier);
	return err;
}

static void xe_pm_runtime_fini(struct xe_device *xe)
{
	struct device *dev = xe->drm.dev;

	/* Our current VFs do not support RPM. so, disable it */
	if (IS_SRIOV_VF(xe))
		return;

	pm_runtime_get_sync(dev);
	pm_runtime_forbid(dev);
}

/**
 * xe_pm_fini - Finalize PM
 * @xe: xe device instance
 */
void xe_pm_fini(struct xe_device *xe)
{
	if (xe_device_uc_enabled(xe))
		xe_pm_runtime_fini(xe);

	unregister_pm_notifier(&xe->pm_notifier);
}

static void xe_pm_write_callback_task(struct xe_device *xe,
				      struct task_struct *task)
{
	WRITE_ONCE(xe->pm_callback_task, task);

	/*
	 * Just in case it's somehow possible for our writes to be reordered to
	 * the extent that something else re-uses the task written in
	 * pm_callback_task. For example after returning from the callback, but
	 * before the reordered write that resets pm_callback_task back to NULL.
	 */
	smp_mb(); /* pairs with xe_pm_read_callback_task */
}

struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)
{
	smp_mb(); /* pairs with xe_pm_write_callback_task */

	return READ_ONCE(xe->pm_callback_task);
}

/**
 * xe_pm_runtime_suspended - Check if runtime_pm state is suspended
 * @xe: xe device instance
 *
 * This does not provide any guarantee that the device is going to remain
 * suspended as it might be racing with the runtime state transitions.
 * It can be used only as a non-reliable assertion, to ensure that we are not in
 * the sleep state while trying to access some memory for instance.
 *
 * Returns true if PCI device is suspended, false otherwise.
 */
bool xe_pm_runtime_suspended(struct xe_device *xe)
{
	return pm_runtime_suspended(xe->drm.dev);
}

/**
 * xe_pm_runtime_suspend - Prepare our device for D3hot/D3Cold
 * @xe: xe device instance
 *
 * Returns 0 for success, negative error code otherwise.
 */
int xe_pm_runtime_suspend(struct xe_device *xe)
{
	struct xe_bo *bo, *on;
	struct xe_gt *gt;
	u8 id;
	int err = 0;

	trace_xe_pm_runtime_suspend(xe, __builtin_return_address(0));
	/* Disable access_ongoing asserts and prevent recursive pm calls */
	xe_pm_write_callback_task(xe, current);

	/*
	 * The actual xe_pm_runtime_put() is always async underneath, so
	 * exactly where that is called should makes no difference to us. However
	 * we still need to be very careful with the locks that this callback
	 * acquires and the locks that are acquired and held by any callers of
	 * xe_runtime_pm_get(). We already have the matching annotation
	 * on that side, but we also need it here. For example lockdep should be
	 * able to tell us if the following scenario is in theory possible:
	 *
	 * CPU0                          | CPU1 (kworker)
	 * lock(A)                       |
	 *                               | xe_pm_runtime_suspend()
	 *                               |      lock(A)
	 * xe_pm_runtime_get()           |
	 *
	 * This will clearly deadlock since rpm core needs to wait for
	 * xe_pm_runtime_suspend() to complete, but here we are holding lock(A)
	 * on CPU0 which prevents CPU1 making forward progress.  With the
	 * annotation here and in xe_pm_runtime_get() lockdep will see
	 * the potential lock inversion and give us a nice splat.
	 */
	xe_rpm_lockmap_acquire(xe);

	err = xe_pxp_pm_suspend(xe->pxp);
	if (err)
		goto out;

	/*
	 * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify
	 * also checks and deletes bo entry from user fault list.
	 */
	mutex_lock(&xe->mem_access.vram_userfault.lock);
	list_for_each_entry_safe(bo, on,
				 &xe->mem_access.vram_userfault.list, vram_userfault_link)
		xe_bo_runtime_pm_release_mmap_offset(bo);
	mutex_unlock(&xe->mem_access.vram_userfault.lock);

	xe_display_pm_runtime_suspend(xe);

	if (xe->d3cold.allowed) {
		err = xe_bo_evict_all(xe);
		if (err)
			goto out_resume;
	}

	for_each_gt(gt, xe, id) {
		err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt);
		if (err)
			goto out_resume;
	}

	xe_irq_suspend(xe);

	xe_display_pm_runtime_suspend_late(xe);

	xe_i2c_pm_suspend(xe);

	xe_rpm_lockmap_release(xe);
	xe_pm_write_callback_task(xe, NULL);
	return 0;

out_resume:
	xe_display_pm_runtime_resume(xe);
	xe_pxp_pm_resume(xe->pxp);
out:
	xe_rpm_lockmap_release(xe);
	xe_pm_write_callback_task(xe, NULL);
	return err;
}

/**
 * xe_pm_runtime_resume - Waking up from D3hot/D3Cold
 * @xe: xe device instance
 *
 * Returns 0 for success, negative error code otherwise.
 */
int xe_pm_runtime_resume(struct xe_device *xe)
{
	struct xe_gt *gt;
	u8 id;
	int err = 0;

	trace_xe_pm_runtime_resume(xe, __builtin_return_address(0));
	/* Disable access_ongoing asserts and prevent recursive pm calls */
	xe_pm_write_callback_task(xe, current);

	xe_rpm_lockmap_acquire(xe);

	if (xe->d3cold.allowed) {
		for_each_gt(gt, xe, id)
			xe_gt_idle_disable_c6(gt);

		err = xe_pcode_ready(xe, true);
		if (err)
			goto out;

		xe_display_pm_resume_early(xe);

		/*
		 * This only restores pinned memory which is the memory
		 * required for the GT(s) to resume.
		 */
		err = xe_bo_restore_early(xe);
		if (err)
			goto out;
	}

	xe_i2c_pm_resume(xe, xe->d3cold.allowed);

	if (xe->d3cold.allowed)
		xe_sysctrl_pm_resume(xe);

	xe_irq_resume(xe);

	for_each_gt(gt, xe, id) {
		err = xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt);
		if (err)
			break;
	}

	/*
	 * Try to bring up display before bailing from GT resume failure,
	 * so we don't leave the user clueless with a blank screen.
	 */
	xe_display_pm_runtime_resume(xe);
	if (err)
		goto out;

	if (xe->d3cold.allowed) {
		err = xe_bo_restore_late(xe);
		if (err)
			goto out;
	}

	xe_pxp_pm_resume(xe->pxp);

	if (IS_VF_CCS_READY(xe))
		xe_sriov_vf_ccs_register_context(xe);

	if (xe->d3cold.allowed)
		xe_late_bind_fw_load(&xe->late_bind);

out:
	xe_rpm_lockmap_release(xe);
	xe_pm_write_callback_task(xe, NULL);
	return err;
}

/*
 * For places where resume is synchronous it can be quite easy to deadlock
 * if we are not careful. Also in practice it might be quite timing
 * sensitive to ever see the 0 -> 1 transition with the callers locks
 * held, so deadlocks might exist but are hard for lockdep to ever see.
 * With this in mind, help lockdep learn about the potentially scary
 * stuff that can happen inside the runtime_resume callback by acquiring
 * a dummy lock (it doesn't protect anything and gets compiled out on
 * non-debug builds).  Lockdep then only needs to see the
 * xe_pm_runtime_xxx_map -> runtime_resume callback once, and then can
 * hopefully validate all the (callers_locks) -> xe_pm_runtime_xxx_map.
 * For example if the (callers_locks) are ever grabbed in the
 * runtime_resume callback, lockdep should give us a nice splat.
 */
static void xe_rpm_might_enter_cb(const struct xe_device *xe)
{
	xe_rpm_lockmap_acquire(xe);
	xe_rpm_lockmap_release(xe);
}

/*
 * Prime the lockdep maps for known locking orders that need to
 * be supported but that may not always occur on all systems.
 */
static void xe_pm_runtime_lockdep_prime(void)
{
	struct dma_resv lockdep_resv;

	dma_resv_init(&lockdep_resv);
	lock_map_acquire(&xe_pm_runtime_d3cold_map);
	/* D3Cold takes the dma_resv locks to evict bos */
	dma_resv_lock(&lockdep_resv, NULL);
	dma_resv_unlock(&lockdep_resv);
	lock_map_release(&xe_pm_runtime_d3cold_map);

	/* Shrinkers might like to wake up the device under reclaim. */
	fs_reclaim_acquire(GFP_KERNEL);
	lock_map_acquire(&xe_pm_runtime_nod3cold_map);
	lock_map_release(&xe_pm_runtime_nod3cold_map);
	fs_reclaim_release(GFP_KERNEL);
}

/**
 * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously
 * @xe: xe device instance
 *
 * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is
 * be preferred over direct usage of this function.  Manual get/put handling
 * should only be used when the function contains goto-based logic which
 * can break scope-based handling, or when the lifetime of the runtime PM
 * reference does not match a specific scope (e.g., runtime PM obtained in one
 * function and released in a different one).
 */
void xe_pm_runtime_get(struct xe_device *xe)
{
	trace_xe_pm_runtime_get(xe, __builtin_return_address(0));
	pm_runtime_get_noresume(xe->drm.dev);

	if (xe_pm_read_callback_task(xe) == current)
		return;

	xe_rpm_might_enter_cb(xe);
	pm_runtime_resume(xe->drm.dev);
}

/**
 * xe_pm_runtime_put - Put the runtime_pm reference back and mark as idle
 * @xe: xe device instance
 */
void xe_pm_runtime_put(struct xe_device *xe)
{
	trace_xe_pm_runtime_put(xe, __builtin_return_address(0));
	if (xe_pm_read_callback_task(xe) == current) {
		pm_runtime_put_noidle(xe->drm.dev);
	} else {
		pm_runtime_mark_last_busy(xe->drm.dev);
		pm_runtime_put(xe->drm.dev);
	}
}

/**
 * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl
 * @xe: xe device instance
 *
 * When possible, scope-based runtime PM (through
 * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this
 * function.  Manual get/put handling should only be used when the function
 * contains goto-based logic which can break scope-based handling, or when the
 * lifetime of the runtime PM reference does not match a specific scope (e.g.,
 * runtime PM obtained in one function and released in a different one).
 *
 * Returns: Any number greater than or equal to 0 for success, negative error
 * code otherwise.
 */
int xe_pm_runtime_get_ioctl(struct xe_device *xe)
{
	trace_xe_pm_runtime_get_ioctl(xe, __builtin_return_address(0));
	if (WARN_ON(xe_pm_read_callback_task(xe) == current))
		return -ELOOP;

	xe_rpm_might_enter_cb(xe);
	return pm_runtime_get_sync(xe->drm.dev);
}

/**
 * xe_pm_runtime_get_if_active - Get a runtime_pm reference if device active
 * @xe: xe device instance
 *
 * Return: True if device is awake (regardless the previous number of references)
 * and a new reference was taken, false otherwise.
 */
bool xe_pm_runtime_get_if_active(struct xe_device *xe)
{
	return pm_runtime_get_if_active(xe->drm.dev) > 0;
}

/**
 * xe_pm_runtime_get_if_in_use - Get a new reference if device is active with previous ref taken
 * @xe: xe device instance
 *
 * Return: True if device is awake, a previous reference had been already taken,
 * and a new reference was now taken, false otherwise.
 */
bool xe_pm_runtime_get_if_in_use(struct xe_device *xe)
{
	if (xe_pm_read_callback_task(xe) == current) {
		/* The device is awake, grab the ref and move on */
		pm_runtime_get_noresume(xe->drm.dev);
		return true;
	}

	return pm_runtime_get_if_in_use(xe->drm.dev) > 0;
}

/*
 * Very unreliable! Should only be used to suppress the false positive case
 * in the missing outer rpm protection warning.
 */
static bool xe_pm_suspending_or_resuming(struct xe_device *xe)
{
#ifdef CONFIG_PM
	struct device *dev = xe->drm.dev;

	return dev->power.runtime_status == RPM_SUSPENDING ||
		dev->power.runtime_status == RPM_RESUMING ||
		pm_suspend_in_progress();
#else
	return false;
#endif
}

/**
 * xe_pm_runtime_get_noresume - Bump runtime PM usage counter without resuming
 * @xe: xe device instance
 *
 * This function should be used in inner places where it is surely already
 * protected by outer-bound callers of `xe_pm_runtime_get`.
 * It will warn if not protected.
 * The reference should be put back after this function regardless, since it
 * will always bump the usage counter, regardless.
 *
 * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume))
 * is be preferred over direct usage of this function.  Manual get/put handling
 * should only be used when the function contains goto-based logic which can
 * break scope-based handling, or when the lifetime of the runtime PM reference
 * does not match a specific scope (e.g., runtime PM obtained in one function
 * and released in a different one).
 */
void xe_pm_runtime_get_noresume(struct xe_device *xe)
{
	bool ref;

	ref = xe_pm_runtime_get_if_in_use(xe);

	if (!ref) {
		pm_runtime_get_noresume(xe->drm.dev);
		drm_WARN(&xe->drm, !xe_pm_suspending_or_resuming(xe),
			 "Missing outer runtime PM protection\n");
	}
}

/**
 * xe_pm_runtime_resume_and_get - Resume, then get a runtime_pm ref if awake.
 * @xe: xe device instance
 *
 * Returns: True if device is awake and the reference was taken, false otherwise.
 */
bool xe_pm_runtime_resume_and_get(struct xe_device *xe)
{
	if (xe_pm_read_callback_task(xe) == current) {
		/* The device is awake, grab the ref and move on */
		pm_runtime_get_noresume(xe->drm.dev);
		return true;
	}

	xe_rpm_might_enter_cb(xe);
	return pm_runtime_resume_and_get(xe->drm.dev) >= 0;
}

/**
 * xe_pm_assert_unbounded_bridge - Disable PM on unbounded pcie parent bridge
 * @xe: xe device instance
 */
void xe_pm_assert_unbounded_bridge(struct xe_device *xe)
{
	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
	struct pci_dev *bridge = pci_upstream_bridge(pdev);

	if (!bridge)
		return;

	if (!bridge->driver) {
		drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n");
		device_set_pm_not_required(&pdev->dev);
	}
}

/**
 * xe_pm_set_vram_threshold - Set a VRAM threshold for allowing/blocking D3Cold
 * @xe: xe device instance
 * @threshold: VRAM size in MiB for the D3cold threshold
 *
 * Return:
 * * 0		- success
 * * -EINVAL	- invalid argument
 */
int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
{
	struct ttm_resource_manager *man;
	u32 vram_total_mb = 0;
	int i;

	for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
		man = ttm_manager_type(&xe->ttm, i);
		if (man)
			vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024);
	}

	drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb);

	if (threshold > vram_total_mb)
		return -EINVAL;

	mutex_lock(&xe->d3cold.lock);
	xe->d3cold.vram_threshold = threshold;
	mutex_unlock(&xe->d3cold.lock);

	return 0;
}

/**
 * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed
 * @xe: xe device instance
 *
 * To be called during runtime_pm idle callback.
 * Check for all the D3Cold conditions ahead of runtime suspend.
 */
void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)
{
	struct ttm_resource_manager *man;
	u32 total_vram_used_mb = 0;
	u64 vram_used;
	int i;

	if (!xe->d3cold.capable) {
		xe->d3cold.allowed = false;
		return;
	}

	for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
		man = ttm_manager_type(&xe->ttm, i);
		if (man) {
			vram_used = ttm_resource_manager_usage(man);
			total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
		}
	}

	mutex_lock(&xe->d3cold.lock);

	if (total_vram_used_mb < xe->d3cold.vram_threshold)
		xe->d3cold.allowed = true;
	else
		xe->d3cold.allowed = false;

	mutex_unlock(&xe->d3cold.lock);
}

/**
 * xe_pm_module_init() - Perform xe_pm specific module initialization.
 *
 * Return: 0 on success. Currently doesn't fail.
 */
int __init xe_pm_module_init(void)
{
	xe_pm_runtime_lockdep_prime();
	return 0;
}