drm/xe/xe_tlb_inval.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2023 Intel Corporation
 */

#include <drm/drm_managed.h>

#include "xe_device_types.h"
#include "xe_force_wake.h"
#include "xe_gt_stats.h"
#include "xe_gt_types.h"
#include "xe_guc_ct.h"
#include "xe_guc_tlb_inval.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_tlb_inval.h"
#include "xe_trace.h"

/**
 * DOC: Xe TLB invalidation
 *
 * Xe TLB invalidation is implemented in two layers. The first is the frontend
 * API, which provides an interface for TLB invalidations to the driver code.
 * The frontend handles seqno assignment, synchronization (fences), and the
 * timeout mechanism. The frontend is implemented via an embedded structure
 * xe_tlb_inval that includes a set of ops hooking into the backend. The backend
 * interacts with the hardware (or firmware) to perform the actual invalidation.
 */

#define FENCE_STACK_BIT		DMA_FENCE_FLAG_USER_BITS

static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
{
	if (WARN_ON_ONCE(!fence->tlb_inval))
		return;

	xe_pm_runtime_put(fence->tlb_inval->xe);
	fence->tlb_inval = NULL; /* fini() should be called once */
}

static void
xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
{
	struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
	bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);

	lockdep_assert_held(&fence->tlb_inval->pending_lock);

	list_del(&fence->link);
	if (list_empty(&tlb_inval->pending_fences))
		cancel_delayed_work(&tlb_inval->fence_tdr);
	trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence);
	xe_tlb_inval_fence_fini(fence);
	dma_fence_signal(&fence->base);
	if (!stack)
		dma_fence_put(&fence->base);
}

static void
xe_tlb_inval_fence_signal_unlocked(struct xe_tlb_inval_fence *fence)
{
	struct xe_tlb_inval *tlb_inval = fence->tlb_inval;

	spin_lock_irq(&tlb_inval->pending_lock);
	xe_tlb_inval_fence_signal(fence);
	spin_unlock_irq(&tlb_inval->pending_lock);
}

static void xe_tlb_inval_fence_timeout(struct work_struct *work)
{
	struct xe_tlb_inval *tlb_inval = container_of(work, struct xe_tlb_inval,
						      fence_tdr.work);
	struct xe_device *xe = tlb_inval->xe;
	struct xe_tlb_inval_fence *fence, *next;
	long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval);

	tlb_inval->ops->flush(tlb_inval);

	spin_lock_irq(&tlb_inval->pending_lock);
	list_for_each_entry_safe(fence, next,
				 &tlb_inval->pending_fences, link) {
		s64 since_inval_ms = ktime_ms_delta(ktime_get(),
						    fence->inval_time);

		if (msecs_to_jiffies(since_inval_ms) < timeout_delay)
			break;

		trace_xe_tlb_inval_fence_timeout(xe, fence);
		drm_err(&xe->drm,
			"TLB invalidation fence timeout, seqno=%d recv=%d",
			fence->seqno, tlb_inval->seqno_recv);

		fence->base.error = -ETIME;
		xe_tlb_inval_fence_signal(fence);
	}
	if (!list_empty(&tlb_inval->pending_fences))
		queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
				   timeout_delay);
	spin_unlock_irq(&tlb_inval->pending_lock);
}

/**
 * tlb_inval_fini - Clean up TLB invalidation state
 * @drm: @drm_device
 * @arg: pointer to struct @xe_tlb_inval
 *
 * Cancel pending fence workers and clean up any additional
 * TLB invalidation state.
 */
static void tlb_inval_fini(struct drm_device *drm, void *arg)
{
	struct xe_tlb_inval *tlb_inval = arg;

	xe_tlb_inval_reset(tlb_inval);
}

static void primelockdep(struct xe_tlb_inval *tlb_inval)
{
	if (!IS_ENABLED(CONFIG_LOCKDEP))
		return;

	fs_reclaim_acquire(GFP_KERNEL);
	might_lock(&tlb_inval->seqno_lock);
	fs_reclaim_release(GFP_KERNEL);
}

/**
 * xe_gt_tlb_inval_init_early() - Initialize TLB invalidation state
 * @gt: GT structure
 *
 * Initialize TLB invalidation state, purely software initialization, should
 * be called once during driver load.
 *
 * Return: 0 on success, negative error code on error.
 */
int xe_gt_tlb_inval_init_early(struct xe_gt *gt)
{
	struct xe_device *xe = gt_to_xe(gt);
	struct xe_tlb_inval *tlb_inval = &gt->tlb_inval;
	int err;

	tlb_inval->xe = xe;
	tlb_inval->seqno = 1;
	INIT_LIST_HEAD(&tlb_inval->pending_fences);
	spin_lock_init(&tlb_inval->pending_lock);
	spin_lock_init(&tlb_inval->lock);
	INIT_DELAYED_WORK(&tlb_inval->fence_tdr, xe_tlb_inval_fence_timeout);

	err = drmm_mutex_init(&xe->drm, &tlb_inval->seqno_lock);
	if (err)
		return err;

	primelockdep(tlb_inval);

	tlb_inval->job_wq = drmm_alloc_ordered_workqueue(&xe->drm,
							 "gt-tbl-inval-job-wq",
							 WQ_MEM_RECLAIM);
	if (IS_ERR(tlb_inval->job_wq))
		return PTR_ERR(tlb_inval->job_wq);

	tlb_inval->timeout_wq = gt->ordered_wq;
	if (IS_ERR(tlb_inval->timeout_wq))
		return PTR_ERR(tlb_inval->timeout_wq);

	/* XXX: Blindly setting up backend to GuC */
	xe_guc_tlb_inval_init_early(&gt->uc.guc, tlb_inval);

	return drmm_add_action_or_reset(&xe->drm, tlb_inval_fini, tlb_inval);
}

/**
 * xe_tlb_inval_reset() - TLB invalidation reset
 * @tlb_inval: TLB invalidation client
 *
 * Signal any pending invalidation fences, should be called during a GT reset
 */
void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
{
	struct xe_tlb_inval_fence *fence, *next;
	int pending_seqno;

	/*
	 * we can get here before the backends are even initialized if we're
	 * wedging very early, in which case there are not going to be any
	 * pendind fences so we can bail immediately.
	 */
	if (!tlb_inval->ops->initialized(tlb_inval))
		return;

	/*
	 * Backend is already disabled at this point. No new TLB requests can
	 * appear.
	 */

	mutex_lock(&tlb_inval->seqno_lock);
	spin_lock_irq(&tlb_inval->pending_lock);
	cancel_delayed_work(&tlb_inval->fence_tdr);
	/*
	 * We might have various kworkers waiting for TLB flushes to complete
	 * which are not tracked with an explicit TLB fence, however at this
	 * stage that will never happen since the backend is already disabled,
	 * so make sure we signal them here under the assumption that we have
	 * completed a full GT reset.
	 */
	if (tlb_inval->seqno == 1)
		pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1;
	else
		pending_seqno = tlb_inval->seqno - 1;
	WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno);

	list_for_each_entry_safe(fence, next,
				 &tlb_inval->pending_fences, link)
		xe_tlb_inval_fence_signal(fence);
	spin_unlock_irq(&tlb_inval->pending_lock);
	mutex_unlock(&tlb_inval->seqno_lock);
}

/**
 * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout
 * @tlb_inval: TLB invalidation client
 *
 * Reset the TLB invalidation timeout timer.
 */
static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval)
{
	lockdep_assert_held(&tlb_inval->pending_lock);

	mod_delayed_work(system_wq, &tlb_inval->fence_tdr,
			 tlb_inval->ops->timeout_delay(tlb_inval));
}

static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
{
	int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);

	lockdep_assert_held(&tlb_inval->pending_lock);

	if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2))
		return false;

	if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2))
		return true;

	return seqno_recv >= seqno;
}

static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
{
	struct xe_tlb_inval *tlb_inval = fence->tlb_inval;

	fence->seqno = tlb_inval->seqno;
	trace_xe_tlb_inval_fence_send(tlb_inval->xe, fence);

	spin_lock_irq(&tlb_inval->pending_lock);
	fence->inval_time = ktime_get();
	list_add_tail(&fence->link, &tlb_inval->pending_fences);

	if (list_is_singular(&tlb_inval->pending_fences))
		queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
				   tlb_inval->ops->timeout_delay(tlb_inval));
	spin_unlock_irq(&tlb_inval->pending_lock);

	tlb_inval->seqno = (tlb_inval->seqno + 1) %
		TLB_INVALIDATION_SEQNO_MAX;
	if (!tlb_inval->seqno)
		tlb_inval->seqno = 1;
}

#define xe_tlb_inval_issue(__tlb_inval, __fence, op, args...)	\
({								\
	int __ret;						\
								\
	xe_assert((__tlb_inval)->xe, (__tlb_inval)->ops);	\
	xe_assert((__tlb_inval)->xe, (__fence));		\
								\
	mutex_lock(&(__tlb_inval)->seqno_lock); 		\
	xe_tlb_inval_fence_prep((__fence));			\
	__ret = op((__tlb_inval), (__fence)->seqno, ##args);	\
	if (__ret < 0)						\
		xe_tlb_inval_fence_signal_unlocked((__fence));	\
	mutex_unlock(&(__tlb_inval)->seqno_lock);		\
								\
	__ret == -ECANCELED ? 0 : __ret;			\
})

/**
 * xe_tlb_inval_all() - Issue a TLB invalidation for all TLBs
 * @tlb_inval: TLB invalidation client
 * @fence: invalidation fence which will be signal on TLB invalidation
 * completion
 *
 * Issue a TLB invalidation for all TLBs. Completion of TLB is asynchronous and
 * caller can use the invalidation fence to wait for completion.
 *
 * Return: 0 on success, negative error code on error
 */
int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
		     struct xe_tlb_inval_fence *fence)
{
	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->all);
}

/**
 * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT
 * @tlb_inval: TLB invalidation client
 *
 * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and
 * caller can use the invalidation fence to wait for completion.
 *
 * Return: 0 on success, negative error code on error
 */
int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
{
	struct xe_tlb_inval_fence fence, *fence_ptr = &fence;
	int ret;

	xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true);
	ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt);
	xe_tlb_inval_fence_wait(fence_ptr);

	return ret;
}

/**
 * xe_tlb_inval_range() - Issue a TLB invalidation for an address range
 * @tlb_inval: TLB invalidation client
 * @fence: invalidation fence which will be signal on TLB invalidation
 * completion
 * @start: start address
 * @end: end address
 * @asid: address space id
 * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush
 *
 * Issue a range based TLB invalidation if supported, if not fallback to a full
 * TLB invalidation. Completion of TLB is asynchronous and caller can use
 * the invalidation fence to wait for completion.
 *
 * Return: Negative error code on error, 0 on success
 */
int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
		       struct xe_tlb_inval_fence *fence, u64 start, u64 end,
		       u32 asid, struct drm_suballoc *prl_sa)
{
	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
				  start, end, asid, prl_sa);
}

/**
 * xe_tlb_inval_vm() - Issue a TLB invalidation for a VM
 * @tlb_inval: TLB invalidation client
 * @vm: VM to invalidate
 *
 * Invalidate entire VM's address space
 */
void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm)
{
	struct xe_tlb_inval_fence fence;
	u64 range = 1ull << vm->xe->info.va_bits;

	xe_tlb_inval_fence_init(tlb_inval, &fence, true);
	xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL);
	xe_tlb_inval_fence_wait(&fence);
}

/**
 * xe_tlb_inval_done_handler() - TLB invalidation done handler
 * @tlb_inval: TLB invalidation client
 * @seqno: seqno of invalidation that is done
 *
 * Update recv seqno, signal any TLB invalidation fences, and restart TDR
 */
void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
{
	struct xe_device *xe = tlb_inval->xe;
	struct xe_tlb_inval_fence *fence, *next;
	unsigned long flags;

	/*
	 * This can also be run both directly from the IRQ handler and also in
	 * process_g2h_msg(). Only one may process any individual CT message,
	 * however the order they are processed here could result in skipping a
	 * seqno. To handle that we just process all the seqnos from the last
	 * seqno_recv up to and including the one in msg[0]. The delta should be
	 * very small so there shouldn't be much of pending_fences we actually
	 * need to iterate over here.
	 *
	 * From GuC POV we expect the seqnos to always appear in-order, so if we
	 * see something later in the timeline we can be sure that anything
	 * appearing earlier has already signalled, just that we have yet to
	 * officially process the CT message like if racing against
	 * process_g2h_msg().
	 */
	spin_lock_irqsave(&tlb_inval->pending_lock, flags);
	if (seqno == TLB_INVALIDATION_SEQNO_INVALID) {
		xe_tlb_inval_reset_timeout(tlb_inval);
		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
		return;
	}

	if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
		return;
	}

	WRITE_ONCE(tlb_inval->seqno_recv, seqno);

	list_for_each_entry_safe(fence, next,
				 &tlb_inval->pending_fences, link) {
		trace_xe_tlb_inval_fence_recv(xe, fence);

		if (!xe_tlb_inval_seqno_past(tlb_inval, fence->seqno))
			break;

		xe_tlb_inval_fence_signal(fence);
	}

	if (!list_empty(&tlb_inval->pending_fences))
		mod_delayed_work(tlb_inval->timeout_wq,
				 &tlb_inval->fence_tdr,
				 tlb_inval->ops->timeout_delay(tlb_inval));
	else
		cancel_delayed_work(&tlb_inval->fence_tdr);

	spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
}

static const char *
xe_inval_fence_get_driver_name(struct dma_fence *dma_fence)
{
	return "xe";
}

static const char *
xe_inval_fence_get_timeline_name(struct dma_fence *dma_fence)
{
	return "tlb_inval_fence";
}

static const struct dma_fence_ops inval_fence_ops = {
	.get_driver_name = xe_inval_fence_get_driver_name,
	.get_timeline_name = xe_inval_fence_get_timeline_name,
};

/**
 * xe_tlb_inval_fence_init() - Initialize TLB invalidation fence
 * @tlb_inval: TLB invalidation client
 * @fence: TLB invalidation fence to initialize
 * @stack: fence is stack variable
 *
 * Initialize TLB invalidation fence for use. xe_tlb_inval_fence_fini
 * will be automatically called when fence is signalled (all fences must signal),
 * even on error.
 */
void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
			     struct xe_tlb_inval_fence *fence,
			     bool stack)
{
	xe_pm_runtime_get_noresume(tlb_inval->xe);

	spin_lock_irq(&tlb_inval->lock);
	dma_fence_init(&fence->base, &inval_fence_ops, &tlb_inval->lock,
		       dma_fence_context_alloc(1), 1);
	spin_unlock_irq(&tlb_inval->lock);
	INIT_LIST_HEAD(&fence->link);
	if (stack)
		set_bit(FENCE_STACK_BIT, &fence->base.flags);
	else
		dma_fence_get(&fence->base);
	fence->tlb_inval = tlb_inval;
}

/**
 * xe_tlb_inval_idle() - Initialize TLB invalidation is idle
 * @tlb_inval: TLB invalidation client
 *
 * Check the TLB invalidation seqno to determine if it is idle (i.e., no TLB
 * invalidations are in flight). Expected to be called in the backend after the
 * fence has been added to the pending list, and takes this into account.
 *
 * Return: True if TLB invalidation client is idle, False otherwise
 */
bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval)
{
	lockdep_assert_held(&tlb_inval->seqno_lock);

	guard(spinlock_irq)(&tlb_inval->pending_lock);
	return list_is_singular(&tlb_inval->pending_fences);
}

/**
 * xe_tlb_inval_batch_wait() - Wait for all fences in a TLB invalidation batch
 * @batch: Batch of TLB invalidation fences to wait on
 *
 * Waits for every fence in @batch to signal, then resets @batch so it can be
 * reused for a subsequent invalidation.
 */
void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch)
{
	struct xe_tlb_inval_fence *fence = &batch->fence[0];
	unsigned int i;

	for (i = 0; i < batch->num_fences; ++i)
		xe_tlb_inval_fence_wait(fence++);

	batch->num_fences = 0;
}

/**
 * xe_tlb_inval_range_tilemask_submit() - Submit TLB invalidations for an
 * address range on a tile mask
 * @xe: The xe device
 * @asid: Address space ID
 * @start: start address
 * @end: end address
 * @tile_mask: mask for which gt's issue tlb invalidation
 * @batch: Batch of tlb invalidate fences
 *
 * Issue a range based TLB invalidation for gt's in tilemask
 * If the function returns an error, there is no need to call
 * xe_tlb_inval_batch_wait() on @batch.
 *
 * Returns 0 for success, negative error code otherwise.
 */
int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid,
				       u64 start, u64 end, u8 tile_mask,
				       struct xe_tlb_inval_batch *batch)
{
	struct xe_tlb_inval_fence *fence = &batch->fence[0];
	struct xe_tile *tile;
	u32 fence_id = 0;
	u8 id;
	int err;

	batch->num_fences = 0;
	if (!tile_mask)
		return 0;

	for_each_tile(tile, xe, id) {
		if (!(tile_mask & BIT(id)))
			continue;

		xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
					&fence[fence_id], true);

		err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
					 &fence[fence_id], start, end,
					 asid, NULL);
		if (err)
			goto wait;
		++fence_id;

		if (!tile->media_gt)
			continue;

		xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
					&fence[fence_id], true);

		err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
					 &fence[fence_id], start, end,
					 asid, NULL);
		if (err)
			goto wait;
		++fence_id;
	}

wait:
	batch->num_fences = fence_id;
	if (err)
		xe_tlb_inval_batch_wait(batch);

	return err;
}