1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include <linux/bitfield.h> 7 #include <linux/kref.h> 8 #include <linux/mm.h> 9 #include <linux/slab.h> 10 11 #include "xe_page_reclaim.h" 12 13 #include "xe_gt_stats.h" 14 #include "xe_guc_tlb_inval.h" 15 #include "xe_macros.h" 16 #include "xe_pat.h" 17 #include "xe_sa.h" 18 #include "xe_tlb_inval_types.h" 19 20 /** 21 * xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA 22 * @tile: Tile owning the VMA 23 * @vma: VMA under consideration 24 * 25 * PPC flushing may be handled by HW for specific PAT encodings. 26 * Skip PPC flushing/Page Reclaim for scenarios below due to redundant 27 * flushes. 28 * - pat_index is transient display (1) 29 * 30 * For cases of NULL VMA, there should be no corresponding PRL entry 31 * so skip over. 32 * 33 * Return: true when page reclamation is unnecessary, false otherwise. 34 */ 35 bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma) 36 { 37 u8 l3_policy; 38 39 if (xe_vma_is_null(vma)) 40 return true; 41 42 l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index); 43 44 /* 45 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC 46 * Transient display flushes is taken care by HW, l3_policy = 1. 47 * 48 * HW will sequence these transient flushes at various sync points so 49 * any event of page reclamation will hit these sync points before 50 * page reclamation could execute. 51 */ 52 return (l3_policy == XE_L3_POLICY_XD); 53 } 54 55 /** 56 * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO 57 * @tlb_inval: TLB invalidation frontend associated with the request 58 * @prl: page reclaim list data that bo will copy from 59 * @fence: tlb invalidation fence that page reclaim action is paired to 60 * 61 * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU 62 * copy into the BO and queues the buffer for release when @fence signals. 63 * 64 * Return: struct drm_suballoc pointer on success or ERR_PTR on failure. 65 */ 66 struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, 67 struct xe_page_reclaim_list *prl, 68 struct xe_tlb_inval_fence *fence) 69 { 70 struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval); 71 struct xe_tile *tile = gt_to_tile(gt); 72 /* (+1) for NULL page_reclaim_entry to indicate end of list */ 73 int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) * 74 sizeof(struct xe_guc_page_reclaim_entry); 75 struct drm_suballoc *prl_sa; 76 77 /* Maximum size of PRL is 1 4K-page */ 78 prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool, 79 prl_size, GFP_ATOMIC); 80 if (IS_ERR(prl_sa)) 81 return prl_sa; 82 83 memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries, 84 prl_size); 85 xe_sa_bo_flush_write(prl_sa); 86 /* Queue up sa_bo_free on tlb invalidation fence signal */ 87 xe_sa_bo_free(prl_sa, &fence->base); 88 89 return prl_sa; 90 } 91 92 /** 93 * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid 94 * @prl: Page reclaim list to reset 95 * 96 * Clears the entries pointer and marks the list as invalid so 97 * future use knows PRL is unusable. It is expected that the entries 98 * have already been released. 99 */ 100 void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl) 101 { 102 xe_page_reclaim_entries_put(prl->entries); 103 prl->entries = NULL; 104 prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST; 105 } 106 107 /** 108 * xe_page_reclaim_list_init() - Initialize a page reclaim list 109 * @prl: Page reclaim list to initialize 110 * 111 * NULLs both values in list to prepare on initalization. 112 */ 113 void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl) 114 { 115 prl->entries = NULL; 116 prl->num_entries = 0; 117 } 118 119 /** 120 * xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries 121 * @prl: Page reclaim list to allocate entries for 122 * 123 * Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL. 124 */ 125 int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl) 126 { 127 struct page *page; 128 129 if (XE_WARN_ON(prl->entries)) 130 return 0; 131 132 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 133 if (page) { 134 prl->entries = page_address(page); 135 prl->num_entries = 0; 136 } 137 138 return page ? 0 : -ENOMEM; 139 } 140 141 /** 142 * xe_guc_page_reclaim_done_handler() - Page reclaim done handler 143 * @guc: guc 144 * @msg: message indicating page reclamation done 145 * @len: length of message 146 * 147 * Page reclamation is an extension of TLB invalidation. Both 148 * operations share the same seqno and fence. When either 149 * action completes, we need to signal the corresponding 150 * fence. Since the handling logic is currently identical, this 151 * function delegates to the TLB invalidation handler. 152 * 153 * Return: 0 on success, -EPROTO for malformed messages. 154 */ 155 int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 156 { 157 return xe_guc_tlb_inval_done_handler(guc, msg, len); 158 } 159