1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #ifndef I915_SCATTERLIST_H 8 #define I915_SCATTERLIST_H 9 10 #include <linux/pfn.h> 11 #include <linux/scatterlist.h> 12 #include <linux/swiotlb.h> 13 14 #include "i915_gem.h" 15 16 struct drm_mm_node; 17 struct ttm_resource; 18 19 /* 20 * Optimised SGL iterator for GEM objects 21 */ 22 static __always_inline struct sgt_iter { 23 struct scatterlist *sgp; 24 union { 25 unsigned long pfn; 26 dma_addr_t dma; 27 }; 28 unsigned int curr; 29 unsigned int max; 30 } __sgt_iter(struct scatterlist *sgl, bool dma) { 31 struct sgt_iter s = { .sgp = sgl }; 32 33 if (dma && s.sgp && sg_dma_len(s.sgp) == 0) { 34 s.sgp = NULL; 35 } else if (s.sgp) { 36 s.max = s.curr = s.sgp->offset; 37 if (dma) { 38 s.dma = sg_dma_address(s.sgp); 39 s.max += sg_dma_len(s.sgp); 40 } else { 41 s.pfn = page_to_pfn(sg_page(s.sgp)); 42 s.max += s.sgp->length; 43 } 44 } 45 46 return s; 47 } 48 49 static inline int __sg_page_count(const struct scatterlist *sg) 50 { 51 return sg->length >> PAGE_SHIFT; 52 } 53 54 static inline int __sg_dma_page_count(const struct scatterlist *sg) 55 { 56 return sg_dma_len(sg) >> PAGE_SHIFT; 57 } 58 59 static inline struct scatterlist *____sg_next(struct scatterlist *sg) 60 { 61 ++sg; 62 if (unlikely(sg_is_chain(sg))) 63 sg = sg_chain_ptr(sg); 64 return sg; 65 } 66 67 /** 68 * __sg_next - return the next scatterlist entry in a list 69 * @sg: The current sg entry 70 * 71 * Description: 72 * If the entry is the last, return NULL; otherwise, step to the next 73 * element in the array (@sg@+1). If that's a chain pointer, follow it; 74 * otherwise just return the pointer to the current element. 75 **/ 76 static inline struct scatterlist *__sg_next(struct scatterlist *sg) 77 { 78 return sg_is_last(sg) ? NULL : ____sg_next(sg); 79 } 80 81 /** 82 * __for_each_sgt_daddr - iterate over the device addresses of the given sg_table 83 * @__dp: Device address (output) 84 * @__iter: 'struct sgt_iter' (iterator state, internal) 85 * @__sgt: sg_table to iterate over (input) 86 * @__step: step size 87 */ 88 #define __for_each_sgt_daddr(__dp, __iter, __sgt, __step) \ 89 for ((__iter) = __sgt_iter((__sgt)->sgl, true); \ 90 ((__dp) = (__iter).dma + (__iter).curr), (__iter).sgp; \ 91 (((__iter).curr += (__step)) >= (__iter).max) ? \ 92 (__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0 : 0) 93 94 /** 95 * for_each_sgt_page - iterate over the pages of the given sg_table 96 * @__pp: page pointer (output) 97 * @__iter: 'struct sgt_iter' (iterator state, internal) 98 * @__sgt: sg_table to iterate over (input) 99 */ 100 #define for_each_sgt_page(__pp, __iter, __sgt) \ 101 for ((__iter) = __sgt_iter((__sgt)->sgl, false); \ 102 ((__pp) = (__iter).pfn == 0 ? NULL : \ 103 pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \ 104 (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \ 105 (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0) 106 107 /** 108 * i915_sg_dma_sizes - Record the dma segment sizes of a scatterlist 109 * @sg: The scatterlist 110 * 111 * Return: An unsigned int with segment sizes logically or'ed together. 112 * A caller can use this information to determine what hardware page table 113 * entry sizes can be used to map the memory represented by the scatterlist. 114 */ 115 static inline unsigned int i915_sg_dma_sizes(struct scatterlist *sg) 116 { 117 unsigned int page_sizes; 118 119 page_sizes = 0; 120 while (sg && sg_dma_len(sg)) { 121 GEM_BUG_ON(sg->offset); 122 GEM_BUG_ON(!IS_ALIGNED(sg_dma_len(sg), PAGE_SIZE)); 123 page_sizes |= sg_dma_len(sg); 124 sg = __sg_next(sg); 125 } 126 127 return page_sizes; 128 } 129 130 static inline unsigned int i915_sg_segment_size(void) 131 { 132 unsigned int size = swiotlb_max_segment(); 133 134 if (size == 0) 135 size = UINT_MAX; 136 137 size = rounddown(size, PAGE_SIZE); 138 /* swiotlb_max_segment_size can return 1 byte when it means one page. */ 139 if (size < PAGE_SIZE) 140 size = PAGE_SIZE; 141 142 return size; 143 } 144 145 bool i915_sg_trim(struct sg_table *orig_st); 146 147 /** 148 * struct i915_refct_sgt_ops - Operations structure for struct i915_refct_sgt 149 */ 150 struct i915_refct_sgt_ops { 151 /** 152 * release() - Free the memory of the struct i915_refct_sgt 153 * @ref: struct kref that is embedded in the struct i915_refct_sgt 154 */ 155 void (*release)(struct kref *ref); 156 }; 157 158 /** 159 * struct i915_refct_sgt - A refcounted scatter-gather table 160 * @kref: struct kref for refcounting 161 * @table: struct sg_table holding the scatter-gather table itself. Note that 162 * @table->sgl = NULL can be used to determine whether a scatter-gather table 163 * is present or not. 164 * @size: The size in bytes of the underlying memory buffer 165 * @ops: The operations structure. 166 */ 167 struct i915_refct_sgt { 168 struct kref kref; 169 struct sg_table table; 170 size_t size; 171 const struct i915_refct_sgt_ops *ops; 172 }; 173 174 /** 175 * i915_refct_sgt_put - Put a refcounted sg-table 176 * @rsgt the struct i915_refct_sgt to put. 177 */ 178 static inline void i915_refct_sgt_put(struct i915_refct_sgt *rsgt) 179 { 180 if (rsgt) 181 kref_put(&rsgt->kref, rsgt->ops->release); 182 } 183 184 /** 185 * i915_refct_sgt_get - Get a refcounted sg-table 186 * @rsgt the struct i915_refct_sgt to get. 187 */ 188 static inline struct i915_refct_sgt * 189 i915_refct_sgt_get(struct i915_refct_sgt *rsgt) 190 { 191 kref_get(&rsgt->kref); 192 return rsgt; 193 } 194 195 /** 196 * __i915_refct_sgt_init - Initialize a refcounted sg-list with a custom 197 * operations structure 198 * @rsgt The struct i915_refct_sgt to initialize. 199 * @size: Size in bytes of the underlying memory buffer. 200 * @ops: A customized operations structure in case the refcounted sg-list 201 * is embedded into another structure. 202 */ 203 static inline void __i915_refct_sgt_init(struct i915_refct_sgt *rsgt, 204 size_t size, 205 const struct i915_refct_sgt_ops *ops) 206 { 207 kref_init(&rsgt->kref); 208 rsgt->table.sgl = NULL; 209 rsgt->size = size; 210 rsgt->ops = ops; 211 } 212 213 void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size); 214 215 struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, 216 u64 region_start, 217 u32 page_alignment); 218 219 struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, 220 u64 region_start, 221 u32 page_alignment); 222 223 #endif 224