3  * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
28  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
34 #include <linux/iosys-map.h>
79  * drm_clflush_pages - Flush dcache lines of a set of pages.
114 	WARN_ONCE(1, "Architecture has no drm_cache.c support\n");  in drm_clflush_pages()
120  * drm_clflush_sg - Flush dcache lines pointing to a scather-gather.
143 	WARN_ONCE(1, "Architecture has no drm_cache.c support\n");  in drm_clflush_sg()
149  * drm_clflush_virt_range - Flush dcache lines of a region
164 		addr = (void *)(((unsigned long)addr) & -size);  in drm_clflush_virt_range()
168 		clflushopt(end - 1); /* force serialisation */  in drm_clflush_virt_range()
175 	WARN_ONCE(1, "Architecture has no drm_cache.c support\n");  in drm_clflush_virt_range()
204 	for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling)  in drm_need_swiotlb()
205 		max_iomem = max(max_iomem,  tmp->end);  in drm_need_swiotlb()
212 			    const struct iosys_map *src,  in memcpy_fallback()  argument
215 	if (!dst->is_iomem && !src->is_iomem) {  in memcpy_fallback()
216 		memcpy(dst->vaddr, src->vaddr, len);  in memcpy_fallback()
217 	} else if (!src->is_iomem) {  in memcpy_fallback()
218 		iosys_map_memcpy_to(dst, 0, src->vaddr, len);  in memcpy_fallback()
219 	} else if (!dst->is_iomem) {  in memcpy_fallback()
220 		memcpy_fromio(dst->vaddr, src->vaddr_iomem, len);  in memcpy_fallback()
228 		void __iomem *_src = src->vaddr_iomem;  in memcpy_fallback()
229 		void __iomem *_dst = dst->vaddr_iomem;  in memcpy_fallback()
236 			len -= MEMCPY_BOUNCE_SIZE;  in memcpy_fallback()
249 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)  in __memcpy_ntdqa()  argument
254 		asm("movntdqa	(%0), %%xmm0\n"  in __memcpy_ntdqa()
255 		    "movntdqa 16(%0), %%xmm1\n"  in __memcpy_ntdqa()
256 		    "movntdqa 32(%0), %%xmm2\n"  in __memcpy_ntdqa()
257 		    "movntdqa 48(%0), %%xmm3\n"  in __memcpy_ntdqa()
258 		    "movaps %%xmm0,   (%1)\n"  in __memcpy_ntdqa()
259 		    "movaps %%xmm1, 16(%1)\n"  in __memcpy_ntdqa()
260 		    "movaps %%xmm2, 32(%1)\n"  in __memcpy_ntdqa()
261 		    "movaps %%xmm3, 48(%1)\n"  in __memcpy_ntdqa()
262 		    :: "r" (src), "r" (dst) : "memory");  in __memcpy_ntdqa()
263 		src += 64;  in __memcpy_ntdqa()
265 		len -= 4;  in __memcpy_ntdqa()
267 	while (len--) {  in __memcpy_ntdqa()
268 		asm("movntdqa (%0), %%xmm0\n"  in __memcpy_ntdqa()
269 		    "movaps %%xmm0, (%1)\n"  in __memcpy_ntdqa()
270 		    :: "r" (src), "r" (dst) : "memory");  in __memcpy_ntdqa()
271 		src += 16;  in __memcpy_ntdqa()
279  * __drm_memcpy_from_wc copies @len bytes from @src to @dst using
280  * non-temporal instructions where available. Note that all arguments
281  * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
284 static void __drm_memcpy_from_wc(void *dst, const void *src, unsigned long len)  in __drm_memcpy_from_wc()  argument
286 	if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))  in __drm_memcpy_from_wc()
287 		memcpy(dst, src, len);  in __drm_memcpy_from_wc()
289 		__memcpy_ntdqa(dst, src, len >> 4);  in __drm_memcpy_from_wc()
293  * drm_memcpy_from_wc - Perform the fastest available memcpy from a source
296  * @src: The source pointer
303 			const struct iosys_map *src,  in drm_memcpy_from_wc()  argument
307 		memcpy_fallback(dst, src, len);  in drm_memcpy_from_wc()
312 		__drm_memcpy_from_wc(dst->is_iomem ?  in drm_memcpy_from_wc()
313 				     (void __force *)dst->vaddr_iomem :  in drm_memcpy_from_wc()
314 				     dst->vaddr,  in drm_memcpy_from_wc()
315 				     src->is_iomem ?  in drm_memcpy_from_wc()
316 				     (void const __force *)src->vaddr_iomem :  in drm_memcpy_from_wc()
317 				     src->vaddr,  in drm_memcpy_from_wc()
322 	memcpy_fallback(dst, src, len);  in drm_memcpy_from_wc()
327  * drm_memcpy_init_early - One time initialization of the WC memcpy code
332 	 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions  in drm_memcpy_init_early()
341 			const struct iosys_map *src,  in drm_memcpy_from_wc()  argument
346 	memcpy_fallback(dst, src, len);  in drm_memcpy_from_wc()