1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3 *
4 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 #include "vmwgfx_drv.h"
30
31 #include "vmwgfx_bo.h"
32 #include <linux/highmem.h>
33 #include <linux/overflow.h>
34
35 /*
36 * Template that implements find_first_diff() for a generic
37 * unsigned integer type. @size and return value are in bytes.
38 */
39 #define VMW_FIND_FIRST_DIFF(_type) \
40 static size_t vmw_find_first_diff_ ## _type \
41 (const _type * dst, const _type * src, size_t size)\
42 { \
43 size_t i; \
44 \
45 for (i = 0; i < size; i += sizeof(_type)) { \
46 if (*dst++ != *src++) \
47 break; \
48 } \
49 \
50 return i; \
51 }
52
53
54 /*
55 * Template that implements find_last_diff() for a generic
56 * unsigned integer type. Pointers point to the item following the
57 * *end* of the area to be examined. @size and return value are in
58 * bytes.
59 */
60 #define VMW_FIND_LAST_DIFF(_type) \
61 static ssize_t vmw_find_last_diff_ ## _type( \
62 const _type * dst, const _type * src, size_t size) \
63 { \
64 while (size) { \
65 if (*--dst != *--src) \
66 break; \
67 \
68 size -= sizeof(_type); \
69 } \
70 return size; \
71 }
72
73
74 /*
75 * Instantiate find diff functions for relevant unsigned integer sizes,
76 * assuming that wider integers are faster (including aligning) up to the
77 * architecture native width, which is assumed to be 32 bit unless
78 * CONFIG_64BIT is defined.
79 */
80 VMW_FIND_FIRST_DIFF(u8);
81 VMW_FIND_LAST_DIFF(u8);
82
83 VMW_FIND_FIRST_DIFF(u16);
84 VMW_FIND_LAST_DIFF(u16);
85
86 VMW_FIND_FIRST_DIFF(u32);
87 VMW_FIND_LAST_DIFF(u32);
88
89 #ifdef CONFIG_64BIT
90 VMW_FIND_FIRST_DIFF(u64);
91 VMW_FIND_LAST_DIFF(u64);
92 #endif
93
94
95 /* We use size aligned copies. This computes (addr - align(addr)) */
96 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
97
98
99 /*
100 * Template to compute find_first_diff() for a certain integer type
101 * including a head copy for alignment, and adjustment of parameters
102 * for tail find or increased resolution find using an unsigned integer find
103 * of smaller width. If finding is complete, and resolution is sufficient,
104 * the macro executes a return statement. Otherwise it falls through.
105 */
106 #define VMW_TRY_FIND_FIRST_DIFF(_type) \
107 do { \
108 unsigned int spill = SPILL(dst, _type); \
109 size_t diff_offs; \
110 \
111 if (spill && spill == SPILL(src, _type) && \
112 sizeof(_type) - spill <= size) { \
113 spill = sizeof(_type) - spill; \
114 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \
115 if (diff_offs < spill) \
116 return round_down(offset + diff_offs, granularity); \
117 \
118 dst += spill; \
119 src += spill; \
120 size -= spill; \
121 offset += spill; \
122 spill = 0; \
123 } \
124 if (!spill && !SPILL(src, _type)) { \
125 size_t to_copy = size & ~(sizeof(_type) - 1); \
126 \
127 diff_offs = vmw_find_first_diff_ ## _type \
128 ((_type *) dst, (_type *) src, to_copy); \
129 if (diff_offs >= size || granularity == sizeof(_type)) \
130 return (offset + diff_offs); \
131 \
132 dst += diff_offs; \
133 src += diff_offs; \
134 size -= diff_offs; \
135 offset += diff_offs; \
136 } \
137 } while (0) \
138
139
140 /**
141 * vmw_find_first_diff - find the first difference between dst and src
142 *
143 * @dst: The destination address
144 * @src: The source address
145 * @size: Number of bytes to compare
146 * @granularity: The granularity needed for the return value in bytes.
147 * return: The offset from find start where the first difference was
148 * encountered in bytes. If no difference was found, the function returns
149 * a value >= @size.
150 */
vmw_find_first_diff(const u8 * dst,const u8 * src,size_t size,size_t granularity)151 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
152 size_t granularity)
153 {
154 size_t offset = 0;
155
156 /*
157 * Try finding with large integers if alignment allows, or we can
158 * fix it. Fall through if we need better resolution or alignment
159 * was bad.
160 */
161 #ifdef CONFIG_64BIT
162 VMW_TRY_FIND_FIRST_DIFF(u64);
163 #endif
164 VMW_TRY_FIND_FIRST_DIFF(u32);
165 VMW_TRY_FIND_FIRST_DIFF(u16);
166
167 return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
168 granularity);
169 }
170
171
172 /*
173 * Template to compute find_last_diff() for a certain integer type
174 * including a tail copy for alignment, and adjustment of parameters
175 * for head find or increased resolution find using an unsigned integer find
176 * of smaller width. If finding is complete, and resolution is sufficient,
177 * the macro executes a return statement. Otherwise it falls through.
178 */
179 #define VMW_TRY_FIND_LAST_DIFF(_type) \
180 do { \
181 unsigned int spill = SPILL(dst, _type); \
182 ssize_t location; \
183 ssize_t diff_offs; \
184 \
185 if (spill && spill <= size && spill == SPILL(src, _type)) { \
186 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
187 if (diff_offs) { \
188 location = size - spill + diff_offs - 1; \
189 return round_down(location, granularity); \
190 } \
191 \
192 dst -= spill; \
193 src -= spill; \
194 size -= spill; \
195 spill = 0; \
196 } \
197 if (!spill && !SPILL(src, _type)) { \
198 size_t to_copy = round_down(size, sizeof(_type)); \
199 \
200 diff_offs = vmw_find_last_diff_ ## _type \
201 ((_type *) dst, (_type *) src, to_copy); \
202 location = size - to_copy + diff_offs - sizeof(_type); \
203 if (location < 0 || granularity == sizeof(_type)) \
204 return location; \
205 \
206 dst -= to_copy - diff_offs; \
207 src -= to_copy - diff_offs; \
208 size -= to_copy - diff_offs; \
209 } \
210 } while (0)
211
212
213 /**
214 * vmw_find_last_diff - find the last difference between dst and src
215 *
216 * @dst: The destination address
217 * @src: The source address
218 * @size: Number of bytes to compare
219 * @granularity: The granularity needed for the return value in bytes.
220 * return: The offset from find start where the last difference was
221 * encountered in bytes, or a negative value if no difference was found.
222 */
vmw_find_last_diff(const u8 * dst,const u8 * src,size_t size,size_t granularity)223 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
224 size_t granularity)
225 {
226 dst += size;
227 src += size;
228
229 #ifdef CONFIG_64BIT
230 VMW_TRY_FIND_LAST_DIFF(u64);
231 #endif
232 VMW_TRY_FIND_LAST_DIFF(u32);
233 VMW_TRY_FIND_LAST_DIFF(u16);
234
235 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
236 granularity);
237 }
238
239
240 /**
241 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
242 * struct vmw_diff_cpy.
243 *
244 * @diff: The struct vmw_diff_cpy closure argument (unused).
245 * @dest: The copy destination.
246 * @src: The copy source.
247 * @n: Number of bytes to copy.
248 */
vmw_memcpy(struct vmw_diff_cpy * diff,u8 * dest,const u8 * src,size_t n)249 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
250 {
251 memcpy(dest, src, n);
252 }
253
254
255 /**
256 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
257 *
258 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
259 * @diff_offs: The offset from @diff->line_offset where the difference was
260 * found.
261 */
vmw_adjust_rect(struct vmw_diff_cpy * diff,size_t diff_offs)262 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
263 {
264 size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
265 struct drm_rect *rect = &diff->rect;
266
267 rect->x1 = min_t(int, rect->x1, offs);
268 rect->x2 = max_t(int, rect->x2, offs + 1);
269 rect->y1 = min_t(int, rect->y1, diff->line);
270 rect->y2 = max_t(int, rect->y2, diff->line + 1);
271 }
272
273 /**
274 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
275 *
276 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
277 * @dest: The copy destination.
278 * @src: The copy source.
279 * @n: Number of bytes to copy.
280 *
281 * In order to correctly track the modified content, the field @diff->line must
282 * be pre-loaded with the current line number, the field @diff->line_offset must
283 * be pre-loaded with the line offset in bytes where the copy starts, and
284 * finally the field @diff->cpp need to be preloaded with the number of bytes
285 * per unit in the horizontal direction of the area we're examining.
286 * Typically bytes per pixel.
287 * This is needed to know the needed granularity of the difference computing
288 * operations. A higher cpp generally leads to faster execution at the cost of
289 * bounding box width precision.
290 */
vmw_diff_memcpy(struct vmw_diff_cpy * diff,u8 * dest,const u8 * src,size_t n)291 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
292 size_t n)
293 {
294 ssize_t csize, byte_len;
295
296 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
297 return;
298
299 /* TODO: Possibly use a single vmw_find_first_diff per line? */
300 csize = vmw_find_first_diff(dest, src, n, diff->cpp);
301 if (csize < n) {
302 vmw_adjust_rect(diff, csize);
303 byte_len = diff->cpp;
304
305 /*
306 * Starting from where first difference was found, find
307 * location of last difference, and then copy.
308 */
309 diff->line_offset += csize;
310 dest += csize;
311 src += csize;
312 n -= csize;
313 csize = vmw_find_last_diff(dest, src, n, diff->cpp);
314 if (csize >= 0) {
315 byte_len += csize;
316 vmw_adjust_rect(diff, csize);
317 }
318 memcpy(dest, src, byte_len);
319 }
320 diff->line_offset += n;
321 }
322
323 /**
324 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
325 *
326 * @mapped_dst: Already mapped destination page index in @dst_pages.
327 * @dst_addr: Kernel virtual address of mapped destination page.
328 * @dst_pages: Array of destination bo pages.
329 * @dst_num_pages: Number of destination bo pages.
330 * @dst_prot: Destination bo page protection.
331 * @mapped_src: Already mapped source page index in @dst_pages.
332 * @src_addr: Kernel virtual address of mapped source page.
333 * @src_pages: Array of source bo pages.
334 * @src_num_pages: Number of source bo pages.
335 * @src_prot: Source bo page protection.
336 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
337 */
338 struct vmw_bo_blit_line_data {
339 u32 mapped_dst;
340 u8 *dst_addr;
341 struct page **dst_pages;
342 u32 dst_num_pages;
343 pgprot_t dst_prot;
344 u32 mapped_src;
345 u8 *src_addr;
346 struct page **src_pages;
347 u32 src_num_pages;
348 pgprot_t src_prot;
349 struct vmw_diff_cpy *diff;
350 };
351
352 /**
353 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
354 *
355 * @d: Blit data as described above.
356 * @dst_offset: Destination copy start offset from start of bo.
357 * @src_offset: Source copy start offset from start of bo.
358 * @bytes_to_copy: Number of bytes to copy in this line.
359 */
vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data * d,u32 dst_offset,u32 src_offset,u32 bytes_to_copy)360 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
361 u32 dst_offset,
362 u32 src_offset,
363 u32 bytes_to_copy)
364 {
365 struct vmw_diff_cpy *diff = d->diff;
366
367 while (bytes_to_copy) {
368 u32 copy_size = bytes_to_copy;
369 u32 dst_page = dst_offset >> PAGE_SHIFT;
370 u32 src_page = src_offset >> PAGE_SHIFT;
371 u32 dst_page_offset = dst_offset & ~PAGE_MASK;
372 u32 src_page_offset = src_offset & ~PAGE_MASK;
373 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
374 bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
375 unmap_dst);
376
377 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
378 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
379
380 if (unmap_src) {
381 kunmap_atomic(d->src_addr);
382 d->src_addr = NULL;
383 }
384
385 if (unmap_dst) {
386 kunmap_atomic(d->dst_addr);
387 d->dst_addr = NULL;
388 }
389
390 if (!d->dst_addr) {
391 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
392 return -EINVAL;
393
394 d->dst_addr =
395 kmap_atomic_prot(d->dst_pages[dst_page],
396 d->dst_prot);
397 if (!d->dst_addr)
398 return -ENOMEM;
399
400 d->mapped_dst = dst_page;
401 }
402
403 if (!d->src_addr) {
404 if (WARN_ON_ONCE(src_page >= d->src_num_pages))
405 return -EINVAL;
406
407 d->src_addr =
408 kmap_atomic_prot(d->src_pages[src_page],
409 d->src_prot);
410 if (!d->src_addr)
411 return -ENOMEM;
412
413 d->mapped_src = src_page;
414 }
415 diff->do_cpy(diff, d->dst_addr + dst_page_offset,
416 d->src_addr + src_page_offset, copy_size);
417
418 bytes_to_copy -= copy_size;
419 dst_offset += copy_size;
420 src_offset += copy_size;
421 }
422
423 return 0;
424 }
425
map_external(struct vmw_bo * bo,struct iosys_map * map)426 static void *map_external(struct vmw_bo *bo, struct iosys_map *map)
427 {
428 struct vmw_private *vmw =
429 container_of(bo->tbo.bdev, struct vmw_private, bdev);
430 void *ptr = NULL;
431 int ret;
432
433 if (drm_gem_is_imported(&bo->tbo.base)) {
434 ret = dma_buf_vmap(bo->tbo.base.dma_buf, map);
435 if (ret) {
436 drm_dbg_driver(&vmw->drm,
437 "Wasn't able to map external bo!\n");
438 goto out;
439 }
440 ptr = map->vaddr;
441 } else {
442 ptr = vmw_bo_map_and_cache(bo);
443 }
444
445 out:
446 return ptr;
447 }
448
unmap_external(struct vmw_bo * bo,struct iosys_map * map)449 static void unmap_external(struct vmw_bo *bo, struct iosys_map *map)
450 {
451 if (drm_gem_is_imported(&bo->tbo.base))
452 dma_buf_vunmap(bo->tbo.base.dma_buf, map);
453 else
454 vmw_bo_unmap(bo);
455 }
456
vmw_external_bo_copy(struct vmw_bo * dst,u32 dst_offset,u32 dst_stride,struct vmw_bo * src,u32 src_offset,u32 src_stride,u32 width_in_bytes,u32 height,struct vmw_diff_cpy * diff)457 static int vmw_external_bo_copy(struct vmw_bo *dst, u32 dst_offset,
458 u32 dst_stride, struct vmw_bo *src,
459 u32 src_offset, u32 src_stride,
460 u32 width_in_bytes, u32 height,
461 struct vmw_diff_cpy *diff)
462 {
463 struct vmw_private *vmw =
464 container_of(dst->tbo.bdev, struct vmw_private, bdev);
465 size_t dst_size = dst->tbo.resource->size;
466 size_t src_size = src->tbo.resource->size;
467 size_t dst_end, src_end;
468 struct iosys_map dst_map = {0};
469 struct iosys_map src_map = {0};
470 bool dst_mapped = false;
471 bool src_mapped = false;
472 int ret, i;
473 int x_in_bytes;
474 u8 *vsrc;
475 u8 *vdst;
476
477 if (!height || !width_in_bytes)
478 return 0;
479
480 if (!dst_stride || !src_stride)
481 return -EINVAL;
482 if (dst_stride < width_in_bytes || src_stride < width_in_bytes)
483 return -EINVAL;
484 if (check_mul_overflow((size_t)dst_stride, (size_t)height - 1, &dst_end) ||
485 check_add_overflow(dst_end, (size_t)width_in_bytes, &dst_end) ||
486 check_add_overflow((size_t)dst_offset, dst_end, &dst_end) ||
487 dst_end > dst_size ||
488 check_mul_overflow((size_t)src_stride, (size_t)height - 1, &src_end) ||
489 check_add_overflow(src_end, (size_t)width_in_bytes, &src_end) ||
490 check_add_overflow((size_t)src_offset, src_end, &src_end) ||
491 src_end > src_size) {
492 drm_dbg_driver(&vmw->drm, "Out-of-bounds external BO copy\n");
493 return -EINVAL;
494 }
495
496 vsrc = map_external(src, &src_map);
497 if (!vsrc) {
498 drm_dbg_driver(&vmw->drm, "Wasn't able to map src\n");
499 ret = -ENOMEM;
500 goto out;
501 }
502 src_mapped = true;
503
504 vdst = map_external(dst, &dst_map);
505 if (!vdst) {
506 drm_dbg_driver(&vmw->drm, "Wasn't able to map dst\n");
507 ret = -ENOMEM;
508 goto out;
509 }
510 dst_mapped = true;
511
512 vsrc += src_offset;
513 vdst += dst_offset;
514 if (src_stride == dst_stride && width_in_bytes == dst_stride) {
515 memcpy(vdst, vsrc, dst_stride * (size_t)height);
516 } else {
517 for (i = 0; i < height; ++i) {
518 memcpy(vdst, vsrc, width_in_bytes);
519 vsrc += src_stride;
520 vdst += dst_stride;
521 }
522 }
523
524 x_in_bytes = (dst_offset % dst_stride);
525 diff->rect.x1 = x_in_bytes / diff->cpp;
526 diff->rect.y1 = ((dst_offset - x_in_bytes) / dst_stride);
527 diff->rect.x2 = diff->rect.x1 + width_in_bytes / diff->cpp;
528 diff->rect.y2 = diff->rect.y1 + height;
529
530 ret = 0;
531 out:
532 if (src_mapped)
533 unmap_external(src, &src_map);
534 if (dst_mapped)
535 unmap_external(dst, &dst_map);
536
537 return ret;
538 }
539
540 /**
541 * vmw_bo_cpu_blit - in-kernel cpu blit.
542 *
543 * @vmw_dst: Destination buffer object.
544 * @dst_offset: Destination offset of blit start in bytes.
545 * @dst_stride: Destination stride in bytes.
546 * @vmw_src: Source buffer object.
547 * @src_offset: Source offset of blit start in bytes.
548 * @src_stride: Source stride in bytes.
549 * @w: Width of blit.
550 * @h: Height of blit.
551 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
552 * return: Zero on success. Negative error value on failure. Will print out
553 * kernel warnings on caller bugs.
554 *
555 * Performs a CPU blit from one buffer object to another avoiding a full
556 * bo vmap which may exhaust- or fragment vmalloc space.
557 * On supported architectures (x86), we're using kmap_atomic which avoids
558 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
559 * reference already set-up mappings.
560 *
561 * Neither of the buffer objects may be placed in PCI memory
562 * (Fixed memory in TTM terminology) when using this function.
563 */
vmw_bo_cpu_blit(struct vmw_bo * vmw_dst,u32 dst_offset,u32 dst_stride,struct vmw_bo * vmw_src,u32 src_offset,u32 src_stride,u32 w,u32 h,struct vmw_diff_cpy * diff)564 int vmw_bo_cpu_blit(struct vmw_bo *vmw_dst,
565 u32 dst_offset, u32 dst_stride,
566 struct vmw_bo *vmw_src,
567 u32 src_offset, u32 src_stride,
568 u32 w, u32 h,
569 struct vmw_diff_cpy *diff)
570 {
571 struct ttm_buffer_object *src = &vmw_src->tbo;
572 struct ttm_buffer_object *dst = &vmw_dst->tbo;
573 struct ttm_operation_ctx ctx = {
574 .interruptible = false,
575 .no_wait_gpu = false
576 };
577 u32 j, initial_line = dst_offset / dst_stride;
578 struct vmw_bo_blit_line_data d = {0};
579 int ret = 0;
580 struct page **dst_pages = NULL;
581 struct page **src_pages = NULL;
582 bool src_external = (src->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) != 0;
583 bool dst_external = (dst->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) != 0;
584
585 if (WARN_ON(dst == src))
586 return -EINVAL;
587
588 /* Buffer objects need to be either pinned or reserved: */
589 if (!(dst->pin_count))
590 dma_resv_assert_held(dst->base.resv);
591 if (!(src->pin_count))
592 dma_resv_assert_held(src->base.resv);
593
594 if (!ttm_tt_is_populated(dst->ttm)) {
595 ret = dst->bdev->funcs->ttm_tt_populate(dst->bdev, dst->ttm, &ctx);
596 if (ret)
597 return ret;
598 }
599
600 if (!ttm_tt_is_populated(src->ttm)) {
601 ret = src->bdev->funcs->ttm_tt_populate(src->bdev, src->ttm, &ctx);
602 if (ret)
603 return ret;
604 }
605
606 if (src_external || dst_external)
607 return vmw_external_bo_copy(vmw_dst, dst_offset, dst_stride,
608 vmw_src, src_offset, src_stride,
609 w, h, diff);
610
611 if (!src->ttm->pages && src->ttm->sg) {
612 src_pages = kvmalloc_objs(struct page *, src->ttm->num_pages);
613 if (!src_pages)
614 return -ENOMEM;
615 ret = drm_prime_sg_to_page_array(src->ttm->sg, src_pages,
616 src->ttm->num_pages);
617 if (ret)
618 goto out;
619 }
620 if (!dst->ttm->pages && dst->ttm->sg) {
621 dst_pages = kvmalloc_objs(struct page *, dst->ttm->num_pages);
622 if (!dst_pages) {
623 ret = -ENOMEM;
624 goto out;
625 }
626 ret = drm_prime_sg_to_page_array(dst->ttm->sg, dst_pages,
627 dst->ttm->num_pages);
628 if (ret)
629 goto out;
630 }
631
632 d.mapped_dst = 0;
633 d.mapped_src = 0;
634 d.dst_addr = NULL;
635 d.src_addr = NULL;
636 d.dst_pages = dst->ttm->pages ? dst->ttm->pages : dst_pages;
637 d.src_pages = src->ttm->pages ? src->ttm->pages : src_pages;
638 d.dst_num_pages = PFN_UP(dst->resource->size);
639 d.src_num_pages = PFN_UP(src->resource->size);
640 d.dst_prot = ttm_io_prot(dst, dst->resource, PAGE_KERNEL);
641 d.src_prot = ttm_io_prot(src, src->resource, PAGE_KERNEL);
642 d.diff = diff;
643
644 for (j = 0; j < h; ++j) {
645 diff->line = j + initial_line;
646 diff->line_offset = dst_offset % dst_stride;
647 ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w);
648 if (ret)
649 goto out;
650
651 dst_offset += dst_stride;
652 src_offset += src_stride;
653 }
654 out:
655 if (d.src_addr)
656 kunmap_atomic(d.src_addr);
657 if (d.dst_addr)
658 kunmap_atomic(d.dst_addr);
659 kvfree(src_pages);
660 kvfree(dst_pages);
661
662 return ret;
663 }
664