1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cryptographic API. 4 * 5 * Cipher operations. 6 * 7 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> 8 * 2002 Adam J. Richter <adam@yggdrasil.com> 9 * 2004 Jean-Luc Cooke <jlcooke@certainkey.com> 10 */ 11 12 #include <crypto/scatterwalk.h> 13 #include <linux/kernel.h> 14 #include <linux/mm.h> 15 #include <linux/module.h> 16 #include <linux/scatterlist.h> 17 18 void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) 19 { 20 struct scatterlist *sg = walk->sg; 21 22 nbytes += walk->offset - sg->offset; 23 24 while (nbytes > sg->length) { 25 nbytes -= sg->length; 26 sg = sg_next(sg); 27 } 28 walk->sg = sg; 29 walk->offset = sg->offset + nbytes; 30 } 31 EXPORT_SYMBOL_GPL(scatterwalk_skip); 32 33 inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, 34 unsigned int nbytes) 35 { 36 do { 37 unsigned int to_copy; 38 39 to_copy = scatterwalk_next(walk, nbytes); 40 memcpy(buf, walk->addr, to_copy); 41 scatterwalk_done_src(walk, to_copy); 42 buf += to_copy; 43 nbytes -= to_copy; 44 } while (nbytes); 45 } 46 EXPORT_SYMBOL_GPL(memcpy_from_scatterwalk); 47 48 inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, 49 unsigned int nbytes) 50 { 51 do { 52 unsigned int to_copy; 53 54 to_copy = scatterwalk_next(walk, nbytes); 55 memcpy(walk->addr, buf, to_copy); 56 scatterwalk_done_dst(walk, to_copy); 57 buf += to_copy; 58 nbytes -= to_copy; 59 } while (nbytes); 60 } 61 EXPORT_SYMBOL_GPL(memcpy_to_scatterwalk); 62 63 void memcpy_from_sglist(void *buf, struct scatterlist *sg, 64 unsigned int start, unsigned int nbytes) 65 { 66 struct scatter_walk walk; 67 68 if (unlikely(nbytes == 0)) /* in case sg == NULL */ 69 return; 70 71 scatterwalk_start_at_pos(&walk, sg, start); 72 memcpy_from_scatterwalk(buf, &walk, nbytes); 73 } 74 EXPORT_SYMBOL_GPL(memcpy_from_sglist); 75 76 void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, 77 const void *buf, unsigned int nbytes) 78 { 79 struct scatter_walk walk; 80 81 if (unlikely(nbytes == 0)) /* in case sg == NULL */ 82 return; 83 84 scatterwalk_start_at_pos(&walk, sg, start); 85 memcpy_to_scatterwalk(&walk, buf, nbytes); 86 } 87 EXPORT_SYMBOL_GPL(memcpy_to_sglist); 88 89 /** 90 * memcpy_sglist() - Copy data from one scatterlist to another 91 * @dst: The destination scatterlist. Can be NULL if @nbytes == 0. 92 * @src: The source scatterlist. Can be NULL if @nbytes == 0. 93 * @nbytes: Number of bytes to copy 94 * 95 * The scatterlists can describe exactly the same memory, in which case this 96 * function is a no-op. No other overlaps are supported. 97 * 98 * Context: Any context 99 */ 100 void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, 101 unsigned int nbytes) 102 { 103 unsigned int src_offset, dst_offset; 104 105 if (unlikely(nbytes == 0)) /* in case src and/or dst is NULL */ 106 return; 107 108 src_offset = src->offset; 109 dst_offset = dst->offset; 110 for (;;) { 111 /* Compute the length to copy this step. */ 112 unsigned int len = min3(src->offset + src->length - src_offset, 113 dst->offset + dst->length - dst_offset, 114 nbytes); 115 struct page *src_page = sg_page(src); 116 struct page *dst_page = sg_page(dst); 117 const void *src_virt; 118 void *dst_virt; 119 120 if (IS_ENABLED(CONFIG_HIGHMEM)) { 121 /* HIGHMEM: we may have to actually map the pages. */ 122 const unsigned int src_oip = offset_in_page(src_offset); 123 const unsigned int dst_oip = offset_in_page(dst_offset); 124 const unsigned int limit = PAGE_SIZE; 125 126 /* Further limit len to not cross a page boundary. */ 127 len = min3(len, limit - src_oip, limit - dst_oip); 128 129 /* Compute the source and destination pages. */ 130 src_page += src_offset / PAGE_SIZE; 131 dst_page += dst_offset / PAGE_SIZE; 132 133 if (src_page != dst_page) { 134 /* Copy between different pages. */ 135 memcpy_page(dst_page, dst_oip, 136 src_page, src_oip, len); 137 flush_dcache_page(dst_page); 138 } else if (src_oip != dst_oip) { 139 /* Copy between different parts of same page. */ 140 dst_virt = kmap_local_page(dst_page); 141 memcpy(dst_virt + dst_oip, dst_virt + src_oip, 142 len); 143 kunmap_local(dst_virt); 144 flush_dcache_page(dst_page); 145 } /* Else, it's the same memory. No action needed. */ 146 } else { 147 /* 148 * !HIGHMEM: no mapping needed. Just work in the linear 149 * buffer of each sg entry. Note that we can cross page 150 * boundaries, as they are not significant in this case. 151 */ 152 src_virt = page_address(src_page) + src_offset; 153 dst_virt = page_address(dst_page) + dst_offset; 154 if (src_virt != dst_virt) { 155 memcpy(dst_virt, src_virt, len); 156 if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) 157 __scatterwalk_flush_dcache_pages( 158 dst_page, dst_offset, len); 159 } /* Else, it's the same memory. No action needed. */ 160 } 161 nbytes -= len; 162 if (nbytes == 0) /* No more to copy? */ 163 break; 164 165 /* 166 * There's more to copy. Advance the offsets by the length 167 * copied this step, and advance the sg entries as needed. 168 */ 169 src_offset += len; 170 if (src_offset >= src->offset + src->length) { 171 src = sg_next(src); 172 src_offset = src->offset; 173 } 174 dst_offset += len; 175 if (dst_offset >= dst->offset + dst->length) { 176 dst = sg_next(dst); 177 dst_offset = dst->offset; 178 } 179 } 180 } 181 EXPORT_SYMBOL_GPL(memcpy_sglist); 182 183 struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], 184 struct scatterlist *src, 185 unsigned int len) 186 { 187 for (;;) { 188 if (!len) 189 return src; 190 191 if (src->length > len) 192 break; 193 194 len -= src->length; 195 src = sg_next(src); 196 } 197 198 sg_init_table(dst, 2); 199 sg_set_page(dst, sg_page(src), src->length - len, src->offset + len); 200 scatterwalk_crypto_chain(dst, sg_next(src), 2); 201 202 return dst; 203 } 204 EXPORT_SYMBOL_GPL(scatterwalk_ffwd); 205