| /linux/include/crypto/ |
| H A D | scatterwalk.h | 29 static inline void scatterwalk_start(struct scatter_walk *walk, in scatterwalk_start() argument
32 walk->sg = sg; in scatterwalk_start()
33 walk->offset = sg->offset; in scatterwalk_start()
40 static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, in scatterwalk_start_at_pos() argument
48 walk->sg = sg; in scatterwalk_start_at_pos()
49 walk->offset = sg->offset + pos; in scatterwalk_start_at_pos()
52 static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, in scatterwalk_clamp() argument
58 if (walk->offset >= walk->sg->offset + walk->sg->length) in scatterwalk_clamp()
59 scatterwalk_start(walk, sg_next(walk->sg)); in scatterwalk_clamp()
60 len_this_sg = walk->sg->offset + walk->sg->length - walk->offset; in scatterwalk_clamp()
[all …]
|
| /linux/arch/arm64/crypto/ |
| H A D | aes-neonbs-glue.c | 92 struct skcipher_walk walk; in __ecb_crypt() local
95 err = skcipher_walk_virt(&walk, req, false); in __ecb_crypt()
97 while (walk.nbytes >= AES_BLOCK_SIZE) { in __ecb_crypt()
98 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; in __ecb_crypt()
100 if (walk.nbytes < walk.total) in __ecb_crypt()
102 walk.stride / AES_BLOCK_SIZE); in __ecb_crypt()
105 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, in __ecb_crypt()
107 err = skcipher_walk_done(&walk, in __ecb_crypt()
108 walk.nbytes - blocks * AES_BLOCK_SIZE); in __ecb_crypt()
154 struct skcipher_walk walk; in cbc_encrypt() local
[all …]
|
| H A D | sm4-neon-glue.c | 38 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
42 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
44 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
45 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
46 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
55 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); in sm4_ecb_do_crypt()
81 struct skcipher_walk walk; in sm4_cbc_encrypt() local
85 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_encrypt()
87 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_encrypt()
88 const u8 *iv = walk.iv; in sm4_cbc_encrypt()
[all …]
|
| H A D | sm4-ce-glue.c | 108 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
112 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
114 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
115 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
116 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
127 err = skcipher_walk_done(&walk, nbytes); in sm4_ecb_do_crypt()
152 struct skcipher_walk walk; in sm4_cbc_crypt() local
156 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_crypt()
160 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_crypt()
161 const u8 *src = walk.src.virt.addr; in sm4_cbc_crypt()
[all …]
|
| H A D | sm4-ce-ccm-glue.c | 96 struct scatter_walk walk; in ccm_calculate_auth_mac() local
111 scatterwalk_start(&walk, req->src); in ccm_calculate_auth_mac()
117 orig_n = scatterwalk_next(&walk, assoclen); in ccm_calculate_auth_mac()
118 p = walk.addr; in ccm_calculate_auth_mac()
151 scatterwalk_done_src(&walk, orig_n); in ccm_calculate_auth_mac()
156 static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk, in ccm_crypt() argument
166 memcpy(ctr0, walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
167 crypto_inc(walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
173 while (walk->nbytes) { in ccm_crypt()
174 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in ccm_crypt()
[all …]
|
| H A D | sm4-ce-gcm-glue.c | 77 struct scatter_walk walk; in gcm_calculate_auth_mac() local
80 scatterwalk_start(&walk, req->src); in gcm_calculate_auth_mac()
86 orig_n = scatterwalk_next(&walk, assoclen); in gcm_calculate_auth_mac()
87 p = walk.addr; in gcm_calculate_auth_mac()
119 scatterwalk_done_src(&walk, orig_n); in gcm_calculate_auth_mac()
130 static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk, in gcm_crypt() argument
145 lengths.b = cpu_to_be64(walk->total * 8); in gcm_crypt()
155 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in gcm_crypt()
156 const u8 *src = walk->src.virt.addr; in gcm_crypt()
157 u8 *dst = walk->dst.virt.addr; in gcm_crypt()
[all …]
|
| /linux/drivers/gpu/drm/xe/ |
| H A D | xe_pt_walk.c | 28 const struct xe_pt_walk *walk) in xe_pt_addr_end() argument
30 u64 size = 1ull << walk->shifts[level]; in xe_pt_addr_end()
37 unsigned int level, const struct xe_pt_walk *walk) in xe_pt_next() argument
42 if (unlikely(walk->shared_pt_mode)) { in xe_pt_next()
43 unsigned int shift = walk->shifts[level]; in xe_pt_next()
74 u64 addr, u64 end, struct xe_pt_walk *walk) in xe_pt_walk_range() argument
76 pgoff_t offset = xe_pt_offset(addr, level, walk); in xe_pt_walk_range()
77 struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) : in xe_pt_walk_range()
79 const struct xe_pt_walk_ops *ops = walk->ops; in xe_pt_walk_range()
86 next = xe_pt_addr_end(addr, end, level, walk); in xe_pt_walk_range()
[all …]
|
| /linux/crypto/ |
| H A D | acompress.c | 464 void acomp_walk_done_src(struct acomp_walk *walk, int used) in acomp_walk_done_src() argument
466 walk->slen -= used; in acomp_walk_done_src()
467 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) in acomp_walk_done_src()
468 scatterwalk_advance(&walk->in, used); in acomp_walk_done_src()
470 scatterwalk_done_src(&walk->in, used); in acomp_walk_done_src()
472 if ((walk->flags & ACOMP_WALK_SLEEP)) in acomp_walk_done_src()
477 void acomp_walk_done_dst(struct acomp_walk *walk, int used) in acomp_walk_done_dst() argument
479 walk->dlen -= used; in acomp_walk_done_dst()
480 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) in acomp_walk_done_dst()
481 scatterwalk_advance(&walk->out, used); in acomp_walk_done_dst()
[all …]
|
| H A D | scatterwalk.c | 18 void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) in scatterwalk_skip() argument
20 struct scatterlist *sg = walk->sg; in scatterwalk_skip()
22 nbytes += walk->offset - sg->offset; in scatterwalk_skip()
28 walk->sg = sg; in scatterwalk_skip()
29 walk->offset = sg->offset + nbytes; in scatterwalk_skip()
33 inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, in memcpy_from_scatterwalk() argument
39 to_copy = scatterwalk_next(walk, nbytes); in memcpy_from_scatterwalk()
40 memcpy(buf, walk->addr, to_copy); in memcpy_from_scatterwalk()
41 scatterwalk_done_src(walk, to_copy); in memcpy_from_scatterwalk()
48 inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, in memcpy_to_scatterwalk() argument
[all …]
|
| H A D | zstd.c | 104 struct acomp_walk walk; in zstd_compress() local
114 ret = acomp_walk_virt(&walk, req, true); in zstd_compress()
125 dcur = acomp_walk_next_dst(&walk); in zstd_compress()
132 outbuf.dst = (u8 *)walk.dst.virt.addr; in zstd_compress()
136 scur = acomp_walk_next_src(&walk); in zstd_compress()
138 ret = zstd_compress_one(req, ctx, walk.src.virt.addr, in zstd_compress()
139 walk.dst.virt.addr, &total_out); in zstd_compress()
140 acomp_walk_done_src(&walk, scur); in zstd_compress()
141 acomp_walk_done_dst(&walk, dcur); in zstd_compress()
147 inbuf.src = walk.src.virt.addr; in zstd_compress()
[all …]
|
| H A D | deflate.c | 66 struct acomp_walk walk; in deflate_compress_one() local
69 ret = acomp_walk_virt(&walk, req, true); in deflate_compress_one()
76 dcur = acomp_walk_next_dst(&walk); in deflate_compress_one()
81 stream->next_out = walk.dst.virt.addr; in deflate_compress_one()
90 scur = acomp_walk_next_src(&walk); in deflate_compress_one()
92 if (acomp_walk_more_src(&walk, scur)) in deflate_compress_one()
95 stream->next_in = walk.src.virt.addr; in deflate_compress_one()
102 acomp_walk_done_src(&walk, scur); in deflate_compress_one()
106 acomp_walk_done_dst(&walk, dcur); in deflate_compress_one()
146 struct acomp_walk walk; in deflate_decompress_one() local
[all …]
|
| H A D | chacha.c | 52 struct skcipher_walk walk; in chacha_stream_xor() local
56 err = skcipher_walk_virt(&walk, req, false); in chacha_stream_xor()
60 while (walk.nbytes > 0) { in chacha_stream_xor()
61 unsigned int nbytes = walk.nbytes; in chacha_stream_xor()
63 if (nbytes < walk.total) in chacha_stream_xor()
66 chacha_crypt(&state, walk.dst.virt.addr, walk.src.virt.addr, in chacha_stream_xor()
68 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); in chacha_stream_xor()
|
| /linux/arch/s390/crypto/ |
| H A D | paes_s390.c | 349 struct skcipher_walk walk; member
401 struct skcipher_walk *walk = &req_ctx->walk; in ecb_paes_do_crypt() local
434 while ((nbytes = walk->nbytes) != 0) { in ecb_paes_do_crypt()
438 walk->dst.virt.addr, walk->src.virt.addr, n); in ecb_paes_do_crypt()
440 rc = skcipher_walk_done(walk, nbytes - k); in ecb_paes_do_crypt()
468 struct skcipher_walk *walk = &req_ctx->walk; in ecb_paes_crypt() local
480 rc = skcipher_walk_virt(walk, req, false); in ecb_paes_crypt()
508 if (cleanup && walk->nbytes) in ecb_paes_crypt()
509 skcipher_walk_done(walk, rc); in ecb_paes_crypt()
553 struct skcipher_walk *walk = &req_ctx->walk; in ecb_paes_do_one_request() local
[all …]
|
| H A D | aes_s390.c | 64 struct scatter_walk walk; member
124 struct skcipher_walk walk; in ecb_aes_crypt() local
131 ret = skcipher_walk_virt(&walk, req, false); in ecb_aes_crypt()
132 while (!ret && ((nbytes = walk.nbytes) != 0)) { in ecb_aes_crypt()
136 walk.dst.virt.addr, walk.src.virt.addr, n); in ecb_aes_crypt()
137 ret = skcipher_walk_done(&walk, nbytes - n); in ecb_aes_crypt()
220 struct skcipher_walk walk; in cbc_aes_crypt() local
231 ret = skcipher_walk_virt(&walk, req, false); in cbc_aes_crypt()
234 memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); in cbc_aes_crypt()
236 while (!ret && ((nbytes = walk.nbytes) != 0)) { in cbc_aes_crypt()
[all …]
|
| /linux/mm/ |
| H A D | hmm.c | 92 struct mm_walk *walk) in hmm_record_fault() argument
94 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_record_fault()
165 __always_unused int depth, struct mm_walk *walk) in hmm_vma_walk_hole() argument
167 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_vma_walk_hole()
178 if (!walk->vma) { in hmm_vma_walk_hole()
184 return hmm_record_fault(addr, end, required_fault, walk); in hmm_vma_walk_hole()
204 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, in hmm_vma_handle_pmd() argument
208 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_vma_handle_pmd()
219 return hmm_record_fault(addr, end, required_fault, walk); in hmm_vma_handle_pmd()
230 int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
[all …]
|
| /linux/arch/x86/crypto/ |
| H A D | ecb_cbc_helpers.h | 18 struct skcipher_walk walk; \
19 int err = skcipher_walk_virt(&walk, (req), false); \
20 while (walk.nbytes > 0) { \
21 unsigned int nbytes = walk.nbytes; \
24 const u8 *src = walk.src.virt.addr; \
25 u8 *dst = walk.dst.virt.addr; \
51 const u8 *__iv = walk.iv; \
58 memcpy(walk.iv, __iv, __bsize); \
72 crypto_xor(dst, walk.iv, __bsize); \
73 memcpy(walk.iv, __iv, __bsize); \
[all …]
|
| H A D | aria_aesni_avx2_glue.c | 91 struct skcipher_walk walk; in aria_avx2_ctr_encrypt() local
95 err = skcipher_walk_virt(&walk, req, false); in aria_avx2_ctr_encrypt()
97 while ((nbytes = walk.nbytes) > 0) { in aria_avx2_ctr_encrypt()
98 const u8 *src = walk.src.virt.addr; in aria_avx2_ctr_encrypt()
99 u8 *dst = walk.dst.virt.addr; in aria_avx2_ctr_encrypt()
105 walk.iv); in aria_avx2_ctr_encrypt()
116 walk.iv); in aria_avx2_ctr_encrypt()
124 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
125 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
137 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx2_ctr_encrypt()
[all …]
|
| H A D | aria_aesni_avx_glue.c | 89 struct skcipher_walk walk; in aria_avx_ctr_encrypt() local
93 err = skcipher_walk_virt(&walk, req, false); in aria_avx_ctr_encrypt()
95 while ((nbytes = walk.nbytes) > 0) { in aria_avx_ctr_encrypt()
96 const u8 *src = walk.src.virt.addr; in aria_avx_ctr_encrypt()
97 u8 *dst = walk.dst.virt.addr; in aria_avx_ctr_encrypt()
103 walk.iv); in aria_avx_ctr_encrypt()
111 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
112 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
124 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx_ctr_encrypt()
125 memcpy(&req_ctx->keystream[0], walk.iv, in aria_avx_ctr_encrypt()
[all …]
|
| /linux/drivers/crypto/ |
| H A D | padlock-aes.c | 348 struct skcipher_walk walk; in ecb_aes_encrypt() local
354 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_encrypt()
356 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_encrypt()
357 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_encrypt()
361 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_encrypt()
373 struct skcipher_walk walk; in ecb_aes_decrypt() local
379 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_decrypt()
381 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_decrypt()
382 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_decrypt()
386 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_decrypt()
[all …]
|
| /linux/mm/damon/ |
| H A D | vaddr.c | 269 unsigned long next, struct mm_walk *walk) in damon_mkold_pmd_entry() argument
274 ptl = pmd_trans_huge_lock(pmd, walk->vma); in damon_mkold_pmd_entry()
279 damon_pmdp_mkold(pmd, walk->vma, addr); in damon_mkold_pmd_entry()
284 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); in damon_mkold_pmd_entry()
289 damon_ptep_mkold(pte, walk->vma, addr); in damon_mkold_pmd_entry()
332 struct mm_walk *walk) in damon_mkold_hugetlb_entry() argument
334 struct hstate *h = hstate_vma(walk->vma); in damon_mkold_hugetlb_entry()
338 ptl = huge_pte_lock(h, walk->mm, pte); in damon_mkold_hugetlb_entry()
339 entry = huge_ptep_get(walk->mm, addr, pte); in damon_mkold_hugetlb_entry()
343 damon_hugetlb_mkold(pte, walk->mm, walk->vma, addr); in damon_mkold_hugetlb_entry()
[all …]
|
| /linux/arch/loongarch/mm/ |
| H A D | pageattr.c | 17 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) in set_pageattr_masks() argument
20 struct pageattr_masks *masks = walk->private; in set_pageattr_masks()
29 unsigned long next, struct mm_walk *walk) in pageattr_pgd_entry() argument
34 val = __pgd(set_pageattr_masks(pgd_val(val), walk)); in pageattr_pgd_entry()
42 unsigned long next, struct mm_walk *walk) in pageattr_p4d_entry() argument
47 val = __p4d(set_pageattr_masks(p4d_val(val), walk)); in pageattr_p4d_entry()
55 unsigned long next, struct mm_walk *walk) in pageattr_pud_entry() argument
60 val = __pud(set_pageattr_masks(pud_val(val), walk)); in pageattr_pud_entry()
68 unsigned long next, struct mm_walk *walk) in pageattr_pmd_entry() argument
73 val = __pmd(set_pageattr_masks(pmd_val(val), walk)); in pageattr_pmd_entry()
[all …]
|
| /linux/drivers/media/mc/ |
| H A D | mc-entity.c | 477 media_pipeline_walk_top(struct media_pipeline_walk *walk) in media_pipeline_walk_top() argument
479 return &walk->stack.entries[walk->stack.top]; in media_pipeline_walk_top()
482 static bool media_pipeline_walk_empty(struct media_pipeline_walk *walk) in media_pipeline_walk_empty() argument
484 return walk->stack.top == -1; in media_pipeline_walk_empty()
488 static int media_pipeline_walk_resize(struct media_pipeline_walk *walk) in media_pipeline_walk_resize() argument
494 if (walk->stack.size >= 256) in media_pipeline_walk_resize()
497 new_size = walk->stack.size + MEDIA_PIPELINE_STACK_GROW_STEP; in media_pipeline_walk_resize()
499 entries = krealloc(walk->stack.entries, in media_pipeline_walk_resize()
500 new_size * sizeof(*walk->stack.entries), in media_pipeline_walk_resize()
505 walk->stack.entries = entries; in media_pipeline_walk_resize()
[all …]
|
| /linux/drivers/gpu/drm/ttm/ |
| H A D | ttm_bo.c | 608 struct ttm_lru_walk walk; member
627 static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) in ttm_bo_evict_cb() argument
630 container_of(walk, typeof(*evict_walk), walk); in ttm_bo_evict_cb()
686 lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); in ttm_bo_evict_cb()
690 lret = ttm_bo_evict(bo, walk->arg.ctx); in ttm_bo_evict_cb()
730 .walk = { in ttm_bo_evict_alloc()
746 evict_walk.walk.arg.trylock_only = true; in ttm_bo_evict_alloc()
747 lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); in ttm_bo_evict_alloc()
756 lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); in ttm_bo_evict_alloc()
764 evict_walk.walk.arg.trylock_only = false; in ttm_bo_evict_alloc()
[all …]
|
| /linux/net/xfrm/ |
| H A D | xfrm_iptfs.c | 335 struct iptfs_skb_frag_walk *walk) in iptfs_skb_prepare_frag_walk() argument
341 walk->initial_offset = initial_offset; in iptfs_skb_prepare_frag_walk()
342 walk->fragi = 0; in iptfs_skb_prepare_frag_walk()
343 walk->past = 0; in iptfs_skb_prepare_frag_walk()
344 walk->total = 0; in iptfs_skb_prepare_frag_walk()
345 walk->nr_frags = 0; in iptfs_skb_prepare_frag_walk()
346 walk->pp_recycle = skb->pp_recycle; in iptfs_skb_prepare_frag_walk()
352 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
356 walk->total += frag->len; in iptfs_skb_prepare_frag_walk()
369 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
[all …]
|
| /linux/security/ |
| H A D | device_cgroup.c | 102 struct dev_exception_item *excopy, *walk; in dev_exception_add() local
110 list_for_each_entry(walk, &dev_cgroup->exceptions, list) { in dev_exception_add()
111 if (walk->type != ex->type) in dev_exception_add()
113 if (walk->major != ex->major) in dev_exception_add()
115 if (walk->minor != ex->minor) in dev_exception_add()
118 walk->access |= ex->access; in dev_exception_add()
134 struct dev_exception_item *walk, *tmp; in dev_exception_rm() local
138 list_for_each_entry_safe(walk, tmp, &dev_cgroup->exceptions, list) { in dev_exception_rm()
139 if (walk->type != ex->type) in dev_exception_rm()
141 if (walk->major != ex->major) in dev_exception_rm()
[all …]
|