| /linux/include/crypto/ |
| H A D | scatterwalk.h | 29 static inline void scatterwalk_start(struct scatter_walk *walk, in scatterwalk_start() argument
32 walk->sg = sg; in scatterwalk_start()
33 walk->offset = sg->offset; in scatterwalk_start()
40 static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, in scatterwalk_start_at_pos() argument
48 walk->sg = sg; in scatterwalk_start_at_pos()
49 walk->offset = sg->offset + pos; in scatterwalk_start_at_pos()
52 static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, in scatterwalk_clamp() argument
58 if (walk->offset >= walk->sg->offset + walk->sg->length) in scatterwalk_clamp()
59 scatterwalk_start(walk, sg_next(walk->sg)); in scatterwalk_clamp()
60 len_this_sg = walk->sg->offset + walk->sg->length - walk->offset; in scatterwalk_clamp()
[all …]
|
| /linux/mm/ |
| H A D | pagewalk.c | 30 unsigned long end, struct mm_walk *walk) in walk_pte_range_inner() argument
32 const struct mm_walk_ops *ops = walk->ops; in walk_pte_range_inner()
40 walk); in walk_pte_range_inner()
44 set_pte_at(walk->mm, addr, pte, new_pte); in walk_pte_range_inner()
46 if (!WARN_ON_ONCE(walk->no_vma)) in walk_pte_range_inner()
47 update_mmu_cache(walk->vma, addr, pte); in walk_pte_range_inner()
49 err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); in walk_pte_range_inner()
62 struct mm_walk *walk) in walk_pte_range() argument
68 if (walk->no_vma) { in walk_pte_range()
75 if (walk->mm == &init_mm || addr >= TASK_SIZE) in walk_pte_range()
[all …]
|
| H A D | mapping_dirty_helpers.c | 35 struct mm_walk *walk) in wp_pte() argument
37 struct wp_walk *wpwalk = walk->private; in wp_pte()
41 pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); in wp_pte()
44 ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent); in wp_pte()
90 unsigned long end, struct mm_walk *walk) in clean_record_pte() argument
92 struct wp_walk *wpwalk = walk->private; in clean_record_pte()
97 pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) + in clean_record_pte()
98 walk->vma->vm_pgoff - cwalk->bitmap_pgoff; in clean_record_pte()
99 pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); in clean_record_pte()
102 ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent); in clean_record_pte()
[all …]
|
| H A D | hmm.c | 74 unsigned int required_fault, struct mm_walk *walk) in hmm_vma_fault() argument
76 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_vma_fault()
77 struct vm_area_struct *vma = walk->vma; in hmm_vma_fault()
158 __always_unused int depth, struct mm_walk *walk) in hmm_vma_walk_hole() argument
160 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_vma_walk_hole()
171 if (!walk->vma) { in hmm_vma_walk_hole()
177 return hmm_vma_fault(addr, end, required_fault, walk); in hmm_vma_walk_hole()
197 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, in hmm_vma_handle_pmd() argument
201 struct hmm_vma_walk *hmm_vma_walk = walk->private; in hmm_vma_handle_pmd()
212 return hmm_vma_fault(addr, end, required_fault, walk); in hmm_vma_handle_pmd()
[all …]
|
| H A D | mincore.c | 27 unsigned long end, struct mm_walk *walk) in mincore_hugetlb() argument
31 unsigned char *vec = walk->private; in mincore_hugetlb()
34 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); in mincore_hugetlb()
43 const pte_t ptep = huge_ptep_get(walk->mm, addr, pte); in mincore_hugetlb()
53 walk->private = vec; in mincore_hugetlb()
156 struct mm_walk *walk) in mincore_unmapped_range() argument
158 walk->private += __mincore_unmapped_range(addr, end, in mincore_unmapped_range()
159 walk->vma, walk->private); in mincore_unmapped_range()
164 struct mm_walk *walk) in mincore_pte_range() argument
167 struct vm_area_struct *vma = walk->vma; in mincore_pte_range()
[all …]
|
| H A D | migrate_device.c | 23 struct mm_walk *walk) in migrate_vma_collect_skip() argument
25 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_skip()
39 struct mm_walk *walk) in migrate_vma_collect_hole() argument
41 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_hole()
45 if (!vma_is_anonymous(walk->vma)) in migrate_vma_collect_hole()
46 return migrate_vma_collect_skip(start, end, walk); in migrate_vma_collect_hole()
62 return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk); in migrate_vma_collect_hole()
137 unsigned long end, struct mm_walk *walk, in migrate_vma_collect_huge_pmd() argument
140 struct mm_struct *mm = walk->mm; in migrate_vma_collect_huge_pmd()
142 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_huge_pmd()
[all …]
|
| /linux/arch/arm64/crypto/ |
| H A D | aes-neonbs-glue.c | 92 struct skcipher_walk walk; in __ecb_crypt() local
95 err = skcipher_walk_virt(&walk, req, false); in __ecb_crypt()
97 while (walk.nbytes >= AES_BLOCK_SIZE) { in __ecb_crypt()
98 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; in __ecb_crypt()
100 if (walk.nbytes < walk.total) in __ecb_crypt()
102 walk.stride / AES_BLOCK_SIZE); in __ecb_crypt()
105 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, in __ecb_crypt()
107 err = skcipher_walk_done(&walk, in __ecb_crypt()
108 walk.nbytes - blocks * AES_BLOCK_SIZE); in __ecb_crypt()
154 struct skcipher_walk walk; in cbc_encrypt() local
[all …]
|
| H A D | sm4-neon-glue.c | 38 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
42 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
44 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
45 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
46 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
55 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); in sm4_ecb_do_crypt()
81 struct skcipher_walk walk; in sm4_cbc_encrypt() local
85 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_encrypt()
87 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_encrypt()
88 const u8 *iv = walk.iv; in sm4_cbc_encrypt()
[all …]
|
| H A D | sm4-ce-glue.c | 108 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
112 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
114 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
115 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
116 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
127 err = skcipher_walk_done(&walk, nbytes); in sm4_ecb_do_crypt()
152 struct skcipher_walk walk; in sm4_cbc_crypt() local
156 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_crypt()
160 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_crypt()
161 const u8 *src = walk.src.virt.addr; in sm4_cbc_crypt()
[all …]
|
| H A D | sm4-ce-ccm-glue.c | 96 struct scatter_walk walk; in ccm_calculate_auth_mac() local
111 scatterwalk_start(&walk, req->src); in ccm_calculate_auth_mac()
117 orig_n = scatterwalk_next(&walk, assoclen); in ccm_calculate_auth_mac()
118 p = walk.addr; in ccm_calculate_auth_mac()
151 scatterwalk_done_src(&walk, orig_n); in ccm_calculate_auth_mac()
156 static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk, in ccm_crypt() argument
166 memcpy(ctr0, walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
167 crypto_inc(walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
173 while (walk->nbytes) { in ccm_crypt()
174 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in ccm_crypt()
[all …]
|
| H A D | sm4-ce-gcm-glue.c | 77 struct scatter_walk walk; in gcm_calculate_auth_mac() local
80 scatterwalk_start(&walk, req->src); in gcm_calculate_auth_mac()
86 orig_n = scatterwalk_next(&walk, assoclen); in gcm_calculate_auth_mac()
87 p = walk.addr; in gcm_calculate_auth_mac()
119 scatterwalk_done_src(&walk, orig_n); in gcm_calculate_auth_mac()
130 static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk, in gcm_crypt() argument
145 lengths.b = cpu_to_be64(walk->total * 8); in gcm_crypt()
155 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in gcm_crypt()
156 const u8 *src = walk->src.virt.addr; in gcm_crypt()
157 u8 *dst = walk->dst.virt.addr; in gcm_crypt()
[all …]
|
| /linux/drivers/gpu/drm/xe/ |
| H A D | xe_pt_walk.c | 28 const struct xe_pt_walk *walk) in xe_pt_addr_end() argument
30 u64 size = 1ull << walk->shifts[level]; in xe_pt_addr_end()
37 unsigned int level, const struct xe_pt_walk *walk) in xe_pt_next() argument
42 if (unlikely(walk->shared_pt_mode)) { in xe_pt_next()
43 unsigned int shift = walk->shifts[level]; in xe_pt_next()
74 u64 addr, u64 end, struct xe_pt_walk *walk) in xe_pt_walk_range() argument
76 pgoff_t offset = xe_pt_offset(addr, level, walk); in xe_pt_walk_range()
77 struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) : in xe_pt_walk_range()
79 const struct xe_pt_walk_ops *ops = walk->ops; in xe_pt_walk_range()
86 next = xe_pt_addr_end(addr, end, level, walk); in xe_pt_walk_range()
[all …]
|
| H A D | xe_pt_walk.h | 67 struct xe_pt_walk *walk);
88 u64 addr, u64 end, struct xe_pt_walk *walk);
91 u64 addr, u64 end, struct xe_pt_walk *walk);
106 const struct xe_pt_walk *walk) in xe_pt_covers() argument
108 u64 pt_size = 1ull << walk->shifts[level]; in xe_pt_covers()
126 const struct xe_pt_walk *walk) in xe_pt_num_entries() argument
128 u64 pt_size = 1ull << walk->shifts[level]; in xe_pt_num_entries()
131 walk->shifts[level]; in xe_pt_num_entries()
144 xe_pt_offset(u64 addr, unsigned int level, const struct xe_pt_walk *walk) in xe_pt_offset() argument
146 if (level < walk->max_level) in xe_pt_offset()
[all …]
|
| /linux/crypto/ |
| H A D | scatterwalk.c | 18 void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) in scatterwalk_skip() argument
20 struct scatterlist *sg = walk->sg; in scatterwalk_skip()
22 nbytes += walk->offset - sg->offset; in scatterwalk_skip()
28 walk->sg = sg; in scatterwalk_skip()
29 walk->offset = sg->offset + nbytes; in scatterwalk_skip()
33 inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, in memcpy_from_scatterwalk() argument
39 to_copy = scatterwalk_next(walk, nbytes); in memcpy_from_scatterwalk()
40 memcpy(buf, walk->addr, to_copy); in memcpy_from_scatterwalk()
41 scatterwalk_done_src(walk, to_copy); in memcpy_from_scatterwalk()
48 inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, in memcpy_to_scatterwalk() argument
[all …]
|
| H A D | zstd.c | 104 struct acomp_walk walk; in zstd_compress() local
114 ret = acomp_walk_virt(&walk, req, true); in zstd_compress()
125 dcur = acomp_walk_next_dst(&walk); in zstd_compress()
132 outbuf.dst = (u8 *)walk.dst.virt.addr; in zstd_compress()
136 scur = acomp_walk_next_src(&walk); in zstd_compress()
138 ret = zstd_compress_one(req, ctx, walk.src.virt.addr, in zstd_compress()
139 walk.dst.virt.addr, &total_out); in zstd_compress()
140 acomp_walk_done_src(&walk, scur); in zstd_compress()
141 acomp_walk_done_dst(&walk, dcur); in zstd_compress()
147 inbuf.src = walk.src.virt.addr; in zstd_compress()
[all …]
|
| H A D | deflate.c | 66 struct acomp_walk walk; in deflate_compress_one() local
69 ret = acomp_walk_virt(&walk, req, true); in deflate_compress_one()
76 dcur = acomp_walk_next_dst(&walk); in deflate_compress_one()
81 stream->next_out = walk.dst.virt.addr; in deflate_compress_one()
90 scur = acomp_walk_next_src(&walk); in deflate_compress_one()
92 if (acomp_walk_more_src(&walk, scur)) in deflate_compress_one()
95 stream->next_in = walk.src.virt.addr; in deflate_compress_one()
102 acomp_walk_done_src(&walk, scur); in deflate_compress_one()
106 acomp_walk_done_dst(&walk, dcur); in deflate_compress_one()
146 struct acomp_walk walk; in deflate_decompress_one() local
[all …]
|
| H A D | chacha.c | 52 struct skcipher_walk walk; in chacha_stream_xor() local
56 err = skcipher_walk_virt(&walk, req, false); in chacha_stream_xor()
60 while (walk.nbytes > 0) { in chacha_stream_xor()
61 unsigned int nbytes = walk.nbytes; in chacha_stream_xor()
63 if (nbytes < walk.total) in chacha_stream_xor()
66 chacha_crypt(&state, walk.dst.virt.addr, walk.src.virt.addr, in chacha_stream_xor()
68 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); in chacha_stream_xor()
|
| /linux/arch/x86/crypto/ |
| H A D | ecb_cbc_helpers.h | 18 struct skcipher_walk walk; \
19 int err = skcipher_walk_virt(&walk, (req), false); \
20 while (walk.nbytes > 0) { \
21 unsigned int nbytes = walk.nbytes; \
24 const u8 *src = walk.src.virt.addr; \
25 u8 *dst = walk.dst.virt.addr; \
51 const u8 *__iv = walk.iv; \
58 memcpy(walk.iv, __iv, __bsize); \
72 crypto_xor(dst, walk.iv, __bsize); \
73 memcpy(walk.iv, __iv, __bsize); \
[all …]
|
| H A D | aria_aesni_avx2_glue.c | 91 struct skcipher_walk walk; in aria_avx2_ctr_encrypt() local
95 err = skcipher_walk_virt(&walk, req, false); in aria_avx2_ctr_encrypt()
97 while ((nbytes = walk.nbytes) > 0) { in aria_avx2_ctr_encrypt()
98 const u8 *src = walk.src.virt.addr; in aria_avx2_ctr_encrypt()
99 u8 *dst = walk.dst.virt.addr; in aria_avx2_ctr_encrypt()
105 walk.iv); in aria_avx2_ctr_encrypt()
116 walk.iv); in aria_avx2_ctr_encrypt()
124 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
125 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
137 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx2_ctr_encrypt()
[all …]
|
| H A D | aria_aesni_avx_glue.c | 89 struct skcipher_walk walk; in aria_avx_ctr_encrypt() local
93 err = skcipher_walk_virt(&walk, req, false); in aria_avx_ctr_encrypt()
95 while ((nbytes = walk.nbytes) > 0) { in aria_avx_ctr_encrypt()
96 const u8 *src = walk.src.virt.addr; in aria_avx_ctr_encrypt()
97 u8 *dst = walk.dst.virt.addr; in aria_avx_ctr_encrypt()
103 walk.iv); in aria_avx_ctr_encrypt()
111 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
112 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
124 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx_ctr_encrypt()
125 memcpy(&req_ctx->keystream[0], walk.iv, in aria_avx_ctr_encrypt()
[all …]
|
| /linux/drivers/crypto/ |
| H A D | padlock-aes.c | 348 struct skcipher_walk walk; in ecb_aes_encrypt() local
354 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_encrypt()
356 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_encrypt()
357 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_encrypt()
361 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_encrypt()
373 struct skcipher_walk walk; in ecb_aes_decrypt() local
379 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_decrypt()
381 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_decrypt()
382 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_decrypt()
386 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_decrypt()
[all …]
|
| /linux/drivers/media/mc/ |
| H A D | mc-entity.c | 477 media_pipeline_walk_top(struct media_pipeline_walk *walk) in media_pipeline_walk_top() argument
479 return &walk->stack.entries[walk->stack.top]; in media_pipeline_walk_top()
482 static bool media_pipeline_walk_empty(struct media_pipeline_walk *walk) in media_pipeline_walk_empty() argument
484 return walk->stack.top == -1; in media_pipeline_walk_empty()
488 static int media_pipeline_walk_resize(struct media_pipeline_walk *walk) in media_pipeline_walk_resize() argument
494 if (walk->stack.size >= 256) in media_pipeline_walk_resize()
497 new_size = walk->stack.size + MEDIA_PIPELINE_STACK_GROW_STEP; in media_pipeline_walk_resize()
499 entries = krealloc(walk->stack.entries, in media_pipeline_walk_resize()
500 new_size * sizeof(*walk->stack.entries), in media_pipeline_walk_resize()
505 walk->stack.entries = entries; in media_pipeline_walk_resize()
[all …]
|
| /linux/arch/s390/kvm/ |
| H A D | dat.c | 535 struct dat_walk *walk) in dat_crste_walk_range() argument
549 walk->last = table->crstes + idx; in dat_crste_walk_range()
550 crste = READ_ONCE(*walk->last); in dat_crste_walk_range()
553 if (!(walk->flags & DAT_WALK_IGN_HOLES)) in dat_crste_walk_range()
555 if (!(walk->flags & DAT_WALK_ANY)) in dat_crste_walk_range()
559 the_op = walk->ops->crste_ops[crste.h.tt]; in dat_crste_walk_range()
561 rc = the_op(walk->last, cur, next, walk); in dat_crste_walk_range()
562 crste = READ_ONCE(*walk->last); in dat_crste_walk_range()
569 _dereference_crste(crste), walk); in dat_crste_walk_range()
570 else if (walk->ops->pte_entry) in dat_crste_walk_range()
[all …]
|
| /linux/net/xfrm/ |
| H A D | xfrm_iptfs.c | 335 struct iptfs_skb_frag_walk *walk) in iptfs_skb_prepare_frag_walk() argument
341 walk->initial_offset = initial_offset; in iptfs_skb_prepare_frag_walk()
342 walk->fragi = 0; in iptfs_skb_prepare_frag_walk()
343 walk->past = 0; in iptfs_skb_prepare_frag_walk()
344 walk->total = 0; in iptfs_skb_prepare_frag_walk()
345 walk->nr_frags = 0; in iptfs_skb_prepare_frag_walk()
346 walk->pp_recycle = skb->pp_recycle; in iptfs_skb_prepare_frag_walk()
352 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
356 walk->total += frag->len; in iptfs_skb_prepare_frag_walk()
369 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
[all …]
|
| /linux/security/ |
| H A D | device_cgroup.c | 102 struct dev_exception_item *excopy, *walk; in dev_exception_add() local
110 list_for_each_entry(walk, &dev_cgroup->exceptions, list) { in dev_exception_add()
111 if (walk->type != ex->type) in dev_exception_add()
113 if (walk->major != ex->major) in dev_exception_add()
115 if (walk->minor != ex->minor) in dev_exception_add()
118 walk->access |= ex->access; in dev_exception_add()
134 struct dev_exception_item *walk, *tmp; in dev_exception_rm() local
138 list_for_each_entry_safe(walk, tmp, &dev_cgroup->exceptions, list) { in dev_exception_rm()
139 if (walk->type != ex->type) in dev_exception_rm()
141 if (walk->major != ex->major) in dev_exception_rm()
[all …]
|