| /linux/include/crypto/ |
| H A D | scatterwalk.h | 29 static inline void scatterwalk_start(struct scatter_walk *walk, in scatterwalk_start() argument
32 walk->sg = sg; in scatterwalk_start()
33 walk->offset = sg->offset; in scatterwalk_start()
40 static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, in scatterwalk_start_at_pos() argument
48 walk->sg = sg; in scatterwalk_start_at_pos()
49 walk->offset = sg->offset + pos; in scatterwalk_start_at_pos()
52 static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, in scatterwalk_clamp() argument
58 if (walk->offset >= walk->sg->offset + walk->sg->length) in scatterwalk_clamp()
59 scatterwalk_start(walk, sg_next(walk->sg)); in scatterwalk_clamp()
60 len_this_sg = walk->sg->offset + walk->sg->length - walk->offset; in scatterwalk_clamp()
[all …]
|
| /linux/mm/ |
| H A D | pagewalk.c | 30 unsigned long end, struct mm_walk *walk) in walk_pte_range_inner() argument
32 const struct mm_walk_ops *ops = walk->ops; in walk_pte_range_inner()
40 walk); in walk_pte_range_inner()
44 set_pte_at(walk->mm, addr, pte, new_pte); in walk_pte_range_inner()
46 if (!WARN_ON_ONCE(walk->no_vma)) in walk_pte_range_inner()
47 update_mmu_cache(walk->vma, addr, pte); in walk_pte_range_inner()
49 err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); in walk_pte_range_inner()
62 struct mm_walk *walk) in walk_pte_range() argument
68 if (walk->no_vma) { in walk_pte_range()
73 * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk the in walk_pte_range()
97 walk_pmd_range(pud_t * pud,unsigned long addr,unsigned long end,struct mm_walk * walk) walk_pmd_range() argument
186 walk_pud_range(p4d_t * p4d,unsigned long addr,unsigned long end,struct mm_walk * walk) walk_pud_range() argument
251 walk_p4d_range(pgd_t * pgd,unsigned long addr,unsigned long end,struct mm_walk * walk) walk_p4d_range() argument
289 walk_pgd_range(unsigned long addr,unsigned long end,struct mm_walk * walk) walk_pgd_range() argument
339 walk_hugetlb_range(unsigned long addr,unsigned long end,struct mm_walk * walk) walk_hugetlb_range() argument
368 walk_hugetlb_range(unsigned long addr,unsigned long end,struct mm_walk * walk) walk_hugetlb_range() argument
382 walk_page_test(unsigned long start,unsigned long end,struct mm_walk * walk) walk_page_test() argument
408 __walk_page_range(unsigned long start,unsigned long end,struct mm_walk * walk) __walk_page_range() argument
481 struct mm_walk walk = { walk_page_range_mm_unsafe() local
651 struct mm_walk walk = { walk_kernel_page_table_range_lockless() local
688 struct mm_walk walk = { walk_page_range_debug() local
725 struct mm_walk walk = { walk_page_range_vma_unsafe() local
755 struct mm_walk walk = { walk_page_vma() local
806 struct mm_walk walk = { walk_page_mapping() local
[all...] |
| H A D | mapping_dirty_helpers.c | 11 * struct wp_walk - Private struct for pagetable walk callbacks
29 * @walk: pagetable walk callback argument
35 struct mm_walk *walk) in wp_pte() argument
37 struct wp_walk *wpwalk = walk->private; in wp_pte()
41 pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); in wp_pte()
44 ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent); in wp_pte()
81 * @walk: pagetable walk callback argument
90 unsigned long end, struct mm_walk *walk) in clean_record_pte() argument
127 wp_clean_pmd_entry(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk) wp_clean_pmd_entry() argument
149 wp_clean_pud_entry(pud_t * pud,unsigned long addr,unsigned long end,struct mm_walk * walk) wp_clean_pud_entry() argument
170 wp_clean_pre_vma(unsigned long start,unsigned long end,struct mm_walk * walk) wp_clean_pre_vma() argument
198 wp_clean_post_vma(struct mm_walk * walk) wp_clean_post_vma() argument
219 wp_clean_test_walk(unsigned long start,unsigned long end,struct mm_walk * walk) wp_clean_test_walk() argument
[all...] |
| H A D | ptdump.c | 17 static inline int note_kasan_page_table(struct mm_walk *walk, in note_kasan_page_table() argument
20 struct ptdump_state *st = walk->private; in note_kasan_page_table()
24 walk->action = ACTION_CONTINUE; in note_kasan_page_table()
31 unsigned long next, struct mm_walk *walk) in ptdump_pgd_entry() argument
33 struct ptdump_state *st = walk->private; in ptdump_pgd_entry()
39 return note_kasan_page_table(walk, addr); in ptdump_pgd_entry()
47 walk->action = ACTION_CONTINUE; in ptdump_pgd_entry()
54 unsigned long next, struct mm_walk *walk) in ptdump_p4d_entry() argument
56 struct ptdump_state *st = walk->private; in ptdump_p4d_entry()
62 return note_kasan_page_table(walk, addr); in ptdump_p4d_entry()
[all …]
|
| /linux/arch/arm64/crypto/ |
| H A D | aes-neonbs-glue.c | 92 struct skcipher_walk walk; in __ecb_crypt() local
95 err = skcipher_walk_virt(&walk, req, false); in __ecb_crypt()
97 while (walk.nbytes >= AES_BLOCK_SIZE) { in __ecb_crypt()
98 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; in __ecb_crypt()
100 if (walk.nbytes < walk.total) in __ecb_crypt()
102 walk.stride / AES_BLOCK_SIZE); in __ecb_crypt()
105 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, in __ecb_crypt()
107 err = skcipher_walk_done(&walk, in __ecb_crypt()
108 walk.nbytes - blocks * AES_BLOCK_SIZE); in __ecb_crypt()
154 struct skcipher_walk walk; in cbc_encrypt() local
[all …]
|
| H A D | sm4-neon-glue.c | 38 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
42 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
44 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
45 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
46 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
55 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); in sm4_ecb_do_crypt()
81 struct skcipher_walk walk; in sm4_cbc_encrypt() local
85 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_encrypt()
87 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_encrypt()
88 const u8 *iv = walk.iv; in sm4_cbc_encrypt()
[all …]
|
| H A D | sm4-ce-glue.c | 108 struct skcipher_walk walk; in sm4_ecb_do_crypt() local
112 err = skcipher_walk_virt(&walk, req, false); in sm4_ecb_do_crypt()
114 while ((nbytes = walk.nbytes) > 0) { in sm4_ecb_do_crypt()
115 const u8 *src = walk.src.virt.addr; in sm4_ecb_do_crypt()
116 u8 *dst = walk.dst.virt.addr; in sm4_ecb_do_crypt()
127 err = skcipher_walk_done(&walk, nbytes); in sm4_ecb_do_crypt()
152 struct skcipher_walk walk; in sm4_cbc_crypt() local
156 err = skcipher_walk_virt(&walk, req, false); in sm4_cbc_crypt()
160 while ((nbytes = walk.nbytes) > 0) { in sm4_cbc_crypt()
161 const u8 *src = walk.src.virt.addr; in sm4_cbc_crypt()
[all …]
|
| H A D | sm4-ce-ccm-glue.c | 96 struct scatter_walk walk; in ccm_calculate_auth_mac() local
111 scatterwalk_start(&walk, req->src); in ccm_calculate_auth_mac()
117 orig_n = scatterwalk_next(&walk, assoclen); in ccm_calculate_auth_mac()
118 p = walk.addr; in ccm_calculate_auth_mac()
151 scatterwalk_done_src(&walk, orig_n); in ccm_calculate_auth_mac()
156 static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk, in ccm_crypt() argument
166 memcpy(ctr0, walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
167 crypto_inc(walk->iv, SM4_BLOCK_SIZE); in ccm_crypt()
173 while (walk->nbytes) { in ccm_crypt()
174 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in ccm_crypt()
[all …]
|
| H A D | sm4-ce-gcm-glue.c | 77 struct scatter_walk walk; in gcm_calculate_auth_mac() local
80 scatterwalk_start(&walk, req->src); in gcm_calculate_auth_mac()
86 orig_n = scatterwalk_next(&walk, assoclen); in gcm_calculate_auth_mac()
87 p = walk.addr; in gcm_calculate_auth_mac()
119 scatterwalk_done_src(&walk, orig_n); in gcm_calculate_auth_mac()
130 static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk, in gcm_crypt() argument
145 lengths.b = cpu_to_be64(walk->total * 8); in gcm_crypt()
155 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE; in gcm_crypt()
156 const u8 *src = walk->src.virt.addr; in gcm_crypt()
157 u8 *dst = walk->dst.virt.addr; in gcm_crypt()
[all …]
|
| /linux/drivers/gpu/drm/xe/ |
| H A D | xe_pt_walk.c | 28 const struct xe_pt_walk *walk) in xe_pt_addr_end() argument
30 u64 size = 1ull << walk->shifts[level]; in xe_pt_addr_end()
37 unsigned int level, const struct xe_pt_walk *walk) in xe_pt_next() argument
42 if (unlikely(walk->shared_pt_mode)) { in xe_pt_next()
43 unsigned int shift = walk->shifts[level]; in xe_pt_next()
74 u64 addr, u64 end, struct xe_pt_walk *walk) in xe_pt_walk_range() argument
76 pgoff_t offset = xe_pt_offset(addr, level, walk); in xe_pt_walk_range()
77 struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) : in xe_pt_walk_range()
79 const struct xe_pt_walk_ops *ops = walk->ops; in xe_pt_walk_range()
86 next = xe_pt_addr_end(addr, end, level, walk); in xe_pt_walk_range()
[all …]
|
| /linux/crypto/ |
| H A D | scatterwalk.c | 18 void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) in scatterwalk_skip() argument
20 struct scatterlist *sg = walk->sg; in scatterwalk_skip()
22 nbytes += walk->offset - sg->offset; in scatterwalk_skip()
28 walk->sg = sg; in scatterwalk_skip()
29 walk->offset = sg->offset + nbytes; in scatterwalk_skip()
33 inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, in memcpy_from_scatterwalk() argument
39 to_copy = scatterwalk_next(walk, nbytes); in memcpy_from_scatterwalk()
40 memcpy(buf, walk->addr, to_copy); in memcpy_from_scatterwalk()
41 scatterwalk_done_src(walk, to_copy); in memcpy_from_scatterwalk()
48 inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, in memcpy_to_scatterwalk() argument
[all …]
|
| H A D | zstd.c | 104 struct acomp_walk walk; in zstd_compress() local
114 ret = acomp_walk_virt(&walk, req, true); in zstd_compress()
125 dcur = acomp_walk_next_dst(&walk); in zstd_compress()
132 outbuf.dst = (u8 *)walk.dst.virt.addr; in zstd_compress()
136 scur = acomp_walk_next_src(&walk); in zstd_compress()
138 ret = zstd_compress_one(req, ctx, walk.src.virt.addr, in zstd_compress()
139 walk.dst.virt.addr, &total_out); in zstd_compress()
140 acomp_walk_done_src(&walk, scur); in zstd_compress()
141 acomp_walk_done_dst(&walk, dcur); in zstd_compress()
147 inbuf.src = walk.src.virt.addr; in zstd_compress()
[all …]
|
| H A D | deflate.c | 66 struct acomp_walk walk; in deflate_compress_one() local
69 ret = acomp_walk_virt(&walk, req, true); in deflate_compress_one()
76 dcur = acomp_walk_next_dst(&walk); in deflate_compress_one()
81 stream->next_out = walk.dst.virt.addr; in deflate_compress_one()
90 scur = acomp_walk_next_src(&walk); in deflate_compress_one()
92 if (acomp_walk_more_src(&walk, scur)) in deflate_compress_one()
95 stream->next_in = walk.src.virt.addr; in deflate_compress_one()
102 acomp_walk_done_src(&walk, scur); in deflate_compress_one()
106 acomp_walk_done_dst(&walk, dcur); in deflate_compress_one()
146 struct acomp_walk walk; in deflate_decompress_one() local
[all …]
|
| H A D | chacha.c | 52 struct skcipher_walk walk; in chacha_stream_xor() local
56 err = skcipher_walk_virt(&walk, req, false); in chacha_stream_xor()
60 while (walk.nbytes > 0) { in chacha_stream_xor()
61 unsigned int nbytes = walk.nbytes; in chacha_stream_xor()
63 if (nbytes < walk.total) in chacha_stream_xor()
66 chacha_crypt(&state, walk.dst.virt.addr, walk.src.virt.addr, in chacha_stream_xor()
68 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); in chacha_stream_xor()
|
| /linux/arch/x86/crypto/ |
| H A D | ecb_cbc_helpers.h | 18 struct skcipher_walk walk; \
19 int err = skcipher_walk_virt(&walk, (req), false); \
20 while (walk.nbytes > 0) { \
21 unsigned int nbytes = walk.nbytes; \
24 const u8 *src = walk.src.virt.addr; \
25 u8 *dst = walk.dst.virt.addr; \
51 const u8 *__iv = walk.iv; \
58 memcpy(walk.iv, __iv, __bsize); \
72 crypto_xor(dst, walk.iv, __bsize); \
73 memcpy(walk.iv, __iv, __bsize); \
[all …]
|
| H A D | aria_aesni_avx2_glue.c | 91 struct skcipher_walk walk; in aria_avx2_ctr_encrypt() local
95 err = skcipher_walk_virt(&walk, req, false); in aria_avx2_ctr_encrypt()
97 while ((nbytes = walk.nbytes) > 0) { in aria_avx2_ctr_encrypt()
98 const u8 *src = walk.src.virt.addr; in aria_avx2_ctr_encrypt()
99 u8 *dst = walk.dst.virt.addr; in aria_avx2_ctr_encrypt()
105 walk.iv); in aria_avx2_ctr_encrypt()
116 walk.iv); in aria_avx2_ctr_encrypt()
124 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
125 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx2_ctr_encrypt()
137 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx2_ctr_encrypt()
[all …]
|
| H A D | aria_aesni_avx_glue.c | 89 struct skcipher_walk walk; in aria_avx_ctr_encrypt() local
93 err = skcipher_walk_virt(&walk, req, false); in aria_avx_ctr_encrypt()
95 while ((nbytes = walk.nbytes) > 0) { in aria_avx_ctr_encrypt()
96 const u8 *src = walk.src.virt.addr; in aria_avx_ctr_encrypt()
97 u8 *dst = walk.dst.virt.addr; in aria_avx_ctr_encrypt()
103 walk.iv); in aria_avx_ctr_encrypt()
111 memcpy(&req_ctx->keystream[0], walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
112 crypto_inc(walk.iv, ARIA_BLOCK_SIZE); in aria_avx_ctr_encrypt()
124 if (walk.nbytes == walk.total && nbytes > 0) { in aria_avx_ctr_encrypt()
125 memcpy(&req_ctx->keystream[0], walk.iv, in aria_avx_ctr_encrypt()
[all …]
|
| /linux/drivers/crypto/ |
| H A D | padlock-aes.c | 348 struct skcipher_walk walk; in ecb_aes_encrypt() local
354 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_encrypt()
356 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_encrypt()
357 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_encrypt()
361 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_encrypt()
373 struct skcipher_walk walk; in ecb_aes_decrypt() local
379 err = skcipher_walk_virt(&walk, req, false); in ecb_aes_decrypt()
381 while ((nbytes = walk.nbytes) != 0) { in ecb_aes_decrypt()
382 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, in ecb_aes_decrypt()
386 err = skcipher_walk_done(&walk, nbytes); in ecb_aes_decrypt()
[all …]
|
| H A D | geode-aes.c | 267 struct skcipher_walk walk; in geode_skcipher_crypt() local
282 err = skcipher_walk_virt(&walk, req, false); in geode_skcipher_crypt()
284 while ((nbytes = walk.nbytes) != 0) { in geode_skcipher_crypt()
285 geode_aes_crypt(tctx, walk.src.virt.addr, walk.dst.virt.addr, in geode_skcipher_crypt()
287 walk.iv, mode, dir); in geode_skcipher_crypt()
288 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); in geode_skcipher_crypt()
|
| /linux/arch/loongarch/mm/ |
| H A D | pageattr.c | 17 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) in set_pageattr_masks() argument
20 struct pageattr_masks *masks = walk->private; in set_pageattr_masks()
29 unsigned long next, struct mm_walk *walk) in pageattr_pgd_entry() argument
34 val = __pgd(set_pageattr_masks(pgd_val(val), walk)); in pageattr_pgd_entry()
42 unsigned long next, struct mm_walk *walk) in pageattr_p4d_entry() argument
47 val = __p4d(set_pageattr_masks(p4d_val(val), walk)); in pageattr_p4d_entry()
55 unsigned long next, struct mm_walk *walk) in pageattr_pud_entry() argument
60 val = __pud(set_pageattr_masks(pud_val(val), walk)); in pageattr_pud_entry()
68 unsigned long next, struct mm_walk *walk) in pageattr_pmd_entry() argument
73 val = __pmd(set_pageattr_masks(pmd_val(val), walk)); in pageattr_pmd_entry()
[all …]
|
| /linux/drivers/media/mc/ |
| H A D | mc-entity.c | 477 media_pipeline_walk_top(struct media_pipeline_walk *walk) in media_pipeline_walk_top() argument
479 return &walk->stack.entries[walk->stack.top]; in media_pipeline_walk_top()
482 static bool media_pipeline_walk_empty(struct media_pipeline_walk *walk) in media_pipeline_walk_empty() argument
484 return walk->stack.top == -1; in media_pipeline_walk_empty()
488 static int media_pipeline_walk_resize(struct media_pipeline_walk *walk) in media_pipeline_walk_resize() argument
494 if (walk->stack.size >= 256) in media_pipeline_walk_resize()
497 new_size = walk->stack.size + MEDIA_PIPELINE_STACK_GROW_STEP; in media_pipeline_walk_resize()
499 entries = krealloc(walk->stack.entries, in media_pipeline_walk_resize()
500 new_size * sizeof(*walk->stack.entries), in media_pipeline_walk_resize()
505 walk->stack.entries = entries; in media_pipeline_walk_resize()
[all …]
|
| /linux/arch/s390/kvm/ |
| H A D | dat.c | 535 struct dat_walk *walk) in dat_crste_walk_range() argument
549 walk->last = table->crstes + idx; in dat_crste_walk_range()
550 crste = READ_ONCE(*walk->last); in dat_crste_walk_range()
553 if (!(walk->flags & DAT_WALK_IGN_HOLES)) in dat_crste_walk_range()
555 if (!(walk->flags & DAT_WALK_ANY)) in dat_crste_walk_range()
559 the_op = walk->ops->crste_ops[crste.h.tt]; in dat_crste_walk_range()
561 rc = the_op(walk->last, cur, next, walk); in dat_crste_walk_range()
562 crste = READ_ONCE(*walk->last); in dat_crste_walk_range()
569 _dereference_crste(crste), walk); in dat_crste_walk_range()
570 else if (walk->ops->pte_entry) in dat_crste_walk_range()
[all …]
|
| /linux/security/ |
| H A D | device_cgroup.c | 102 struct dev_exception_item *excopy, *walk; in dev_exception_add() local
110 list_for_each_entry(walk, &dev_cgroup->exceptions, list) { in dev_exception_add()
111 if (walk->type != ex->type) in dev_exception_add()
113 if (walk->major != ex->major) in dev_exception_add()
115 if (walk->minor != ex->minor) in dev_exception_add()
118 walk->access |= ex->access; in dev_exception_add()
134 struct dev_exception_item *walk, *tmp; in dev_exception_rm() local
138 list_for_each_entry_safe(walk, tmp, &dev_cgroup->exceptions, list) { in dev_exception_rm()
139 if (walk->type != ex->type) in dev_exception_rm()
141 if (walk->major != ex->major) in dev_exception_rm()
[all …]
|
| /linux/net/xfrm/ |
| H A D | xfrm_iptfs.c | 335 struct iptfs_skb_frag_walk *walk) in iptfs_skb_prepare_frag_walk() argument
341 walk->initial_offset = initial_offset; in iptfs_skb_prepare_frag_walk()
342 walk->fragi = 0; in iptfs_skb_prepare_frag_walk()
343 walk->past = 0; in iptfs_skb_prepare_frag_walk()
344 walk->total = 0; in iptfs_skb_prepare_frag_walk()
345 walk->nr_frags = 0; in iptfs_skb_prepare_frag_walk()
346 walk->pp_recycle = skb->pp_recycle; in iptfs_skb_prepare_frag_walk()
352 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
356 walk->total += frag->len; in iptfs_skb_prepare_frag_walk()
369 frag = &walk->frags[walk->nr_frags++]; in iptfs_skb_prepare_frag_walk()
[all …]
|
| H A D | xfrm_policy.c | 367 if (unlikely(xp->walk.dead)) in xfrm_policy_timer()
439 INIT_LIST_HEAD(&policy->walk.all); in xfrm_policy_alloc()
466 BUG_ON(!policy->walk.dead); in xfrm_policy_destroy()
488 policy->walk.dead = 1; in xfrm_policy_kill()
905 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_policy_inexact_list_reinsert()
909 if (policy->walk.dead || !policy->bydst_reinsert) in xfrm_policy_inexact_list_reinsert()
1269 if (policy->walk.dead) in xfrm_policy_is_dead_or_sk()
1306 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1361 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1761 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush_secctx_check()
[all …]
|