120a884f5SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2f848dbd3SHendrik Brueckner /*
3f848dbd3SHendrik Brueckner * Crypto-API module for CRC-32 algorithms implemented with the
4f848dbd3SHendrik Brueckner * z/Architecture Vector Extension Facility.
5f848dbd3SHendrik Brueckner *
6f848dbd3SHendrik Brueckner * Copyright IBM Corp. 2015
7f848dbd3SHendrik Brueckner * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8f848dbd3SHendrik Brueckner */
9f848dbd3SHendrik Brueckner #define KMSG_COMPONENT "crc32-vx"
10f848dbd3SHendrik Brueckner #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11f848dbd3SHendrik Brueckner
12f848dbd3SHendrik Brueckner #include <linux/module.h>
13f848dbd3SHendrik Brueckner #include <linux/cpufeature.h>
14f848dbd3SHendrik Brueckner #include <linux/crc32.h>
15f848dbd3SHendrik Brueckner #include <crypto/internal/hash.h>
16fd2527f2SHeiko Carstens #include <asm/fpu.h>
17c59bf4deSHeiko Carstens #include "crc32-vx.h"
18f848dbd3SHendrik Brueckner
19f848dbd3SHendrik Brueckner #define CRC32_BLOCK_SIZE 1
20f848dbd3SHendrik Brueckner #define CRC32_DIGEST_SIZE 4
21f848dbd3SHendrik Brueckner
22f848dbd3SHendrik Brueckner #define VX_MIN_LEN 64
23f848dbd3SHendrik Brueckner #define VX_ALIGNMENT 16L
24f848dbd3SHendrik Brueckner #define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
25f848dbd3SHendrik Brueckner
26f848dbd3SHendrik Brueckner struct crc_ctx {
27f848dbd3SHendrik Brueckner u32 key;
28f848dbd3SHendrik Brueckner };
29f848dbd3SHendrik Brueckner
30f848dbd3SHendrik Brueckner struct crc_desc_ctx {
31f848dbd3SHendrik Brueckner u32 crc;
32f848dbd3SHendrik Brueckner };
33f848dbd3SHendrik Brueckner
34f848dbd3SHendrik Brueckner /*
35f848dbd3SHendrik Brueckner * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
36f848dbd3SHendrik Brueckner *
37f848dbd3SHendrik Brueckner * Creates a function to perform a particular CRC-32 computation. Depending
38f848dbd3SHendrik Brueckner * on the message buffer, the hardware-accelerated or software implementation
39f848dbd3SHendrik Brueckner * is used. Note that the message buffer is aligned to improve fetch
40f848dbd3SHendrik Brueckner * operations of VECTOR LOAD MULTIPLE instructions.
41f848dbd3SHendrik Brueckner *
42f848dbd3SHendrik Brueckner */
43f848dbd3SHendrik Brueckner #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
44f848dbd3SHendrik Brueckner static u32 __pure ___fname(u32 crc, \
45f848dbd3SHendrik Brueckner unsigned char const *data, size_t datalen) \
46f848dbd3SHendrik Brueckner { \
47f848dbd3SHendrik Brueckner unsigned long prealign, aligned, remaining; \
48066c4091SHeiko Carstens DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
49f848dbd3SHendrik Brueckner \
50134a24cdSChristian Borntraeger if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
51134a24cdSChristian Borntraeger return ___crc32_sw(crc, data, datalen); \
52134a24cdSChristian Borntraeger \
53f848dbd3SHendrik Brueckner if ((unsigned long)data & VX_ALIGN_MASK) { \
54f848dbd3SHendrik Brueckner prealign = VX_ALIGNMENT - \
55f848dbd3SHendrik Brueckner ((unsigned long)data & VX_ALIGN_MASK); \
56f848dbd3SHendrik Brueckner datalen -= prealign; \
57f848dbd3SHendrik Brueckner crc = ___crc32_sw(crc, data, prealign); \
58f848dbd3SHendrik Brueckner data = (void *)((unsigned long)data + prealign); \
59f848dbd3SHendrik Brueckner } \
60f848dbd3SHendrik Brueckner \
61f848dbd3SHendrik Brueckner aligned = datalen & ~VX_ALIGN_MASK; \
62f848dbd3SHendrik Brueckner remaining = datalen & VX_ALIGN_MASK; \
63f848dbd3SHendrik Brueckner \
64f848dbd3SHendrik Brueckner kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
65f848dbd3SHendrik Brueckner crc = ___crc32_vx(crc, data, aligned); \
667f79695cSMartin Schwidefsky kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
67f848dbd3SHendrik Brueckner \
68f848dbd3SHendrik Brueckner if (remaining) \
69f848dbd3SHendrik Brueckner crc = ___crc32_sw(crc, data + aligned, remaining); \
70f848dbd3SHendrik Brueckner \
71f848dbd3SHendrik Brueckner return crc; \
72f848dbd3SHendrik Brueckner }
73f848dbd3SHendrik Brueckner
DEFINE_CRC32_VX(crc32_le_vx,crc32_le_vgfm_16,crc32_le)74f848dbd3SHendrik Brueckner DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
75f848dbd3SHendrik Brueckner DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
76f848dbd3SHendrik Brueckner DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
77f848dbd3SHendrik Brueckner
78f848dbd3SHendrik Brueckner
79f848dbd3SHendrik Brueckner static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
80f848dbd3SHendrik Brueckner {
81f848dbd3SHendrik Brueckner struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
82f848dbd3SHendrik Brueckner
83f848dbd3SHendrik Brueckner mctx->key = 0;
84f848dbd3SHendrik Brueckner return 0;
85f848dbd3SHendrik Brueckner }
86f848dbd3SHendrik Brueckner
crc32_vx_cra_init_invert(struct crypto_tfm * tfm)87f848dbd3SHendrik Brueckner static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
88f848dbd3SHendrik Brueckner {
89f848dbd3SHendrik Brueckner struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
90f848dbd3SHendrik Brueckner
91f848dbd3SHendrik Brueckner mctx->key = ~0;
92f848dbd3SHendrik Brueckner return 0;
93f848dbd3SHendrik Brueckner }
94f848dbd3SHendrik Brueckner
crc32_vx_init(struct shash_desc * desc)95f848dbd3SHendrik Brueckner static int crc32_vx_init(struct shash_desc *desc)
96f848dbd3SHendrik Brueckner {
97f848dbd3SHendrik Brueckner struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
98f848dbd3SHendrik Brueckner struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
99f848dbd3SHendrik Brueckner
100f848dbd3SHendrik Brueckner ctx->crc = mctx->key;
101f848dbd3SHendrik Brueckner return 0;
102f848dbd3SHendrik Brueckner }
103f848dbd3SHendrik Brueckner
crc32_vx_setkey(struct crypto_shash * tfm,const u8 * newkey,unsigned int newkeylen)104f848dbd3SHendrik Brueckner static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
105f848dbd3SHendrik Brueckner unsigned int newkeylen)
106f848dbd3SHendrik Brueckner {
107f848dbd3SHendrik Brueckner struct crc_ctx *mctx = crypto_shash_ctx(tfm);
108f848dbd3SHendrik Brueckner
109674f368aSEric Biggers if (newkeylen != sizeof(mctx->key))
110f848dbd3SHendrik Brueckner return -EINVAL;
111f848dbd3SHendrik Brueckner mctx->key = le32_to_cpu(*(__le32 *)newkey);
112f848dbd3SHendrik Brueckner return 0;
113f848dbd3SHendrik Brueckner }
114f848dbd3SHendrik Brueckner
crc32be_vx_setkey(struct crypto_shash * tfm,const u8 * newkey,unsigned int newkeylen)115f848dbd3SHendrik Brueckner static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
116f848dbd3SHendrik Brueckner unsigned int newkeylen)
117f848dbd3SHendrik Brueckner {
118f848dbd3SHendrik Brueckner struct crc_ctx *mctx = crypto_shash_ctx(tfm);
119f848dbd3SHendrik Brueckner
120674f368aSEric Biggers if (newkeylen != sizeof(mctx->key))
121f848dbd3SHendrik Brueckner return -EINVAL;
122f848dbd3SHendrik Brueckner mctx->key = be32_to_cpu(*(__be32 *)newkey);
123f848dbd3SHendrik Brueckner return 0;
124f848dbd3SHendrik Brueckner }
125f848dbd3SHendrik Brueckner
crc32le_vx_final(struct shash_desc * desc,u8 * out)126f848dbd3SHendrik Brueckner static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
127f848dbd3SHendrik Brueckner {
128f848dbd3SHendrik Brueckner struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
129f848dbd3SHendrik Brueckner
130f848dbd3SHendrik Brueckner *(__le32 *)out = cpu_to_le32p(&ctx->crc);
131f848dbd3SHendrik Brueckner return 0;
132f848dbd3SHendrik Brueckner }
133f848dbd3SHendrik Brueckner
crc32be_vx_final(struct shash_desc * desc,u8 * out)134f848dbd3SHendrik Brueckner static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
135f848dbd3SHendrik Brueckner {
136f848dbd3SHendrik Brueckner struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
137f848dbd3SHendrik Brueckner
138f848dbd3SHendrik Brueckner *(__be32 *)out = cpu_to_be32p(&ctx->crc);
139f848dbd3SHendrik Brueckner return 0;
140f848dbd3SHendrik Brueckner }
141f848dbd3SHendrik Brueckner
crc32c_vx_final(struct shash_desc * desc,u8 * out)142f848dbd3SHendrik Brueckner static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
143f848dbd3SHendrik Brueckner {
144f848dbd3SHendrik Brueckner struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
145f848dbd3SHendrik Brueckner
146f848dbd3SHendrik Brueckner /*
147f848dbd3SHendrik Brueckner * Perform a final XOR with 0xFFFFFFFF to be in sync
148f848dbd3SHendrik Brueckner * with the generic crc32c shash implementation.
149f848dbd3SHendrik Brueckner */
150f848dbd3SHendrik Brueckner *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
151f848dbd3SHendrik Brueckner return 0;
152f848dbd3SHendrik Brueckner }
153f848dbd3SHendrik Brueckner
__crc32le_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)154f848dbd3SHendrik Brueckner static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
155f848dbd3SHendrik Brueckner u8 *out)
156f848dbd3SHendrik Brueckner {
157f848dbd3SHendrik Brueckner *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
158f848dbd3SHendrik Brueckner return 0;
159f848dbd3SHendrik Brueckner }
160f848dbd3SHendrik Brueckner
__crc32be_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)161f848dbd3SHendrik Brueckner static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
162f848dbd3SHendrik Brueckner u8 *out)
163f848dbd3SHendrik Brueckner {
164f848dbd3SHendrik Brueckner *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
165f848dbd3SHendrik Brueckner return 0;
166f848dbd3SHendrik Brueckner }
167f848dbd3SHendrik Brueckner
__crc32c_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)168f848dbd3SHendrik Brueckner static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
169f848dbd3SHendrik Brueckner u8 *out)
170f848dbd3SHendrik Brueckner {
171f848dbd3SHendrik Brueckner /*
172f848dbd3SHendrik Brueckner * Perform a final XOR with 0xFFFFFFFF to be in sync
173f848dbd3SHendrik Brueckner * with the generic crc32c shash implementation.
174f848dbd3SHendrik Brueckner */
175f848dbd3SHendrik Brueckner *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
176f848dbd3SHendrik Brueckner return 0;
177f848dbd3SHendrik Brueckner }
178f848dbd3SHendrik Brueckner
179f848dbd3SHendrik Brueckner
180f848dbd3SHendrik Brueckner #define CRC32_VX_FINUP(alg, func) \
181f848dbd3SHendrik Brueckner static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
182f848dbd3SHendrik Brueckner unsigned int datalen, u8 *out) \
183f848dbd3SHendrik Brueckner { \
184f848dbd3SHendrik Brueckner return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
185f848dbd3SHendrik Brueckner data, datalen, out); \
186f848dbd3SHendrik Brueckner }
187f848dbd3SHendrik Brueckner
188f848dbd3SHendrik Brueckner CRC32_VX_FINUP(crc32le, crc32_le_vx)
189f848dbd3SHendrik Brueckner CRC32_VX_FINUP(crc32be, crc32_be_vx)
190f848dbd3SHendrik Brueckner CRC32_VX_FINUP(crc32c, crc32c_le_vx)
191f848dbd3SHendrik Brueckner
192f848dbd3SHendrik Brueckner #define CRC32_VX_DIGEST(alg, func) \
193f848dbd3SHendrik Brueckner static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
194f848dbd3SHendrik Brueckner unsigned int len, u8 *out) \
195f848dbd3SHendrik Brueckner { \
196f848dbd3SHendrik Brueckner return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
197f848dbd3SHendrik Brueckner data, len, out); \
198f848dbd3SHendrik Brueckner }
199f848dbd3SHendrik Brueckner
200f848dbd3SHendrik Brueckner CRC32_VX_DIGEST(crc32le, crc32_le_vx)
201f848dbd3SHendrik Brueckner CRC32_VX_DIGEST(crc32be, crc32_be_vx)
202f848dbd3SHendrik Brueckner CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
203f848dbd3SHendrik Brueckner
204f848dbd3SHendrik Brueckner #define CRC32_VX_UPDATE(alg, func) \
205f848dbd3SHendrik Brueckner static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
206f848dbd3SHendrik Brueckner unsigned int datalen) \
207f848dbd3SHendrik Brueckner { \
208f848dbd3SHendrik Brueckner struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
209f848dbd3SHendrik Brueckner ctx->crc = func(ctx->crc, data, datalen); \
210f848dbd3SHendrik Brueckner return 0; \
211f848dbd3SHendrik Brueckner }
212f848dbd3SHendrik Brueckner
213f848dbd3SHendrik Brueckner CRC32_VX_UPDATE(crc32le, crc32_le_vx)
214f848dbd3SHendrik Brueckner CRC32_VX_UPDATE(crc32be, crc32_be_vx)
215f848dbd3SHendrik Brueckner CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
216f848dbd3SHendrik Brueckner
217f848dbd3SHendrik Brueckner
218f848dbd3SHendrik Brueckner static struct shash_alg crc32_vx_algs[] = {
219f848dbd3SHendrik Brueckner /* CRC-32 LE */
220f848dbd3SHendrik Brueckner {
221f848dbd3SHendrik Brueckner .init = crc32_vx_init,
222f848dbd3SHendrik Brueckner .setkey = crc32_vx_setkey,
223f848dbd3SHendrik Brueckner .update = crc32le_vx_update,
224f848dbd3SHendrik Brueckner .final = crc32le_vx_final,
225f848dbd3SHendrik Brueckner .finup = crc32le_vx_finup,
226f848dbd3SHendrik Brueckner .digest = crc32le_vx_digest,
227f848dbd3SHendrik Brueckner .descsize = sizeof(struct crc_desc_ctx),
228f848dbd3SHendrik Brueckner .digestsize = CRC32_DIGEST_SIZE,
229f848dbd3SHendrik Brueckner .base = {
230f848dbd3SHendrik Brueckner .cra_name = "crc32",
231f848dbd3SHendrik Brueckner .cra_driver_name = "crc32-vx",
232f848dbd3SHendrik Brueckner .cra_priority = 200,
233a208fa8fSEric Biggers .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
234f848dbd3SHendrik Brueckner .cra_blocksize = CRC32_BLOCK_SIZE,
235f848dbd3SHendrik Brueckner .cra_ctxsize = sizeof(struct crc_ctx),
236f848dbd3SHendrik Brueckner .cra_module = THIS_MODULE,
237f848dbd3SHendrik Brueckner .cra_init = crc32_vx_cra_init_zero,
238f848dbd3SHendrik Brueckner },
239f848dbd3SHendrik Brueckner },
240f848dbd3SHendrik Brueckner /* CRC-32 BE */
241f848dbd3SHendrik Brueckner {
242f848dbd3SHendrik Brueckner .init = crc32_vx_init,
243f848dbd3SHendrik Brueckner .setkey = crc32be_vx_setkey,
244f848dbd3SHendrik Brueckner .update = crc32be_vx_update,
245f848dbd3SHendrik Brueckner .final = crc32be_vx_final,
246f848dbd3SHendrik Brueckner .finup = crc32be_vx_finup,
247f848dbd3SHendrik Brueckner .digest = crc32be_vx_digest,
248f848dbd3SHendrik Brueckner .descsize = sizeof(struct crc_desc_ctx),
249f848dbd3SHendrik Brueckner .digestsize = CRC32_DIGEST_SIZE,
250f848dbd3SHendrik Brueckner .base = {
251f848dbd3SHendrik Brueckner .cra_name = "crc32be",
252f848dbd3SHendrik Brueckner .cra_driver_name = "crc32be-vx",
253f848dbd3SHendrik Brueckner .cra_priority = 200,
254a208fa8fSEric Biggers .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
255f848dbd3SHendrik Brueckner .cra_blocksize = CRC32_BLOCK_SIZE,
256f848dbd3SHendrik Brueckner .cra_ctxsize = sizeof(struct crc_ctx),
257f848dbd3SHendrik Brueckner .cra_module = THIS_MODULE,
258f848dbd3SHendrik Brueckner .cra_init = crc32_vx_cra_init_zero,
259f848dbd3SHendrik Brueckner },
260f848dbd3SHendrik Brueckner },
261f848dbd3SHendrik Brueckner /* CRC-32C LE */
262f848dbd3SHendrik Brueckner {
263f848dbd3SHendrik Brueckner .init = crc32_vx_init,
264f848dbd3SHendrik Brueckner .setkey = crc32_vx_setkey,
265f848dbd3SHendrik Brueckner .update = crc32c_vx_update,
266f848dbd3SHendrik Brueckner .final = crc32c_vx_final,
267f848dbd3SHendrik Brueckner .finup = crc32c_vx_finup,
268f848dbd3SHendrik Brueckner .digest = crc32c_vx_digest,
269f848dbd3SHendrik Brueckner .descsize = sizeof(struct crc_desc_ctx),
270f848dbd3SHendrik Brueckner .digestsize = CRC32_DIGEST_SIZE,
271f848dbd3SHendrik Brueckner .base = {
272f848dbd3SHendrik Brueckner .cra_name = "crc32c",
273f848dbd3SHendrik Brueckner .cra_driver_name = "crc32c-vx",
274f848dbd3SHendrik Brueckner .cra_priority = 200,
275a208fa8fSEric Biggers .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
276f848dbd3SHendrik Brueckner .cra_blocksize = CRC32_BLOCK_SIZE,
277f848dbd3SHendrik Brueckner .cra_ctxsize = sizeof(struct crc_ctx),
278f848dbd3SHendrik Brueckner .cra_module = THIS_MODULE,
279f848dbd3SHendrik Brueckner .cra_init = crc32_vx_cra_init_invert,
280f848dbd3SHendrik Brueckner },
281f848dbd3SHendrik Brueckner },
282f848dbd3SHendrik Brueckner };
283f848dbd3SHendrik Brueckner
284f848dbd3SHendrik Brueckner
crc_vx_mod_init(void)285f848dbd3SHendrik Brueckner static int __init crc_vx_mod_init(void)
286f848dbd3SHendrik Brueckner {
287f848dbd3SHendrik Brueckner return crypto_register_shashes(crc32_vx_algs,
288f848dbd3SHendrik Brueckner ARRAY_SIZE(crc32_vx_algs));
289f848dbd3SHendrik Brueckner }
290f848dbd3SHendrik Brueckner
crc_vx_mod_exit(void)291f848dbd3SHendrik Brueckner static void __exit crc_vx_mod_exit(void)
292f848dbd3SHendrik Brueckner {
293f848dbd3SHendrik Brueckner crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
294f848dbd3SHendrik Brueckner }
295f848dbd3SHendrik Brueckner
2960a5f9b38SHeiko Carstens module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
297f848dbd3SHendrik Brueckner module_exit(crc_vx_mod_exit);
298f848dbd3SHendrik Brueckner
299f848dbd3SHendrik Brueckner MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
300*68d7bb54SJeff Johnson MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
301f848dbd3SHendrik Brueckner MODULE_LICENSE("GPL");
302f848dbd3SHendrik Brueckner
303f848dbd3SHendrik Brueckner MODULE_ALIAS_CRYPTO("crc32");
304f848dbd3SHendrik Brueckner MODULE_ALIAS_CRYPTO("crc32-vx");
305f848dbd3SHendrik Brueckner MODULE_ALIAS_CRYPTO("crc32c");
306f848dbd3SHendrik Brueckner MODULE_ALIAS_CRYPTO("crc32c-vx");
307