1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue code for accelerated AES-GCM stitched implementation for ppc64le. 4 * 5 * Copyright 2022- IBM Inc. All rights reserved 6 */ 7 8 #include <linux/unaligned.h> 9 #include <asm/simd.h> 10 #include <asm/switch_to.h> 11 #include <crypto/gcm.h> 12 #include <crypto/aes.h> 13 #include <crypto/algapi.h> 14 #include <crypto/b128ops.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/skcipher.h> 20 #include <crypto/scatterwalk.h> 21 #include <linux/cpufeature.h> 22 #include <linux/crypto.h> 23 #include <linux/module.h> 24 #include <linux/types.h> 25 26 #define PPC_ALIGN 16 27 #define GCM_IV_SIZE 12 28 #define RFC4106_NONCE_SIZE 4 29 30 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation"); 31 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com"); 32 MODULE_LICENSE("GPL v2"); 33 MODULE_ALIAS_CRYPTO("aes"); 34 35 asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, 36 void *key); 37 asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); 38 asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, 39 void *rkey, u8 *iv, void *Xi); 40 asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, 41 void *rkey, u8 *iv, void *Xi); 42 asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); 43 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, 44 unsigned char *aad, unsigned int alen); 45 asmlinkage void gcm_update(u8 *iv, void *Xi); 46 47 struct aes_key { 48 u8 key[AES_MAX_KEYLENGTH]; 49 u64 rounds; 50 }; 51 52 struct gcm_ctx { 53 u8 iv[16]; 54 u8 ivtag[16]; 55 u8 aad_hash[16]; 56 u64 aadLen; 57 u64 Plen; /* offset 56 - used in aes_p10_gcm_{en/de}crypt */ 58 u8 pblock[16]; 59 }; 60 struct Hash_ctx { 61 u8 H[16]; /* subkey */ 62 u8 Htable[256]; /* Xi, Hash table(offset 32) */ 63 }; 64 65 struct p10_aes_gcm_ctx { 66 struct aes_key enc_key; 67 u8 nonce[RFC4106_NONCE_SIZE]; 68 }; 69 70 static void vsx_begin(void) 71 { 72 preempt_disable(); 73 pagefault_disable(); 74 enable_kernel_vsx(); 75 } 76 77 static void vsx_end(void) 78 { 79 disable_kernel_vsx(); 80 pagefault_enable(); 81 preempt_enable(); 82 } 83 84 static void set_subkey(unsigned char *hash) 85 { 86 *(u64 *)&hash[0] = be64_to_cpup((__be64 *)&hash[0]); 87 *(u64 *)&hash[8] = be64_to_cpup((__be64 *)&hash[8]); 88 } 89 90 /* 91 * Compute aad if any. 92 * - Hash aad and copy to Xi. 93 */ 94 static void set_aad(struct gcm_ctx *gctx, struct Hash_ctx *hash, 95 unsigned char *aad, int alen) 96 { 97 int i; 98 u8 nXi[16] = {0, }; 99 100 gctx->aadLen = alen; 101 i = alen & ~0xf; 102 if (i) { 103 gcm_ghash_p10(nXi, hash->Htable+32, aad, i); 104 aad += i; 105 alen -= i; 106 } 107 if (alen) { 108 for (i = 0; i < alen; i++) 109 nXi[i] ^= aad[i]; 110 111 memset(gctx->aad_hash, 0, 16); 112 gcm_ghash_p10(gctx->aad_hash, hash->Htable+32, nXi, 16); 113 } else { 114 memcpy(gctx->aad_hash, nXi, 16); 115 } 116 117 memcpy(hash->Htable, gctx->aad_hash, 16); 118 } 119 120 static void gcmp10_init(struct gcm_ctx *gctx, u8 *iv, unsigned char *rdkey, 121 struct Hash_ctx *hash, u8 *assoc, unsigned int assoclen) 122 { 123 __be32 counter = cpu_to_be32(1); 124 125 aes_p10_encrypt(hash->H, hash->H, rdkey); 126 set_subkey(hash->H); 127 gcm_init_htable(hash->Htable+32, hash->H); 128 129 *((__be32 *)(iv+12)) = counter; 130 131 gctx->Plen = 0; 132 133 /* 134 * Encrypt counter vector as iv tag and increment counter. 135 */ 136 aes_p10_encrypt(iv, gctx->ivtag, rdkey); 137 138 counter = cpu_to_be32(2); 139 *((__be32 *)(iv+12)) = counter; 140 memcpy(gctx->iv, iv, 16); 141 142 gctx->aadLen = assoclen; 143 memset(gctx->aad_hash, 0, 16); 144 if (assoclen) 145 set_aad(gctx, hash, assoc, assoclen); 146 } 147 148 static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len) 149 { 150 int i; 151 unsigned char len_ac[16 + PPC_ALIGN]; 152 unsigned char *aclen = PTR_ALIGN((void *)len_ac, PPC_ALIGN); 153 __be64 clen = cpu_to_be64(len << 3); 154 __be64 alen = cpu_to_be64(gctx->aadLen << 3); 155 156 if (len == 0 && gctx->aadLen == 0) { 157 memcpy(hash->Htable, gctx->ivtag, 16); 158 return; 159 } 160 161 /* 162 * Len is in bits. 163 */ 164 *((__be64 *)(aclen)) = alen; 165 *((__be64 *)(aclen+8)) = clen; 166 167 /* 168 * hash (AAD len and len) 169 */ 170 gcm_ghash_p10(hash->Htable, hash->Htable+32, aclen, 16); 171 172 for (i = 0; i < 16; i++) 173 hash->Htable[i] ^= gctx->ivtag[i]; 174 } 175 176 static int set_authsize(struct crypto_aead *tfm, unsigned int authsize) 177 { 178 switch (authsize) { 179 case 4: 180 case 8: 181 case 12: 182 case 13: 183 case 14: 184 case 15: 185 case 16: 186 break; 187 default: 188 return -EINVAL; 189 } 190 191 return 0; 192 } 193 194 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 195 unsigned int keylen) 196 { 197 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 198 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 199 int ret; 200 201 vsx_begin(); 202 ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key); 203 vsx_end(); 204 205 return ret ? -EINVAL : 0; 206 } 207 208 static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, 209 int assoclen, int enc) 210 { 211 struct crypto_tfm *tfm = req->base.tfm; 212 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 213 u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN]; 214 struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN); 215 u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN]; 216 struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN); 217 struct scatter_walk assoc_sg_walk; 218 struct skcipher_walk walk; 219 u8 *assocmem = NULL; 220 u8 *assoc; 221 unsigned int cryptlen = req->cryptlen; 222 unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN]; 223 unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN); 224 int ret; 225 unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm)); 226 u8 otag[16]; 227 int total_processed = 0; 228 int nbytes; 229 230 memset(databuf, 0, sizeof(databuf)); 231 memset(hashbuf, 0, sizeof(hashbuf)); 232 memset(ivbuf, 0, sizeof(ivbuf)); 233 memcpy(iv, riv, GCM_IV_SIZE); 234 235 /* Linearize assoc, if not already linear */ 236 if (req->src->length >= assoclen && req->src->length) { 237 scatterwalk_start(&assoc_sg_walk, req->src); 238 assoc = scatterwalk_map(&assoc_sg_walk); 239 } else { 240 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 241 GFP_KERNEL : GFP_ATOMIC; 242 243 /* assoc can be any length, so must be on heap */ 244 assocmem = kmalloc(assoclen, flags); 245 if (unlikely(!assocmem)) 246 return -ENOMEM; 247 assoc = assocmem; 248 249 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 250 } 251 252 vsx_begin(); 253 gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen); 254 vsx_end(); 255 256 if (!assocmem) 257 scatterwalk_unmap(assoc); 258 else 259 kfree(assocmem); 260 261 if (enc) 262 ret = skcipher_walk_aead_encrypt(&walk, req, false); 263 else 264 ret = skcipher_walk_aead_decrypt(&walk, req, false); 265 if (ret) 266 return ret; 267 268 while ((nbytes = walk.nbytes) > 0 && ret == 0) { 269 u8 *src = walk.src.virt.addr; 270 u8 *dst = walk.dst.virt.addr; 271 u8 buf[AES_BLOCK_SIZE]; 272 273 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 274 src = dst = memcpy(buf, src, nbytes); 275 276 vsx_begin(); 277 if (enc) 278 aes_p10_gcm_encrypt(src, dst, nbytes, 279 &ctx->enc_key, gctx->iv, hash->Htable); 280 else 281 aes_p10_gcm_decrypt(src, dst, nbytes, 282 &ctx->enc_key, gctx->iv, hash->Htable); 283 284 if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) 285 memcpy(walk.dst.virt.addr, buf, nbytes); 286 287 vsx_end(); 288 289 total_processed += walk.nbytes; 290 ret = skcipher_walk_done(&walk, 0); 291 } 292 293 if (ret) 294 return ret; 295 296 /* Finalize hash */ 297 vsx_begin(); 298 gcm_update(gctx->iv, hash->Htable); 299 finish_tag(gctx, hash, total_processed); 300 vsx_end(); 301 302 /* copy Xi to end of dst */ 303 if (enc) 304 scatterwalk_map_and_copy(hash->Htable, req->dst, req->assoclen + cryptlen, 305 auth_tag_len, 1); 306 else { 307 scatterwalk_map_and_copy(otag, req->src, 308 req->assoclen + cryptlen - auth_tag_len, 309 auth_tag_len, 0); 310 311 if (crypto_memneq(otag, hash->Htable, auth_tag_len)) { 312 memzero_explicit(hash->Htable, 16); 313 return -EBADMSG; 314 } 315 } 316 317 return 0; 318 } 319 320 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 321 unsigned int keylen) 322 { 323 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 324 int err; 325 326 keylen -= RFC4106_NONCE_SIZE; 327 err = p10_aes_gcm_setkey(tfm, inkey, keylen); 328 if (err) 329 return err; 330 331 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 332 return 0; 333 } 334 335 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 336 { 337 return crypto_rfc4106_check_authsize(authsize); 338 } 339 340 static int rfc4106_encrypt(struct aead_request *req) 341 { 342 struct crypto_aead *aead = crypto_aead_reqtfm(req); 343 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 344 u8 iv[AES_BLOCK_SIZE]; 345 346 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 347 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 348 349 return crypto_ipsec_check_assoclen(req->assoclen) ?: 350 p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 1); 351 } 352 353 static int rfc4106_decrypt(struct aead_request *req) 354 { 355 struct crypto_aead *aead = crypto_aead_reqtfm(req); 356 struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); 357 u8 iv[AES_BLOCK_SIZE]; 358 359 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 360 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 361 362 return crypto_ipsec_check_assoclen(req->assoclen) ?: 363 p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 0); 364 } 365 366 static int p10_aes_gcm_encrypt(struct aead_request *req) 367 { 368 return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 1); 369 } 370 371 static int p10_aes_gcm_decrypt(struct aead_request *req) 372 { 373 return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 0); 374 } 375 376 static struct aead_alg gcm_aes_algs[] = {{ 377 .ivsize = GCM_IV_SIZE, 378 .maxauthsize = 16, 379 380 .setauthsize = set_authsize, 381 .setkey = p10_aes_gcm_setkey, 382 .encrypt = p10_aes_gcm_encrypt, 383 .decrypt = p10_aes_gcm_decrypt, 384 385 .base.cra_name = "__gcm(aes)", 386 .base.cra_driver_name = "__aes_gcm_p10", 387 .base.cra_priority = 2100, 388 .base.cra_blocksize = 1, 389 .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx)+ 390 4 * sizeof(u64[2]), 391 .base.cra_module = THIS_MODULE, 392 .base.cra_flags = CRYPTO_ALG_INTERNAL, 393 }, { 394 .ivsize = GCM_RFC4106_IV_SIZE, 395 .maxauthsize = 16, 396 .setkey = rfc4106_setkey, 397 .setauthsize = rfc4106_setauthsize, 398 .encrypt = rfc4106_encrypt, 399 .decrypt = rfc4106_decrypt, 400 401 .base.cra_name = "__rfc4106(gcm(aes))", 402 .base.cra_driver_name = "__rfc4106_aes_gcm_p10", 403 .base.cra_priority = 2100, 404 .base.cra_blocksize = 1, 405 .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx) + 406 4 * sizeof(u64[2]), 407 .base.cra_module = THIS_MODULE, 408 .base.cra_flags = CRYPTO_ALG_INTERNAL, 409 }}; 410 411 static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)]; 412 413 static int __init p10_init(void) 414 { 415 int ret; 416 417 if (!cpu_has_feature(CPU_FTR_ARCH_31)) 418 return 0; 419 420 ret = simd_register_aeads_compat(gcm_aes_algs, 421 ARRAY_SIZE(gcm_aes_algs), 422 p10_simd_aeads); 423 if (ret) { 424 simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 425 p10_simd_aeads); 426 return ret; 427 } 428 return 0; 429 } 430 431 static void __exit p10_exit(void) 432 { 433 simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), 434 p10_simd_aeads); 435 } 436 437 module_init(p10_init); 438 module_exit(p10_exit); 439