1*10930333SDanny Tsen // SPDX-License-Identifier: GPL-2.0-only 2*10930333SDanny Tsen /* 3*10930333SDanny Tsen * AES XTS routines supporting VMX In-core instructions on Power 8 4*10930333SDanny Tsen * 5*10930333SDanny Tsen * Copyright (C) 2015 International Business Machines Inc. 6*10930333SDanny Tsen * 7*10930333SDanny Tsen * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com> 8*10930333SDanny Tsen */ 9*10930333SDanny Tsen 10*10930333SDanny Tsen #include <asm/simd.h> 11*10930333SDanny Tsen #include <asm/switch_to.h> 12*10930333SDanny Tsen #include <crypto/aes.h> 13*10930333SDanny Tsen #include <crypto/internal/simd.h> 14*10930333SDanny Tsen #include <crypto/internal/skcipher.h> 15*10930333SDanny Tsen #include <crypto/xts.h> 16*10930333SDanny Tsen 17*10930333SDanny Tsen #include "aesp8-ppc.h" 18*10930333SDanny Tsen 19*10930333SDanny Tsen struct p8_aes_xts_ctx { 20*10930333SDanny Tsen struct crypto_skcipher *fallback; 21*10930333SDanny Tsen struct aes_key enc_key; 22*10930333SDanny Tsen struct aes_key dec_key; 23*10930333SDanny Tsen struct aes_key tweak_key; 24*10930333SDanny Tsen }; 25*10930333SDanny Tsen 26*10930333SDanny Tsen static int p8_aes_xts_init(struct crypto_skcipher *tfm) 27*10930333SDanny Tsen { 28*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 29*10930333SDanny Tsen struct crypto_skcipher *fallback; 30*10930333SDanny Tsen 31*10930333SDanny Tsen fallback = crypto_alloc_skcipher("xts(aes)", 0, 32*10930333SDanny Tsen CRYPTO_ALG_NEED_FALLBACK | 33*10930333SDanny Tsen CRYPTO_ALG_ASYNC); 34*10930333SDanny Tsen if (IS_ERR(fallback)) { 35*10930333SDanny Tsen pr_err("Failed to allocate xts(aes) fallback: %ld\n", 36*10930333SDanny Tsen PTR_ERR(fallback)); 37*10930333SDanny Tsen return PTR_ERR(fallback); 38*10930333SDanny Tsen } 39*10930333SDanny Tsen 40*10930333SDanny Tsen crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + 41*10930333SDanny Tsen crypto_skcipher_reqsize(fallback)); 42*10930333SDanny Tsen ctx->fallback = fallback; 43*10930333SDanny Tsen return 0; 44*10930333SDanny Tsen } 45*10930333SDanny Tsen 46*10930333SDanny Tsen static void p8_aes_xts_exit(struct crypto_skcipher *tfm) 47*10930333SDanny Tsen { 48*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 49*10930333SDanny Tsen 50*10930333SDanny Tsen crypto_free_skcipher(ctx->fallback); 51*10930333SDanny Tsen } 52*10930333SDanny Tsen 53*10930333SDanny Tsen static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 54*10930333SDanny Tsen unsigned int keylen) 55*10930333SDanny Tsen { 56*10930333SDanny Tsen struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 57*10930333SDanny Tsen int ret; 58*10930333SDanny Tsen 59*10930333SDanny Tsen ret = xts_verify_key(tfm, key, keylen); 60*10930333SDanny Tsen if (ret) 61*10930333SDanny Tsen return ret; 62*10930333SDanny Tsen 63*10930333SDanny Tsen preempt_disable(); 64*10930333SDanny Tsen pagefault_disable(); 65*10930333SDanny Tsen enable_kernel_vsx(); 66*10930333SDanny Tsen ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key); 67*10930333SDanny Tsen ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key); 68*10930333SDanny Tsen ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key); 69*10930333SDanny Tsen disable_kernel_vsx(); 70*10930333SDanny Tsen pagefault_enable(); 71*10930333SDanny Tsen preempt_enable(); 72*10930333SDanny Tsen 73*10930333SDanny Tsen ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); 74*10930333SDanny Tsen 75*10930333SDanny Tsen return ret ? -EINVAL : 0; 76*10930333SDanny Tsen } 77*10930333SDanny Tsen 78*10930333SDanny Tsen static int p8_aes_xts_crypt(struct skcipher_request *req, int enc) 79*10930333SDanny Tsen { 80*10930333SDanny Tsen struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 81*10930333SDanny Tsen const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 82*10930333SDanny Tsen struct skcipher_walk walk; 83*10930333SDanny Tsen unsigned int nbytes; 84*10930333SDanny Tsen u8 tweak[AES_BLOCK_SIZE]; 85*10930333SDanny Tsen int ret; 86*10930333SDanny Tsen 87*10930333SDanny Tsen if (req->cryptlen < AES_BLOCK_SIZE) 88*10930333SDanny Tsen return -EINVAL; 89*10930333SDanny Tsen 90*10930333SDanny Tsen if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) { 91*10930333SDanny Tsen struct skcipher_request *subreq = skcipher_request_ctx(req); 92*10930333SDanny Tsen 93*10930333SDanny Tsen *subreq = *req; 94*10930333SDanny Tsen skcipher_request_set_tfm(subreq, ctx->fallback); 95*10930333SDanny Tsen return enc ? crypto_skcipher_encrypt(subreq) : 96*10930333SDanny Tsen crypto_skcipher_decrypt(subreq); 97*10930333SDanny Tsen } 98*10930333SDanny Tsen 99*10930333SDanny Tsen ret = skcipher_walk_virt(&walk, req, false); 100*10930333SDanny Tsen if (ret) 101*10930333SDanny Tsen return ret; 102*10930333SDanny Tsen 103*10930333SDanny Tsen preempt_disable(); 104*10930333SDanny Tsen pagefault_disable(); 105*10930333SDanny Tsen enable_kernel_vsx(); 106*10930333SDanny Tsen 107*10930333SDanny Tsen aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key); 108*10930333SDanny Tsen 109*10930333SDanny Tsen disable_kernel_vsx(); 110*10930333SDanny Tsen pagefault_enable(); 111*10930333SDanny Tsen preempt_enable(); 112*10930333SDanny Tsen 113*10930333SDanny Tsen while ((nbytes = walk.nbytes) != 0) { 114*10930333SDanny Tsen preempt_disable(); 115*10930333SDanny Tsen pagefault_disable(); 116*10930333SDanny Tsen enable_kernel_vsx(); 117*10930333SDanny Tsen if (enc) 118*10930333SDanny Tsen aes_p8_xts_encrypt(walk.src.virt.addr, 119*10930333SDanny Tsen walk.dst.virt.addr, 120*10930333SDanny Tsen round_down(nbytes, AES_BLOCK_SIZE), 121*10930333SDanny Tsen &ctx->enc_key, NULL, tweak); 122*10930333SDanny Tsen else 123*10930333SDanny Tsen aes_p8_xts_decrypt(walk.src.virt.addr, 124*10930333SDanny Tsen walk.dst.virt.addr, 125*10930333SDanny Tsen round_down(nbytes, AES_BLOCK_SIZE), 126*10930333SDanny Tsen &ctx->dec_key, NULL, tweak); 127*10930333SDanny Tsen disable_kernel_vsx(); 128*10930333SDanny Tsen pagefault_enable(); 129*10930333SDanny Tsen preempt_enable(); 130*10930333SDanny Tsen 131*10930333SDanny Tsen ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); 132*10930333SDanny Tsen } 133*10930333SDanny Tsen return ret; 134*10930333SDanny Tsen } 135*10930333SDanny Tsen 136*10930333SDanny Tsen static int p8_aes_xts_encrypt(struct skcipher_request *req) 137*10930333SDanny Tsen { 138*10930333SDanny Tsen return p8_aes_xts_crypt(req, 1); 139*10930333SDanny Tsen } 140*10930333SDanny Tsen 141*10930333SDanny Tsen static int p8_aes_xts_decrypt(struct skcipher_request *req) 142*10930333SDanny Tsen { 143*10930333SDanny Tsen return p8_aes_xts_crypt(req, 0); 144*10930333SDanny Tsen } 145*10930333SDanny Tsen 146*10930333SDanny Tsen struct skcipher_alg p8_aes_xts_alg = { 147*10930333SDanny Tsen .base.cra_name = "xts(aes)", 148*10930333SDanny Tsen .base.cra_driver_name = "p8_aes_xts", 149*10930333SDanny Tsen .base.cra_module = THIS_MODULE, 150*10930333SDanny Tsen .base.cra_priority = 2000, 151*10930333SDanny Tsen .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, 152*10930333SDanny Tsen .base.cra_blocksize = AES_BLOCK_SIZE, 153*10930333SDanny Tsen .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx), 154*10930333SDanny Tsen .setkey = p8_aes_xts_setkey, 155*10930333SDanny Tsen .encrypt = p8_aes_xts_encrypt, 156*10930333SDanny Tsen .decrypt = p8_aes_xts_decrypt, 157*10930333SDanny Tsen .init = p8_aes_xts_init, 158*10930333SDanny Tsen .exit = p8_aes_xts_exit, 159*10930333SDanny Tsen .min_keysize = 2 * AES_MIN_KEY_SIZE, 160*10930333SDanny Tsen .max_keysize = 2 * AES_MAX_KEY_SIZE, 161*10930333SDanny Tsen .ivsize = AES_BLOCK_SIZE, 162*10930333SDanny Tsen }; 163