1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AES CTR routines supporting VMX instructions on the Power 8 4 * 5 * Copyright (C) 2015 International Business Machines Inc. 6 * 7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> 8 */ 9 10 #include <asm/simd.h> 11 #include <asm/switch_to.h> 12 #include <crypto/aes.h> 13 #include <crypto/internal/simd.h> 14 #include <crypto/internal/skcipher.h> 15 #include <linux/err.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/uaccess.h> 19 20 #include "aesp8-ppc.h" 21 22 struct p8_aes_ctr_ctx { 23 struct crypto_skcipher *fallback; 24 struct aes_key enc_key; 25 }; 26 27 static int p8_aes_ctr_init(struct crypto_skcipher *tfm) 28 { 29 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 30 struct crypto_skcipher *fallback; 31 32 fallback = crypto_alloc_skcipher("ctr(aes)", 0, 33 CRYPTO_ALG_NEED_FALLBACK | 34 CRYPTO_ALG_ASYNC); 35 if (IS_ERR(fallback)) { 36 pr_err("Failed to allocate ctr(aes) fallback: %ld\n", 37 PTR_ERR(fallback)); 38 return PTR_ERR(fallback); 39 } 40 41 crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + 42 crypto_skcipher_reqsize(fallback)); 43 ctx->fallback = fallback; 44 return 0; 45 } 46 47 static void p8_aes_ctr_exit(struct crypto_skcipher *tfm) 48 { 49 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 50 51 crypto_free_skcipher(ctx->fallback); 52 } 53 54 static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, 55 unsigned int keylen) 56 { 57 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 58 int ret; 59 60 preempt_disable(); 61 pagefault_disable(); 62 enable_kernel_vsx(); 63 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); 64 disable_kernel_vsx(); 65 pagefault_enable(); 66 preempt_enable(); 67 68 ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); 69 70 return ret ? -EINVAL : 0; 71 } 72 73 static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, 74 struct skcipher_walk *walk) 75 { 76 const u8 *src = walk->src.virt.addr; 77 u8 *ctrblk = walk->iv; 78 u8 keystream[AES_BLOCK_SIZE]; 79 u8 *dst = walk->dst.virt.addr; 80 unsigned int nbytes = walk->nbytes; 81 82 preempt_disable(); 83 pagefault_disable(); 84 enable_kernel_vsx(); 85 aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key); 86 disable_kernel_vsx(); 87 pagefault_enable(); 88 preempt_enable(); 89 90 crypto_xor_cpy(dst, keystream, src, nbytes); 91 crypto_inc(ctrblk, AES_BLOCK_SIZE); 92 } 93 94 static int p8_aes_ctr_crypt(struct skcipher_request *req) 95 { 96 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 97 const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 98 struct skcipher_walk walk; 99 unsigned int nbytes; 100 int ret; 101 102 if (!crypto_simd_usable()) { 103 struct skcipher_request *subreq = skcipher_request_ctx(req); 104 105 *subreq = *req; 106 skcipher_request_set_tfm(subreq, ctx->fallback); 107 return crypto_skcipher_encrypt(subreq); 108 } 109 110 ret = skcipher_walk_virt(&walk, req, false); 111 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 112 preempt_disable(); 113 pagefault_disable(); 114 enable_kernel_vsx(); 115 aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, 116 walk.dst.virt.addr, 117 nbytes / AES_BLOCK_SIZE, 118 &ctx->enc_key, walk.iv); 119 disable_kernel_vsx(); 120 pagefault_enable(); 121 preempt_enable(); 122 123 do { 124 crypto_inc(walk.iv, AES_BLOCK_SIZE); 125 } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE); 126 127 ret = skcipher_walk_done(&walk, nbytes); 128 } 129 if (nbytes) { 130 p8_aes_ctr_final(ctx, &walk); 131 ret = skcipher_walk_done(&walk, 0); 132 } 133 return ret; 134 } 135 136 struct skcipher_alg p8_aes_ctr_alg = { 137 .base.cra_name = "ctr(aes)", 138 .base.cra_driver_name = "p8_aes_ctr", 139 .base.cra_module = THIS_MODULE, 140 .base.cra_priority = 2000, 141 .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, 142 .base.cra_blocksize = 1, 143 .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), 144 .setkey = p8_aes_ctr_setkey, 145 .encrypt = p8_aes_ctr_crypt, 146 .decrypt = p8_aes_ctr_crypt, 147 .init = p8_aes_ctr_init, 148 .exit = p8_aes_ctr_exit, 149 .min_keysize = AES_MIN_KEY_SIZE, 150 .max_keysize = AES_MAX_KEY_SIZE, 151 .ivsize = AES_BLOCK_SIZE, 152 .chunksize = AES_BLOCK_SIZE, 153 }; 154