1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support 4 * 5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. 6 * 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * Author: Tom Lendacky <thomas.lendacky@amd.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/sched.h> 13 #include <linux/delay.h> 14 #include <linux/scatterlist.h> 15 #include <linux/string.h> 16 #include <crypto/aes.h> 17 #include <crypto/xts.h> 18 #include <crypto/internal/skcipher.h> 19 #include <crypto/scatterwalk.h> 20 21 #include "ccp-crypto.h" 22 23 struct ccp_aes_xts_def { 24 const char *name; 25 const char *drv_name; 26 }; 27 28 static const struct ccp_aes_xts_def aes_xts_algs[] = { 29 { 30 .name = "xts(aes)", 31 .drv_name = "xts-aes-ccp", 32 }, 33 }; 34 35 struct ccp_unit_size_map { 36 unsigned int size; 37 u32 value; 38 }; 39 40 static struct ccp_unit_size_map xts_unit_sizes[] = { 41 { 42 .size = 16, 43 .value = CCP_XTS_AES_UNIT_SIZE_16, 44 }, 45 { 46 .size = 512, 47 .value = CCP_XTS_AES_UNIT_SIZE_512, 48 }, 49 { 50 .size = 1024, 51 .value = CCP_XTS_AES_UNIT_SIZE_1024, 52 }, 53 { 54 .size = 2048, 55 .value = CCP_XTS_AES_UNIT_SIZE_2048, 56 }, 57 { 58 .size = 4096, 59 .value = CCP_XTS_AES_UNIT_SIZE_4096, 60 }, 61 }; 62 63 static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret) 64 { 65 struct skcipher_request *req = skcipher_request_cast(async_req); 66 struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); 67 68 if (ret) 69 return ret; 70 71 memcpy(req->iv, rctx->iv, AES_BLOCK_SIZE); 72 73 return 0; 74 } 75 76 static int ccp_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 77 unsigned int key_len) 78 { 79 struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 80 unsigned int ccpversion = ccp_version(); 81 int ret; 82 83 ret = xts_verify_key(tfm, key, key_len); 84 if (ret) 85 return ret; 86 87 /* Version 3 devices support 128-bit keys; version 5 devices can 88 * accommodate 128- and 256-bit keys. 89 */ 90 switch (key_len) { 91 case AES_KEYSIZE_128 * 2: 92 memcpy(ctx->u.aes.key, key, key_len); 93 break; 94 case AES_KEYSIZE_256 * 2: 95 if (ccpversion > CCP_VERSION(3, 0)) 96 memcpy(ctx->u.aes.key, key, key_len); 97 break; 98 } 99 ctx->u.aes.key_len = key_len / 2; 100 sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); 101 102 return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len); 103 } 104 105 static int ccp_aes_xts_crypt(struct skcipher_request *req, 106 unsigned int encrypt) 107 { 108 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 109 struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 110 struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); 111 unsigned int ccpversion = ccp_version(); 112 unsigned int fallback = 0; 113 unsigned int unit; 114 u32 unit_size; 115 int ret; 116 117 if (!ctx->u.aes.key_len) 118 return -EINVAL; 119 120 if (!req->iv) 121 return -EINVAL; 122 123 /* Check conditions under which the CCP can fulfill a request. The 124 * device can handle input plaintext of a length that is a multiple 125 * of the unit_size, bug the crypto implementation only supports 126 * the unit_size being equal to the input length. This limits the 127 * number of scenarios we can handle. 128 */ 129 unit_size = CCP_XTS_AES_UNIT_SIZE__LAST; 130 for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) { 131 if (req->cryptlen == xts_unit_sizes[unit].size) { 132 unit_size = unit; 133 break; 134 } 135 } 136 /* The CCP has restrictions on block sizes. Also, a version 3 device 137 * only supports AES-128 operations; version 5 CCPs support both 138 * AES-128 and -256 operations. 139 */ 140 if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) 141 fallback = 1; 142 if ((ccpversion < CCP_VERSION(5, 0)) && 143 (ctx->u.aes.key_len != AES_KEYSIZE_128)) 144 fallback = 1; 145 if ((ctx->u.aes.key_len != AES_KEYSIZE_128) && 146 (ctx->u.aes.key_len != AES_KEYSIZE_256)) 147 fallback = 1; 148 if (fallback) { 149 /* Use the fallback to process the request for any 150 * unsupported unit sizes or key sizes 151 */ 152 skcipher_request_set_tfm(&rctx->fallback_req, 153 ctx->u.aes.tfm_skcipher); 154 skcipher_request_set_callback(&rctx->fallback_req, 155 req->base.flags, 156 req->base.complete, 157 req->base.data); 158 skcipher_request_set_crypt(&rctx->fallback_req, req->src, 159 req->dst, req->cryptlen, req->iv); 160 ret = encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : 161 crypto_skcipher_decrypt(&rctx->fallback_req); 162 return ret; 163 } 164 165 memcpy(rctx->iv, req->iv, AES_BLOCK_SIZE); 166 sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE); 167 168 memset(&rctx->cmd, 0, sizeof(rctx->cmd)); 169 INIT_LIST_HEAD(&rctx->cmd.entry); 170 rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; 171 rctx->cmd.u.xts.type = CCP_AES_TYPE_128; 172 rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT 173 : CCP_AES_ACTION_DECRYPT; 174 rctx->cmd.u.xts.unit_size = unit_size; 175 rctx->cmd.u.xts.key = &ctx->u.aes.key_sg; 176 rctx->cmd.u.xts.key_len = ctx->u.aes.key_len; 177 rctx->cmd.u.xts.iv = &rctx->iv_sg; 178 rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE; 179 rctx->cmd.u.xts.src = req->src; 180 rctx->cmd.u.xts.src_len = req->cryptlen; 181 rctx->cmd.u.xts.dst = req->dst; 182 183 ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); 184 185 return ret; 186 } 187 188 static int ccp_aes_xts_encrypt(struct skcipher_request *req) 189 { 190 return ccp_aes_xts_crypt(req, 1); 191 } 192 193 static int ccp_aes_xts_decrypt(struct skcipher_request *req) 194 { 195 return ccp_aes_xts_crypt(req, 0); 196 } 197 198 static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm) 199 { 200 struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 201 struct crypto_skcipher *fallback_tfm; 202 203 ctx->complete = ccp_aes_xts_complete; 204 ctx->u.aes.key_len = 0; 205 206 fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0, 207 CRYPTO_ALG_NEED_FALLBACK); 208 if (IS_ERR(fallback_tfm)) { 209 pr_warn("could not load fallback driver xts(aes)\n"); 210 return PTR_ERR(fallback_tfm); 211 } 212 ctx->u.aes.tfm_skcipher = fallback_tfm; 213 214 crypto_skcipher_set_reqsize_dma(tfm, 215 sizeof(struct ccp_aes_req_ctx) + 216 crypto_skcipher_reqsize(fallback_tfm)); 217 218 return 0; 219 } 220 221 static void ccp_aes_xts_exit_tfm(struct crypto_skcipher *tfm) 222 { 223 struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 224 225 crypto_free_skcipher(ctx->u.aes.tfm_skcipher); 226 } 227 228 static int ccp_register_aes_xts_alg(struct list_head *head, 229 const struct ccp_aes_xts_def *def) 230 { 231 struct ccp_crypto_skcipher_alg *ccp_alg; 232 struct skcipher_alg *alg; 233 int ret; 234 235 ccp_alg = kzalloc_obj(*ccp_alg); 236 if (!ccp_alg) 237 return -ENOMEM; 238 239 INIT_LIST_HEAD(&ccp_alg->entry); 240 241 alg = &ccp_alg->alg; 242 243 strscpy(alg->base.cra_name, def->name); 244 strscpy(alg->base.cra_driver_name, def->drv_name); 245 alg->base.cra_flags = CRYPTO_ALG_ASYNC | 246 CRYPTO_ALG_ALLOCATES_MEMORY | 247 CRYPTO_ALG_KERN_DRIVER_ONLY | 248 CRYPTO_ALG_NEED_FALLBACK; 249 alg->base.cra_blocksize = AES_BLOCK_SIZE; 250 alg->base.cra_ctxsize = sizeof(struct ccp_ctx) + 251 crypto_dma_padding(); 252 alg->base.cra_priority = CCP_CRA_PRIORITY; 253 alg->base.cra_module = THIS_MODULE; 254 255 alg->setkey = ccp_aes_xts_setkey; 256 alg->encrypt = ccp_aes_xts_encrypt; 257 alg->decrypt = ccp_aes_xts_decrypt; 258 alg->min_keysize = AES_MIN_KEY_SIZE * 2; 259 alg->max_keysize = AES_MAX_KEY_SIZE * 2; 260 alg->ivsize = AES_BLOCK_SIZE; 261 alg->init = ccp_aes_xts_init_tfm; 262 alg->exit = ccp_aes_xts_exit_tfm; 263 264 ret = crypto_register_skcipher(alg); 265 if (ret) { 266 pr_err("%s skcipher algorithm registration error (%d)\n", 267 alg->base.cra_name, ret); 268 kfree(ccp_alg); 269 return ret; 270 } 271 272 list_add(&ccp_alg->entry, head); 273 274 return 0; 275 } 276 277 int ccp_register_aes_xts_algs(struct list_head *head) 278 { 279 int i, ret; 280 281 for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) { 282 ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]); 283 if (ret) 284 return ret; 285 } 286 287 return 0; 288 } 289