1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cryptographic API. 4 * 5 * Support for OMAP AES GCM HW acceleration. 6 * 7 * Copyright (c) 2016 Texas Instruments Incorporated 8 */ 9 10 #include <crypto/aes.h> 11 #include <crypto/engine.h> 12 #include <crypto/gcm.h> 13 #include <crypto/internal/aead.h> 14 #include <crypto/scatterwalk.h> 15 #include <crypto/skcipher.h> 16 #include <linux/errno.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmaengine.h> 19 #include <linux/interrupt.h> 20 #include <linux/kernel.h> 21 #include <linux/omap-dma.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/scatterlist.h> 24 #include <linux/string.h> 25 26 #include "omap-crypto.h" 27 #include "omap-aes.h" 28 29 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 30 struct aead_request *req); 31 32 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) 33 { 34 struct aead_request *req = dd->aead_req; 35 36 dd->in_sg = NULL; 37 dd->out_sg = NULL; 38 39 crypto_finalize_aead_request(dd->engine, req, ret); 40 41 pm_runtime_mark_last_busy(dd->dev); 42 pm_runtime_put_autosuspend(dd->dev); 43 } 44 45 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) 46 { 47 u8 *tag; 48 int alen, clen, i, ret = 0, nsg; 49 struct omap_aes_reqctx *rctx; 50 51 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); 52 clen = ALIGN(dd->total, AES_BLOCK_SIZE); 53 rctx = aead_request_ctx(dd->aead_req); 54 55 nsg = !!(dd->assoc_len && dd->total); 56 57 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, 58 DMA_FROM_DEVICE); 59 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); 60 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); 61 omap_aes_crypt_dma_stop(dd); 62 63 omap_crypto_cleanup(dd->out_sg, dd->orig_out, 64 dd->aead_req->assoclen, dd->total, 65 FLAGS_OUT_DATA_ST_SHIFT, dd->flags); 66 67 if (dd->flags & FLAGS_ENCRYPT) 68 scatterwalk_map_and_copy(rctx->auth_tag, 69 dd->aead_req->dst, 70 dd->total + dd->aead_req->assoclen, 71 dd->authsize, 1); 72 73 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, 74 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); 75 76 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, 77 FLAGS_IN_DATA_ST_SHIFT, dd->flags); 78 79 if (!(dd->flags & FLAGS_ENCRYPT)) { 80 tag = (u8 *)rctx->auth_tag; 81 for (i = 0; i < dd->authsize; i++) { 82 if (tag[i]) { 83 ret = -EBADMSG; 84 } 85 } 86 } 87 88 omap_aes_gcm_finish_req(dd, ret); 89 } 90 91 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, 92 struct aead_request *req) 93 { 94 int alen, clen, cryptlen, assoclen, ret; 95 struct crypto_aead *aead = crypto_aead_reqtfm(req); 96 unsigned int authlen = crypto_aead_authsize(aead); 97 struct scatterlist *tmp, sg_arr[2]; 98 int nsg; 99 u16 flags; 100 101 assoclen = req->assoclen; 102 cryptlen = req->cryptlen; 103 104 if (dd->flags & FLAGS_RFC4106_GCM) 105 assoclen -= 8; 106 107 if (!(dd->flags & FLAGS_ENCRYPT)) 108 cryptlen -= authlen; 109 110 alen = ALIGN(assoclen, AES_BLOCK_SIZE); 111 clen = ALIGN(cryptlen, AES_BLOCK_SIZE); 112 113 nsg = !!(assoclen && cryptlen); 114 115 omap_aes_clear_copy_flags(dd); 116 117 sg_init_table(dd->in_sgl, nsg + 1); 118 if (assoclen) { 119 tmp = req->src; 120 ret = omap_crypto_align_sg(&tmp, assoclen, 121 AES_BLOCK_SIZE, dd->in_sgl, 122 OMAP_CRYPTO_COPY_DATA | 123 OMAP_CRYPTO_ZERO_BUF | 124 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 125 FLAGS_ASSOC_DATA_ST_SHIFT, 126 &dd->flags); 127 if (ret) 128 return ret; 129 } 130 131 if (cryptlen) { 132 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); 133 134 if (nsg) 135 sg_unmark_end(dd->in_sgl); 136 137 ret = omap_crypto_align_sg(&tmp, cryptlen, 138 AES_BLOCK_SIZE, &dd->in_sgl[nsg], 139 OMAP_CRYPTO_COPY_DATA | 140 OMAP_CRYPTO_ZERO_BUF | 141 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 142 FLAGS_IN_DATA_ST_SHIFT, 143 &dd->flags); 144 if (ret) 145 return ret; 146 } 147 148 dd->in_sg = dd->in_sgl; 149 dd->total = cryptlen; 150 dd->assoc_len = assoclen; 151 dd->authsize = authlen; 152 153 dd->out_sg = req->dst; 154 dd->orig_out = req->dst; 155 156 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen); 157 158 flags = 0; 159 if (req->src == req->dst || dd->out_sg == sg_arr) 160 flags |= OMAP_CRYPTO_FORCE_COPY; 161 162 if (cryptlen) { 163 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, 164 AES_BLOCK_SIZE, &dd->out_sgl, 165 flags, 166 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); 167 if (ret) 168 return ret; 169 } 170 171 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); 172 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); 173 174 return 0; 175 } 176 177 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) 178 { 179 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 180 181 aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv); 182 return 0; 183 } 184 185 void omap_aes_gcm_dma_out_callback(void *data) 186 { 187 struct omap_aes_dev *dd = data; 188 struct omap_aes_reqctx *rctx; 189 int i, val; 190 u32 *auth_tag, tag[4]; 191 192 if (!(dd->flags & FLAGS_ENCRYPT)) 193 scatterwalk_map_and_copy(tag, dd->aead_req->src, 194 dd->total + dd->aead_req->assoclen, 195 dd->authsize, 0); 196 197 rctx = aead_request_ctx(dd->aead_req); 198 auth_tag = (u32 *)rctx->auth_tag; 199 for (i = 0; i < 4; i++) { 200 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); 201 auth_tag[i] = val ^ auth_tag[i]; 202 if (!(dd->flags & FLAGS_ENCRYPT)) 203 auth_tag[i] = auth_tag[i] ^ tag[i]; 204 } 205 206 omap_aes_gcm_done_task(dd); 207 } 208 209 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 210 struct aead_request *req) 211 { 212 if (req) 213 return crypto_transfer_aead_request_to_engine(dd->engine, req); 214 215 return 0; 216 } 217 218 static int omap_aes_gcm_prepare_req(struct aead_request *req, 219 struct omap_aes_dev *dd) 220 { 221 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 222 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 223 int err; 224 225 dd->aead_req = req; 226 227 rctx->mode &= FLAGS_MODE_MASK; 228 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; 229 230 err = omap_aes_gcm_copy_buffers(dd, req); 231 if (err) 232 return err; 233 234 dd->ctx = &ctx->octx; 235 236 return omap_aes_write_ctrl(dd); 237 } 238 239 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) 240 { 241 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 242 struct crypto_aead *aead = crypto_aead_reqtfm(req); 243 unsigned int authlen = crypto_aead_authsize(aead); 244 struct omap_aes_dev *dd; 245 __be32 counter = cpu_to_be32(1); 246 int err, assoclen; 247 248 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); 249 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); 250 251 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); 252 if (err) 253 return err; 254 255 if (mode & FLAGS_RFC4106_GCM) 256 assoclen = req->assoclen - 8; 257 else 258 assoclen = req->assoclen; 259 if (assoclen + req->cryptlen == 0) { 260 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, 261 1); 262 return 0; 263 } 264 265 dd = omap_aes_find_dev(rctx); 266 if (!dd) 267 return -ENODEV; 268 rctx->mode = mode; 269 270 return omap_aes_gcm_handle_queue(dd, req); 271 } 272 273 int omap_aes_gcm_encrypt(struct aead_request *req) 274 { 275 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 276 277 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 278 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); 279 } 280 281 int omap_aes_gcm_decrypt(struct aead_request *req) 282 { 283 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 284 285 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 286 return omap_aes_gcm_crypt(req, FLAGS_GCM); 287 } 288 289 int omap_aes_4106gcm_encrypt(struct aead_request *req) 290 { 291 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 292 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 293 294 memcpy(rctx->iv, ctx->octx.nonce, 4); 295 memcpy(rctx->iv + 4, req->iv, 8); 296 return crypto_ipsec_check_assoclen(req->assoclen) ?: 297 omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | 298 FLAGS_RFC4106_GCM); 299 } 300 301 int omap_aes_4106gcm_decrypt(struct aead_request *req) 302 { 303 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 304 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 305 306 memcpy(rctx->iv, ctx->octx.nonce, 4); 307 memcpy(rctx->iv + 4, req->iv, 8); 308 return crypto_ipsec_check_assoclen(req->assoclen) ?: 309 omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); 310 } 311 312 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 313 unsigned int keylen) 314 { 315 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 316 int ret; 317 318 ret = aes_expandkey(&ctx->actx, key, keylen); 319 if (ret) 320 return ret; 321 322 memcpy(ctx->octx.key, key, keylen); 323 ctx->octx.keylen = keylen; 324 325 return 0; 326 } 327 328 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, 329 unsigned int keylen) 330 { 331 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 332 int ret; 333 334 if (keylen < 4) 335 return -EINVAL; 336 keylen -= 4; 337 338 ret = aes_expandkey(&ctx->actx, key, keylen); 339 if (ret) 340 return ret; 341 342 memcpy(ctx->octx.key, key, keylen); 343 memcpy(ctx->octx.nonce, key + keylen, 4); 344 ctx->octx.keylen = keylen; 345 346 return 0; 347 } 348 349 int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 350 { 351 return crypto_gcm_check_authsize(authsize); 352 } 353 354 int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, 355 unsigned int authsize) 356 { 357 return crypto_rfc4106_check_authsize(authsize); 358 } 359 360 int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) 361 { 362 struct aead_request *req = container_of(areq, struct aead_request, 363 base); 364 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 365 struct omap_aes_dev *dd = rctx->dd; 366 int ret; 367 368 if (!dd) 369 return -ENODEV; 370 371 ret = omap_aes_gcm_prepare_req(req, dd); 372 if (ret) 373 return ret; 374 375 if (dd->in_sg_len) 376 ret = omap_aes_crypt_dma_start(dd); 377 else 378 omap_aes_gcm_dma_out_callback(dd); 379 380 return ret; 381 } 382 383 int omap_aes_gcm_cra_init(struct crypto_aead *tfm) 384 { 385 crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx)); 386 387 return 0; 388 } 389