1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * StarFive AES acceleration driver 4 * 5 * Copyright (c) 2022 StarFive Technology 6 */ 7 8 #include <crypto/engine.h> 9 #include <crypto/gcm.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/skcipher.h> 12 #include <crypto/scatterwalk.h> 13 #include "jh7110-cryp.h" 14 #include <linux/err.h> 15 #include <linux/iopoll.h> 16 #include <linux/kernel.h> 17 #include <linux/slab.h> 18 #include <linux/string.h> 19 20 #define STARFIVE_AES_REGS_OFFSET 0x100 21 #define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0) 22 #define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4) 23 #define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8) 24 #define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC) 25 #define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10) 26 #define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14) 27 #define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18) 28 #define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C) 29 #define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20) 30 #define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24) 31 #define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28) 32 #define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C) 33 #define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30) 34 #define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34) 35 #define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C) 36 #define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40) 37 #define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44) 38 #define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48) 39 #define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C) 40 #define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50) 41 #define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54) 42 #define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58) 43 #define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C) 44 45 #define FLG_MODE_MASK GENMASK(2, 0) 46 #define FLG_ENCRYPT BIT(4) 47 48 /* Misc */ 49 #define CCM_B0_ADATA 0x40 50 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 51 52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp) 53 { 54 u32 status; 55 56 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 57 !(status & STARFIVE_AES_BUSY), 10, 100000); 58 } 59 60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp) 61 { 62 u32 status; 63 64 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 65 (status & STARFIVE_AES_KEY_DONE), 10, 100000); 66 } 67 68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp) 69 { 70 u32 status; 71 72 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 73 (status & STARFIVE_AES_GCM_DONE), 10, 100000); 74 } 75 76 static inline int is_gcm(struct starfive_cryp_dev *cryp) 77 { 78 return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; 79 } 80 81 static inline bool is_encrypt(struct starfive_cryp_dev *cryp) 82 { 83 return cryp->flags & FLG_ENCRYPT; 84 } 85 86 static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode) 87 { 88 struct starfive_cryp_dev *cryp = ctx->cryp; 89 unsigned int value; 90 91 switch (hw_mode) { 92 case STARFIVE_AES_MODE_GCM: 93 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); 94 value |= STARFIVE_AES_GCM_START; 95 writel(value, cryp->base + STARFIVE_AES_CSR); 96 starfive_aes_wait_gcmdone(cryp); 97 break; 98 case STARFIVE_AES_MODE_CCM: 99 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); 100 value |= STARFIVE_AES_CCM_START; 101 writel(value, cryp->base + STARFIVE_AES_CSR); 102 break; 103 } 104 } 105 106 static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) 107 { 108 struct starfive_cryp_dev *cryp = ctx->cryp; 109 110 writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0); 111 writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1); 112 } 113 114 static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx) 115 { 116 struct starfive_cryp_dev *cryp = ctx->cryp; 117 118 writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0); 119 writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1); 120 } 121 122 static inline int starfive_aes_ccm_check_iv(const u8 *iv) 123 { 124 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 125 if (iv[0] < 1 || iv[0] > 7) 126 return -EINVAL; 127 128 return 0; 129 } 130 131 static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv) 132 { 133 struct starfive_cryp_dev *cryp = ctx->cryp; 134 135 writel(iv[0], cryp->base + STARFIVE_AES_IV0); 136 writel(iv[1], cryp->base + STARFIVE_AES_IV1); 137 writel(iv[2], cryp->base + STARFIVE_AES_IV2); 138 139 if (is_gcm(cryp)) { 140 if (starfive_aes_wait_gcmdone(cryp)) 141 return -ETIMEDOUT; 142 143 return 0; 144 } 145 146 writel(iv[3], cryp->base + STARFIVE_AES_IV3); 147 148 return 0; 149 } 150 151 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv) 152 { 153 iv[0] = readl(cryp->base + STARFIVE_AES_IV0); 154 iv[1] = readl(cryp->base + STARFIVE_AES_IV1); 155 iv[2] = readl(cryp->base + STARFIVE_AES_IV2); 156 iv[3] = readl(cryp->base + STARFIVE_AES_IV3); 157 } 158 159 static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce) 160 { 161 struct starfive_cryp_dev *cryp = ctx->cryp; 162 163 writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0); 164 writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1); 165 writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2); 166 writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3); 167 } 168 169 static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx) 170 { 171 struct starfive_cryp_dev *cryp = ctx->cryp; 172 u32 *key = (u32 *)ctx->key; 173 174 if (ctx->keylen >= AES_KEYSIZE_128) { 175 writel(key[0], cryp->base + STARFIVE_AES_KEY0); 176 writel(key[1], cryp->base + STARFIVE_AES_KEY1); 177 writel(key[2], cryp->base + STARFIVE_AES_KEY2); 178 writel(key[3], cryp->base + STARFIVE_AES_KEY3); 179 } 180 181 if (ctx->keylen >= AES_KEYSIZE_192) { 182 writel(key[4], cryp->base + STARFIVE_AES_KEY4); 183 writel(key[5], cryp->base + STARFIVE_AES_KEY5); 184 } 185 186 if (ctx->keylen >= AES_KEYSIZE_256) { 187 writel(key[6], cryp->base + STARFIVE_AES_KEY6); 188 writel(key[7], cryp->base + STARFIVE_AES_KEY7); 189 } 190 191 if (starfive_aes_wait_keydone(cryp)) 192 return -ETIMEDOUT; 193 194 return 0; 195 } 196 197 static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx) 198 { 199 struct starfive_cryp_dev *cryp = ctx->cryp; 200 u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; 201 unsigned int textlen; 202 203 memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE); 204 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 205 206 /* Build B0 */ 207 memcpy(b0, iv, AES_BLOCK_SIZE); 208 209 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 210 211 if (cryp->assoclen) 212 b0[0] |= CCM_B0_ADATA; 213 214 textlen = cryp->total_in; 215 216 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 217 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 218 219 starfive_aes_write_nonce(ctx, (u32 *)b0); 220 221 return 0; 222 } 223 224 static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) 225 { 226 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 227 struct starfive_cryp_dev *cryp = ctx->cryp; 228 u32 hw_mode; 229 230 /* reset */ 231 rctx->csr.aes.v = 0; 232 rctx->csr.aes.aesrst = 1; 233 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); 234 235 /* csr setup */ 236 hw_mode = cryp->flags & FLG_MODE_MASK; 237 238 rctx->csr.aes.v = 0; 239 240 switch (ctx->keylen) { 241 case AES_KEYSIZE_128: 242 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128; 243 break; 244 case AES_KEYSIZE_192: 245 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192; 246 break; 247 case AES_KEYSIZE_256: 248 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256; 249 break; 250 } 251 252 rctx->csr.aes.mode = hw_mode; 253 rctx->csr.aes.cmode = !is_encrypt(cryp); 254 rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; 255 256 if (cryp->side_chan) { 257 rctx->csr.aes.delay_aes = 1; 258 rctx->csr.aes.vaes_start = 1; 259 } 260 261 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); 262 263 cryp->err = starfive_aes_write_key(ctx); 264 if (cryp->err) 265 return cryp->err; 266 267 switch (hw_mode) { 268 case STARFIVE_AES_MODE_GCM: 269 starfive_aes_set_alen(ctx); 270 starfive_aes_set_mlen(ctx); 271 writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); 272 starfive_aes_aead_hw_start(ctx, hw_mode); 273 starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); 274 break; 275 case STARFIVE_AES_MODE_CCM: 276 starfive_aes_set_alen(ctx); 277 starfive_aes_set_mlen(ctx); 278 starfive_aes_ccm_init(ctx); 279 starfive_aes_aead_hw_start(ctx, hw_mode); 280 break; 281 case STARFIVE_AES_MODE_CBC: 282 case STARFIVE_AES_MODE_CTR: 283 starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); 284 break; 285 default: 286 break; 287 } 288 289 return cryp->err; 290 } 291 292 static int starfive_aes_read_authtag(struct starfive_cryp_ctx *ctx) 293 { 294 struct starfive_cryp_dev *cryp = ctx->cryp; 295 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 296 int i; 297 298 if (starfive_aes_wait_busy(cryp)) 299 return dev_err_probe(cryp->dev, -ETIMEDOUT, 300 "Timeout waiting for tag generation."); 301 302 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) { 303 cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0); 304 cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1); 305 cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2); 306 cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3); 307 } else { 308 for (i = 0; i < AES_BLOCK_32; i++) 309 cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); 310 } 311 312 if (is_encrypt(cryp)) { 313 scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg, 314 cryp->total_in, cryp->authsize, 1); 315 } else { 316 if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) 317 return -EBADMSG; 318 } 319 320 return 0; 321 } 322 323 static void starfive_aes_finish_req(struct starfive_cryp_ctx *ctx) 324 { 325 struct starfive_cryp_dev *cryp = ctx->cryp; 326 int err = cryp->err; 327 328 if (!err && cryp->authsize) 329 err = starfive_aes_read_authtag(ctx); 330 331 if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || 332 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) 333 starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); 334 335 if (cryp->authsize) 336 crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); 337 else 338 crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq, 339 err); 340 } 341 342 static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx) 343 { 344 struct starfive_cryp_dev *cryp = ctx->cryp; 345 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 346 u32 *buffer; 347 int total_len, loop; 348 349 total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int); 350 buffer = (u32 *)rctx->adata; 351 352 for (loop = 0; loop < total_len; loop += 4) { 353 writel(*buffer, cryp->base + STARFIVE_AES_NONCE0); 354 buffer++; 355 writel(*buffer, cryp->base + STARFIVE_AES_NONCE1); 356 buffer++; 357 writel(*buffer, cryp->base + STARFIVE_AES_NONCE2); 358 buffer++; 359 writel(*buffer, cryp->base + STARFIVE_AES_NONCE3); 360 buffer++; 361 } 362 363 if (starfive_aes_wait_gcmdone(cryp)) 364 return dev_err_probe(cryp->dev, -ETIMEDOUT, 365 "Timeout processing gcm aad block"); 366 367 return 0; 368 } 369 370 static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx) 371 { 372 struct starfive_cryp_dev *cryp = ctx->cryp; 373 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 374 u32 *buffer; 375 u8 *ci; 376 int total_len, loop; 377 378 total_len = cryp->assoclen; 379 380 ci = rctx->adata; 381 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); 382 ci++; 383 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); 384 ci++; 385 total_len -= 2; 386 buffer = (u32 *)ci; 387 388 for (loop = 0; loop < 3; loop++, buffer++) 389 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); 390 391 total_len -= 12; 392 393 while (total_len > 0) { 394 for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++) 395 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); 396 397 total_len -= AES_BLOCK_SIZE; 398 } 399 400 if (starfive_aes_wait_busy(cryp)) 401 return dev_err_probe(cryp->dev, -ETIMEDOUT, 402 "Timeout processing ccm aad block"); 403 404 return 0; 405 } 406 407 static void starfive_aes_dma_done(void *param) 408 { 409 struct starfive_cryp_dev *cryp = param; 410 411 complete(&cryp->dma_done); 412 } 413 414 static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp) 415 { 416 cryp->cfg_in.direction = DMA_MEM_TO_DEV; 417 cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; 418 cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 419 cryp->cfg_in.src_maxburst = cryp->dma_maxburst; 420 cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; 421 cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; 422 423 dmaengine_slave_config(cryp->tx, &cryp->cfg_in); 424 425 cryp->cfg_out.direction = DMA_DEV_TO_MEM; 426 cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 427 cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; 428 cryp->cfg_out.src_maxburst = 4; 429 cryp->cfg_out.dst_maxburst = 4; 430 cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; 431 432 dmaengine_slave_config(cryp->rx, &cryp->cfg_out); 433 434 init_completion(&cryp->dma_done); 435 } 436 437 static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp, 438 struct scatterlist *src, 439 struct scatterlist *dst, 440 int len) 441 { 442 struct dma_async_tx_descriptor *in_desc, *out_desc; 443 union starfive_alg_cr alg_cr; 444 int ret = 0, in_save, out_save; 445 446 alg_cr.v = 0; 447 alg_cr.start = 1; 448 alg_cr.aes_dma_en = 1; 449 writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); 450 451 in_save = sg_dma_len(src); 452 out_save = sg_dma_len(dst); 453 454 writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); 455 writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET); 456 457 sg_dma_len(src) = ALIGN(len, AES_BLOCK_SIZE); 458 sg_dma_len(dst) = ALIGN(len, AES_BLOCK_SIZE); 459 460 out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM, 461 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 462 if (!out_desc) { 463 ret = -EINVAL; 464 goto dma_err; 465 } 466 467 out_desc->callback = starfive_aes_dma_done; 468 out_desc->callback_param = cryp; 469 470 reinit_completion(&cryp->dma_done); 471 dmaengine_submit(out_desc); 472 dma_async_issue_pending(cryp->rx); 473 474 in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV, 475 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 476 if (!in_desc) { 477 ret = -EINVAL; 478 goto dma_err; 479 } 480 481 dmaengine_submit(in_desc); 482 dma_async_issue_pending(cryp->tx); 483 484 if (!wait_for_completion_timeout(&cryp->dma_done, 485 msecs_to_jiffies(1000))) 486 ret = -ETIMEDOUT; 487 488 dma_err: 489 sg_dma_len(src) = in_save; 490 sg_dma_len(dst) = out_save; 491 492 alg_cr.v = 0; 493 alg_cr.clear = 1; 494 writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); 495 496 return ret; 497 } 498 499 static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, 500 struct scatterlist *src, 501 struct scatterlist *dst) 502 { 503 struct scatterlist *stsg, *dtsg; 504 struct scatterlist _src[2], _dst[2]; 505 unsigned int remain = cryp->total_in; 506 unsigned int len, src_nents, dst_nents; 507 int ret; 508 509 if (src == dst) { 510 for (stsg = src, dtsg = dst; remain > 0; 511 stsg = sg_next(stsg), dtsg = sg_next(dtsg)) { 512 src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); 513 if (src_nents == 0) 514 return -ENOMEM; 515 516 dst_nents = src_nents; 517 len = min(sg_dma_len(stsg), remain); 518 519 ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); 520 dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); 521 if (ret) 522 return ret; 523 524 remain -= len; 525 } 526 } else { 527 for (stsg = src, dtsg = dst;;) { 528 src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); 529 if (src_nents == 0) 530 return -ENOMEM; 531 532 dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); 533 if (dst_nents == 0) 534 return -ENOMEM; 535 536 len = min(sg_dma_len(stsg), sg_dma_len(dtsg)); 537 len = min(len, remain); 538 539 ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); 540 dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); 541 dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); 542 if (ret) 543 return ret; 544 545 remain -= len; 546 if (remain == 0) 547 break; 548 549 if (sg_dma_len(stsg) - len) { 550 stsg = scatterwalk_ffwd(_src, stsg, len); 551 dtsg = sg_next(dtsg); 552 } else if (sg_dma_len(dtsg) - len) { 553 dtsg = scatterwalk_ffwd(_dst, dtsg, len); 554 stsg = sg_next(stsg); 555 } else { 556 stsg = sg_next(stsg); 557 dtsg = sg_next(dtsg); 558 } 559 } 560 } 561 562 return 0; 563 } 564 565 static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) 566 { 567 struct skcipher_request *req = 568 container_of(areq, struct skcipher_request, base); 569 struct starfive_cryp_ctx *ctx = 570 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 571 struct starfive_cryp_request_ctx *rctx = skcipher_request_ctx(req); 572 struct starfive_cryp_dev *cryp = ctx->cryp; 573 int ret; 574 575 cryp->req.sreq = req; 576 cryp->total_in = req->cryptlen; 577 cryp->total_out = req->cryptlen; 578 cryp->assoclen = 0; 579 cryp->authsize = 0; 580 581 rctx->in_sg = req->src; 582 rctx->out_sg = req->dst; 583 584 ctx->rctx = rctx; 585 586 ret = starfive_aes_hw_init(ctx); 587 if (ret) 588 return ret; 589 590 if (!cryp->total_in) 591 goto finish_req; 592 593 starfive_aes_dma_init(cryp); 594 595 ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); 596 if (ret) 597 return ret; 598 599 finish_req: 600 starfive_aes_finish_req(ctx); 601 602 return 0; 603 } 604 605 static int starfive_aes_init_tfm(struct crypto_skcipher *tfm, 606 const char *alg_name) 607 { 608 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 609 610 ctx->cryp = starfive_cryp_find_dev(ctx); 611 if (!ctx->cryp) 612 return -ENODEV; 613 614 ctx->skcipher_fbk = crypto_alloc_skcipher(alg_name, 0, 615 CRYPTO_ALG_NEED_FALLBACK); 616 if (IS_ERR(ctx->skcipher_fbk)) 617 return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk), 618 "%s() failed to allocate fallback for %s\n", 619 __func__, alg_name); 620 621 crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + 622 crypto_skcipher_reqsize(ctx->skcipher_fbk)); 623 624 return 0; 625 } 626 627 static void starfive_aes_exit_tfm(struct crypto_skcipher *tfm) 628 { 629 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 630 631 crypto_free_skcipher(ctx->skcipher_fbk); 632 } 633 634 static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) 635 { 636 struct aead_request *req = 637 container_of(areq, struct aead_request, base); 638 struct starfive_cryp_ctx *ctx = 639 crypto_aead_ctx(crypto_aead_reqtfm(req)); 640 struct starfive_cryp_dev *cryp = ctx->cryp; 641 struct starfive_cryp_request_ctx *rctx = aead_request_ctx(req); 642 struct scatterlist _src[2], _dst[2]; 643 int ret; 644 645 cryp->req.areq = req; 646 cryp->assoclen = req->assoclen; 647 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 648 649 rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen); 650 if (req->src == req->dst) 651 rctx->out_sg = rctx->in_sg; 652 else 653 rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen); 654 655 if (is_encrypt(cryp)) { 656 cryp->total_in = req->cryptlen; 657 cryp->total_out = req->cryptlen; 658 } else { 659 cryp->total_in = req->cryptlen - cryp->authsize; 660 cryp->total_out = cryp->total_in; 661 scatterwalk_map_and_copy(cryp->tag_in, req->src, 662 cryp->total_in + cryp->assoclen, 663 cryp->authsize, 0); 664 } 665 666 if (cryp->assoclen) { 667 rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); 668 if (!rctx->adata) 669 return -ENOMEM; 670 671 if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), 672 rctx->adata, cryp->assoclen) != cryp->assoclen) { 673 kfree(rctx->adata); 674 return -EINVAL; 675 } 676 } 677 678 if (cryp->total_in) 679 sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg), 680 sg_dma_len(rctx->in_sg) - cryp->total_in, 681 cryp->total_in); 682 683 ctx->rctx = rctx; 684 685 ret = starfive_aes_hw_init(ctx); 686 if (ret) { 687 if (cryp->assoclen) 688 kfree(rctx->adata); 689 return ret; 690 } 691 692 if (!cryp->assoclen) 693 goto write_text; 694 695 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) 696 ret = starfive_aes_ccm_write_adata(ctx); 697 else 698 ret = starfive_aes_gcm_write_adata(ctx); 699 700 kfree(rctx->adata); 701 702 if (ret) 703 return ret; 704 705 write_text: 706 if (!cryp->total_in) 707 goto finish_req; 708 709 starfive_aes_dma_init(cryp); 710 711 ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); 712 if (ret) 713 return ret; 714 715 finish_req: 716 starfive_aes_finish_req(ctx); 717 return 0; 718 } 719 720 static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm, 721 const char *alg_name) 722 { 723 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 724 725 ctx->cryp = starfive_cryp_find_dev(ctx); 726 if (!ctx->cryp) 727 return -ENODEV; 728 729 ctx->aead_fbk = crypto_alloc_aead(alg_name, 0, 730 CRYPTO_ALG_NEED_FALLBACK); 731 if (IS_ERR(ctx->aead_fbk)) 732 return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk), 733 "%s() failed to allocate fallback for %s\n", 734 __func__, alg_name); 735 736 crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + 737 crypto_aead_reqsize(ctx->aead_fbk)); 738 739 return 0; 740 } 741 742 static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm) 743 { 744 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 745 746 crypto_free_aead(ctx->aead_fbk); 747 } 748 749 static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp, 750 struct scatterlist *src, 751 struct scatterlist *dst) 752 { 753 struct scatterlist *tsg; 754 int i; 755 756 for_each_sg(src, tsg, sg_nents(src), i) 757 if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || 758 (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && 759 !sg_is_last(tsg))) 760 return true; 761 762 if (src != dst) 763 for_each_sg(dst, tsg, sg_nents(dst), i) 764 if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || 765 (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && 766 !sg_is_last(tsg))) 767 return true; 768 769 return false; 770 } 771 772 static int starfive_aes_do_fallback(struct skcipher_request *req, bool enc) 773 { 774 struct starfive_cryp_ctx *ctx = 775 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 776 struct skcipher_request *subreq = skcipher_request_ctx(req); 777 778 skcipher_request_set_tfm(subreq, ctx->skcipher_fbk); 779 skcipher_request_set_callback(subreq, req->base.flags, 780 req->base.complete, 781 req->base.data); 782 skcipher_request_set_crypt(subreq, req->src, req->dst, 783 req->cryptlen, req->iv); 784 785 return enc ? crypto_skcipher_encrypt(subreq) : 786 crypto_skcipher_decrypt(subreq); 787 } 788 789 static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) 790 { 791 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 792 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 793 struct starfive_cryp_dev *cryp = ctx->cryp; 794 unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1; 795 796 cryp->flags = flags; 797 798 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB || 799 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC) 800 if (req->cryptlen & blocksize_align) 801 return -EINVAL; 802 803 if (starfive_aes_check_unaligned(cryp, req->src, req->dst)) 804 return starfive_aes_do_fallback(req, is_encrypt(cryp)); 805 806 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); 807 } 808 809 static int starfive_aes_aead_do_fallback(struct aead_request *req, bool enc) 810 { 811 struct starfive_cryp_ctx *ctx = 812 crypto_aead_ctx(crypto_aead_reqtfm(req)); 813 struct aead_request *subreq = aead_request_ctx(req); 814 815 aead_request_set_tfm(subreq, ctx->aead_fbk); 816 aead_request_set_callback(subreq, req->base.flags, 817 req->base.complete, 818 req->base.data); 819 aead_request_set_crypt(subreq, req->src, req->dst, 820 req->cryptlen, req->iv); 821 aead_request_set_ad(subreq, req->assoclen); 822 823 return enc ? crypto_aead_encrypt(subreq) : 824 crypto_aead_decrypt(subreq); 825 } 826 827 static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags) 828 { 829 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 830 struct starfive_cryp_dev *cryp = ctx->cryp; 831 struct scatterlist *src, *dst, _src[2], _dst[2]; 832 833 cryp->flags = flags; 834 835 /* aes-ccm does not support tag verification for non-aligned text, 836 * use fallback for ccm decryption instead. 837 */ 838 if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) && 839 !is_encrypt(cryp)) 840 return starfive_aes_aead_do_fallback(req, 0); 841 842 src = scatterwalk_ffwd(_src, req->src, req->assoclen); 843 844 if (req->src == req->dst) 845 dst = src; 846 else 847 dst = scatterwalk_ffwd(_dst, req->dst, req->assoclen); 848 849 if (starfive_aes_check_unaligned(cryp, src, dst)) 850 return starfive_aes_aead_do_fallback(req, is_encrypt(cryp)); 851 852 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 853 } 854 855 static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 856 unsigned int keylen) 857 { 858 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 859 860 if (!key || !keylen) 861 return -EINVAL; 862 863 if (keylen != AES_KEYSIZE_128 && 864 keylen != AES_KEYSIZE_192 && 865 keylen != AES_KEYSIZE_256) 866 return -EINVAL; 867 868 memcpy(ctx->key, key, keylen); 869 ctx->keylen = keylen; 870 871 return crypto_skcipher_setkey(ctx->skcipher_fbk, key, keylen); 872 } 873 874 static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 875 unsigned int keylen) 876 { 877 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 878 879 if (!key || !keylen) 880 return -EINVAL; 881 882 if (keylen != AES_KEYSIZE_128 && 883 keylen != AES_KEYSIZE_192 && 884 keylen != AES_KEYSIZE_256) 885 return -EINVAL; 886 887 memcpy(ctx->key, key, keylen); 888 ctx->keylen = keylen; 889 890 return crypto_aead_setkey(ctx->aead_fbk, key, keylen); 891 } 892 893 static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm, 894 unsigned int authsize) 895 { 896 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 897 int ret; 898 899 ret = crypto_gcm_check_authsize(authsize); 900 if (ret) 901 return ret; 902 903 return crypto_aead_setauthsize(ctx->aead_fbk, authsize); 904 } 905 906 static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm, 907 unsigned int authsize) 908 { 909 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 910 911 switch (authsize) { 912 case 4: 913 case 6: 914 case 8: 915 case 10: 916 case 12: 917 case 14: 918 case 16: 919 break; 920 default: 921 return -EINVAL; 922 } 923 924 return crypto_aead_setauthsize(ctx->aead_fbk, authsize); 925 } 926 927 static int starfive_aes_ecb_encrypt(struct skcipher_request *req) 928 { 929 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT); 930 } 931 932 static int starfive_aes_ecb_decrypt(struct skcipher_request *req) 933 { 934 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB); 935 } 936 937 static int starfive_aes_cbc_encrypt(struct skcipher_request *req) 938 { 939 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT); 940 } 941 942 static int starfive_aes_cbc_decrypt(struct skcipher_request *req) 943 { 944 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC); 945 } 946 947 static int starfive_aes_ctr_encrypt(struct skcipher_request *req) 948 { 949 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT); 950 } 951 952 static int starfive_aes_ctr_decrypt(struct skcipher_request *req) 953 { 954 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR); 955 } 956 957 static int starfive_aes_gcm_encrypt(struct aead_request *req) 958 { 959 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT); 960 } 961 962 static int starfive_aes_gcm_decrypt(struct aead_request *req) 963 { 964 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM); 965 } 966 967 static int starfive_aes_ccm_encrypt(struct aead_request *req) 968 { 969 int ret; 970 971 ret = starfive_aes_ccm_check_iv(req->iv); 972 if (ret) 973 return ret; 974 975 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT); 976 } 977 978 static int starfive_aes_ccm_decrypt(struct aead_request *req) 979 { 980 int ret; 981 982 ret = starfive_aes_ccm_check_iv(req->iv); 983 if (ret) 984 return ret; 985 986 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM); 987 } 988 989 static int starfive_aes_ecb_init_tfm(struct crypto_skcipher *tfm) 990 { 991 return starfive_aes_init_tfm(tfm, "ecb(aes-lib)"); 992 } 993 994 static int starfive_aes_cbc_init_tfm(struct crypto_skcipher *tfm) 995 { 996 return starfive_aes_init_tfm(tfm, "cbc(aes-lib)"); 997 } 998 999 static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm) 1000 { 1001 return starfive_aes_init_tfm(tfm, "ctr(aes-lib)"); 1002 } 1003 1004 static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) 1005 { 1006 return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac(aes-lib))"); 1007 } 1008 1009 static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) 1010 { 1011 return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-generic)"); 1012 } 1013 1014 static struct skcipher_engine_alg skcipher_algs[] = { 1015 { 1016 .base.init = starfive_aes_ecb_init_tfm, 1017 .base.exit = starfive_aes_exit_tfm, 1018 .base.setkey = starfive_aes_setkey, 1019 .base.encrypt = starfive_aes_ecb_encrypt, 1020 .base.decrypt = starfive_aes_ecb_decrypt, 1021 .base.min_keysize = AES_MIN_KEY_SIZE, 1022 .base.max_keysize = AES_MAX_KEY_SIZE, 1023 .base.base = { 1024 .cra_name = "ecb(aes)", 1025 .cra_driver_name = "starfive-ecb-aes", 1026 .cra_priority = 200, 1027 .cra_flags = CRYPTO_ALG_ASYNC | 1028 CRYPTO_ALG_NEED_FALLBACK, 1029 .cra_blocksize = AES_BLOCK_SIZE, 1030 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 1031 .cra_alignmask = 0xf, 1032 .cra_module = THIS_MODULE, 1033 }, 1034 .op = { 1035 .do_one_request = starfive_aes_do_one_req, 1036 }, 1037 }, { 1038 .base.init = starfive_aes_cbc_init_tfm, 1039 .base.exit = starfive_aes_exit_tfm, 1040 .base.setkey = starfive_aes_setkey, 1041 .base.encrypt = starfive_aes_cbc_encrypt, 1042 .base.decrypt = starfive_aes_cbc_decrypt, 1043 .base.min_keysize = AES_MIN_KEY_SIZE, 1044 .base.max_keysize = AES_MAX_KEY_SIZE, 1045 .base.ivsize = AES_BLOCK_SIZE, 1046 .base.base = { 1047 .cra_name = "cbc(aes)", 1048 .cra_driver_name = "starfive-cbc-aes", 1049 .cra_priority = 200, 1050 .cra_flags = CRYPTO_ALG_ASYNC | 1051 CRYPTO_ALG_NEED_FALLBACK, 1052 .cra_blocksize = AES_BLOCK_SIZE, 1053 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 1054 .cra_alignmask = 0xf, 1055 .cra_module = THIS_MODULE, 1056 }, 1057 .op = { 1058 .do_one_request = starfive_aes_do_one_req, 1059 }, 1060 }, { 1061 .base.init = starfive_aes_ctr_init_tfm, 1062 .base.exit = starfive_aes_exit_tfm, 1063 .base.setkey = starfive_aes_setkey, 1064 .base.encrypt = starfive_aes_ctr_encrypt, 1065 .base.decrypt = starfive_aes_ctr_decrypt, 1066 .base.min_keysize = AES_MIN_KEY_SIZE, 1067 .base.max_keysize = AES_MAX_KEY_SIZE, 1068 .base.ivsize = AES_BLOCK_SIZE, 1069 .base.base = { 1070 .cra_name = "ctr(aes)", 1071 .cra_driver_name = "starfive-ctr-aes", 1072 .cra_priority = 200, 1073 .cra_flags = CRYPTO_ALG_ASYNC | 1074 CRYPTO_ALG_NEED_FALLBACK, 1075 .cra_blocksize = 1, 1076 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 1077 .cra_alignmask = 0xf, 1078 .cra_module = THIS_MODULE, 1079 }, 1080 .op = { 1081 .do_one_request = starfive_aes_do_one_req, 1082 }, 1083 }, 1084 }; 1085 1086 static struct aead_engine_alg aead_algs[] = { 1087 { 1088 .base.setkey = starfive_aes_aead_setkey, 1089 .base.setauthsize = starfive_aes_gcm_setauthsize, 1090 .base.encrypt = starfive_aes_gcm_encrypt, 1091 .base.decrypt = starfive_aes_gcm_decrypt, 1092 .base.init = starfive_aes_gcm_init_tfm, 1093 .base.exit = starfive_aes_aead_exit_tfm, 1094 .base.ivsize = GCM_AES_IV_SIZE, 1095 .base.maxauthsize = AES_BLOCK_SIZE, 1096 .base.base = { 1097 .cra_name = "gcm(aes)", 1098 .cra_driver_name = "starfive-gcm-aes", 1099 .cra_priority = 200, 1100 .cra_flags = CRYPTO_ALG_ASYNC | 1101 CRYPTO_ALG_NEED_FALLBACK, 1102 .cra_blocksize = 1, 1103 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 1104 .cra_alignmask = 0xf, 1105 .cra_module = THIS_MODULE, 1106 }, 1107 .op = { 1108 .do_one_request = starfive_aes_aead_do_one_req, 1109 }, 1110 }, { 1111 .base.setkey = starfive_aes_aead_setkey, 1112 .base.setauthsize = starfive_aes_ccm_setauthsize, 1113 .base.encrypt = starfive_aes_ccm_encrypt, 1114 .base.decrypt = starfive_aes_ccm_decrypt, 1115 .base.init = starfive_aes_ccm_init_tfm, 1116 .base.exit = starfive_aes_aead_exit_tfm, 1117 .base.ivsize = AES_BLOCK_SIZE, 1118 .base.maxauthsize = AES_BLOCK_SIZE, 1119 .base.base = { 1120 .cra_name = "ccm(aes)", 1121 .cra_driver_name = "starfive-ccm-aes", 1122 .cra_priority = 200, 1123 .cra_flags = CRYPTO_ALG_ASYNC | 1124 CRYPTO_ALG_NEED_FALLBACK, 1125 .cra_blocksize = 1, 1126 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 1127 .cra_alignmask = 0xf, 1128 .cra_module = THIS_MODULE, 1129 }, 1130 .op = { 1131 .do_one_request = starfive_aes_aead_do_one_req, 1132 }, 1133 }, 1134 }; 1135 1136 int starfive_aes_register_algs(void) 1137 { 1138 int ret; 1139 1140 ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 1141 if (ret) 1142 return ret; 1143 1144 ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 1145 if (ret) 1146 crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 1147 1148 return ret; 1149 } 1150 1151 void starfive_aes_unregister_algs(void) 1152 { 1153 crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 1154 crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 1155 } 1156