1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * StarFive AES acceleration driver 4 * 5 * Copyright (c) 2022 StarFive Technology 6 */ 7 8 #include <crypto/engine.h> 9 #include <crypto/gcm.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/skcipher.h> 12 #include <crypto/scatterwalk.h> 13 #include "jh7110-cryp.h" 14 #include <linux/err.h> 15 #include <linux/iopoll.h> 16 #include <linux/kernel.h> 17 #include <linux/slab.h> 18 #include <linux/string.h> 19 20 #define STARFIVE_AES_REGS_OFFSET 0x100 21 #define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0) 22 #define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4) 23 #define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8) 24 #define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC) 25 #define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10) 26 #define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14) 27 #define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18) 28 #define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C) 29 #define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20) 30 #define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24) 31 #define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28) 32 #define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C) 33 #define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30) 34 #define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34) 35 #define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C) 36 #define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40) 37 #define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44) 38 #define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48) 39 #define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C) 40 #define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50) 41 #define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54) 42 #define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58) 43 #define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C) 44 45 #define FLG_MODE_MASK GENMASK(2, 0) 46 #define FLG_ENCRYPT BIT(4) 47 48 /* Misc */ 49 #define CCM_B0_ADATA 0x40 50 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 51 52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp) 53 { 54 u32 status; 55 56 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 57 !(status & STARFIVE_AES_BUSY), 10, 100000); 58 } 59 60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp) 61 { 62 u32 status; 63 64 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 65 (status & STARFIVE_AES_KEY_DONE), 10, 100000); 66 } 67 68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp) 69 { 70 u32 status; 71 72 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, 73 (status & STARFIVE_AES_GCM_DONE), 10, 100000); 74 } 75 76 static inline int is_gcm(struct starfive_cryp_dev *cryp) 77 { 78 return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; 79 } 80 81 static inline int is_encrypt(struct starfive_cryp_dev *cryp) 82 { 83 return cryp->flags & FLG_ENCRYPT; 84 } 85 86 static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode) 87 { 88 struct starfive_cryp_dev *cryp = ctx->cryp; 89 unsigned int value; 90 91 switch (hw_mode) { 92 case STARFIVE_AES_MODE_GCM: 93 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); 94 value |= STARFIVE_AES_GCM_START; 95 writel(value, cryp->base + STARFIVE_AES_CSR); 96 starfive_aes_wait_gcmdone(cryp); 97 break; 98 case STARFIVE_AES_MODE_CCM: 99 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); 100 value |= STARFIVE_AES_CCM_START; 101 writel(value, cryp->base + STARFIVE_AES_CSR); 102 break; 103 } 104 } 105 106 static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx) 107 { 108 struct starfive_cryp_dev *cryp = ctx->cryp; 109 110 if (is_gcm(cryp)) 111 writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); 112 else 113 writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN); 114 } 115 116 static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) 117 { 118 struct starfive_cryp_dev *cryp = ctx->cryp; 119 120 writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0); 121 writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1); 122 } 123 124 static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx) 125 { 126 struct starfive_cryp_dev *cryp = ctx->cryp; 127 128 writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0); 129 writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1); 130 } 131 132 static inline int starfive_aes_ccm_check_iv(const u8 *iv) 133 { 134 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 135 if (iv[0] < 1 || iv[0] > 7) 136 return -EINVAL; 137 138 return 0; 139 } 140 141 static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv) 142 { 143 struct starfive_cryp_dev *cryp = ctx->cryp; 144 145 writel(iv[0], cryp->base + STARFIVE_AES_IV0); 146 writel(iv[1], cryp->base + STARFIVE_AES_IV1); 147 writel(iv[2], cryp->base + STARFIVE_AES_IV2); 148 149 if (is_gcm(cryp)) { 150 if (starfive_aes_wait_gcmdone(cryp)) 151 return -ETIMEDOUT; 152 153 return 0; 154 } 155 156 writel(iv[3], cryp->base + STARFIVE_AES_IV3); 157 158 return 0; 159 } 160 161 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv) 162 { 163 iv[0] = readl(cryp->base + STARFIVE_AES_IV0); 164 iv[1] = readl(cryp->base + STARFIVE_AES_IV1); 165 iv[2] = readl(cryp->base + STARFIVE_AES_IV2); 166 iv[3] = readl(cryp->base + STARFIVE_AES_IV3); 167 } 168 169 static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce) 170 { 171 struct starfive_cryp_dev *cryp = ctx->cryp; 172 173 writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0); 174 writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1); 175 writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2); 176 writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3); 177 } 178 179 static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx) 180 { 181 struct starfive_cryp_dev *cryp = ctx->cryp; 182 u32 *key = (u32 *)ctx->key; 183 184 if (ctx->keylen >= AES_KEYSIZE_128) { 185 writel(key[0], cryp->base + STARFIVE_AES_KEY0); 186 writel(key[1], cryp->base + STARFIVE_AES_KEY1); 187 writel(key[2], cryp->base + STARFIVE_AES_KEY2); 188 writel(key[3], cryp->base + STARFIVE_AES_KEY3); 189 } 190 191 if (ctx->keylen >= AES_KEYSIZE_192) { 192 writel(key[4], cryp->base + STARFIVE_AES_KEY4); 193 writel(key[5], cryp->base + STARFIVE_AES_KEY5); 194 } 195 196 if (ctx->keylen >= AES_KEYSIZE_256) { 197 writel(key[6], cryp->base + STARFIVE_AES_KEY6); 198 writel(key[7], cryp->base + STARFIVE_AES_KEY7); 199 } 200 201 if (starfive_aes_wait_keydone(cryp)) 202 return -ETIMEDOUT; 203 204 return 0; 205 } 206 207 static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx) 208 { 209 struct starfive_cryp_dev *cryp = ctx->cryp; 210 u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; 211 unsigned int textlen; 212 213 memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE); 214 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 215 216 /* Build B0 */ 217 memcpy(b0, iv, AES_BLOCK_SIZE); 218 219 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 220 221 if (cryp->assoclen) 222 b0[0] |= CCM_B0_ADATA; 223 224 textlen = cryp->total_in; 225 226 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 227 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 228 229 starfive_aes_write_nonce(ctx, (u32 *)b0); 230 231 return 0; 232 } 233 234 static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) 235 { 236 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 237 struct starfive_cryp_dev *cryp = ctx->cryp; 238 u32 hw_mode; 239 240 /* reset */ 241 rctx->csr.aes.v = 0; 242 rctx->csr.aes.aesrst = 1; 243 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); 244 245 /* csr setup */ 246 hw_mode = cryp->flags & FLG_MODE_MASK; 247 248 rctx->csr.aes.v = 0; 249 250 switch (ctx->keylen) { 251 case AES_KEYSIZE_128: 252 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128; 253 break; 254 case AES_KEYSIZE_192: 255 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192; 256 break; 257 case AES_KEYSIZE_256: 258 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256; 259 break; 260 } 261 262 rctx->csr.aes.mode = hw_mode; 263 rctx->csr.aes.cmode = !is_encrypt(cryp); 264 rctx->csr.aes.ie = 1; 265 rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; 266 267 if (cryp->side_chan) { 268 rctx->csr.aes.delay_aes = 1; 269 rctx->csr.aes.vaes_start = 1; 270 } 271 272 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); 273 274 cryp->err = starfive_aes_write_key(ctx); 275 if (cryp->err) 276 return cryp->err; 277 278 switch (hw_mode) { 279 case STARFIVE_AES_MODE_GCM: 280 starfive_aes_set_alen(ctx); 281 starfive_aes_set_mlen(ctx); 282 starfive_aes_set_ivlen(ctx); 283 starfive_aes_aead_hw_start(ctx, hw_mode); 284 starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); 285 break; 286 case STARFIVE_AES_MODE_CCM: 287 starfive_aes_set_alen(ctx); 288 starfive_aes_set_mlen(ctx); 289 starfive_aes_ccm_init(ctx); 290 starfive_aes_aead_hw_start(ctx, hw_mode); 291 break; 292 case STARFIVE_AES_MODE_CBC: 293 case STARFIVE_AES_MODE_CTR: 294 starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); 295 break; 296 default: 297 break; 298 } 299 300 return cryp->err; 301 } 302 303 static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp) 304 { 305 int i, start_addr; 306 307 if (starfive_aes_wait_busy(cryp)) 308 return dev_err_probe(cryp->dev, -ETIMEDOUT, 309 "Timeout waiting for tag generation."); 310 311 start_addr = STARFIVE_AES_NONCE0; 312 313 if (is_gcm(cryp)) 314 for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4) 315 cryp->tag_out[i] = readl(cryp->base + start_addr); 316 else 317 for (i = 0; i < AES_BLOCK_32; i++) 318 cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); 319 320 if (is_encrypt(cryp)) { 321 scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1); 322 } else { 323 scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0); 324 325 if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) 326 return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n"); 327 } 328 329 return 0; 330 } 331 332 static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp) 333 { 334 union starfive_aes_csr csr; 335 int err = cryp->err; 336 337 if (!err && cryp->authsize) 338 err = starfive_aes_read_authtag(cryp); 339 340 if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || 341 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) 342 starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); 343 344 /* reset irq flags*/ 345 csr.v = 0; 346 csr.aesrst = 1; 347 writel(csr.v, cryp->base + STARFIVE_AES_CSR); 348 349 if (cryp->authsize) 350 crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); 351 else 352 crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq, 353 err); 354 } 355 356 void starfive_aes_done_task(unsigned long param) 357 { 358 struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; 359 u32 block[AES_BLOCK_32]; 360 u32 stat; 361 int i; 362 363 for (i = 0; i < AES_BLOCK_32; i++) 364 block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); 365 366 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE, 367 cryp->total_out), 1); 368 369 cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out); 370 371 if (!cryp->total_out) { 372 starfive_aes_finish_req(cryp); 373 return; 374 } 375 376 memset(block, 0, AES_BLOCK_SIZE); 377 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, 378 cryp->total_in), 0); 379 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); 380 381 for (i = 0; i < AES_BLOCK_32; i++) 382 writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); 383 384 stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); 385 stat &= ~STARFIVE_IE_MASK_AES_DONE; 386 writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); 387 } 388 389 static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx) 390 { 391 struct starfive_cryp_dev *cryp = ctx->cryp; 392 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 393 u32 *buffer; 394 int total_len, loop; 395 396 total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int); 397 buffer = (u32 *)rctx->adata; 398 399 for (loop = 0; loop < total_len; loop += 4) { 400 writel(*buffer, cryp->base + STARFIVE_AES_NONCE0); 401 buffer++; 402 writel(*buffer, cryp->base + STARFIVE_AES_NONCE1); 403 buffer++; 404 writel(*buffer, cryp->base + STARFIVE_AES_NONCE2); 405 buffer++; 406 writel(*buffer, cryp->base + STARFIVE_AES_NONCE3); 407 buffer++; 408 } 409 410 if (starfive_aes_wait_gcmdone(cryp)) 411 return dev_err_probe(cryp->dev, -ETIMEDOUT, 412 "Timeout processing gcm aad block"); 413 414 return 0; 415 } 416 417 static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx) 418 { 419 struct starfive_cryp_dev *cryp = ctx->cryp; 420 struct starfive_cryp_request_ctx *rctx = ctx->rctx; 421 u32 *buffer; 422 u8 *ci; 423 int total_len, loop; 424 425 total_len = cryp->assoclen; 426 427 ci = rctx->adata; 428 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); 429 ci++; 430 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); 431 ci++; 432 total_len -= 2; 433 buffer = (u32 *)ci; 434 435 for (loop = 0; loop < 3; loop++, buffer++) 436 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); 437 438 total_len -= 12; 439 440 while (total_len > 0) { 441 for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++) 442 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); 443 444 total_len -= AES_BLOCK_SIZE; 445 } 446 447 if (starfive_aes_wait_busy(cryp)) 448 return dev_err_probe(cryp->dev, -ETIMEDOUT, 449 "Timeout processing ccm aad block"); 450 451 return 0; 452 } 453 454 static int starfive_aes_prepare_req(struct skcipher_request *req, 455 struct aead_request *areq) 456 { 457 struct starfive_cryp_ctx *ctx; 458 struct starfive_cryp_request_ctx *rctx; 459 struct starfive_cryp_dev *cryp; 460 461 if (!req && !areq) 462 return -EINVAL; 463 464 ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : 465 crypto_aead_ctx(crypto_aead_reqtfm(areq)); 466 467 cryp = ctx->cryp; 468 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); 469 470 if (req) { 471 cryp->req.sreq = req; 472 cryp->total_in = req->cryptlen; 473 cryp->total_out = req->cryptlen; 474 cryp->assoclen = 0; 475 cryp->authsize = 0; 476 } else { 477 cryp->req.areq = areq; 478 cryp->assoclen = areq->assoclen; 479 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); 480 if (is_encrypt(cryp)) { 481 cryp->total_in = areq->cryptlen; 482 cryp->total_out = areq->cryptlen; 483 } else { 484 cryp->total_in = areq->cryptlen - cryp->authsize; 485 cryp->total_out = cryp->total_in; 486 } 487 } 488 489 rctx->in_sg = req ? req->src : areq->src; 490 scatterwalk_start(&cryp->in_walk, rctx->in_sg); 491 492 rctx->out_sg = req ? req->dst : areq->dst; 493 scatterwalk_start(&cryp->out_walk, rctx->out_sg); 494 495 if (cryp->assoclen) { 496 rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); 497 if (!rctx->adata) 498 return dev_err_probe(cryp->dev, -ENOMEM, 499 "Failed to alloc memory for adata"); 500 501 scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0); 502 scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2); 503 } 504 505 ctx->rctx = rctx; 506 507 return starfive_aes_hw_init(ctx); 508 } 509 510 static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) 511 { 512 struct skcipher_request *req = 513 container_of(areq, struct skcipher_request, base); 514 struct starfive_cryp_ctx *ctx = 515 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 516 struct starfive_cryp_dev *cryp = ctx->cryp; 517 u32 block[AES_BLOCK_32]; 518 u32 stat; 519 int err; 520 int i; 521 522 err = starfive_aes_prepare_req(req, NULL); 523 if (err) 524 return err; 525 526 /* 527 * Write first plain/ciphertext block to start the module 528 * then let irq tasklet handle the rest of the data blocks. 529 */ 530 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, 531 cryp->total_in), 0); 532 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); 533 534 for (i = 0; i < AES_BLOCK_32; i++) 535 writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); 536 537 stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); 538 stat &= ~STARFIVE_IE_MASK_AES_DONE; 539 writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); 540 541 return 0; 542 } 543 544 static int starfive_aes_init_tfm(struct crypto_skcipher *tfm) 545 { 546 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 547 548 ctx->cryp = starfive_cryp_find_dev(ctx); 549 if (!ctx->cryp) 550 return -ENODEV; 551 552 crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + 553 sizeof(struct skcipher_request)); 554 555 return 0; 556 } 557 558 static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) 559 { 560 struct aead_request *req = 561 container_of(areq, struct aead_request, base); 562 struct starfive_cryp_ctx *ctx = 563 crypto_aead_ctx(crypto_aead_reqtfm(req)); 564 struct starfive_cryp_dev *cryp = ctx->cryp; 565 struct starfive_cryp_request_ctx *rctx; 566 u32 block[AES_BLOCK_32]; 567 u32 stat; 568 int err; 569 int i; 570 571 err = starfive_aes_prepare_req(NULL, req); 572 if (err) 573 return err; 574 575 rctx = ctx->rctx; 576 577 if (!cryp->assoclen) 578 goto write_text; 579 580 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) 581 cryp->err = starfive_aes_ccm_write_adata(ctx); 582 else 583 cryp->err = starfive_aes_gcm_write_adata(ctx); 584 585 kfree(rctx->adata); 586 587 if (cryp->err) 588 return cryp->err; 589 590 write_text: 591 if (!cryp->total_in) 592 goto finish_req; 593 594 /* 595 * Write first plain/ciphertext block to start the module 596 * then let irq tasklet handle the rest of the data blocks. 597 */ 598 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, 599 cryp->total_in), 0); 600 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); 601 602 for (i = 0; i < AES_BLOCK_32; i++) 603 writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); 604 605 stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); 606 stat &= ~STARFIVE_IE_MASK_AES_DONE; 607 writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); 608 609 return 0; 610 611 finish_req: 612 starfive_aes_finish_req(cryp); 613 return 0; 614 } 615 616 static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm) 617 { 618 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 619 struct starfive_cryp_dev *cryp = ctx->cryp; 620 struct crypto_tfm *aead = crypto_aead_tfm(tfm); 621 struct crypto_alg *alg = aead->__crt_alg; 622 623 ctx->cryp = starfive_cryp_find_dev(ctx); 624 if (!ctx->cryp) 625 return -ENODEV; 626 627 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 628 ctx->aead_fbk = crypto_alloc_aead(alg->cra_name, 0, 629 CRYPTO_ALG_NEED_FALLBACK); 630 if (IS_ERR(ctx->aead_fbk)) 631 return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk), 632 "%s() failed to allocate fallback for %s\n", 633 __func__, alg->cra_name); 634 } 635 636 crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_ctx) + 637 sizeof(struct aead_request)); 638 639 return 0; 640 } 641 642 static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm) 643 { 644 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 645 646 crypto_free_aead(ctx->aead_fbk); 647 } 648 649 static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) 650 { 651 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 652 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 653 struct starfive_cryp_dev *cryp = ctx->cryp; 654 unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1; 655 656 cryp->flags = flags; 657 658 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB || 659 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC) 660 if (req->cryptlen & blocksize_align) 661 return -EINVAL; 662 663 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); 664 } 665 666 static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags) 667 { 668 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 669 struct starfive_cryp_dev *cryp = ctx->cryp; 670 671 cryp->flags = flags; 672 673 /* 674 * HW engine could not perform CCM tag verification on 675 * non-blocksize aligned text, use fallback algo instead 676 */ 677 if (ctx->aead_fbk && !is_encrypt(cryp)) { 678 struct aead_request *subreq = aead_request_ctx(req); 679 680 aead_request_set_tfm(subreq, ctx->aead_fbk); 681 aead_request_set_callback(subreq, req->base.flags, 682 req->base.complete, req->base.data); 683 aead_request_set_crypt(subreq, req->src, 684 req->dst, req->cryptlen, req->iv); 685 aead_request_set_ad(subreq, req->assoclen); 686 687 return crypto_aead_decrypt(subreq); 688 } 689 690 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 691 } 692 693 static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 694 unsigned int keylen) 695 { 696 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 697 698 if (!key || !keylen) 699 return -EINVAL; 700 701 if (keylen != AES_KEYSIZE_128 && 702 keylen != AES_KEYSIZE_192 && 703 keylen != AES_KEYSIZE_256) 704 return -EINVAL; 705 706 memcpy(ctx->key, key, keylen); 707 ctx->keylen = keylen; 708 709 return 0; 710 } 711 712 static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 713 unsigned int keylen) 714 { 715 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 716 717 if (!key || !keylen) 718 return -EINVAL; 719 720 if (keylen != AES_KEYSIZE_128 && 721 keylen != AES_KEYSIZE_192 && 722 keylen != AES_KEYSIZE_256) 723 return -EINVAL; 724 725 memcpy(ctx->key, key, keylen); 726 ctx->keylen = keylen; 727 728 if (ctx->aead_fbk) 729 return crypto_aead_setkey(ctx->aead_fbk, key, keylen); 730 731 return 0; 732 } 733 734 static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm, 735 unsigned int authsize) 736 { 737 return crypto_gcm_check_authsize(authsize); 738 } 739 740 static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm, 741 unsigned int authsize) 742 { 743 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); 744 745 switch (authsize) { 746 case 4: 747 case 6: 748 case 8: 749 case 10: 750 case 12: 751 case 14: 752 case 16: 753 break; 754 default: 755 return -EINVAL; 756 } 757 758 return crypto_aead_setauthsize(ctx->aead_fbk, authsize); 759 } 760 761 static int starfive_aes_ecb_encrypt(struct skcipher_request *req) 762 { 763 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT); 764 } 765 766 static int starfive_aes_ecb_decrypt(struct skcipher_request *req) 767 { 768 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB); 769 } 770 771 static int starfive_aes_cbc_encrypt(struct skcipher_request *req) 772 { 773 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT); 774 } 775 776 static int starfive_aes_cbc_decrypt(struct skcipher_request *req) 777 { 778 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC); 779 } 780 781 static int starfive_aes_ctr_encrypt(struct skcipher_request *req) 782 { 783 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT); 784 } 785 786 static int starfive_aes_ctr_decrypt(struct skcipher_request *req) 787 { 788 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR); 789 } 790 791 static int starfive_aes_gcm_encrypt(struct aead_request *req) 792 { 793 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT); 794 } 795 796 static int starfive_aes_gcm_decrypt(struct aead_request *req) 797 { 798 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM); 799 } 800 801 static int starfive_aes_ccm_encrypt(struct aead_request *req) 802 { 803 int ret; 804 805 ret = starfive_aes_ccm_check_iv(req->iv); 806 if (ret) 807 return ret; 808 809 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT); 810 } 811 812 static int starfive_aes_ccm_decrypt(struct aead_request *req) 813 { 814 int ret; 815 816 ret = starfive_aes_ccm_check_iv(req->iv); 817 if (ret) 818 return ret; 819 820 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM); 821 } 822 823 static struct skcipher_engine_alg skcipher_algs[] = { 824 { 825 .base.init = starfive_aes_init_tfm, 826 .base.setkey = starfive_aes_setkey, 827 .base.encrypt = starfive_aes_ecb_encrypt, 828 .base.decrypt = starfive_aes_ecb_decrypt, 829 .base.min_keysize = AES_MIN_KEY_SIZE, 830 .base.max_keysize = AES_MAX_KEY_SIZE, 831 .base.base = { 832 .cra_name = "ecb(aes)", 833 .cra_driver_name = "starfive-ecb-aes", 834 .cra_priority = 200, 835 .cra_flags = CRYPTO_ALG_ASYNC, 836 .cra_blocksize = AES_BLOCK_SIZE, 837 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 838 .cra_alignmask = 0xf, 839 .cra_module = THIS_MODULE, 840 }, 841 .op = { 842 .do_one_request = starfive_aes_do_one_req, 843 }, 844 }, { 845 .base.init = starfive_aes_init_tfm, 846 .base.setkey = starfive_aes_setkey, 847 .base.encrypt = starfive_aes_cbc_encrypt, 848 .base.decrypt = starfive_aes_cbc_decrypt, 849 .base.min_keysize = AES_MIN_KEY_SIZE, 850 .base.max_keysize = AES_MAX_KEY_SIZE, 851 .base.ivsize = AES_BLOCK_SIZE, 852 .base.base = { 853 .cra_name = "cbc(aes)", 854 .cra_driver_name = "starfive-cbc-aes", 855 .cra_priority = 200, 856 .cra_flags = CRYPTO_ALG_ASYNC, 857 .cra_blocksize = AES_BLOCK_SIZE, 858 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 859 .cra_alignmask = 0xf, 860 .cra_module = THIS_MODULE, 861 }, 862 .op = { 863 .do_one_request = starfive_aes_do_one_req, 864 }, 865 }, { 866 .base.init = starfive_aes_init_tfm, 867 .base.setkey = starfive_aes_setkey, 868 .base.encrypt = starfive_aes_ctr_encrypt, 869 .base.decrypt = starfive_aes_ctr_decrypt, 870 .base.min_keysize = AES_MIN_KEY_SIZE, 871 .base.max_keysize = AES_MAX_KEY_SIZE, 872 .base.ivsize = AES_BLOCK_SIZE, 873 .base.base = { 874 .cra_name = "ctr(aes)", 875 .cra_driver_name = "starfive-ctr-aes", 876 .cra_priority = 200, 877 .cra_flags = CRYPTO_ALG_ASYNC, 878 .cra_blocksize = 1, 879 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 880 .cra_alignmask = 0xf, 881 .cra_module = THIS_MODULE, 882 }, 883 .op = { 884 .do_one_request = starfive_aes_do_one_req, 885 }, 886 }, 887 }; 888 889 static struct aead_engine_alg aead_algs[] = { 890 { 891 .base.setkey = starfive_aes_aead_setkey, 892 .base.setauthsize = starfive_aes_gcm_setauthsize, 893 .base.encrypt = starfive_aes_gcm_encrypt, 894 .base.decrypt = starfive_aes_gcm_decrypt, 895 .base.init = starfive_aes_aead_init_tfm, 896 .base.exit = starfive_aes_aead_exit_tfm, 897 .base.ivsize = GCM_AES_IV_SIZE, 898 .base.maxauthsize = AES_BLOCK_SIZE, 899 .base.base = { 900 .cra_name = "gcm(aes)", 901 .cra_driver_name = "starfive-gcm-aes", 902 .cra_priority = 200, 903 .cra_flags = CRYPTO_ALG_ASYNC, 904 .cra_blocksize = 1, 905 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 906 .cra_alignmask = 0xf, 907 .cra_module = THIS_MODULE, 908 }, 909 .op = { 910 .do_one_request = starfive_aes_aead_do_one_req, 911 }, 912 }, { 913 .base.setkey = starfive_aes_aead_setkey, 914 .base.setauthsize = starfive_aes_ccm_setauthsize, 915 .base.encrypt = starfive_aes_ccm_encrypt, 916 .base.decrypt = starfive_aes_ccm_decrypt, 917 .base.init = starfive_aes_aead_init_tfm, 918 .base.exit = starfive_aes_aead_exit_tfm, 919 .base.ivsize = AES_BLOCK_SIZE, 920 .base.maxauthsize = AES_BLOCK_SIZE, 921 .base.base = { 922 .cra_name = "ccm(aes)", 923 .cra_driver_name = "starfive-ccm-aes", 924 .cra_priority = 200, 925 .cra_flags = CRYPTO_ALG_ASYNC | 926 CRYPTO_ALG_NEED_FALLBACK, 927 .cra_blocksize = 1, 928 .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 929 .cra_alignmask = 0xf, 930 .cra_module = THIS_MODULE, 931 }, 932 .op = { 933 .do_one_request = starfive_aes_aead_do_one_req, 934 }, 935 }, 936 }; 937 938 int starfive_aes_register_algs(void) 939 { 940 int ret; 941 942 ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 943 if (ret) 944 return ret; 945 946 ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 947 if (ret) 948 crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 949 950 return ret; 951 } 952 953 void starfive_aes_unregister_algs(void) 954 { 955 crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 956 crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 957 } 958