1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) STMicroelectronics SA 2017 4 * Author: Fabien Dessenne <fabien.dessenne@st.com> 5 * Ux500 support taken from snippets in the old Ux500 cryp driver 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/engine.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/internal/des.h> 12 #include <crypto/internal/skcipher.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/bottom_half.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmaengine.h> 19 #include <linux/err.h> 20 #include <linux/iopoll.h> 21 #include <linux/interrupt.h> 22 #include <linux/kernel.h> 23 #include <linux/module.h> 24 #include <linux/minmax.h> 25 #include <linux/of.h> 26 #include <linux/platform_device.h> 27 #include <linux/pm_runtime.h> 28 #include <linux/reset.h> 29 #include <linux/string.h> 30 31 #define DRIVER_NAME "stm32-cryp" 32 33 /* Bit [0] encrypt / decrypt */ 34 #define FLG_ENCRYPT BIT(0) 35 /* Bit [8..1] algo & operation mode */ 36 #define FLG_AES BIT(1) 37 #define FLG_DES BIT(2) 38 #define FLG_TDES BIT(3) 39 #define FLG_ECB BIT(4) 40 #define FLG_CBC BIT(5) 41 #define FLG_CTR BIT(6) 42 #define FLG_GCM BIT(7) 43 #define FLG_CCM BIT(8) 44 /* Mode mask = bits [15..0] */ 45 #define FLG_MODE_MASK GENMASK(15, 0) 46 /* Bit [31..16] status */ 47 #define FLG_IN_OUT_DMA BIT(16) 48 #define FLG_HEADER_DMA BIT(17) 49 50 /* Registers */ 51 #define CRYP_CR 0x00000000 52 #define CRYP_SR 0x00000004 53 #define CRYP_DIN 0x00000008 54 #define CRYP_DOUT 0x0000000C 55 #define CRYP_DMACR 0x00000010 56 #define CRYP_IMSCR 0x00000014 57 #define CRYP_RISR 0x00000018 58 #define CRYP_MISR 0x0000001C 59 #define CRYP_K0LR 0x00000020 60 #define CRYP_K0RR 0x00000024 61 #define CRYP_K1LR 0x00000028 62 #define CRYP_K1RR 0x0000002C 63 #define CRYP_K2LR 0x00000030 64 #define CRYP_K2RR 0x00000034 65 #define CRYP_K3LR 0x00000038 66 #define CRYP_K3RR 0x0000003C 67 #define CRYP_IV0LR 0x00000040 68 #define CRYP_IV0RR 0x00000044 69 #define CRYP_IV1LR 0x00000048 70 #define CRYP_IV1RR 0x0000004C 71 #define CRYP_CSGCMCCM0R 0x00000050 72 #define CRYP_CSGCM0R 0x00000070 73 74 #define UX500_CRYP_CR 0x00000000 75 #define UX500_CRYP_SR 0x00000004 76 #define UX500_CRYP_DIN 0x00000008 77 #define UX500_CRYP_DINSIZE 0x0000000C 78 #define UX500_CRYP_DOUT 0x00000010 79 #define UX500_CRYP_DOUSIZE 0x00000014 80 #define UX500_CRYP_DMACR 0x00000018 81 #define UX500_CRYP_IMSC 0x0000001C 82 #define UX500_CRYP_RIS 0x00000020 83 #define UX500_CRYP_MIS 0x00000024 84 #define UX500_CRYP_K1L 0x00000028 85 #define UX500_CRYP_K1R 0x0000002C 86 #define UX500_CRYP_K2L 0x00000030 87 #define UX500_CRYP_K2R 0x00000034 88 #define UX500_CRYP_K3L 0x00000038 89 #define UX500_CRYP_K3R 0x0000003C 90 #define UX500_CRYP_K4L 0x00000040 91 #define UX500_CRYP_K4R 0x00000044 92 #define UX500_CRYP_IV0L 0x00000048 93 #define UX500_CRYP_IV0R 0x0000004C 94 #define UX500_CRYP_IV1L 0x00000050 95 #define UX500_CRYP_IV1R 0x00000054 96 97 /* Registers values */ 98 #define CR_DEC_NOT_ENC 0x00000004 99 #define CR_TDES_ECB 0x00000000 100 #define CR_TDES_CBC 0x00000008 101 #define CR_DES_ECB 0x00000010 102 #define CR_DES_CBC 0x00000018 103 #define CR_AES_ECB 0x00000020 104 #define CR_AES_CBC 0x00000028 105 #define CR_AES_CTR 0x00000030 106 #define CR_AES_KP 0x00000038 /* Not on Ux500 */ 107 #define CR_AES_XTS 0x00000038 /* Only on Ux500 */ 108 #define CR_AES_GCM 0x00080000 109 #define CR_AES_CCM 0x00080008 110 #define CR_AES_UNKNOWN 0xFFFFFFFF 111 #define CR_ALGO_MASK 0x00080038 112 #define CR_DATA32 0x00000000 113 #define CR_DATA16 0x00000040 114 #define CR_DATA8 0x00000080 115 #define CR_DATA1 0x000000C0 116 #define CR_KEY128 0x00000000 117 #define CR_KEY192 0x00000100 118 #define CR_KEY256 0x00000200 119 #define CR_KEYRDEN 0x00000400 /* Only on Ux500 */ 120 #define CR_KSE 0x00000800 /* Only on Ux500 */ 121 #define CR_FFLUSH 0x00004000 122 #define CR_CRYPEN 0x00008000 123 #define CR_PH_INIT 0x00000000 124 #define CR_PH_HEADER 0x00010000 125 #define CR_PH_PAYLOAD 0x00020000 126 #define CR_PH_FINAL 0x00030000 127 #define CR_PH_MASK 0x00030000 128 #define CR_NBPBL_SHIFT 20 129 130 #define SR_IFNF BIT(1) 131 #define SR_OFNE BIT(2) 132 #define SR_BUSY BIT(8) 133 134 #define DMACR_DIEN BIT(0) 135 #define DMACR_DOEN BIT(1) 136 137 #define IMSCR_IN BIT(0) 138 #define IMSCR_OUT BIT(1) 139 140 #define MISR_IN BIT(0) 141 #define MISR_OUT BIT(1) 142 143 /* Misc */ 144 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 145 #define GCM_CTR_INIT 2 146 #define CRYP_AUTOSUSPEND_DELAY 50 147 148 #define CRYP_DMA_BURST_REG 4 149 150 enum stm32_dma_mode { 151 NO_DMA, 152 DMA_PLAIN_SG, 153 DMA_NEED_SG_TRUNC 154 }; 155 156 struct stm32_cryp_caps { 157 bool aeads_support; 158 bool linear_aes_key; 159 bool kp_mode; 160 bool iv_protection; 161 bool swap_final; 162 bool padding_wa; 163 u32 cr; 164 u32 sr; 165 u32 din; 166 u32 dout; 167 u32 dmacr; 168 u32 imsc; 169 u32 mis; 170 u32 k1l; 171 u32 k1r; 172 u32 k3r; 173 u32 iv0l; 174 u32 iv0r; 175 u32 iv1l; 176 u32 iv1r; 177 }; 178 179 struct stm32_cryp_ctx { 180 struct stm32_cryp *cryp; 181 int keylen; 182 __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; 183 unsigned long flags; 184 }; 185 186 struct stm32_cryp_reqctx { 187 unsigned long mode; 188 }; 189 190 struct stm32_cryp { 191 struct list_head list; 192 struct device *dev; 193 void __iomem *regs; 194 phys_addr_t phys_base; 195 struct clk *clk; 196 unsigned long flags; 197 u32 irq_status; 198 const struct stm32_cryp_caps *caps; 199 struct stm32_cryp_ctx *ctx; 200 201 struct crypto_engine *engine; 202 203 struct skcipher_request *req; 204 struct aead_request *areq; 205 206 size_t authsize; 207 size_t hw_blocksize; 208 209 size_t payload_in; 210 size_t header_in; 211 size_t payload_out; 212 213 /* DMA process fields */ 214 struct scatterlist *in_sg; 215 struct scatterlist *header_sg; 216 struct scatterlist *out_sg; 217 size_t in_sg_len; 218 size_t header_sg_len; 219 size_t out_sg_len; 220 struct completion dma_completion; 221 222 struct dma_chan *dma_lch_in; 223 struct dma_chan *dma_lch_out; 224 enum stm32_dma_mode dma_mode; 225 226 /* IT process fields */ 227 struct scatter_walk in_walk; 228 struct scatter_walk out_walk; 229 230 __be32 last_ctr[4]; 231 u32 gcm_ctr; 232 }; 233 234 struct stm32_cryp_list { 235 struct list_head dev_list; 236 spinlock_t lock; /* protect dev_list */ 237 }; 238 239 static struct stm32_cryp_list cryp_list = { 240 .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), 241 .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), 242 }; 243 244 static inline bool is_aes(struct stm32_cryp *cryp) 245 { 246 return cryp->flags & FLG_AES; 247 } 248 249 static inline bool is_des(struct stm32_cryp *cryp) 250 { 251 return cryp->flags & FLG_DES; 252 } 253 254 static inline bool is_tdes(struct stm32_cryp *cryp) 255 { 256 return cryp->flags & FLG_TDES; 257 } 258 259 static inline bool is_ecb(struct stm32_cryp *cryp) 260 { 261 return cryp->flags & FLG_ECB; 262 } 263 264 static inline bool is_cbc(struct stm32_cryp *cryp) 265 { 266 return cryp->flags & FLG_CBC; 267 } 268 269 static inline bool is_ctr(struct stm32_cryp *cryp) 270 { 271 return cryp->flags & FLG_CTR; 272 } 273 274 static inline bool is_gcm(struct stm32_cryp *cryp) 275 { 276 return cryp->flags & FLG_GCM; 277 } 278 279 static inline bool is_ccm(struct stm32_cryp *cryp) 280 { 281 return cryp->flags & FLG_CCM; 282 } 283 284 static inline bool is_encrypt(struct stm32_cryp *cryp) 285 { 286 return cryp->flags & FLG_ENCRYPT; 287 } 288 289 static inline bool is_decrypt(struct stm32_cryp *cryp) 290 { 291 return !is_encrypt(cryp); 292 } 293 294 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) 295 { 296 return readl_relaxed(cryp->regs + ofst); 297 } 298 299 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) 300 { 301 writel_relaxed(val, cryp->regs + ofst); 302 } 303 304 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) 305 { 306 u32 status; 307 308 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, 309 !(status & SR_BUSY), 10, 100000); 310 } 311 312 static inline void stm32_cryp_enable(struct stm32_cryp *cryp) 313 { 314 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN, 315 cryp->regs + cryp->caps->cr); 316 } 317 318 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) 319 { 320 u32 status; 321 322 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status, 323 !(status & CR_CRYPEN), 10, 100000); 324 } 325 326 static inline int stm32_cryp_wait_input(struct stm32_cryp *cryp) 327 { 328 u32 status; 329 330 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, 331 status & SR_IFNF, 1, 10); 332 } 333 334 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) 335 { 336 u32 status; 337 338 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, 339 status & SR_OFNE, 1, 10); 340 } 341 342 static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp) 343 { 344 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN, 345 cryp->regs + cryp->caps->cr); 346 } 347 348 static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp) 349 { 350 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN, 351 cryp->regs + cryp->caps->cr); 352 } 353 354 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp); 355 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp); 356 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp); 357 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); 358 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err); 359 static int stm32_cryp_dma_start(struct stm32_cryp *cryp); 360 static int stm32_cryp_it_start(struct stm32_cryp *cryp); 361 362 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) 363 { 364 struct stm32_cryp *cryp; 365 366 spin_lock_bh(&cryp_list.lock); 367 if (!ctx->cryp) 368 ctx->cryp = list_first_entry_or_null(&cryp_list.dev_list, 369 struct stm32_cryp, list); 370 cryp = ctx->cryp; 371 spin_unlock_bh(&cryp_list.lock); 372 373 return cryp; 374 } 375 376 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) 377 { 378 if (!iv) 379 return; 380 381 stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++)); 382 stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++)); 383 384 if (is_aes(cryp)) { 385 stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++)); 386 stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++)); 387 } 388 } 389 390 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) 391 { 392 struct skcipher_request *req = cryp->req; 393 __be32 *tmp = (void *)req->iv; 394 395 if (!tmp) 396 return; 397 398 if (cryp->caps->iv_protection) 399 stm32_cryp_key_read_enable(cryp); 400 401 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); 402 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); 403 404 if (is_aes(cryp)) { 405 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); 406 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); 407 } 408 409 if (cryp->caps->iv_protection) 410 stm32_cryp_key_read_disable(cryp); 411 } 412 413 /** 414 * ux500_swap_bits_in_byte() - mirror the bits in a byte 415 * @b: the byte to be mirrored 416 * 417 * The bits are swapped the following way: 418 * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and 419 * nibble 2 (n2) bits 4-7. 420 * 421 * Nibble 1 (n1): 422 * (The "old" (moved) bit is replaced with a zero) 423 * 1. Move bit 6 and 7, 4 positions to the left. 424 * 2. Move bit 3 and 5, 2 positions to the left. 425 * 3. Move bit 1-4, 1 position to the left. 426 * 427 * Nibble 2 (n2): 428 * 1. Move bit 0 and 1, 4 positions to the right. 429 * 2. Move bit 2 and 4, 2 positions to the right. 430 * 3. Move bit 3-6, 1 position to the right. 431 * 432 * Combine the two nibbles to a complete and swapped byte. 433 */ 434 static inline u8 ux500_swap_bits_in_byte(u8 b) 435 { 436 #define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */ 437 #define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5, 438 right shift 2 */ 439 #define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4, 440 right shift 1 */ 441 #define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */ 442 #define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4, 443 left shift 2 */ 444 #define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6, 445 left shift 1 */ 446 447 u8 n1; 448 u8 n2; 449 450 /* Swap most significant nibble */ 451 /* Right shift 4, bits 6 and 7 */ 452 n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4)); 453 /* Right shift 2, bits 3 and 5 */ 454 n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2)); 455 /* Right shift 1, bits 1-4 */ 456 n1 = (n1 & R_SHIFT_1_MASK) >> 1; 457 458 /* Swap least significant nibble */ 459 /* Left shift 4, bits 0 and 1 */ 460 n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4)); 461 /* Left shift 2, bits 2 and 4 */ 462 n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2)); 463 /* Left shift 1, bits 3-6 */ 464 n2 = (n2 & L_SHIFT_1_MASK) << 1; 465 466 return n1 | n2; 467 } 468 469 /** 470 * ux500_swizzle_key() - Shuffle around words and bits in the AES key 471 * @in: key to swizzle 472 * @out: swizzled key 473 * @len: length of key, in bytes 474 * 475 * This "key swizzling procedure" is described in the examples in the 476 * DB8500 design specification. There is no real description of why 477 * the bits have been arranged like this in the hardware. 478 */ 479 static inline void ux500_swizzle_key(const u8 *in, u8 *out, u32 len) 480 { 481 int i = 0; 482 int bpw = sizeof(u32); 483 int j; 484 int index = 0; 485 486 j = len - bpw; 487 while (j >= 0) { 488 for (i = 0; i < bpw; i++) { 489 index = len - j - bpw + i; 490 out[j + i] = 491 ux500_swap_bits_in_byte(in[index]); 492 } 493 j -= bpw; 494 } 495 } 496 497 static void stm32_cryp_hw_write_key(struct stm32_cryp *c) 498 { 499 unsigned int i; 500 int r_id; 501 502 if (is_des(c)) { 503 stm32_cryp_write(c, c->caps->k1l, be32_to_cpu(c->ctx->key[0])); 504 stm32_cryp_write(c, c->caps->k1r, be32_to_cpu(c->ctx->key[1])); 505 return; 506 } 507 508 /* 509 * On the Ux500 the AES key is considered as a single bit sequence 510 * of 128, 192 or 256 bits length. It is written linearly into the 511 * registers from K1L and down, and need to be processed to become 512 * a proper big-endian bit sequence. 513 */ 514 if (is_aes(c) && c->caps->linear_aes_key) { 515 u32 tmpkey[8]; 516 517 ux500_swizzle_key((u8 *)c->ctx->key, 518 (u8 *)tmpkey, c->ctx->keylen); 519 520 r_id = c->caps->k1l; 521 for (i = 0; i < c->ctx->keylen / sizeof(u32); i++, r_id += 4) 522 stm32_cryp_write(c, r_id, tmpkey[i]); 523 524 return; 525 } 526 527 r_id = c->caps->k3r; 528 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) 529 stm32_cryp_write(c, r_id, be32_to_cpu(c->ctx->key[i - 1])); 530 } 531 532 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) 533 { 534 if (is_aes(cryp) && is_ecb(cryp)) 535 return CR_AES_ECB; 536 537 if (is_aes(cryp) && is_cbc(cryp)) 538 return CR_AES_CBC; 539 540 if (is_aes(cryp) && is_ctr(cryp)) 541 return CR_AES_CTR; 542 543 if (is_aes(cryp) && is_gcm(cryp)) 544 return CR_AES_GCM; 545 546 if (is_aes(cryp) && is_ccm(cryp)) 547 return CR_AES_CCM; 548 549 if (is_des(cryp) && is_ecb(cryp)) 550 return CR_DES_ECB; 551 552 if (is_des(cryp) && is_cbc(cryp)) 553 return CR_DES_CBC; 554 555 if (is_tdes(cryp) && is_ecb(cryp)) 556 return CR_TDES_ECB; 557 558 if (is_tdes(cryp) && is_cbc(cryp)) 559 return CR_TDES_CBC; 560 561 dev_err(cryp->dev, "Unknown mode\n"); 562 return CR_AES_UNKNOWN; 563 } 564 565 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) 566 { 567 return is_encrypt(cryp) ? cryp->areq->cryptlen : 568 cryp->areq->cryptlen - cryp->authsize; 569 } 570 571 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) 572 { 573 int ret; 574 __be32 iv[4]; 575 576 /* Phase 1 : init */ 577 memcpy(iv, cryp->areq->iv, 12); 578 iv[3] = cpu_to_be32(GCM_CTR_INIT); 579 cryp->gcm_ctr = GCM_CTR_INIT; 580 stm32_cryp_hw_write_iv(cryp, iv); 581 582 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); 583 584 /* Wait for end of processing */ 585 ret = stm32_cryp_wait_enable(cryp); 586 if (ret) { 587 dev_err(cryp->dev, "Timeout (gcm init)\n"); 588 return ret; 589 } 590 591 /* Prepare next phase */ 592 if (cryp->areq->assoclen) { 593 cfg |= CR_PH_HEADER; 594 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 595 } else if (stm32_cryp_get_input_text_len(cryp)) { 596 cfg |= CR_PH_PAYLOAD; 597 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 598 } 599 600 return 0; 601 } 602 603 static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) 604 { 605 u32 cfg; 606 int err; 607 608 /* Check if whole header written */ 609 if (!cryp->header_in) { 610 /* Wait for completion */ 611 err = stm32_cryp_wait_busy(cryp); 612 if (err) { 613 dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); 614 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 615 stm32_cryp_finish_req(cryp, err); 616 return; 617 } 618 619 if (stm32_cryp_get_input_text_len(cryp)) { 620 /* Phase 3 : payload */ 621 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 622 cfg &= ~CR_CRYPEN; 623 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 624 625 cfg &= ~CR_PH_MASK; 626 cfg |= CR_PH_PAYLOAD | CR_CRYPEN; 627 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 628 } else { 629 /* 630 * Phase 4 : tag. 631 * Nothing to read, nothing to write, caller have to 632 * end request 633 */ 634 } 635 } 636 } 637 638 static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) 639 { 640 size_t written; 641 size_t len; 642 u32 alen = cryp->areq->assoclen; 643 u32 block[AES_BLOCK_32] = {0}; 644 u8 *b8 = (u8 *)block; 645 646 if (alen <= 65280) { 647 /* Write first u32 of B1 */ 648 b8[0] = (alen >> 8) & 0xFF; 649 b8[1] = alen & 0xFF; 650 len = 2; 651 } else { 652 /* Build the two first u32 of B1 */ 653 b8[0] = 0xFF; 654 b8[1] = 0xFE; 655 b8[2] = (alen & 0xFF000000) >> 24; 656 b8[3] = (alen & 0x00FF0000) >> 16; 657 b8[4] = (alen & 0x0000FF00) >> 8; 658 b8[5] = alen & 0x000000FF; 659 len = 6; 660 } 661 662 written = min_t(size_t, AES_BLOCK_SIZE - len, alen); 663 664 memcpy_from_scatterwalk((char *)block + len, &cryp->in_walk, written); 665 666 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 667 668 cryp->header_in -= written; 669 670 stm32_crypt_gcmccm_end_header(cryp); 671 } 672 673 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) 674 { 675 int ret; 676 u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32]; 677 u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32; 678 __be32 *bd; 679 u32 *d; 680 unsigned int i, textlen; 681 682 /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ 683 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 684 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 685 iv[AES_BLOCK_SIZE - 1] = 1; 686 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); 687 688 /* Build B0 */ 689 memcpy(b0, iv, AES_BLOCK_SIZE); 690 691 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 692 693 if (cryp->areq->assoclen) 694 b0[0] |= 0x40; 695 696 textlen = stm32_cryp_get_input_text_len(cryp); 697 698 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 699 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 700 701 /* Enable HW */ 702 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); 703 704 /* Write B0 */ 705 d = (u32 *)b0; 706 bd = (__be32 *)b0; 707 708 for (i = 0; i < AES_BLOCK_32; i++) { 709 u32 xd = d[i]; 710 711 if (!cryp->caps->padding_wa) 712 xd = be32_to_cpu(bd[i]); 713 stm32_cryp_write(cryp, cryp->caps->din, xd); 714 } 715 716 /* Wait for end of processing */ 717 ret = stm32_cryp_wait_enable(cryp); 718 if (ret) { 719 dev_err(cryp->dev, "Timeout (ccm init)\n"); 720 return ret; 721 } 722 723 /* Prepare next phase */ 724 if (cryp->areq->assoclen) { 725 cfg |= CR_PH_HEADER | CR_CRYPEN; 726 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 727 728 /* Write first (special) block (may move to next phase [payload]) */ 729 stm32_cryp_write_ccm_first_header(cryp); 730 } else if (stm32_cryp_get_input_text_len(cryp)) { 731 cfg |= CR_PH_PAYLOAD; 732 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 733 } 734 735 return 0; 736 } 737 738 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) 739 { 740 int ret; 741 u32 cfg, hw_mode; 742 743 pm_runtime_get_sync(cryp->dev); 744 745 /* Disable interrupt */ 746 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 747 748 /* Set configuration */ 749 cfg = CR_DATA8 | CR_FFLUSH; 750 751 switch (cryp->ctx->keylen) { 752 case AES_KEYSIZE_128: 753 cfg |= CR_KEY128; 754 break; 755 756 case AES_KEYSIZE_192: 757 cfg |= CR_KEY192; 758 break; 759 760 default: 761 case AES_KEYSIZE_256: 762 cfg |= CR_KEY256; 763 break; 764 } 765 766 hw_mode = stm32_cryp_get_hw_mode(cryp); 767 if (hw_mode == CR_AES_UNKNOWN) 768 return -EINVAL; 769 770 /* AES ECB/CBC decrypt: run key preparation first */ 771 if (is_decrypt(cryp) && 772 ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { 773 /* Configure in key preparation mode */ 774 if (cryp->caps->kp_mode) 775 stm32_cryp_write(cryp, cryp->caps->cr, 776 cfg | CR_AES_KP); 777 else 778 stm32_cryp_write(cryp, 779 cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE); 780 781 /* Set key only after full configuration done */ 782 stm32_cryp_hw_write_key(cryp); 783 784 /* Start prepare key */ 785 stm32_cryp_enable(cryp); 786 /* Wait for end of processing */ 787 ret = stm32_cryp_wait_busy(cryp); 788 if (ret) { 789 dev_err(cryp->dev, "Timeout (key preparation)\n"); 790 return ret; 791 } 792 793 cfg |= hw_mode | CR_DEC_NOT_ENC; 794 795 /* Apply updated config (Decrypt + algo) and flush */ 796 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 797 } else { 798 cfg |= hw_mode; 799 if (is_decrypt(cryp)) 800 cfg |= CR_DEC_NOT_ENC; 801 802 /* Apply config and flush */ 803 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 804 805 /* Set key only after configuration done */ 806 stm32_cryp_hw_write_key(cryp); 807 } 808 809 switch (hw_mode) { 810 case CR_AES_GCM: 811 case CR_AES_CCM: 812 /* Phase 1 : init */ 813 if (hw_mode == CR_AES_CCM) 814 ret = stm32_cryp_ccm_init(cryp, cfg); 815 else 816 ret = stm32_cryp_gcm_init(cryp, cfg); 817 818 if (ret) 819 return ret; 820 821 break; 822 823 case CR_DES_CBC: 824 case CR_TDES_CBC: 825 case CR_AES_CBC: 826 case CR_AES_CTR: 827 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); 828 break; 829 830 default: 831 break; 832 } 833 834 /* Enable now */ 835 stm32_cryp_enable(cryp); 836 837 return 0; 838 } 839 840 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) 841 { 842 if (!err && (is_gcm(cryp) || is_ccm(cryp))) 843 /* Phase 4 : output tag */ 844 err = stm32_cryp_read_auth_tag(cryp); 845 846 if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) 847 stm32_cryp_get_iv(cryp); 848 849 pm_runtime_put_autosuspend(cryp->dev); 850 851 if (is_gcm(cryp) || is_ccm(cryp)) 852 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); 853 else 854 crypto_finalize_skcipher_request(cryp->engine, cryp->req, err); 855 } 856 857 static void stm32_cryp_header_dma_callback(void *param) 858 { 859 struct stm32_cryp *cryp = (struct stm32_cryp *)param; 860 int ret; 861 u32 reg; 862 863 dma_unmap_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); 864 865 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); 866 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); 867 868 kfree(cryp->header_sg); 869 870 reg = stm32_cryp_read(cryp, cryp->caps->cr); 871 872 if (cryp->header_in) { 873 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); 874 875 ret = stm32_cryp_wait_input(cryp); 876 if (ret) { 877 dev_err(cryp->dev, "input header ready timeout after dma\n"); 878 stm32_cryp_finish_req(cryp, ret); 879 return; 880 } 881 stm32_cryp_irq_write_gcmccm_header(cryp); 882 WARN_ON(cryp->header_in); 883 } 884 885 if (stm32_cryp_get_input_text_len(cryp)) { 886 /* Phase 3 : payload */ 887 reg = stm32_cryp_read(cryp, cryp->caps->cr); 888 stm32_cryp_write(cryp, cryp->caps->cr, reg & ~CR_CRYPEN); 889 890 reg &= ~CR_PH_MASK; 891 reg |= CR_PH_PAYLOAD | CR_CRYPEN; 892 stm32_cryp_write(cryp, cryp->caps->cr, reg); 893 894 if (cryp->flags & FLG_IN_OUT_DMA) { 895 ret = stm32_cryp_dma_start(cryp); 896 if (ret) 897 stm32_cryp_finish_req(cryp, ret); 898 } else { 899 stm32_cryp_it_start(cryp); 900 } 901 } else { 902 /* 903 * Phase 4 : tag. 904 * Nothing to read, nothing to write => end request 905 */ 906 stm32_cryp_finish_req(cryp, 0); 907 } 908 } 909 910 static void stm32_cryp_dma_callback(void *param) 911 { 912 struct stm32_cryp *cryp = (struct stm32_cryp *)param; 913 int ret; 914 u32 reg; 915 916 complete(&cryp->dma_completion); /* completion to indicate no timeout */ 917 918 dma_sync_sg_for_device(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); 919 920 if (cryp->in_sg != cryp->out_sg) 921 dma_unmap_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); 922 923 dma_unmap_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); 924 925 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); 926 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); 927 928 reg = stm32_cryp_read(cryp, cryp->caps->cr); 929 930 if (is_gcm(cryp) || is_ccm(cryp)) { 931 kfree(cryp->in_sg); 932 kfree(cryp->out_sg); 933 } else { 934 if (cryp->in_sg != cryp->req->src) 935 kfree(cryp->in_sg); 936 if (cryp->out_sg != cryp->req->dst) 937 kfree(cryp->out_sg); 938 } 939 940 if (cryp->payload_in) { 941 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); 942 943 ret = stm32_cryp_wait_input(cryp); 944 if (ret) { 945 dev_err(cryp->dev, "input ready timeout after dma\n"); 946 stm32_cryp_finish_req(cryp, ret); 947 return; 948 } 949 stm32_cryp_irq_write_data(cryp); 950 951 ret = stm32_cryp_wait_output(cryp); 952 if (ret) { 953 dev_err(cryp->dev, "output ready timeout after dma\n"); 954 stm32_cryp_finish_req(cryp, ret); 955 return; 956 } 957 stm32_cryp_irq_read_data(cryp); 958 } 959 960 stm32_cryp_finish_req(cryp, 0); 961 } 962 963 static int stm32_cryp_header_dma_start(struct stm32_cryp *cryp) 964 { 965 int ret; 966 struct dma_async_tx_descriptor *tx_in; 967 u32 reg; 968 size_t align_size; 969 970 ret = dma_map_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); 971 if (!ret) { 972 dev_err(cryp->dev, "dma_map_sg() error\n"); 973 return -ENOMEM; 974 } 975 976 dma_sync_sg_for_device(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); 977 978 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->header_sg, cryp->header_sg_len, 979 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 980 if (!tx_in) { 981 dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); 982 return -EINVAL; 983 } 984 985 tx_in->callback_param = cryp; 986 tx_in->callback = stm32_cryp_header_dma_callback; 987 988 /* Advance scatterwalk to not DMA'ed data */ 989 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); 990 scatterwalk_skip(&cryp->in_walk, align_size); 991 cryp->header_in -= align_size; 992 993 ret = dma_submit_error(dmaengine_submit(tx_in)); 994 if (ret < 0) { 995 dev_err(cryp->dev, "DMA in submit failed\n"); 996 return ret; 997 } 998 dma_async_issue_pending(cryp->dma_lch_in); 999 1000 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); 1001 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DIEN); 1002 1003 return 0; 1004 } 1005 1006 static int stm32_cryp_dma_start(struct stm32_cryp *cryp) 1007 { 1008 int ret; 1009 size_t align_size; 1010 struct dma_async_tx_descriptor *tx_in, *tx_out; 1011 u32 reg; 1012 1013 if (cryp->in_sg != cryp->out_sg) { 1014 ret = dma_map_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); 1015 if (!ret) { 1016 dev_err(cryp->dev, "dma_map_sg() error\n"); 1017 return -ENOMEM; 1018 } 1019 } 1020 1021 ret = dma_map_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); 1022 if (!ret) { 1023 dev_err(cryp->dev, "dma_map_sg() error\n"); 1024 return -ENOMEM; 1025 } 1026 1027 dma_sync_sg_for_device(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); 1028 1029 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->in_sg, cryp->in_sg_len, 1030 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1031 if (!tx_in) { 1032 dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); 1033 return -EINVAL; 1034 } 1035 1036 /* No callback necessary */ 1037 tx_in->callback_param = cryp; 1038 tx_in->callback = NULL; 1039 1040 tx_out = dmaengine_prep_slave_sg(cryp->dma_lch_out, cryp->out_sg, cryp->out_sg_len, 1041 DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1042 if (!tx_out) { 1043 dev_err(cryp->dev, "OUT prep_slave_sg() failed\n"); 1044 return -EINVAL; 1045 } 1046 1047 reinit_completion(&cryp->dma_completion); 1048 tx_out->callback = stm32_cryp_dma_callback; 1049 tx_out->callback_param = cryp; 1050 1051 /* Advance scatterwalk to not DMA'ed data */ 1052 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); 1053 scatterwalk_skip(&cryp->in_walk, align_size); 1054 cryp->payload_in -= align_size; 1055 1056 ret = dma_submit_error(dmaengine_submit(tx_in)); 1057 if (ret < 0) { 1058 dev_err(cryp->dev, "DMA in submit failed\n"); 1059 return ret; 1060 } 1061 dma_async_issue_pending(cryp->dma_lch_in); 1062 1063 /* Advance scatterwalk to not DMA'ed data */ 1064 scatterwalk_skip(&cryp->out_walk, align_size); 1065 cryp->payload_out -= align_size; 1066 ret = dma_submit_error(dmaengine_submit(tx_out)); 1067 if (ret < 0) { 1068 dev_err(cryp->dev, "DMA out submit failed\n"); 1069 return ret; 1070 } 1071 dma_async_issue_pending(cryp->dma_lch_out); 1072 1073 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); 1074 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DOEN | DMACR_DIEN); 1075 1076 if (!wait_for_completion_timeout(&cryp->dma_completion, msecs_to_jiffies(1000))) { 1077 dev_err(cryp->dev, "DMA out timed out\n"); 1078 dmaengine_terminate_sync(cryp->dma_lch_out); 1079 return -ETIMEDOUT; 1080 } 1081 1082 return 0; 1083 } 1084 1085 static int stm32_cryp_it_start(struct stm32_cryp *cryp) 1086 { 1087 /* Enable interrupt and let the IRQ handler do everything */ 1088 stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT); 1089 1090 return 0; 1091 } 1092 1093 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); 1094 1095 static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm) 1096 { 1097 crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); 1098 1099 return 0; 1100 } 1101 1102 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); 1103 1104 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) 1105 { 1106 crypto_aead_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); 1107 1108 return 0; 1109 } 1110 1111 static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode) 1112 { 1113 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 1114 crypto_skcipher_reqtfm(req)); 1115 struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req); 1116 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 1117 1118 if (!cryp) 1119 return -ENODEV; 1120 1121 rctx->mode = mode; 1122 1123 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); 1124 } 1125 1126 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode) 1127 { 1128 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1129 struct stm32_cryp_reqctx *rctx = aead_request_ctx(req); 1130 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 1131 1132 if (!cryp) 1133 return -ENODEV; 1134 1135 rctx->mode = mode; 1136 1137 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 1138 } 1139 1140 static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key, 1141 unsigned int keylen) 1142 { 1143 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 1144 1145 memcpy(ctx->key, key, keylen); 1146 ctx->keylen = keylen; 1147 1148 return 0; 1149 } 1150 1151 static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 1152 unsigned int keylen) 1153 { 1154 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 1155 keylen != AES_KEYSIZE_256) 1156 return -EINVAL; 1157 else 1158 return stm32_cryp_setkey(tfm, key, keylen); 1159 } 1160 1161 static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 1162 unsigned int keylen) 1163 { 1164 return verify_skcipher_des_key(tfm, key) ?: 1165 stm32_cryp_setkey(tfm, key, keylen); 1166 } 1167 1168 static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 1169 unsigned int keylen) 1170 { 1171 return verify_skcipher_des3_key(tfm, key) ?: 1172 stm32_cryp_setkey(tfm, key, keylen); 1173 } 1174 1175 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 1176 unsigned int keylen) 1177 { 1178 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 1179 1180 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 1181 keylen != AES_KEYSIZE_256) 1182 return -EINVAL; 1183 1184 memcpy(ctx->key, key, keylen); 1185 ctx->keylen = keylen; 1186 1187 return 0; 1188 } 1189 1190 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, 1191 unsigned int authsize) 1192 { 1193 switch (authsize) { 1194 case 4: 1195 case 8: 1196 case 12: 1197 case 13: 1198 case 14: 1199 case 15: 1200 case 16: 1201 break; 1202 default: 1203 return -EINVAL; 1204 } 1205 1206 return 0; 1207 } 1208 1209 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, 1210 unsigned int authsize) 1211 { 1212 switch (authsize) { 1213 case 4: 1214 case 6: 1215 case 8: 1216 case 10: 1217 case 12: 1218 case 14: 1219 case 16: 1220 break; 1221 default: 1222 return -EINVAL; 1223 } 1224 1225 return 0; 1226 } 1227 1228 static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req) 1229 { 1230 if (req->cryptlen % AES_BLOCK_SIZE) 1231 return -EINVAL; 1232 1233 if (req->cryptlen == 0) 1234 return 0; 1235 1236 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); 1237 } 1238 1239 static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req) 1240 { 1241 if (req->cryptlen % AES_BLOCK_SIZE) 1242 return -EINVAL; 1243 1244 if (req->cryptlen == 0) 1245 return 0; 1246 1247 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); 1248 } 1249 1250 static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req) 1251 { 1252 if (req->cryptlen % AES_BLOCK_SIZE) 1253 return -EINVAL; 1254 1255 if (req->cryptlen == 0) 1256 return 0; 1257 1258 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); 1259 } 1260 1261 static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req) 1262 { 1263 if (req->cryptlen % AES_BLOCK_SIZE) 1264 return -EINVAL; 1265 1266 if (req->cryptlen == 0) 1267 return 0; 1268 1269 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); 1270 } 1271 1272 static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req) 1273 { 1274 if (req->cryptlen == 0) 1275 return 0; 1276 1277 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); 1278 } 1279 1280 static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req) 1281 { 1282 if (req->cryptlen == 0) 1283 return 0; 1284 1285 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); 1286 } 1287 1288 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req) 1289 { 1290 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT); 1291 } 1292 1293 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) 1294 { 1295 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); 1296 } 1297 1298 static inline int crypto_ccm_check_iv(const u8 *iv) 1299 { 1300 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1301 if (iv[0] < 1 || iv[0] > 7) 1302 return -EINVAL; 1303 1304 return 0; 1305 } 1306 1307 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) 1308 { 1309 int err; 1310 1311 err = crypto_ccm_check_iv(req->iv); 1312 if (err) 1313 return err; 1314 1315 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); 1316 } 1317 1318 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) 1319 { 1320 int err; 1321 1322 err = crypto_ccm_check_iv(req->iv); 1323 if (err) 1324 return err; 1325 1326 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); 1327 } 1328 1329 static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req) 1330 { 1331 if (req->cryptlen % DES_BLOCK_SIZE) 1332 return -EINVAL; 1333 1334 if (req->cryptlen == 0) 1335 return 0; 1336 1337 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); 1338 } 1339 1340 static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req) 1341 { 1342 if (req->cryptlen % DES_BLOCK_SIZE) 1343 return -EINVAL; 1344 1345 if (req->cryptlen == 0) 1346 return 0; 1347 1348 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); 1349 } 1350 1351 static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req) 1352 { 1353 if (req->cryptlen % DES_BLOCK_SIZE) 1354 return -EINVAL; 1355 1356 if (req->cryptlen == 0) 1357 return 0; 1358 1359 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); 1360 } 1361 1362 static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req) 1363 { 1364 if (req->cryptlen % DES_BLOCK_SIZE) 1365 return -EINVAL; 1366 1367 if (req->cryptlen == 0) 1368 return 0; 1369 1370 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); 1371 } 1372 1373 static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req) 1374 { 1375 if (req->cryptlen % DES_BLOCK_SIZE) 1376 return -EINVAL; 1377 1378 if (req->cryptlen == 0) 1379 return 0; 1380 1381 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); 1382 } 1383 1384 static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req) 1385 { 1386 if (req->cryptlen % DES_BLOCK_SIZE) 1387 return -EINVAL; 1388 1389 if (req->cryptlen == 0) 1390 return 0; 1391 1392 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); 1393 } 1394 1395 static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req) 1396 { 1397 if (req->cryptlen % DES_BLOCK_SIZE) 1398 return -EINVAL; 1399 1400 if (req->cryptlen == 0) 1401 return 0; 1402 1403 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); 1404 } 1405 1406 static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req) 1407 { 1408 if (req->cryptlen % DES_BLOCK_SIZE) 1409 return -EINVAL; 1410 1411 if (req->cryptlen == 0) 1412 return 0; 1413 1414 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); 1415 } 1416 1417 static enum stm32_dma_mode stm32_cryp_dma_check_sg(struct scatterlist *test_sg, size_t len, 1418 size_t block_size) 1419 { 1420 struct scatterlist *sg; 1421 int i; 1422 1423 if (len <= 16) 1424 return NO_DMA; /* Faster */ 1425 1426 for_each_sg(test_sg, sg, sg_nents(test_sg), i) { 1427 if (!IS_ALIGNED(sg->length, block_size) && !sg_is_last(sg)) 1428 return NO_DMA; 1429 1430 if (sg->offset % sizeof(u32)) 1431 return NO_DMA; 1432 1433 if (sg_is_last(sg) && !IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) 1434 return DMA_NEED_SG_TRUNC; 1435 } 1436 1437 return DMA_PLAIN_SG; 1438 } 1439 1440 static enum stm32_dma_mode stm32_cryp_dma_check(struct stm32_cryp *cryp, struct scatterlist *in_sg, 1441 struct scatterlist *out_sg) 1442 { 1443 enum stm32_dma_mode ret = DMA_PLAIN_SG; 1444 1445 if (!is_aes(cryp)) 1446 return NO_DMA; 1447 1448 if (!cryp->dma_lch_in || !cryp->dma_lch_out) 1449 return NO_DMA; 1450 1451 ret = stm32_cryp_dma_check_sg(in_sg, cryp->payload_in, AES_BLOCK_SIZE); 1452 if (ret == NO_DMA) 1453 return ret; 1454 1455 ret = stm32_cryp_dma_check_sg(out_sg, cryp->payload_out, AES_BLOCK_SIZE); 1456 if (ret == NO_DMA) 1457 return ret; 1458 1459 /* Check CTR counter overflow */ 1460 if (is_aes(cryp) && is_ctr(cryp)) { 1461 u32 c; 1462 __be32 iv3; 1463 1464 memcpy(&iv3, &cryp->req->iv[3 * sizeof(u32)], sizeof(iv3)); 1465 c = be32_to_cpu(iv3); 1466 if ((c + cryp->payload_in) < cryp->payload_in) 1467 return NO_DMA; 1468 } 1469 1470 /* Workaround */ 1471 if (is_aes(cryp) && is_ctr(cryp) && ret == DMA_NEED_SG_TRUNC) 1472 return NO_DMA; 1473 1474 return ret; 1475 } 1476 1477 static int stm32_cryp_truncate_sg(struct scatterlist **new_sg, size_t *new_sg_len, 1478 struct scatterlist *sg, off_t skip, size_t size) 1479 { 1480 struct scatterlist *cur; 1481 int alloc_sg_len; 1482 1483 *new_sg_len = 0; 1484 1485 if (!sg || !size) { 1486 *new_sg = NULL; 1487 return 0; 1488 } 1489 1490 alloc_sg_len = sg_nents_for_len(sg, skip + size); 1491 if (alloc_sg_len < 0) 1492 return alloc_sg_len; 1493 1494 /* We allocate to much sg entry, but it is easier */ 1495 *new_sg = kmalloc_objs(struct scatterlist, (size_t)alloc_sg_len); 1496 if (!*new_sg) 1497 return -ENOMEM; 1498 1499 sg_init_table(*new_sg, (unsigned int)alloc_sg_len); 1500 1501 cur = *new_sg; 1502 while (sg && size) { 1503 unsigned int len = sg->length; 1504 unsigned int offset = sg->offset; 1505 1506 if (skip > len) { 1507 skip -= len; 1508 sg = sg_next(sg); 1509 continue; 1510 } 1511 1512 if (skip) { 1513 len -= skip; 1514 offset += skip; 1515 skip = 0; 1516 } 1517 1518 if (size < len) 1519 len = size; 1520 1521 if (len > 0) { 1522 (*new_sg_len)++; 1523 size -= len; 1524 sg_set_page(cur, sg_page(sg), len, offset); 1525 if (size == 0) 1526 sg_mark_end(cur); 1527 cur = sg_next(cur); 1528 } 1529 1530 sg = sg_next(sg); 1531 } 1532 1533 return 0; 1534 } 1535 1536 static int stm32_cryp_cipher_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, 1537 struct scatterlist *out_sg) 1538 { 1539 size_t align_size; 1540 int ret; 1541 1542 cryp->dma_mode = stm32_cryp_dma_check(cryp, in_sg, out_sg); 1543 1544 scatterwalk_start(&cryp->in_walk, in_sg); 1545 scatterwalk_start(&cryp->out_walk, out_sg); 1546 1547 if (cryp->dma_mode == NO_DMA) { 1548 cryp->flags &= ~FLG_IN_OUT_DMA; 1549 1550 if (is_ctr(cryp)) 1551 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); 1552 1553 } else if (cryp->dma_mode == DMA_NEED_SG_TRUNC) { 1554 1555 cryp->flags |= FLG_IN_OUT_DMA; 1556 1557 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); 1558 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, 0, align_size); 1559 if (ret) 1560 return ret; 1561 1562 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, 0, 1563 align_size); 1564 if (ret) { 1565 kfree(cryp->in_sg); 1566 return ret; 1567 } 1568 } else { 1569 cryp->flags |= FLG_IN_OUT_DMA; 1570 1571 cryp->in_sg = in_sg; 1572 cryp->out_sg = out_sg; 1573 1574 ret = sg_nents_for_len(cryp->in_sg, cryp->payload_in); 1575 if (ret < 0) 1576 return ret; 1577 cryp->in_sg_len = (size_t)ret; 1578 1579 ret = sg_nents_for_len(out_sg, cryp->payload_out); 1580 if (ret < 0) 1581 return ret; 1582 cryp->out_sg_len = (size_t)ret; 1583 } 1584 1585 return 0; 1586 } 1587 1588 static int stm32_cryp_aead_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, 1589 struct scatterlist *out_sg) 1590 { 1591 size_t align_size; 1592 off_t skip; 1593 int ret, ret2; 1594 1595 cryp->header_sg = NULL; 1596 cryp->in_sg = NULL; 1597 cryp->out_sg = NULL; 1598 1599 if (!cryp->dma_lch_in || !cryp->dma_lch_out) { 1600 cryp->dma_mode = NO_DMA; 1601 cryp->flags &= ~(FLG_IN_OUT_DMA | FLG_HEADER_DMA); 1602 1603 return 0; 1604 } 1605 1606 /* CCM hw_init may have advanced in header */ 1607 skip = cryp->areq->assoclen - cryp->header_in; 1608 1609 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); 1610 ret = stm32_cryp_truncate_sg(&cryp->header_sg, &cryp->header_sg_len, in_sg, skip, 1611 align_size); 1612 if (ret) 1613 return ret; 1614 1615 ret = stm32_cryp_dma_check_sg(cryp->header_sg, align_size, AES_BLOCK_SIZE); 1616 if (ret == NO_DMA) { 1617 /* We cannot DMA the header */ 1618 kfree(cryp->header_sg); 1619 cryp->header_sg = NULL; 1620 1621 cryp->flags &= ~FLG_HEADER_DMA; 1622 } else { 1623 cryp->flags |= FLG_HEADER_DMA; 1624 } 1625 1626 /* Now skip all header to be at payload start */ 1627 skip = cryp->areq->assoclen; 1628 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); 1629 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, skip, align_size); 1630 if (ret) { 1631 kfree(cryp->header_sg); 1632 return ret; 1633 } 1634 1635 /* For out buffer align_size is same as in buffer */ 1636 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, skip, align_size); 1637 if (ret) { 1638 kfree(cryp->header_sg); 1639 kfree(cryp->in_sg); 1640 return ret; 1641 } 1642 1643 ret = stm32_cryp_dma_check_sg(cryp->in_sg, align_size, AES_BLOCK_SIZE); 1644 ret2 = stm32_cryp_dma_check_sg(cryp->out_sg, align_size, AES_BLOCK_SIZE); 1645 if (ret == NO_DMA || ret2 == NO_DMA) { 1646 kfree(cryp->in_sg); 1647 cryp->in_sg = NULL; 1648 1649 kfree(cryp->out_sg); 1650 cryp->out_sg = NULL; 1651 1652 cryp->flags &= ~FLG_IN_OUT_DMA; 1653 } else { 1654 cryp->flags |= FLG_IN_OUT_DMA; 1655 } 1656 1657 return 0; 1658 } 1659 1660 static int stm32_cryp_prepare_req(struct skcipher_request *req, 1661 struct aead_request *areq) 1662 { 1663 struct stm32_cryp_ctx *ctx; 1664 struct stm32_cryp *cryp; 1665 struct stm32_cryp_reqctx *rctx; 1666 struct scatterlist *in_sg, *out_sg; 1667 int ret; 1668 1669 if (!req && !areq) 1670 return -EINVAL; 1671 1672 ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : 1673 crypto_aead_ctx(crypto_aead_reqtfm(areq)); 1674 1675 cryp = ctx->cryp; 1676 1677 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); 1678 rctx->mode &= FLG_MODE_MASK; 1679 1680 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; 1681 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; 1682 cryp->ctx = ctx; 1683 1684 if (req) { 1685 cryp->req = req; 1686 cryp->areq = NULL; 1687 cryp->header_in = 0; 1688 cryp->payload_in = req->cryptlen; 1689 cryp->payload_out = req->cryptlen; 1690 cryp->authsize = 0; 1691 1692 in_sg = req->src; 1693 out_sg = req->dst; 1694 1695 ret = stm32_cryp_cipher_prepare(cryp, in_sg, out_sg); 1696 if (ret) 1697 return ret; 1698 1699 ret = stm32_cryp_hw_init(cryp); 1700 } else { 1701 /* 1702 * Length of input and output data: 1703 * Encryption case: 1704 * INPUT = AssocData || PlainText 1705 * <- assoclen -> <- cryptlen -> 1706 * 1707 * OUTPUT = AssocData || CipherText || AuthTag 1708 * <- assoclen -> <-- cryptlen --> <- authsize -> 1709 * 1710 * Decryption case: 1711 * INPUT = AssocData || CipherTex || AuthTag 1712 * <- assoclen ---> <---------- cryptlen ----------> 1713 * 1714 * OUTPUT = AssocData || PlainText 1715 * <- assoclen -> <- cryptlen - authsize -> 1716 */ 1717 cryp->areq = areq; 1718 cryp->req = NULL; 1719 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); 1720 if (is_encrypt(cryp)) { 1721 cryp->payload_in = areq->cryptlen; 1722 cryp->header_in = areq->assoclen; 1723 cryp->payload_out = areq->cryptlen; 1724 } else { 1725 cryp->payload_in = areq->cryptlen - cryp->authsize; 1726 cryp->header_in = areq->assoclen; 1727 cryp->payload_out = cryp->payload_in; 1728 } 1729 1730 in_sg = areq->src; 1731 out_sg = areq->dst; 1732 1733 scatterwalk_start(&cryp->in_walk, in_sg); 1734 /* In output, jump after assoc data */ 1735 scatterwalk_start_at_pos(&cryp->out_walk, out_sg, 1736 areq->assoclen); 1737 1738 ret = stm32_cryp_hw_init(cryp); 1739 if (ret) 1740 return ret; 1741 1742 ret = stm32_cryp_aead_prepare(cryp, in_sg, out_sg); 1743 } 1744 1745 return ret; 1746 } 1747 1748 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) 1749 { 1750 struct skcipher_request *req = container_of(areq, 1751 struct skcipher_request, 1752 base); 1753 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 1754 crypto_skcipher_reqtfm(req)); 1755 struct stm32_cryp *cryp = ctx->cryp; 1756 int ret; 1757 1758 if (!cryp) 1759 return -ENODEV; 1760 1761 ret = stm32_cryp_prepare_req(req, NULL); 1762 if (ret) 1763 return ret; 1764 1765 if (cryp->flags & FLG_IN_OUT_DMA) 1766 ret = stm32_cryp_dma_start(cryp); 1767 else 1768 ret = stm32_cryp_it_start(cryp); 1769 1770 if (ret == -ETIMEDOUT) 1771 stm32_cryp_finish_req(cryp, ret); 1772 1773 return ret; 1774 } 1775 1776 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) 1777 { 1778 struct aead_request *req = container_of(areq, struct aead_request, 1779 base); 1780 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1781 struct stm32_cryp *cryp = ctx->cryp; 1782 int err; 1783 1784 if (!cryp) 1785 return -ENODEV; 1786 1787 err = stm32_cryp_prepare_req(NULL, req); 1788 if (err) 1789 return err; 1790 1791 if (!stm32_cryp_get_input_text_len(cryp) && !cryp->header_in && 1792 !(cryp->flags & FLG_HEADER_DMA)) { 1793 /* No input data to process: get tag and finish */ 1794 stm32_cryp_finish_req(cryp, 0); 1795 return 0; 1796 } 1797 1798 if (cryp->flags & FLG_HEADER_DMA) 1799 return stm32_cryp_header_dma_start(cryp); 1800 1801 if (!cryp->header_in && cryp->flags & FLG_IN_OUT_DMA) 1802 return stm32_cryp_dma_start(cryp); 1803 1804 return stm32_cryp_it_start(cryp); 1805 } 1806 1807 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) 1808 { 1809 u32 cfg, size_bit; 1810 unsigned int i; 1811 int ret = 0; 1812 1813 /* Update Config */ 1814 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1815 1816 cfg &= ~CR_PH_MASK; 1817 cfg |= CR_PH_FINAL; 1818 cfg &= ~CR_DEC_NOT_ENC; 1819 cfg |= CR_CRYPEN; 1820 1821 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1822 1823 if (is_gcm(cryp)) { 1824 /* GCM: write aad and payload size (in bits) */ 1825 size_bit = cryp->areq->assoclen * 8; 1826 if (cryp->caps->swap_final) 1827 size_bit = (__force u32)cpu_to_be32(size_bit); 1828 1829 stm32_cryp_write(cryp, cryp->caps->din, 0); 1830 stm32_cryp_write(cryp, cryp->caps->din, size_bit); 1831 1832 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : 1833 cryp->areq->cryptlen - cryp->authsize; 1834 size_bit *= 8; 1835 if (cryp->caps->swap_final) 1836 size_bit = (__force u32)cpu_to_be32(size_bit); 1837 1838 stm32_cryp_write(cryp, cryp->caps->din, 0); 1839 stm32_cryp_write(cryp, cryp->caps->din, size_bit); 1840 } else { 1841 /* CCM: write CTR0 */ 1842 u32 iv32[AES_BLOCK_32]; 1843 u8 *iv = (u8 *)iv32; 1844 __be32 *biv = (__be32 *)iv32; 1845 1846 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 1847 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 1848 1849 for (i = 0; i < AES_BLOCK_32; i++) { 1850 u32 xiv = iv32[i]; 1851 1852 if (!cryp->caps->padding_wa) 1853 xiv = be32_to_cpu(biv[i]); 1854 stm32_cryp_write(cryp, cryp->caps->din, xiv); 1855 } 1856 } 1857 1858 /* Wait for output data */ 1859 ret = stm32_cryp_wait_output(cryp); 1860 if (ret) { 1861 dev_err(cryp->dev, "Timeout (read tag)\n"); 1862 return ret; 1863 } 1864 1865 if (is_encrypt(cryp)) { 1866 u32 out_tag[AES_BLOCK_32]; 1867 1868 /* Get and write tag */ 1869 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); 1870 memcpy_to_scatterwalk(&cryp->out_walk, out_tag, cryp->authsize); 1871 } else { 1872 /* Get and check tag */ 1873 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; 1874 1875 memcpy_from_scatterwalk(in_tag, &cryp->in_walk, cryp->authsize); 1876 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); 1877 1878 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) 1879 ret = -EBADMSG; 1880 } 1881 1882 /* Disable cryp */ 1883 cfg &= ~CR_CRYPEN; 1884 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1885 1886 return ret; 1887 } 1888 1889 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) 1890 { 1891 u32 cr; 1892 1893 if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { 1894 /* 1895 * In this case, we need to increment manually the ctr counter, 1896 * as HW doesn't handle the U32 carry. 1897 */ 1898 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); 1899 1900 cr = stm32_cryp_read(cryp, cryp->caps->cr); 1901 stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN); 1902 1903 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); 1904 1905 stm32_cryp_write(cryp, cryp->caps->cr, cr); 1906 } 1907 1908 /* The IV registers are BE */ 1909 cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); 1910 cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); 1911 cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); 1912 cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); 1913 } 1914 1915 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) 1916 { 1917 u32 block[AES_BLOCK_32]; 1918 1919 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1920 memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 1921 cryp->payload_out)); 1922 cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 1923 } 1924 1925 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) 1926 { 1927 u32 block[AES_BLOCK_32] = {0}; 1928 1929 memcpy_from_scatterwalk(block, &cryp->in_walk, min(cryp->hw_blocksize, 1930 cryp->payload_in)); 1931 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); 1932 cryp->payload_in -= min(cryp->hw_blocksize, cryp->payload_in); 1933 } 1934 1935 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) 1936 { 1937 int err; 1938 u32 cfg, block[AES_BLOCK_32] = {0}; 1939 unsigned int i; 1940 1941 /* 'Special workaround' procedure described in the datasheet */ 1942 1943 /* a) disable ip */ 1944 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 1945 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1946 cfg &= ~CR_CRYPEN; 1947 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1948 1949 /* b) Update IV1R */ 1950 stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2); 1951 1952 /* c) change mode to CTR */ 1953 cfg &= ~CR_ALGO_MASK; 1954 cfg |= CR_AES_CTR; 1955 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1956 1957 /* a) enable IP */ 1958 cfg |= CR_CRYPEN; 1959 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1960 1961 /* b) pad and write the last block */ 1962 stm32_cryp_irq_write_block(cryp); 1963 /* wait end of process */ 1964 err = stm32_cryp_wait_output(cryp); 1965 if (err) { 1966 dev_err(cryp->dev, "Timeout (write gcm last data)\n"); 1967 return stm32_cryp_finish_req(cryp, err); 1968 } 1969 1970 /* c) get and store encrypted data */ 1971 /* 1972 * Same code as stm32_cryp_irq_read_data(), but we want to store 1973 * block value 1974 */ 1975 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1976 1977 memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 1978 cryp->payload_out)); 1979 cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 1980 1981 /* d) change mode back to AES GCM */ 1982 cfg &= ~CR_ALGO_MASK; 1983 cfg |= CR_AES_GCM; 1984 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1985 1986 /* e) change phase to Final */ 1987 cfg &= ~CR_PH_MASK; 1988 cfg |= CR_PH_FINAL; 1989 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1990 1991 /* f) write padded data */ 1992 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 1993 1994 /* g) Empty fifo out */ 1995 err = stm32_cryp_wait_output(cryp); 1996 if (err) { 1997 dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); 1998 return stm32_cryp_finish_req(cryp, err); 1999 } 2000 2001 for (i = 0; i < AES_BLOCK_32; i++) 2002 stm32_cryp_read(cryp, cryp->caps->dout); 2003 2004 /* h) run the he normal Final phase */ 2005 stm32_cryp_finish_req(cryp, 0); 2006 } 2007 2008 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) 2009 { 2010 u32 cfg; 2011 2012 /* disable ip, set NPBLB and reneable ip */ 2013 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 2014 cfg &= ~CR_CRYPEN; 2015 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2016 2017 cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; 2018 cfg |= CR_CRYPEN; 2019 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2020 } 2021 2022 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) 2023 { 2024 int err = 0; 2025 u32 cfg, iv1tmp; 2026 u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32]; 2027 u32 block[AES_BLOCK_32] = {0}; 2028 unsigned int i; 2029 2030 /* 'Special workaround' procedure described in the datasheet */ 2031 2032 /* a) disable ip */ 2033 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 2034 2035 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 2036 cfg &= ~CR_CRYPEN; 2037 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2038 2039 /* b) get IV1 from CRYP_CSGCMCCM7 */ 2040 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); 2041 2042 /* c) Load CRYP_CSGCMCCMxR */ 2043 for (i = 0; i < ARRAY_SIZE(cstmp1); i++) 2044 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 2045 2046 /* d) Write IV1R */ 2047 stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp); 2048 2049 /* e) change mode to CTR */ 2050 cfg &= ~CR_ALGO_MASK; 2051 cfg |= CR_AES_CTR; 2052 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2053 2054 /* a) enable IP */ 2055 cfg |= CR_CRYPEN; 2056 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2057 2058 /* b) pad and write the last block */ 2059 stm32_cryp_irq_write_block(cryp); 2060 /* wait end of process */ 2061 err = stm32_cryp_wait_output(cryp); 2062 if (err) { 2063 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); 2064 return stm32_cryp_finish_req(cryp, err); 2065 } 2066 2067 /* c) get and store decrypted data */ 2068 /* 2069 * Same code as stm32_cryp_irq_read_data(), but we want to store 2070 * block value 2071 */ 2072 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 2073 2074 memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 2075 cryp->payload_out)); 2076 cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 2077 2078 /* d) Load again CRYP_CSGCMCCMxR */ 2079 for (i = 0; i < ARRAY_SIZE(cstmp2); i++) 2080 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 2081 2082 /* e) change mode back to AES CCM */ 2083 cfg &= ~CR_ALGO_MASK; 2084 cfg |= CR_AES_CCM; 2085 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2086 2087 /* f) change phase to header */ 2088 cfg &= ~CR_PH_MASK; 2089 cfg |= CR_PH_HEADER; 2090 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 2091 2092 /* g) XOR and write padded data */ 2093 for (i = 0; i < ARRAY_SIZE(block); i++) { 2094 block[i] ^= cstmp1[i]; 2095 block[i] ^= cstmp2[i]; 2096 stm32_cryp_write(cryp, cryp->caps->din, block[i]); 2097 } 2098 2099 /* h) wait for completion */ 2100 err = stm32_cryp_wait_busy(cryp); 2101 if (err) 2102 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); 2103 2104 /* i) run the he normal Final phase */ 2105 stm32_cryp_finish_req(cryp, err); 2106 } 2107 2108 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) 2109 { 2110 if (unlikely(!cryp->payload_in)) { 2111 dev_warn(cryp->dev, "No more data to process\n"); 2112 return; 2113 } 2114 2115 if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && 2116 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && 2117 is_encrypt(cryp))) { 2118 /* Padding for AES GCM encryption */ 2119 if (cryp->caps->padding_wa) { 2120 /* Special case 1 */ 2121 stm32_cryp_irq_write_gcm_padded_data(cryp); 2122 return; 2123 } 2124 2125 /* Setting padding bytes (NBBLB) */ 2126 stm32_cryp_irq_set_npblb(cryp); 2127 } 2128 2129 if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && 2130 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && 2131 is_decrypt(cryp))) { 2132 /* Padding for AES CCM decryption */ 2133 if (cryp->caps->padding_wa) { 2134 /* Special case 2 */ 2135 stm32_cryp_irq_write_ccm_padded_data(cryp); 2136 return; 2137 } 2138 2139 /* Setting padding bytes (NBBLB) */ 2140 stm32_cryp_irq_set_npblb(cryp); 2141 } 2142 2143 if (is_aes(cryp) && is_ctr(cryp)) 2144 stm32_cryp_check_ctr_counter(cryp); 2145 2146 stm32_cryp_irq_write_block(cryp); 2147 } 2148 2149 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) 2150 { 2151 u32 block[AES_BLOCK_32] = {0}; 2152 size_t written; 2153 2154 written = min(AES_BLOCK_SIZE, cryp->header_in); 2155 2156 memcpy_from_scatterwalk(block, &cryp->in_walk, written); 2157 2158 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 2159 2160 cryp->header_in -= written; 2161 2162 stm32_crypt_gcmccm_end_header(cryp); 2163 } 2164 2165 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) 2166 { 2167 struct stm32_cryp *cryp = arg; 2168 u32 ph; 2169 u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc); 2170 2171 if (cryp->irq_status & MISR_OUT) 2172 /* Output FIFO IRQ: read data */ 2173 stm32_cryp_irq_read_data(cryp); 2174 2175 if (cryp->irq_status & MISR_IN) { 2176 if (is_gcm(cryp) || is_ccm(cryp)) { 2177 ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK; 2178 if (unlikely(ph == CR_PH_HEADER)) 2179 /* Write Header */ 2180 stm32_cryp_irq_write_gcmccm_header(cryp); 2181 else 2182 /* Input FIFO IRQ: write data */ 2183 stm32_cryp_irq_write_data(cryp); 2184 if (is_gcm(cryp)) 2185 cryp->gcm_ctr++; 2186 } else { 2187 /* Input FIFO IRQ: write data */ 2188 stm32_cryp_irq_write_data(cryp); 2189 } 2190 } 2191 2192 /* Mask useless interrupts */ 2193 if (!cryp->payload_in && !cryp->header_in) 2194 it_mask &= ~IMSCR_IN; 2195 if (!cryp->payload_out) 2196 it_mask &= ~IMSCR_OUT; 2197 stm32_cryp_write(cryp, cryp->caps->imsc, it_mask); 2198 2199 if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) { 2200 local_bh_disable(); 2201 stm32_cryp_finish_req(cryp, 0); 2202 local_bh_enable(); 2203 } 2204 2205 return IRQ_HANDLED; 2206 } 2207 2208 static irqreturn_t stm32_cryp_irq(int irq, void *arg) 2209 { 2210 struct stm32_cryp *cryp = arg; 2211 2212 cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis); 2213 2214 return IRQ_WAKE_THREAD; 2215 } 2216 2217 static int stm32_cryp_dma_init(struct stm32_cryp *cryp) 2218 { 2219 struct dma_slave_config dma_conf; 2220 struct dma_chan *chan; 2221 int ret; 2222 2223 memset(&dma_conf, 0, sizeof(dma_conf)); 2224 2225 dma_conf.direction = DMA_MEM_TO_DEV; 2226 dma_conf.dst_addr = cryp->phys_base + cryp->caps->din; 2227 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2228 dma_conf.dst_maxburst = CRYP_DMA_BURST_REG; 2229 dma_conf.device_fc = false; 2230 2231 chan = dma_request_chan(cryp->dev, "in"); 2232 if (IS_ERR(chan)) 2233 return PTR_ERR(chan); 2234 2235 cryp->dma_lch_in = chan; 2236 ret = dmaengine_slave_config(cryp->dma_lch_in, &dma_conf); 2237 if (ret) { 2238 dma_release_channel(cryp->dma_lch_in); 2239 cryp->dma_lch_in = NULL; 2240 dev_err(cryp->dev, "Couldn't configure DMA in slave.\n"); 2241 return ret; 2242 } 2243 2244 memset(&dma_conf, 0, sizeof(dma_conf)); 2245 2246 dma_conf.direction = DMA_DEV_TO_MEM; 2247 dma_conf.src_addr = cryp->phys_base + cryp->caps->dout; 2248 dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2249 dma_conf.src_maxburst = CRYP_DMA_BURST_REG; 2250 dma_conf.device_fc = false; 2251 2252 chan = dma_request_chan(cryp->dev, "out"); 2253 if (IS_ERR(chan)) { 2254 dma_release_channel(cryp->dma_lch_in); 2255 cryp->dma_lch_in = NULL; 2256 return PTR_ERR(chan); 2257 } 2258 2259 cryp->dma_lch_out = chan; 2260 2261 ret = dmaengine_slave_config(cryp->dma_lch_out, &dma_conf); 2262 if (ret) { 2263 dma_release_channel(cryp->dma_lch_out); 2264 cryp->dma_lch_out = NULL; 2265 dev_err(cryp->dev, "Couldn't configure DMA out slave.\n"); 2266 dma_release_channel(cryp->dma_lch_in); 2267 cryp->dma_lch_in = NULL; 2268 return ret; 2269 } 2270 2271 init_completion(&cryp->dma_completion); 2272 2273 return 0; 2274 } 2275 2276 static struct skcipher_engine_alg crypto_algs[] = { 2277 { 2278 .base = { 2279 .base.cra_name = "ecb(aes)", 2280 .base.cra_driver_name = "stm32-ecb-aes", 2281 .base.cra_priority = 300, 2282 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2283 .base.cra_blocksize = AES_BLOCK_SIZE, 2284 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2285 .base.cra_alignmask = 0, 2286 .base.cra_module = THIS_MODULE, 2287 2288 .init = stm32_cryp_init_tfm, 2289 .min_keysize = AES_MIN_KEY_SIZE, 2290 .max_keysize = AES_MAX_KEY_SIZE, 2291 .setkey = stm32_cryp_aes_setkey, 2292 .encrypt = stm32_cryp_aes_ecb_encrypt, 2293 .decrypt = stm32_cryp_aes_ecb_decrypt, 2294 }, 2295 .op = { 2296 .do_one_request = stm32_cryp_cipher_one_req, 2297 }, 2298 }, 2299 { 2300 .base = { 2301 .base.cra_name = "cbc(aes)", 2302 .base.cra_driver_name = "stm32-cbc-aes", 2303 .base.cra_priority = 300, 2304 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2305 .base.cra_blocksize = AES_BLOCK_SIZE, 2306 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2307 .base.cra_alignmask = 0, 2308 .base.cra_module = THIS_MODULE, 2309 2310 .init = stm32_cryp_init_tfm, 2311 .min_keysize = AES_MIN_KEY_SIZE, 2312 .max_keysize = AES_MAX_KEY_SIZE, 2313 .ivsize = AES_BLOCK_SIZE, 2314 .setkey = stm32_cryp_aes_setkey, 2315 .encrypt = stm32_cryp_aes_cbc_encrypt, 2316 .decrypt = stm32_cryp_aes_cbc_decrypt, 2317 }, 2318 .op = { 2319 .do_one_request = stm32_cryp_cipher_one_req, 2320 }, 2321 }, 2322 { 2323 .base = { 2324 .base.cra_name = "ctr(aes)", 2325 .base.cra_driver_name = "stm32-ctr-aes", 2326 .base.cra_priority = 300, 2327 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2328 .base.cra_blocksize = 1, 2329 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2330 .base.cra_alignmask = 0, 2331 .base.cra_module = THIS_MODULE, 2332 2333 .init = stm32_cryp_init_tfm, 2334 .min_keysize = AES_MIN_KEY_SIZE, 2335 .max_keysize = AES_MAX_KEY_SIZE, 2336 .ivsize = AES_BLOCK_SIZE, 2337 .setkey = stm32_cryp_aes_setkey, 2338 .encrypt = stm32_cryp_aes_ctr_encrypt, 2339 .decrypt = stm32_cryp_aes_ctr_decrypt, 2340 }, 2341 .op = { 2342 .do_one_request = stm32_cryp_cipher_one_req, 2343 }, 2344 }, 2345 { 2346 .base = { 2347 .base.cra_name = "ecb(des)", 2348 .base.cra_driver_name = "stm32-ecb-des", 2349 .base.cra_priority = 300, 2350 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2351 .base.cra_blocksize = DES_BLOCK_SIZE, 2352 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2353 .base.cra_alignmask = 0, 2354 .base.cra_module = THIS_MODULE, 2355 2356 .init = stm32_cryp_init_tfm, 2357 .min_keysize = DES_BLOCK_SIZE, 2358 .max_keysize = DES_BLOCK_SIZE, 2359 .setkey = stm32_cryp_des_setkey, 2360 .encrypt = stm32_cryp_des_ecb_encrypt, 2361 .decrypt = stm32_cryp_des_ecb_decrypt, 2362 }, 2363 .op = { 2364 .do_one_request = stm32_cryp_cipher_one_req, 2365 }, 2366 }, 2367 { 2368 .base = { 2369 .base.cra_name = "cbc(des)", 2370 .base.cra_driver_name = "stm32-cbc-des", 2371 .base.cra_priority = 300, 2372 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2373 .base.cra_blocksize = DES_BLOCK_SIZE, 2374 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2375 .base.cra_alignmask = 0, 2376 .base.cra_module = THIS_MODULE, 2377 2378 .init = stm32_cryp_init_tfm, 2379 .min_keysize = DES_BLOCK_SIZE, 2380 .max_keysize = DES_BLOCK_SIZE, 2381 .ivsize = DES_BLOCK_SIZE, 2382 .setkey = stm32_cryp_des_setkey, 2383 .encrypt = stm32_cryp_des_cbc_encrypt, 2384 .decrypt = stm32_cryp_des_cbc_decrypt, 2385 }, 2386 .op = { 2387 .do_one_request = stm32_cryp_cipher_one_req, 2388 }, 2389 }, 2390 { 2391 .base = { 2392 .base.cra_name = "ecb(des3_ede)", 2393 .base.cra_driver_name = "stm32-ecb-des3", 2394 .base.cra_priority = 300, 2395 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2396 .base.cra_blocksize = DES_BLOCK_SIZE, 2397 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2398 .base.cra_alignmask = 0, 2399 .base.cra_module = THIS_MODULE, 2400 2401 .init = stm32_cryp_init_tfm, 2402 .min_keysize = 3 * DES_BLOCK_SIZE, 2403 .max_keysize = 3 * DES_BLOCK_SIZE, 2404 .setkey = stm32_cryp_tdes_setkey, 2405 .encrypt = stm32_cryp_tdes_ecb_encrypt, 2406 .decrypt = stm32_cryp_tdes_ecb_decrypt, 2407 }, 2408 .op = { 2409 .do_one_request = stm32_cryp_cipher_one_req, 2410 }, 2411 }, 2412 { 2413 .base = { 2414 .base.cra_name = "cbc(des3_ede)", 2415 .base.cra_driver_name = "stm32-cbc-des3", 2416 .base.cra_priority = 300, 2417 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2418 .base.cra_blocksize = DES_BLOCK_SIZE, 2419 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2420 .base.cra_alignmask = 0, 2421 .base.cra_module = THIS_MODULE, 2422 2423 .init = stm32_cryp_init_tfm, 2424 .min_keysize = 3 * DES_BLOCK_SIZE, 2425 .max_keysize = 3 * DES_BLOCK_SIZE, 2426 .ivsize = DES_BLOCK_SIZE, 2427 .setkey = stm32_cryp_tdes_setkey, 2428 .encrypt = stm32_cryp_tdes_cbc_encrypt, 2429 .decrypt = stm32_cryp_tdes_cbc_decrypt, 2430 }, 2431 .op = { 2432 .do_one_request = stm32_cryp_cipher_one_req, 2433 }, 2434 }, 2435 }; 2436 2437 static struct aead_engine_alg aead_algs[] = { 2438 { 2439 .base.setkey = stm32_cryp_aes_aead_setkey, 2440 .base.setauthsize = stm32_cryp_aes_gcm_setauthsize, 2441 .base.encrypt = stm32_cryp_aes_gcm_encrypt, 2442 .base.decrypt = stm32_cryp_aes_gcm_decrypt, 2443 .base.init = stm32_cryp_aes_aead_init, 2444 .base.ivsize = 12, 2445 .base.maxauthsize = AES_BLOCK_SIZE, 2446 2447 .base.base = { 2448 .cra_name = "gcm(aes)", 2449 .cra_driver_name = "stm32-gcm-aes", 2450 .cra_priority = 300, 2451 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2452 .cra_blocksize = 1, 2453 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2454 .cra_alignmask = 0, 2455 .cra_module = THIS_MODULE, 2456 }, 2457 .op = { 2458 .do_one_request = stm32_cryp_aead_one_req, 2459 }, 2460 }, 2461 { 2462 .base.setkey = stm32_cryp_aes_aead_setkey, 2463 .base.setauthsize = stm32_cryp_aes_ccm_setauthsize, 2464 .base.encrypt = stm32_cryp_aes_ccm_encrypt, 2465 .base.decrypt = stm32_cryp_aes_ccm_decrypt, 2466 .base.init = stm32_cryp_aes_aead_init, 2467 .base.ivsize = AES_BLOCK_SIZE, 2468 .base.maxauthsize = AES_BLOCK_SIZE, 2469 2470 .base.base = { 2471 .cra_name = "ccm(aes)", 2472 .cra_driver_name = "stm32-ccm-aes", 2473 .cra_priority = 300, 2474 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, 2475 .cra_blocksize = 1, 2476 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 2477 .cra_alignmask = 0, 2478 .cra_module = THIS_MODULE, 2479 }, 2480 .op = { 2481 .do_one_request = stm32_cryp_aead_one_req, 2482 }, 2483 }, 2484 }; 2485 2486 static const struct stm32_cryp_caps ux500_data = { 2487 .aeads_support = false, 2488 .linear_aes_key = true, 2489 .kp_mode = false, 2490 .iv_protection = true, 2491 .swap_final = true, 2492 .padding_wa = true, 2493 .cr = UX500_CRYP_CR, 2494 .sr = UX500_CRYP_SR, 2495 .din = UX500_CRYP_DIN, 2496 .dout = UX500_CRYP_DOUT, 2497 .dmacr = UX500_CRYP_DMACR, 2498 .imsc = UX500_CRYP_IMSC, 2499 .mis = UX500_CRYP_MIS, 2500 .k1l = UX500_CRYP_K1L, 2501 .k1r = UX500_CRYP_K1R, 2502 .k3r = UX500_CRYP_K3R, 2503 .iv0l = UX500_CRYP_IV0L, 2504 .iv0r = UX500_CRYP_IV0R, 2505 .iv1l = UX500_CRYP_IV1L, 2506 .iv1r = UX500_CRYP_IV1R, 2507 }; 2508 2509 static const struct stm32_cryp_caps f7_data = { 2510 .aeads_support = true, 2511 .linear_aes_key = false, 2512 .kp_mode = true, 2513 .iv_protection = false, 2514 .swap_final = true, 2515 .padding_wa = true, 2516 .cr = CRYP_CR, 2517 .sr = CRYP_SR, 2518 .din = CRYP_DIN, 2519 .dout = CRYP_DOUT, 2520 .dmacr = CRYP_DMACR, 2521 .imsc = CRYP_IMSCR, 2522 .mis = CRYP_MISR, 2523 .k1l = CRYP_K1LR, 2524 .k1r = CRYP_K1RR, 2525 .k3r = CRYP_K3RR, 2526 .iv0l = CRYP_IV0LR, 2527 .iv0r = CRYP_IV0RR, 2528 .iv1l = CRYP_IV1LR, 2529 .iv1r = CRYP_IV1RR, 2530 }; 2531 2532 static const struct stm32_cryp_caps mp1_data = { 2533 .aeads_support = true, 2534 .linear_aes_key = false, 2535 .kp_mode = true, 2536 .iv_protection = false, 2537 .swap_final = false, 2538 .padding_wa = false, 2539 .cr = CRYP_CR, 2540 .sr = CRYP_SR, 2541 .din = CRYP_DIN, 2542 .dout = CRYP_DOUT, 2543 .dmacr = CRYP_DMACR, 2544 .imsc = CRYP_IMSCR, 2545 .mis = CRYP_MISR, 2546 .k1l = CRYP_K1LR, 2547 .k1r = CRYP_K1RR, 2548 .k3r = CRYP_K3RR, 2549 .iv0l = CRYP_IV0LR, 2550 .iv0r = CRYP_IV0RR, 2551 .iv1l = CRYP_IV1LR, 2552 .iv1r = CRYP_IV1RR, 2553 }; 2554 2555 static const struct of_device_id stm32_dt_ids[] = { 2556 { .compatible = "stericsson,ux500-cryp", .data = &ux500_data}, 2557 { .compatible = "st,stm32f756-cryp", .data = &f7_data}, 2558 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data}, 2559 {}, 2560 }; 2561 MODULE_DEVICE_TABLE(of, stm32_dt_ids); 2562 2563 static int stm32_cryp_probe(struct platform_device *pdev) 2564 { 2565 struct device *dev = &pdev->dev; 2566 struct stm32_cryp *cryp; 2567 struct reset_control *rst; 2568 int irq, ret; 2569 2570 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); 2571 if (!cryp) 2572 return -ENOMEM; 2573 2574 cryp->caps = of_device_get_match_data(dev); 2575 if (!cryp->caps) 2576 return -ENODEV; 2577 2578 cryp->dev = dev; 2579 2580 cryp->regs = devm_platform_ioremap_resource(pdev, 0); 2581 if (IS_ERR(cryp->regs)) 2582 return PTR_ERR(cryp->regs); 2583 2584 cryp->phys_base = platform_get_resource(pdev, IORESOURCE_MEM, 0)->start; 2585 2586 irq = platform_get_irq(pdev, 0); 2587 if (irq < 0) 2588 return irq; 2589 2590 ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, 2591 stm32_cryp_irq_thread, IRQF_ONESHOT, 2592 dev_name(dev), cryp); 2593 if (ret) { 2594 dev_err(dev, "Cannot grab IRQ\n"); 2595 return ret; 2596 } 2597 2598 cryp->clk = devm_clk_get(dev, NULL); 2599 if (IS_ERR(cryp->clk)) { 2600 dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); 2601 2602 return PTR_ERR(cryp->clk); 2603 } 2604 2605 ret = clk_prepare_enable(cryp->clk); 2606 if (ret) { 2607 dev_err(cryp->dev, "Failed to enable clock\n"); 2608 return ret; 2609 } 2610 2611 pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY); 2612 pm_runtime_use_autosuspend(dev); 2613 2614 pm_runtime_get_noresume(dev); 2615 pm_runtime_set_active(dev); 2616 pm_runtime_enable(dev); 2617 2618 rst = devm_reset_control_get(dev, NULL); 2619 if (IS_ERR(rst)) { 2620 ret = PTR_ERR(rst); 2621 if (ret == -EPROBE_DEFER) 2622 goto err_rst; 2623 } else { 2624 reset_control_assert(rst); 2625 udelay(2); 2626 reset_control_deassert(rst); 2627 } 2628 2629 platform_set_drvdata(pdev, cryp); 2630 2631 ret = stm32_cryp_dma_init(cryp); 2632 switch (ret) { 2633 case 0: 2634 break; 2635 case -ENODEV: 2636 dev_dbg(dev, "DMA mode not available\n"); 2637 break; 2638 default: 2639 goto err_dma; 2640 } 2641 2642 spin_lock(&cryp_list.lock); 2643 list_add(&cryp->list, &cryp_list.dev_list); 2644 spin_unlock(&cryp_list.lock); 2645 2646 /* Initialize crypto engine */ 2647 cryp->engine = crypto_engine_alloc_init(dev, 1); 2648 if (!cryp->engine) { 2649 dev_err(dev, "Could not init crypto engine\n"); 2650 ret = -ENOMEM; 2651 goto err_engine1; 2652 } 2653 2654 ret = crypto_engine_start(cryp->engine); 2655 if (ret) { 2656 dev_err(dev, "Could not start crypto engine\n"); 2657 goto err_engine2; 2658 } 2659 2660 ret = crypto_engine_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2661 if (ret) { 2662 dev_err(dev, "Could not register algs\n"); 2663 goto err_algs; 2664 } 2665 2666 if (cryp->caps->aeads_support) { 2667 ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2668 if (ret) 2669 goto err_aead_algs; 2670 } 2671 2672 dev_info(dev, "Initialized\n"); 2673 2674 pm_runtime_put_sync(dev); 2675 2676 return 0; 2677 2678 err_aead_algs: 2679 crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2680 err_algs: 2681 err_engine2: 2682 crypto_engine_exit(cryp->engine); 2683 err_engine1: 2684 spin_lock(&cryp_list.lock); 2685 list_del(&cryp->list); 2686 spin_unlock(&cryp_list.lock); 2687 2688 if (cryp->dma_lch_in) 2689 dma_release_channel(cryp->dma_lch_in); 2690 if (cryp->dma_lch_out) 2691 dma_release_channel(cryp->dma_lch_out); 2692 err_dma: 2693 err_rst: 2694 pm_runtime_disable(dev); 2695 pm_runtime_put_noidle(dev); 2696 2697 clk_disable_unprepare(cryp->clk); 2698 2699 return ret; 2700 } 2701 2702 static void stm32_cryp_remove(struct platform_device *pdev) 2703 { 2704 struct stm32_cryp *cryp = platform_get_drvdata(pdev); 2705 int ret; 2706 2707 ret = pm_runtime_get_sync(cryp->dev); 2708 2709 if (cryp->caps->aeads_support) 2710 crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2711 crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2712 2713 crypto_engine_exit(cryp->engine); 2714 2715 spin_lock(&cryp_list.lock); 2716 list_del(&cryp->list); 2717 spin_unlock(&cryp_list.lock); 2718 2719 if (cryp->dma_lch_in) 2720 dma_release_channel(cryp->dma_lch_in); 2721 2722 if (cryp->dma_lch_out) 2723 dma_release_channel(cryp->dma_lch_out); 2724 2725 pm_runtime_disable(cryp->dev); 2726 pm_runtime_put_noidle(cryp->dev); 2727 2728 if (ret >= 0) 2729 clk_disable_unprepare(cryp->clk); 2730 } 2731 2732 #ifdef CONFIG_PM 2733 static int stm32_cryp_runtime_suspend(struct device *dev) 2734 { 2735 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2736 2737 clk_disable_unprepare(cryp->clk); 2738 2739 return 0; 2740 } 2741 2742 static int stm32_cryp_runtime_resume(struct device *dev) 2743 { 2744 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2745 int ret; 2746 2747 ret = clk_prepare_enable(cryp->clk); 2748 if (ret) { 2749 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); 2750 return ret; 2751 } 2752 2753 return 0; 2754 } 2755 #endif 2756 2757 static const struct dev_pm_ops stm32_cryp_pm_ops = { 2758 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 2759 pm_runtime_force_resume) 2760 SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend, 2761 stm32_cryp_runtime_resume, NULL) 2762 }; 2763 2764 static struct platform_driver stm32_cryp_driver = { 2765 .probe = stm32_cryp_probe, 2766 .remove = stm32_cryp_remove, 2767 .driver = { 2768 .name = DRIVER_NAME, 2769 .pm = &stm32_cryp_pm_ops, 2770 .of_match_table = stm32_dt_ids, 2771 }, 2772 }; 2773 2774 module_platform_driver(stm32_cryp_driver); 2775 2776 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); 2777 MODULE_DESCRIPTION("STMicroelectronics STM32 CRYP hardware driver"); 2778 MODULE_LICENSE("GPL"); 2779