1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel Keem Bay OCS AES Crypto Driver. 4 * 5 * Copyright (C) 2018-2020 Intel Corporation 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/interrupt.h> 10 #include <linux/kernel.h> 11 #include <linux/platform_device.h> 12 #include <linux/slab.h> 13 #include <linux/swab.h> 14 15 #include <asm/byteorder.h> 16 #include <asm/errno.h> 17 18 #include <crypto/aes.h> 19 #include <crypto/gcm.h> 20 21 #include "ocs-aes.h" 22 23 #define AES_COMMAND_OFFSET 0x0000 24 #define AES_KEY_0_OFFSET 0x0004 25 #define AES_KEY_1_OFFSET 0x0008 26 #define AES_KEY_2_OFFSET 0x000C 27 #define AES_KEY_3_OFFSET 0x0010 28 #define AES_KEY_4_OFFSET 0x0014 29 #define AES_KEY_5_OFFSET 0x0018 30 #define AES_KEY_6_OFFSET 0x001C 31 #define AES_KEY_7_OFFSET 0x0020 32 #define AES_IV_0_OFFSET 0x0024 33 #define AES_IV_1_OFFSET 0x0028 34 #define AES_IV_2_OFFSET 0x002C 35 #define AES_IV_3_OFFSET 0x0030 36 #define AES_ACTIVE_OFFSET 0x0034 37 #define AES_STATUS_OFFSET 0x0038 38 #define AES_KEY_SIZE_OFFSET 0x0044 39 #define AES_IER_OFFSET 0x0048 40 #define AES_ISR_OFFSET 0x005C 41 #define AES_MULTIPURPOSE1_0_OFFSET 0x0200 42 #define AES_MULTIPURPOSE1_1_OFFSET 0x0204 43 #define AES_MULTIPURPOSE1_2_OFFSET 0x0208 44 #define AES_MULTIPURPOSE1_3_OFFSET 0x020C 45 #define AES_MULTIPURPOSE2_0_OFFSET 0x0220 46 #define AES_MULTIPURPOSE2_1_OFFSET 0x0224 47 #define AES_MULTIPURPOSE2_2_OFFSET 0x0228 48 #define AES_MULTIPURPOSE2_3_OFFSET 0x022C 49 #define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 50 #define AES_TLEN_OFFSET 0x0300 51 #define AES_T_MAC_0_OFFSET 0x0304 52 #define AES_T_MAC_1_OFFSET 0x0308 53 #define AES_T_MAC_2_OFFSET 0x030C 54 #define AES_T_MAC_3_OFFSET 0x0310 55 #define AES_PLEN_OFFSET 0x0314 56 #define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 57 #define AES_A_DMA_DST_ADDR_OFFSET 0x0404 58 #define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 59 #define AES_A_DMA_DST_SIZE_OFFSET 0x040C 60 #define AES_A_DMA_DMA_MODE_OFFSET 0x0410 61 #define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 62 #define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C 63 #define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 64 #define AES_A_DMA_LOG_OFFSET 0x0424 65 #define AES_A_DMA_STATUS_OFFSET 0x0428 66 #define AES_A_DMA_PERF_CNTR_OFFSET 0x042C 67 #define AES_A_DMA_MSI_ISR_OFFSET 0x0480 68 #define AES_A_DMA_MSI_IER_OFFSET 0x0484 69 #define AES_A_DMA_MSI_MASK_OFFSET 0x0488 70 #define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 71 #define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 72 73 /* 74 * AES_A_DMA_DMA_MODE register. 75 * Default: 0x00000000. 76 * bit[31] ACTIVE 77 * This bit activates the DMA. When the DMA finishes, it resets 78 * this bit to zero. 79 * bit[30:26] Unused by this driver. 80 * bit[25] SRC_LINK_LIST_EN 81 * Source link list enable bit. When the linked list is terminated 82 * this bit is reset by the DMA. 83 * bit[24] DST_LINK_LIST_EN 84 * Destination link list enable bit. When the linked list is 85 * terminated this bit is reset by the DMA. 86 * bit[23:0] Unused by this driver. 87 */ 88 #define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) 89 #define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) 90 #define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) 91 92 /* 93 * AES_ACTIVE register 94 * default 0x00000000 95 * bit[31:10] Reserved 96 * bit[9] LAST_ADATA 97 * bit[8] LAST_GCX 98 * bit[7:2] Reserved 99 * bit[1] TERMINATION 100 * bit[0] TRIGGER 101 */ 102 #define AES_ACTIVE_LAST_ADATA BIT(9) 103 #define AES_ACTIVE_LAST_CCM_GCM BIT(8) 104 #define AES_ACTIVE_TERMINATION BIT(1) 105 #define AES_ACTIVE_TRIGGER BIT(0) 106 107 #define AES_DISABLE_INT 0x00000000 108 #define AES_DMA_CPD_ERR_INT BIT(8) 109 #define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) 110 #define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) 111 #define AES_DMA_INBUF_RD_ERR_INT BIT(5) 112 #define AES_DMA_INBUF_WR_ERR_INT BIT(4) 113 #define AES_DMA_BAD_COMP_INT BIT(3) 114 #define AES_DMA_SAI_INT BIT(2) 115 #define AES_DMA_SRC_DONE_INT BIT(0) 116 #define AES_COMPLETE_INT BIT(1) 117 118 #define AES_DMA_MSI_MASK_CLEAR BIT(0) 119 120 #define AES_128_BIT_KEY 0x00000000 121 #define AES_256_BIT_KEY BIT(0) 122 123 #define AES_DEACTIVATE_PERF_CNTR 0x00000000 124 #define AES_ACTIVATE_PERF_CNTR BIT(0) 125 126 #define AES_MAX_TAG_SIZE_U32 4 127 128 #define OCS_LL_DMA_FLAG_TERMINATE BIT(31) 129 130 /* 131 * There is an inconsistency in the documentation. This is documented as a 132 * 11-bit value, but it is actually 10-bits. 133 */ 134 #define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF 135 136 /* 137 * During CCM decrypt, the OCS block needs to finish processing the ciphertext 138 * before the tag is written. For 128-bit mode this required delay is 28 OCS 139 * clock cycles. For 256-bit mode it is 36 OCS clock cycles. 140 */ 141 #define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL 142 143 /* 144 * During CCM decrypt there must be a delay of at least 42 OCS clock cycles 145 * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM 146 * bit in the same register (as stated in the OCS databook) 147 */ 148 #define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL 149 150 /* See RFC3610 section 2.2 */ 151 #define L_PRIME_MIN (1) 152 #define L_PRIME_MAX (7) 153 /* 154 * CCM IV format from RFC 3610 section 2.3 155 * 156 * Octet Number Contents 157 * ------------ --------- 158 * 0 Flags 159 * 1 ... 15-L Nonce N 160 * 16-L ... 15 Counter i 161 * 162 * Flags = L' = L - 1 163 */ 164 #define L_PRIME_IDX 0 165 #define COUNTER_START(lprime) (16 - ((lprime) + 1)) 166 #define COUNTER_LEN(lprime) ((lprime) + 1) 167 168 enum aes_counter_mode { 169 AES_CTR_M_NO_INC = 0, 170 AES_CTR_M_32_INC = 1, 171 AES_CTR_M_64_INC = 2, 172 AES_CTR_M_128_INC = 3, 173 }; 174 175 /** 176 * struct ocs_dma_linked_list - OCS DMA linked list entry. 177 * @src_addr: Source address of the data. 178 * @src_len: Length of data to be fetched. 179 * @next: Next dma_list to fetch. 180 * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. 181 */ 182 struct ocs_dma_linked_list { 183 u32 src_addr; 184 u32 src_len; 185 u32 next; 186 u32 ll_flags; 187 } __packed; 188 189 /* 190 * Set endianness of inputs and outputs 191 * AES_BYTE_ORDER_CFG 192 * default 0x00000000 193 * bit [10] - KEY_HI_LO_SWAP 194 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD 195 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD 196 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD 197 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD 198 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD 199 * bit [4] - IV_SWAP_BYTES_IN_DWORD 200 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD 201 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD 202 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD 203 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD 204 */ 205 static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) 206 { 207 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); 208 } 209 210 /* Trigger AES process start. */ 211 static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) 212 { 213 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); 214 } 215 216 /* Indicate last bulk of data. */ 217 static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) 218 { 219 iowrite32(AES_ACTIVE_TERMINATION, 220 aes_dev->base_reg + AES_ACTIVE_OFFSET); 221 } 222 223 /* 224 * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. 225 * 226 * Called when DMA is programmed to fetch the last batch of data. 227 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext 228 * data. 229 * - For AES-GCM, it is called for the last batch of Plaintext data and 230 * Ciphertext data. 231 */ 232 static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) 233 { 234 iowrite32(AES_ACTIVE_LAST_CCM_GCM, 235 aes_dev->base_reg + AES_ACTIVE_OFFSET); 236 } 237 238 /* Wait for LAST_CCM_GCM bit to be unset. */ 239 static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) 240 { 241 u32 aes_active_reg; 242 243 do { 244 aes_active_reg = ioread32(aes_dev->base_reg + 245 AES_ACTIVE_OFFSET); 246 } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); 247 } 248 249 /* Wait for 10 bits of input occupancy. */ 250 static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) 251 { 252 u32 reg; 253 254 do { 255 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); 256 } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); 257 } 258 259 /* 260 * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all 261 * other bits). 262 * 263 * Called when DMA is programmed to fetch the last batch of Associated Data 264 * (CCM case) or Additional Authenticated Data (GCM case). 265 */ 266 static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) 267 { 268 iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, 269 aes_dev->base_reg + AES_ACTIVE_OFFSET); 270 } 271 272 /* Set DMA src and dst transfer size to 0 */ 273 static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) 274 { 275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); 276 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); 277 } 278 279 /* Activate DMA for zero-byte transfer case. */ 280 static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) 281 { 282 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, 283 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 284 } 285 286 /* Activate DMA and enable src linked list */ 287 static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) 288 { 289 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 290 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, 291 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 292 } 293 294 /* Activate DMA and enable dst linked list */ 295 static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) 296 { 297 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 298 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, 299 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 300 } 301 302 /* Activate DMA and enable src and dst linked lists */ 303 static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) 304 { 305 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 306 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | 307 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, 308 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 309 } 310 311 /* Reset PERF_CNTR to 0 and activate it */ 312 static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) 313 { 314 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); 315 iowrite32(AES_ACTIVATE_PERF_CNTR, 316 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); 317 } 318 319 /* Wait until PERF_CNTR is > delay, then deactivate it */ 320 static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, 321 int delay) 322 { 323 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) 324 ; 325 iowrite32(AES_DEACTIVATE_PERF_CNTR, 326 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); 327 } 328 329 /* Disable AES and DMA IRQ. */ 330 static void aes_irq_disable(struct ocs_aes_dev *aes_dev) 331 { 332 u32 isr_val = 0; 333 334 /* Disable interrupts */ 335 iowrite32(AES_DISABLE_INT, 336 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 337 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); 338 339 /* Clear any pending interrupt */ 340 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 341 if (isr_val) 342 iowrite32(isr_val, 343 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 344 345 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); 346 if (isr_val) 347 iowrite32(isr_val, 348 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); 349 350 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); 351 if (isr_val) 352 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); 353 } 354 355 /* Enable AES or DMA IRQ. IRQ is disabled once fired. */ 356 static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) 357 { 358 if (irq == AES_COMPLETE_INT) { 359 /* Ensure DMA error interrupts are enabled */ 360 iowrite32(AES_DMA_CPD_ERR_INT | 361 AES_DMA_OUTBUF_RD_ERR_INT | 362 AES_DMA_OUTBUF_WR_ERR_INT | 363 AES_DMA_INBUF_RD_ERR_INT | 364 AES_DMA_INBUF_WR_ERR_INT | 365 AES_DMA_BAD_COMP_INT | 366 AES_DMA_SAI_INT, 367 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 368 /* 369 * AES_IER 370 * default 0x00000000 371 * bits [31:3] - reserved 372 * bit [2] - EN_SKS_ERR 373 * bit [1] - EN_AES_COMPLETE 374 * bit [0] - reserved 375 */ 376 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); 377 return; 378 } 379 if (irq == AES_DMA_SRC_DONE_INT) { 380 /* Ensure AES interrupts are disabled */ 381 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); 382 /* 383 * DMA_MSI_IER 384 * default 0x00000000 385 * bits [31:9] - reserved 386 * bit [8] - CPD_ERR_INT_EN 387 * bit [7] - OUTBUF_RD_ERR_INT_EN 388 * bit [6] - OUTBUF_WR_ERR_INT_EN 389 * bit [5] - INBUF_RD_ERR_INT_EN 390 * bit [4] - INBUF_WR_ERR_INT_EN 391 * bit [3] - BAD_COMP_INT_EN 392 * bit [2] - SAI_INT_EN 393 * bit [1] - DST_DONE_INT_EN 394 * bit [0] - SRC_DONE_INT_EN 395 */ 396 iowrite32(AES_DMA_CPD_ERR_INT | 397 AES_DMA_OUTBUF_RD_ERR_INT | 398 AES_DMA_OUTBUF_WR_ERR_INT | 399 AES_DMA_INBUF_RD_ERR_INT | 400 AES_DMA_INBUF_WR_ERR_INT | 401 AES_DMA_BAD_COMP_INT | 402 AES_DMA_SAI_INT | 403 AES_DMA_SRC_DONE_INT, 404 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 405 } 406 } 407 408 /* Enable and wait for IRQ (either from OCS AES engine or DMA) */ 409 static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) 410 { 411 int rc; 412 413 reinit_completion(&aes_dev->irq_completion); 414 aes_irq_enable(aes_dev, irq); 415 rc = wait_for_completion_interruptible(&aes_dev->irq_completion); 416 if (rc) 417 return rc; 418 419 return aes_dev->dma_err_mask ? -EIO : 0; 420 } 421 422 /* Configure DMA to OCS, linked list mode */ 423 static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, 424 dma_addr_t dma_list) 425 { 426 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); 427 iowrite32(dma_list, 428 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); 429 } 430 431 /* Configure DMA from OCS, linked list mode */ 432 static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, 433 dma_addr_t dma_list) 434 { 435 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); 436 iowrite32(dma_list, 437 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); 438 } 439 440 irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) 441 { 442 struct ocs_aes_dev *aes_dev = dev_id; 443 u32 aes_dma_isr; 444 445 /* Read DMA ISR status. */ 446 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 447 448 /* Disable and clear interrupts. */ 449 aes_irq_disable(aes_dev); 450 451 /* Save DMA error status. */ 452 aes_dev->dma_err_mask = aes_dma_isr & 453 (AES_DMA_CPD_ERR_INT | 454 AES_DMA_OUTBUF_RD_ERR_INT | 455 AES_DMA_OUTBUF_WR_ERR_INT | 456 AES_DMA_INBUF_RD_ERR_INT | 457 AES_DMA_INBUF_WR_ERR_INT | 458 AES_DMA_BAD_COMP_INT | 459 AES_DMA_SAI_INT); 460 461 /* Signal IRQ completion. */ 462 complete(&aes_dev->irq_completion); 463 464 return IRQ_HANDLED; 465 } 466 467 /** 468 * ocs_aes_set_key() - Write key into OCS AES hardware. 469 * @aes_dev: The OCS AES device to write the key to. 470 * @key_size: The size of the key (in bytes). 471 * @key: The key to write. 472 * @cipher: The cipher the key is for. 473 * 474 * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. 475 * 476 * Return: 0 on success, negative error code otherwise. 477 */ 478 int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, 479 enum ocs_cipher cipher) 480 { 481 const u32 *key_u32; 482 u32 val; 483 int i; 484 485 /* OCS AES supports 128-bit and 256-bit keys only. */ 486 if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { 487 dev_err(aes_dev->dev, 488 "%d-bit keys not supported by AES cipher\n", 489 key_size * 8); 490 return -EINVAL; 491 } 492 /* OCS SM4 supports 128-bit keys only. */ 493 if (cipher == OCS_SM4 && key_size != 16) { 494 dev_err(aes_dev->dev, 495 "%d-bit keys not supported for SM4 cipher\n", 496 key_size * 8); 497 return -EINVAL; 498 } 499 500 if (!key) 501 return -EINVAL; 502 503 key_u32 = (const u32 *)key; 504 505 /* Write key to AES_KEY[0-7] registers */ 506 for (i = 0; i < (key_size / sizeof(u32)); i++) { 507 iowrite32(key_u32[i], 508 aes_dev->base_reg + AES_KEY_0_OFFSET + 509 (i * sizeof(u32))); 510 } 511 /* 512 * Write key size 513 * bits [31:1] - reserved 514 * bit [0] - AES_KEY_SIZE 515 * 0 - 128 bit key 516 * 1 - 256 bit key 517 */ 518 val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; 519 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); 520 521 return 0; 522 } 523 524 /* Write AES_COMMAND */ 525 static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, 526 enum ocs_cipher cipher, 527 enum ocs_mode mode, 528 enum ocs_instruction instruction) 529 { 530 u32 val; 531 532 /* AES_COMMAND 533 * default 0x000000CC 534 * bit [14] - CIPHER_SELECT 535 * 0 - AES 536 * 1 - SM4 537 * bits [11:8] - OCS_AES_MODE 538 * 0000 - ECB 539 * 0001 - CBC 540 * 0010 - CTR 541 * 0110 - CCM 542 * 0111 - GCM 543 * 1001 - CTS 544 * bits [7:6] - AES_INSTRUCTION 545 * 00 - ENCRYPT 546 * 01 - DECRYPT 547 * 10 - EXPAND 548 * 11 - BYPASS 549 * bits [3:2] - CTR_M_BITS 550 * 00 - No increment 551 * 01 - Least significant 32 bits are incremented 552 * 10 - Least significant 64 bits are incremented 553 * 11 - Full 128 bits are incremented 554 */ 555 val = (cipher << 14) | (mode << 8) | (instruction << 6) | 556 (AES_CTR_M_128_INC << 2); 557 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); 558 } 559 560 static void ocs_aes_init(struct ocs_aes_dev *aes_dev, 561 enum ocs_mode mode, 562 enum ocs_cipher cipher, 563 enum ocs_instruction instruction) 564 { 565 /* Ensure interrupts are disabled and pending interrupts cleared. */ 566 aes_irq_disable(aes_dev); 567 568 /* Set endianness recommended by data-sheet. */ 569 aes_a_set_endianness(aes_dev); 570 571 /* Set AES_COMMAND register. */ 572 set_ocs_aes_command(aes_dev, cipher, mode, instruction); 573 } 574 575 /* 576 * Write the byte length of the last AES/SM4 block of Payload data (without 577 * zero padding and without the length of the MAC) in register AES_PLEN. 578 */ 579 static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, 580 u32 size) 581 { 582 u32 val; 583 584 if (size == 0) { 585 val = 0; 586 goto exit; 587 } 588 589 val = size % AES_BLOCK_SIZE; 590 if (val == 0) 591 val = AES_BLOCK_SIZE; 592 593 exit: 594 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); 595 } 596 597 /* 598 * Validate inputs according to mode. 599 * If OK return 0; else return -EINVAL. 600 */ 601 static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, 602 const u8 *iv, u32 iv_size, 603 dma_addr_t aad_dma_list, u32 aad_size, 604 const u8 *tag, u32 tag_size, 605 enum ocs_cipher cipher, enum ocs_mode mode, 606 enum ocs_instruction instruction, 607 dma_addr_t dst_dma_list) 608 { 609 /* Ensure cipher, mode and instruction are valid. */ 610 if (!(cipher == OCS_AES || cipher == OCS_SM4)) 611 return -EINVAL; 612 613 if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && 614 mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && 615 mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) 616 return -EINVAL; 617 618 if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && 619 instruction != OCS_EXPAND && instruction != OCS_BYPASS) 620 return -EINVAL; 621 622 /* 623 * When instruction is OCS_BYPASS, OCS simply copies data from source 624 * to destination using DMA. 625 * 626 * AES mode is irrelevant, but both source and destination DMA 627 * linked-list must be defined. 628 */ 629 if (instruction == OCS_BYPASS) { 630 if (src_dma_list == DMA_MAPPING_ERROR || 631 dst_dma_list == DMA_MAPPING_ERROR) 632 return -EINVAL; 633 634 return 0; 635 } 636 637 /* 638 * For performance reasons switch based on mode to limit unnecessary 639 * conditionals for each mode 640 */ 641 switch (mode) { 642 case OCS_MODE_ECB: 643 /* Ensure input length is multiple of block size */ 644 if (src_size % AES_BLOCK_SIZE != 0) 645 return -EINVAL; 646 647 /* Ensure source and destination linked lists are created */ 648 if (src_dma_list == DMA_MAPPING_ERROR || 649 dst_dma_list == DMA_MAPPING_ERROR) 650 return -EINVAL; 651 652 return 0; 653 654 case OCS_MODE_CBC: 655 /* Ensure input length is multiple of block size */ 656 if (src_size % AES_BLOCK_SIZE != 0) 657 return -EINVAL; 658 659 /* Ensure source and destination linked lists are created */ 660 if (src_dma_list == DMA_MAPPING_ERROR || 661 dst_dma_list == DMA_MAPPING_ERROR) 662 return -EINVAL; 663 664 /* Ensure IV is present and block size in length */ 665 if (!iv || iv_size != AES_BLOCK_SIZE) 666 return -EINVAL; 667 668 return 0; 669 670 case OCS_MODE_CTR: 671 /* Ensure input length of 1 byte or greater */ 672 if (src_size == 0) 673 return -EINVAL; 674 675 /* Ensure source and destination linked lists are created */ 676 if (src_dma_list == DMA_MAPPING_ERROR || 677 dst_dma_list == DMA_MAPPING_ERROR) 678 return -EINVAL; 679 680 /* Ensure IV is present and block size in length */ 681 if (!iv || iv_size != AES_BLOCK_SIZE) 682 return -EINVAL; 683 684 return 0; 685 686 case OCS_MODE_CTS: 687 /* Ensure input length >= block size */ 688 if (src_size < AES_BLOCK_SIZE) 689 return -EINVAL; 690 691 /* Ensure source and destination linked lists are created */ 692 if (src_dma_list == DMA_MAPPING_ERROR || 693 dst_dma_list == DMA_MAPPING_ERROR) 694 return -EINVAL; 695 696 /* Ensure IV is present and block size in length */ 697 if (!iv || iv_size != AES_BLOCK_SIZE) 698 return -EINVAL; 699 700 return 0; 701 702 case OCS_MODE_GCM: 703 /* Ensure IV is present and GCM_AES_IV_SIZE in length */ 704 if (!iv || iv_size != GCM_AES_IV_SIZE) 705 return -EINVAL; 706 707 /* 708 * If input data present ensure source and destination linked 709 * lists are created 710 */ 711 if (src_size && (src_dma_list == DMA_MAPPING_ERROR || 712 dst_dma_list == DMA_MAPPING_ERROR)) 713 return -EINVAL; 714 715 /* If aad present ensure aad linked list is created */ 716 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) 717 return -EINVAL; 718 719 /* Ensure tag destination is set */ 720 if (!tag) 721 return -EINVAL; 722 723 /* Just ensure that tag_size doesn't cause overflows. */ 724 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) 725 return -EINVAL; 726 727 return 0; 728 729 case OCS_MODE_CCM: 730 /* Ensure IV is present and block size in length */ 731 if (!iv || iv_size != AES_BLOCK_SIZE) 732 return -EINVAL; 733 734 /* 2 <= L <= 8, so 1 <= L' <= 7 */ 735 if (iv[L_PRIME_IDX] < L_PRIME_MIN || 736 iv[L_PRIME_IDX] > L_PRIME_MAX) 737 return -EINVAL; 738 739 /* If aad present ensure aad linked list is created */ 740 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) 741 return -EINVAL; 742 743 /* Just ensure that tag_size doesn't cause overflows. */ 744 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) 745 return -EINVAL; 746 747 if (instruction == OCS_DECRYPT) { 748 /* 749 * If input data present ensure source and destination 750 * linked lists are created 751 */ 752 if (src_size && (src_dma_list == DMA_MAPPING_ERROR || 753 dst_dma_list == DMA_MAPPING_ERROR)) 754 return -EINVAL; 755 756 /* Ensure input tag is present */ 757 if (!tag) 758 return -EINVAL; 759 760 return 0; 761 } 762 763 /* Instruction == OCS_ENCRYPT */ 764 765 /* 766 * Destination linked list always required (for tag even if no 767 * input data) 768 */ 769 if (dst_dma_list == DMA_MAPPING_ERROR) 770 return -EINVAL; 771 772 /* If input data present ensure src linked list is created */ 773 if (src_size && src_dma_list == DMA_MAPPING_ERROR) 774 return -EINVAL; 775 776 return 0; 777 778 default: 779 return -EINVAL; 780 } 781 } 782 783 /** 784 * ocs_aes_op() - Perform AES/SM4 operation. 785 * @aes_dev: The OCS AES device to use. 786 * @mode: The mode to use (ECB, CBC, CTR, or CTS). 787 * @cipher: The cipher to use (AES or SM4). 788 * @instruction: The instruction to perform (encrypt or decrypt). 789 * @dst_dma_list: The OCS DMA list mapping output memory. 790 * @src_dma_list: The OCS DMA list mapping input payload data. 791 * @src_size: The amount of data mapped by @src_dma_list. 792 * @iv: The IV vector. 793 * @iv_size: The size (in bytes) of @iv. 794 * 795 * Return: 0 on success, negative error code otherwise. 796 */ 797 int ocs_aes_op(struct ocs_aes_dev *aes_dev, 798 enum ocs_mode mode, 799 enum ocs_cipher cipher, 800 enum ocs_instruction instruction, 801 dma_addr_t dst_dma_list, 802 dma_addr_t src_dma_list, 803 u32 src_size, 804 u8 *iv, 805 u32 iv_size) 806 { 807 u32 *iv32; 808 int rc; 809 810 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, 811 NULL, 0, cipher, mode, instruction, 812 dst_dma_list); 813 if (rc) 814 return rc; 815 /* 816 * ocs_aes_validate_inputs() is a generic check, now ensure mode is not 817 * GCM or CCM. 818 */ 819 if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) 820 return -EINVAL; 821 822 /* Cast IV to u32 array. */ 823 iv32 = (u32 *)iv; 824 825 ocs_aes_init(aes_dev, mode, cipher, instruction); 826 827 if (mode == OCS_MODE_CTS) { 828 /* Write the byte length of the last data block to engine. */ 829 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 830 } 831 832 /* ECB is the only mode that doesn't use IV. */ 833 if (mode != OCS_MODE_ECB) { 834 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); 835 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); 836 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); 837 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); 838 } 839 840 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 841 aes_a_op_trigger(aes_dev); 842 843 /* Configure and activate input / output DMA. */ 844 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 845 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 846 aes_a_dma_active_src_dst_ll_en(aes_dev); 847 848 if (mode == OCS_MODE_CTS) { 849 /* 850 * For CTS mode, instruct engine to activate ciphertext 851 * stealing if last block of data is incomplete. 852 */ 853 aes_a_set_last_gcx(aes_dev); 854 } else { 855 /* For all other modes, just write the 'termination' bit. */ 856 aes_a_op_termination(aes_dev); 857 } 858 859 /* Wait for engine to complete processing. */ 860 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 861 if (rc) 862 return rc; 863 864 if (mode == OCS_MODE_CTR) { 865 /* Read back IV for streaming mode */ 866 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); 867 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); 868 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); 869 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); 870 } 871 872 return 0; 873 } 874 875 /* Compute and write J0 to engine registers. */ 876 static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, 877 const u8 *iv) 878 { 879 const u32 *j0 = (u32 *)iv; 880 881 /* 882 * IV must be 12 bytes; Other sizes not supported as Linux crypto API 883 * does only expects/allows 12 byte IV for GCM 884 */ 885 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); 886 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); 887 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); 888 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); 889 } 890 891 /* Read GCM tag from engine registers. */ 892 static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, 893 u8 *tag, u32 tag_size) 894 { 895 u32 tag_u32[AES_MAX_TAG_SIZE_U32]; 896 897 /* 898 * The Authentication Tag T is stored in Little Endian order in the 899 * registers with the most significant bytes stored from AES_T_MAC[3] 900 * downward. 901 */ 902 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); 903 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); 904 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); 905 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); 906 907 memcpy(tag, tag_u32, tag_size); 908 } 909 910 /** 911 * ocs_aes_gcm_op() - Perform GCM operation. 912 * @aes_dev: The OCS AES device to use. 913 * @cipher: The Cipher to use (AES or SM4). 914 * @instruction: The instruction to perform (encrypt or decrypt). 915 * @dst_dma_list: The OCS DMA list mapping output memory. 916 * @src_dma_list: The OCS DMA list mapping input payload data. 917 * @src_size: The amount of data mapped by @src_dma_list. 918 * @iv: The input IV vector. 919 * @aad_dma_list: The OCS DMA list mapping input AAD data. 920 * @aad_size: The amount of data mapped by @aad_dma_list. 921 * @out_tag: Where to store computed tag. 922 * @tag_size: The size (in bytes) of @out_tag. 923 * 924 * Return: 0 on success, negative error code otherwise. 925 */ 926 int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, 927 enum ocs_cipher cipher, 928 enum ocs_instruction instruction, 929 dma_addr_t dst_dma_list, 930 dma_addr_t src_dma_list, 931 u32 src_size, 932 const u8 *iv, 933 dma_addr_t aad_dma_list, 934 u32 aad_size, 935 u8 *out_tag, 936 u32 tag_size) 937 { 938 u64 bit_len; 939 u32 val; 940 int rc; 941 942 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, 943 GCM_AES_IV_SIZE, aad_dma_list, 944 aad_size, out_tag, tag_size, cipher, 945 OCS_MODE_GCM, instruction, 946 dst_dma_list); 947 if (rc) 948 return rc; 949 950 ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); 951 952 /* Compute and write J0 to OCS HW. */ 953 ocs_aes_gcm_write_j0(aes_dev, iv); 954 955 /* Write out_tag byte length */ 956 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); 957 958 /* Write the byte length of the last plaintext / ciphertext block. */ 959 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 960 961 /* Write ciphertext bit length */ 962 bit_len = (u64)src_size * 8; 963 val = bit_len & 0xFFFFFFFF; 964 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); 965 val = bit_len >> 32; 966 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); 967 968 /* Write aad bit length */ 969 bit_len = (u64)aad_size * 8; 970 val = bit_len & 0xFFFFFFFF; 971 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); 972 val = bit_len >> 32; 973 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); 974 975 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 976 aes_a_op_trigger(aes_dev); 977 978 /* Process AAD. */ 979 if (aad_size) { 980 /* If aad present, configure DMA to feed it to the engine. */ 981 dma_to_ocs_aes_ll(aes_dev, aad_dma_list); 982 aes_a_dma_active_src_ll_en(aes_dev); 983 984 /* Instructs engine to pad last block of aad, if needed. */ 985 aes_a_set_last_gcx_and_adata(aes_dev); 986 987 /* Wait for DMA transfer to complete. */ 988 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 989 if (rc) 990 return rc; 991 } else { 992 aes_a_set_last_gcx_and_adata(aes_dev); 993 } 994 995 /* Wait until adata (if present) has been processed. */ 996 aes_a_wait_last_gcx(aes_dev); 997 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 998 999 /* Now process payload. */ 1000 if (src_size) { 1001 /* Configure and activate DMA for both input and output data. */ 1002 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1003 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1004 aes_a_dma_active_src_dst_ll_en(aes_dev); 1005 } else { 1006 aes_a_dma_set_xfer_size_zero(aes_dev); 1007 aes_a_dma_active(aes_dev); 1008 } 1009 1010 /* Instruct AES/SMA4 engine payload processing is over. */ 1011 aes_a_set_last_gcx(aes_dev); 1012 1013 /* Wait for OCS AES engine to complete processing. */ 1014 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1015 if (rc) 1016 return rc; 1017 1018 ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); 1019 1020 return 0; 1021 } 1022 1023 /* Write encrypted tag to AES/SM4 engine. */ 1024 static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, 1025 const u8 *in_tag, u32 tag_size) 1026 { 1027 int i; 1028 1029 /* Ensure DMA input buffer is empty */ 1030 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 1031 1032 /* 1033 * During CCM decrypt, the OCS block needs to finish processing the 1034 * ciphertext before the tag is written. So delay needed after DMA has 1035 * completed writing the ciphertext 1036 */ 1037 aes_a_dma_reset_and_activate_perf_cntr(aes_dev); 1038 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, 1039 CCM_DECRYPT_DELAY_TAG_CLK_COUNT); 1040 1041 /* Write encrypted tag to AES/SM4 engine. */ 1042 for (i = 0; i < tag_size; i++) { 1043 iowrite8(in_tag[i], aes_dev->base_reg + 1044 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1045 } 1046 } 1047 1048 /* 1049 * Write B0 CCM block to OCS AES HW. 1050 * 1051 * Note: B0 format is documented in NIST Special Publication 800-38C 1052 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf 1053 * (see Section A.2.1) 1054 */ 1055 static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, 1056 const u8 *iv, u32 adata_size, u32 tag_size, 1057 u32 cryptlen) 1058 { 1059 u8 b0[16]; /* CCM B0 block is 16 bytes long. */ 1060 int i, q; 1061 1062 /* Initialize B0 to 0. */ 1063 memset(b0, 0, sizeof(b0)); 1064 1065 /* 1066 * B0[0] is the 'Flags Octet' and has the following structure: 1067 * bit 7: Reserved 1068 * bit 6: Adata flag 1069 * bit 5-3: t value encoded as (t-2)/2 1070 * bit 2-0: q value encoded as q - 1 1071 */ 1072 /* If there is AAD data, set the Adata flag. */ 1073 if (adata_size) 1074 b0[0] |= BIT(6); 1075 /* 1076 * t denotes the octet length of T. 1077 * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is 1078 * encoded as (t - 2) / 2 1079 */ 1080 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; 1081 /* 1082 * q is the octet length of Q. 1083 * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as 1084 * q - 1 == iv[0] & 0x7; 1085 */ 1086 b0[0] |= iv[0] & 0x7; 1087 /* 1088 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] 1089 * and must be copied to b0[1]..b0[15-q]. 1090 * q == (iv[0] & 0x7) + 1 1091 */ 1092 q = (iv[0] & 0x7) + 1; 1093 for (i = 1; i <= 15 - q; i++) 1094 b0[i] = iv[i]; 1095 /* 1096 * The rest of B0 must contain Q, i.e., the message length. 1097 * Q is encoded in q octets, in big-endian order, so to write it, we 1098 * start from the end of B0 and we move backward. 1099 */ 1100 i = sizeof(b0) - 1; 1101 while (q) { 1102 b0[i] = cryptlen & 0xff; 1103 cryptlen >>= 8; 1104 i--; 1105 q--; 1106 } 1107 /* 1108 * If cryptlen is not zero at this point, it means that its original 1109 * value was too big. 1110 */ 1111 if (cryptlen) 1112 return -EOVERFLOW; 1113 /* Now write B0 to OCS AES input buffer. */ 1114 for (i = 0; i < sizeof(b0); i++) 1115 iowrite8(b0[i], aes_dev->base_reg + 1116 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1117 return 0; 1118 } 1119 1120 /* 1121 * Write adata length to OCS AES HW. 1122 * 1123 * Note: adata len encoding is documented in NIST Special Publication 800-38C 1124 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf 1125 * (see Section A.2.2) 1126 */ 1127 static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, 1128 u64 adata_len) 1129 { 1130 u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ 1131 int i, len; 1132 1133 /* 1134 * adata_len ('a') is encoded as follows: 1135 * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets 1136 * (big endian). 1137 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, 1138 * i.e., six octets (big endian). 1139 * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, 1140 * i.e., ten octets (big endian). 1141 */ 1142 if (adata_len < 65280) { 1143 len = 2; 1144 *(__be16 *)enc_a = cpu_to_be16(adata_len); 1145 } else if (adata_len <= 0xFFFFFFFF) { 1146 len = 6; 1147 *(__be16 *)enc_a = cpu_to_be16(0xfffe); 1148 *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); 1149 } else { /* adata_len >= 2^32 */ 1150 len = 10; 1151 *(__be16 *)enc_a = cpu_to_be16(0xffff); 1152 *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); 1153 } 1154 for (i = 0; i < len; i++) 1155 iowrite8(enc_a[i], 1156 aes_dev->base_reg + 1157 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1158 } 1159 1160 static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, 1161 dma_addr_t adata_dma_list, u32 adata_size) 1162 { 1163 int rc; 1164 1165 if (!adata_size) { 1166 /* Since no aad the LAST_GCX bit can be set now */ 1167 aes_a_set_last_gcx_and_adata(aes_dev); 1168 goto exit; 1169 } 1170 1171 /* Adata case. */ 1172 1173 /* 1174 * Form the encoding of the Associated data length and write it 1175 * to the AES/SM4 input buffer. 1176 */ 1177 ocs_aes_ccm_write_adata_len(aes_dev, adata_size); 1178 1179 /* Configure the AES/SM4 DMA to fetch the Associated Data */ 1180 dma_to_ocs_aes_ll(aes_dev, adata_dma_list); 1181 1182 /* Activate DMA to fetch Associated data. */ 1183 aes_a_dma_active_src_ll_en(aes_dev); 1184 1185 /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ 1186 aes_a_set_last_gcx_and_adata(aes_dev); 1187 1188 /* Wait for DMA transfer to complete. */ 1189 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 1190 if (rc) 1191 return rc; 1192 1193 exit: 1194 /* Wait until adata (if present) has been processed. */ 1195 aes_a_wait_last_gcx(aes_dev); 1196 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 1197 1198 return 0; 1199 } 1200 1201 static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, 1202 dma_addr_t dst_dma_list, 1203 dma_addr_t src_dma_list, 1204 u32 src_size) 1205 { 1206 if (src_size) { 1207 /* 1208 * Configure and activate DMA for both input and output 1209 * data. 1210 */ 1211 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1212 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1213 aes_a_dma_active_src_dst_ll_en(aes_dev); 1214 } else { 1215 /* Configure and activate DMA for output data only. */ 1216 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1217 aes_a_dma_active_dst_ll_en(aes_dev); 1218 } 1219 1220 /* 1221 * Set the LAST GCX bit in AES_ACTIVE Register to instruct 1222 * AES/SM4 engine to pad the last block of data. 1223 */ 1224 aes_a_set_last_gcx(aes_dev); 1225 1226 /* We are done, wait for IRQ and return. */ 1227 return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1228 } 1229 1230 static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, 1231 dma_addr_t dst_dma_list, 1232 dma_addr_t src_dma_list, 1233 u32 src_size) 1234 { 1235 if (!src_size) { 1236 /* Let engine process 0-length input. */ 1237 aes_a_dma_set_xfer_size_zero(aes_dev); 1238 aes_a_dma_active(aes_dev); 1239 aes_a_set_last_gcx(aes_dev); 1240 1241 return 0; 1242 } 1243 1244 /* 1245 * Configure and activate DMA for both input and output 1246 * data. 1247 */ 1248 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1249 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1250 aes_a_dma_active_src_dst_ll_en(aes_dev); 1251 /* 1252 * Set the LAST GCX bit in AES_ACTIVE Register; this allows the 1253 * AES/SM4 engine to differentiate between encrypted data and 1254 * encrypted MAC. 1255 */ 1256 aes_a_set_last_gcx(aes_dev); 1257 /* 1258 * Enable DMA DONE interrupt; once DMA transfer is over, 1259 * interrupt handler will process the MAC/tag. 1260 */ 1261 return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 1262 } 1263 1264 /* 1265 * Compare Tag to Yr. 1266 * 1267 * Only used at the end of CCM decrypt. If tag == yr, message authentication 1268 * has succeeded. 1269 */ 1270 static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, 1271 u8 tag_size_bytes) 1272 { 1273 u32 tag[AES_MAX_TAG_SIZE_U32]; 1274 u32 yr[AES_MAX_TAG_SIZE_U32]; 1275 u8 i; 1276 1277 /* Read Tag and Yr from AES registers. */ 1278 for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { 1279 tag[i] = ioread32(aes_dev->base_reg + 1280 AES_T_MAC_0_OFFSET + (i * sizeof(u32))); 1281 yr[i] = ioread32(aes_dev->base_reg + 1282 AES_MULTIPURPOSE2_0_OFFSET + 1283 (i * sizeof(u32))); 1284 } 1285 1286 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; 1287 } 1288 1289 /** 1290 * ocs_aes_ccm_op() - Perform CCM operation. 1291 * @aes_dev: The OCS AES device to use. 1292 * @cipher: The Cipher to use (AES or SM4). 1293 * @instruction: The instruction to perform (encrypt or decrypt). 1294 * @dst_dma_list: The OCS DMA list mapping output memory. 1295 * @src_dma_list: The OCS DMA list mapping input payload data. 1296 * @src_size: The amount of data mapped by @src_dma_list. 1297 * @iv: The input IV vector. 1298 * @adata_dma_list: The OCS DMA list mapping input A-data. 1299 * @adata_size: The amount of data mapped by @adata_dma_list. 1300 * @in_tag: Input tag. 1301 * @tag_size: The size (in bytes) of @in_tag. 1302 * 1303 * Note: for encrypt the tag is appended to the ciphertext (in the memory 1304 * mapped by @dst_dma_list). 1305 * 1306 * Return: 0 on success, negative error code otherwise. 1307 */ 1308 int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, 1309 enum ocs_cipher cipher, 1310 enum ocs_instruction instruction, 1311 dma_addr_t dst_dma_list, 1312 dma_addr_t src_dma_list, 1313 u32 src_size, 1314 u8 *iv, 1315 dma_addr_t adata_dma_list, 1316 u32 adata_size, 1317 u8 *in_tag, 1318 u32 tag_size) 1319 { 1320 u32 *iv_32; 1321 u8 lprime; 1322 int rc; 1323 1324 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, 1325 AES_BLOCK_SIZE, adata_dma_list, adata_size, 1326 in_tag, tag_size, cipher, OCS_MODE_CCM, 1327 instruction, dst_dma_list); 1328 if (rc) 1329 return rc; 1330 1331 ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); 1332 1333 /* 1334 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt 1335 * auth tag so ensure this is the case 1336 */ 1337 lprime = iv[L_PRIME_IDX]; 1338 memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); 1339 1340 /* 1341 * Nonce is already converted to ctr0 before being passed into this 1342 * function as iv. 1343 */ 1344 iv_32 = (u32 *)iv; 1345 iowrite32(__swab32(iv_32[0]), 1346 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); 1347 iowrite32(__swab32(iv_32[1]), 1348 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); 1349 iowrite32(__swab32(iv_32[2]), 1350 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); 1351 iowrite32(__swab32(iv_32[3]), 1352 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); 1353 1354 /* Write MAC/tag length in register AES_TLEN */ 1355 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); 1356 /* 1357 * Write the byte length of the last AES/SM4 block of Payload data 1358 * (without zero padding and without the length of the MAC) in register 1359 * AES_PLEN. 1360 */ 1361 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 1362 1363 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 1364 aes_a_op_trigger(aes_dev); 1365 1366 aes_a_dma_reset_and_activate_perf_cntr(aes_dev); 1367 1368 /* Form block B0 and write it to the AES/SM4 input buffer. */ 1369 rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); 1370 if (rc) 1371 return rc; 1372 /* 1373 * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 1374 * clock cycles since TRIGGER bit was set 1375 */ 1376 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, 1377 CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); 1378 1379 /* Process Adata. */ 1380 ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); 1381 1382 /* For Encrypt case we just process the payload and return. */ 1383 if (instruction == OCS_ENCRYPT) { 1384 return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, 1385 src_dma_list, src_size); 1386 } 1387 /* For Decypt we need to process the payload and then the tag. */ 1388 rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, 1389 src_dma_list, src_size); 1390 if (rc) 1391 return rc; 1392 1393 /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ 1394 ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); 1395 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1396 if (rc) 1397 return rc; 1398 1399 return ccm_compare_tag_to_yr(aes_dev, tag_size); 1400 } 1401 1402 /** 1403 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. 1404 * @aes_dev: The OCS AES device the list will be created for. 1405 * @sg: The SG list OCS DMA linked list will be created from. When 1406 * passed to this function, @sg must have been already mapped 1407 * with dma_map_sg(). 1408 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the 1409 * value returned by dma_map_sg() when @sg was mapped. 1410 * @dll_desc: The OCS DMA dma_list to use to store information about the 1411 * created linked list. 1412 * @data_size: The size of the data (from the SG list) to be mapped into the 1413 * OCS DMA linked list. 1414 * @data_offset: The offset (within the SG list) of the data to be mapped. 1415 * 1416 * Return: 0 on success, negative error code otherwise. 1417 */ 1418 int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, 1419 struct scatterlist *sg, 1420 int sg_dma_count, 1421 struct ocs_dll_desc *dll_desc, 1422 size_t data_size, size_t data_offset) 1423 { 1424 struct ocs_dma_linked_list *ll = NULL; 1425 struct scatterlist *sg_tmp; 1426 unsigned int tmp; 1427 int dma_nents; 1428 int i; 1429 1430 if (!dll_desc || !sg || !aes_dev) 1431 return -EINVAL; 1432 1433 /* Default values for when no ddl_desc is created. */ 1434 dll_desc->vaddr = NULL; 1435 dll_desc->dma_addr = DMA_MAPPING_ERROR; 1436 dll_desc->size = 0; 1437 1438 if (data_size == 0) 1439 return 0; 1440 1441 /* Loop over sg_list until we reach entry at specified offset. */ 1442 while (data_offset >= sg_dma_len(sg)) { 1443 data_offset -= sg_dma_len(sg); 1444 sg_dma_count--; 1445 sg = sg_next(sg); 1446 /* If we reach the end of the list, offset was invalid. */ 1447 if (!sg || sg_dma_count == 0) 1448 return -EINVAL; 1449 } 1450 1451 /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ 1452 dma_nents = 0; 1453 tmp = 0; 1454 sg_tmp = sg; 1455 while (tmp < data_offset + data_size) { 1456 /* If we reach the end of the list, data_size was invalid. */ 1457 if (!sg_tmp) 1458 return -EINVAL; 1459 tmp += sg_dma_len(sg_tmp); 1460 dma_nents++; 1461 sg_tmp = sg_next(sg_tmp); 1462 } 1463 if (dma_nents > sg_dma_count) 1464 return -EINVAL; 1465 1466 /* Allocate the DMA list, one entry for each SG entry. */ 1467 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; 1468 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, 1469 &dll_desc->dma_addr, GFP_KERNEL); 1470 if (!dll_desc->vaddr) 1471 return -ENOMEM; 1472 1473 /* Populate DMA linked list entries. */ 1474 ll = dll_desc->vaddr; 1475 for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { 1476 ll[i].src_addr = sg_dma_address(sg) + data_offset; 1477 ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size); 1478 data_offset = 0; 1479 data_size -= ll[i].src_len; 1480 /* Current element points to the DMA address of the next one. */ 1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); 1482 ll[i].ll_flags = 0; 1483 } 1484 /* Terminate last element. */ 1485 ll[i - 1].next = 0; 1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; 1487 1488 return 0; 1489 } 1490