Lines Matching +full:dma +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2018-2020 Intel Corporation
8 #include <linux/dma-mapping.h>
21 #include "ocs-aes.h"
77 * This bit activates the DMA. When the DMA finishes, it resets
82 * this bit is reset by the DMA.
85 * terminated this bit is reset by the DMA.
132 * 11-bit value, but it is actually 10-bits.
138 * before the tag is written. For 128-bit mode this required delay is 28 OCS
139 * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
157 * ------------ ---------
159 * 1 ... 15-L Nonce N
160 * 16-L ... 15 Counter i
162 * Flags = L' = L - 1
165 #define COUNTER_START(lprime) (16 - ((lprime) + 1))
176 * struct ocs_dma_linked_list - OCS DMA linked list entry.
180 * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
193 * bit [10] - KEY_HI_LO_SWAP
194 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
195 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
196 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
197 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
198 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
199 * bit [4] - IV_SWAP_BYTES_IN_DWORD
200 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
201 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
202 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
203 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
207 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
213 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
220 aes_dev->base_reg + AES_ACTIVE_OFFSET);
226 * Called when DMA is programmed to fetch the last batch of data.
227 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
229 * - For AES-GCM, it is called for the last batch of Plaintext data and
235 aes_dev->base_reg + AES_ACTIVE_OFFSET);
244 aes_active_reg = ioread32(aes_dev->base_reg +
255 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
263 * Called when DMA is programmed to fetch the last batch of Associated Data
269 aes_dev->base_reg + AES_ACTIVE_OFFSET);
272 /* Set DMA src and dst transfer size to 0 */
275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
276 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
279 /* Activate DMA for zero-byte transfer case. */
283 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
286 /* Activate DMA and enable src linked list */
291 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
294 /* Activate DMA and enable dst linked list */
299 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
302 /* Activate DMA and enable src and dst linked lists */
308 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
314 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
316 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
323 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
326 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
329 /* Disable AES and DMA IRQ. */
336 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
337 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
340 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
343 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
345 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
348 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
350 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
352 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
355 /* Enable AES or DMA IRQ. IRQ is disabled once fired. */
359 /* Ensure DMA error interrupts are enabled */
367 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
371 * bits [31:3] - reserved
372 * bit [2] - EN_SKS_ERR
373 * bit [1] - EN_AES_COMPLETE
374 * bit [0] - reserved
376 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
381 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
385 * bits [31:9] - reserved
386 * bit [8] - CPD_ERR_INT_EN
387 * bit [7] - OUTBUF_RD_ERR_INT_EN
388 * bit [6] - OUTBUF_WR_ERR_INT_EN
389 * bit [5] - INBUF_RD_ERR_INT_EN
390 * bit [4] - INBUF_WR_ERR_INT_EN
391 * bit [3] - BAD_COMP_INT_EN
392 * bit [2] - SAI_INT_EN
393 * bit [1] - DST_DONE_INT_EN
394 * bit [0] - SRC_DONE_INT_EN
404 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
408 /* Enable and wait for IRQ (either from OCS AES engine or DMA) */
413 reinit_completion(&aes_dev->irq_completion);
415 rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
419 return aes_dev->dma_err_mask ? -EIO : 0;
422 /* Configure DMA to OCS, linked list mode */
426 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
428 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
431 /* Configure DMA from OCS, linked list mode */
435 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
437 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
445 /* Read DMA ISR status. */
446 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
451 /* Save DMA error status. */
452 aes_dev->dma_err_mask = aes_dma_isr &
462 complete(&aes_dev->irq_completion);
468 * ocs_aes_set_key() - Write key into OCS AES hardware.
485 /* OCS AES supports 128-bit and 256-bit keys only. */
487 dev_err(aes_dev->dev,
488 "%d-bit keys not supported by AES cipher\n",
490 return -EINVAL;
492 /* OCS SM4 supports 128-bit keys only. */
494 dev_err(aes_dev->dev,
495 "%d-bit keys not supported for SM4 cipher\n",
497 return -EINVAL;
501 return -EINVAL;
505 /* Write key to AES_KEY[0-7] registers */
508 aes_dev->base_reg + AES_KEY_0_OFFSET +
513 * bits [31:1] - reserved
514 * bit [0] - AES_KEY_SIZE
515 * 0 - 128 bit key
516 * 1 - 256 bit key
519 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
534 * bit [14] - CIPHER_SELECT
535 * 0 - AES
536 * 1 - SM4
537 * bits [11:8] - OCS_AES_MODE
538 * 0000 - ECB
539 * 0001 - CBC
540 * 0010 - CTR
541 * 0110 - CCM
542 * 0111 - GCM
543 * 1001 - CTS
544 * bits [7:6] - AES_INSTRUCTION
545 * 00 - ENCRYPT
546 * 01 - DECRYPT
547 * 10 - EXPAND
548 * 11 - BYPASS
549 * bits [3:2] - CTR_M_BITS
550 * 00 - No increment
551 * 01 - Least significant 32 bits are incremented
552 * 10 - Least significant 64 bits are incremented
553 * 11 - Full 128 bits are incremented
557 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
568 /* Set endianness recommended by data-sheet. */
594 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
599 * If OK return 0; else return -EINVAL.
611 return -EINVAL;
616 return -EINVAL;
620 return -EINVAL;
624 * to destination using DMA.
626 * AES mode is irrelevant, but both source and destination DMA
627 * linked-list must be defined.
632 return -EINVAL;
645 return -EINVAL;
650 return -EINVAL;
657 return -EINVAL;
662 return -EINVAL;
666 return -EINVAL;
673 return -EINVAL;
678 return -EINVAL;
682 return -EINVAL;
689 return -EINVAL;
694 return -EINVAL;
698 return -EINVAL;
705 return -EINVAL;
713 return -EINVAL;
717 return -EINVAL;
721 return -EINVAL;
725 return -EINVAL;
732 return -EINVAL;
737 return -EINVAL;
741 return -EINVAL;
745 return -EINVAL;
754 return -EINVAL;
758 return -EINVAL;
770 return -EINVAL;
774 return -EINVAL;
779 return -EINVAL;
784 * ocs_aes_op() - Perform AES/SM4 operation.
789 * @dst_dma_list: The OCS DMA list mapping output memory.
790 * @src_dma_list: The OCS DMA list mapping input payload data.
820 return -EINVAL;
828 /* Write the byte length of the last data block to engine. */
834 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
835 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
836 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
837 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
843 /* Configure and activate input / output DMA. */
850 * For CTS mode, instruct engine to activate ciphertext
859 /* Wait for engine to complete processing. */
866 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
867 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
868 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
869 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
875 /* Compute and write J0 to engine registers. */
885 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
886 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
887 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
888 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
891 /* Read GCM tag from engine registers. */
902 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
903 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
904 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
905 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
911 * ocs_aes_gcm_op() - Perform GCM operation.
915 * @dst_dma_list: The OCS DMA list mapping output memory.
916 * @src_dma_list: The OCS DMA list mapping input payload data.
919 * @aad_dma_list: The OCS DMA list mapping input AAD data.
956 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
964 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
966 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
971 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
973 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
980 /* If aad present, configure DMA to feed it to the engine. */
984 /* Instructs engine to pad last block of aad, if needed. */
987 /* Wait for DMA transfer to complete. */
1001 /* Configure and activate DMA for both input and output data. */
1010 /* Instruct AES/SMA4 engine payload processing is over. */
1013 /* Wait for OCS AES engine to complete processing. */
1023 /* Write encrypted tag to AES/SM4 engine. */
1029 /* Ensure DMA input buffer is empty */
1034 * ciphertext before the tag is written. So delay needed after DMA has
1041 /* Write encrypted tag to AES/SM4 engine. */
1043 iowrite8(in_tag[i], aes_dev->base_reg +
1051 * Note: B0 format is documented in NIST Special Publication 800-38C
1052 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1069 * bit 5-3: t value encoded as (t-2)/2
1070 * bit 2-0: q value encoded as q - 1
1078 * encoded as (t - 2) / 2
1080 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
1084 * q - 1 == iv[0] & 0x7;
1088 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
1089 * and must be copied to b0[1]..b0[15-q].
1093 for (i = 1; i <= 15 - q; i++)
1097 * Q is encoded in q octets, in big-endian order, so to write it, we
1100 i = sizeof(b0) - 1;
1104 i--;
1105 q--;
1112 return -EOVERFLOW;
1115 iowrite8(b0[i], aes_dev->base_reg +
1123 * Note: adata len encoding is documented in NIST Special Publication 800-38C
1124 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1135 * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets
1137 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
1156 aes_dev->base_reg +
1179 /* Configure the AES/SM4 DMA to fetch the Associated Data */
1182 /* Activate DMA to fetch Associated data. */
1188 /* Wait for DMA transfer to complete. */
1208 * Configure and activate DMA for both input and output
1215 /* Configure and activate DMA for output data only. */
1222 * AES/SM4 engine to pad the last block of data.
1236 /* Let engine process 0-length input. */
1245 * Configure and activate DMA for both input and output
1253 * AES/SM4 engine to differentiate between encrypted data and
1258 * Enable DMA DONE interrupt; once DMA transfer is over,
1279 tag[i] = ioread32(aes_dev->base_reg +
1281 yr[i] = ioread32(aes_dev->base_reg +
1286 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
1290 * ocs_aes_ccm_op() - Perform CCM operation.
1294 * @dst_dma_list: The OCS DMA list mapping output memory.
1295 * @src_dma_list: The OCS DMA list mapping input payload data.
1298 * @adata_dma_list: The OCS DMA list mapping input A-data.
1334 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
1346 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
1348 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
1350 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
1352 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
1355 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
1393 /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
1403 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
1405 * @sg: The SG list OCS DMA linked list will be created from. When
1408 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
1410 * @dll_desc: The OCS DMA dma_list to use to store information about the
1413 * OCS DMA linked list.
1431 return -EINVAL;
1434 dll_desc->vaddr = NULL;
1435 dll_desc->dma_addr = DMA_MAPPING_ERROR;
1436 dll_desc->size = 0;
1443 data_offset -= sg_dma_len(sg);
1444 sg_dma_count--;
1448 return -EINVAL;
1451 /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
1458 return -EINVAL;
1464 return -EINVAL;
1466 /* Allocate the DMA list, one entry for each SG entry. */
1467 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
1468 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
1469 &dll_desc->dma_addr, GFP_KERNEL);
1470 if (!dll_desc->vaddr)
1471 return -ENOMEM;
1473 /* Populate DMA linked list entries. */
1474 ll = dll_desc->vaddr;
1477 ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size);
1479 data_size -= ll[i].src_len;
1480 /* Current element points to the DMA address of the next one. */
1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
1485 ll[i - 1].next = 0;
1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;