1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ccm.h> 10 #include <crypto/aes-ctr.h> 11 #include <crypto/aes-ecb.h> 12 #include <crypto/aes-gcm.h> 13 #include <crypto/aes-xts.h> 14 #include <crypto/aes.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/utils.h> 17 #include <linux/cache.h> 18 #include <linux/crypto.h> 19 #include <linux/export.h> 20 #include <linux/module.h> 21 #include <linux/unaligned.h> 22 #include "fips-aes.h" 23 24 static const u8 ____cacheline_aligned aes_sbox[] = { 25 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 26 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 27 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 28 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 29 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 30 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 31 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 32 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 33 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 34 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 35 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 36 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 37 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 38 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 39 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 40 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 41 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 42 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 43 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 44 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 45 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 46 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 47 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 48 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 49 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 50 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 51 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 52 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 53 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 54 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 55 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 56 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 57 }; 58 59 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 60 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 61 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 62 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 63 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 64 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 65 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 66 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 67 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 68 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 69 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 70 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 71 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 72 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 73 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 74 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 75 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 76 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 77 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 78 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 79 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 80 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 81 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 82 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 83 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 84 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 85 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 86 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 87 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 88 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 89 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 90 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 91 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 92 }; 93 94 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 95 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 96 97 EXPORT_SYMBOL(crypto_aes_sbox); 98 EXPORT_SYMBOL(crypto_aes_inv_sbox); 99 100 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 101 const u32 ____cacheline_aligned aes_enc_tab[256] = { 102 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 103 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 104 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 105 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 106 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 107 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 108 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 109 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 110 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 111 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 112 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 113 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 114 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 115 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 116 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 117 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 118 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 119 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 120 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 121 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 122 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 123 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 124 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 125 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 126 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 127 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 128 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 129 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 130 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 131 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 132 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 133 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 134 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 135 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 136 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 137 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 138 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 139 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 140 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 141 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 142 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 143 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 144 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 145 }; 146 EXPORT_SYMBOL(aes_enc_tab); 147 148 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 149 const u32 ____cacheline_aligned aes_dec_tab[256] = { 150 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 151 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 152 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 153 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 154 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 155 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 156 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 157 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 158 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 159 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 160 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 161 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 162 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 163 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 164 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 165 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 166 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 167 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 168 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 169 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 170 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 171 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 172 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 173 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 174 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 175 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 176 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 177 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 178 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 179 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 180 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 181 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 182 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 183 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 184 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 185 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 186 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 187 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 188 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 189 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 190 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 191 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 192 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 193 }; 194 EXPORT_SYMBOL(aes_dec_tab); 195 196 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 197 static __always_inline void aes_prefetch(const void *mem, size_t len) 198 { 199 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 200 *(volatile const u8 *)(mem + i); 201 barrier(); 202 } 203 204 static u32 mul_by_x(u32 w) 205 { 206 u32 x = w & 0x7f7f7f7f; 207 u32 y = w & 0x80808080; 208 209 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 210 return (x << 1) ^ (y >> 7) * 0x1b; 211 } 212 213 static u32 mul_by_x2(u32 w) 214 { 215 u32 x = w & 0x3f3f3f3f; 216 u32 y = w & 0x80808080; 217 u32 z = w & 0x40404040; 218 219 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 220 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 221 } 222 223 static u32 mix_columns(u32 x) 224 { 225 /* 226 * Perform the following matrix multiplication in GF(2^8) 227 * 228 * | 0x2 0x3 0x1 0x1 | | x[0] | 229 * | 0x1 0x2 0x3 0x1 | | x[1] | 230 * | 0x1 0x1 0x2 0x3 | x | x[2] | 231 * | 0x3 0x1 0x1 0x2 | | x[3] | 232 */ 233 u32 y = mul_by_x(x) ^ ror32(x, 16); 234 235 return y ^ ror32(x ^ y, 8); 236 } 237 238 static u32 inv_mix_columns(u32 x) 239 { 240 /* 241 * Perform the following matrix multiplication in GF(2^8) 242 * 243 * | 0xe 0xb 0xd 0x9 | | x[0] | 244 * | 0x9 0xe 0xb 0xd | | x[1] | 245 * | 0xd 0x9 0xe 0xb | x | x[2] | 246 * | 0xb 0xd 0x9 0xe | | x[3] | 247 * 248 * which can conveniently be reduced to 249 * 250 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 251 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 252 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 253 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 254 */ 255 u32 y = mul_by_x2(x); 256 257 return mix_columns(x ^ y ^ ror32(y, 16)); 258 } 259 260 static u32 subw(u32 in) 261 { 262 return (aes_sbox[in & 0xff]) ^ 263 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 264 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 265 (aes_sbox[(in >> 24) & 0xff] << 24); 266 } 267 268 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 269 const u8 *in_key, int key_len) 270 { 271 u32 kwords = key_len / sizeof(u32); 272 u32 rc, i, j; 273 274 for (i = 0; i < kwords; i++) 275 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 276 277 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 278 u32 *rki = &rndkeys[i * kwords]; 279 u32 *rko = rki + kwords; 280 281 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 282 rko[1] = rko[0] ^ rki[1]; 283 rko[2] = rko[1] ^ rki[2]; 284 rko[3] = rko[2] ^ rki[3]; 285 286 if (key_len == AES_KEYSIZE_192) { 287 if (i >= 7) 288 break; 289 rko[4] = rko[3] ^ rki[4]; 290 rko[5] = rko[4] ^ rki[5]; 291 } else if (key_len == AES_KEYSIZE_256) { 292 if (i >= 6) 293 break; 294 rko[4] = subw(rko[3]) ^ rki[4]; 295 rko[5] = rko[4] ^ rki[5]; 296 rko[6] = rko[5] ^ rki[6]; 297 rko[7] = rko[6] ^ rki[7]; 298 } 299 } 300 301 /* 302 * Generate the decryption keys for the Equivalent Inverse Cipher. 303 * This involves reversing the order of the round keys, and applying 304 * the Inverse Mix Columns transformation to all but the first and 305 * the last one. 306 */ 307 if (inv_rndkeys) { 308 inv_rndkeys[0] = rndkeys[key_len + 24]; 309 inv_rndkeys[1] = rndkeys[key_len + 25]; 310 inv_rndkeys[2] = rndkeys[key_len + 26]; 311 inv_rndkeys[3] = rndkeys[key_len + 27]; 312 313 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 314 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 315 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 316 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 317 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 318 } 319 320 inv_rndkeys[i] = rndkeys[0]; 321 inv_rndkeys[i + 1] = rndkeys[1]; 322 inv_rndkeys[i + 2] = rndkeys[2]; 323 inv_rndkeys[i + 3] = rndkeys[3]; 324 } 325 } 326 327 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 328 unsigned int key_len) 329 { 330 if (aes_check_keylen(key_len) != 0) 331 return -EINVAL; 332 ctx->key_length = key_len; 333 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 334 return 0; 335 } 336 EXPORT_SYMBOL(aes_expandkey); 337 338 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 339 { 340 return rk ^ aes_enc_tab[(u8)w[i]] ^ 341 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 342 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 343 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 344 } 345 346 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 347 { 348 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 349 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 350 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 351 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 352 } 353 354 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 355 u8 out[AES_BLOCK_SIZE], 356 const u8 in[AES_BLOCK_SIZE]) 357 { 358 const u32 *rkp = rndkeys; 359 int n = nrounds - 1; 360 u32 w[4]; 361 362 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 363 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 364 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 365 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 366 367 /* 368 * Prefetch the table before doing data and key-dependent loads from it. 369 * 370 * This is intended only as a basic constant-time hardening measure that 371 * avoids interfering with performance too much. Its effectiveness is 372 * not guaranteed. For proper constant-time AES, a CPU that supports 373 * AES instructions should be used instead. 374 */ 375 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 376 377 do { 378 u32 w0 = enc_quarterround(w, 0, *rkp++); 379 u32 w1 = enc_quarterround(w, 1, *rkp++); 380 u32 w2 = enc_quarterround(w, 2, *rkp++); 381 u32 w3 = enc_quarterround(w, 3, *rkp++); 382 383 w[0] = w0; 384 w[1] = w1; 385 w[2] = w2; 386 w[3] = w3; 387 } while (--n); 388 389 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 390 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 391 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 392 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 393 } 394 395 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 396 { 397 return rk ^ aes_dec_tab[(u8)w[i]] ^ 398 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 399 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 400 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 401 } 402 403 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 404 { 405 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 406 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 407 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 408 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 409 } 410 411 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 412 int nrounds, 413 u8 out[AES_BLOCK_SIZE], 414 const u8 in[AES_BLOCK_SIZE]) 415 { 416 const u32 *rkp = inv_rndkeys; 417 int n = nrounds - 1; 418 u32 w[4]; 419 420 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 421 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 422 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 423 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 424 425 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 426 427 do { 428 u32 w0 = dec_quarterround(w, 0, *rkp++); 429 u32 w1 = dec_quarterround(w, 1, *rkp++); 430 u32 w2 = dec_quarterround(w, 2, *rkp++); 431 u32 w3 = dec_quarterround(w, 3, *rkp++); 432 433 w[0] = w0; 434 w[1] = w1; 435 w[2] = w2; 436 w[3] = w3; 437 } while (--n); 438 439 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 440 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 441 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 442 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 443 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 444 } 445 446 /* 447 * Note: the aes_prepare*key_* names reflect the fact that the implementation 448 * might not actually expand the key. (The s390 code for example doesn't.) 449 * Where the key is expanded we use the more specific names aes_expandkey_*. 450 * 451 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 452 * the area to store the prepared decryption key. It will be NULL if the user 453 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 454 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 455 */ 456 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 457 /* An arch-specific implementation of AES is available. Include it. */ 458 #include "aes.h" /* $(SRCARCH)/aes.h */ 459 #else 460 /* No arch-specific implementation of AES is available. Use generic code. */ 461 462 static void aes_preparekey_arch(union aes_enckey_arch *k, 463 union aes_invkey_arch *inv_k, 464 const u8 *in_key, int key_len, int nrounds) 465 { 466 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 467 in_key, key_len); 468 } 469 470 static void aes_encrypt_arch(const struct aes_enckey *key, 471 u8 out[AES_BLOCK_SIZE], 472 const u8 in[AES_BLOCK_SIZE]) 473 { 474 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 475 } 476 477 static void aes_decrypt_arch(const struct aes_key *key, 478 u8 out[AES_BLOCK_SIZE], 479 const u8 in[AES_BLOCK_SIZE]) 480 { 481 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 482 } 483 #endif 484 485 static int __aes_preparekey(struct aes_enckey *enc_key, 486 union aes_invkey_arch *inv_k, 487 const u8 *in_key, size_t key_len) 488 { 489 if (aes_check_keylen(key_len) != 0) 490 return -EINVAL; 491 enc_key->len = key_len; 492 enc_key->nrounds = 6 + key_len / 4; 493 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 494 enc_key->nrounds); 495 return 0; 496 } 497 498 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 499 { 500 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 501 in_key, key_len); 502 } 503 EXPORT_SYMBOL(aes_preparekey); 504 505 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 506 { 507 return __aes_preparekey(key, NULL, in_key, key_len); 508 } 509 EXPORT_SYMBOL(aes_prepareenckey); 510 511 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 512 const u8 in[AES_BLOCK_SIZE]) 513 { 514 aes_encrypt_arch(key.enc_key, out, in); 515 } 516 EXPORT_SYMBOL(aes_encrypt); 517 518 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 519 const u8 in[AES_BLOCK_SIZE]) 520 { 521 aes_decrypt_arch(key, out, in); 522 } 523 EXPORT_SYMBOL(aes_decrypt); 524 525 /* FIPS cryptographic algorithm self-test for "bare" AES */ 526 static void __init aes_fips_test(void) 527 { 528 struct aes_key key; 529 u8 data[AES_BLOCK_SIZE]; 530 531 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 532 panic("aes: FIPS self-test failed (preparekey)\n"); 533 534 aes_encrypt(&key, data, fips_test_data); 535 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 536 panic("aes: FIPS self-test failed (wrong ciphertext)\n"); 537 538 aes_decrypt(&key, data, data); 539 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 540 panic("aes: FIPS self-test failed (wrong plaintext)\n"); 541 542 aes_zeroize_key(&key); 543 } 544 545 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 546 547 #ifndef aes_cbcmac_blocks_arch 548 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 549 const struct aes_enckey *key, const u8 *data, 550 size_t nblocks, bool enc_before, 551 bool enc_after) 552 { 553 return false; 554 } 555 #endif 556 557 /* This assumes nblocks >= 1. */ 558 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 559 const struct aes_enckey *key, const u8 *data, 560 size_t nblocks, bool enc_before, bool enc_after) 561 { 562 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 563 enc_after)) 564 return; 565 566 if (enc_before) 567 aes_encrypt(key, h, h); 568 for (; nblocks > 1; nblocks--) { 569 crypto_xor(h, data, AES_BLOCK_SIZE); 570 data += AES_BLOCK_SIZE; 571 aes_encrypt(key, h, h); 572 } 573 crypto_xor(h, data, AES_BLOCK_SIZE); 574 if (enc_after) 575 aes_encrypt(key, h, h); 576 } 577 578 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 579 size_t key_len) 580 { 581 u64 hi, lo, mask; 582 int err; 583 584 /* Prepare the AES key. */ 585 err = aes_prepareenckey(&key->aes, in_key, key_len); 586 if (err) 587 return err; 588 589 /* 590 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 591 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 592 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 593 */ 594 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 595 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 596 hi = be64_to_cpu(key->k_final[0].w[0]); 597 lo = be64_to_cpu(key->k_final[0].w[1]); 598 for (int i = 0; i < 2; i++) { 599 mask = ((s64)hi >> 63) & 0x87; 600 hi = (hi << 1) ^ (lo >> 63); 601 lo = (lo << 1) ^ mask; 602 key->k_final[i].w[0] = cpu_to_be64(hi); 603 key->k_final[i].w[1] = cpu_to_be64(lo); 604 } 605 return 0; 606 } 607 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 608 609 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 610 const u8 in_key[AES_KEYSIZE_128]) 611 { 612 static const u8 constants[3][AES_BLOCK_SIZE] = { 613 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 614 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 615 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 616 }; 617 u8 new_aes_key[AES_BLOCK_SIZE]; 618 619 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 620 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 621 aes_encrypt(&key->aes, new_aes_key, constants[0]); 622 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 623 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 624 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 625 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 626 } 627 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 628 629 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 630 { 631 bool enc_before = false; 632 size_t nblocks; 633 634 if (ctx->partial_len) { 635 /* XOR data into a pending block. */ 636 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 637 638 crypto_xor(&ctx->h[ctx->partial_len], data, l); 639 data += l; 640 data_len -= l; 641 ctx->partial_len += l; 642 if (data_len == 0) { 643 /* 644 * Either the pending block hasn't been filled yet, or 645 * no more data was given so it's not yet known whether 646 * the block is the final block. 647 */ 648 return; 649 } 650 /* Pending block has been filled and isn't the final block. */ 651 enc_before = true; 652 } 653 654 nblocks = data_len / AES_BLOCK_SIZE; 655 data_len %= AES_BLOCK_SIZE; 656 if (nblocks == 0) { 657 /* 0 additional full blocks, then optionally a partial block */ 658 if (enc_before) 659 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 660 crypto_xor(ctx->h, data, data_len); 661 ctx->partial_len = data_len; 662 } else if (data_len != 0) { 663 /* 1 or more additional full blocks, then a partial block */ 664 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 665 enc_before, /* enc_after= */ true); 666 data += nblocks * AES_BLOCK_SIZE; 667 crypto_xor(ctx->h, data, data_len); 668 ctx->partial_len = data_len; 669 } else { 670 /* 671 * 1 or more additional full blocks only. Encryption of the 672 * last block is delayed until it's known whether it's the final 673 * block in the message or not. 674 */ 675 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 676 enc_before, /* enc_after= */ false); 677 ctx->partial_len = AES_BLOCK_SIZE; 678 } 679 } 680 EXPORT_SYMBOL_GPL(aes_cmac_update); 681 682 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 683 { 684 if (ctx->partial_len == AES_BLOCK_SIZE) { 685 /* Final block is a full block. Use k_final[0]. */ 686 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 687 } else { 688 /* Final block is a partial block. Pad, and use k_final[1]. */ 689 ctx->h[ctx->partial_len] ^= 0x80; 690 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 691 } 692 aes_encrypt(&ctx->key->aes, out, ctx->h); 693 memzero_explicit(ctx, sizeof(*ctx)); 694 } 695 EXPORT_SYMBOL_GPL(aes_cmac_final); 696 697 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 698 size_t data_len) 699 { 700 bool enc_before = false; 701 size_t nblocks; 702 703 if (ctx->partial_len) { 704 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 705 706 crypto_xor(&ctx->h[ctx->partial_len], data, l); 707 data += l; 708 data_len -= l; 709 ctx->partial_len += l; 710 if (ctx->partial_len < AES_BLOCK_SIZE) 711 return; 712 enc_before = true; 713 } 714 715 nblocks = data_len / AES_BLOCK_SIZE; 716 data_len %= AES_BLOCK_SIZE; 717 if (nblocks == 0) { 718 if (enc_before) 719 aes_encrypt(ctx->key, ctx->h, ctx->h); 720 } else { 721 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 722 /* enc_after= */ true); 723 data += nblocks * AES_BLOCK_SIZE; 724 } 725 crypto_xor(ctx->h, data, data_len); 726 ctx->partial_len = data_len; 727 } 728 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 729 730 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 731 { 732 if (ctx->partial_len) 733 aes_encrypt(ctx->key, out, ctx->h); 734 else 735 memcpy(out, ctx->h, AES_BLOCK_SIZE); 736 memzero_explicit(ctx, sizeof(*ctx)); 737 } 738 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 739 740 /* FIPS cryptographic algorithm self-test for AES-CMAC */ 741 static void __init aes_cmac_fips_test(void) 742 { 743 struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key); 744 u8 mac[AES_BLOCK_SIZE]; 745 746 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 747 0) 748 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 749 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 750 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 751 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 752 } 753 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 754 static inline void aes_cmac_fips_test(void) 755 { 756 } 757 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 758 759 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 760 /* 761 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 762 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 763 * causes the fallback implementation to be used instead. 764 */ 765 #ifndef aes_ecb_encrypt_arch 766 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 767 const struct aes_enckey *key) 768 { 769 return false; 770 } 771 #endif 772 #ifndef aes_ecb_decrypt_arch 773 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 774 const struct aes_key *key) 775 { 776 return false; 777 } 778 #endif 779 780 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 781 { 782 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 783 len = round_down(len, AES_BLOCK_SIZE); 784 785 if (unlikely(len == 0)) 786 return; 787 788 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 789 return; 790 791 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 792 aes_encrypt(key, &dst[i], &src[i]); 793 } 794 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 795 796 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 797 const struct aes_key *key) 798 { 799 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 800 len = round_down(len, AES_BLOCK_SIZE); 801 802 if (unlikely(len == 0)) 803 return; 804 805 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 806 return; 807 808 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 809 aes_decrypt(key, &dst[i], &src[i]); 810 } 811 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 812 813 /* FIPS cryptographic algorithm self-test for AES-ECB */ 814 static void __init aes_ecb_fips_test(void) 815 { 816 struct aes_key key; 817 u8 data[sizeof(fips_test_data)]; 818 819 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 820 panic("aes: ECB FIPS self-test failed (preparekey)\n"); 821 822 aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key); 823 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 824 panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n"); 825 826 aes_ecb_decrypt(data, data, sizeof(data), &key); 827 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 828 panic("aes: ECB FIPS self-test failed (wrong plaintext)\n"); 829 830 aes_zeroize_key(&key); 831 } 832 #else /* CONFIG_CRYPTO_LIB_AES_ECB */ 833 static inline void aes_ecb_fips_test(void) 834 { 835 } 836 #endif /* !CONFIG_CRYPTO_LIB_AES_ECB */ 837 838 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 839 /* 840 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 841 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 842 * causes the fallback implementation to be used instead. 843 */ 844 #ifndef aes_cbc_encrypt_arch 845 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 846 u8 iv[AES_BLOCK_SIZE], 847 const struct aes_enckey *key) 848 { 849 return false; 850 } 851 #endif 852 #ifndef aes_cbc_decrypt_arch 853 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 854 u8 iv[AES_BLOCK_SIZE], 855 const struct aes_key *key) 856 { 857 return false; 858 } 859 #endif 860 861 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 862 aes_encrypt_arg key) 863 { 864 const u8 *prev = iv; 865 866 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 867 len = round_down(len, AES_BLOCK_SIZE); 868 869 if (unlikely(len == 0)) 870 return; 871 872 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 873 return; 874 875 do { 876 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 877 aes_encrypt(key, dst, dst); 878 prev = dst; 879 dst += AES_BLOCK_SIZE; 880 src += AES_BLOCK_SIZE; 881 len -= AES_BLOCK_SIZE; 882 } while (len); 883 memcpy(iv, prev, AES_BLOCK_SIZE); 884 } 885 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 886 887 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 888 const struct aes_key *key) 889 { 890 u8 next_iv[AES_BLOCK_SIZE]; 891 892 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 893 len = round_down(len, AES_BLOCK_SIZE); 894 895 if (unlikely(len == 0)) 896 return; 897 898 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 899 return; 900 901 len -= AES_BLOCK_SIZE; 902 dst += len; 903 src += len; 904 memcpy(next_iv, src, AES_BLOCK_SIZE); 905 for (;;) { 906 aes_decrypt(key, dst, src); 907 if (len == 0) 908 break; 909 src -= AES_BLOCK_SIZE; 910 crypto_xor(dst, src, AES_BLOCK_SIZE); 911 dst -= AES_BLOCK_SIZE; 912 len -= AES_BLOCK_SIZE; 913 } 914 crypto_xor(dst, iv, AES_BLOCK_SIZE); 915 memcpy(iv, next_iv, AES_BLOCK_SIZE); 916 } 917 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 918 919 /* 920 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 921 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 922 * causes the fallback implementation to be used instead. The fallback 923 * implementation still uses the arch-optimized AES-CBC code if available, but 924 * direct implementation of AES-CBC-CTS is helpful on short messages. 925 */ 926 #ifndef aes_cbc_cts_encrypt_arch 927 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 928 u8 iv[AES_BLOCK_SIZE], 929 const struct aes_enckey *key) 930 { 931 return false; 932 } 933 #endif 934 #ifndef aes_cbc_cts_decrypt_arch 935 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 936 u8 iv[AES_BLOCK_SIZE], 937 const struct aes_key *key) 938 { 939 return false; 940 } 941 #endif 942 943 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 944 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 945 { 946 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 947 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 948 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 949 size_t pn_len = len - pn_offset; 950 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 951 u8 *pad; 952 953 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 954 return; 955 956 if (len == AES_BLOCK_SIZE) { 957 aes_cbc_encrypt(dst, src, len, iv, key); 958 return; 959 } 960 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 961 return; 962 963 /* CBC-encrypt all blocks except the last. */ 964 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 965 966 /* 967 * Compute C[n] and C[n - 1]. 968 * 969 * Careful: src may equal dst (i.e., the encryption can be in-place), so 970 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 971 */ 972 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 973 memcpy(tmp, pad, AES_BLOCK_SIZE); 974 crypto_xor(tmp, &src[pn_offset], pn_len); 975 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 976 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 977 978 memzero_explicit(tmp, sizeof(tmp)); 979 } 980 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 981 982 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 983 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 984 { 985 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 986 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 987 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 988 size_t pn_len = len - pn_offset; 989 u8 *pad; 990 991 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 992 return; 993 994 if (len == AES_BLOCK_SIZE) { 995 aes_cbc_decrypt(dst, src, len, iv, key); 996 return; 997 } 998 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 999 return; 1000 1001 /* Compute P[0]..P[n - 2]. */ 1002 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 1003 1004 /* 1005 * Compute P[n] and P[n - 1]. 1006 * 1007 * Careful: src may equal dst (i.e., the decryption can be in-place), so 1008 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 1009 * 1010 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 1011 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 1012 */ 1013 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 1014 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 1015 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 1016 pn_len); /* P[n] */ 1017 crypto_xor(pad, &dst[pn_offset], pn_len); 1018 aes_decrypt(key, pad, pad); 1019 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 1020 } 1021 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 1022 1023 /* FIPS cryptographic algorithm self-test for AES-CBC */ 1024 static void __init aes_cbc_fips_test(void) 1025 { 1026 struct aes_key key; 1027 u8 iv[AES_BLOCK_SIZE]; 1028 u8 data[sizeof(fips_test_data)]; 1029 1030 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1031 panic("aes: CBC FIPS self-test failed (preparekey)\n"); 1032 1033 memcpy(iv, fips_test_iv, sizeof(iv)); 1034 aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key); 1035 if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0) 1036 panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n"); 1037 1038 memcpy(iv, fips_test_iv, sizeof(iv)); 1039 aes_cbc_decrypt(data, data, sizeof(data), iv, &key); 1040 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1041 panic("aes: CBC FIPS self-test failed (wrong plaintext)\n"); 1042 1043 aes_zeroize_key(&key); 1044 } 1045 1046 /* FIPS cryptographic algorithm self-test for AES-CBC-CTS */ 1047 static void __init aes_cbc_cts_fips_test(void) 1048 { 1049 struct aes_key key; 1050 u8 iv[AES_BLOCK_SIZE]; 1051 const size_t data_len = 2 * AES_BLOCK_SIZE; 1052 u8 ptext[2 * AES_BLOCK_SIZE]; 1053 u8 data[2 * AES_BLOCK_SIZE]; 1054 1055 /* ptext = fips_test_data || fips_test_data */ 1056 memcpy(ptext, fips_test_data, AES_BLOCK_SIZE); 1057 memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE); 1058 1059 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1060 panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n"); 1061 1062 memcpy(iv, fips_test_iv, sizeof(iv)); 1063 aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key); 1064 if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0) 1065 panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n"); 1066 1067 memcpy(iv, fips_test_iv, sizeof(iv)); 1068 aes_cbc_cts_decrypt(data, data, data_len, iv, &key); 1069 if (memcmp(ptext, data, data_len) != 0) 1070 panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n"); 1071 1072 aes_zeroize_key(&key); 1073 } 1074 #else /* CONFIG_CRYPTO_LIB_AES_CBC */ 1075 static inline void aes_cbc_fips_test(void) 1076 { 1077 } 1078 static inline void aes_cbc_cts_fips_test(void) 1079 { 1080 } 1081 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC */ 1082 1083 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 1084 /* 1085 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 1086 * architecture. They are called with any len >= 0. Returning false causes the 1087 * fallback implementation to be used instead. 1088 */ 1089 #ifndef aes_ctr_arch 1090 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 1091 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 1092 { 1093 return false; 1094 } 1095 #endif 1096 #ifndef aes_xctr_arch 1097 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr, 1098 const u8 iv[AES_BLOCK_SIZE], 1099 const struct aes_enckey *key) 1100 { 1101 return false; 1102 } 1103 #endif 1104 1105 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1106 { 1107 /* 1108 * 255 times out of 256 the first iteration is enough, so unroll the 1109 * first iteration as a micro-optimization. 1110 */ 1111 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1112 return; 1113 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1114 if (++ctr[i] != 0) 1115 break; 1116 } 1117 } 1118 1119 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1120 aes_encrypt_arg key) 1121 { 1122 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1123 1124 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1125 return; 1126 1127 /* Handle the full blocks. */ 1128 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1129 aes_encrypt(key, keystream, ctr); 1130 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1131 inc_be128_ctr(ctr); 1132 dst += AES_BLOCK_SIZE; 1133 src += AES_BLOCK_SIZE; 1134 } 1135 /* Handle any partial block at the end. */ 1136 if (len) { 1137 aes_encrypt(key, keystream, ctr); 1138 crypto_xor_cpy(dst, src, keystream, len); 1139 /* Counter is incremented even with just a partial block. */ 1140 inc_be128_ctr(ctr); 1141 } 1142 memzero_explicit(keystream, sizeof(keystream)); 1143 } 1144 EXPORT_SYMBOL_GPL(aes_ctr); 1145 1146 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1147 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1148 { 1149 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1150 __le64 aes_input[2]; 1151 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1152 1153 if (likely(aes_xctr_arch(dst, src, len, *ctr, iv, key.enc_key))) { 1154 *ctr += DIV_ROUND_UP(len, AES_BLOCK_SIZE); 1155 return; 1156 } 1157 1158 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1159 /* Handle the full blocks. */ 1160 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1161 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1162 aes_encrypt(key, keystream, (const u8 *)aes_input); 1163 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1164 dst += AES_BLOCK_SIZE; 1165 src += AES_BLOCK_SIZE; 1166 } 1167 /* Handle any partial block at the end. */ 1168 if (len) { 1169 /* Counter is incremented even with just a partial block. */ 1170 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1171 aes_encrypt(key, keystream, (const u8 *)aes_input); 1172 crypto_xor_cpy(dst, src, keystream, len); 1173 } 1174 memzero_explicit(keystream, sizeof(keystream)); 1175 memzero_explicit(aes_input, sizeof(aes_input)); 1176 } 1177 EXPORT_SYMBOL_GPL(aes_xctr); 1178 1179 /* FIPS cryptographic algorithm self-test for AES-CTR */ 1180 static void __init aes_ctr_fips_test(void) 1181 { 1182 struct aes_enckey key; 1183 u8 ctr[AES_BLOCK_SIZE]; 1184 u8 data[sizeof(fips_test_data)]; 1185 1186 if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1187 panic("aes: CTR FIPS self-test failed (preparekey)\n"); 1188 1189 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1190 aes_ctr(data, fips_test_data, sizeof(data), ctr, &key); 1191 if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0) 1192 panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n"); 1193 1194 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1195 aes_ctr(data, data, sizeof(data), ctr, &key); 1196 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1197 panic("aes: CTR FIPS self-test failed (wrong plaintext)\n"); 1198 1199 aes_zeroize_enckey(&key); 1200 } 1201 #else /* CONFIG_CRYPTO_LIB_AES_CTR */ 1202 static inline void aes_ctr_fips_test(void) 1203 { 1204 } 1205 #endif /* !CONFIG_CRYPTO_LIB_AES_CTR */ 1206 1207 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 1208 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, 1209 size_t key_len, int flags) 1210 { 1211 int err; 1212 1213 err = __xts_verify_key(in_key, key_len, flags); 1214 if (unlikely(err)) 1215 goto out_zeroize; 1216 /* First half of XTS key is the main key */ 1217 err = aes_preparekey(&key->main_key, in_key, key_len / 2); 1218 if (unlikely(err)) 1219 goto out_zeroize; 1220 /* Second half of XTS key is the tweak key */ 1221 err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], 1222 key_len / 2); 1223 if (unlikely(err)) 1224 goto out_zeroize; 1225 return 0; 1226 1227 out_zeroize: 1228 aes_xts_zeroize_key(key); 1229 return err; 1230 } 1231 EXPORT_SYMBOL_GPL(aes_xts_preparekey); 1232 1233 /* 1234 * Hooks for optimized AES-XTS implementations, overridable by the architecture. 1235 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, 1236 * they aren't expected to handle ciphertext stealing or empty inputs. 1237 * Returning false causes the fallback implementation to be used instead. 1238 * 1239 * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole 1240 * numbers of blocks anyway, with len >= 512. So there's no need to heavily 1241 * optimize ciphertext stealing for short messages.) 1242 */ 1243 #ifndef aes_xts_encrypt_arch 1244 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 1245 u8 tweak[AES_BLOCK_SIZE], 1246 const struct aes_xts_key *key, bool cont) 1247 { 1248 return false; 1249 } 1250 #endif 1251 #ifndef aes_xts_decrypt_arch 1252 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 1253 u8 tweak[AES_BLOCK_SIZE], 1254 const struct aes_xts_key *key, bool cont) 1255 { 1256 return false; 1257 } 1258 #endif 1259 1260 static noinline void aes_xts_crypt_nocts_blockbyblock( 1261 u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 1262 const struct aes_xts_key *key, bool cont, bool enc) 1263 { 1264 le128 t; 1265 1266 if (cont) 1267 memcpy(&t, tweak, sizeof(t)); 1268 else 1269 aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); 1270 do { 1271 crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); 1272 if (enc) 1273 aes_encrypt(&key->main_key, dst, dst); 1274 else 1275 aes_decrypt(&key->main_key, dst, dst); 1276 crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); 1277 gf128mul_x_ble(&t, &t); 1278 dst += AES_BLOCK_SIZE; 1279 src += AES_BLOCK_SIZE; 1280 len -= AES_BLOCK_SIZE; 1281 } while (len); 1282 memcpy(tweak, &t, sizeof(t)); 1283 memzero_explicit(&t, sizeof(t)); 1284 } 1285 1286 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1287 static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, 1288 size_t len, 1289 u8 tweak[AES_BLOCK_SIZE], 1290 const struct aes_xts_key *key, 1291 bool cont) 1292 { 1293 if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) 1294 return; 1295 1296 /* 1297 * For the fallback, just go block-by-block. It could be implemented on 1298 * top of AES-ECB, which could be significantly faster than this if the 1299 * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS 1300 * performance is important enough that it needs to be (and has been) 1301 * implemented directly by every non-obsolete arch anyway. 1302 */ 1303 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1304 /* enc= */ true); 1305 } 1306 1307 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1308 static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, 1309 size_t len, 1310 u8 tweak[AES_BLOCK_SIZE], 1311 const struct aes_xts_key *key, 1312 bool cont) 1313 { 1314 if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) 1315 return; 1316 1317 /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ 1318 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1319 /* enc= */ false); 1320 } 1321 1322 static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, 1323 u8 tweak[AES_BLOCK_SIZE], 1324 const struct aes_xts_key *key, 1325 bool cont) 1326 { 1327 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1328 size_t nocts_len = round_down(len, AES_BLOCK_SIZE); 1329 u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1330 1331 /* Encrypt all full blocks. */ 1332 aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1333 dst += nocts_len - AES_BLOCK_SIZE; 1334 src += nocts_len - AES_BLOCK_SIZE; 1335 1336 /* 1337 * Swap the partial block with the first 'partial_len' bytes of the 1338 * encrypted last full block. Note that a temporary buffer is needed to 1339 * support in-place encryption. 1340 */ 1341 memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); 1342 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1343 memcpy(dst, tmp_block, partial_len); 1344 1345 /* Encrypt the last full block again. */ 1346 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1347 aes_encrypt(&key->main_key, dst, dst); 1348 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1349 memzero_explicit(tmp_block, sizeof(tmp_block)); 1350 } 1351 1352 static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, 1353 u8 tweak[AES_BLOCK_SIZE], 1354 const struct aes_xts_key *key, 1355 bool cont) 1356 { 1357 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1358 size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; 1359 union { 1360 u8 block[AES_BLOCK_SIZE]; 1361 le128 tweak; 1362 } tmp __aligned(__alignof__(long)); 1363 1364 /* 1365 * Decrypt all blocks except the last full block and the partial block. 1366 * The last full block has to be handled specially because decryption 1367 * ciphertext stealing uses the last two tweaks in reverse order. 1368 * 1369 * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() 1370 * doesn't handle (so that the length doesn't get checked redundantly in 1371 * the fast path). So handle that case specially as well. 1372 */ 1373 if (nocts_len) 1374 aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1375 else if (!cont) 1376 aes_encrypt(&key->tweak_key, tweak, tweak); 1377 dst += nocts_len; 1378 src += nocts_len; 1379 1380 /* Copy the tweak, advance it again, then decrypt last full block. */ 1381 memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); 1382 gf128mul_x_ble(&tmp.tweak, &tmp.tweak); 1383 crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); 1384 aes_decrypt(&key->main_key, dst, dst); 1385 crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); 1386 1387 /* 1388 * Swap the partial block with the first 'partial_len' bytes of the 1389 * decrypted last full block. Note that a temporary buffer is needed to 1390 * support in-place decryption. 1391 */ 1392 memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); 1393 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1394 memcpy(dst, tmp.block, partial_len); 1395 1396 /* Decrypt the last full block again. */ 1397 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1398 aes_decrypt(&key->main_key, dst, dst); 1399 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1400 memzero_explicit(&tmp, sizeof(tmp)); 1401 } 1402 1403 void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, 1404 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1405 bool cont) 1406 { 1407 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1408 return; 1409 1410 if (unlikely(len % AES_BLOCK_SIZE)) { 1411 aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); 1412 return; 1413 } 1414 1415 aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); 1416 } 1417 EXPORT_SYMBOL_GPL(aes_xts_encrypt); 1418 1419 void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, 1420 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1421 bool cont) 1422 { 1423 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1424 return; 1425 1426 if (unlikely(len % AES_BLOCK_SIZE)) { 1427 aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); 1428 return; 1429 } 1430 1431 aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); 1432 } 1433 EXPORT_SYMBOL_GPL(aes_xts_decrypt); 1434 1435 /* FIPS cryptographic algorithm self-test for AES-XTS */ 1436 static void __init aes_xts_fips_test(void) 1437 { 1438 struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key); 1439 u8 tweak[AES_BLOCK_SIZE]; 1440 u8 data[sizeof(fips_test_data)]; 1441 1442 if (key == NULL) 1443 panic("aes: XTS FIPS self-test failed (kmalloc)\n"); 1444 1445 if (aes_xts_preparekey(key, fips_test_xts_key, 1446 sizeof(fips_test_xts_key), 0) != 0) 1447 panic("aes: XTS FIPS self-test failed (preparekey)\n"); 1448 1449 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1450 aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false); 1451 if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0) 1452 panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n"); 1453 1454 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1455 aes_xts_decrypt(data, data, sizeof(data), tweak, key, false); 1456 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1457 panic("aes: XTS FIPS self-test failed (wrong plaintext)\n"); 1458 } 1459 #else /* CONFIG_CRYPTO_LIB_AES_XTS */ 1460 static inline void aes_xts_fips_test(void) 1461 { 1462 } 1463 #endif /* !CONFIG_CRYPTO_LIB_AES_XTS */ 1464 1465 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) 1466 /* 1467 * Hooks for optimized AES-GCM implementations, overridable by the architecture. 1468 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't 1469 * expected to handle empty inputs or partial blocks, as those cases are handled 1470 * by non-arch-specific code instead. 1471 * 1472 * The GHASH accumulator is provided in POLYVAL format. The counter is provided 1473 * in big endian format, and it's read-only, as the caller handles updating it. 1474 * 1475 * Returning false causes the fallback implementation to be used instead. 1476 * 1477 * These hooks are used only for en/decrypted data. For the associated data the 1478 * GHASH functions are called instead, so those should be implemented too. 1479 */ 1480 #ifndef aes_gcm_encrypt_update_arch 1481 static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1482 struct polyval_elem *ghash_acc, 1483 const __be32 ctr32[4], 1484 const struct aes_enckey *aes_key, 1485 const struct ghash_key *ghash_key) 1486 { 1487 return false; 1488 } 1489 #endif 1490 #ifndef aes_gcm_decrypt_update_arch 1491 static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1492 struct polyval_elem *ghash_acc, 1493 const __be32 ctr32[4], 1494 const struct aes_enckey *aes_key, 1495 const struct ghash_key *ghash_key) 1496 { 1497 return false; 1498 } 1499 #endif 1500 1501 int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, 1502 size_t key_len, size_t authtag_len) 1503 { 1504 u8 h[AES_BLOCK_SIZE] = { 0 }; 1505 int err; 1506 1507 err = crypto_gcm_check_authsize(authtag_len); 1508 if (unlikely(err)) 1509 return err; 1510 1511 err = aes_prepareenckey(&key->aes, in_key, key_len); 1512 if (unlikely(err)) 1513 return err; 1514 1515 aes_encrypt(&key->aes, h, h); 1516 ghash_preparekey(&key->ghash, h); 1517 1518 key->authtag_len = authtag_len; 1519 1520 memzero_explicit(h, sizeof(h)); 1521 return 0; 1522 } 1523 EXPORT_SYMBOL_GPL(aes_gcm_preparekey); 1524 1525 void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], 1526 const struct aes_gcm_key *key) 1527 { 1528 ctx->key = key; 1529 ctx->ad_len = 0; 1530 ctx->data_len = 0; 1531 ghash_init(&ctx->ghash, &key->ghash); 1532 memset(ctx->keystream, 0, sizeof(ctx->keystream)); 1533 1534 memcpy(ctx->ctr32, nonce, 12); 1535 ctx->ctr32[3] = cpu_to_be32(1); 1536 1537 aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); 1538 ctx->ctr32[3] = cpu_to_be32(2); 1539 } 1540 EXPORT_SYMBOL_GPL(aes_gcm_init); 1541 1542 void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) 1543 { 1544 WARN_ON_ONCE(ctx->data_len != 0); 1545 if (len) { 1546 ghash_update(&ctx->ghash, ad, len); 1547 ctx->ad_len += len; 1548 } 1549 } 1550 EXPORT_SYMBOL_GPL(aes_gcm_auth_update); 1551 1552 static const u8 gcm_zeroes[AES_BLOCK_SIZE]; 1553 1554 static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) 1555 { 1556 if (len % AES_BLOCK_SIZE) 1557 ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); 1558 } 1559 1560 static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, 1561 u8 *dst, const u8 *src, 1562 size_t len, bool enc) 1563 { 1564 size_t partial_len, n; 1565 1566 if (unlikely(len == 0)) 1567 return; 1568 1569 partial_len = ctx->data_len % AES_BLOCK_SIZE; 1570 if (ctx->data_len == 0) 1571 ghash_pad(&ctx->ghash, ctx->ad_len); 1572 ctx->data_len += len; 1573 1574 if (unlikely(partial_len != 0)) { 1575 /* 1576 * The previous call ended on a non-block-aligned data_len, so 1577 * continue using a previously-generated keystream block. 1578 */ 1579 n = min(len, AES_BLOCK_SIZE - partial_len); 1580 if (enc) { 1581 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1582 n); 1583 ghash_update(&ctx->ghash, dst, n); 1584 } else { 1585 ghash_update(&ctx->ghash, src, n); 1586 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1587 n); 1588 } 1589 dst += n; 1590 src += n; 1591 len -= n; 1592 } 1593 1594 if (len >= AES_BLOCK_SIZE) { 1595 n = round_down(len, AES_BLOCK_SIZE); 1596 if (enc) { 1597 if (likely(aes_gcm_encrypt_update_arch( 1598 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1599 &ctx->key->aes, &ctx->key->ghash))) { 1600 be32_add_cpu(&ctx->ctr32[3], 1601 n / AES_BLOCK_SIZE); 1602 } else { 1603 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1604 ghash_update(&ctx->ghash, dst, n); 1605 } 1606 } else { 1607 if (likely(aes_gcm_decrypt_update_arch( 1608 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1609 &ctx->key->aes, &ctx->key->ghash))) { 1610 be32_add_cpu(&ctx->ctr32[3], 1611 n / AES_BLOCK_SIZE); 1612 } else { 1613 ghash_update(&ctx->ghash, src, n); 1614 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1615 } 1616 } 1617 dst += n; 1618 src += n; 1619 len -= n; 1620 } 1621 1622 if (len != 0) { 1623 /* 1624 * Ending on a non-block aligned data_len. Generate the next 1625 * keystream block, use the needed portion of it, and leave it 1626 * cached in ctx->keystream in case this isn't the final call. 1627 */ 1628 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1629 be32_add_cpu(&ctx->ctr32[3], 1); 1630 if (enc) { 1631 crypto_xor_cpy(dst, src, ctx->keystream, len); 1632 ghash_update(&ctx->ghash, dst, len); 1633 } else { 1634 ghash_update(&ctx->ghash, src, len); 1635 crypto_xor_cpy(dst, src, ctx->keystream, len); 1636 } 1637 } 1638 } 1639 1640 void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1641 size_t len) 1642 { 1643 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1644 } 1645 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); 1646 1647 void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1648 size_t len) 1649 { 1650 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); 1651 } 1652 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); 1653 1654 /* Maximum AES-GCM associated data length in bytes */ 1655 #define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) 1656 /* Maximum AES-GCM en/decrypted data length in bytes */ 1657 #define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) 1658 1659 void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) 1660 { 1661 __be64 tail[2]; 1662 1663 WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); 1664 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); 1665 1666 ghash_pad(&ctx->ghash, 1667 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1668 1669 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1670 tail[1] = cpu_to_be64(ctx->data_len * 8); 1671 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1672 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1673 1674 crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1675 aes_gcm_zeroize_ctx(ctx); 1676 } 1677 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); 1678 1679 int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) 1680 { 1681 __be64 tail[2]; 1682 int err; 1683 1684 if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || 1685 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { 1686 err = -EBADMSG; 1687 goto out; 1688 } 1689 1690 ghash_pad(&ctx->ghash, 1691 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1692 1693 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1694 tail[1] = cpu_to_be64(ctx->data_len * 8); 1695 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1696 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1697 crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1698 err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? 1699 -EBADMSG : 1700 0; 1701 out: 1702 aes_gcm_zeroize_ctx(ctx); 1703 return err; 1704 } 1705 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); 1706 1707 void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 1708 const u8 *ad, size_t ad_len, const u8 nonce[12], 1709 const struct aes_gcm_key *key) 1710 { 1711 struct aes_gcm_ctx ctx; 1712 1713 aes_gcm_init(&ctx, nonce, key); 1714 aes_gcm_auth_update(&ctx, ad, ad_len); 1715 aes_gcm_encrypt_update(&ctx, dst, src, data_len); 1716 aes_gcm_encrypt_final(&ctx, authtag); 1717 } 1718 EXPORT_SYMBOL_GPL(aes_gcm_encrypt); 1719 1720 int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 1721 const u8 *ad, size_t ad_len, const u8 nonce[12], 1722 const struct aes_gcm_key *key) 1723 { 1724 struct aes_gcm_ctx ctx; 1725 int err; 1726 1727 aes_gcm_init(&ctx, nonce, key); 1728 aes_gcm_auth_update(&ctx, ad, ad_len); 1729 aes_gcm_decrypt_update(&ctx, dst, src, data_len); 1730 err = aes_gcm_decrypt_final(&ctx, authtag); 1731 if (unlikely(err) && data_len) { 1732 /* 1733 * Clear the inauthentic decrypted data so that callers won't 1734 * receive it even if they fail to correctly handle errors. 1735 */ 1736 memset(dst, 0, data_len); 1737 } 1738 return err; 1739 } 1740 EXPORT_SYMBOL_GPL(aes_gcm_decrypt); 1741 1742 /* FIPS cryptographic algorithm self-test for AES-GCM */ 1743 static void __init aes_gcm_fips_test(void) 1744 { 1745 const size_t data_len = sizeof(fips_test_data); 1746 u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; 1747 struct aes_gcm_key key __cleanup(aes_gcm_zeroize_key); 1748 int err; 1749 1750 if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key), 1751 AES_BLOCK_SIZE) != 0) 1752 panic("aes: GCM FIPS self-test failed (preparekey)\n"); 1753 1754 aes_gcm_encrypt(buf, fips_test_data, data_len, &buf[data_len], 1755 fips_test_ad, sizeof(fips_test_ad), fips_test_iv, &key); 1756 if (memcmp(fips_test_aes_gcm_ctext_and_tag, buf, sizeof(buf)) != 0) 1757 panic("aes: GCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); 1758 1759 err = aes_gcm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, 1760 sizeof(fips_test_ad), fips_test_iv, &key); 1761 if (err != 0) 1762 panic("aes: GCM FIPS self-test failed (decryption failed)\n"); 1763 if (memcmp(fips_test_data, buf, data_len) != 0) 1764 panic("aes: GCM FIPS self-test failed (wrong plaintext)\n"); 1765 } 1766 #else /* CONFIG_CRYPTO_LIB_AES_GCM */ 1767 static inline void aes_gcm_fips_test(void) 1768 { 1769 } 1770 #endif /* !CONFIG_CRYPTO_LIB_AES_GCM */ 1771 1772 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) 1773 int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, 1774 size_t key_len, size_t authtag_len) 1775 { 1776 int err; 1777 1778 if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) 1779 return -EINVAL; 1780 1781 err = aes_prepareenckey(&key->aes, in_key, key_len); 1782 if (unlikely(err)) 1783 return err; 1784 1785 key->authtag_len = authtag_len; 1786 return 0; 1787 } 1788 EXPORT_SYMBOL_GPL(aes_ccm_preparekey); 1789 1790 int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, 1791 const u8 *nonce, size_t nonce_len, 1792 const struct aes_ccm_key *key) 1793 { 1794 /* 1795 * This is the value L defined in the CCM specification. It determines 1796 * the maximum allowed message length, and it is itself determined by 1797 * the nonce length. They are inversely related, i.e. the longer the 1798 * nonce the smaller the maximum message length is. 1799 */ 1800 unsigned int l = 15 - nonce_len; 1801 1802 if (unlikely(nonce_len < 7 || nonce_len > 13)) 1803 return -EINVAL; 1804 /* Thus 2 <= l <= 8. */ 1805 1806 /* Check whether data_len can be represented in 'l' bytes. */ 1807 if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) 1808 return -EOVERFLOW; 1809 1810 ctx->key = key; 1811 ctx->ad_remaining = ad_len; 1812 ctx->data_remaining = data_len; 1813 ctx->ad_padded = false; 1814 1815 /* 1816 * Initialize the zero-th counter block to: 1817 * 1818 * L - 1 || nonce || 0 1819 * 1820 * ... and the zero-th CBC-MAC block to: 1821 * 1822 * Flags || nonce || data_len 1823 */ 1824 *(__be64 *)&ctx->ctr[8] = 0; 1825 *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); 1826 ctx->ctr[0] = l - 1; 1827 ctx->mac[0] = (ad_len ? 0x40 : 0) | 1828 (((key->authtag_len - 2) / 2) << 3) | (l - 1); 1829 memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ 1830 memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ 1831 1832 /* 1833 * Generate S_0 by encrypting the counter (this is used to encrypt the 1834 * auth tag later), and encrypt the zero-th CBC-MAC block. 1835 */ 1836 aes_encrypt(&key->aes, ctx->s0, ctx->ctr); 1837 aes_encrypt(&key->aes, ctx->mac, ctx->mac); 1838 1839 /* Increment the counter from 0 to 1. */ 1840 ctx->ctr[15] = 1; 1841 1842 if (ad_len) { 1843 /* 1844 * Update CBC-MAC with the associated data length, represented 1845 * using either 2, 6, or 10 bytes depending on the length. 1846 */ 1847 if (likely(ad_len < 0xff00)) { 1848 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); 1849 ctx->partial_len = 2; 1850 } else if (ad_len <= U32_MAX) { 1851 __be32 *p = (__be32 *)&ctx->mac[2]; 1852 1853 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); 1854 put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), 1855 p); 1856 ctx->partial_len = 6; 1857 } else { 1858 __be64 *p = (__be64 *)&ctx->mac[2]; 1859 1860 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); 1861 put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), 1862 p); 1863 ctx->partial_len = 10; 1864 } 1865 } else { 1866 ctx->partial_len = 0; 1867 } 1868 return 0; 1869 } 1870 EXPORT_SYMBOL_GPL(aes_ccm_init); 1871 1872 void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) 1873 { 1874 size_t partial_len = ctx->partial_len; 1875 bool enc_before = false; 1876 size_t nblocks; 1877 1878 WARN_ON_ONCE(ctx->ad_padded); 1879 1880 /* 1881 * We could warn on len > ad_remaining here, but underflow will be 1882 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1883 * going to underflow all the way back to 0.) 1884 */ 1885 ctx->ad_remaining -= len; 1886 1887 if (partial_len) { 1888 size_t n = min(len, AES_BLOCK_SIZE - partial_len); 1889 1890 crypto_xor(&ctx->mac[partial_len], ad, n); 1891 ad += n; 1892 len -= n; 1893 partial_len += n; 1894 if (partial_len < AES_BLOCK_SIZE) { 1895 ctx->partial_len = partial_len; 1896 return; 1897 } 1898 enc_before = true; 1899 } 1900 1901 nblocks = len / AES_BLOCK_SIZE; 1902 len %= AES_BLOCK_SIZE; 1903 if (nblocks == 0) { 1904 if (enc_before) 1905 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1906 } else { 1907 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, 1908 enc_before, /* enc_after= */ true); 1909 ad += nblocks * AES_BLOCK_SIZE; 1910 } 1911 crypto_xor(ctx->mac, ad, len); 1912 ctx->partial_len = len; 1913 } 1914 EXPORT_SYMBOL_GPL(aes_ccm_auth_update); 1915 1916 static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, 1917 u8 *dst, const u8 *src, 1918 size_t len, bool enc) 1919 { 1920 size_t partial_len = ctx->partial_len; 1921 size_t n, nblocks; 1922 1923 if (unlikely(len == 0)) 1924 return; 1925 1926 WARN_ON_ONCE(ctx->ad_remaining != 0); 1927 1928 /* 1929 * We could warn on len > data_remaining here, but underflow will be 1930 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1931 * going to underflow all the way back to 0.) 1932 */ 1933 ctx->data_remaining -= len; 1934 1935 if (!ctx->ad_padded) { 1936 ctx->ad_padded = true; 1937 if (partial_len) 1938 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1939 } else if (partial_len) { 1940 /* 1941 * The previous call ended on a non-block-aligned data_len, so 1942 * continue using a previously-generated keystream block. 1943 */ 1944 n = min(len, AES_BLOCK_SIZE - partial_len); 1945 if (enc) 1946 crypto_xor(&ctx->mac[partial_len], src, n); 1947 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); 1948 if (!enc) 1949 crypto_xor(&ctx->mac[partial_len], dst, n); 1950 dst += n; 1951 src += n; 1952 len -= n; 1953 partial_len += n; 1954 if (partial_len < AES_BLOCK_SIZE) { 1955 ctx->partial_len = partial_len; 1956 return; 1957 } 1958 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1959 } 1960 1961 if (len >= AES_BLOCK_SIZE) { 1962 n = round_down(len, AES_BLOCK_SIZE); 1963 nblocks = len / AES_BLOCK_SIZE; 1964 if (enc) 1965 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, 1966 nblocks, /* enc_before= */ false, 1967 /* enc_after= */ true); 1968 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1969 if (!enc) 1970 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, 1971 nblocks, /* enc_before= */ false, 1972 /* enc_after= */ true); 1973 dst += n; 1974 src += n; 1975 len -= n; 1976 } 1977 1978 if (len) { 1979 /* 1980 * Ending on a non-block aligned data_len. Generate the next 1981 * keystream block, use the needed portion of it, and leave it 1982 * cached in ctx->keystream in case this isn't the final call. 1983 */ 1984 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1985 inc_be128_ctr(ctx->ctr); 1986 if (enc) 1987 crypto_xor(ctx->mac, src, len); 1988 crypto_xor_cpy(dst, src, ctx->keystream, len); 1989 if (!enc) 1990 crypto_xor(ctx->mac, dst, len); 1991 } 1992 ctx->partial_len = len; 1993 } 1994 1995 void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 1996 size_t len) 1997 { 1998 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1999 } 2000 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); 2001 2002 void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 2003 size_t len) 2004 { 2005 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); 2006 } 2007 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); 2008 2009 void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) 2010 { 2011 WARN_ON_ONCE(ctx->ad_remaining != 0); 2012 WARN_ON_ONCE(ctx->data_remaining != 0); 2013 if (ctx->partial_len) 2014 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 2015 crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); 2016 aes_ccm_zeroize_ctx(ctx); 2017 } 2018 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); 2019 2020 int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) 2021 { 2022 int err; 2023 2024 if (WARN_ON_ONCE(ctx->ad_remaining != 0) || 2025 WARN_ON_ONCE(ctx->data_remaining != 0)) { 2026 err = -EBADMSG; 2027 goto out; 2028 } 2029 2030 if (ctx->partial_len) 2031 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 2032 crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); 2033 err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? 2034 -EBADMSG : 2035 0; 2036 out: 2037 aes_ccm_zeroize_ctx(ctx); 2038 return err; 2039 } 2040 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); 2041 2042 int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 2043 const u8 *ad, size_t ad_len, const u8 *nonce, 2044 size_t nonce_len, const struct aes_ccm_key *key) 2045 { 2046 struct aes_ccm_ctx ctx; 2047 int err; 2048 2049 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2050 if (unlikely(err)) 2051 return err; 2052 aes_ccm_auth_update(&ctx, ad, ad_len); 2053 aes_ccm_encrypt_update(&ctx, dst, src, data_len); 2054 aes_ccm_encrypt_final(&ctx, authtag); 2055 return 0; 2056 } 2057 EXPORT_SYMBOL_GPL(aes_ccm_encrypt); 2058 2059 int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 2060 const u8 *ad, size_t ad_len, const u8 *nonce, 2061 size_t nonce_len, const struct aes_ccm_key *key) 2062 { 2063 struct aes_ccm_ctx ctx; 2064 int err; 2065 2066 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2067 if (unlikely(err)) 2068 return err; 2069 aes_ccm_auth_update(&ctx, ad, ad_len); 2070 aes_ccm_decrypt_update(&ctx, dst, src, data_len); 2071 err = aes_ccm_decrypt_final(&ctx, authtag); 2072 if (unlikely(err) && data_len) { 2073 /* 2074 * Clear the inauthentic decrypted data so that callers won't 2075 * receive it even if they fail to correctly handle errors. 2076 */ 2077 memset(dst, 0, data_len); 2078 } 2079 return err; 2080 } 2081 EXPORT_SYMBOL_GPL(aes_ccm_decrypt); 2082 2083 /* FIPS cryptographic algorithm self-test for AES-CCM */ 2084 static void __init aes_ccm_fips_test(void) 2085 { 2086 const size_t data_len = sizeof(fips_test_data); 2087 const size_t nonce_len = 13; 2088 u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; 2089 struct aes_ccm_key key __cleanup(aes_ccm_zeroize_key); 2090 int err; 2091 2092 if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key), 2093 AES_BLOCK_SIZE) != 0) 2094 panic("aes: CCM FIPS self-test failed (preparekey)\n"); 2095 2096 err = aes_ccm_encrypt(buf, fips_test_data, data_len, &buf[data_len], 2097 fips_test_ad, sizeof(fips_test_ad), fips_test_iv, 2098 nonce_len, &key); 2099 if (err != 0) 2100 panic("aes: CCM FIPS self-test failed (encryption failed)\n"); 2101 if (memcmp(fips_test_aes_ccm_ctext_and_tag, buf, sizeof(buf)) != 0) 2102 panic("aes: CCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); 2103 2104 err = aes_ccm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, 2105 sizeof(fips_test_ad), fips_test_iv, nonce_len, 2106 &key); 2107 if (err != 0) 2108 panic("aes: CCM FIPS self-test failed (decryption failed)\n"); 2109 if (memcmp(fips_test_data, buf, data_len) != 0) 2110 panic("aes: CCM FIPS self-test failed (wrong plaintext)\n"); 2111 } 2112 #else /* CONFIG_CRYPTO_LIB_AES_CCM */ 2113 static inline void aes_ccm_fips_test(void) 2114 { 2115 } 2116 #endif /* !CONFIG_CRYPTO_LIB_AES_CCM */ 2117 2118 static int __init aes_mod_init(void) 2119 { 2120 #ifdef aes_mod_init_arch 2121 aes_mod_init_arch(); 2122 #endif 2123 if (fips_enabled) { 2124 aes_fips_test(); 2125 aes_cmac_fips_test(); 2126 aes_ecb_fips_test(); 2127 aes_cbc_fips_test(); 2128 aes_cbc_cts_fips_test(); 2129 aes_ctr_fips_test(); 2130 aes_xts_fips_test(); 2131 aes_gcm_fips_test(); 2132 aes_ccm_fips_test(); 2133 } 2134 return 0; 2135 } 2136 subsys_initcall(aes_mod_init); 2137 2138 static void __exit aes_mod_exit(void) 2139 { 2140 } 2141 module_exit(aes_mod_exit); 2142 2143 MODULE_DESCRIPTION("AES block cipher"); 2144 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 2145 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 2146 MODULE_LICENSE("GPL v2"); 2147