1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes.h> 9 #include <crypto/utils.h> 10 #include <linux/cache.h> 11 #include <linux/crypto.h> 12 #include <linux/export.h> 13 #include <linux/module.h> 14 #include <linux/unaligned.h> 15 16 static const u8 ____cacheline_aligned aes_sbox[] = { 17 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 18 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 19 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 20 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 21 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 22 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 23 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 24 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 25 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 26 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 27 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 28 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 29 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 30 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 31 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 32 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 33 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 34 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 35 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 36 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 37 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 38 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 39 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 40 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 41 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 42 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 43 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 44 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 45 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 46 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 47 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 48 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 49 }; 50 51 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 52 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 53 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 54 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 55 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 56 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 57 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 58 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 59 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 60 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 61 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 62 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 63 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 64 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 65 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 66 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 67 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 68 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 69 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 70 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 71 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 72 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 73 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 74 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 75 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 76 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 77 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 78 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 79 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 80 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 81 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 82 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 83 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 84 }; 85 86 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 87 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 88 89 EXPORT_SYMBOL(crypto_aes_sbox); 90 EXPORT_SYMBOL(crypto_aes_inv_sbox); 91 92 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 93 const u32 ____cacheline_aligned aes_enc_tab[256] = { 94 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 95 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 96 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 97 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 98 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 99 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 100 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 101 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 102 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 103 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 104 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 105 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 106 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 107 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 108 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 109 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 110 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 111 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 112 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 113 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 114 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 115 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 116 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 117 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 118 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 119 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 120 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 121 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 122 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 123 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 124 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 125 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 126 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 127 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 128 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 129 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 130 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 131 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 132 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 133 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 134 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 135 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 136 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 137 }; 138 EXPORT_SYMBOL(aes_enc_tab); 139 140 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 141 const u32 ____cacheline_aligned aes_dec_tab[256] = { 142 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 143 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 144 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 145 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 146 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 147 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 148 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 149 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 150 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 151 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 152 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 153 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 154 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 155 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 156 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 157 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 158 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 159 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 160 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 161 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 162 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 163 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 164 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 165 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 166 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 167 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 168 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 169 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 170 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 171 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 172 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 173 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 174 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 175 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 176 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 177 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 178 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 179 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 180 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 181 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 182 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 183 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 184 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 185 }; 186 EXPORT_SYMBOL(aes_dec_tab); 187 188 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 189 static __always_inline void aes_prefetch(const void *mem, size_t len) 190 { 191 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 192 *(volatile const u8 *)(mem + i); 193 barrier(); 194 } 195 196 static u32 mul_by_x(u32 w) 197 { 198 u32 x = w & 0x7f7f7f7f; 199 u32 y = w & 0x80808080; 200 201 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 202 return (x << 1) ^ (y >> 7) * 0x1b; 203 } 204 205 static u32 mul_by_x2(u32 w) 206 { 207 u32 x = w & 0x3f3f3f3f; 208 u32 y = w & 0x80808080; 209 u32 z = w & 0x40404040; 210 211 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 212 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 213 } 214 215 static u32 mix_columns(u32 x) 216 { 217 /* 218 * Perform the following matrix multiplication in GF(2^8) 219 * 220 * | 0x2 0x3 0x1 0x1 | | x[0] | 221 * | 0x1 0x2 0x3 0x1 | | x[1] | 222 * | 0x1 0x1 0x2 0x3 | x | x[2] | 223 * | 0x3 0x1 0x1 0x2 | | x[3] | 224 */ 225 u32 y = mul_by_x(x) ^ ror32(x, 16); 226 227 return y ^ ror32(x ^ y, 8); 228 } 229 230 static u32 inv_mix_columns(u32 x) 231 { 232 /* 233 * Perform the following matrix multiplication in GF(2^8) 234 * 235 * | 0xe 0xb 0xd 0x9 | | x[0] | 236 * | 0x9 0xe 0xb 0xd | | x[1] | 237 * | 0xd 0x9 0xe 0xb | x | x[2] | 238 * | 0xb 0xd 0x9 0xe | | x[3] | 239 * 240 * which can conveniently be reduced to 241 * 242 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 243 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 244 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 245 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 246 */ 247 u32 y = mul_by_x2(x); 248 249 return mix_columns(x ^ y ^ ror32(y, 16)); 250 } 251 252 static u32 subw(u32 in) 253 { 254 return (aes_sbox[in & 0xff]) ^ 255 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 256 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 257 (aes_sbox[(in >> 24) & 0xff] << 24); 258 } 259 260 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 261 const u8 *in_key, int key_len) 262 { 263 u32 kwords = key_len / sizeof(u32); 264 u32 rc, i, j; 265 266 for (i = 0; i < kwords; i++) 267 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 268 269 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 270 u32 *rki = &rndkeys[i * kwords]; 271 u32 *rko = rki + kwords; 272 273 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 274 rko[1] = rko[0] ^ rki[1]; 275 rko[2] = rko[1] ^ rki[2]; 276 rko[3] = rko[2] ^ rki[3]; 277 278 if (key_len == AES_KEYSIZE_192) { 279 if (i >= 7) 280 break; 281 rko[4] = rko[3] ^ rki[4]; 282 rko[5] = rko[4] ^ rki[5]; 283 } else if (key_len == AES_KEYSIZE_256) { 284 if (i >= 6) 285 break; 286 rko[4] = subw(rko[3]) ^ rki[4]; 287 rko[5] = rko[4] ^ rki[5]; 288 rko[6] = rko[5] ^ rki[6]; 289 rko[7] = rko[6] ^ rki[7]; 290 } 291 } 292 293 /* 294 * Generate the decryption keys for the Equivalent Inverse Cipher. 295 * This involves reversing the order of the round keys, and applying 296 * the Inverse Mix Columns transformation to all but the first and 297 * the last one. 298 */ 299 if (inv_rndkeys) { 300 inv_rndkeys[0] = rndkeys[key_len + 24]; 301 inv_rndkeys[1] = rndkeys[key_len + 25]; 302 inv_rndkeys[2] = rndkeys[key_len + 26]; 303 inv_rndkeys[3] = rndkeys[key_len + 27]; 304 305 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 306 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 307 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 308 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 309 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 310 } 311 312 inv_rndkeys[i] = rndkeys[0]; 313 inv_rndkeys[i + 1] = rndkeys[1]; 314 inv_rndkeys[i + 2] = rndkeys[2]; 315 inv_rndkeys[i + 3] = rndkeys[3]; 316 } 317 } 318 319 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 320 unsigned int key_len) 321 { 322 if (aes_check_keylen(key_len) != 0) 323 return -EINVAL; 324 ctx->key_length = key_len; 325 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 326 return 0; 327 } 328 EXPORT_SYMBOL(aes_expandkey); 329 330 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 331 { 332 return rk ^ aes_enc_tab[(u8)w[i]] ^ 333 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 334 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 335 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 336 } 337 338 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 339 { 340 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 341 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 342 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 343 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 344 } 345 346 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 347 u8 out[AES_BLOCK_SIZE], 348 const u8 in[AES_BLOCK_SIZE]) 349 { 350 const u32 *rkp = rndkeys; 351 int n = nrounds - 1; 352 u32 w[4]; 353 354 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 355 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 356 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 357 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 358 359 /* 360 * Prefetch the table before doing data and key-dependent loads from it. 361 * 362 * This is intended only as a basic constant-time hardening measure that 363 * avoids interfering with performance too much. Its effectiveness is 364 * not guaranteed. For proper constant-time AES, a CPU that supports 365 * AES instructions should be used instead. 366 */ 367 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 368 369 do { 370 u32 w0 = enc_quarterround(w, 0, *rkp++); 371 u32 w1 = enc_quarterround(w, 1, *rkp++); 372 u32 w2 = enc_quarterround(w, 2, *rkp++); 373 u32 w3 = enc_quarterround(w, 3, *rkp++); 374 375 w[0] = w0; 376 w[1] = w1; 377 w[2] = w2; 378 w[3] = w3; 379 } while (--n); 380 381 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 382 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 383 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 384 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 385 } 386 387 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 388 { 389 return rk ^ aes_dec_tab[(u8)w[i]] ^ 390 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 391 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 392 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 393 } 394 395 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 396 { 397 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 398 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 399 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 400 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 401 } 402 403 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 404 int nrounds, 405 u8 out[AES_BLOCK_SIZE], 406 const u8 in[AES_BLOCK_SIZE]) 407 { 408 const u32 *rkp = inv_rndkeys; 409 int n = nrounds - 1; 410 u32 w[4]; 411 412 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 413 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 414 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 415 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 416 417 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 418 419 do { 420 u32 w0 = dec_quarterround(w, 0, *rkp++); 421 u32 w1 = dec_quarterround(w, 1, *rkp++); 422 u32 w2 = dec_quarterround(w, 2, *rkp++); 423 u32 w3 = dec_quarterround(w, 3, *rkp++); 424 425 w[0] = w0; 426 w[1] = w1; 427 w[2] = w2; 428 w[3] = w3; 429 } while (--n); 430 431 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 432 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 433 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 434 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 435 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 436 } 437 438 /* 439 * Note: the aes_prepare*key_* names reflect the fact that the implementation 440 * might not actually expand the key. (The s390 code for example doesn't.) 441 * Where the key is expanded we use the more specific names aes_expandkey_*. 442 * 443 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 444 * the area to store the prepared decryption key. It will be NULL if the user 445 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 446 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 447 */ 448 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 449 /* An arch-specific implementation of AES is available. Include it. */ 450 #include "aes.h" /* $(SRCARCH)/aes.h */ 451 #else 452 /* No arch-specific implementation of AES is available. Use generic code. */ 453 454 static void aes_preparekey_arch(union aes_enckey_arch *k, 455 union aes_invkey_arch *inv_k, 456 const u8 *in_key, int key_len, int nrounds) 457 { 458 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 459 in_key, key_len); 460 } 461 462 static void aes_encrypt_arch(const struct aes_enckey *key, 463 u8 out[AES_BLOCK_SIZE], 464 const u8 in[AES_BLOCK_SIZE]) 465 { 466 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 467 } 468 469 static void aes_decrypt_arch(const struct aes_key *key, 470 u8 out[AES_BLOCK_SIZE], 471 const u8 in[AES_BLOCK_SIZE]) 472 { 473 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 474 } 475 #endif 476 477 static int __aes_preparekey(struct aes_enckey *enc_key, 478 union aes_invkey_arch *inv_k, 479 const u8 *in_key, size_t key_len) 480 { 481 if (aes_check_keylen(key_len) != 0) 482 return -EINVAL; 483 enc_key->len = key_len; 484 enc_key->nrounds = 6 + key_len / 4; 485 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 486 enc_key->nrounds); 487 return 0; 488 } 489 490 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 491 { 492 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 493 in_key, key_len); 494 } 495 EXPORT_SYMBOL(aes_preparekey); 496 497 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 498 { 499 return __aes_preparekey(key, NULL, in_key, key_len); 500 } 501 EXPORT_SYMBOL(aes_prepareenckey); 502 503 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 504 const u8 in[AES_BLOCK_SIZE]) 505 { 506 aes_encrypt_arch(key.enc_key, out, in); 507 } 508 EXPORT_SYMBOL(aes_encrypt); 509 510 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 511 const u8 in[AES_BLOCK_SIZE]) 512 { 513 aes_decrypt_arch(key, out, in); 514 } 515 EXPORT_SYMBOL(aes_decrypt); 516 517 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 518 519 #ifndef aes_cbcmac_blocks_arch 520 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 521 const struct aes_enckey *key, const u8 *data, 522 size_t nblocks, bool enc_before, 523 bool enc_after) 524 { 525 return false; 526 } 527 #endif 528 529 /* This assumes nblocks >= 1. */ 530 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 531 const struct aes_enckey *key, const u8 *data, 532 size_t nblocks, bool enc_before, bool enc_after) 533 { 534 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 535 enc_after)) 536 return; 537 538 if (enc_before) 539 aes_encrypt(key, h, h); 540 for (; nblocks > 1; nblocks--) { 541 crypto_xor(h, data, AES_BLOCK_SIZE); 542 data += AES_BLOCK_SIZE; 543 aes_encrypt(key, h, h); 544 } 545 crypto_xor(h, data, AES_BLOCK_SIZE); 546 if (enc_after) 547 aes_encrypt(key, h, h); 548 } 549 550 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 551 size_t key_len) 552 { 553 u64 hi, lo, mask; 554 int err; 555 556 /* Prepare the AES key. */ 557 err = aes_prepareenckey(&key->aes, in_key, key_len); 558 if (err) 559 return err; 560 561 /* 562 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 563 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 564 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 565 */ 566 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 567 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 568 hi = be64_to_cpu(key->k_final[0].w[0]); 569 lo = be64_to_cpu(key->k_final[0].w[1]); 570 for (int i = 0; i < 2; i++) { 571 mask = ((s64)hi >> 63) & 0x87; 572 hi = (hi << 1) ^ (lo >> 63); 573 lo = (lo << 1) ^ mask; 574 key->k_final[i].w[0] = cpu_to_be64(hi); 575 key->k_final[i].w[1] = cpu_to_be64(lo); 576 } 577 return 0; 578 } 579 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 580 581 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 582 const u8 in_key[AES_KEYSIZE_128]) 583 { 584 static const u8 constants[3][AES_BLOCK_SIZE] = { 585 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 586 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 587 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 588 }; 589 u8 new_aes_key[AES_BLOCK_SIZE]; 590 591 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 592 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 593 aes_encrypt(&key->aes, new_aes_key, constants[0]); 594 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 595 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 596 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 597 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 598 } 599 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 600 601 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 602 { 603 bool enc_before = false; 604 size_t nblocks; 605 606 if (ctx->partial_len) { 607 /* XOR data into a pending block. */ 608 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 609 610 crypto_xor(&ctx->h[ctx->partial_len], data, l); 611 data += l; 612 data_len -= l; 613 ctx->partial_len += l; 614 if (data_len == 0) { 615 /* 616 * Either the pending block hasn't been filled yet, or 617 * no more data was given so it's not yet known whether 618 * the block is the final block. 619 */ 620 return; 621 } 622 /* Pending block has been filled and isn't the final block. */ 623 enc_before = true; 624 } 625 626 nblocks = data_len / AES_BLOCK_SIZE; 627 data_len %= AES_BLOCK_SIZE; 628 if (nblocks == 0) { 629 /* 0 additional full blocks, then optionally a partial block */ 630 if (enc_before) 631 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 632 crypto_xor(ctx->h, data, data_len); 633 ctx->partial_len = data_len; 634 } else if (data_len != 0) { 635 /* 1 or more additional full blocks, then a partial block */ 636 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 637 enc_before, /* enc_after= */ true); 638 data += nblocks * AES_BLOCK_SIZE; 639 crypto_xor(ctx->h, data, data_len); 640 ctx->partial_len = data_len; 641 } else { 642 /* 643 * 1 or more additional full blocks only. Encryption of the 644 * last block is delayed until it's known whether it's the final 645 * block in the message or not. 646 */ 647 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 648 enc_before, /* enc_after= */ false); 649 ctx->partial_len = AES_BLOCK_SIZE; 650 } 651 } 652 EXPORT_SYMBOL_GPL(aes_cmac_update); 653 654 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 655 { 656 if (ctx->partial_len == AES_BLOCK_SIZE) { 657 /* Final block is a full block. Use k_final[0]. */ 658 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 659 } else { 660 /* Final block is a partial block. Pad, and use k_final[1]. */ 661 ctx->h[ctx->partial_len] ^= 0x80; 662 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 663 } 664 aes_encrypt(&ctx->key->aes, out, ctx->h); 665 memzero_explicit(ctx, sizeof(*ctx)); 666 } 667 EXPORT_SYMBOL_GPL(aes_cmac_final); 668 669 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 670 size_t data_len) 671 { 672 bool enc_before = false; 673 size_t nblocks; 674 675 if (ctx->partial_len) { 676 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 677 678 crypto_xor(&ctx->h[ctx->partial_len], data, l); 679 data += l; 680 data_len -= l; 681 ctx->partial_len += l; 682 if (ctx->partial_len < AES_BLOCK_SIZE) 683 return; 684 enc_before = true; 685 } 686 687 nblocks = data_len / AES_BLOCK_SIZE; 688 data_len %= AES_BLOCK_SIZE; 689 if (nblocks == 0) { 690 if (enc_before) 691 aes_encrypt(ctx->key, ctx->h, ctx->h); 692 } else { 693 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 694 /* enc_after= */ true); 695 data += nblocks * AES_BLOCK_SIZE; 696 } 697 crypto_xor(ctx->h, data, data_len); 698 ctx->partial_len = data_len; 699 } 700 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 701 702 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 703 { 704 if (ctx->partial_len) 705 aes_encrypt(ctx->key, out, ctx->h); 706 else 707 memcpy(out, ctx->h, AES_BLOCK_SIZE); 708 memzero_explicit(ctx, sizeof(*ctx)); 709 } 710 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 711 #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 712 713 #ifdef aes_mod_init_arch 714 static int __init aes_mod_init(void) 715 { 716 aes_mod_init_arch(); 717 return 0; 718 } 719 subsys_initcall(aes_mod_init); 720 721 static void __exit aes_mod_exit(void) 722 { 723 } 724 module_exit(aes_mod_exit); 725 #endif 726 727 MODULE_DESCRIPTION("AES block cipher"); 728 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 729 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 730 MODULE_LICENSE("GPL v2"); 731