1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes.h> 9 #include <crypto/utils.h> 10 #include <linux/cache.h> 11 #include <linux/crypto.h> 12 #include <linux/export.h> 13 #include <linux/module.h> 14 #include <linux/unaligned.h> 15 #include "fips.h" 16 17 static const u8 ____cacheline_aligned aes_sbox[] = { 18 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 19 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 20 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 21 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 22 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 23 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 24 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 25 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 26 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 27 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 28 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 29 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 30 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 31 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 32 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 33 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 34 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 35 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 36 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 37 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 38 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 39 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 40 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 41 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 42 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 43 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 44 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 45 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 46 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 47 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 48 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 49 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 50 }; 51 52 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 53 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 54 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 55 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 56 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 57 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 58 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 59 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 60 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 61 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 62 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 63 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 64 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 65 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 66 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 67 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 68 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 69 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 70 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 71 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 72 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 73 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 74 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 75 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 76 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 77 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 78 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 79 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 80 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 81 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 82 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 83 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 84 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 85 }; 86 87 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 88 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 89 90 EXPORT_SYMBOL(crypto_aes_sbox); 91 EXPORT_SYMBOL(crypto_aes_inv_sbox); 92 93 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 94 const u32 ____cacheline_aligned aes_enc_tab[256] = { 95 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 96 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 97 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 98 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 99 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 100 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 101 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 102 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 103 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 104 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 105 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 106 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 107 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 108 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 109 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 110 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 111 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 112 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 113 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 114 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 115 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 116 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 117 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 118 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 119 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 120 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 121 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 122 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 123 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 124 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 125 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 126 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 127 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 128 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 129 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 130 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 131 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 132 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 133 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 134 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 135 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 136 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 137 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 138 }; 139 EXPORT_SYMBOL(aes_enc_tab); 140 141 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 142 const u32 ____cacheline_aligned aes_dec_tab[256] = { 143 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 144 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 145 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 146 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 147 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 148 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 149 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 150 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 151 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 152 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 153 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 154 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 155 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 156 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 157 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 158 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 159 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 160 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 161 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 162 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 163 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 164 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 165 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 166 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 167 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 168 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 169 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 170 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 171 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 172 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 173 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 174 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 175 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 176 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 177 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 178 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 179 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 180 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 181 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 182 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 183 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 184 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 185 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 186 }; 187 EXPORT_SYMBOL(aes_dec_tab); 188 189 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 190 static __always_inline void aes_prefetch(const void *mem, size_t len) 191 { 192 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 193 *(volatile const u8 *)(mem + i); 194 barrier(); 195 } 196 197 static u32 mul_by_x(u32 w) 198 { 199 u32 x = w & 0x7f7f7f7f; 200 u32 y = w & 0x80808080; 201 202 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 203 return (x << 1) ^ (y >> 7) * 0x1b; 204 } 205 206 static u32 mul_by_x2(u32 w) 207 { 208 u32 x = w & 0x3f3f3f3f; 209 u32 y = w & 0x80808080; 210 u32 z = w & 0x40404040; 211 212 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 213 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 214 } 215 216 static u32 mix_columns(u32 x) 217 { 218 /* 219 * Perform the following matrix multiplication in GF(2^8) 220 * 221 * | 0x2 0x3 0x1 0x1 | | x[0] | 222 * | 0x1 0x2 0x3 0x1 | | x[1] | 223 * | 0x1 0x1 0x2 0x3 | x | x[2] | 224 * | 0x3 0x1 0x1 0x2 | | x[3] | 225 */ 226 u32 y = mul_by_x(x) ^ ror32(x, 16); 227 228 return y ^ ror32(x ^ y, 8); 229 } 230 231 static u32 inv_mix_columns(u32 x) 232 { 233 /* 234 * Perform the following matrix multiplication in GF(2^8) 235 * 236 * | 0xe 0xb 0xd 0x9 | | x[0] | 237 * | 0x9 0xe 0xb 0xd | | x[1] | 238 * | 0xd 0x9 0xe 0xb | x | x[2] | 239 * | 0xb 0xd 0x9 0xe | | x[3] | 240 * 241 * which can conveniently be reduced to 242 * 243 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 244 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 245 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 246 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 247 */ 248 u32 y = mul_by_x2(x); 249 250 return mix_columns(x ^ y ^ ror32(y, 16)); 251 } 252 253 static u32 subw(u32 in) 254 { 255 return (aes_sbox[in & 0xff]) ^ 256 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 257 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 258 (aes_sbox[(in >> 24) & 0xff] << 24); 259 } 260 261 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 262 const u8 *in_key, int key_len) 263 { 264 u32 kwords = key_len / sizeof(u32); 265 u32 rc, i, j; 266 267 for (i = 0; i < kwords; i++) 268 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 269 270 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 271 u32 *rki = &rndkeys[i * kwords]; 272 u32 *rko = rki + kwords; 273 274 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 275 rko[1] = rko[0] ^ rki[1]; 276 rko[2] = rko[1] ^ rki[2]; 277 rko[3] = rko[2] ^ rki[3]; 278 279 if (key_len == AES_KEYSIZE_192) { 280 if (i >= 7) 281 break; 282 rko[4] = rko[3] ^ rki[4]; 283 rko[5] = rko[4] ^ rki[5]; 284 } else if (key_len == AES_KEYSIZE_256) { 285 if (i >= 6) 286 break; 287 rko[4] = subw(rko[3]) ^ rki[4]; 288 rko[5] = rko[4] ^ rki[5]; 289 rko[6] = rko[5] ^ rki[6]; 290 rko[7] = rko[6] ^ rki[7]; 291 } 292 } 293 294 /* 295 * Generate the decryption keys for the Equivalent Inverse Cipher. 296 * This involves reversing the order of the round keys, and applying 297 * the Inverse Mix Columns transformation to all but the first and 298 * the last one. 299 */ 300 if (inv_rndkeys) { 301 inv_rndkeys[0] = rndkeys[key_len + 24]; 302 inv_rndkeys[1] = rndkeys[key_len + 25]; 303 inv_rndkeys[2] = rndkeys[key_len + 26]; 304 inv_rndkeys[3] = rndkeys[key_len + 27]; 305 306 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 307 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 308 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 309 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 310 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 311 } 312 313 inv_rndkeys[i] = rndkeys[0]; 314 inv_rndkeys[i + 1] = rndkeys[1]; 315 inv_rndkeys[i + 2] = rndkeys[2]; 316 inv_rndkeys[i + 3] = rndkeys[3]; 317 } 318 } 319 320 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 321 unsigned int key_len) 322 { 323 if (aes_check_keylen(key_len) != 0) 324 return -EINVAL; 325 ctx->key_length = key_len; 326 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 327 return 0; 328 } 329 EXPORT_SYMBOL(aes_expandkey); 330 331 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 332 { 333 return rk ^ aes_enc_tab[(u8)w[i]] ^ 334 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 335 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 336 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 337 } 338 339 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 340 { 341 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 342 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 343 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 344 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 345 } 346 347 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 348 u8 out[AES_BLOCK_SIZE], 349 const u8 in[AES_BLOCK_SIZE]) 350 { 351 const u32 *rkp = rndkeys; 352 int n = nrounds - 1; 353 u32 w[4]; 354 355 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 356 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 357 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 358 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 359 360 /* 361 * Prefetch the table before doing data and key-dependent loads from it. 362 * 363 * This is intended only as a basic constant-time hardening measure that 364 * avoids interfering with performance too much. Its effectiveness is 365 * not guaranteed. For proper constant-time AES, a CPU that supports 366 * AES instructions should be used instead. 367 */ 368 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 369 370 do { 371 u32 w0 = enc_quarterround(w, 0, *rkp++); 372 u32 w1 = enc_quarterround(w, 1, *rkp++); 373 u32 w2 = enc_quarterround(w, 2, *rkp++); 374 u32 w3 = enc_quarterround(w, 3, *rkp++); 375 376 w[0] = w0; 377 w[1] = w1; 378 w[2] = w2; 379 w[3] = w3; 380 } while (--n); 381 382 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 383 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 384 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 385 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 386 } 387 388 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 389 { 390 return rk ^ aes_dec_tab[(u8)w[i]] ^ 391 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 392 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 393 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 394 } 395 396 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 397 { 398 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 399 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 400 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 401 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 402 } 403 404 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 405 int nrounds, 406 u8 out[AES_BLOCK_SIZE], 407 const u8 in[AES_BLOCK_SIZE]) 408 { 409 const u32 *rkp = inv_rndkeys; 410 int n = nrounds - 1; 411 u32 w[4]; 412 413 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 414 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 415 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 416 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 417 418 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 419 420 do { 421 u32 w0 = dec_quarterround(w, 0, *rkp++); 422 u32 w1 = dec_quarterround(w, 1, *rkp++); 423 u32 w2 = dec_quarterround(w, 2, *rkp++); 424 u32 w3 = dec_quarterround(w, 3, *rkp++); 425 426 w[0] = w0; 427 w[1] = w1; 428 w[2] = w2; 429 w[3] = w3; 430 } while (--n); 431 432 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 433 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 434 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 435 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 436 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 437 } 438 439 /* 440 * Note: the aes_prepare*key_* names reflect the fact that the implementation 441 * might not actually expand the key. (The s390 code for example doesn't.) 442 * Where the key is expanded we use the more specific names aes_expandkey_*. 443 * 444 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 445 * the area to store the prepared decryption key. It will be NULL if the user 446 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 447 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 448 */ 449 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 450 /* An arch-specific implementation of AES is available. Include it. */ 451 #include "aes.h" /* $(SRCARCH)/aes.h */ 452 #else 453 /* No arch-specific implementation of AES is available. Use generic code. */ 454 455 static void aes_preparekey_arch(union aes_enckey_arch *k, 456 union aes_invkey_arch *inv_k, 457 const u8 *in_key, int key_len, int nrounds) 458 { 459 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 460 in_key, key_len); 461 } 462 463 static void aes_encrypt_arch(const struct aes_enckey *key, 464 u8 out[AES_BLOCK_SIZE], 465 const u8 in[AES_BLOCK_SIZE]) 466 { 467 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 468 } 469 470 static void aes_decrypt_arch(const struct aes_key *key, 471 u8 out[AES_BLOCK_SIZE], 472 const u8 in[AES_BLOCK_SIZE]) 473 { 474 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 475 } 476 #endif 477 478 static int __aes_preparekey(struct aes_enckey *enc_key, 479 union aes_invkey_arch *inv_k, 480 const u8 *in_key, size_t key_len) 481 { 482 if (aes_check_keylen(key_len) != 0) 483 return -EINVAL; 484 enc_key->len = key_len; 485 enc_key->nrounds = 6 + key_len / 4; 486 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 487 enc_key->nrounds); 488 return 0; 489 } 490 491 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 492 { 493 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 494 in_key, key_len); 495 } 496 EXPORT_SYMBOL(aes_preparekey); 497 498 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 499 { 500 return __aes_preparekey(key, NULL, in_key, key_len); 501 } 502 EXPORT_SYMBOL(aes_prepareenckey); 503 504 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 505 const u8 in[AES_BLOCK_SIZE]) 506 { 507 aes_encrypt_arch(key.enc_key, out, in); 508 } 509 EXPORT_SYMBOL(aes_encrypt); 510 511 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 512 const u8 in[AES_BLOCK_SIZE]) 513 { 514 aes_decrypt_arch(key, out, in); 515 } 516 EXPORT_SYMBOL(aes_decrypt); 517 518 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 519 520 #ifndef aes_cbcmac_blocks_arch 521 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 522 const struct aes_enckey *key, const u8 *data, 523 size_t nblocks, bool enc_before, 524 bool enc_after) 525 { 526 return false; 527 } 528 #endif 529 530 /* This assumes nblocks >= 1. */ 531 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 532 const struct aes_enckey *key, const u8 *data, 533 size_t nblocks, bool enc_before, bool enc_after) 534 { 535 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 536 enc_after)) 537 return; 538 539 if (enc_before) 540 aes_encrypt(key, h, h); 541 for (; nblocks > 1; nblocks--) { 542 crypto_xor(h, data, AES_BLOCK_SIZE); 543 data += AES_BLOCK_SIZE; 544 aes_encrypt(key, h, h); 545 } 546 crypto_xor(h, data, AES_BLOCK_SIZE); 547 if (enc_after) 548 aes_encrypt(key, h, h); 549 } 550 551 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 552 size_t key_len) 553 { 554 u64 hi, lo, mask; 555 int err; 556 557 /* Prepare the AES key. */ 558 err = aes_prepareenckey(&key->aes, in_key, key_len); 559 if (err) 560 return err; 561 562 /* 563 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 564 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 565 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 566 */ 567 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 568 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 569 hi = be64_to_cpu(key->k_final[0].w[0]); 570 lo = be64_to_cpu(key->k_final[0].w[1]); 571 for (int i = 0; i < 2; i++) { 572 mask = ((s64)hi >> 63) & 0x87; 573 hi = (hi << 1) ^ (lo >> 63); 574 lo = (lo << 1) ^ mask; 575 key->k_final[i].w[0] = cpu_to_be64(hi); 576 key->k_final[i].w[1] = cpu_to_be64(lo); 577 } 578 return 0; 579 } 580 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 581 582 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 583 const u8 in_key[AES_KEYSIZE_128]) 584 { 585 static const u8 constants[3][AES_BLOCK_SIZE] = { 586 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 587 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 588 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 589 }; 590 u8 new_aes_key[AES_BLOCK_SIZE]; 591 592 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 593 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 594 aes_encrypt(&key->aes, new_aes_key, constants[0]); 595 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 596 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 597 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 598 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 599 } 600 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 601 602 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 603 { 604 bool enc_before = false; 605 size_t nblocks; 606 607 if (ctx->partial_len) { 608 /* XOR data into a pending block. */ 609 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 610 611 crypto_xor(&ctx->h[ctx->partial_len], data, l); 612 data += l; 613 data_len -= l; 614 ctx->partial_len += l; 615 if (data_len == 0) { 616 /* 617 * Either the pending block hasn't been filled yet, or 618 * no more data was given so it's not yet known whether 619 * the block is the final block. 620 */ 621 return; 622 } 623 /* Pending block has been filled and isn't the final block. */ 624 enc_before = true; 625 } 626 627 nblocks = data_len / AES_BLOCK_SIZE; 628 data_len %= AES_BLOCK_SIZE; 629 if (nblocks == 0) { 630 /* 0 additional full blocks, then optionally a partial block */ 631 if (enc_before) 632 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 633 crypto_xor(ctx->h, data, data_len); 634 ctx->partial_len = data_len; 635 } else if (data_len != 0) { 636 /* 1 or more additional full blocks, then a partial block */ 637 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 638 enc_before, /* enc_after= */ true); 639 data += nblocks * AES_BLOCK_SIZE; 640 crypto_xor(ctx->h, data, data_len); 641 ctx->partial_len = data_len; 642 } else { 643 /* 644 * 1 or more additional full blocks only. Encryption of the 645 * last block is delayed until it's known whether it's the final 646 * block in the message or not. 647 */ 648 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 649 enc_before, /* enc_after= */ false); 650 ctx->partial_len = AES_BLOCK_SIZE; 651 } 652 } 653 EXPORT_SYMBOL_GPL(aes_cmac_update); 654 655 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 656 { 657 if (ctx->partial_len == AES_BLOCK_SIZE) { 658 /* Final block is a full block. Use k_final[0]. */ 659 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 660 } else { 661 /* Final block is a partial block. Pad, and use k_final[1]. */ 662 ctx->h[ctx->partial_len] ^= 0x80; 663 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 664 } 665 aes_encrypt(&ctx->key->aes, out, ctx->h); 666 memzero_explicit(ctx, sizeof(*ctx)); 667 } 668 EXPORT_SYMBOL_GPL(aes_cmac_final); 669 670 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 671 size_t data_len) 672 { 673 bool enc_before = false; 674 size_t nblocks; 675 676 if (ctx->partial_len) { 677 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 678 679 crypto_xor(&ctx->h[ctx->partial_len], data, l); 680 data += l; 681 data_len -= l; 682 ctx->partial_len += l; 683 if (ctx->partial_len < AES_BLOCK_SIZE) 684 return; 685 enc_before = true; 686 } 687 688 nblocks = data_len / AES_BLOCK_SIZE; 689 data_len %= AES_BLOCK_SIZE; 690 if (nblocks == 0) { 691 if (enc_before) 692 aes_encrypt(ctx->key, ctx->h, ctx->h); 693 } else { 694 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 695 /* enc_after= */ true); 696 data += nblocks * AES_BLOCK_SIZE; 697 } 698 crypto_xor(ctx->h, data, data_len); 699 ctx->partial_len = data_len; 700 } 701 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 702 703 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 704 { 705 if (ctx->partial_len) 706 aes_encrypt(ctx->key, out, ctx->h); 707 else 708 memcpy(out, ctx->h, AES_BLOCK_SIZE); 709 memzero_explicit(ctx, sizeof(*ctx)); 710 } 711 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 712 713 /* 714 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 715 * Implementation Guidance, a cryptographic algorithm self-test for at least one 716 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 717 * is implemented. This fulfills that requirement via AES-CMAC. 718 * 719 * This is just for FIPS. The full tests are in the KUnit test suite. 720 */ 721 static void __init aes_cmac_fips_test(void) 722 { 723 struct aes_cmac_key key; 724 u8 mac[AES_BLOCK_SIZE]; 725 726 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 727 0) 728 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 729 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 730 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 731 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 732 memzero_explicit(&key, sizeof(key)); 733 } 734 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 735 static inline void aes_cmac_fips_test(void) 736 { 737 } 738 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 739 740 static int __init aes_mod_init(void) 741 { 742 #ifdef aes_mod_init_arch 743 aes_mod_init_arch(); 744 #endif 745 if (fips_enabled) 746 aes_cmac_fips_test(); 747 return 0; 748 } 749 subsys_initcall(aes_mod_init); 750 751 static void __exit aes_mod_exit(void) 752 { 753 } 754 module_exit(aes_mod_exit); 755 756 MODULE_DESCRIPTION("AES block cipher"); 757 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 758 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 759 MODULE_LICENSE("GPL v2"); 760