1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ctr.h> 10 #include <crypto/aes-ecb.h> 11 #include <crypto/aes.h> 12 #include <crypto/utils.h> 13 #include <linux/cache.h> 14 #include <linux/crypto.h> 15 #include <linux/export.h> 16 #include <linux/module.h> 17 #include <linux/unaligned.h> 18 #include "fips.h" 19 20 static const u8 ____cacheline_aligned aes_sbox[] = { 21 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 22 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 23 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 24 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 25 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 26 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 27 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 28 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 29 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 30 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 31 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 32 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 33 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 34 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 35 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 36 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 37 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 38 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 39 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 40 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 41 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 42 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 43 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 44 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 45 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 46 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 47 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 48 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 49 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 50 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 51 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 52 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 53 }; 54 55 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 56 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 57 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 58 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 59 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 60 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 61 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 62 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 63 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 64 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 65 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 66 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 67 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 68 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 69 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 70 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 71 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 72 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 73 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 74 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 75 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 76 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 77 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 78 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 79 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 80 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 81 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 82 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 83 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 84 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 85 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 86 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 87 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 88 }; 89 90 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 91 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 92 93 EXPORT_SYMBOL(crypto_aes_sbox); 94 EXPORT_SYMBOL(crypto_aes_inv_sbox); 95 96 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 97 const u32 ____cacheline_aligned aes_enc_tab[256] = { 98 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 99 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 100 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 101 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 102 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 103 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 104 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 105 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 106 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 107 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 108 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 109 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 110 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 111 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 112 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 113 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 114 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 115 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 116 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 117 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 118 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 119 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 120 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 121 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 122 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 123 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 124 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 125 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 126 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 127 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 128 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 129 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 130 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 131 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 132 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 133 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 134 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 135 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 136 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 137 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 138 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 139 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 140 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 141 }; 142 EXPORT_SYMBOL(aes_enc_tab); 143 144 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 145 const u32 ____cacheline_aligned aes_dec_tab[256] = { 146 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 147 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 148 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 149 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 150 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 151 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 152 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 153 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 154 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 155 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 156 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 157 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 158 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 159 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 160 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 161 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 162 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 163 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 164 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 165 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 166 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 167 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 168 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 169 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 170 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 171 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 172 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 173 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 174 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 175 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 176 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 177 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 178 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 179 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 180 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 181 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 182 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 183 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 184 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 185 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 186 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 187 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 188 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 189 }; 190 EXPORT_SYMBOL(aes_dec_tab); 191 192 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 193 static __always_inline void aes_prefetch(const void *mem, size_t len) 194 { 195 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 196 *(volatile const u8 *)(mem + i); 197 barrier(); 198 } 199 200 static u32 mul_by_x(u32 w) 201 { 202 u32 x = w & 0x7f7f7f7f; 203 u32 y = w & 0x80808080; 204 205 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 206 return (x << 1) ^ (y >> 7) * 0x1b; 207 } 208 209 static u32 mul_by_x2(u32 w) 210 { 211 u32 x = w & 0x3f3f3f3f; 212 u32 y = w & 0x80808080; 213 u32 z = w & 0x40404040; 214 215 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 216 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 217 } 218 219 static u32 mix_columns(u32 x) 220 { 221 /* 222 * Perform the following matrix multiplication in GF(2^8) 223 * 224 * | 0x2 0x3 0x1 0x1 | | x[0] | 225 * | 0x1 0x2 0x3 0x1 | | x[1] | 226 * | 0x1 0x1 0x2 0x3 | x | x[2] | 227 * | 0x3 0x1 0x1 0x2 | | x[3] | 228 */ 229 u32 y = mul_by_x(x) ^ ror32(x, 16); 230 231 return y ^ ror32(x ^ y, 8); 232 } 233 234 static u32 inv_mix_columns(u32 x) 235 { 236 /* 237 * Perform the following matrix multiplication in GF(2^8) 238 * 239 * | 0xe 0xb 0xd 0x9 | | x[0] | 240 * | 0x9 0xe 0xb 0xd | | x[1] | 241 * | 0xd 0x9 0xe 0xb | x | x[2] | 242 * | 0xb 0xd 0x9 0xe | | x[3] | 243 * 244 * which can conveniently be reduced to 245 * 246 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 247 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 248 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 249 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 250 */ 251 u32 y = mul_by_x2(x); 252 253 return mix_columns(x ^ y ^ ror32(y, 16)); 254 } 255 256 static u32 subw(u32 in) 257 { 258 return (aes_sbox[in & 0xff]) ^ 259 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 260 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 261 (aes_sbox[(in >> 24) & 0xff] << 24); 262 } 263 264 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 265 const u8 *in_key, int key_len) 266 { 267 u32 kwords = key_len / sizeof(u32); 268 u32 rc, i, j; 269 270 for (i = 0; i < kwords; i++) 271 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 272 273 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 274 u32 *rki = &rndkeys[i * kwords]; 275 u32 *rko = rki + kwords; 276 277 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 278 rko[1] = rko[0] ^ rki[1]; 279 rko[2] = rko[1] ^ rki[2]; 280 rko[3] = rko[2] ^ rki[3]; 281 282 if (key_len == AES_KEYSIZE_192) { 283 if (i >= 7) 284 break; 285 rko[4] = rko[3] ^ rki[4]; 286 rko[5] = rko[4] ^ rki[5]; 287 } else if (key_len == AES_KEYSIZE_256) { 288 if (i >= 6) 289 break; 290 rko[4] = subw(rko[3]) ^ rki[4]; 291 rko[5] = rko[4] ^ rki[5]; 292 rko[6] = rko[5] ^ rki[6]; 293 rko[7] = rko[6] ^ rki[7]; 294 } 295 } 296 297 /* 298 * Generate the decryption keys for the Equivalent Inverse Cipher. 299 * This involves reversing the order of the round keys, and applying 300 * the Inverse Mix Columns transformation to all but the first and 301 * the last one. 302 */ 303 if (inv_rndkeys) { 304 inv_rndkeys[0] = rndkeys[key_len + 24]; 305 inv_rndkeys[1] = rndkeys[key_len + 25]; 306 inv_rndkeys[2] = rndkeys[key_len + 26]; 307 inv_rndkeys[3] = rndkeys[key_len + 27]; 308 309 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 310 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 311 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 312 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 313 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 314 } 315 316 inv_rndkeys[i] = rndkeys[0]; 317 inv_rndkeys[i + 1] = rndkeys[1]; 318 inv_rndkeys[i + 2] = rndkeys[2]; 319 inv_rndkeys[i + 3] = rndkeys[3]; 320 } 321 } 322 323 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 324 unsigned int key_len) 325 { 326 if (aes_check_keylen(key_len) != 0) 327 return -EINVAL; 328 ctx->key_length = key_len; 329 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 330 return 0; 331 } 332 EXPORT_SYMBOL(aes_expandkey); 333 334 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 335 { 336 return rk ^ aes_enc_tab[(u8)w[i]] ^ 337 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 338 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 339 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 340 } 341 342 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 343 { 344 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 345 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 346 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 347 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 348 } 349 350 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 351 u8 out[AES_BLOCK_SIZE], 352 const u8 in[AES_BLOCK_SIZE]) 353 { 354 const u32 *rkp = rndkeys; 355 int n = nrounds - 1; 356 u32 w[4]; 357 358 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 359 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 360 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 361 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 362 363 /* 364 * Prefetch the table before doing data and key-dependent loads from it. 365 * 366 * This is intended only as a basic constant-time hardening measure that 367 * avoids interfering with performance too much. Its effectiveness is 368 * not guaranteed. For proper constant-time AES, a CPU that supports 369 * AES instructions should be used instead. 370 */ 371 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 372 373 do { 374 u32 w0 = enc_quarterround(w, 0, *rkp++); 375 u32 w1 = enc_quarterround(w, 1, *rkp++); 376 u32 w2 = enc_quarterround(w, 2, *rkp++); 377 u32 w3 = enc_quarterround(w, 3, *rkp++); 378 379 w[0] = w0; 380 w[1] = w1; 381 w[2] = w2; 382 w[3] = w3; 383 } while (--n); 384 385 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 386 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 387 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 388 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 389 } 390 391 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 392 { 393 return rk ^ aes_dec_tab[(u8)w[i]] ^ 394 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 395 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 396 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 397 } 398 399 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 400 { 401 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 402 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 403 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 404 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 405 } 406 407 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 408 int nrounds, 409 u8 out[AES_BLOCK_SIZE], 410 const u8 in[AES_BLOCK_SIZE]) 411 { 412 const u32 *rkp = inv_rndkeys; 413 int n = nrounds - 1; 414 u32 w[4]; 415 416 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 417 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 418 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 419 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 420 421 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 422 423 do { 424 u32 w0 = dec_quarterround(w, 0, *rkp++); 425 u32 w1 = dec_quarterround(w, 1, *rkp++); 426 u32 w2 = dec_quarterround(w, 2, *rkp++); 427 u32 w3 = dec_quarterround(w, 3, *rkp++); 428 429 w[0] = w0; 430 w[1] = w1; 431 w[2] = w2; 432 w[3] = w3; 433 } while (--n); 434 435 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 436 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 437 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 438 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 439 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 440 } 441 442 /* 443 * Note: the aes_prepare*key_* names reflect the fact that the implementation 444 * might not actually expand the key. (The s390 code for example doesn't.) 445 * Where the key is expanded we use the more specific names aes_expandkey_*. 446 * 447 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 448 * the area to store the prepared decryption key. It will be NULL if the user 449 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 450 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 451 */ 452 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 453 /* An arch-specific implementation of AES is available. Include it. */ 454 #include "aes.h" /* $(SRCARCH)/aes.h */ 455 #else 456 /* No arch-specific implementation of AES is available. Use generic code. */ 457 458 static void aes_preparekey_arch(union aes_enckey_arch *k, 459 union aes_invkey_arch *inv_k, 460 const u8 *in_key, int key_len, int nrounds) 461 { 462 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 463 in_key, key_len); 464 } 465 466 static void aes_encrypt_arch(const struct aes_enckey *key, 467 u8 out[AES_BLOCK_SIZE], 468 const u8 in[AES_BLOCK_SIZE]) 469 { 470 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 471 } 472 473 static void aes_decrypt_arch(const struct aes_key *key, 474 u8 out[AES_BLOCK_SIZE], 475 const u8 in[AES_BLOCK_SIZE]) 476 { 477 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 478 } 479 #endif 480 481 static int __aes_preparekey(struct aes_enckey *enc_key, 482 union aes_invkey_arch *inv_k, 483 const u8 *in_key, size_t key_len) 484 { 485 if (aes_check_keylen(key_len) != 0) 486 return -EINVAL; 487 enc_key->len = key_len; 488 enc_key->nrounds = 6 + key_len / 4; 489 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 490 enc_key->nrounds); 491 return 0; 492 } 493 494 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 495 { 496 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 497 in_key, key_len); 498 } 499 EXPORT_SYMBOL(aes_preparekey); 500 501 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 502 { 503 return __aes_preparekey(key, NULL, in_key, key_len); 504 } 505 EXPORT_SYMBOL(aes_prepareenckey); 506 507 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 508 const u8 in[AES_BLOCK_SIZE]) 509 { 510 aes_encrypt_arch(key.enc_key, out, in); 511 } 512 EXPORT_SYMBOL(aes_encrypt); 513 514 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 515 const u8 in[AES_BLOCK_SIZE]) 516 { 517 aes_decrypt_arch(key, out, in); 518 } 519 EXPORT_SYMBOL(aes_decrypt); 520 521 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 522 523 #ifndef aes_cbcmac_blocks_arch 524 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 525 const struct aes_enckey *key, const u8 *data, 526 size_t nblocks, bool enc_before, 527 bool enc_after) 528 { 529 return false; 530 } 531 #endif 532 533 /* This assumes nblocks >= 1. */ 534 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 535 const struct aes_enckey *key, const u8 *data, 536 size_t nblocks, bool enc_before, bool enc_after) 537 { 538 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 539 enc_after)) 540 return; 541 542 if (enc_before) 543 aes_encrypt(key, h, h); 544 for (; nblocks > 1; nblocks--) { 545 crypto_xor(h, data, AES_BLOCK_SIZE); 546 data += AES_BLOCK_SIZE; 547 aes_encrypt(key, h, h); 548 } 549 crypto_xor(h, data, AES_BLOCK_SIZE); 550 if (enc_after) 551 aes_encrypt(key, h, h); 552 } 553 554 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 555 size_t key_len) 556 { 557 u64 hi, lo, mask; 558 int err; 559 560 /* Prepare the AES key. */ 561 err = aes_prepareenckey(&key->aes, in_key, key_len); 562 if (err) 563 return err; 564 565 /* 566 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 567 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 568 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 569 */ 570 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 571 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 572 hi = be64_to_cpu(key->k_final[0].w[0]); 573 lo = be64_to_cpu(key->k_final[0].w[1]); 574 for (int i = 0; i < 2; i++) { 575 mask = ((s64)hi >> 63) & 0x87; 576 hi = (hi << 1) ^ (lo >> 63); 577 lo = (lo << 1) ^ mask; 578 key->k_final[i].w[0] = cpu_to_be64(hi); 579 key->k_final[i].w[1] = cpu_to_be64(lo); 580 } 581 return 0; 582 } 583 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 584 585 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 586 const u8 in_key[AES_KEYSIZE_128]) 587 { 588 static const u8 constants[3][AES_BLOCK_SIZE] = { 589 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 590 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 591 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 592 }; 593 u8 new_aes_key[AES_BLOCK_SIZE]; 594 595 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 596 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 597 aes_encrypt(&key->aes, new_aes_key, constants[0]); 598 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 599 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 600 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 601 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 602 } 603 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 604 605 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 606 { 607 bool enc_before = false; 608 size_t nblocks; 609 610 if (ctx->partial_len) { 611 /* XOR data into a pending block. */ 612 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 613 614 crypto_xor(&ctx->h[ctx->partial_len], data, l); 615 data += l; 616 data_len -= l; 617 ctx->partial_len += l; 618 if (data_len == 0) { 619 /* 620 * Either the pending block hasn't been filled yet, or 621 * no more data was given so it's not yet known whether 622 * the block is the final block. 623 */ 624 return; 625 } 626 /* Pending block has been filled and isn't the final block. */ 627 enc_before = true; 628 } 629 630 nblocks = data_len / AES_BLOCK_SIZE; 631 data_len %= AES_BLOCK_SIZE; 632 if (nblocks == 0) { 633 /* 0 additional full blocks, then optionally a partial block */ 634 if (enc_before) 635 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 636 crypto_xor(ctx->h, data, data_len); 637 ctx->partial_len = data_len; 638 } else if (data_len != 0) { 639 /* 1 or more additional full blocks, then a partial block */ 640 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 641 enc_before, /* enc_after= */ true); 642 data += nblocks * AES_BLOCK_SIZE; 643 crypto_xor(ctx->h, data, data_len); 644 ctx->partial_len = data_len; 645 } else { 646 /* 647 * 1 or more additional full blocks only. Encryption of the 648 * last block is delayed until it's known whether it's the final 649 * block in the message or not. 650 */ 651 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 652 enc_before, /* enc_after= */ false); 653 ctx->partial_len = AES_BLOCK_SIZE; 654 } 655 } 656 EXPORT_SYMBOL_GPL(aes_cmac_update); 657 658 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 659 { 660 if (ctx->partial_len == AES_BLOCK_SIZE) { 661 /* Final block is a full block. Use k_final[0]. */ 662 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 663 } else { 664 /* Final block is a partial block. Pad, and use k_final[1]. */ 665 ctx->h[ctx->partial_len] ^= 0x80; 666 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 667 } 668 aes_encrypt(&ctx->key->aes, out, ctx->h); 669 memzero_explicit(ctx, sizeof(*ctx)); 670 } 671 EXPORT_SYMBOL_GPL(aes_cmac_final); 672 673 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 674 size_t data_len) 675 { 676 bool enc_before = false; 677 size_t nblocks; 678 679 if (ctx->partial_len) { 680 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 681 682 crypto_xor(&ctx->h[ctx->partial_len], data, l); 683 data += l; 684 data_len -= l; 685 ctx->partial_len += l; 686 if (ctx->partial_len < AES_BLOCK_SIZE) 687 return; 688 enc_before = true; 689 } 690 691 nblocks = data_len / AES_BLOCK_SIZE; 692 data_len %= AES_BLOCK_SIZE; 693 if (nblocks == 0) { 694 if (enc_before) 695 aes_encrypt(ctx->key, ctx->h, ctx->h); 696 } else { 697 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 698 /* enc_after= */ true); 699 data += nblocks * AES_BLOCK_SIZE; 700 } 701 crypto_xor(ctx->h, data, data_len); 702 ctx->partial_len = data_len; 703 } 704 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 705 706 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 707 { 708 if (ctx->partial_len) 709 aes_encrypt(ctx->key, out, ctx->h); 710 else 711 memcpy(out, ctx->h, AES_BLOCK_SIZE); 712 memzero_explicit(ctx, sizeof(*ctx)); 713 } 714 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 715 716 /* 717 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 718 * Implementation Guidance, a cryptographic algorithm self-test for at least one 719 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 720 * is implemented. This fulfills that requirement via AES-CMAC. 721 * 722 * This is just for FIPS. The full tests are in the KUnit test suite. 723 */ 724 static void __init aes_cmac_fips_test(void) 725 { 726 struct aes_cmac_key key; 727 u8 mac[AES_BLOCK_SIZE]; 728 729 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 730 0) 731 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 732 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 733 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 734 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 735 memzero_explicit(&key, sizeof(key)); 736 } 737 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 738 static inline void aes_cmac_fips_test(void) 739 { 740 } 741 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 742 743 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 744 /* 745 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 746 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 747 * causes the fallback implementation to be used instead. 748 */ 749 #ifndef aes_ecb_encrypt_arch 750 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 751 const struct aes_enckey *key) 752 { 753 return false; 754 } 755 #endif 756 #ifndef aes_ecb_decrypt_arch 757 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 758 const struct aes_key *key) 759 { 760 return false; 761 } 762 #endif 763 764 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 765 { 766 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 767 len = round_down(len, AES_BLOCK_SIZE); 768 769 if (unlikely(len == 0)) 770 return; 771 772 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 773 return; 774 775 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 776 aes_encrypt(key, &dst[i], &src[i]); 777 } 778 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 779 780 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 781 const struct aes_key *key) 782 { 783 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 784 len = round_down(len, AES_BLOCK_SIZE); 785 786 if (unlikely(len == 0)) 787 return; 788 789 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 790 return; 791 792 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 793 aes_decrypt(key, &dst[i], &src[i]); 794 } 795 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 796 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 797 798 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 799 /* 800 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 801 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 802 * causes the fallback implementation to be used instead. 803 */ 804 #ifndef aes_cbc_encrypt_arch 805 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 806 u8 iv[AES_BLOCK_SIZE], 807 const struct aes_enckey *key) 808 { 809 return false; 810 } 811 #endif 812 #ifndef aes_cbc_decrypt_arch 813 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 814 u8 iv[AES_BLOCK_SIZE], 815 const struct aes_key *key) 816 { 817 return false; 818 } 819 #endif 820 821 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 822 aes_encrypt_arg key) 823 { 824 const u8 *prev = iv; 825 826 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 827 len = round_down(len, AES_BLOCK_SIZE); 828 829 if (unlikely(len == 0)) 830 return; 831 832 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 833 return; 834 835 do { 836 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 837 aes_encrypt(key, dst, dst); 838 prev = dst; 839 dst += AES_BLOCK_SIZE; 840 src += AES_BLOCK_SIZE; 841 len -= AES_BLOCK_SIZE; 842 } while (len); 843 memcpy(iv, prev, AES_BLOCK_SIZE); 844 } 845 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 846 847 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 848 const struct aes_key *key) 849 { 850 u8 next_iv[AES_BLOCK_SIZE]; 851 852 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 853 len = round_down(len, AES_BLOCK_SIZE); 854 855 if (unlikely(len == 0)) 856 return; 857 858 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 859 return; 860 861 len -= AES_BLOCK_SIZE; 862 dst += len; 863 src += len; 864 memcpy(next_iv, src, AES_BLOCK_SIZE); 865 for (;;) { 866 aes_decrypt(key, dst, src); 867 if (len == 0) 868 break; 869 src -= AES_BLOCK_SIZE; 870 crypto_xor(dst, src, AES_BLOCK_SIZE); 871 dst -= AES_BLOCK_SIZE; 872 len -= AES_BLOCK_SIZE; 873 } 874 crypto_xor(dst, iv, AES_BLOCK_SIZE); 875 memcpy(iv, next_iv, AES_BLOCK_SIZE); 876 } 877 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 878 879 /* 880 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 881 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 882 * causes the fallback implementation to be used instead. The fallback 883 * implementation still uses the arch-optimized AES-CBC code if available, but 884 * direct implementation of AES-CBC-CTS is helpful on short messages. 885 */ 886 #ifndef aes_cbc_cts_encrypt_arch 887 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 888 u8 iv[AES_BLOCK_SIZE], 889 const struct aes_enckey *key) 890 { 891 return false; 892 } 893 #endif 894 #ifndef aes_cbc_cts_decrypt_arch 895 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 896 u8 iv[AES_BLOCK_SIZE], 897 const struct aes_key *key) 898 { 899 return false; 900 } 901 #endif 902 903 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 904 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 905 { 906 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 907 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 908 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 909 size_t pn_len = len - pn_offset; 910 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 911 u8 *pad; 912 913 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 914 return; 915 916 if (len == AES_BLOCK_SIZE) { 917 aes_cbc_encrypt(dst, src, len, iv, key); 918 return; 919 } 920 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 921 return; 922 923 /* CBC-encrypt all blocks except the last. */ 924 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 925 926 /* 927 * Compute C[n] and C[n - 1]. 928 * 929 * Careful: src may equal dst (i.e., the encryption can be in-place), so 930 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 931 */ 932 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 933 memcpy(tmp, pad, AES_BLOCK_SIZE); 934 crypto_xor(tmp, &src[pn_offset], pn_len); 935 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 936 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 937 938 memzero_explicit(tmp, sizeof(tmp)); 939 } 940 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 941 942 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 943 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 944 { 945 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 946 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 947 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 948 size_t pn_len = len - pn_offset; 949 u8 *pad; 950 951 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 952 return; 953 954 if (len == AES_BLOCK_SIZE) { 955 aes_cbc_decrypt(dst, src, len, iv, key); 956 return; 957 } 958 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 959 return; 960 961 /* Compute P[0]..P[n - 2]. */ 962 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 963 964 /* 965 * Compute P[n] and P[n - 1]. 966 * 967 * Careful: src may equal dst (i.e., the decryption can be in-place), so 968 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 969 * 970 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 971 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 972 */ 973 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 974 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 975 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 976 pn_len); /* P[n] */ 977 crypto_xor(pad, &dst[pn_offset], pn_len); 978 aes_decrypt(key, pad, pad); 979 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 980 } 981 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 982 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 983 984 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 985 /* 986 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 987 * architecture. They are called with any len >= 0. Returning false causes the 988 * fallback implementation to be used instead. 989 */ 990 #ifndef aes_ctr_arch 991 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 992 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 993 { 994 return false; 995 } 996 #endif 997 #ifndef aes_xctr_arch 998 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, 999 const u8 iv[AES_BLOCK_SIZE], 1000 const struct aes_enckey *key) 1001 { 1002 return false; 1003 } 1004 #endif 1005 1006 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1007 { 1008 /* 1009 * 255 times out of 256 the first iteration is enough, so unroll the 1010 * first iteration as a micro-optimization. 1011 */ 1012 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1013 return; 1014 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1015 if (++ctr[i] != 0) 1016 break; 1017 } 1018 } 1019 1020 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1021 aes_encrypt_arg key) 1022 { 1023 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1024 1025 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1026 return; 1027 1028 /* Handle the full blocks. */ 1029 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1030 aes_encrypt(key, keystream, ctr); 1031 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1032 inc_be128_ctr(ctr); 1033 dst += AES_BLOCK_SIZE; 1034 src += AES_BLOCK_SIZE; 1035 } 1036 /* Handle any partial block at the end. */ 1037 if (len) { 1038 aes_encrypt(key, keystream, ctr); 1039 crypto_xor_cpy(dst, src, keystream, len); 1040 /* Counter is incremented even with just a partial block. */ 1041 inc_be128_ctr(ctr); 1042 } 1043 memzero_explicit(keystream, sizeof(keystream)); 1044 } 1045 EXPORT_SYMBOL_GPL(aes_ctr); 1046 1047 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1048 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1049 { 1050 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1051 __le64 aes_input[2]; 1052 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1053 1054 if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) 1055 return; 1056 1057 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1058 /* Handle the full blocks. */ 1059 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1060 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1061 aes_encrypt(key, keystream, (const u8 *)aes_input); 1062 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1063 dst += AES_BLOCK_SIZE; 1064 src += AES_BLOCK_SIZE; 1065 } 1066 /* Handle any partial block at the end. */ 1067 if (len) { 1068 /* Counter is incremented even with just a partial block. */ 1069 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1070 aes_encrypt(key, keystream, (const u8 *)aes_input); 1071 crypto_xor_cpy(dst, src, keystream, len); 1072 } 1073 memzero_explicit(keystream, sizeof(keystream)); 1074 memzero_explicit(aes_input, sizeof(aes_input)); 1075 } 1076 EXPORT_SYMBOL_GPL(aes_xctr); 1077 #endif /* CONFIG_CRYPTO_LIB_AES_CTR */ 1078 1079 static int __init aes_mod_init(void) 1080 { 1081 #ifdef aes_mod_init_arch 1082 aes_mod_init_arch(); 1083 #endif 1084 if (fips_enabled) 1085 aes_cmac_fips_test(); 1086 return 0; 1087 } 1088 subsys_initcall(aes_mod_init); 1089 1090 static void __exit aes_mod_exit(void) 1091 { 1092 } 1093 module_exit(aes_mod_exit); 1094 1095 MODULE_DESCRIPTION("AES block cipher"); 1096 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 1097 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 1098 MODULE_LICENSE("GPL v2"); 1099