1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ctr.h> 10 #include <crypto/aes-ecb.h> 11 #include <crypto/aes-xts.h> 12 #include <crypto/aes.h> 13 #include <crypto/gf128mul.h> 14 #include <crypto/utils.h> 15 #include <linux/cache.h> 16 #include <linux/crypto.h> 17 #include <linux/export.h> 18 #include <linux/module.h> 19 #include <linux/unaligned.h> 20 #include "fips.h" 21 22 static const u8 ____cacheline_aligned aes_sbox[] = { 23 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 24 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 25 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 26 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 27 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 28 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 29 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 30 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 31 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 32 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 33 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 34 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 35 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 36 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 37 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 38 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 39 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 40 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 41 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 42 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 43 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 44 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 45 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 46 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 47 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 48 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 49 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 50 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 51 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 52 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 53 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 54 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 55 }; 56 57 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 58 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 59 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 60 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 61 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 62 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 63 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 64 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 65 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 66 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 67 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 68 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 69 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 70 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 71 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 72 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 73 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 74 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 75 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 76 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 77 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 78 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 79 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 80 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 81 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 82 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 83 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 84 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 85 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 86 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 87 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 88 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 89 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 90 }; 91 92 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 93 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 94 95 EXPORT_SYMBOL(crypto_aes_sbox); 96 EXPORT_SYMBOL(crypto_aes_inv_sbox); 97 98 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 99 const u32 ____cacheline_aligned aes_enc_tab[256] = { 100 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 101 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 102 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 103 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 104 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 105 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 106 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 107 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 108 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 109 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 110 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 111 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 112 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 113 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 114 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 115 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 116 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 117 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 118 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 119 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 120 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 121 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 122 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 123 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 124 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 125 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 126 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 127 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 128 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 129 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 130 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 131 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 132 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 133 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 134 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 135 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 136 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 137 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 138 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 139 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 140 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 141 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 142 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 143 }; 144 EXPORT_SYMBOL(aes_enc_tab); 145 146 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 147 const u32 ____cacheline_aligned aes_dec_tab[256] = { 148 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 149 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 150 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 151 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 152 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 153 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 154 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 155 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 156 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 157 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 158 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 159 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 160 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 161 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 162 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 163 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 164 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 165 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 166 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 167 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 168 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 169 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 170 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 171 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 172 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 173 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 174 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 175 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 176 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 177 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 178 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 179 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 180 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 181 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 182 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 183 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 184 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 185 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 186 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 187 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 188 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 189 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 190 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 191 }; 192 EXPORT_SYMBOL(aes_dec_tab); 193 194 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 195 static __always_inline void aes_prefetch(const void *mem, size_t len) 196 { 197 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 198 *(volatile const u8 *)(mem + i); 199 barrier(); 200 } 201 202 static u32 mul_by_x(u32 w) 203 { 204 u32 x = w & 0x7f7f7f7f; 205 u32 y = w & 0x80808080; 206 207 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 208 return (x << 1) ^ (y >> 7) * 0x1b; 209 } 210 211 static u32 mul_by_x2(u32 w) 212 { 213 u32 x = w & 0x3f3f3f3f; 214 u32 y = w & 0x80808080; 215 u32 z = w & 0x40404040; 216 217 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 218 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 219 } 220 221 static u32 mix_columns(u32 x) 222 { 223 /* 224 * Perform the following matrix multiplication in GF(2^8) 225 * 226 * | 0x2 0x3 0x1 0x1 | | x[0] | 227 * | 0x1 0x2 0x3 0x1 | | x[1] | 228 * | 0x1 0x1 0x2 0x3 | x | x[2] | 229 * | 0x3 0x1 0x1 0x2 | | x[3] | 230 */ 231 u32 y = mul_by_x(x) ^ ror32(x, 16); 232 233 return y ^ ror32(x ^ y, 8); 234 } 235 236 static u32 inv_mix_columns(u32 x) 237 { 238 /* 239 * Perform the following matrix multiplication in GF(2^8) 240 * 241 * | 0xe 0xb 0xd 0x9 | | x[0] | 242 * | 0x9 0xe 0xb 0xd | | x[1] | 243 * | 0xd 0x9 0xe 0xb | x | x[2] | 244 * | 0xb 0xd 0x9 0xe | | x[3] | 245 * 246 * which can conveniently be reduced to 247 * 248 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 249 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 250 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 251 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 252 */ 253 u32 y = mul_by_x2(x); 254 255 return mix_columns(x ^ y ^ ror32(y, 16)); 256 } 257 258 static u32 subw(u32 in) 259 { 260 return (aes_sbox[in & 0xff]) ^ 261 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 262 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 263 (aes_sbox[(in >> 24) & 0xff] << 24); 264 } 265 266 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 267 const u8 *in_key, int key_len) 268 { 269 u32 kwords = key_len / sizeof(u32); 270 u32 rc, i, j; 271 272 for (i = 0; i < kwords; i++) 273 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 274 275 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 276 u32 *rki = &rndkeys[i * kwords]; 277 u32 *rko = rki + kwords; 278 279 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 280 rko[1] = rko[0] ^ rki[1]; 281 rko[2] = rko[1] ^ rki[2]; 282 rko[3] = rko[2] ^ rki[3]; 283 284 if (key_len == AES_KEYSIZE_192) { 285 if (i >= 7) 286 break; 287 rko[4] = rko[3] ^ rki[4]; 288 rko[5] = rko[4] ^ rki[5]; 289 } else if (key_len == AES_KEYSIZE_256) { 290 if (i >= 6) 291 break; 292 rko[4] = subw(rko[3]) ^ rki[4]; 293 rko[5] = rko[4] ^ rki[5]; 294 rko[6] = rko[5] ^ rki[6]; 295 rko[7] = rko[6] ^ rki[7]; 296 } 297 } 298 299 /* 300 * Generate the decryption keys for the Equivalent Inverse Cipher. 301 * This involves reversing the order of the round keys, and applying 302 * the Inverse Mix Columns transformation to all but the first and 303 * the last one. 304 */ 305 if (inv_rndkeys) { 306 inv_rndkeys[0] = rndkeys[key_len + 24]; 307 inv_rndkeys[1] = rndkeys[key_len + 25]; 308 inv_rndkeys[2] = rndkeys[key_len + 26]; 309 inv_rndkeys[3] = rndkeys[key_len + 27]; 310 311 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 312 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 313 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 314 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 315 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 316 } 317 318 inv_rndkeys[i] = rndkeys[0]; 319 inv_rndkeys[i + 1] = rndkeys[1]; 320 inv_rndkeys[i + 2] = rndkeys[2]; 321 inv_rndkeys[i + 3] = rndkeys[3]; 322 } 323 } 324 325 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 326 unsigned int key_len) 327 { 328 if (aes_check_keylen(key_len) != 0) 329 return -EINVAL; 330 ctx->key_length = key_len; 331 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 332 return 0; 333 } 334 EXPORT_SYMBOL(aes_expandkey); 335 336 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 337 { 338 return rk ^ aes_enc_tab[(u8)w[i]] ^ 339 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 340 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 341 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 342 } 343 344 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 345 { 346 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 347 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 348 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 349 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 350 } 351 352 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 353 u8 out[AES_BLOCK_SIZE], 354 const u8 in[AES_BLOCK_SIZE]) 355 { 356 const u32 *rkp = rndkeys; 357 int n = nrounds - 1; 358 u32 w[4]; 359 360 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 361 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 362 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 363 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 364 365 /* 366 * Prefetch the table before doing data and key-dependent loads from it. 367 * 368 * This is intended only as a basic constant-time hardening measure that 369 * avoids interfering with performance too much. Its effectiveness is 370 * not guaranteed. For proper constant-time AES, a CPU that supports 371 * AES instructions should be used instead. 372 */ 373 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 374 375 do { 376 u32 w0 = enc_quarterround(w, 0, *rkp++); 377 u32 w1 = enc_quarterround(w, 1, *rkp++); 378 u32 w2 = enc_quarterround(w, 2, *rkp++); 379 u32 w3 = enc_quarterround(w, 3, *rkp++); 380 381 w[0] = w0; 382 w[1] = w1; 383 w[2] = w2; 384 w[3] = w3; 385 } while (--n); 386 387 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 388 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 389 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 390 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 391 } 392 393 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 394 { 395 return rk ^ aes_dec_tab[(u8)w[i]] ^ 396 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 397 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 398 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 399 } 400 401 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 402 { 403 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 404 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 405 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 406 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 407 } 408 409 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 410 int nrounds, 411 u8 out[AES_BLOCK_SIZE], 412 const u8 in[AES_BLOCK_SIZE]) 413 { 414 const u32 *rkp = inv_rndkeys; 415 int n = nrounds - 1; 416 u32 w[4]; 417 418 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 419 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 420 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 421 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 422 423 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 424 425 do { 426 u32 w0 = dec_quarterround(w, 0, *rkp++); 427 u32 w1 = dec_quarterround(w, 1, *rkp++); 428 u32 w2 = dec_quarterround(w, 2, *rkp++); 429 u32 w3 = dec_quarterround(w, 3, *rkp++); 430 431 w[0] = w0; 432 w[1] = w1; 433 w[2] = w2; 434 w[3] = w3; 435 } while (--n); 436 437 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 438 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 439 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 440 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 441 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 442 } 443 444 /* 445 * Note: the aes_prepare*key_* names reflect the fact that the implementation 446 * might not actually expand the key. (The s390 code for example doesn't.) 447 * Where the key is expanded we use the more specific names aes_expandkey_*. 448 * 449 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 450 * the area to store the prepared decryption key. It will be NULL if the user 451 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 452 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 453 */ 454 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 455 /* An arch-specific implementation of AES is available. Include it. */ 456 #include "aes.h" /* $(SRCARCH)/aes.h */ 457 #else 458 /* No arch-specific implementation of AES is available. Use generic code. */ 459 460 static void aes_preparekey_arch(union aes_enckey_arch *k, 461 union aes_invkey_arch *inv_k, 462 const u8 *in_key, int key_len, int nrounds) 463 { 464 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 465 in_key, key_len); 466 } 467 468 static void aes_encrypt_arch(const struct aes_enckey *key, 469 u8 out[AES_BLOCK_SIZE], 470 const u8 in[AES_BLOCK_SIZE]) 471 { 472 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 473 } 474 475 static void aes_decrypt_arch(const struct aes_key *key, 476 u8 out[AES_BLOCK_SIZE], 477 const u8 in[AES_BLOCK_SIZE]) 478 { 479 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 480 } 481 #endif 482 483 static int __aes_preparekey(struct aes_enckey *enc_key, 484 union aes_invkey_arch *inv_k, 485 const u8 *in_key, size_t key_len) 486 { 487 if (aes_check_keylen(key_len) != 0) 488 return -EINVAL; 489 enc_key->len = key_len; 490 enc_key->nrounds = 6 + key_len / 4; 491 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 492 enc_key->nrounds); 493 return 0; 494 } 495 496 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 497 { 498 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 499 in_key, key_len); 500 } 501 EXPORT_SYMBOL(aes_preparekey); 502 503 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 504 { 505 return __aes_preparekey(key, NULL, in_key, key_len); 506 } 507 EXPORT_SYMBOL(aes_prepareenckey); 508 509 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 510 const u8 in[AES_BLOCK_SIZE]) 511 { 512 aes_encrypt_arch(key.enc_key, out, in); 513 } 514 EXPORT_SYMBOL(aes_encrypt); 515 516 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 517 const u8 in[AES_BLOCK_SIZE]) 518 { 519 aes_decrypt_arch(key, out, in); 520 } 521 EXPORT_SYMBOL(aes_decrypt); 522 523 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 524 525 #ifndef aes_cbcmac_blocks_arch 526 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 527 const struct aes_enckey *key, const u8 *data, 528 size_t nblocks, bool enc_before, 529 bool enc_after) 530 { 531 return false; 532 } 533 #endif 534 535 /* This assumes nblocks >= 1. */ 536 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 537 const struct aes_enckey *key, const u8 *data, 538 size_t nblocks, bool enc_before, bool enc_after) 539 { 540 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 541 enc_after)) 542 return; 543 544 if (enc_before) 545 aes_encrypt(key, h, h); 546 for (; nblocks > 1; nblocks--) { 547 crypto_xor(h, data, AES_BLOCK_SIZE); 548 data += AES_BLOCK_SIZE; 549 aes_encrypt(key, h, h); 550 } 551 crypto_xor(h, data, AES_BLOCK_SIZE); 552 if (enc_after) 553 aes_encrypt(key, h, h); 554 } 555 556 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 557 size_t key_len) 558 { 559 u64 hi, lo, mask; 560 int err; 561 562 /* Prepare the AES key. */ 563 err = aes_prepareenckey(&key->aes, in_key, key_len); 564 if (err) 565 return err; 566 567 /* 568 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 569 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 570 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 571 */ 572 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 573 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 574 hi = be64_to_cpu(key->k_final[0].w[0]); 575 lo = be64_to_cpu(key->k_final[0].w[1]); 576 for (int i = 0; i < 2; i++) { 577 mask = ((s64)hi >> 63) & 0x87; 578 hi = (hi << 1) ^ (lo >> 63); 579 lo = (lo << 1) ^ mask; 580 key->k_final[i].w[0] = cpu_to_be64(hi); 581 key->k_final[i].w[1] = cpu_to_be64(lo); 582 } 583 return 0; 584 } 585 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 586 587 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 588 const u8 in_key[AES_KEYSIZE_128]) 589 { 590 static const u8 constants[3][AES_BLOCK_SIZE] = { 591 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 592 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 593 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 594 }; 595 u8 new_aes_key[AES_BLOCK_SIZE]; 596 597 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 598 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 599 aes_encrypt(&key->aes, new_aes_key, constants[0]); 600 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 601 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 602 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 603 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 604 } 605 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 606 607 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 608 { 609 bool enc_before = false; 610 size_t nblocks; 611 612 if (ctx->partial_len) { 613 /* XOR data into a pending block. */ 614 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 615 616 crypto_xor(&ctx->h[ctx->partial_len], data, l); 617 data += l; 618 data_len -= l; 619 ctx->partial_len += l; 620 if (data_len == 0) { 621 /* 622 * Either the pending block hasn't been filled yet, or 623 * no more data was given so it's not yet known whether 624 * the block is the final block. 625 */ 626 return; 627 } 628 /* Pending block has been filled and isn't the final block. */ 629 enc_before = true; 630 } 631 632 nblocks = data_len / AES_BLOCK_SIZE; 633 data_len %= AES_BLOCK_SIZE; 634 if (nblocks == 0) { 635 /* 0 additional full blocks, then optionally a partial block */ 636 if (enc_before) 637 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 638 crypto_xor(ctx->h, data, data_len); 639 ctx->partial_len = data_len; 640 } else if (data_len != 0) { 641 /* 1 or more additional full blocks, then a partial block */ 642 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 643 enc_before, /* enc_after= */ true); 644 data += nblocks * AES_BLOCK_SIZE; 645 crypto_xor(ctx->h, data, data_len); 646 ctx->partial_len = data_len; 647 } else { 648 /* 649 * 1 or more additional full blocks only. Encryption of the 650 * last block is delayed until it's known whether it's the final 651 * block in the message or not. 652 */ 653 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 654 enc_before, /* enc_after= */ false); 655 ctx->partial_len = AES_BLOCK_SIZE; 656 } 657 } 658 EXPORT_SYMBOL_GPL(aes_cmac_update); 659 660 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 661 { 662 if (ctx->partial_len == AES_BLOCK_SIZE) { 663 /* Final block is a full block. Use k_final[0]. */ 664 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 665 } else { 666 /* Final block is a partial block. Pad, and use k_final[1]. */ 667 ctx->h[ctx->partial_len] ^= 0x80; 668 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 669 } 670 aes_encrypt(&ctx->key->aes, out, ctx->h); 671 memzero_explicit(ctx, sizeof(*ctx)); 672 } 673 EXPORT_SYMBOL_GPL(aes_cmac_final); 674 675 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 676 size_t data_len) 677 { 678 bool enc_before = false; 679 size_t nblocks; 680 681 if (ctx->partial_len) { 682 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 683 684 crypto_xor(&ctx->h[ctx->partial_len], data, l); 685 data += l; 686 data_len -= l; 687 ctx->partial_len += l; 688 if (ctx->partial_len < AES_BLOCK_SIZE) 689 return; 690 enc_before = true; 691 } 692 693 nblocks = data_len / AES_BLOCK_SIZE; 694 data_len %= AES_BLOCK_SIZE; 695 if (nblocks == 0) { 696 if (enc_before) 697 aes_encrypt(ctx->key, ctx->h, ctx->h); 698 } else { 699 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 700 /* enc_after= */ true); 701 data += nblocks * AES_BLOCK_SIZE; 702 } 703 crypto_xor(ctx->h, data, data_len); 704 ctx->partial_len = data_len; 705 } 706 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 707 708 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 709 { 710 if (ctx->partial_len) 711 aes_encrypt(ctx->key, out, ctx->h); 712 else 713 memcpy(out, ctx->h, AES_BLOCK_SIZE); 714 memzero_explicit(ctx, sizeof(*ctx)); 715 } 716 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 717 718 /* 719 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 720 * Implementation Guidance, a cryptographic algorithm self-test for at least one 721 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 722 * is implemented. This fulfills that requirement via AES-CMAC. 723 * 724 * This is just for FIPS. The full tests are in the KUnit test suite. 725 */ 726 static void __init aes_cmac_fips_test(void) 727 { 728 struct aes_cmac_key key; 729 u8 mac[AES_BLOCK_SIZE]; 730 731 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 732 0) 733 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 734 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 735 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 736 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 737 memzero_explicit(&key, sizeof(key)); 738 } 739 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 740 static inline void aes_cmac_fips_test(void) 741 { 742 } 743 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 744 745 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 746 /* 747 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 748 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 749 * causes the fallback implementation to be used instead. 750 */ 751 #ifndef aes_ecb_encrypt_arch 752 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 753 const struct aes_enckey *key) 754 { 755 return false; 756 } 757 #endif 758 #ifndef aes_ecb_decrypt_arch 759 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 760 const struct aes_key *key) 761 { 762 return false; 763 } 764 #endif 765 766 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 767 { 768 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 769 len = round_down(len, AES_BLOCK_SIZE); 770 771 if (unlikely(len == 0)) 772 return; 773 774 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 775 return; 776 777 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 778 aes_encrypt(key, &dst[i], &src[i]); 779 } 780 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 781 782 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 783 const struct aes_key *key) 784 { 785 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 786 len = round_down(len, AES_BLOCK_SIZE); 787 788 if (unlikely(len == 0)) 789 return; 790 791 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 792 return; 793 794 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 795 aes_decrypt(key, &dst[i], &src[i]); 796 } 797 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 798 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 799 800 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 801 /* 802 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 803 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 804 * causes the fallback implementation to be used instead. 805 */ 806 #ifndef aes_cbc_encrypt_arch 807 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 808 u8 iv[AES_BLOCK_SIZE], 809 const struct aes_enckey *key) 810 { 811 return false; 812 } 813 #endif 814 #ifndef aes_cbc_decrypt_arch 815 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 816 u8 iv[AES_BLOCK_SIZE], 817 const struct aes_key *key) 818 { 819 return false; 820 } 821 #endif 822 823 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 824 aes_encrypt_arg key) 825 { 826 const u8 *prev = iv; 827 828 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 829 len = round_down(len, AES_BLOCK_SIZE); 830 831 if (unlikely(len == 0)) 832 return; 833 834 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 835 return; 836 837 do { 838 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 839 aes_encrypt(key, dst, dst); 840 prev = dst; 841 dst += AES_BLOCK_SIZE; 842 src += AES_BLOCK_SIZE; 843 len -= AES_BLOCK_SIZE; 844 } while (len); 845 memcpy(iv, prev, AES_BLOCK_SIZE); 846 } 847 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 848 849 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 850 const struct aes_key *key) 851 { 852 u8 next_iv[AES_BLOCK_SIZE]; 853 854 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 855 len = round_down(len, AES_BLOCK_SIZE); 856 857 if (unlikely(len == 0)) 858 return; 859 860 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 861 return; 862 863 len -= AES_BLOCK_SIZE; 864 dst += len; 865 src += len; 866 memcpy(next_iv, src, AES_BLOCK_SIZE); 867 for (;;) { 868 aes_decrypt(key, dst, src); 869 if (len == 0) 870 break; 871 src -= AES_BLOCK_SIZE; 872 crypto_xor(dst, src, AES_BLOCK_SIZE); 873 dst -= AES_BLOCK_SIZE; 874 len -= AES_BLOCK_SIZE; 875 } 876 crypto_xor(dst, iv, AES_BLOCK_SIZE); 877 memcpy(iv, next_iv, AES_BLOCK_SIZE); 878 } 879 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 880 881 /* 882 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 883 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 884 * causes the fallback implementation to be used instead. The fallback 885 * implementation still uses the arch-optimized AES-CBC code if available, but 886 * direct implementation of AES-CBC-CTS is helpful on short messages. 887 */ 888 #ifndef aes_cbc_cts_encrypt_arch 889 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 890 u8 iv[AES_BLOCK_SIZE], 891 const struct aes_enckey *key) 892 { 893 return false; 894 } 895 #endif 896 #ifndef aes_cbc_cts_decrypt_arch 897 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 898 u8 iv[AES_BLOCK_SIZE], 899 const struct aes_key *key) 900 { 901 return false; 902 } 903 #endif 904 905 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 906 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 907 { 908 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 909 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 910 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 911 size_t pn_len = len - pn_offset; 912 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 913 u8 *pad; 914 915 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 916 return; 917 918 if (len == AES_BLOCK_SIZE) { 919 aes_cbc_encrypt(dst, src, len, iv, key); 920 return; 921 } 922 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 923 return; 924 925 /* CBC-encrypt all blocks except the last. */ 926 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 927 928 /* 929 * Compute C[n] and C[n - 1]. 930 * 931 * Careful: src may equal dst (i.e., the encryption can be in-place), so 932 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 933 */ 934 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 935 memcpy(tmp, pad, AES_BLOCK_SIZE); 936 crypto_xor(tmp, &src[pn_offset], pn_len); 937 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 938 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 939 940 memzero_explicit(tmp, sizeof(tmp)); 941 } 942 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 943 944 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 945 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 946 { 947 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 948 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 949 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 950 size_t pn_len = len - pn_offset; 951 u8 *pad; 952 953 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 954 return; 955 956 if (len == AES_BLOCK_SIZE) { 957 aes_cbc_decrypt(dst, src, len, iv, key); 958 return; 959 } 960 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 961 return; 962 963 /* Compute P[0]..P[n - 2]. */ 964 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 965 966 /* 967 * Compute P[n] and P[n - 1]. 968 * 969 * Careful: src may equal dst (i.e., the decryption can be in-place), so 970 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 971 * 972 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 973 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 974 */ 975 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 976 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 977 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 978 pn_len); /* P[n] */ 979 crypto_xor(pad, &dst[pn_offset], pn_len); 980 aes_decrypt(key, pad, pad); 981 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 982 } 983 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 984 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 985 986 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 987 /* 988 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 989 * architecture. They are called with any len >= 0. Returning false causes the 990 * fallback implementation to be used instead. 991 */ 992 #ifndef aes_ctr_arch 993 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 994 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 995 { 996 return false; 997 } 998 #endif 999 #ifndef aes_xctr_arch 1000 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1001 const u8 iv[AES_BLOCK_SIZE], 1002 const struct aes_enckey *key) 1003 { 1004 return false; 1005 } 1006 #endif 1007 1008 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1009 { 1010 /* 1011 * 255 times out of 256 the first iteration is enough, so unroll the 1012 * first iteration as a micro-optimization. 1013 */ 1014 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1015 return; 1016 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1017 if (++ctr[i] != 0) 1018 break; 1019 } 1020 } 1021 1022 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1023 aes_encrypt_arg key) 1024 { 1025 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1026 1027 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1028 return; 1029 1030 /* Handle the full blocks. */ 1031 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1032 aes_encrypt(key, keystream, ctr); 1033 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1034 inc_be128_ctr(ctr); 1035 dst += AES_BLOCK_SIZE; 1036 src += AES_BLOCK_SIZE; 1037 } 1038 /* Handle any partial block at the end. */ 1039 if (len) { 1040 aes_encrypt(key, keystream, ctr); 1041 crypto_xor_cpy(dst, src, keystream, len); 1042 /* Counter is incremented even with just a partial block. */ 1043 inc_be128_ctr(ctr); 1044 } 1045 memzero_explicit(keystream, sizeof(keystream)); 1046 } 1047 EXPORT_SYMBOL_GPL(aes_ctr); 1048 1049 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1050 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1051 { 1052 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1053 __le64 aes_input[2]; 1054 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1055 1056 if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) 1057 return; 1058 1059 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1060 /* Handle the full blocks. */ 1061 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1062 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1063 aes_encrypt(key, keystream, (const u8 *)aes_input); 1064 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1065 dst += AES_BLOCK_SIZE; 1066 src += AES_BLOCK_SIZE; 1067 } 1068 /* Handle any partial block at the end. */ 1069 if (len) { 1070 /* Counter is incremented even with just a partial block. */ 1071 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1072 aes_encrypt(key, keystream, (const u8 *)aes_input); 1073 crypto_xor_cpy(dst, src, keystream, len); 1074 } 1075 memzero_explicit(keystream, sizeof(keystream)); 1076 memzero_explicit(aes_input, sizeof(aes_input)); 1077 } 1078 EXPORT_SYMBOL_GPL(aes_xctr); 1079 #endif /* CONFIG_CRYPTO_LIB_AES_CTR */ 1080 1081 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 1082 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, 1083 size_t key_len, int flags) 1084 { 1085 int err; 1086 1087 err = __xts_verify_key(in_key, key_len, flags); 1088 if (unlikely(err)) 1089 goto out_zeroize; 1090 /* First half of XTS key is the main key */ 1091 err = aes_preparekey(&key->main_key, in_key, key_len / 2); 1092 if (unlikely(err)) 1093 goto out_zeroize; 1094 /* Second half of XTS key is the tweak key */ 1095 err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], 1096 key_len / 2); 1097 if (unlikely(err)) 1098 goto out_zeroize; 1099 return 0; 1100 1101 out_zeroize: 1102 memzero_explicit(key, sizeof(*key)); 1103 return err; 1104 } 1105 EXPORT_SYMBOL_GPL(aes_xts_preparekey); 1106 1107 /* 1108 * Hooks for optimized AES-XTS implementations, overridable by the architecture. 1109 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, 1110 * they aren't expected to handle ciphertext stealing or empty inputs. 1111 * Returning false causes the fallback implementation to be used instead. 1112 * 1113 * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole 1114 * numbers of blocks anyway, with len >= 512. So there's no need to heavily 1115 * optimize ciphertext stealing for short messages.) 1116 */ 1117 #ifndef aes_xts_encrypt_arch 1118 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 1119 u8 tweak[AES_BLOCK_SIZE], 1120 const struct aes_xts_key *key, bool cont) 1121 { 1122 return false; 1123 } 1124 #endif 1125 #ifndef aes_xts_decrypt_arch 1126 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 1127 u8 tweak[AES_BLOCK_SIZE], 1128 const struct aes_xts_key *key, bool cont) 1129 { 1130 return false; 1131 } 1132 #endif 1133 1134 static noinline void aes_xts_crypt_nocts_blockbyblock( 1135 u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 1136 const struct aes_xts_key *key, bool cont, bool enc) 1137 { 1138 le128 t; 1139 1140 if (cont) 1141 memcpy(&t, tweak, sizeof(t)); 1142 else 1143 aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); 1144 do { 1145 crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); 1146 if (enc) 1147 aes_encrypt(&key->main_key, dst, dst); 1148 else 1149 aes_decrypt(&key->main_key, dst, dst); 1150 crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); 1151 gf128mul_x_ble(&t, &t); 1152 dst += AES_BLOCK_SIZE; 1153 src += AES_BLOCK_SIZE; 1154 len -= AES_BLOCK_SIZE; 1155 } while (len); 1156 memcpy(tweak, &t, sizeof(t)); 1157 memzero_explicit(&t, sizeof(t)); 1158 } 1159 1160 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1161 static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, 1162 size_t len, 1163 u8 tweak[AES_BLOCK_SIZE], 1164 const struct aes_xts_key *key, 1165 bool cont) 1166 { 1167 if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) 1168 return; 1169 1170 /* 1171 * For the fallback, just go block-by-block. It could be implemented on 1172 * top of AES-ECB, which could be significantly faster than this if the 1173 * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS 1174 * performance is important enough that it needs to be (and has been) 1175 * implemented directly by every non-obsolete arch anyway. 1176 */ 1177 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1178 /* enc= */ true); 1179 } 1180 1181 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1182 static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, 1183 size_t len, 1184 u8 tweak[AES_BLOCK_SIZE], 1185 const struct aes_xts_key *key, 1186 bool cont) 1187 { 1188 if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) 1189 return; 1190 1191 /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ 1192 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1193 /* enc= */ false); 1194 } 1195 1196 static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, 1197 u8 tweak[AES_BLOCK_SIZE], 1198 const struct aes_xts_key *key, 1199 bool cont) 1200 { 1201 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1202 size_t nocts_len = round_down(len, AES_BLOCK_SIZE); 1203 u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1204 1205 /* Encrypt all full blocks. */ 1206 aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1207 dst += nocts_len - AES_BLOCK_SIZE; 1208 src += nocts_len - AES_BLOCK_SIZE; 1209 1210 /* 1211 * Swap the partial block with the first 'partial_len' bytes of the 1212 * encrypted last full block. Note that a temporary buffer is needed to 1213 * support in-place encryption. 1214 */ 1215 memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); 1216 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1217 memcpy(dst, tmp_block, partial_len); 1218 1219 /* Encrypt the last full block again. */ 1220 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1221 aes_encrypt(&key->main_key, dst, dst); 1222 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1223 memzero_explicit(tmp_block, sizeof(tmp_block)); 1224 } 1225 1226 static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, 1227 u8 tweak[AES_BLOCK_SIZE], 1228 const struct aes_xts_key *key, 1229 bool cont) 1230 { 1231 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1232 size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; 1233 union { 1234 u8 block[AES_BLOCK_SIZE]; 1235 le128 tweak; 1236 } tmp __aligned(__alignof__(long)); 1237 1238 /* 1239 * Decrypt all blocks except the last full block and the partial block. 1240 * The last full block has to be handled specially because decryption 1241 * ciphertext stealing uses the last two tweaks in reverse order. 1242 * 1243 * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() 1244 * doesn't handle (so that the length doesn't get checked redundantly in 1245 * the fast path). So handle that case specially as well. 1246 */ 1247 if (nocts_len) 1248 aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1249 else if (!cont) 1250 aes_encrypt(&key->tweak_key, tweak, tweak); 1251 dst += nocts_len; 1252 src += nocts_len; 1253 1254 /* Copy the tweak, advance it again, then decrypt last full block. */ 1255 memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); 1256 gf128mul_x_ble(&tmp.tweak, &tmp.tweak); 1257 crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); 1258 aes_decrypt(&key->main_key, dst, dst); 1259 crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); 1260 1261 /* 1262 * Swap the partial block with the first 'partial_len' bytes of the 1263 * decrypted last full block. Note that a temporary buffer is needed to 1264 * support in-place decryption. 1265 */ 1266 memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); 1267 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1268 memcpy(dst, tmp.block, partial_len); 1269 1270 /* Decrypt the last full block again. */ 1271 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1272 aes_decrypt(&key->main_key, dst, dst); 1273 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1274 memzero_explicit(&tmp, sizeof(tmp)); 1275 } 1276 1277 void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, 1278 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1279 bool cont) 1280 { 1281 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1282 return; 1283 1284 if (unlikely(len % AES_BLOCK_SIZE)) { 1285 aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); 1286 return; 1287 } 1288 1289 aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); 1290 } 1291 EXPORT_SYMBOL_GPL(aes_xts_encrypt); 1292 1293 void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, 1294 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1295 bool cont) 1296 { 1297 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1298 return; 1299 1300 if (unlikely(len % AES_BLOCK_SIZE)) { 1301 aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); 1302 return; 1303 } 1304 1305 aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); 1306 } 1307 EXPORT_SYMBOL_GPL(aes_xts_decrypt); 1308 #endif /* CONFIG_CRYPTO_LIB_AES_XTS */ 1309 1310 static int __init aes_mod_init(void) 1311 { 1312 #ifdef aes_mod_init_arch 1313 aes_mod_init_arch(); 1314 #endif 1315 if (fips_enabled) 1316 aes_cmac_fips_test(); 1317 return 0; 1318 } 1319 subsys_initcall(aes_mod_init); 1320 1321 static void __exit aes_mod_exit(void) 1322 { 1323 } 1324 module_exit(aes_mod_exit); 1325 1326 MODULE_DESCRIPTION("AES block cipher"); 1327 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 1328 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 1329 MODULE_LICENSE("GPL v2"); 1330