1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ecb.h> 10 #include <crypto/aes.h> 11 #include <crypto/utils.h> 12 #include <linux/cache.h> 13 #include <linux/crypto.h> 14 #include <linux/export.h> 15 #include <linux/module.h> 16 #include <linux/unaligned.h> 17 #include "fips.h" 18 19 static const u8 ____cacheline_aligned aes_sbox[] = { 20 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 21 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 22 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 23 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 24 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 25 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 26 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 27 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 28 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 29 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 30 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 31 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 32 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 33 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 34 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 35 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 36 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 37 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 38 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 39 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 40 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 41 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 42 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 43 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 44 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 45 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 46 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 47 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 48 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 49 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 50 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 51 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 52 }; 53 54 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 55 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 56 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 57 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 58 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 59 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 60 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 61 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 62 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 63 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 64 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 65 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 66 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 67 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 68 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 69 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 70 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 71 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 72 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 73 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 74 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 75 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 76 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 77 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 78 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 79 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 80 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 81 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 82 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 83 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 84 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 85 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 86 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 87 }; 88 89 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 90 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 91 92 EXPORT_SYMBOL(crypto_aes_sbox); 93 EXPORT_SYMBOL(crypto_aes_inv_sbox); 94 95 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 96 const u32 ____cacheline_aligned aes_enc_tab[256] = { 97 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 98 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 99 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 100 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 101 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 102 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 103 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 104 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 105 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 106 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 107 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 108 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 109 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 110 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 111 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 112 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 113 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 114 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 115 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 116 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 117 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 118 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 119 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 120 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 121 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 122 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 123 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 124 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 125 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 126 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 127 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 128 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 129 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 130 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 131 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 132 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 133 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 134 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 135 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 136 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 137 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 138 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 139 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 140 }; 141 EXPORT_SYMBOL(aes_enc_tab); 142 143 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 144 const u32 ____cacheline_aligned aes_dec_tab[256] = { 145 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 146 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 147 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 148 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 149 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 150 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 151 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 152 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 153 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 154 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 155 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 156 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 157 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 158 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 159 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 160 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 161 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 162 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 163 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 164 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 165 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 166 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 167 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 168 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 169 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 170 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 171 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 172 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 173 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 174 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 175 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 176 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 177 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 178 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 179 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 180 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 181 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 182 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 183 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 184 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 185 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 186 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 187 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 188 }; 189 EXPORT_SYMBOL(aes_dec_tab); 190 191 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 192 static __always_inline void aes_prefetch(const void *mem, size_t len) 193 { 194 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 195 *(volatile const u8 *)(mem + i); 196 barrier(); 197 } 198 199 static u32 mul_by_x(u32 w) 200 { 201 u32 x = w & 0x7f7f7f7f; 202 u32 y = w & 0x80808080; 203 204 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 205 return (x << 1) ^ (y >> 7) * 0x1b; 206 } 207 208 static u32 mul_by_x2(u32 w) 209 { 210 u32 x = w & 0x3f3f3f3f; 211 u32 y = w & 0x80808080; 212 u32 z = w & 0x40404040; 213 214 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 215 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 216 } 217 218 static u32 mix_columns(u32 x) 219 { 220 /* 221 * Perform the following matrix multiplication in GF(2^8) 222 * 223 * | 0x2 0x3 0x1 0x1 | | x[0] | 224 * | 0x1 0x2 0x3 0x1 | | x[1] | 225 * | 0x1 0x1 0x2 0x3 | x | x[2] | 226 * | 0x3 0x1 0x1 0x2 | | x[3] | 227 */ 228 u32 y = mul_by_x(x) ^ ror32(x, 16); 229 230 return y ^ ror32(x ^ y, 8); 231 } 232 233 static u32 inv_mix_columns(u32 x) 234 { 235 /* 236 * Perform the following matrix multiplication in GF(2^8) 237 * 238 * | 0xe 0xb 0xd 0x9 | | x[0] | 239 * | 0x9 0xe 0xb 0xd | | x[1] | 240 * | 0xd 0x9 0xe 0xb | x | x[2] | 241 * | 0xb 0xd 0x9 0xe | | x[3] | 242 * 243 * which can conveniently be reduced to 244 * 245 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 246 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 247 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 248 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 249 */ 250 u32 y = mul_by_x2(x); 251 252 return mix_columns(x ^ y ^ ror32(y, 16)); 253 } 254 255 static u32 subw(u32 in) 256 { 257 return (aes_sbox[in & 0xff]) ^ 258 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 259 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 260 (aes_sbox[(in >> 24) & 0xff] << 24); 261 } 262 263 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 264 const u8 *in_key, int key_len) 265 { 266 u32 kwords = key_len / sizeof(u32); 267 u32 rc, i, j; 268 269 for (i = 0; i < kwords; i++) 270 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 271 272 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 273 u32 *rki = &rndkeys[i * kwords]; 274 u32 *rko = rki + kwords; 275 276 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 277 rko[1] = rko[0] ^ rki[1]; 278 rko[2] = rko[1] ^ rki[2]; 279 rko[3] = rko[2] ^ rki[3]; 280 281 if (key_len == AES_KEYSIZE_192) { 282 if (i >= 7) 283 break; 284 rko[4] = rko[3] ^ rki[4]; 285 rko[5] = rko[4] ^ rki[5]; 286 } else if (key_len == AES_KEYSIZE_256) { 287 if (i >= 6) 288 break; 289 rko[4] = subw(rko[3]) ^ rki[4]; 290 rko[5] = rko[4] ^ rki[5]; 291 rko[6] = rko[5] ^ rki[6]; 292 rko[7] = rko[6] ^ rki[7]; 293 } 294 } 295 296 /* 297 * Generate the decryption keys for the Equivalent Inverse Cipher. 298 * This involves reversing the order of the round keys, and applying 299 * the Inverse Mix Columns transformation to all but the first and 300 * the last one. 301 */ 302 if (inv_rndkeys) { 303 inv_rndkeys[0] = rndkeys[key_len + 24]; 304 inv_rndkeys[1] = rndkeys[key_len + 25]; 305 inv_rndkeys[2] = rndkeys[key_len + 26]; 306 inv_rndkeys[3] = rndkeys[key_len + 27]; 307 308 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 309 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 310 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 311 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 312 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 313 } 314 315 inv_rndkeys[i] = rndkeys[0]; 316 inv_rndkeys[i + 1] = rndkeys[1]; 317 inv_rndkeys[i + 2] = rndkeys[2]; 318 inv_rndkeys[i + 3] = rndkeys[3]; 319 } 320 } 321 322 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 323 unsigned int key_len) 324 { 325 if (aes_check_keylen(key_len) != 0) 326 return -EINVAL; 327 ctx->key_length = key_len; 328 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 329 return 0; 330 } 331 EXPORT_SYMBOL(aes_expandkey); 332 333 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 334 { 335 return rk ^ aes_enc_tab[(u8)w[i]] ^ 336 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 337 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 338 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 339 } 340 341 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 342 { 343 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 344 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 345 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 346 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 347 } 348 349 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 350 u8 out[AES_BLOCK_SIZE], 351 const u8 in[AES_BLOCK_SIZE]) 352 { 353 const u32 *rkp = rndkeys; 354 int n = nrounds - 1; 355 u32 w[4]; 356 357 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 358 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 359 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 360 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 361 362 /* 363 * Prefetch the table before doing data and key-dependent loads from it. 364 * 365 * This is intended only as a basic constant-time hardening measure that 366 * avoids interfering with performance too much. Its effectiveness is 367 * not guaranteed. For proper constant-time AES, a CPU that supports 368 * AES instructions should be used instead. 369 */ 370 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 371 372 do { 373 u32 w0 = enc_quarterround(w, 0, *rkp++); 374 u32 w1 = enc_quarterround(w, 1, *rkp++); 375 u32 w2 = enc_quarterround(w, 2, *rkp++); 376 u32 w3 = enc_quarterround(w, 3, *rkp++); 377 378 w[0] = w0; 379 w[1] = w1; 380 w[2] = w2; 381 w[3] = w3; 382 } while (--n); 383 384 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 385 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 386 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 387 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 388 } 389 390 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 391 { 392 return rk ^ aes_dec_tab[(u8)w[i]] ^ 393 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 394 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 395 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 396 } 397 398 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 399 { 400 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 401 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 402 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 403 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 404 } 405 406 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 407 int nrounds, 408 u8 out[AES_BLOCK_SIZE], 409 const u8 in[AES_BLOCK_SIZE]) 410 { 411 const u32 *rkp = inv_rndkeys; 412 int n = nrounds - 1; 413 u32 w[4]; 414 415 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 416 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 417 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 418 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 419 420 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 421 422 do { 423 u32 w0 = dec_quarterround(w, 0, *rkp++); 424 u32 w1 = dec_quarterround(w, 1, *rkp++); 425 u32 w2 = dec_quarterround(w, 2, *rkp++); 426 u32 w3 = dec_quarterround(w, 3, *rkp++); 427 428 w[0] = w0; 429 w[1] = w1; 430 w[2] = w2; 431 w[3] = w3; 432 } while (--n); 433 434 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 435 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 436 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 437 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 438 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 439 } 440 441 /* 442 * Note: the aes_prepare*key_* names reflect the fact that the implementation 443 * might not actually expand the key. (The s390 code for example doesn't.) 444 * Where the key is expanded we use the more specific names aes_expandkey_*. 445 * 446 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 447 * the area to store the prepared decryption key. It will be NULL if the user 448 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 449 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 450 */ 451 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 452 /* An arch-specific implementation of AES is available. Include it. */ 453 #include "aes.h" /* $(SRCARCH)/aes.h */ 454 #else 455 /* No arch-specific implementation of AES is available. Use generic code. */ 456 457 static void aes_preparekey_arch(union aes_enckey_arch *k, 458 union aes_invkey_arch *inv_k, 459 const u8 *in_key, int key_len, int nrounds) 460 { 461 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 462 in_key, key_len); 463 } 464 465 static void aes_encrypt_arch(const struct aes_enckey *key, 466 u8 out[AES_BLOCK_SIZE], 467 const u8 in[AES_BLOCK_SIZE]) 468 { 469 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 470 } 471 472 static void aes_decrypt_arch(const struct aes_key *key, 473 u8 out[AES_BLOCK_SIZE], 474 const u8 in[AES_BLOCK_SIZE]) 475 { 476 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 477 } 478 #endif 479 480 static int __aes_preparekey(struct aes_enckey *enc_key, 481 union aes_invkey_arch *inv_k, 482 const u8 *in_key, size_t key_len) 483 { 484 if (aes_check_keylen(key_len) != 0) 485 return -EINVAL; 486 enc_key->len = key_len; 487 enc_key->nrounds = 6 + key_len / 4; 488 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 489 enc_key->nrounds); 490 return 0; 491 } 492 493 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 494 { 495 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 496 in_key, key_len); 497 } 498 EXPORT_SYMBOL(aes_preparekey); 499 500 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 501 { 502 return __aes_preparekey(key, NULL, in_key, key_len); 503 } 504 EXPORT_SYMBOL(aes_prepareenckey); 505 506 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 507 const u8 in[AES_BLOCK_SIZE]) 508 { 509 aes_encrypt_arch(key.enc_key, out, in); 510 } 511 EXPORT_SYMBOL(aes_encrypt); 512 513 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 514 const u8 in[AES_BLOCK_SIZE]) 515 { 516 aes_decrypt_arch(key, out, in); 517 } 518 EXPORT_SYMBOL(aes_decrypt); 519 520 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 521 522 #ifndef aes_cbcmac_blocks_arch 523 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 524 const struct aes_enckey *key, const u8 *data, 525 size_t nblocks, bool enc_before, 526 bool enc_after) 527 { 528 return false; 529 } 530 #endif 531 532 /* This assumes nblocks >= 1. */ 533 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 534 const struct aes_enckey *key, const u8 *data, 535 size_t nblocks, bool enc_before, bool enc_after) 536 { 537 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 538 enc_after)) 539 return; 540 541 if (enc_before) 542 aes_encrypt(key, h, h); 543 for (; nblocks > 1; nblocks--) { 544 crypto_xor(h, data, AES_BLOCK_SIZE); 545 data += AES_BLOCK_SIZE; 546 aes_encrypt(key, h, h); 547 } 548 crypto_xor(h, data, AES_BLOCK_SIZE); 549 if (enc_after) 550 aes_encrypt(key, h, h); 551 } 552 553 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 554 size_t key_len) 555 { 556 u64 hi, lo, mask; 557 int err; 558 559 /* Prepare the AES key. */ 560 err = aes_prepareenckey(&key->aes, in_key, key_len); 561 if (err) 562 return err; 563 564 /* 565 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 566 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 567 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 568 */ 569 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 570 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 571 hi = be64_to_cpu(key->k_final[0].w[0]); 572 lo = be64_to_cpu(key->k_final[0].w[1]); 573 for (int i = 0; i < 2; i++) { 574 mask = ((s64)hi >> 63) & 0x87; 575 hi = (hi << 1) ^ (lo >> 63); 576 lo = (lo << 1) ^ mask; 577 key->k_final[i].w[0] = cpu_to_be64(hi); 578 key->k_final[i].w[1] = cpu_to_be64(lo); 579 } 580 return 0; 581 } 582 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 583 584 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 585 const u8 in_key[AES_KEYSIZE_128]) 586 { 587 static const u8 constants[3][AES_BLOCK_SIZE] = { 588 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 589 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 590 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 591 }; 592 u8 new_aes_key[AES_BLOCK_SIZE]; 593 594 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 595 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 596 aes_encrypt(&key->aes, new_aes_key, constants[0]); 597 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 598 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 599 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 600 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 601 } 602 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 603 604 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 605 { 606 bool enc_before = false; 607 size_t nblocks; 608 609 if (ctx->partial_len) { 610 /* XOR data into a pending block. */ 611 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 612 613 crypto_xor(&ctx->h[ctx->partial_len], data, l); 614 data += l; 615 data_len -= l; 616 ctx->partial_len += l; 617 if (data_len == 0) { 618 /* 619 * Either the pending block hasn't been filled yet, or 620 * no more data was given so it's not yet known whether 621 * the block is the final block. 622 */ 623 return; 624 } 625 /* Pending block has been filled and isn't the final block. */ 626 enc_before = true; 627 } 628 629 nblocks = data_len / AES_BLOCK_SIZE; 630 data_len %= AES_BLOCK_SIZE; 631 if (nblocks == 0) { 632 /* 0 additional full blocks, then optionally a partial block */ 633 if (enc_before) 634 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 635 crypto_xor(ctx->h, data, data_len); 636 ctx->partial_len = data_len; 637 } else if (data_len != 0) { 638 /* 1 or more additional full blocks, then a partial block */ 639 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 640 enc_before, /* enc_after= */ true); 641 data += nblocks * AES_BLOCK_SIZE; 642 crypto_xor(ctx->h, data, data_len); 643 ctx->partial_len = data_len; 644 } else { 645 /* 646 * 1 or more additional full blocks only. Encryption of the 647 * last block is delayed until it's known whether it's the final 648 * block in the message or not. 649 */ 650 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 651 enc_before, /* enc_after= */ false); 652 ctx->partial_len = AES_BLOCK_SIZE; 653 } 654 } 655 EXPORT_SYMBOL_GPL(aes_cmac_update); 656 657 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 658 { 659 if (ctx->partial_len == AES_BLOCK_SIZE) { 660 /* Final block is a full block. Use k_final[0]. */ 661 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 662 } else { 663 /* Final block is a partial block. Pad, and use k_final[1]. */ 664 ctx->h[ctx->partial_len] ^= 0x80; 665 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 666 } 667 aes_encrypt(&ctx->key->aes, out, ctx->h); 668 memzero_explicit(ctx, sizeof(*ctx)); 669 } 670 EXPORT_SYMBOL_GPL(aes_cmac_final); 671 672 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 673 size_t data_len) 674 { 675 bool enc_before = false; 676 size_t nblocks; 677 678 if (ctx->partial_len) { 679 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 680 681 crypto_xor(&ctx->h[ctx->partial_len], data, l); 682 data += l; 683 data_len -= l; 684 ctx->partial_len += l; 685 if (ctx->partial_len < AES_BLOCK_SIZE) 686 return; 687 enc_before = true; 688 } 689 690 nblocks = data_len / AES_BLOCK_SIZE; 691 data_len %= AES_BLOCK_SIZE; 692 if (nblocks == 0) { 693 if (enc_before) 694 aes_encrypt(ctx->key, ctx->h, ctx->h); 695 } else { 696 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 697 /* enc_after= */ true); 698 data += nblocks * AES_BLOCK_SIZE; 699 } 700 crypto_xor(ctx->h, data, data_len); 701 ctx->partial_len = data_len; 702 } 703 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 704 705 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 706 { 707 if (ctx->partial_len) 708 aes_encrypt(ctx->key, out, ctx->h); 709 else 710 memcpy(out, ctx->h, AES_BLOCK_SIZE); 711 memzero_explicit(ctx, sizeof(*ctx)); 712 } 713 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 714 715 /* 716 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 717 * Implementation Guidance, a cryptographic algorithm self-test for at least one 718 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 719 * is implemented. This fulfills that requirement via AES-CMAC. 720 * 721 * This is just for FIPS. The full tests are in the KUnit test suite. 722 */ 723 static void __init aes_cmac_fips_test(void) 724 { 725 struct aes_cmac_key key; 726 u8 mac[AES_BLOCK_SIZE]; 727 728 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 729 0) 730 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 731 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 732 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 733 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 734 memzero_explicit(&key, sizeof(key)); 735 } 736 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 737 static inline void aes_cmac_fips_test(void) 738 { 739 } 740 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 741 742 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 743 /* 744 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 745 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 746 * causes the fallback implementation to be used instead. 747 */ 748 #ifndef aes_ecb_encrypt_arch 749 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 750 const struct aes_enckey *key) 751 { 752 return false; 753 } 754 #endif 755 #ifndef aes_ecb_decrypt_arch 756 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 757 const struct aes_key *key) 758 { 759 return false; 760 } 761 #endif 762 763 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 764 { 765 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 766 len = round_down(len, AES_BLOCK_SIZE); 767 768 if (unlikely(len == 0)) 769 return; 770 771 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 772 return; 773 774 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 775 aes_encrypt(key, &dst[i], &src[i]); 776 } 777 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 778 779 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 780 const struct aes_key *key) 781 { 782 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 783 len = round_down(len, AES_BLOCK_SIZE); 784 785 if (unlikely(len == 0)) 786 return; 787 788 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 789 return; 790 791 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 792 aes_decrypt(key, &dst[i], &src[i]); 793 } 794 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 795 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 796 797 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 798 /* 799 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 800 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 801 * causes the fallback implementation to be used instead. 802 */ 803 #ifndef aes_cbc_encrypt_arch 804 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 805 u8 iv[AES_BLOCK_SIZE], 806 const struct aes_enckey *key) 807 { 808 return false; 809 } 810 #endif 811 #ifndef aes_cbc_decrypt_arch 812 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 813 u8 iv[AES_BLOCK_SIZE], 814 const struct aes_key *key) 815 { 816 return false; 817 } 818 #endif 819 820 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 821 aes_encrypt_arg key) 822 { 823 const u8 *prev = iv; 824 825 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 826 len = round_down(len, AES_BLOCK_SIZE); 827 828 if (unlikely(len == 0)) 829 return; 830 831 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 832 return; 833 834 do { 835 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 836 aes_encrypt(key, dst, dst); 837 prev = dst; 838 dst += AES_BLOCK_SIZE; 839 src += AES_BLOCK_SIZE; 840 len -= AES_BLOCK_SIZE; 841 } while (len); 842 memcpy(iv, prev, AES_BLOCK_SIZE); 843 } 844 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 845 846 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 847 const struct aes_key *key) 848 { 849 u8 next_iv[AES_BLOCK_SIZE]; 850 851 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 852 len = round_down(len, AES_BLOCK_SIZE); 853 854 if (unlikely(len == 0)) 855 return; 856 857 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 858 return; 859 860 len -= AES_BLOCK_SIZE; 861 dst += len; 862 src += len; 863 memcpy(next_iv, src, AES_BLOCK_SIZE); 864 for (;;) { 865 aes_decrypt(key, dst, src); 866 if (len == 0) 867 break; 868 src -= AES_BLOCK_SIZE; 869 crypto_xor(dst, src, AES_BLOCK_SIZE); 870 dst -= AES_BLOCK_SIZE; 871 len -= AES_BLOCK_SIZE; 872 } 873 crypto_xor(dst, iv, AES_BLOCK_SIZE); 874 memcpy(iv, next_iv, AES_BLOCK_SIZE); 875 } 876 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 877 878 /* 879 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 880 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 881 * causes the fallback implementation to be used instead. The fallback 882 * implementation still uses the arch-optimized AES-CBC code if available, but 883 * direct implementation of AES-CBC-CTS is helpful on short messages. 884 */ 885 #ifndef aes_cbc_cts_encrypt_arch 886 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 887 u8 iv[AES_BLOCK_SIZE], 888 const struct aes_enckey *key) 889 { 890 return false; 891 } 892 #endif 893 #ifndef aes_cbc_cts_decrypt_arch 894 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 895 u8 iv[AES_BLOCK_SIZE], 896 const struct aes_key *key) 897 { 898 return false; 899 } 900 #endif 901 902 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 903 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 904 { 905 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 906 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 907 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 908 size_t pn_len = len - pn_offset; 909 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 910 u8 *pad; 911 912 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 913 return; 914 915 if (len == AES_BLOCK_SIZE) { 916 aes_cbc_encrypt(dst, src, len, iv, key); 917 return; 918 } 919 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 920 return; 921 922 /* CBC-encrypt all blocks except the last. */ 923 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 924 925 /* 926 * Compute C[n] and C[n - 1]. 927 * 928 * Careful: src may equal dst (i.e., the encryption can be in-place), so 929 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 930 */ 931 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 932 memcpy(tmp, pad, AES_BLOCK_SIZE); 933 crypto_xor(tmp, &src[pn_offset], pn_len); 934 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 935 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 936 937 memzero_explicit(tmp, sizeof(tmp)); 938 } 939 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 940 941 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 942 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 943 { 944 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 945 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 946 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 947 size_t pn_len = len - pn_offset; 948 u8 *pad; 949 950 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 951 return; 952 953 if (len == AES_BLOCK_SIZE) { 954 aes_cbc_decrypt(dst, src, len, iv, key); 955 return; 956 } 957 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 958 return; 959 960 /* Compute P[0]..P[n - 2]. */ 961 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 962 963 /* 964 * Compute P[n] and P[n - 1]. 965 * 966 * Careful: src may equal dst (i.e., the decryption can be in-place), so 967 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 968 * 969 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 970 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 971 */ 972 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 973 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 974 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 975 pn_len); /* P[n] */ 976 crypto_xor(pad, &dst[pn_offset], pn_len); 977 aes_decrypt(key, pad, pad); 978 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 979 } 980 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 981 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 982 983 static int __init aes_mod_init(void) 984 { 985 #ifdef aes_mod_init_arch 986 aes_mod_init_arch(); 987 #endif 988 if (fips_enabled) 989 aes_cmac_fips_test(); 990 return 0; 991 } 992 subsys_initcall(aes_mod_init); 993 994 static void __exit aes_mod_exit(void) 995 { 996 } 997 module_exit(aes_mod_exit); 998 999 MODULE_DESCRIPTION("AES block cipher"); 1000 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 1001 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 1002 MODULE_LICENSE("GPL v2"); 1003