1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ccm.h> 10 #include <crypto/aes-ctr.h> 11 #include <crypto/aes-ecb.h> 12 #include <crypto/aes-gcm.h> 13 #include <crypto/aes-xts.h> 14 #include <crypto/aes.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/utils.h> 17 #include <linux/cache.h> 18 #include <linux/crypto.h> 19 #include <linux/export.h> 20 #include <linux/module.h> 21 #include <linux/unaligned.h> 22 #include "fips-aes.h" 23 24 static const u8 ____cacheline_aligned aes_sbox[] = { 25 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 26 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 27 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 28 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 29 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 30 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 31 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 32 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 33 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 34 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 35 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 36 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 37 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 38 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 39 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 40 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 41 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 42 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 43 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 44 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 45 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 46 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 47 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 48 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 49 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 50 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 51 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 52 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 53 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 54 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 55 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 56 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 57 }; 58 59 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 60 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 61 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 62 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 63 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 64 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 65 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 66 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 67 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 68 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 69 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 70 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 71 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 72 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 73 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 74 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 75 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 76 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 77 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 78 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 79 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 80 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 81 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 82 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 83 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 84 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 85 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 86 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 87 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 88 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 89 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 90 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 91 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 92 }; 93 94 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 95 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 96 97 EXPORT_SYMBOL(crypto_aes_sbox); 98 EXPORT_SYMBOL(crypto_aes_inv_sbox); 99 100 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 101 const u32 ____cacheline_aligned aes_enc_tab[256] = { 102 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 103 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 104 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 105 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 106 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 107 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 108 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 109 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 110 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 111 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 112 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 113 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 114 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 115 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 116 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 117 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 118 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 119 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 120 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 121 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 122 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 123 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 124 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 125 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 126 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 127 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 128 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 129 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 130 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 131 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 132 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 133 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 134 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 135 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 136 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 137 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 138 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 139 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 140 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 141 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 142 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 143 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 144 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 145 }; 146 EXPORT_SYMBOL(aes_enc_tab); 147 148 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 149 const u32 ____cacheline_aligned aes_dec_tab[256] = { 150 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 151 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 152 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 153 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 154 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 155 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 156 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 157 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 158 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 159 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 160 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 161 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 162 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 163 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 164 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 165 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 166 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 167 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 168 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 169 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 170 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 171 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 172 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 173 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 174 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 175 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 176 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 177 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 178 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 179 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 180 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 181 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 182 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 183 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 184 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 185 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 186 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 187 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 188 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 189 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 190 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 191 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 192 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 193 }; 194 EXPORT_SYMBOL(aes_dec_tab); 195 196 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 197 static __always_inline void aes_prefetch(const void *mem, size_t len) 198 { 199 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 200 *(volatile const u8 *)(mem + i); 201 barrier(); 202 } 203 204 static u32 mul_by_x(u32 w) 205 { 206 u32 x = w & 0x7f7f7f7f; 207 u32 y = w & 0x80808080; 208 209 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 210 return (x << 1) ^ (y >> 7) * 0x1b; 211 } 212 213 static u32 mul_by_x2(u32 w) 214 { 215 u32 x = w & 0x3f3f3f3f; 216 u32 y = w & 0x80808080; 217 u32 z = w & 0x40404040; 218 219 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 220 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 221 } 222 223 static u32 mix_columns(u32 x) 224 { 225 /* 226 * Perform the following matrix multiplication in GF(2^8) 227 * 228 * | 0x2 0x3 0x1 0x1 | | x[0] | 229 * | 0x1 0x2 0x3 0x1 | | x[1] | 230 * | 0x1 0x1 0x2 0x3 | x | x[2] | 231 * | 0x3 0x1 0x1 0x2 | | x[3] | 232 */ 233 u32 y = mul_by_x(x) ^ ror32(x, 16); 234 235 return y ^ ror32(x ^ y, 8); 236 } 237 238 static u32 inv_mix_columns(u32 x) 239 { 240 /* 241 * Perform the following matrix multiplication in GF(2^8) 242 * 243 * | 0xe 0xb 0xd 0x9 | | x[0] | 244 * | 0x9 0xe 0xb 0xd | | x[1] | 245 * | 0xd 0x9 0xe 0xb | x | x[2] | 246 * | 0xb 0xd 0x9 0xe | | x[3] | 247 * 248 * which can conveniently be reduced to 249 * 250 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 251 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 252 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 253 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 254 */ 255 u32 y = mul_by_x2(x); 256 257 return mix_columns(x ^ y ^ ror32(y, 16)); 258 } 259 260 static u32 subw(u32 in) 261 { 262 return (aes_sbox[in & 0xff]) ^ 263 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 264 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 265 (aes_sbox[(in >> 24) & 0xff] << 24); 266 } 267 268 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 269 const u8 *in_key, int key_len) 270 { 271 u32 kwords = key_len / sizeof(u32); 272 u32 rc, i, j; 273 274 for (i = 0; i < kwords; i++) 275 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 276 277 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 278 u32 *rki = &rndkeys[i * kwords]; 279 u32 *rko = rki + kwords; 280 281 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 282 rko[1] = rko[0] ^ rki[1]; 283 rko[2] = rko[1] ^ rki[2]; 284 rko[3] = rko[2] ^ rki[3]; 285 286 if (key_len == AES_KEYSIZE_192) { 287 if (i >= 7) 288 break; 289 rko[4] = rko[3] ^ rki[4]; 290 rko[5] = rko[4] ^ rki[5]; 291 } else if (key_len == AES_KEYSIZE_256) { 292 if (i >= 6) 293 break; 294 rko[4] = subw(rko[3]) ^ rki[4]; 295 rko[5] = rko[4] ^ rki[5]; 296 rko[6] = rko[5] ^ rki[6]; 297 rko[7] = rko[6] ^ rki[7]; 298 } 299 } 300 301 /* 302 * Generate the decryption keys for the Equivalent Inverse Cipher. 303 * This involves reversing the order of the round keys, and applying 304 * the Inverse Mix Columns transformation to all but the first and 305 * the last one. 306 */ 307 if (inv_rndkeys) { 308 inv_rndkeys[0] = rndkeys[key_len + 24]; 309 inv_rndkeys[1] = rndkeys[key_len + 25]; 310 inv_rndkeys[2] = rndkeys[key_len + 26]; 311 inv_rndkeys[3] = rndkeys[key_len + 27]; 312 313 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 314 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 315 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 316 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 317 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 318 } 319 320 inv_rndkeys[i] = rndkeys[0]; 321 inv_rndkeys[i + 1] = rndkeys[1]; 322 inv_rndkeys[i + 2] = rndkeys[2]; 323 inv_rndkeys[i + 3] = rndkeys[3]; 324 } 325 } 326 327 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 328 unsigned int key_len) 329 { 330 if (aes_check_keylen(key_len) != 0) 331 return -EINVAL; 332 ctx->key_length = key_len; 333 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 334 return 0; 335 } 336 EXPORT_SYMBOL(aes_expandkey); 337 338 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 339 { 340 return rk ^ aes_enc_tab[(u8)w[i]] ^ 341 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 342 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 343 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 344 } 345 346 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 347 { 348 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 349 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 350 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 351 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 352 } 353 354 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 355 u8 out[AES_BLOCK_SIZE], 356 const u8 in[AES_BLOCK_SIZE]) 357 { 358 const u32 *rkp = rndkeys; 359 int n = nrounds - 1; 360 u32 w[4]; 361 362 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 363 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 364 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 365 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 366 367 /* 368 * Prefetch the table before doing data and key-dependent loads from it. 369 * 370 * This is intended only as a basic constant-time hardening measure that 371 * avoids interfering with performance too much. Its effectiveness is 372 * not guaranteed. For proper constant-time AES, a CPU that supports 373 * AES instructions should be used instead. 374 */ 375 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 376 377 do { 378 u32 w0 = enc_quarterround(w, 0, *rkp++); 379 u32 w1 = enc_quarterround(w, 1, *rkp++); 380 u32 w2 = enc_quarterround(w, 2, *rkp++); 381 u32 w3 = enc_quarterround(w, 3, *rkp++); 382 383 w[0] = w0; 384 w[1] = w1; 385 w[2] = w2; 386 w[3] = w3; 387 } while (--n); 388 389 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 390 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 391 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 392 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 393 } 394 395 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 396 { 397 return rk ^ aes_dec_tab[(u8)w[i]] ^ 398 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 399 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 400 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 401 } 402 403 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 404 { 405 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 406 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 407 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 408 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 409 } 410 411 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 412 int nrounds, 413 u8 out[AES_BLOCK_SIZE], 414 const u8 in[AES_BLOCK_SIZE]) 415 { 416 const u32 *rkp = inv_rndkeys; 417 int n = nrounds - 1; 418 u32 w[4]; 419 420 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 421 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 422 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 423 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 424 425 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 426 427 do { 428 u32 w0 = dec_quarterround(w, 0, *rkp++); 429 u32 w1 = dec_quarterround(w, 1, *rkp++); 430 u32 w2 = dec_quarterround(w, 2, *rkp++); 431 u32 w3 = dec_quarterround(w, 3, *rkp++); 432 433 w[0] = w0; 434 w[1] = w1; 435 w[2] = w2; 436 w[3] = w3; 437 } while (--n); 438 439 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 440 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 441 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 442 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 443 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 444 } 445 446 /* 447 * Note: the aes_prepare*key_* names reflect the fact that the implementation 448 * might not actually expand the key. (The s390 code for example doesn't.) 449 * Where the key is expanded we use the more specific names aes_expandkey_*. 450 * 451 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 452 * the area to store the prepared decryption key. It will be NULL if the user 453 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 454 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 455 */ 456 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 457 /* An arch-specific implementation of AES is available. Include it. */ 458 #include "aes.h" /* $(SRCARCH)/aes.h */ 459 #else 460 /* No arch-specific implementation of AES is available. Use generic code. */ 461 462 static void aes_preparekey_arch(union aes_enckey_arch *k, 463 union aes_invkey_arch *inv_k, 464 const u8 *in_key, int key_len, int nrounds) 465 { 466 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 467 in_key, key_len); 468 } 469 470 static void aes_encrypt_arch(const struct aes_enckey *key, 471 u8 out[AES_BLOCK_SIZE], 472 const u8 in[AES_BLOCK_SIZE]) 473 { 474 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 475 } 476 477 static void aes_decrypt_arch(const struct aes_key *key, 478 u8 out[AES_BLOCK_SIZE], 479 const u8 in[AES_BLOCK_SIZE]) 480 { 481 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 482 } 483 #endif 484 485 static int __aes_preparekey(struct aes_enckey *enc_key, 486 union aes_invkey_arch *inv_k, 487 const u8 *in_key, size_t key_len) 488 { 489 if (aes_check_keylen(key_len) != 0) 490 return -EINVAL; 491 enc_key->len = key_len; 492 enc_key->nrounds = 6 + key_len / 4; 493 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 494 enc_key->nrounds); 495 return 0; 496 } 497 498 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 499 { 500 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 501 in_key, key_len); 502 } 503 EXPORT_SYMBOL(aes_preparekey); 504 505 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 506 { 507 return __aes_preparekey(key, NULL, in_key, key_len); 508 } 509 EXPORT_SYMBOL(aes_prepareenckey); 510 511 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 512 const u8 in[AES_BLOCK_SIZE]) 513 { 514 aes_encrypt_arch(key.enc_key, out, in); 515 } 516 EXPORT_SYMBOL(aes_encrypt); 517 518 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 519 const u8 in[AES_BLOCK_SIZE]) 520 { 521 aes_decrypt_arch(key, out, in); 522 } 523 EXPORT_SYMBOL(aes_decrypt); 524 525 /* FIPS cryptographic algorithm self-test for "bare" AES */ 526 static void __init aes_fips_test(void) 527 { 528 struct aes_key key; 529 u8 data[AES_BLOCK_SIZE]; 530 531 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 532 panic("aes: FIPS self-test failed (preparekey)\n"); 533 534 aes_encrypt(&key, data, fips_test_data); 535 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 536 panic("aes: FIPS self-test failed (wrong ciphertext)\n"); 537 538 aes_decrypt(&key, data, data); 539 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 540 panic("aes: FIPS self-test failed (wrong plaintext)\n"); 541 542 memzero_explicit(&key, sizeof(key)); 543 } 544 545 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 546 547 #ifndef aes_cbcmac_blocks_arch 548 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 549 const struct aes_enckey *key, const u8 *data, 550 size_t nblocks, bool enc_before, 551 bool enc_after) 552 { 553 return false; 554 } 555 #endif 556 557 /* This assumes nblocks >= 1. */ 558 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 559 const struct aes_enckey *key, const u8 *data, 560 size_t nblocks, bool enc_before, bool enc_after) 561 { 562 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 563 enc_after)) 564 return; 565 566 if (enc_before) 567 aes_encrypt(key, h, h); 568 for (; nblocks > 1; nblocks--) { 569 crypto_xor(h, data, AES_BLOCK_SIZE); 570 data += AES_BLOCK_SIZE; 571 aes_encrypt(key, h, h); 572 } 573 crypto_xor(h, data, AES_BLOCK_SIZE); 574 if (enc_after) 575 aes_encrypt(key, h, h); 576 } 577 578 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 579 size_t key_len) 580 { 581 u64 hi, lo, mask; 582 int err; 583 584 /* Prepare the AES key. */ 585 err = aes_prepareenckey(&key->aes, in_key, key_len); 586 if (err) 587 return err; 588 589 /* 590 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 591 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 592 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 593 */ 594 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 595 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 596 hi = be64_to_cpu(key->k_final[0].w[0]); 597 lo = be64_to_cpu(key->k_final[0].w[1]); 598 for (int i = 0; i < 2; i++) { 599 mask = ((s64)hi >> 63) & 0x87; 600 hi = (hi << 1) ^ (lo >> 63); 601 lo = (lo << 1) ^ mask; 602 key->k_final[i].w[0] = cpu_to_be64(hi); 603 key->k_final[i].w[1] = cpu_to_be64(lo); 604 } 605 return 0; 606 } 607 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 608 609 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 610 const u8 in_key[AES_KEYSIZE_128]) 611 { 612 static const u8 constants[3][AES_BLOCK_SIZE] = { 613 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 614 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 615 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 616 }; 617 u8 new_aes_key[AES_BLOCK_SIZE]; 618 619 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 620 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 621 aes_encrypt(&key->aes, new_aes_key, constants[0]); 622 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 623 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 624 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 625 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 626 } 627 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 628 629 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 630 { 631 bool enc_before = false; 632 size_t nblocks; 633 634 if (ctx->partial_len) { 635 /* XOR data into a pending block. */ 636 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 637 638 crypto_xor(&ctx->h[ctx->partial_len], data, l); 639 data += l; 640 data_len -= l; 641 ctx->partial_len += l; 642 if (data_len == 0) { 643 /* 644 * Either the pending block hasn't been filled yet, or 645 * no more data was given so it's not yet known whether 646 * the block is the final block. 647 */ 648 return; 649 } 650 /* Pending block has been filled and isn't the final block. */ 651 enc_before = true; 652 } 653 654 nblocks = data_len / AES_BLOCK_SIZE; 655 data_len %= AES_BLOCK_SIZE; 656 if (nblocks == 0) { 657 /* 0 additional full blocks, then optionally a partial block */ 658 if (enc_before) 659 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 660 crypto_xor(ctx->h, data, data_len); 661 ctx->partial_len = data_len; 662 } else if (data_len != 0) { 663 /* 1 or more additional full blocks, then a partial block */ 664 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 665 enc_before, /* enc_after= */ true); 666 data += nblocks * AES_BLOCK_SIZE; 667 crypto_xor(ctx->h, data, data_len); 668 ctx->partial_len = data_len; 669 } else { 670 /* 671 * 1 or more additional full blocks only. Encryption of the 672 * last block is delayed until it's known whether it's the final 673 * block in the message or not. 674 */ 675 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 676 enc_before, /* enc_after= */ false); 677 ctx->partial_len = AES_BLOCK_SIZE; 678 } 679 } 680 EXPORT_SYMBOL_GPL(aes_cmac_update); 681 682 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 683 { 684 if (ctx->partial_len == AES_BLOCK_SIZE) { 685 /* Final block is a full block. Use k_final[0]. */ 686 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 687 } else { 688 /* Final block is a partial block. Pad, and use k_final[1]. */ 689 ctx->h[ctx->partial_len] ^= 0x80; 690 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 691 } 692 aes_encrypt(&ctx->key->aes, out, ctx->h); 693 memzero_explicit(ctx, sizeof(*ctx)); 694 } 695 EXPORT_SYMBOL_GPL(aes_cmac_final); 696 697 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 698 size_t data_len) 699 { 700 bool enc_before = false; 701 size_t nblocks; 702 703 if (ctx->partial_len) { 704 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 705 706 crypto_xor(&ctx->h[ctx->partial_len], data, l); 707 data += l; 708 data_len -= l; 709 ctx->partial_len += l; 710 if (ctx->partial_len < AES_BLOCK_SIZE) 711 return; 712 enc_before = true; 713 } 714 715 nblocks = data_len / AES_BLOCK_SIZE; 716 data_len %= AES_BLOCK_SIZE; 717 if (nblocks == 0) { 718 if (enc_before) 719 aes_encrypt(ctx->key, ctx->h, ctx->h); 720 } else { 721 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 722 /* enc_after= */ true); 723 data += nblocks * AES_BLOCK_SIZE; 724 } 725 crypto_xor(ctx->h, data, data_len); 726 ctx->partial_len = data_len; 727 } 728 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 729 730 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 731 { 732 if (ctx->partial_len) 733 aes_encrypt(ctx->key, out, ctx->h); 734 else 735 memcpy(out, ctx->h, AES_BLOCK_SIZE); 736 memzero_explicit(ctx, sizeof(*ctx)); 737 } 738 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 739 740 /* 741 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 742 * Implementation Guidance, a cryptographic algorithm self-test for at least one 743 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 744 * is implemented. This fulfills that requirement via AES-CMAC. 745 * 746 * This is just for FIPS. The full tests are in the KUnit test suite. 747 */ 748 static void __init aes_cmac_fips_test(void) 749 { 750 struct aes_cmac_key key; 751 u8 mac[AES_BLOCK_SIZE]; 752 753 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 754 0) 755 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 756 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 757 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 758 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 759 memzero_explicit(&key, sizeof(key)); 760 } 761 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 762 static inline void aes_cmac_fips_test(void) 763 { 764 } 765 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 766 767 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 768 /* 769 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 770 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 771 * causes the fallback implementation to be used instead. 772 */ 773 #ifndef aes_ecb_encrypt_arch 774 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 775 const struct aes_enckey *key) 776 { 777 return false; 778 } 779 #endif 780 #ifndef aes_ecb_decrypt_arch 781 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 782 const struct aes_key *key) 783 { 784 return false; 785 } 786 #endif 787 788 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 789 { 790 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 791 len = round_down(len, AES_BLOCK_SIZE); 792 793 if (unlikely(len == 0)) 794 return; 795 796 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 797 return; 798 799 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 800 aes_encrypt(key, &dst[i], &src[i]); 801 } 802 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 803 804 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 805 const struct aes_key *key) 806 { 807 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 808 len = round_down(len, AES_BLOCK_SIZE); 809 810 if (unlikely(len == 0)) 811 return; 812 813 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 814 return; 815 816 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 817 aes_decrypt(key, &dst[i], &src[i]); 818 } 819 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 820 821 /* FIPS cryptographic algorithm self-test for AES-ECB */ 822 static void __init aes_ecb_fips_test(void) 823 { 824 struct aes_key key; 825 u8 data[sizeof(fips_test_data)]; 826 827 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 828 panic("aes: ECB FIPS self-test failed (preparekey)\n"); 829 830 aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key); 831 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 832 panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n"); 833 834 aes_ecb_decrypt(data, data, sizeof(data), &key); 835 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 836 panic("aes: ECB FIPS self-test failed (wrong plaintext)\n"); 837 838 memzero_explicit(&key, sizeof(key)); 839 } 840 #else /* CONFIG_CRYPTO_LIB_AES_ECB */ 841 static inline void aes_ecb_fips_test(void) 842 { 843 } 844 #endif /* !CONFIG_CRYPTO_LIB_AES_ECB */ 845 846 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 847 /* 848 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 849 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 850 * causes the fallback implementation to be used instead. 851 */ 852 #ifndef aes_cbc_encrypt_arch 853 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 854 u8 iv[AES_BLOCK_SIZE], 855 const struct aes_enckey *key) 856 { 857 return false; 858 } 859 #endif 860 #ifndef aes_cbc_decrypt_arch 861 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 862 u8 iv[AES_BLOCK_SIZE], 863 const struct aes_key *key) 864 { 865 return false; 866 } 867 #endif 868 869 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 870 aes_encrypt_arg key) 871 { 872 const u8 *prev = iv; 873 874 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 875 len = round_down(len, AES_BLOCK_SIZE); 876 877 if (unlikely(len == 0)) 878 return; 879 880 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 881 return; 882 883 do { 884 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 885 aes_encrypt(key, dst, dst); 886 prev = dst; 887 dst += AES_BLOCK_SIZE; 888 src += AES_BLOCK_SIZE; 889 len -= AES_BLOCK_SIZE; 890 } while (len); 891 memcpy(iv, prev, AES_BLOCK_SIZE); 892 } 893 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 894 895 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 896 const struct aes_key *key) 897 { 898 u8 next_iv[AES_BLOCK_SIZE]; 899 900 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 901 len = round_down(len, AES_BLOCK_SIZE); 902 903 if (unlikely(len == 0)) 904 return; 905 906 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 907 return; 908 909 len -= AES_BLOCK_SIZE; 910 dst += len; 911 src += len; 912 memcpy(next_iv, src, AES_BLOCK_SIZE); 913 for (;;) { 914 aes_decrypt(key, dst, src); 915 if (len == 0) 916 break; 917 src -= AES_BLOCK_SIZE; 918 crypto_xor(dst, src, AES_BLOCK_SIZE); 919 dst -= AES_BLOCK_SIZE; 920 len -= AES_BLOCK_SIZE; 921 } 922 crypto_xor(dst, iv, AES_BLOCK_SIZE); 923 memcpy(iv, next_iv, AES_BLOCK_SIZE); 924 } 925 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 926 927 /* 928 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 929 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 930 * causes the fallback implementation to be used instead. The fallback 931 * implementation still uses the arch-optimized AES-CBC code if available, but 932 * direct implementation of AES-CBC-CTS is helpful on short messages. 933 */ 934 #ifndef aes_cbc_cts_encrypt_arch 935 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 936 u8 iv[AES_BLOCK_SIZE], 937 const struct aes_enckey *key) 938 { 939 return false; 940 } 941 #endif 942 #ifndef aes_cbc_cts_decrypt_arch 943 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 944 u8 iv[AES_BLOCK_SIZE], 945 const struct aes_key *key) 946 { 947 return false; 948 } 949 #endif 950 951 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 952 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 953 { 954 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 955 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 956 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 957 size_t pn_len = len - pn_offset; 958 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 959 u8 *pad; 960 961 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 962 return; 963 964 if (len == AES_BLOCK_SIZE) { 965 aes_cbc_encrypt(dst, src, len, iv, key); 966 return; 967 } 968 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 969 return; 970 971 /* CBC-encrypt all blocks except the last. */ 972 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 973 974 /* 975 * Compute C[n] and C[n - 1]. 976 * 977 * Careful: src may equal dst (i.e., the encryption can be in-place), so 978 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 979 */ 980 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 981 memcpy(tmp, pad, AES_BLOCK_SIZE); 982 crypto_xor(tmp, &src[pn_offset], pn_len); 983 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 984 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 985 986 memzero_explicit(tmp, sizeof(tmp)); 987 } 988 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 989 990 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 991 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 992 { 993 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 994 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 995 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 996 size_t pn_len = len - pn_offset; 997 u8 *pad; 998 999 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1000 return; 1001 1002 if (len == AES_BLOCK_SIZE) { 1003 aes_cbc_decrypt(dst, src, len, iv, key); 1004 return; 1005 } 1006 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 1007 return; 1008 1009 /* Compute P[0]..P[n - 2]. */ 1010 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 1011 1012 /* 1013 * Compute P[n] and P[n - 1]. 1014 * 1015 * Careful: src may equal dst (i.e., the decryption can be in-place), so 1016 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 1017 * 1018 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 1019 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 1020 */ 1021 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 1022 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 1023 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 1024 pn_len); /* P[n] */ 1025 crypto_xor(pad, &dst[pn_offset], pn_len); 1026 aes_decrypt(key, pad, pad); 1027 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 1028 } 1029 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 1030 1031 /* FIPS cryptographic algorithm self-test for AES-CBC */ 1032 static void __init aes_cbc_fips_test(void) 1033 { 1034 struct aes_key key; 1035 u8 iv[AES_BLOCK_SIZE]; 1036 u8 data[sizeof(fips_test_data)]; 1037 1038 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1039 panic("aes: CBC FIPS self-test failed (preparekey)\n"); 1040 1041 memcpy(iv, fips_test_iv, sizeof(iv)); 1042 aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key); 1043 if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0) 1044 panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n"); 1045 1046 memcpy(iv, fips_test_iv, sizeof(iv)); 1047 aes_cbc_decrypt(data, data, sizeof(data), iv, &key); 1048 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1049 panic("aes: CBC FIPS self-test failed (wrong plaintext)\n"); 1050 1051 memzero_explicit(&key, sizeof(key)); 1052 } 1053 1054 /* FIPS cryptographic algorithm self-test for AES-CBC-CTS */ 1055 static void __init aes_cbc_cts_fips_test(void) 1056 { 1057 struct aes_key key; 1058 u8 iv[AES_BLOCK_SIZE]; 1059 const size_t data_len = 2 * AES_BLOCK_SIZE; 1060 u8 ptext[2 * AES_BLOCK_SIZE]; 1061 u8 data[2 * AES_BLOCK_SIZE]; 1062 1063 /* ptext = fips_test_data || fips_test_data */ 1064 memcpy(ptext, fips_test_data, AES_BLOCK_SIZE); 1065 memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE); 1066 1067 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1068 panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n"); 1069 1070 memcpy(iv, fips_test_iv, sizeof(iv)); 1071 aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key); 1072 if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0) 1073 panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n"); 1074 1075 memcpy(iv, fips_test_iv, sizeof(iv)); 1076 aes_cbc_cts_decrypt(data, data, data_len, iv, &key); 1077 if (memcmp(ptext, data, data_len) != 0) 1078 panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n"); 1079 1080 memzero_explicit(&key, sizeof(key)); 1081 } 1082 #else /* CONFIG_CRYPTO_LIB_AES_CBC */ 1083 static inline void aes_cbc_fips_test(void) 1084 { 1085 } 1086 static inline void aes_cbc_cts_fips_test(void) 1087 { 1088 } 1089 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC */ 1090 1091 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 1092 /* 1093 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 1094 * architecture. They are called with any len >= 0. Returning false causes the 1095 * fallback implementation to be used instead. 1096 */ 1097 #ifndef aes_ctr_arch 1098 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 1099 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 1100 { 1101 return false; 1102 } 1103 #endif 1104 #ifndef aes_xctr_arch 1105 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1106 const u8 iv[AES_BLOCK_SIZE], 1107 const struct aes_enckey *key) 1108 { 1109 return false; 1110 } 1111 #endif 1112 1113 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1114 { 1115 /* 1116 * 255 times out of 256 the first iteration is enough, so unroll the 1117 * first iteration as a micro-optimization. 1118 */ 1119 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1120 return; 1121 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1122 if (++ctr[i] != 0) 1123 break; 1124 } 1125 } 1126 1127 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1128 aes_encrypt_arg key) 1129 { 1130 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1131 1132 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1133 return; 1134 1135 /* Handle the full blocks. */ 1136 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1137 aes_encrypt(key, keystream, ctr); 1138 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1139 inc_be128_ctr(ctr); 1140 dst += AES_BLOCK_SIZE; 1141 src += AES_BLOCK_SIZE; 1142 } 1143 /* Handle any partial block at the end. */ 1144 if (len) { 1145 aes_encrypt(key, keystream, ctr); 1146 crypto_xor_cpy(dst, src, keystream, len); 1147 /* Counter is incremented even with just a partial block. */ 1148 inc_be128_ctr(ctr); 1149 } 1150 memzero_explicit(keystream, sizeof(keystream)); 1151 } 1152 EXPORT_SYMBOL_GPL(aes_ctr); 1153 1154 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1155 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1156 { 1157 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1158 __le64 aes_input[2]; 1159 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1160 1161 if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) 1162 return; 1163 1164 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1165 /* Handle the full blocks. */ 1166 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1167 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1168 aes_encrypt(key, keystream, (const u8 *)aes_input); 1169 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1170 dst += AES_BLOCK_SIZE; 1171 src += AES_BLOCK_SIZE; 1172 } 1173 /* Handle any partial block at the end. */ 1174 if (len) { 1175 /* Counter is incremented even with just a partial block. */ 1176 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1177 aes_encrypt(key, keystream, (const u8 *)aes_input); 1178 crypto_xor_cpy(dst, src, keystream, len); 1179 } 1180 memzero_explicit(keystream, sizeof(keystream)); 1181 memzero_explicit(aes_input, sizeof(aes_input)); 1182 } 1183 EXPORT_SYMBOL_GPL(aes_xctr); 1184 1185 /* FIPS cryptographic algorithm self-test for AES-CTR */ 1186 static void __init aes_ctr_fips_test(void) 1187 { 1188 struct aes_enckey key; 1189 u8 ctr[AES_BLOCK_SIZE]; 1190 u8 data[sizeof(fips_test_data)]; 1191 1192 if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1193 panic("aes: CTR FIPS self-test failed (preparekey)\n"); 1194 1195 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1196 aes_ctr(data, fips_test_data, sizeof(data), ctr, &key); 1197 if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0) 1198 panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n"); 1199 1200 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1201 aes_ctr(data, data, sizeof(data), ctr, &key); 1202 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1203 panic("aes: CTR FIPS self-test failed (wrong plaintext)\n"); 1204 1205 memzero_explicit(&key, sizeof(key)); 1206 } 1207 #else /* CONFIG_CRYPTO_LIB_AES_CTR */ 1208 static inline void aes_ctr_fips_test(void) 1209 { 1210 } 1211 #endif /* !CONFIG_CRYPTO_LIB_AES_CTR */ 1212 1213 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 1214 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, 1215 size_t key_len, int flags) 1216 { 1217 int err; 1218 1219 err = __xts_verify_key(in_key, key_len, flags); 1220 if (unlikely(err)) 1221 goto out_zeroize; 1222 /* First half of XTS key is the main key */ 1223 err = aes_preparekey(&key->main_key, in_key, key_len / 2); 1224 if (unlikely(err)) 1225 goto out_zeroize; 1226 /* Second half of XTS key is the tweak key */ 1227 err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], 1228 key_len / 2); 1229 if (unlikely(err)) 1230 goto out_zeroize; 1231 return 0; 1232 1233 out_zeroize: 1234 memzero_explicit(key, sizeof(*key)); 1235 return err; 1236 } 1237 EXPORT_SYMBOL_GPL(aes_xts_preparekey); 1238 1239 /* 1240 * Hooks for optimized AES-XTS implementations, overridable by the architecture. 1241 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, 1242 * they aren't expected to handle ciphertext stealing or empty inputs. 1243 * Returning false causes the fallback implementation to be used instead. 1244 * 1245 * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole 1246 * numbers of blocks anyway, with len >= 512. So there's no need to heavily 1247 * optimize ciphertext stealing for short messages.) 1248 */ 1249 #ifndef aes_xts_encrypt_arch 1250 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 1251 u8 tweak[AES_BLOCK_SIZE], 1252 const struct aes_xts_key *key, bool cont) 1253 { 1254 return false; 1255 } 1256 #endif 1257 #ifndef aes_xts_decrypt_arch 1258 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 1259 u8 tweak[AES_BLOCK_SIZE], 1260 const struct aes_xts_key *key, bool cont) 1261 { 1262 return false; 1263 } 1264 #endif 1265 1266 static noinline void aes_xts_crypt_nocts_blockbyblock( 1267 u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 1268 const struct aes_xts_key *key, bool cont, bool enc) 1269 { 1270 le128 t; 1271 1272 if (cont) 1273 memcpy(&t, tweak, sizeof(t)); 1274 else 1275 aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); 1276 do { 1277 crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); 1278 if (enc) 1279 aes_encrypt(&key->main_key, dst, dst); 1280 else 1281 aes_decrypt(&key->main_key, dst, dst); 1282 crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); 1283 gf128mul_x_ble(&t, &t); 1284 dst += AES_BLOCK_SIZE; 1285 src += AES_BLOCK_SIZE; 1286 len -= AES_BLOCK_SIZE; 1287 } while (len); 1288 memcpy(tweak, &t, sizeof(t)); 1289 memzero_explicit(&t, sizeof(t)); 1290 } 1291 1292 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1293 static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, 1294 size_t len, 1295 u8 tweak[AES_BLOCK_SIZE], 1296 const struct aes_xts_key *key, 1297 bool cont) 1298 { 1299 if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) 1300 return; 1301 1302 /* 1303 * For the fallback, just go block-by-block. It could be implemented on 1304 * top of AES-ECB, which could be significantly faster than this if the 1305 * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS 1306 * performance is important enough that it needs to be (and has been) 1307 * implemented directly by every non-obsolete arch anyway. 1308 */ 1309 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1310 /* enc= */ true); 1311 } 1312 1313 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1314 static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, 1315 size_t len, 1316 u8 tweak[AES_BLOCK_SIZE], 1317 const struct aes_xts_key *key, 1318 bool cont) 1319 { 1320 if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) 1321 return; 1322 1323 /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ 1324 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1325 /* enc= */ false); 1326 } 1327 1328 static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, 1329 u8 tweak[AES_BLOCK_SIZE], 1330 const struct aes_xts_key *key, 1331 bool cont) 1332 { 1333 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1334 size_t nocts_len = round_down(len, AES_BLOCK_SIZE); 1335 u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1336 1337 /* Encrypt all full blocks. */ 1338 aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1339 dst += nocts_len - AES_BLOCK_SIZE; 1340 src += nocts_len - AES_BLOCK_SIZE; 1341 1342 /* 1343 * Swap the partial block with the first 'partial_len' bytes of the 1344 * encrypted last full block. Note that a temporary buffer is needed to 1345 * support in-place encryption. 1346 */ 1347 memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); 1348 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1349 memcpy(dst, tmp_block, partial_len); 1350 1351 /* Encrypt the last full block again. */ 1352 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1353 aes_encrypt(&key->main_key, dst, dst); 1354 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1355 memzero_explicit(tmp_block, sizeof(tmp_block)); 1356 } 1357 1358 static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, 1359 u8 tweak[AES_BLOCK_SIZE], 1360 const struct aes_xts_key *key, 1361 bool cont) 1362 { 1363 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1364 size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; 1365 union { 1366 u8 block[AES_BLOCK_SIZE]; 1367 le128 tweak; 1368 } tmp __aligned(__alignof__(long)); 1369 1370 /* 1371 * Decrypt all blocks except the last full block and the partial block. 1372 * The last full block has to be handled specially because decryption 1373 * ciphertext stealing uses the last two tweaks in reverse order. 1374 * 1375 * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() 1376 * doesn't handle (so that the length doesn't get checked redundantly in 1377 * the fast path). So handle that case specially as well. 1378 */ 1379 if (nocts_len) 1380 aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1381 else if (!cont) 1382 aes_encrypt(&key->tweak_key, tweak, tweak); 1383 dst += nocts_len; 1384 src += nocts_len; 1385 1386 /* Copy the tweak, advance it again, then decrypt last full block. */ 1387 memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); 1388 gf128mul_x_ble(&tmp.tweak, &tmp.tweak); 1389 crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); 1390 aes_decrypt(&key->main_key, dst, dst); 1391 crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); 1392 1393 /* 1394 * Swap the partial block with the first 'partial_len' bytes of the 1395 * decrypted last full block. Note that a temporary buffer is needed to 1396 * support in-place decryption. 1397 */ 1398 memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); 1399 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1400 memcpy(dst, tmp.block, partial_len); 1401 1402 /* Decrypt the last full block again. */ 1403 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1404 aes_decrypt(&key->main_key, dst, dst); 1405 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1406 memzero_explicit(&tmp, sizeof(tmp)); 1407 } 1408 1409 void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, 1410 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1411 bool cont) 1412 { 1413 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1414 return; 1415 1416 if (unlikely(len % AES_BLOCK_SIZE)) { 1417 aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); 1418 return; 1419 } 1420 1421 aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); 1422 } 1423 EXPORT_SYMBOL_GPL(aes_xts_encrypt); 1424 1425 void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, 1426 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1427 bool cont) 1428 { 1429 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1430 return; 1431 1432 if (unlikely(len % AES_BLOCK_SIZE)) { 1433 aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); 1434 return; 1435 } 1436 1437 aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); 1438 } 1439 EXPORT_SYMBOL_GPL(aes_xts_decrypt); 1440 1441 /* FIPS cryptographic algorithm self-test for AES-XTS */ 1442 static void __init aes_xts_fips_test(void) 1443 { 1444 struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key); 1445 u8 tweak[AES_BLOCK_SIZE]; 1446 u8 data[sizeof(fips_test_data)]; 1447 1448 if (key == NULL) 1449 panic("aes: XTS FIPS self-test failed (kmalloc)\n"); 1450 1451 if (aes_xts_preparekey(key, fips_test_xts_key, 1452 sizeof(fips_test_xts_key), 0) != 0) 1453 panic("aes: XTS FIPS self-test failed (preparekey)\n"); 1454 1455 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1456 aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false); 1457 if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0) 1458 panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n"); 1459 1460 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1461 aes_xts_decrypt(data, data, sizeof(data), tweak, key, false); 1462 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1463 panic("aes: XTS FIPS self-test failed (wrong plaintext)\n"); 1464 } 1465 #else /* CONFIG_CRYPTO_LIB_AES_XTS */ 1466 static inline void aes_xts_fips_test(void) 1467 { 1468 } 1469 #endif /* !CONFIG_CRYPTO_LIB_AES_XTS */ 1470 1471 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) 1472 /* 1473 * Hooks for optimized AES-GCM implementations, overridable by the architecture. 1474 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't 1475 * expected to handle empty inputs or partial blocks, as those cases are handled 1476 * by non-arch-specific code instead. 1477 * 1478 * The GHASH accumulator is provided in POLYVAL format. The counter is provided 1479 * in big endian format, and it's read-only, as the caller handles updating it. 1480 * 1481 * Returning false causes the fallback implementation to be used instead. 1482 * 1483 * These hooks are used only for en/decrypted data. For the associated data the 1484 * GHASH functions are called instead, so those should be implemented too. 1485 */ 1486 #ifndef aes_gcm_encrypt_update_arch 1487 static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1488 struct polyval_elem *ghash_acc, 1489 const __be32 ctr32[4], 1490 const struct aes_enckey *aes_key, 1491 const struct ghash_key *ghash_key) 1492 { 1493 return false; 1494 } 1495 #endif 1496 #ifndef aes_gcm_decrypt_update_arch 1497 static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1498 struct polyval_elem *ghash_acc, 1499 const __be32 ctr32[4], 1500 const struct aes_enckey *aes_key, 1501 const struct ghash_key *ghash_key) 1502 { 1503 return false; 1504 } 1505 #endif 1506 1507 int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, 1508 size_t key_len, size_t authtag_len) 1509 { 1510 u8 h[AES_BLOCK_SIZE] = { 0 }; 1511 int err; 1512 1513 err = crypto_gcm_check_authsize(authtag_len); 1514 if (unlikely(err)) 1515 return err; 1516 1517 err = aes_prepareenckey(&key->aes, in_key, key_len); 1518 if (unlikely(err)) 1519 return err; 1520 1521 aes_encrypt(&key->aes, h, h); 1522 ghash_preparekey(&key->ghash, h); 1523 1524 key->authtag_len = authtag_len; 1525 1526 memzero_explicit(h, sizeof(h)); 1527 return 0; 1528 } 1529 EXPORT_SYMBOL_GPL(aes_gcm_preparekey); 1530 1531 void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], 1532 const struct aes_gcm_key *key) 1533 { 1534 ctx->key = key; 1535 ctx->ad_len = 0; 1536 ctx->data_len = 0; 1537 ghash_init(&ctx->ghash, &key->ghash); 1538 memset(ctx->keystream, 0, sizeof(ctx->keystream)); 1539 1540 memcpy(ctx->ctr32, nonce, 12); 1541 ctx->ctr32[3] = cpu_to_be32(1); 1542 1543 aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); 1544 ctx->ctr32[3] = cpu_to_be32(2); 1545 } 1546 EXPORT_SYMBOL_GPL(aes_gcm_init); 1547 1548 void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) 1549 { 1550 WARN_ON_ONCE(ctx->data_len != 0); 1551 if (len) { 1552 ghash_update(&ctx->ghash, ad, len); 1553 ctx->ad_len += len; 1554 } 1555 } 1556 EXPORT_SYMBOL_GPL(aes_gcm_auth_update); 1557 1558 static const u8 gcm_zeroes[AES_BLOCK_SIZE]; 1559 1560 static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) 1561 { 1562 if (len % AES_BLOCK_SIZE) 1563 ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); 1564 } 1565 1566 static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, 1567 u8 *dst, const u8 *src, 1568 size_t len, bool enc) 1569 { 1570 size_t partial_len, n; 1571 1572 if (unlikely(len == 0)) 1573 return; 1574 1575 partial_len = ctx->data_len % AES_BLOCK_SIZE; 1576 if (ctx->data_len == 0) 1577 ghash_pad(&ctx->ghash, ctx->ad_len); 1578 ctx->data_len += len; 1579 1580 if (unlikely(partial_len != 0)) { 1581 /* 1582 * The previous call ended on a non-block-aligned data_len, so 1583 * continue using a previously-generated keystream block. 1584 */ 1585 n = min(len, AES_BLOCK_SIZE - partial_len); 1586 if (enc) { 1587 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1588 n); 1589 ghash_update(&ctx->ghash, dst, n); 1590 } else { 1591 ghash_update(&ctx->ghash, src, n); 1592 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1593 n); 1594 } 1595 dst += n; 1596 src += n; 1597 len -= n; 1598 } 1599 1600 if (len >= AES_BLOCK_SIZE) { 1601 n = round_down(len, AES_BLOCK_SIZE); 1602 if (enc) { 1603 if (likely(aes_gcm_encrypt_update_arch( 1604 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1605 &ctx->key->aes, &ctx->key->ghash))) { 1606 be32_add_cpu(&ctx->ctr32[3], 1607 n / AES_BLOCK_SIZE); 1608 } else { 1609 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1610 ghash_update(&ctx->ghash, dst, n); 1611 } 1612 } else { 1613 if (likely(aes_gcm_decrypt_update_arch( 1614 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1615 &ctx->key->aes, &ctx->key->ghash))) { 1616 be32_add_cpu(&ctx->ctr32[3], 1617 n / AES_BLOCK_SIZE); 1618 } else { 1619 ghash_update(&ctx->ghash, src, n); 1620 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1621 } 1622 } 1623 dst += n; 1624 src += n; 1625 len -= n; 1626 } 1627 1628 if (len != 0) { 1629 /* 1630 * Ending on a non-block aligned data_len. Generate the next 1631 * keystream block, use the needed portion of it, and leave it 1632 * cached in ctx->keystream in case this isn't the final call. 1633 */ 1634 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1635 be32_add_cpu(&ctx->ctr32[3], 1); 1636 if (enc) { 1637 crypto_xor_cpy(dst, src, ctx->keystream, len); 1638 ghash_update(&ctx->ghash, dst, len); 1639 } else { 1640 ghash_update(&ctx->ghash, src, len); 1641 crypto_xor_cpy(dst, src, ctx->keystream, len); 1642 } 1643 } 1644 } 1645 1646 void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1647 size_t len) 1648 { 1649 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1650 } 1651 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); 1652 1653 void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1654 size_t len) 1655 { 1656 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); 1657 } 1658 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); 1659 1660 /* Maximum AES-GCM associated data length in bytes */ 1661 #define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) 1662 /* Maximum AES-GCM en/decrypted data length in bytes */ 1663 #define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) 1664 1665 void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) 1666 { 1667 __be64 tail[2]; 1668 1669 WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); 1670 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); 1671 1672 ghash_pad(&ctx->ghash, 1673 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1674 1675 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1676 tail[1] = cpu_to_be64(ctx->data_len * 8); 1677 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1678 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1679 1680 crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1681 memzero_explicit(ctx, sizeof(*ctx)); 1682 } 1683 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); 1684 1685 int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) 1686 { 1687 __be64 tail[2]; 1688 int err; 1689 1690 if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || 1691 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { 1692 err = -EBADMSG; 1693 goto out; 1694 } 1695 1696 ghash_pad(&ctx->ghash, 1697 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1698 1699 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1700 tail[1] = cpu_to_be64(ctx->data_len * 8); 1701 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1702 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1703 crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1704 err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? 1705 -EBADMSG : 1706 0; 1707 out: 1708 memzero_explicit(ctx, sizeof(*ctx)); 1709 return err; 1710 } 1711 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); 1712 1713 void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 1714 const u8 *ad, size_t ad_len, const u8 nonce[12], 1715 const struct aes_gcm_key *key) 1716 { 1717 struct aes_gcm_ctx ctx; 1718 1719 aes_gcm_init(&ctx, nonce, key); 1720 aes_gcm_auth_update(&ctx, ad, ad_len); 1721 aes_gcm_encrypt_update(&ctx, dst, src, data_len); 1722 aes_gcm_encrypt_final(&ctx, authtag); 1723 } 1724 EXPORT_SYMBOL_GPL(aes_gcm_encrypt); 1725 1726 int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 1727 const u8 *ad, size_t ad_len, const u8 nonce[12], 1728 const struct aes_gcm_key *key) 1729 { 1730 struct aes_gcm_ctx ctx; 1731 int err; 1732 1733 aes_gcm_init(&ctx, nonce, key); 1734 aes_gcm_auth_update(&ctx, ad, ad_len); 1735 aes_gcm_decrypt_update(&ctx, dst, src, data_len); 1736 err = aes_gcm_decrypt_final(&ctx, authtag); 1737 if (unlikely(err) && data_len) { 1738 /* 1739 * Clear the inauthentic decrypted data so that callers won't 1740 * receive it even if they fail to correctly handle errors. 1741 */ 1742 memset(dst, 0, data_len); 1743 } 1744 return err; 1745 } 1746 EXPORT_SYMBOL_GPL(aes_gcm_decrypt); 1747 1748 #endif /* CONFIG_CRYPTO_LIB_AES_GCM */ 1749 1750 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) 1751 int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, 1752 size_t key_len, size_t authtag_len) 1753 { 1754 int err; 1755 1756 if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) 1757 return -EINVAL; 1758 1759 err = aes_prepareenckey(&key->aes, in_key, key_len); 1760 if (unlikely(err)) 1761 return err; 1762 1763 key->authtag_len = authtag_len; 1764 return 0; 1765 } 1766 EXPORT_SYMBOL_GPL(aes_ccm_preparekey); 1767 1768 int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, 1769 const u8 *nonce, size_t nonce_len, 1770 const struct aes_ccm_key *key) 1771 { 1772 /* 1773 * This is the value L defined in the CCM specification. It determines 1774 * the maximum allowed message length, and it is itself determined by 1775 * the nonce length. They are inversely related, i.e. the longer the 1776 * nonce the smaller the maximum message length is. 1777 */ 1778 unsigned int l = 15 - nonce_len; 1779 1780 if (unlikely(nonce_len < 7 || nonce_len > 13)) 1781 return -EINVAL; 1782 /* Thus 2 <= l <= 8. */ 1783 1784 /* Check whether data_len can be represented in 'l' bytes. */ 1785 if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) 1786 return -EOVERFLOW; 1787 1788 ctx->key = key; 1789 ctx->ad_remaining = ad_len; 1790 ctx->data_remaining = data_len; 1791 ctx->ad_padded = false; 1792 1793 /* 1794 * Initialize the zero-th counter block to: 1795 * 1796 * L - 1 || nonce || 0 1797 * 1798 * ... and the zero-th CBC-MAC block to: 1799 * 1800 * Flags || nonce || data_len 1801 */ 1802 *(__be64 *)&ctx->ctr[8] = 0; 1803 *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); 1804 ctx->ctr[0] = l - 1; 1805 ctx->mac[0] = (ad_len ? 0x40 : 0) | 1806 (((key->authtag_len - 2) / 2) << 3) | (l - 1); 1807 memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ 1808 memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ 1809 1810 /* 1811 * Generate S_0 by encrypting the counter (this is used to encrypt the 1812 * auth tag later), and encrypt the zero-th CBC-MAC block. 1813 */ 1814 aes_encrypt(&key->aes, ctx->s0, ctx->ctr); 1815 aes_encrypt(&key->aes, ctx->mac, ctx->mac); 1816 1817 /* Increment the counter from 0 to 1. */ 1818 ctx->ctr[15] = 1; 1819 1820 if (ad_len) { 1821 /* 1822 * Update CBC-MAC with the associated data length, represented 1823 * using either 2, 6, or 10 bytes depending on the length. 1824 */ 1825 if (likely(ad_len < 0xff00)) { 1826 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); 1827 ctx->partial_len = 2; 1828 } else if (ad_len <= U32_MAX) { 1829 __be32 *p = (__be32 *)&ctx->mac[2]; 1830 1831 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); 1832 put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), 1833 p); 1834 ctx->partial_len = 6; 1835 } else { 1836 __be64 *p = (__be64 *)&ctx->mac[2]; 1837 1838 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); 1839 put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), 1840 p); 1841 ctx->partial_len = 10; 1842 } 1843 } else { 1844 ctx->partial_len = 0; 1845 } 1846 return 0; 1847 } 1848 EXPORT_SYMBOL_GPL(aes_ccm_init); 1849 1850 void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) 1851 { 1852 size_t partial_len = ctx->partial_len; 1853 bool enc_before = false; 1854 size_t nblocks; 1855 1856 WARN_ON_ONCE(ctx->ad_padded); 1857 1858 /* 1859 * We could warn on len > ad_remaining here, but underflow will be 1860 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1861 * going to underflow all the way back to 0.) 1862 */ 1863 ctx->ad_remaining -= len; 1864 1865 if (partial_len) { 1866 size_t n = min(len, AES_BLOCK_SIZE - partial_len); 1867 1868 crypto_xor(&ctx->mac[partial_len], ad, n); 1869 ad += n; 1870 len -= n; 1871 partial_len += n; 1872 if (partial_len < AES_BLOCK_SIZE) { 1873 ctx->partial_len = partial_len; 1874 return; 1875 } 1876 enc_before = true; 1877 } 1878 1879 nblocks = len / AES_BLOCK_SIZE; 1880 len %= AES_BLOCK_SIZE; 1881 if (nblocks == 0) { 1882 if (enc_before) 1883 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1884 } else { 1885 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, 1886 enc_before, /* enc_after= */ true); 1887 ad += nblocks * AES_BLOCK_SIZE; 1888 } 1889 crypto_xor(ctx->mac, ad, len); 1890 ctx->partial_len = len; 1891 } 1892 EXPORT_SYMBOL_GPL(aes_ccm_auth_update); 1893 1894 static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, 1895 u8 *dst, const u8 *src, 1896 size_t len, bool enc) 1897 { 1898 size_t partial_len = ctx->partial_len; 1899 size_t n, nblocks; 1900 1901 if (unlikely(len == 0)) 1902 return; 1903 1904 WARN_ON_ONCE(ctx->ad_remaining != 0); 1905 1906 /* 1907 * We could warn on len > data_remaining here, but underflow will be 1908 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1909 * going to underflow all the way back to 0.) 1910 */ 1911 ctx->data_remaining -= len; 1912 1913 if (!ctx->ad_padded) { 1914 ctx->ad_padded = true; 1915 if (partial_len) 1916 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1917 } else if (partial_len) { 1918 /* 1919 * The previous call ended on a non-block-aligned data_len, so 1920 * continue using a previously-generated keystream block. 1921 */ 1922 n = min(len, AES_BLOCK_SIZE - partial_len); 1923 if (enc) 1924 crypto_xor(&ctx->mac[partial_len], src, n); 1925 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); 1926 if (!enc) 1927 crypto_xor(&ctx->mac[partial_len], dst, n); 1928 dst += n; 1929 src += n; 1930 len -= n; 1931 partial_len += n; 1932 if (partial_len < AES_BLOCK_SIZE) { 1933 ctx->partial_len = partial_len; 1934 return; 1935 } 1936 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1937 } 1938 1939 if (len >= AES_BLOCK_SIZE) { 1940 n = round_down(len, AES_BLOCK_SIZE); 1941 nblocks = len / AES_BLOCK_SIZE; 1942 if (enc) 1943 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, 1944 nblocks, /* enc_before= */ false, 1945 /* enc_after= */ true); 1946 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1947 if (!enc) 1948 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, 1949 nblocks, /* enc_before= */ false, 1950 /* enc_after= */ true); 1951 dst += n; 1952 src += n; 1953 len -= n; 1954 } 1955 1956 if (len) { 1957 /* 1958 * Ending on a non-block aligned data_len. Generate the next 1959 * keystream block, use the needed portion of it, and leave it 1960 * cached in ctx->keystream in case this isn't the final call. 1961 */ 1962 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1963 inc_be128_ctr(ctx->ctr); 1964 if (enc) 1965 crypto_xor(ctx->mac, src, len); 1966 crypto_xor_cpy(dst, src, ctx->keystream, len); 1967 if (!enc) 1968 crypto_xor(ctx->mac, dst, len); 1969 } 1970 ctx->partial_len = len; 1971 } 1972 1973 void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 1974 size_t len) 1975 { 1976 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1977 } 1978 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); 1979 1980 void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 1981 size_t len) 1982 { 1983 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); 1984 } 1985 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); 1986 1987 void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) 1988 { 1989 WARN_ON_ONCE(ctx->ad_remaining != 0); 1990 WARN_ON_ONCE(ctx->data_remaining != 0); 1991 if (ctx->partial_len) 1992 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1993 crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); 1994 memzero_explicit(ctx, sizeof(*ctx)); 1995 } 1996 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); 1997 1998 int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) 1999 { 2000 int err; 2001 2002 if (WARN_ON_ONCE(ctx->ad_remaining != 0) || 2003 WARN_ON_ONCE(ctx->data_remaining != 0)) { 2004 err = -EBADMSG; 2005 goto out; 2006 } 2007 2008 if (ctx->partial_len) 2009 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 2010 crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); 2011 err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? 2012 -EBADMSG : 2013 0; 2014 out: 2015 memzero_explicit(ctx, sizeof(*ctx)); 2016 return err; 2017 } 2018 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); 2019 2020 int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 2021 const u8 *ad, size_t ad_len, const u8 *nonce, 2022 size_t nonce_len, const struct aes_ccm_key *key) 2023 { 2024 struct aes_ccm_ctx ctx; 2025 int err; 2026 2027 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2028 if (unlikely(err)) 2029 return err; 2030 aes_ccm_auth_update(&ctx, ad, ad_len); 2031 aes_ccm_encrypt_update(&ctx, dst, src, data_len); 2032 aes_ccm_encrypt_final(&ctx, authtag); 2033 return 0; 2034 } 2035 EXPORT_SYMBOL_GPL(aes_ccm_encrypt); 2036 2037 int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 2038 const u8 *ad, size_t ad_len, const u8 *nonce, 2039 size_t nonce_len, const struct aes_ccm_key *key) 2040 { 2041 struct aes_ccm_ctx ctx; 2042 int err; 2043 2044 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2045 if (unlikely(err)) 2046 return err; 2047 aes_ccm_auth_update(&ctx, ad, ad_len); 2048 aes_ccm_decrypt_update(&ctx, dst, src, data_len); 2049 err = aes_ccm_decrypt_final(&ctx, authtag); 2050 if (unlikely(err) && data_len) { 2051 /* 2052 * Clear the inauthentic decrypted data so that callers won't 2053 * receive it even if they fail to correctly handle errors. 2054 */ 2055 memset(dst, 0, data_len); 2056 } 2057 return err; 2058 } 2059 EXPORT_SYMBOL_GPL(aes_ccm_decrypt); 2060 #endif /* CONFIG_CRYPTO_LIB_AES_CCM */ 2061 2062 static int __init aes_mod_init(void) 2063 { 2064 #ifdef aes_mod_init_arch 2065 aes_mod_init_arch(); 2066 #endif 2067 if (fips_enabled) { 2068 aes_fips_test(); 2069 aes_cmac_fips_test(); 2070 aes_ecb_fips_test(); 2071 aes_cbc_fips_test(); 2072 aes_cbc_cts_fips_test(); 2073 aes_ctr_fips_test(); 2074 aes_xts_fips_test(); 2075 } 2076 return 0; 2077 } 2078 subsys_initcall(aes_mod_init); 2079 2080 static void __exit aes_mod_exit(void) 2081 { 2082 } 2083 module_exit(aes_mod_exit); 2084 2085 MODULE_DESCRIPTION("AES block cipher"); 2086 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 2087 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 2088 MODULE_LICENSE("GPL v2"); 2089