1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ccm.h> 10 #include <crypto/aes-ctr.h> 11 #include <crypto/aes-ecb.h> 12 #include <crypto/aes-gcm.h> 13 #include <crypto/aes-xts.h> 14 #include <crypto/aes.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/utils.h> 17 #include <linux/cache.h> 18 #include <linux/crypto.h> 19 #include <linux/export.h> 20 #include <linux/module.h> 21 #include <linux/unaligned.h> 22 #include "fips-aes.h" 23 24 static const u8 ____cacheline_aligned aes_sbox[] = { 25 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 26 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 27 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 28 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 29 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 30 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 31 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 32 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 33 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 34 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 35 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 36 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 37 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 38 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 39 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 40 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 41 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 42 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 43 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 44 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 45 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 46 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 47 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 48 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 49 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 50 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 51 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 52 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 53 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 54 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 55 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 56 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 57 }; 58 59 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 60 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 61 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 62 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 63 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 64 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 65 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 66 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 67 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 68 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 69 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 70 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 71 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 72 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 73 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 74 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 75 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 76 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 77 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 78 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 79 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 80 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 81 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 82 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 83 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 84 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 85 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 86 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 87 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 88 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 89 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 90 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 91 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 92 }; 93 94 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 95 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 96 97 EXPORT_SYMBOL(crypto_aes_sbox); 98 EXPORT_SYMBOL(crypto_aes_inv_sbox); 99 100 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 101 const u32 ____cacheline_aligned aes_enc_tab[256] = { 102 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 103 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 104 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 105 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 106 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 107 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 108 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 109 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 110 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 111 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 112 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 113 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 114 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 115 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 116 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 117 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 118 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 119 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 120 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 121 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 122 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 123 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 124 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 125 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 126 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 127 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 128 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 129 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 130 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 131 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 132 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 133 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 134 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 135 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 136 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 137 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 138 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 139 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 140 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 141 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 142 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 143 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 144 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 145 }; 146 EXPORT_SYMBOL(aes_enc_tab); 147 148 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 149 const u32 ____cacheline_aligned aes_dec_tab[256] = { 150 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 151 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 152 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 153 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 154 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 155 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 156 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 157 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 158 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 159 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 160 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 161 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 162 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 163 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 164 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 165 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 166 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 167 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 168 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 169 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 170 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 171 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 172 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 173 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 174 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 175 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 176 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 177 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 178 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 179 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 180 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 181 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 182 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 183 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 184 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 185 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 186 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 187 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 188 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 189 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 190 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 191 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 192 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 193 }; 194 EXPORT_SYMBOL(aes_dec_tab); 195 196 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 197 static __always_inline void aes_prefetch(const void *mem, size_t len) 198 { 199 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 200 *(volatile const u8 *)(mem + i); 201 barrier(); 202 } 203 204 static u32 mul_by_x(u32 w) 205 { 206 u32 x = w & 0x7f7f7f7f; 207 u32 y = w & 0x80808080; 208 209 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 210 return (x << 1) ^ (y >> 7) * 0x1b; 211 } 212 213 static u32 mul_by_x2(u32 w) 214 { 215 u32 x = w & 0x3f3f3f3f; 216 u32 y = w & 0x80808080; 217 u32 z = w & 0x40404040; 218 219 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 220 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 221 } 222 223 static u32 mix_columns(u32 x) 224 { 225 /* 226 * Perform the following matrix multiplication in GF(2^8) 227 * 228 * | 0x2 0x3 0x1 0x1 | | x[0] | 229 * | 0x1 0x2 0x3 0x1 | | x[1] | 230 * | 0x1 0x1 0x2 0x3 | x | x[2] | 231 * | 0x3 0x1 0x1 0x2 | | x[3] | 232 */ 233 u32 y = mul_by_x(x) ^ ror32(x, 16); 234 235 return y ^ ror32(x ^ y, 8); 236 } 237 238 static u32 inv_mix_columns(u32 x) 239 { 240 /* 241 * Perform the following matrix multiplication in GF(2^8) 242 * 243 * | 0xe 0xb 0xd 0x9 | | x[0] | 244 * | 0x9 0xe 0xb 0xd | | x[1] | 245 * | 0xd 0x9 0xe 0xb | x | x[2] | 246 * | 0xb 0xd 0x9 0xe | | x[3] | 247 * 248 * which can conveniently be reduced to 249 * 250 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 251 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 252 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 253 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 254 */ 255 u32 y = mul_by_x2(x); 256 257 return mix_columns(x ^ y ^ ror32(y, 16)); 258 } 259 260 static u32 subw(u32 in) 261 { 262 return (aes_sbox[in & 0xff]) ^ 263 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 264 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 265 (aes_sbox[(in >> 24) & 0xff] << 24); 266 } 267 268 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 269 const u8 *in_key, int key_len) 270 { 271 u32 kwords = key_len / sizeof(u32); 272 u32 rc, i, j; 273 274 for (i = 0; i < kwords; i++) 275 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 276 277 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 278 u32 *rki = &rndkeys[i * kwords]; 279 u32 *rko = rki + kwords; 280 281 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 282 rko[1] = rko[0] ^ rki[1]; 283 rko[2] = rko[1] ^ rki[2]; 284 rko[3] = rko[2] ^ rki[3]; 285 286 if (key_len == AES_KEYSIZE_192) { 287 if (i >= 7) 288 break; 289 rko[4] = rko[3] ^ rki[4]; 290 rko[5] = rko[4] ^ rki[5]; 291 } else if (key_len == AES_KEYSIZE_256) { 292 if (i >= 6) 293 break; 294 rko[4] = subw(rko[3]) ^ rki[4]; 295 rko[5] = rko[4] ^ rki[5]; 296 rko[6] = rko[5] ^ rki[6]; 297 rko[7] = rko[6] ^ rki[7]; 298 } 299 } 300 301 /* 302 * Generate the decryption keys for the Equivalent Inverse Cipher. 303 * This involves reversing the order of the round keys, and applying 304 * the Inverse Mix Columns transformation to all but the first and 305 * the last one. 306 */ 307 if (inv_rndkeys) { 308 inv_rndkeys[0] = rndkeys[key_len + 24]; 309 inv_rndkeys[1] = rndkeys[key_len + 25]; 310 inv_rndkeys[2] = rndkeys[key_len + 26]; 311 inv_rndkeys[3] = rndkeys[key_len + 27]; 312 313 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 314 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 315 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 316 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 317 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 318 } 319 320 inv_rndkeys[i] = rndkeys[0]; 321 inv_rndkeys[i + 1] = rndkeys[1]; 322 inv_rndkeys[i + 2] = rndkeys[2]; 323 inv_rndkeys[i + 3] = rndkeys[3]; 324 } 325 } 326 327 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 328 unsigned int key_len) 329 { 330 if (aes_check_keylen(key_len) != 0) 331 return -EINVAL; 332 ctx->key_length = key_len; 333 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 334 return 0; 335 } 336 EXPORT_SYMBOL(aes_expandkey); 337 338 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 339 { 340 return rk ^ aes_enc_tab[(u8)w[i]] ^ 341 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 342 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 343 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 344 } 345 346 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 347 { 348 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 349 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 350 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 351 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 352 } 353 354 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 355 u8 out[AES_BLOCK_SIZE], 356 const u8 in[AES_BLOCK_SIZE]) 357 { 358 const u32 *rkp = rndkeys; 359 int n = nrounds - 1; 360 u32 w[4]; 361 362 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 363 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 364 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 365 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 366 367 /* 368 * Prefetch the table before doing data and key-dependent loads from it. 369 * 370 * This is intended only as a basic constant-time hardening measure that 371 * avoids interfering with performance too much. Its effectiveness is 372 * not guaranteed. For proper constant-time AES, a CPU that supports 373 * AES instructions should be used instead. 374 */ 375 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 376 377 do { 378 u32 w0 = enc_quarterround(w, 0, *rkp++); 379 u32 w1 = enc_quarterround(w, 1, *rkp++); 380 u32 w2 = enc_quarterround(w, 2, *rkp++); 381 u32 w3 = enc_quarterround(w, 3, *rkp++); 382 383 w[0] = w0; 384 w[1] = w1; 385 w[2] = w2; 386 w[3] = w3; 387 } while (--n); 388 389 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 390 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 391 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 392 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 393 } 394 395 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 396 { 397 return rk ^ aes_dec_tab[(u8)w[i]] ^ 398 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 399 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 400 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 401 } 402 403 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 404 { 405 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 406 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 407 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 408 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 409 } 410 411 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 412 int nrounds, 413 u8 out[AES_BLOCK_SIZE], 414 const u8 in[AES_BLOCK_SIZE]) 415 { 416 const u32 *rkp = inv_rndkeys; 417 int n = nrounds - 1; 418 u32 w[4]; 419 420 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 421 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 422 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 423 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 424 425 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 426 427 do { 428 u32 w0 = dec_quarterround(w, 0, *rkp++); 429 u32 w1 = dec_quarterround(w, 1, *rkp++); 430 u32 w2 = dec_quarterround(w, 2, *rkp++); 431 u32 w3 = dec_quarterround(w, 3, *rkp++); 432 433 w[0] = w0; 434 w[1] = w1; 435 w[2] = w2; 436 w[3] = w3; 437 } while (--n); 438 439 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 440 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 441 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 442 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 443 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 444 } 445 446 /* 447 * Note: the aes_prepare*key_* names reflect the fact that the implementation 448 * might not actually expand the key. (The s390 code for example doesn't.) 449 * Where the key is expanded we use the more specific names aes_expandkey_*. 450 * 451 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 452 * the area to store the prepared decryption key. It will be NULL if the user 453 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 454 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 455 */ 456 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 457 /* An arch-specific implementation of AES is available. Include it. */ 458 #include "aes.h" /* $(SRCARCH)/aes.h */ 459 #else 460 /* No arch-specific implementation of AES is available. Use generic code. */ 461 462 static void aes_preparekey_arch(union aes_enckey_arch *k, 463 union aes_invkey_arch *inv_k, 464 const u8 *in_key, int key_len, int nrounds) 465 { 466 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 467 in_key, key_len); 468 } 469 470 static void aes_encrypt_arch(const struct aes_enckey *key, 471 u8 out[AES_BLOCK_SIZE], 472 const u8 in[AES_BLOCK_SIZE]) 473 { 474 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 475 } 476 477 static void aes_decrypt_arch(const struct aes_key *key, 478 u8 out[AES_BLOCK_SIZE], 479 const u8 in[AES_BLOCK_SIZE]) 480 { 481 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 482 } 483 #endif 484 485 static int __aes_preparekey(struct aes_enckey *enc_key, 486 union aes_invkey_arch *inv_k, 487 const u8 *in_key, size_t key_len) 488 { 489 if (aes_check_keylen(key_len) != 0) 490 return -EINVAL; 491 enc_key->len = key_len; 492 enc_key->nrounds = 6 + key_len / 4; 493 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 494 enc_key->nrounds); 495 return 0; 496 } 497 498 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 499 { 500 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 501 in_key, key_len); 502 } 503 EXPORT_SYMBOL(aes_preparekey); 504 505 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 506 { 507 return __aes_preparekey(key, NULL, in_key, key_len); 508 } 509 EXPORT_SYMBOL(aes_prepareenckey); 510 511 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 512 const u8 in[AES_BLOCK_SIZE]) 513 { 514 aes_encrypt_arch(key.enc_key, out, in); 515 } 516 EXPORT_SYMBOL(aes_encrypt); 517 518 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 519 const u8 in[AES_BLOCK_SIZE]) 520 { 521 aes_decrypt_arch(key, out, in); 522 } 523 EXPORT_SYMBOL(aes_decrypt); 524 525 /* FIPS cryptographic algorithm self-test for "bare" AES */ 526 static void __init aes_fips_test(void) 527 { 528 struct aes_key key; 529 u8 data[AES_BLOCK_SIZE]; 530 531 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 532 panic("aes: FIPS self-test failed (preparekey)\n"); 533 534 aes_encrypt(&key, data, fips_test_data); 535 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 536 panic("aes: FIPS self-test failed (wrong ciphertext)\n"); 537 538 aes_decrypt(&key, data, data); 539 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 540 panic("aes: FIPS self-test failed (wrong plaintext)\n"); 541 542 memzero_explicit(&key, sizeof(key)); 543 } 544 545 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 546 547 #ifndef aes_cbcmac_blocks_arch 548 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 549 const struct aes_enckey *key, const u8 *data, 550 size_t nblocks, bool enc_before, 551 bool enc_after) 552 { 553 return false; 554 } 555 #endif 556 557 /* This assumes nblocks >= 1. */ 558 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 559 const struct aes_enckey *key, const u8 *data, 560 size_t nblocks, bool enc_before, bool enc_after) 561 { 562 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 563 enc_after)) 564 return; 565 566 if (enc_before) 567 aes_encrypt(key, h, h); 568 for (; nblocks > 1; nblocks--) { 569 crypto_xor(h, data, AES_BLOCK_SIZE); 570 data += AES_BLOCK_SIZE; 571 aes_encrypt(key, h, h); 572 } 573 crypto_xor(h, data, AES_BLOCK_SIZE); 574 if (enc_after) 575 aes_encrypt(key, h, h); 576 } 577 578 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 579 size_t key_len) 580 { 581 u64 hi, lo, mask; 582 int err; 583 584 /* Prepare the AES key. */ 585 err = aes_prepareenckey(&key->aes, in_key, key_len); 586 if (err) 587 return err; 588 589 /* 590 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 591 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 592 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 593 */ 594 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 595 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 596 hi = be64_to_cpu(key->k_final[0].w[0]); 597 lo = be64_to_cpu(key->k_final[0].w[1]); 598 for (int i = 0; i < 2; i++) { 599 mask = ((s64)hi >> 63) & 0x87; 600 hi = (hi << 1) ^ (lo >> 63); 601 lo = (lo << 1) ^ mask; 602 key->k_final[i].w[0] = cpu_to_be64(hi); 603 key->k_final[i].w[1] = cpu_to_be64(lo); 604 } 605 return 0; 606 } 607 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 608 609 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 610 const u8 in_key[AES_KEYSIZE_128]) 611 { 612 static const u8 constants[3][AES_BLOCK_SIZE] = { 613 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 614 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 615 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 616 }; 617 u8 new_aes_key[AES_BLOCK_SIZE]; 618 619 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 620 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 621 aes_encrypt(&key->aes, new_aes_key, constants[0]); 622 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 623 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 624 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 625 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 626 } 627 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 628 629 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 630 { 631 bool enc_before = false; 632 size_t nblocks; 633 634 if (ctx->partial_len) { 635 /* XOR data into a pending block. */ 636 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 637 638 crypto_xor(&ctx->h[ctx->partial_len], data, l); 639 data += l; 640 data_len -= l; 641 ctx->partial_len += l; 642 if (data_len == 0) { 643 /* 644 * Either the pending block hasn't been filled yet, or 645 * no more data was given so it's not yet known whether 646 * the block is the final block. 647 */ 648 return; 649 } 650 /* Pending block has been filled and isn't the final block. */ 651 enc_before = true; 652 } 653 654 nblocks = data_len / AES_BLOCK_SIZE; 655 data_len %= AES_BLOCK_SIZE; 656 if (nblocks == 0) { 657 /* 0 additional full blocks, then optionally a partial block */ 658 if (enc_before) 659 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 660 crypto_xor(ctx->h, data, data_len); 661 ctx->partial_len = data_len; 662 } else if (data_len != 0) { 663 /* 1 or more additional full blocks, then a partial block */ 664 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 665 enc_before, /* enc_after= */ true); 666 data += nblocks * AES_BLOCK_SIZE; 667 crypto_xor(ctx->h, data, data_len); 668 ctx->partial_len = data_len; 669 } else { 670 /* 671 * 1 or more additional full blocks only. Encryption of the 672 * last block is delayed until it's known whether it's the final 673 * block in the message or not. 674 */ 675 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 676 enc_before, /* enc_after= */ false); 677 ctx->partial_len = AES_BLOCK_SIZE; 678 } 679 } 680 EXPORT_SYMBOL_GPL(aes_cmac_update); 681 682 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 683 { 684 if (ctx->partial_len == AES_BLOCK_SIZE) { 685 /* Final block is a full block. Use k_final[0]. */ 686 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 687 } else { 688 /* Final block is a partial block. Pad, and use k_final[1]. */ 689 ctx->h[ctx->partial_len] ^= 0x80; 690 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 691 } 692 aes_encrypt(&ctx->key->aes, out, ctx->h); 693 memzero_explicit(ctx, sizeof(*ctx)); 694 } 695 EXPORT_SYMBOL_GPL(aes_cmac_final); 696 697 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 698 size_t data_len) 699 { 700 bool enc_before = false; 701 size_t nblocks; 702 703 if (ctx->partial_len) { 704 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 705 706 crypto_xor(&ctx->h[ctx->partial_len], data, l); 707 data += l; 708 data_len -= l; 709 ctx->partial_len += l; 710 if (ctx->partial_len < AES_BLOCK_SIZE) 711 return; 712 enc_before = true; 713 } 714 715 nblocks = data_len / AES_BLOCK_SIZE; 716 data_len %= AES_BLOCK_SIZE; 717 if (nblocks == 0) { 718 if (enc_before) 719 aes_encrypt(ctx->key, ctx->h, ctx->h); 720 } else { 721 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 722 /* enc_after= */ true); 723 data += nblocks * AES_BLOCK_SIZE; 724 } 725 crypto_xor(ctx->h, data, data_len); 726 ctx->partial_len = data_len; 727 } 728 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 729 730 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 731 { 732 if (ctx->partial_len) 733 aes_encrypt(ctx->key, out, ctx->h); 734 else 735 memcpy(out, ctx->h, AES_BLOCK_SIZE); 736 memzero_explicit(ctx, sizeof(*ctx)); 737 } 738 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 739 740 /* FIPS cryptographic algorithm self-test for AES-CMAC */ 741 static void __init aes_cmac_fips_test(void) 742 { 743 struct aes_cmac_key key; 744 u8 mac[AES_BLOCK_SIZE]; 745 746 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 747 0) 748 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 749 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 750 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 751 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 752 memzero_explicit(&key, sizeof(key)); 753 } 754 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 755 static inline void aes_cmac_fips_test(void) 756 { 757 } 758 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 759 760 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 761 /* 762 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 763 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 764 * causes the fallback implementation to be used instead. 765 */ 766 #ifndef aes_ecb_encrypt_arch 767 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 768 const struct aes_enckey *key) 769 { 770 return false; 771 } 772 #endif 773 #ifndef aes_ecb_decrypt_arch 774 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 775 const struct aes_key *key) 776 { 777 return false; 778 } 779 #endif 780 781 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 782 { 783 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 784 len = round_down(len, AES_BLOCK_SIZE); 785 786 if (unlikely(len == 0)) 787 return; 788 789 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 790 return; 791 792 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 793 aes_encrypt(key, &dst[i], &src[i]); 794 } 795 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 796 797 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 798 const struct aes_key *key) 799 { 800 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 801 len = round_down(len, AES_BLOCK_SIZE); 802 803 if (unlikely(len == 0)) 804 return; 805 806 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 807 return; 808 809 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 810 aes_decrypt(key, &dst[i], &src[i]); 811 } 812 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 813 814 /* FIPS cryptographic algorithm self-test for AES-ECB */ 815 static void __init aes_ecb_fips_test(void) 816 { 817 struct aes_key key; 818 u8 data[sizeof(fips_test_data)]; 819 820 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 821 panic("aes: ECB FIPS self-test failed (preparekey)\n"); 822 823 aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key); 824 if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) 825 panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n"); 826 827 aes_ecb_decrypt(data, data, sizeof(data), &key); 828 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 829 panic("aes: ECB FIPS self-test failed (wrong plaintext)\n"); 830 831 memzero_explicit(&key, sizeof(key)); 832 } 833 #else /* CONFIG_CRYPTO_LIB_AES_ECB */ 834 static inline void aes_ecb_fips_test(void) 835 { 836 } 837 #endif /* !CONFIG_CRYPTO_LIB_AES_ECB */ 838 839 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 840 /* 841 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 842 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 843 * causes the fallback implementation to be used instead. 844 */ 845 #ifndef aes_cbc_encrypt_arch 846 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 847 u8 iv[AES_BLOCK_SIZE], 848 const struct aes_enckey *key) 849 { 850 return false; 851 } 852 #endif 853 #ifndef aes_cbc_decrypt_arch 854 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 855 u8 iv[AES_BLOCK_SIZE], 856 const struct aes_key *key) 857 { 858 return false; 859 } 860 #endif 861 862 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 863 aes_encrypt_arg key) 864 { 865 const u8 *prev = iv; 866 867 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 868 len = round_down(len, AES_BLOCK_SIZE); 869 870 if (unlikely(len == 0)) 871 return; 872 873 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 874 return; 875 876 do { 877 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 878 aes_encrypt(key, dst, dst); 879 prev = dst; 880 dst += AES_BLOCK_SIZE; 881 src += AES_BLOCK_SIZE; 882 len -= AES_BLOCK_SIZE; 883 } while (len); 884 memcpy(iv, prev, AES_BLOCK_SIZE); 885 } 886 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 887 888 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 889 const struct aes_key *key) 890 { 891 u8 next_iv[AES_BLOCK_SIZE]; 892 893 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 894 len = round_down(len, AES_BLOCK_SIZE); 895 896 if (unlikely(len == 0)) 897 return; 898 899 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 900 return; 901 902 len -= AES_BLOCK_SIZE; 903 dst += len; 904 src += len; 905 memcpy(next_iv, src, AES_BLOCK_SIZE); 906 for (;;) { 907 aes_decrypt(key, dst, src); 908 if (len == 0) 909 break; 910 src -= AES_BLOCK_SIZE; 911 crypto_xor(dst, src, AES_BLOCK_SIZE); 912 dst -= AES_BLOCK_SIZE; 913 len -= AES_BLOCK_SIZE; 914 } 915 crypto_xor(dst, iv, AES_BLOCK_SIZE); 916 memcpy(iv, next_iv, AES_BLOCK_SIZE); 917 } 918 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 919 920 /* 921 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 922 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 923 * causes the fallback implementation to be used instead. The fallback 924 * implementation still uses the arch-optimized AES-CBC code if available, but 925 * direct implementation of AES-CBC-CTS is helpful on short messages. 926 */ 927 #ifndef aes_cbc_cts_encrypt_arch 928 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 929 u8 iv[AES_BLOCK_SIZE], 930 const struct aes_enckey *key) 931 { 932 return false; 933 } 934 #endif 935 #ifndef aes_cbc_cts_decrypt_arch 936 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 937 u8 iv[AES_BLOCK_SIZE], 938 const struct aes_key *key) 939 { 940 return false; 941 } 942 #endif 943 944 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 945 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 946 { 947 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 948 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 949 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 950 size_t pn_len = len - pn_offset; 951 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 952 u8 *pad; 953 954 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 955 return; 956 957 if (len == AES_BLOCK_SIZE) { 958 aes_cbc_encrypt(dst, src, len, iv, key); 959 return; 960 } 961 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 962 return; 963 964 /* CBC-encrypt all blocks except the last. */ 965 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 966 967 /* 968 * Compute C[n] and C[n - 1]. 969 * 970 * Careful: src may equal dst (i.e., the encryption can be in-place), so 971 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 972 */ 973 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 974 memcpy(tmp, pad, AES_BLOCK_SIZE); 975 crypto_xor(tmp, &src[pn_offset], pn_len); 976 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 977 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 978 979 memzero_explicit(tmp, sizeof(tmp)); 980 } 981 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 982 983 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 984 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 985 { 986 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 987 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 988 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 989 size_t pn_len = len - pn_offset; 990 u8 *pad; 991 992 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 993 return; 994 995 if (len == AES_BLOCK_SIZE) { 996 aes_cbc_decrypt(dst, src, len, iv, key); 997 return; 998 } 999 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 1000 return; 1001 1002 /* Compute P[0]..P[n - 2]. */ 1003 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 1004 1005 /* 1006 * Compute P[n] and P[n - 1]. 1007 * 1008 * Careful: src may equal dst (i.e., the decryption can be in-place), so 1009 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 1010 * 1011 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 1012 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 1013 */ 1014 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 1015 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 1016 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 1017 pn_len); /* P[n] */ 1018 crypto_xor(pad, &dst[pn_offset], pn_len); 1019 aes_decrypt(key, pad, pad); 1020 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 1021 } 1022 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 1023 1024 /* FIPS cryptographic algorithm self-test for AES-CBC */ 1025 static void __init aes_cbc_fips_test(void) 1026 { 1027 struct aes_key key; 1028 u8 iv[AES_BLOCK_SIZE]; 1029 u8 data[sizeof(fips_test_data)]; 1030 1031 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1032 panic("aes: CBC FIPS self-test failed (preparekey)\n"); 1033 1034 memcpy(iv, fips_test_iv, sizeof(iv)); 1035 aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key); 1036 if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0) 1037 panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n"); 1038 1039 memcpy(iv, fips_test_iv, sizeof(iv)); 1040 aes_cbc_decrypt(data, data, sizeof(data), iv, &key); 1041 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1042 panic("aes: CBC FIPS self-test failed (wrong plaintext)\n"); 1043 1044 memzero_explicit(&key, sizeof(key)); 1045 } 1046 1047 /* FIPS cryptographic algorithm self-test for AES-CBC-CTS */ 1048 static void __init aes_cbc_cts_fips_test(void) 1049 { 1050 struct aes_key key; 1051 u8 iv[AES_BLOCK_SIZE]; 1052 const size_t data_len = 2 * AES_BLOCK_SIZE; 1053 u8 ptext[2 * AES_BLOCK_SIZE]; 1054 u8 data[2 * AES_BLOCK_SIZE]; 1055 1056 /* ptext = fips_test_data || fips_test_data */ 1057 memcpy(ptext, fips_test_data, AES_BLOCK_SIZE); 1058 memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE); 1059 1060 if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1061 panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n"); 1062 1063 memcpy(iv, fips_test_iv, sizeof(iv)); 1064 aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key); 1065 if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0) 1066 panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n"); 1067 1068 memcpy(iv, fips_test_iv, sizeof(iv)); 1069 aes_cbc_cts_decrypt(data, data, data_len, iv, &key); 1070 if (memcmp(ptext, data, data_len) != 0) 1071 panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n"); 1072 1073 memzero_explicit(&key, sizeof(key)); 1074 } 1075 #else /* CONFIG_CRYPTO_LIB_AES_CBC */ 1076 static inline void aes_cbc_fips_test(void) 1077 { 1078 } 1079 static inline void aes_cbc_cts_fips_test(void) 1080 { 1081 } 1082 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC */ 1083 1084 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 1085 /* 1086 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 1087 * architecture. They are called with any len >= 0. Returning false causes the 1088 * fallback implementation to be used instead. 1089 */ 1090 #ifndef aes_ctr_arch 1091 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 1092 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 1093 { 1094 return false; 1095 } 1096 #endif 1097 #ifndef aes_xctr_arch 1098 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1099 const u8 iv[AES_BLOCK_SIZE], 1100 const struct aes_enckey *key) 1101 { 1102 return false; 1103 } 1104 #endif 1105 1106 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1107 { 1108 /* 1109 * 255 times out of 256 the first iteration is enough, so unroll the 1110 * first iteration as a micro-optimization. 1111 */ 1112 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1113 return; 1114 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1115 if (++ctr[i] != 0) 1116 break; 1117 } 1118 } 1119 1120 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1121 aes_encrypt_arg key) 1122 { 1123 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1124 1125 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1126 return; 1127 1128 /* Handle the full blocks. */ 1129 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1130 aes_encrypt(key, keystream, ctr); 1131 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1132 inc_be128_ctr(ctr); 1133 dst += AES_BLOCK_SIZE; 1134 src += AES_BLOCK_SIZE; 1135 } 1136 /* Handle any partial block at the end. */ 1137 if (len) { 1138 aes_encrypt(key, keystream, ctr); 1139 crypto_xor_cpy(dst, src, keystream, len); 1140 /* Counter is incremented even with just a partial block. */ 1141 inc_be128_ctr(ctr); 1142 } 1143 memzero_explicit(keystream, sizeof(keystream)); 1144 } 1145 EXPORT_SYMBOL_GPL(aes_ctr); 1146 1147 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1148 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1149 { 1150 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1151 __le64 aes_input[2]; 1152 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1153 1154 if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) 1155 return; 1156 1157 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1158 /* Handle the full blocks. */ 1159 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1160 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1161 aes_encrypt(key, keystream, (const u8 *)aes_input); 1162 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1163 dst += AES_BLOCK_SIZE; 1164 src += AES_BLOCK_SIZE; 1165 } 1166 /* Handle any partial block at the end. */ 1167 if (len) { 1168 /* Counter is incremented even with just a partial block. */ 1169 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1170 aes_encrypt(key, keystream, (const u8 *)aes_input); 1171 crypto_xor_cpy(dst, src, keystream, len); 1172 } 1173 memzero_explicit(keystream, sizeof(keystream)); 1174 memzero_explicit(aes_input, sizeof(aes_input)); 1175 } 1176 EXPORT_SYMBOL_GPL(aes_xctr); 1177 1178 /* FIPS cryptographic algorithm self-test for AES-CTR */ 1179 static void __init aes_ctr_fips_test(void) 1180 { 1181 struct aes_enckey key; 1182 u8 ctr[AES_BLOCK_SIZE]; 1183 u8 data[sizeof(fips_test_data)]; 1184 1185 if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0) 1186 panic("aes: CTR FIPS self-test failed (preparekey)\n"); 1187 1188 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1189 aes_ctr(data, fips_test_data, sizeof(data), ctr, &key); 1190 if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0) 1191 panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n"); 1192 1193 memcpy(ctr, fips_test_iv, sizeof(ctr)); 1194 aes_ctr(data, data, sizeof(data), ctr, &key); 1195 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1196 panic("aes: CTR FIPS self-test failed (wrong plaintext)\n"); 1197 1198 memzero_explicit(&key, sizeof(key)); 1199 } 1200 #else /* CONFIG_CRYPTO_LIB_AES_CTR */ 1201 static inline void aes_ctr_fips_test(void) 1202 { 1203 } 1204 #endif /* !CONFIG_CRYPTO_LIB_AES_CTR */ 1205 1206 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 1207 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, 1208 size_t key_len, int flags) 1209 { 1210 int err; 1211 1212 err = __xts_verify_key(in_key, key_len, flags); 1213 if (unlikely(err)) 1214 goto out_zeroize; 1215 /* First half of XTS key is the main key */ 1216 err = aes_preparekey(&key->main_key, in_key, key_len / 2); 1217 if (unlikely(err)) 1218 goto out_zeroize; 1219 /* Second half of XTS key is the tweak key */ 1220 err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], 1221 key_len / 2); 1222 if (unlikely(err)) 1223 goto out_zeroize; 1224 return 0; 1225 1226 out_zeroize: 1227 memzero_explicit(key, sizeof(*key)); 1228 return err; 1229 } 1230 EXPORT_SYMBOL_GPL(aes_xts_preparekey); 1231 1232 /* 1233 * Hooks for optimized AES-XTS implementations, overridable by the architecture. 1234 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, 1235 * they aren't expected to handle ciphertext stealing or empty inputs. 1236 * Returning false causes the fallback implementation to be used instead. 1237 * 1238 * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole 1239 * numbers of blocks anyway, with len >= 512. So there's no need to heavily 1240 * optimize ciphertext stealing for short messages.) 1241 */ 1242 #ifndef aes_xts_encrypt_arch 1243 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 1244 u8 tweak[AES_BLOCK_SIZE], 1245 const struct aes_xts_key *key, bool cont) 1246 { 1247 return false; 1248 } 1249 #endif 1250 #ifndef aes_xts_decrypt_arch 1251 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 1252 u8 tweak[AES_BLOCK_SIZE], 1253 const struct aes_xts_key *key, bool cont) 1254 { 1255 return false; 1256 } 1257 #endif 1258 1259 static noinline void aes_xts_crypt_nocts_blockbyblock( 1260 u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 1261 const struct aes_xts_key *key, bool cont, bool enc) 1262 { 1263 le128 t; 1264 1265 if (cont) 1266 memcpy(&t, tweak, sizeof(t)); 1267 else 1268 aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); 1269 do { 1270 crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); 1271 if (enc) 1272 aes_encrypt(&key->main_key, dst, dst); 1273 else 1274 aes_decrypt(&key->main_key, dst, dst); 1275 crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); 1276 gf128mul_x_ble(&t, &t); 1277 dst += AES_BLOCK_SIZE; 1278 src += AES_BLOCK_SIZE; 1279 len -= AES_BLOCK_SIZE; 1280 } while (len); 1281 memcpy(tweak, &t, sizeof(t)); 1282 memzero_explicit(&t, sizeof(t)); 1283 } 1284 1285 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1286 static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, 1287 size_t len, 1288 u8 tweak[AES_BLOCK_SIZE], 1289 const struct aes_xts_key *key, 1290 bool cont) 1291 { 1292 if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) 1293 return; 1294 1295 /* 1296 * For the fallback, just go block-by-block. It could be implemented on 1297 * top of AES-ECB, which could be significantly faster than this if the 1298 * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS 1299 * performance is important enough that it needs to be (and has been) 1300 * implemented directly by every non-obsolete arch anyway. 1301 */ 1302 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1303 /* enc= */ true); 1304 } 1305 1306 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1307 static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, 1308 size_t len, 1309 u8 tweak[AES_BLOCK_SIZE], 1310 const struct aes_xts_key *key, 1311 bool cont) 1312 { 1313 if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) 1314 return; 1315 1316 /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ 1317 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1318 /* enc= */ false); 1319 } 1320 1321 static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, 1322 u8 tweak[AES_BLOCK_SIZE], 1323 const struct aes_xts_key *key, 1324 bool cont) 1325 { 1326 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1327 size_t nocts_len = round_down(len, AES_BLOCK_SIZE); 1328 u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1329 1330 /* Encrypt all full blocks. */ 1331 aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1332 dst += nocts_len - AES_BLOCK_SIZE; 1333 src += nocts_len - AES_BLOCK_SIZE; 1334 1335 /* 1336 * Swap the partial block with the first 'partial_len' bytes of the 1337 * encrypted last full block. Note that a temporary buffer is needed to 1338 * support in-place encryption. 1339 */ 1340 memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); 1341 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1342 memcpy(dst, tmp_block, partial_len); 1343 1344 /* Encrypt the last full block again. */ 1345 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1346 aes_encrypt(&key->main_key, dst, dst); 1347 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1348 memzero_explicit(tmp_block, sizeof(tmp_block)); 1349 } 1350 1351 static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, 1352 u8 tweak[AES_BLOCK_SIZE], 1353 const struct aes_xts_key *key, 1354 bool cont) 1355 { 1356 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1357 size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; 1358 union { 1359 u8 block[AES_BLOCK_SIZE]; 1360 le128 tweak; 1361 } tmp __aligned(__alignof__(long)); 1362 1363 /* 1364 * Decrypt all blocks except the last full block and the partial block. 1365 * The last full block has to be handled specially because decryption 1366 * ciphertext stealing uses the last two tweaks in reverse order. 1367 * 1368 * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() 1369 * doesn't handle (so that the length doesn't get checked redundantly in 1370 * the fast path). So handle that case specially as well. 1371 */ 1372 if (nocts_len) 1373 aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1374 else if (!cont) 1375 aes_encrypt(&key->tweak_key, tweak, tweak); 1376 dst += nocts_len; 1377 src += nocts_len; 1378 1379 /* Copy the tweak, advance it again, then decrypt last full block. */ 1380 memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); 1381 gf128mul_x_ble(&tmp.tweak, &tmp.tweak); 1382 crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); 1383 aes_decrypt(&key->main_key, dst, dst); 1384 crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); 1385 1386 /* 1387 * Swap the partial block with the first 'partial_len' bytes of the 1388 * decrypted last full block. Note that a temporary buffer is needed to 1389 * support in-place decryption. 1390 */ 1391 memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); 1392 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1393 memcpy(dst, tmp.block, partial_len); 1394 1395 /* Decrypt the last full block again. */ 1396 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1397 aes_decrypt(&key->main_key, dst, dst); 1398 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1399 memzero_explicit(&tmp, sizeof(tmp)); 1400 } 1401 1402 void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, 1403 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1404 bool cont) 1405 { 1406 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1407 return; 1408 1409 if (unlikely(len % AES_BLOCK_SIZE)) { 1410 aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); 1411 return; 1412 } 1413 1414 aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); 1415 } 1416 EXPORT_SYMBOL_GPL(aes_xts_encrypt); 1417 1418 void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, 1419 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1420 bool cont) 1421 { 1422 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1423 return; 1424 1425 if (unlikely(len % AES_BLOCK_SIZE)) { 1426 aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); 1427 return; 1428 } 1429 1430 aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); 1431 } 1432 EXPORT_SYMBOL_GPL(aes_xts_decrypt); 1433 1434 /* FIPS cryptographic algorithm self-test for AES-XTS */ 1435 static void __init aes_xts_fips_test(void) 1436 { 1437 struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key); 1438 u8 tweak[AES_BLOCK_SIZE]; 1439 u8 data[sizeof(fips_test_data)]; 1440 1441 if (key == NULL) 1442 panic("aes: XTS FIPS self-test failed (kmalloc)\n"); 1443 1444 if (aes_xts_preparekey(key, fips_test_xts_key, 1445 sizeof(fips_test_xts_key), 0) != 0) 1446 panic("aes: XTS FIPS self-test failed (preparekey)\n"); 1447 1448 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1449 aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false); 1450 if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0) 1451 panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n"); 1452 1453 memcpy(tweak, fips_test_iv, sizeof(tweak)); 1454 aes_xts_decrypt(data, data, sizeof(data), tweak, key, false); 1455 if (memcmp(fips_test_data, data, sizeof(data)) != 0) 1456 panic("aes: XTS FIPS self-test failed (wrong plaintext)\n"); 1457 } 1458 #else /* CONFIG_CRYPTO_LIB_AES_XTS */ 1459 static inline void aes_xts_fips_test(void) 1460 { 1461 } 1462 #endif /* !CONFIG_CRYPTO_LIB_AES_XTS */ 1463 1464 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) 1465 /* 1466 * Hooks for optimized AES-GCM implementations, overridable by the architecture. 1467 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't 1468 * expected to handle empty inputs or partial blocks, as those cases are handled 1469 * by non-arch-specific code instead. 1470 * 1471 * The GHASH accumulator is provided in POLYVAL format. The counter is provided 1472 * in big endian format, and it's read-only, as the caller handles updating it. 1473 * 1474 * Returning false causes the fallback implementation to be used instead. 1475 * 1476 * These hooks are used only for en/decrypted data. For the associated data the 1477 * GHASH functions are called instead, so those should be implemented too. 1478 */ 1479 #ifndef aes_gcm_encrypt_update_arch 1480 static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1481 struct polyval_elem *ghash_acc, 1482 const __be32 ctr32[4], 1483 const struct aes_enckey *aes_key, 1484 const struct ghash_key *ghash_key) 1485 { 1486 return false; 1487 } 1488 #endif 1489 #ifndef aes_gcm_decrypt_update_arch 1490 static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1491 struct polyval_elem *ghash_acc, 1492 const __be32 ctr32[4], 1493 const struct aes_enckey *aes_key, 1494 const struct ghash_key *ghash_key) 1495 { 1496 return false; 1497 } 1498 #endif 1499 1500 int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, 1501 size_t key_len, size_t authtag_len) 1502 { 1503 u8 h[AES_BLOCK_SIZE] = { 0 }; 1504 int err; 1505 1506 err = crypto_gcm_check_authsize(authtag_len); 1507 if (unlikely(err)) 1508 return err; 1509 1510 err = aes_prepareenckey(&key->aes, in_key, key_len); 1511 if (unlikely(err)) 1512 return err; 1513 1514 aes_encrypt(&key->aes, h, h); 1515 ghash_preparekey(&key->ghash, h); 1516 1517 key->authtag_len = authtag_len; 1518 1519 memzero_explicit(h, sizeof(h)); 1520 return 0; 1521 } 1522 EXPORT_SYMBOL_GPL(aes_gcm_preparekey); 1523 1524 void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], 1525 const struct aes_gcm_key *key) 1526 { 1527 ctx->key = key; 1528 ctx->ad_len = 0; 1529 ctx->data_len = 0; 1530 ghash_init(&ctx->ghash, &key->ghash); 1531 memset(ctx->keystream, 0, sizeof(ctx->keystream)); 1532 1533 memcpy(ctx->ctr32, nonce, 12); 1534 ctx->ctr32[3] = cpu_to_be32(1); 1535 1536 aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); 1537 ctx->ctr32[3] = cpu_to_be32(2); 1538 } 1539 EXPORT_SYMBOL_GPL(aes_gcm_init); 1540 1541 void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) 1542 { 1543 WARN_ON_ONCE(ctx->data_len != 0); 1544 if (len) { 1545 ghash_update(&ctx->ghash, ad, len); 1546 ctx->ad_len += len; 1547 } 1548 } 1549 EXPORT_SYMBOL_GPL(aes_gcm_auth_update); 1550 1551 static const u8 gcm_zeroes[AES_BLOCK_SIZE]; 1552 1553 static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) 1554 { 1555 if (len % AES_BLOCK_SIZE) 1556 ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); 1557 } 1558 1559 static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, 1560 u8 *dst, const u8 *src, 1561 size_t len, bool enc) 1562 { 1563 size_t partial_len, n; 1564 1565 if (unlikely(len == 0)) 1566 return; 1567 1568 partial_len = ctx->data_len % AES_BLOCK_SIZE; 1569 if (ctx->data_len == 0) 1570 ghash_pad(&ctx->ghash, ctx->ad_len); 1571 ctx->data_len += len; 1572 1573 if (unlikely(partial_len != 0)) { 1574 /* 1575 * The previous call ended on a non-block-aligned data_len, so 1576 * continue using a previously-generated keystream block. 1577 */ 1578 n = min(len, AES_BLOCK_SIZE - partial_len); 1579 if (enc) { 1580 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1581 n); 1582 ghash_update(&ctx->ghash, dst, n); 1583 } else { 1584 ghash_update(&ctx->ghash, src, n); 1585 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1586 n); 1587 } 1588 dst += n; 1589 src += n; 1590 len -= n; 1591 } 1592 1593 if (len >= AES_BLOCK_SIZE) { 1594 n = round_down(len, AES_BLOCK_SIZE); 1595 if (enc) { 1596 if (likely(aes_gcm_encrypt_update_arch( 1597 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1598 &ctx->key->aes, &ctx->key->ghash))) { 1599 be32_add_cpu(&ctx->ctr32[3], 1600 n / AES_BLOCK_SIZE); 1601 } else { 1602 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1603 ghash_update(&ctx->ghash, dst, n); 1604 } 1605 } else { 1606 if (likely(aes_gcm_decrypt_update_arch( 1607 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1608 &ctx->key->aes, &ctx->key->ghash))) { 1609 be32_add_cpu(&ctx->ctr32[3], 1610 n / AES_BLOCK_SIZE); 1611 } else { 1612 ghash_update(&ctx->ghash, src, n); 1613 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1614 } 1615 } 1616 dst += n; 1617 src += n; 1618 len -= n; 1619 } 1620 1621 if (len != 0) { 1622 /* 1623 * Ending on a non-block aligned data_len. Generate the next 1624 * keystream block, use the needed portion of it, and leave it 1625 * cached in ctx->keystream in case this isn't the final call. 1626 */ 1627 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1628 be32_add_cpu(&ctx->ctr32[3], 1); 1629 if (enc) { 1630 crypto_xor_cpy(dst, src, ctx->keystream, len); 1631 ghash_update(&ctx->ghash, dst, len); 1632 } else { 1633 ghash_update(&ctx->ghash, src, len); 1634 crypto_xor_cpy(dst, src, ctx->keystream, len); 1635 } 1636 } 1637 } 1638 1639 void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1640 size_t len) 1641 { 1642 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1643 } 1644 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); 1645 1646 void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1647 size_t len) 1648 { 1649 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); 1650 } 1651 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); 1652 1653 /* Maximum AES-GCM associated data length in bytes */ 1654 #define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) 1655 /* Maximum AES-GCM en/decrypted data length in bytes */ 1656 #define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) 1657 1658 void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) 1659 { 1660 __be64 tail[2]; 1661 1662 WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); 1663 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); 1664 1665 ghash_pad(&ctx->ghash, 1666 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1667 1668 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1669 tail[1] = cpu_to_be64(ctx->data_len * 8); 1670 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1671 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1672 1673 crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1674 memzero_explicit(ctx, sizeof(*ctx)); 1675 } 1676 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); 1677 1678 int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) 1679 { 1680 __be64 tail[2]; 1681 int err; 1682 1683 if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || 1684 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { 1685 err = -EBADMSG; 1686 goto out; 1687 } 1688 1689 ghash_pad(&ctx->ghash, 1690 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1691 1692 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1693 tail[1] = cpu_to_be64(ctx->data_len * 8); 1694 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1695 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1696 crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1697 err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? 1698 -EBADMSG : 1699 0; 1700 out: 1701 memzero_explicit(ctx, sizeof(*ctx)); 1702 return err; 1703 } 1704 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); 1705 1706 void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 1707 const u8 *ad, size_t ad_len, const u8 nonce[12], 1708 const struct aes_gcm_key *key) 1709 { 1710 struct aes_gcm_ctx ctx; 1711 1712 aes_gcm_init(&ctx, nonce, key); 1713 aes_gcm_auth_update(&ctx, ad, ad_len); 1714 aes_gcm_encrypt_update(&ctx, dst, src, data_len); 1715 aes_gcm_encrypt_final(&ctx, authtag); 1716 } 1717 EXPORT_SYMBOL_GPL(aes_gcm_encrypt); 1718 1719 int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 1720 const u8 *ad, size_t ad_len, const u8 nonce[12], 1721 const struct aes_gcm_key *key) 1722 { 1723 struct aes_gcm_ctx ctx; 1724 int err; 1725 1726 aes_gcm_init(&ctx, nonce, key); 1727 aes_gcm_auth_update(&ctx, ad, ad_len); 1728 aes_gcm_decrypt_update(&ctx, dst, src, data_len); 1729 err = aes_gcm_decrypt_final(&ctx, authtag); 1730 if (unlikely(err) && data_len) { 1731 /* 1732 * Clear the inauthentic decrypted data so that callers won't 1733 * receive it even if they fail to correctly handle errors. 1734 */ 1735 memset(dst, 0, data_len); 1736 } 1737 return err; 1738 } 1739 EXPORT_SYMBOL_GPL(aes_gcm_decrypt); 1740 1741 /* FIPS cryptographic algorithm self-test for AES-GCM */ 1742 static void __init aes_gcm_fips_test(void) 1743 { 1744 const size_t data_len = sizeof(fips_test_data); 1745 u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; 1746 struct aes_gcm_key key; 1747 int err; 1748 1749 if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key), 1750 AES_BLOCK_SIZE) != 0) 1751 panic("aes: GCM FIPS self-test failed (preparekey)\n"); 1752 1753 aes_gcm_encrypt(buf, fips_test_data, data_len, &buf[data_len], 1754 fips_test_ad, sizeof(fips_test_ad), fips_test_iv, &key); 1755 if (memcmp(fips_test_aes_gcm_ctext_and_tag, buf, sizeof(buf)) != 0) 1756 panic("aes: GCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); 1757 1758 err = aes_gcm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, 1759 sizeof(fips_test_ad), fips_test_iv, &key); 1760 if (err != 0) 1761 panic("aes: GCM FIPS self-test failed (decryption failed)\n"); 1762 if (memcmp(fips_test_data, buf, data_len) != 0) 1763 panic("aes: GCM FIPS self-test failed (wrong plaintext)\n"); 1764 1765 memzero_explicit(&key, sizeof(key)); 1766 } 1767 #else /* CONFIG_CRYPTO_LIB_AES_GCM */ 1768 static inline void aes_gcm_fips_test(void) 1769 { 1770 } 1771 #endif /* !CONFIG_CRYPTO_LIB_AES_GCM */ 1772 1773 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) 1774 int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, 1775 size_t key_len, size_t authtag_len) 1776 { 1777 int err; 1778 1779 if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) 1780 return -EINVAL; 1781 1782 err = aes_prepareenckey(&key->aes, in_key, key_len); 1783 if (unlikely(err)) 1784 return err; 1785 1786 key->authtag_len = authtag_len; 1787 return 0; 1788 } 1789 EXPORT_SYMBOL_GPL(aes_ccm_preparekey); 1790 1791 int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, 1792 const u8 *nonce, size_t nonce_len, 1793 const struct aes_ccm_key *key) 1794 { 1795 /* 1796 * This is the value L defined in the CCM specification. It determines 1797 * the maximum allowed message length, and it is itself determined by 1798 * the nonce length. They are inversely related, i.e. the longer the 1799 * nonce the smaller the maximum message length is. 1800 */ 1801 unsigned int l = 15 - nonce_len; 1802 1803 if (unlikely(nonce_len < 7 || nonce_len > 13)) 1804 return -EINVAL; 1805 /* Thus 2 <= l <= 8. */ 1806 1807 /* Check whether data_len can be represented in 'l' bytes. */ 1808 if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) 1809 return -EOVERFLOW; 1810 1811 ctx->key = key; 1812 ctx->ad_remaining = ad_len; 1813 ctx->data_remaining = data_len; 1814 ctx->ad_padded = false; 1815 1816 /* 1817 * Initialize the zero-th counter block to: 1818 * 1819 * L - 1 || nonce || 0 1820 * 1821 * ... and the zero-th CBC-MAC block to: 1822 * 1823 * Flags || nonce || data_len 1824 */ 1825 *(__be64 *)&ctx->ctr[8] = 0; 1826 *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); 1827 ctx->ctr[0] = l - 1; 1828 ctx->mac[0] = (ad_len ? 0x40 : 0) | 1829 (((key->authtag_len - 2) / 2) << 3) | (l - 1); 1830 memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ 1831 memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ 1832 1833 /* 1834 * Generate S_0 by encrypting the counter (this is used to encrypt the 1835 * auth tag later), and encrypt the zero-th CBC-MAC block. 1836 */ 1837 aes_encrypt(&key->aes, ctx->s0, ctx->ctr); 1838 aes_encrypt(&key->aes, ctx->mac, ctx->mac); 1839 1840 /* Increment the counter from 0 to 1. */ 1841 ctx->ctr[15] = 1; 1842 1843 if (ad_len) { 1844 /* 1845 * Update CBC-MAC with the associated data length, represented 1846 * using either 2, 6, or 10 bytes depending on the length. 1847 */ 1848 if (likely(ad_len < 0xff00)) { 1849 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); 1850 ctx->partial_len = 2; 1851 } else if (ad_len <= U32_MAX) { 1852 __be32 *p = (__be32 *)&ctx->mac[2]; 1853 1854 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); 1855 put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), 1856 p); 1857 ctx->partial_len = 6; 1858 } else { 1859 __be64 *p = (__be64 *)&ctx->mac[2]; 1860 1861 *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); 1862 put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), 1863 p); 1864 ctx->partial_len = 10; 1865 } 1866 } else { 1867 ctx->partial_len = 0; 1868 } 1869 return 0; 1870 } 1871 EXPORT_SYMBOL_GPL(aes_ccm_init); 1872 1873 void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) 1874 { 1875 size_t partial_len = ctx->partial_len; 1876 bool enc_before = false; 1877 size_t nblocks; 1878 1879 WARN_ON_ONCE(ctx->ad_padded); 1880 1881 /* 1882 * We could warn on len > ad_remaining here, but underflow will be 1883 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1884 * going to underflow all the way back to 0.) 1885 */ 1886 ctx->ad_remaining -= len; 1887 1888 if (partial_len) { 1889 size_t n = min(len, AES_BLOCK_SIZE - partial_len); 1890 1891 crypto_xor(&ctx->mac[partial_len], ad, n); 1892 ad += n; 1893 len -= n; 1894 partial_len += n; 1895 if (partial_len < AES_BLOCK_SIZE) { 1896 ctx->partial_len = partial_len; 1897 return; 1898 } 1899 enc_before = true; 1900 } 1901 1902 nblocks = len / AES_BLOCK_SIZE; 1903 len %= AES_BLOCK_SIZE; 1904 if (nblocks == 0) { 1905 if (enc_before) 1906 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1907 } else { 1908 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, 1909 enc_before, /* enc_after= */ true); 1910 ad += nblocks * AES_BLOCK_SIZE; 1911 } 1912 crypto_xor(ctx->mac, ad, len); 1913 ctx->partial_len = len; 1914 } 1915 EXPORT_SYMBOL_GPL(aes_ccm_auth_update); 1916 1917 static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, 1918 u8 *dst, const u8 *src, 1919 size_t len, bool enc) 1920 { 1921 size_t partial_len = ctx->partial_len; 1922 size_t n, nblocks; 1923 1924 if (unlikely(len == 0)) 1925 return; 1926 1927 WARN_ON_ONCE(ctx->ad_remaining != 0); 1928 1929 /* 1930 * We could warn on len > data_remaining here, but underflow will be 1931 * caught by the != 0 check at the end anyway. (It's a u64, so it isn't 1932 * going to underflow all the way back to 0.) 1933 */ 1934 ctx->data_remaining -= len; 1935 1936 if (!ctx->ad_padded) { 1937 ctx->ad_padded = true; 1938 if (partial_len) 1939 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1940 } else if (partial_len) { 1941 /* 1942 * The previous call ended on a non-block-aligned data_len, so 1943 * continue using a previously-generated keystream block. 1944 */ 1945 n = min(len, AES_BLOCK_SIZE - partial_len); 1946 if (enc) 1947 crypto_xor(&ctx->mac[partial_len], src, n); 1948 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); 1949 if (!enc) 1950 crypto_xor(&ctx->mac[partial_len], dst, n); 1951 dst += n; 1952 src += n; 1953 len -= n; 1954 partial_len += n; 1955 if (partial_len < AES_BLOCK_SIZE) { 1956 ctx->partial_len = partial_len; 1957 return; 1958 } 1959 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 1960 } 1961 1962 if (len >= AES_BLOCK_SIZE) { 1963 n = round_down(len, AES_BLOCK_SIZE); 1964 nblocks = len / AES_BLOCK_SIZE; 1965 if (enc) 1966 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, 1967 nblocks, /* enc_before= */ false, 1968 /* enc_after= */ true); 1969 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1970 if (!enc) 1971 aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, 1972 nblocks, /* enc_before= */ false, 1973 /* enc_after= */ true); 1974 dst += n; 1975 src += n; 1976 len -= n; 1977 } 1978 1979 if (len) { 1980 /* 1981 * Ending on a non-block aligned data_len. Generate the next 1982 * keystream block, use the needed portion of it, and leave it 1983 * cached in ctx->keystream in case this isn't the final call. 1984 */ 1985 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1986 inc_be128_ctr(ctx->ctr); 1987 if (enc) 1988 crypto_xor(ctx->mac, src, len); 1989 crypto_xor_cpy(dst, src, ctx->keystream, len); 1990 if (!enc) 1991 crypto_xor(ctx->mac, dst, len); 1992 } 1993 ctx->partial_len = len; 1994 } 1995 1996 void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 1997 size_t len) 1998 { 1999 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); 2000 } 2001 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); 2002 2003 void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, 2004 size_t len) 2005 { 2006 aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); 2007 } 2008 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); 2009 2010 void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) 2011 { 2012 WARN_ON_ONCE(ctx->ad_remaining != 0); 2013 WARN_ON_ONCE(ctx->data_remaining != 0); 2014 if (ctx->partial_len) 2015 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 2016 crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); 2017 memzero_explicit(ctx, sizeof(*ctx)); 2018 } 2019 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); 2020 2021 int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) 2022 { 2023 int err; 2024 2025 if (WARN_ON_ONCE(ctx->ad_remaining != 0) || 2026 WARN_ON_ONCE(ctx->data_remaining != 0)) { 2027 err = -EBADMSG; 2028 goto out; 2029 } 2030 2031 if (ctx->partial_len) 2032 aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); 2033 crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); 2034 err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? 2035 -EBADMSG : 2036 0; 2037 out: 2038 memzero_explicit(ctx, sizeof(*ctx)); 2039 return err; 2040 } 2041 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); 2042 2043 int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 2044 const u8 *ad, size_t ad_len, const u8 *nonce, 2045 size_t nonce_len, const struct aes_ccm_key *key) 2046 { 2047 struct aes_ccm_ctx ctx; 2048 int err; 2049 2050 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2051 if (unlikely(err)) 2052 return err; 2053 aes_ccm_auth_update(&ctx, ad, ad_len); 2054 aes_ccm_encrypt_update(&ctx, dst, src, data_len); 2055 aes_ccm_encrypt_final(&ctx, authtag); 2056 return 0; 2057 } 2058 EXPORT_SYMBOL_GPL(aes_ccm_encrypt); 2059 2060 int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 2061 const u8 *ad, size_t ad_len, const u8 *nonce, 2062 size_t nonce_len, const struct aes_ccm_key *key) 2063 { 2064 struct aes_ccm_ctx ctx; 2065 int err; 2066 2067 err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); 2068 if (unlikely(err)) 2069 return err; 2070 aes_ccm_auth_update(&ctx, ad, ad_len); 2071 aes_ccm_decrypt_update(&ctx, dst, src, data_len); 2072 err = aes_ccm_decrypt_final(&ctx, authtag); 2073 if (unlikely(err) && data_len) { 2074 /* 2075 * Clear the inauthentic decrypted data so that callers won't 2076 * receive it even if they fail to correctly handle errors. 2077 */ 2078 memset(dst, 0, data_len); 2079 } 2080 return err; 2081 } 2082 EXPORT_SYMBOL_GPL(aes_ccm_decrypt); 2083 2084 /* FIPS cryptographic algorithm self-test for AES-CCM */ 2085 static void __init aes_ccm_fips_test(void) 2086 { 2087 const size_t data_len = sizeof(fips_test_data); 2088 const size_t nonce_len = 13; 2089 u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; 2090 struct aes_ccm_key key; 2091 int err; 2092 2093 if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key), 2094 AES_BLOCK_SIZE) != 0) 2095 panic("aes: CCM FIPS self-test failed (preparekey)\n"); 2096 2097 err = aes_ccm_encrypt(buf, fips_test_data, data_len, &buf[data_len], 2098 fips_test_ad, sizeof(fips_test_ad), fips_test_iv, 2099 nonce_len, &key); 2100 if (err != 0) 2101 panic("aes: CCM FIPS self-test failed (encryption failed)\n"); 2102 if (memcmp(fips_test_aes_ccm_ctext_and_tag, buf, sizeof(buf)) != 0) 2103 panic("aes: CCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); 2104 2105 err = aes_ccm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, 2106 sizeof(fips_test_ad), fips_test_iv, nonce_len, 2107 &key); 2108 if (err != 0) 2109 panic("aes: CCM FIPS self-test failed (decryption failed)\n"); 2110 if (memcmp(fips_test_data, buf, data_len) != 0) 2111 panic("aes: CCM FIPS self-test failed (wrong plaintext)\n"); 2112 2113 memzero_explicit(&key, sizeof(key)); 2114 } 2115 #else /* CONFIG_CRYPTO_LIB_AES_CCM */ 2116 static inline void aes_ccm_fips_test(void) 2117 { 2118 } 2119 #endif /* !CONFIG_CRYPTO_LIB_AES_CCM */ 2120 2121 static int __init aes_mod_init(void) 2122 { 2123 #ifdef aes_mod_init_arch 2124 aes_mod_init_arch(); 2125 #endif 2126 if (fips_enabled) { 2127 aes_fips_test(); 2128 aes_cmac_fips_test(); 2129 aes_ecb_fips_test(); 2130 aes_cbc_fips_test(); 2131 aes_cbc_cts_fips_test(); 2132 aes_ctr_fips_test(); 2133 aes_xts_fips_test(); 2134 aes_gcm_fips_test(); 2135 aes_ccm_fips_test(); 2136 } 2137 return 0; 2138 } 2139 subsys_initcall(aes_mod_init); 2140 2141 static void __exit aes_mod_exit(void) 2142 { 2143 } 2144 module_exit(aes_mod_exit); 2145 2146 MODULE_DESCRIPTION("AES block cipher"); 2147 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 2148 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 2149 MODULE_LICENSE("GPL v2"); 2150