1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright 2026 Google LLC 5 */ 6 7 #include <crypto/aes-cbc-macs.h> 8 #include <crypto/aes-cbc.h> 9 #include <crypto/aes-ctr.h> 10 #include <crypto/aes-ecb.h> 11 #include <crypto/aes-gcm.h> 12 #include <crypto/aes-xts.h> 13 #include <crypto/aes.h> 14 #include <crypto/gf128mul.h> 15 #include <crypto/utils.h> 16 #include <linux/cache.h> 17 #include <linux/crypto.h> 18 #include <linux/export.h> 19 #include <linux/module.h> 20 #include <linux/unaligned.h> 21 #include "fips.h" 22 23 static const u8 ____cacheline_aligned aes_sbox[] = { 24 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 25 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 26 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 27 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 28 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 29 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 30 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 31 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 32 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 33 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 34 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 35 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 36 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 37 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 38 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 39 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 40 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 41 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 42 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 43 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 44 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 45 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 46 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 47 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 48 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 49 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 50 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 51 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 52 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 53 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 54 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 55 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, 56 }; 57 58 static const u8 ____cacheline_aligned aes_inv_sbox[] = { 59 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 60 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 61 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 62 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 63 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 64 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 65 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 66 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 67 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 68 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 69 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 70 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 71 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 72 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 73 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 74 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 75 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 76 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 77 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 78 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 79 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 80 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 81 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 82 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 83 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 84 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 85 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 86 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 87 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 88 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 89 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 90 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, 91 }; 92 93 extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); 94 extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); 95 96 EXPORT_SYMBOL(crypto_aes_sbox); 97 EXPORT_SYMBOL(crypto_aes_inv_sbox); 98 99 /* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */ 100 const u32 ____cacheline_aligned aes_enc_tab[256] = { 101 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 102 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, 103 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, 104 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, 105 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, 106 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, 107 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, 108 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, 109 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, 110 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, 111 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, 112 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, 113 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, 114 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, 115 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, 116 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, 117 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, 118 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, 119 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, 120 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, 121 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, 122 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, 123 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, 124 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, 125 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, 126 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, 127 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, 128 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, 129 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, 130 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, 131 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, 132 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, 133 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, 134 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, 135 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, 136 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, 137 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, 138 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, 139 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, 140 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, 141 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, 142 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, 143 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, 144 }; 145 EXPORT_SYMBOL(aes_enc_tab); 146 147 /* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */ 148 const u32 ____cacheline_aligned aes_dec_tab[256] = { 149 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 150 0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5, 151 0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25, 152 0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b, 153 0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458, 154 0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927, 155 0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5, 156 0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9, 157 0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72, 158 0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3, 159 0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7, 160 0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4, 161 0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040, 162 0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d, 163 0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6, 164 0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879, 165 0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32, 166 0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36, 167 0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793, 168 0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c, 169 0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2, 170 0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3, 171 0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb, 172 0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684, 173 0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc, 174 0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947, 175 0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9, 176 0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f, 177 0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890, 178 0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf, 179 0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e, 180 0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef, 181 0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a, 182 0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733, 183 0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43, 184 0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546, 185 0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92, 186 0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb, 187 0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255, 188 0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478, 189 0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc, 190 0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664, 191 0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0, 192 }; 193 EXPORT_SYMBOL(aes_dec_tab); 194 195 /* Prefetch data into L1 cache. @mem should be cacheline-aligned. */ 196 static __always_inline void aes_prefetch(const void *mem, size_t len) 197 { 198 for (size_t i = 0; i < len; i += L1_CACHE_BYTES) 199 *(volatile const u8 *)(mem + i); 200 barrier(); 201 } 202 203 static u32 mul_by_x(u32 w) 204 { 205 u32 x = w & 0x7f7f7f7f; 206 u32 y = w & 0x80808080; 207 208 /* multiply by polynomial 'x' (0b10) in GF(2^8) */ 209 return (x << 1) ^ (y >> 7) * 0x1b; 210 } 211 212 static u32 mul_by_x2(u32 w) 213 { 214 u32 x = w & 0x3f3f3f3f; 215 u32 y = w & 0x80808080; 216 u32 z = w & 0x40404040; 217 218 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ 219 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; 220 } 221 222 static u32 mix_columns(u32 x) 223 { 224 /* 225 * Perform the following matrix multiplication in GF(2^8) 226 * 227 * | 0x2 0x3 0x1 0x1 | | x[0] | 228 * | 0x1 0x2 0x3 0x1 | | x[1] | 229 * | 0x1 0x1 0x2 0x3 | x | x[2] | 230 * | 0x3 0x1 0x1 0x2 | | x[3] | 231 */ 232 u32 y = mul_by_x(x) ^ ror32(x, 16); 233 234 return y ^ ror32(x ^ y, 8); 235 } 236 237 static u32 inv_mix_columns(u32 x) 238 { 239 /* 240 * Perform the following matrix multiplication in GF(2^8) 241 * 242 * | 0xe 0xb 0xd 0x9 | | x[0] | 243 * | 0x9 0xe 0xb 0xd | | x[1] | 244 * | 0xd 0x9 0xe 0xb | x | x[2] | 245 * | 0xb 0xd 0x9 0xe | | x[3] | 246 * 247 * which can conveniently be reduced to 248 * 249 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | 250 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | 251 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | 252 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | 253 */ 254 u32 y = mul_by_x2(x); 255 256 return mix_columns(x ^ y ^ ror32(y, 16)); 257 } 258 259 static u32 subw(u32 in) 260 { 261 return (aes_sbox[in & 0xff]) ^ 262 (aes_sbox[(in >> 8) & 0xff] << 8) ^ 263 (aes_sbox[(in >> 16) & 0xff] << 16) ^ 264 (aes_sbox[(in >> 24) & 0xff] << 24); 265 } 266 267 static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys, 268 const u8 *in_key, int key_len) 269 { 270 u32 kwords = key_len / sizeof(u32); 271 u32 rc, i, j; 272 273 for (i = 0; i < kwords; i++) 274 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 275 276 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { 277 u32 *rki = &rndkeys[i * kwords]; 278 u32 *rko = rki + kwords; 279 280 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; 281 rko[1] = rko[0] ^ rki[1]; 282 rko[2] = rko[1] ^ rki[2]; 283 rko[3] = rko[2] ^ rki[3]; 284 285 if (key_len == AES_KEYSIZE_192) { 286 if (i >= 7) 287 break; 288 rko[4] = rko[3] ^ rki[4]; 289 rko[5] = rko[4] ^ rki[5]; 290 } else if (key_len == AES_KEYSIZE_256) { 291 if (i >= 6) 292 break; 293 rko[4] = subw(rko[3]) ^ rki[4]; 294 rko[5] = rko[4] ^ rki[5]; 295 rko[6] = rko[5] ^ rki[6]; 296 rko[7] = rko[6] ^ rki[7]; 297 } 298 } 299 300 /* 301 * Generate the decryption keys for the Equivalent Inverse Cipher. 302 * This involves reversing the order of the round keys, and applying 303 * the Inverse Mix Columns transformation to all but the first and 304 * the last one. 305 */ 306 if (inv_rndkeys) { 307 inv_rndkeys[0] = rndkeys[key_len + 24]; 308 inv_rndkeys[1] = rndkeys[key_len + 25]; 309 inv_rndkeys[2] = rndkeys[key_len + 26]; 310 inv_rndkeys[3] = rndkeys[key_len + 27]; 311 312 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { 313 inv_rndkeys[i] = inv_mix_columns(rndkeys[j]); 314 inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]); 315 inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]); 316 inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]); 317 } 318 319 inv_rndkeys[i] = rndkeys[0]; 320 inv_rndkeys[i + 1] = rndkeys[1]; 321 inv_rndkeys[i + 2] = rndkeys[2]; 322 inv_rndkeys[i + 3] = rndkeys[3]; 323 } 324 } 325 326 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 327 unsigned int key_len) 328 { 329 if (aes_check_keylen(key_len) != 0) 330 return -EINVAL; 331 ctx->key_length = key_len; 332 aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len); 333 return 0; 334 } 335 EXPORT_SYMBOL(aes_expandkey); 336 337 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk) 338 { 339 return rk ^ aes_enc_tab[(u8)w[i]] ^ 340 rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^ 341 rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 342 rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24); 343 } 344 345 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk) 346 { 347 return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^ 348 (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^ 349 ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^ 350 ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16); 351 } 352 353 static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds, 354 u8 out[AES_BLOCK_SIZE], 355 const u8 in[AES_BLOCK_SIZE]) 356 { 357 const u32 *rkp = rndkeys; 358 int n = nrounds - 1; 359 u32 w[4]; 360 361 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 362 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 363 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 364 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 365 366 /* 367 * Prefetch the table before doing data and key-dependent loads from it. 368 * 369 * This is intended only as a basic constant-time hardening measure that 370 * avoids interfering with performance too much. Its effectiveness is 371 * not guaranteed. For proper constant-time AES, a CPU that supports 372 * AES instructions should be used instead. 373 */ 374 aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab)); 375 376 do { 377 u32 w0 = enc_quarterround(w, 0, *rkp++); 378 u32 w1 = enc_quarterround(w, 1, *rkp++); 379 u32 w2 = enc_quarterround(w, 2, *rkp++); 380 u32 w3 = enc_quarterround(w, 3, *rkp++); 381 382 w[0] = w0; 383 w[1] = w1; 384 w[2] = w2; 385 w[3] = w3; 386 } while (--n); 387 388 put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]); 389 put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]); 390 put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]); 391 put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]); 392 } 393 394 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk) 395 { 396 return rk ^ aes_dec_tab[(u8)w[i]] ^ 397 rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^ 398 rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^ 399 rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24); 400 } 401 402 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk) 403 { 404 return rk ^ aes_inv_sbox[(u8)w[i]] ^ 405 ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^ 406 ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^ 407 ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24); 408 } 409 410 static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[], 411 int nrounds, 412 u8 out[AES_BLOCK_SIZE], 413 const u8 in[AES_BLOCK_SIZE]) 414 { 415 const u32 *rkp = inv_rndkeys; 416 int n = nrounds - 1; 417 u32 w[4]; 418 419 w[0] = get_unaligned_le32(&in[0]) ^ *rkp++; 420 w[1] = get_unaligned_le32(&in[4]) ^ *rkp++; 421 w[2] = get_unaligned_le32(&in[8]) ^ *rkp++; 422 w[3] = get_unaligned_le32(&in[12]) ^ *rkp++; 423 424 aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab)); 425 426 do { 427 u32 w0 = dec_quarterround(w, 0, *rkp++); 428 u32 w1 = dec_quarterround(w, 1, *rkp++); 429 u32 w2 = dec_quarterround(w, 2, *rkp++); 430 u32 w3 = dec_quarterround(w, 3, *rkp++); 431 432 w[0] = w0; 433 w[1] = w1; 434 w[2] = w2; 435 w[3] = w3; 436 } while (--n); 437 438 aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox)); 439 put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]); 440 put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]); 441 put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]); 442 put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]); 443 } 444 445 /* 446 * Note: the aes_prepare*key_* names reflect the fact that the implementation 447 * might not actually expand the key. (The s390 code for example doesn't.) 448 * Where the key is expanded we use the more specific names aes_expandkey_*. 449 * 450 * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to 451 * the area to store the prepared decryption key. It will be NULL if the user 452 * is requesting encryption-only. aes_preparekey_arch() is also passed a valid 453 * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256. 454 */ 455 #ifdef CONFIG_CRYPTO_LIB_AES_ARCH 456 /* An arch-specific implementation of AES is available. Include it. */ 457 #include "aes.h" /* $(SRCARCH)/aes.h */ 458 #else 459 /* No arch-specific implementation of AES is available. Use generic code. */ 460 461 static void aes_preparekey_arch(union aes_enckey_arch *k, 462 union aes_invkey_arch *inv_k, 463 const u8 *in_key, int key_len, int nrounds) 464 { 465 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 466 in_key, key_len); 467 } 468 469 static void aes_encrypt_arch(const struct aes_enckey *key, 470 u8 out[AES_BLOCK_SIZE], 471 const u8 in[AES_BLOCK_SIZE]) 472 { 473 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 474 } 475 476 static void aes_decrypt_arch(const struct aes_key *key, 477 u8 out[AES_BLOCK_SIZE], 478 const u8 in[AES_BLOCK_SIZE]) 479 { 480 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in); 481 } 482 #endif 483 484 static int __aes_preparekey(struct aes_enckey *enc_key, 485 union aes_invkey_arch *inv_k, 486 const u8 *in_key, size_t key_len) 487 { 488 if (aes_check_keylen(key_len) != 0) 489 return -EINVAL; 490 enc_key->len = key_len; 491 enc_key->nrounds = 6 + key_len / 4; 492 aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len, 493 enc_key->nrounds); 494 return 0; 495 } 496 497 int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len) 498 { 499 return __aes_preparekey((struct aes_enckey *)key, &key->inv_k, 500 in_key, key_len); 501 } 502 EXPORT_SYMBOL(aes_preparekey); 503 504 int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len) 505 { 506 return __aes_preparekey(key, NULL, in_key, key_len); 507 } 508 EXPORT_SYMBOL(aes_prepareenckey); 509 510 void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE], 511 const u8 in[AES_BLOCK_SIZE]) 512 { 513 aes_encrypt_arch(key.enc_key, out, in); 514 } 515 EXPORT_SYMBOL(aes_encrypt); 516 517 void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 518 const u8 in[AES_BLOCK_SIZE]) 519 { 520 aes_decrypt_arch(key, out, in); 521 } 522 EXPORT_SYMBOL(aes_decrypt); 523 524 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) 525 526 #ifndef aes_cbcmac_blocks_arch 527 static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], 528 const struct aes_enckey *key, const u8 *data, 529 size_t nblocks, bool enc_before, 530 bool enc_after) 531 { 532 return false; 533 } 534 #endif 535 536 /* This assumes nblocks >= 1. */ 537 static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], 538 const struct aes_enckey *key, const u8 *data, 539 size_t nblocks, bool enc_before, bool enc_after) 540 { 541 if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, 542 enc_after)) 543 return; 544 545 if (enc_before) 546 aes_encrypt(key, h, h); 547 for (; nblocks > 1; nblocks--) { 548 crypto_xor(h, data, AES_BLOCK_SIZE); 549 data += AES_BLOCK_SIZE; 550 aes_encrypt(key, h, h); 551 } 552 crypto_xor(h, data, AES_BLOCK_SIZE); 553 if (enc_after) 554 aes_encrypt(key, h, h); 555 } 556 557 int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, 558 size_t key_len) 559 { 560 u64 hi, lo, mask; 561 int err; 562 563 /* Prepare the AES key. */ 564 err = aes_prepareenckey(&key->aes, in_key, key_len); 565 if (err) 566 return err; 567 568 /* 569 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, 570 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). 571 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". 572 */ 573 memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); 574 aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); 575 hi = be64_to_cpu(key->k_final[0].w[0]); 576 lo = be64_to_cpu(key->k_final[0].w[1]); 577 for (int i = 0; i < 2; i++) { 578 mask = ((s64)hi >> 63) & 0x87; 579 hi = (hi << 1) ^ (lo >> 63); 580 lo = (lo << 1) ^ mask; 581 key->k_final[i].w[0] = cpu_to_be64(hi); 582 key->k_final[i].w[1] = cpu_to_be64(lo); 583 } 584 return 0; 585 } 586 EXPORT_SYMBOL_GPL(aes_cmac_preparekey); 587 588 void aes_xcbcmac_preparekey(struct aes_cmac_key *key, 589 const u8 in_key[AES_KEYSIZE_128]) 590 { 591 static const u8 constants[3][AES_BLOCK_SIZE] = { 592 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 593 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 594 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 595 }; 596 u8 new_aes_key[AES_BLOCK_SIZE]; 597 598 static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); 599 aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); 600 aes_encrypt(&key->aes, new_aes_key, constants[0]); 601 aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); 602 aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); 603 aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); 604 memzero_explicit(new_aes_key, AES_BLOCK_SIZE); 605 } 606 EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); 607 608 void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) 609 { 610 bool enc_before = false; 611 size_t nblocks; 612 613 if (ctx->partial_len) { 614 /* XOR data into a pending block. */ 615 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 616 617 crypto_xor(&ctx->h[ctx->partial_len], data, l); 618 data += l; 619 data_len -= l; 620 ctx->partial_len += l; 621 if (data_len == 0) { 622 /* 623 * Either the pending block hasn't been filled yet, or 624 * no more data was given so it's not yet known whether 625 * the block is the final block. 626 */ 627 return; 628 } 629 /* Pending block has been filled and isn't the final block. */ 630 enc_before = true; 631 } 632 633 nblocks = data_len / AES_BLOCK_SIZE; 634 data_len %= AES_BLOCK_SIZE; 635 if (nblocks == 0) { 636 /* 0 additional full blocks, then optionally a partial block */ 637 if (enc_before) 638 aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); 639 crypto_xor(ctx->h, data, data_len); 640 ctx->partial_len = data_len; 641 } else if (data_len != 0) { 642 /* 1 or more additional full blocks, then a partial block */ 643 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 644 enc_before, /* enc_after= */ true); 645 data += nblocks * AES_BLOCK_SIZE; 646 crypto_xor(ctx->h, data, data_len); 647 ctx->partial_len = data_len; 648 } else { 649 /* 650 * 1 or more additional full blocks only. Encryption of the 651 * last block is delayed until it's known whether it's the final 652 * block in the message or not. 653 */ 654 aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, 655 enc_before, /* enc_after= */ false); 656 ctx->partial_len = AES_BLOCK_SIZE; 657 } 658 } 659 EXPORT_SYMBOL_GPL(aes_cmac_update); 660 661 void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 662 { 663 if (ctx->partial_len == AES_BLOCK_SIZE) { 664 /* Final block is a full block. Use k_final[0]. */ 665 crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); 666 } else { 667 /* Final block is a partial block. Pad, and use k_final[1]. */ 668 ctx->h[ctx->partial_len] ^= 0x80; 669 crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); 670 } 671 aes_encrypt(&ctx->key->aes, out, ctx->h); 672 memzero_explicit(ctx, sizeof(*ctx)); 673 } 674 EXPORT_SYMBOL_GPL(aes_cmac_final); 675 676 void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, 677 size_t data_len) 678 { 679 bool enc_before = false; 680 size_t nblocks; 681 682 if (ctx->partial_len) { 683 size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); 684 685 crypto_xor(&ctx->h[ctx->partial_len], data, l); 686 data += l; 687 data_len -= l; 688 ctx->partial_len += l; 689 if (ctx->partial_len < AES_BLOCK_SIZE) 690 return; 691 enc_before = true; 692 } 693 694 nblocks = data_len / AES_BLOCK_SIZE; 695 data_len %= AES_BLOCK_SIZE; 696 if (nblocks == 0) { 697 if (enc_before) 698 aes_encrypt(ctx->key, ctx->h, ctx->h); 699 } else { 700 aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, 701 /* enc_after= */ true); 702 data += nblocks * AES_BLOCK_SIZE; 703 } 704 crypto_xor(ctx->h, data, data_len); 705 ctx->partial_len = data_len; 706 } 707 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); 708 709 void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) 710 { 711 if (ctx->partial_len) 712 aes_encrypt(ctx->key, out, ctx->h); 713 else 714 memcpy(out, ctx->h, AES_BLOCK_SIZE); 715 memzero_explicit(ctx, sizeof(*ctx)); 716 } 717 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); 718 719 /* 720 * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 721 * Implementation Guidance, a cryptographic algorithm self-test for at least one 722 * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes 723 * is implemented. This fulfills that requirement via AES-CMAC. 724 * 725 * This is just for FIPS. The full tests are in the KUnit test suite. 726 */ 727 static void __init aes_cmac_fips_test(void) 728 { 729 struct aes_cmac_key key; 730 u8 mac[AES_BLOCK_SIZE]; 731 732 if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 733 0) 734 panic("aes: CMAC FIPS self-test failed (preparekey)\n"); 735 aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); 736 if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) 737 panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); 738 memzero_explicit(&key, sizeof(key)); 739 } 740 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 741 static inline void aes_cmac_fips_test(void) 742 { 743 } 744 #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ 745 746 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 747 /* 748 * Hooks for optimized AES-ECB implementations, overridable by the architecture. 749 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 750 * causes the fallback implementation to be used instead. 751 */ 752 #ifndef aes_ecb_encrypt_arch 753 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 754 const struct aes_enckey *key) 755 { 756 return false; 757 } 758 #endif 759 #ifndef aes_ecb_decrypt_arch 760 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 761 const struct aes_key *key) 762 { 763 return false; 764 } 765 #endif 766 767 void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) 768 { 769 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 770 len = round_down(len, AES_BLOCK_SIZE); 771 772 if (unlikely(len == 0)) 773 return; 774 775 if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) 776 return; 777 778 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 779 aes_encrypt(key, &dst[i], &src[i]); 780 } 781 EXPORT_SYMBOL_GPL(aes_ecb_encrypt); 782 783 void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, 784 const struct aes_key *key) 785 { 786 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 787 len = round_down(len, AES_BLOCK_SIZE); 788 789 if (unlikely(len == 0)) 790 return; 791 792 if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) 793 return; 794 795 for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) 796 aes_decrypt(key, &dst[i], &src[i]); 797 } 798 EXPORT_SYMBOL_GPL(aes_ecb_decrypt); 799 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 800 801 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 802 /* 803 * Hooks for optimized AES-CBC implementations, overridable by the architecture. 804 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false 805 * causes the fallback implementation to be used instead. 806 */ 807 #ifndef aes_cbc_encrypt_arch 808 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 809 u8 iv[AES_BLOCK_SIZE], 810 const struct aes_enckey *key) 811 { 812 return false; 813 } 814 #endif 815 #ifndef aes_cbc_decrypt_arch 816 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 817 u8 iv[AES_BLOCK_SIZE], 818 const struct aes_key *key) 819 { 820 return false; 821 } 822 #endif 823 824 void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 825 aes_encrypt_arg key) 826 { 827 const u8 *prev = iv; 828 829 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 830 len = round_down(len, AES_BLOCK_SIZE); 831 832 if (unlikely(len == 0)) 833 return; 834 835 if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) 836 return; 837 838 do { 839 crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 840 aes_encrypt(key, dst, dst); 841 prev = dst; 842 dst += AES_BLOCK_SIZE; 843 src += AES_BLOCK_SIZE; 844 len -= AES_BLOCK_SIZE; 845 } while (len); 846 memcpy(iv, prev, AES_BLOCK_SIZE); 847 } 848 EXPORT_SYMBOL_GPL(aes_cbc_encrypt); 849 850 void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], 851 const struct aes_key *key) 852 { 853 u8 next_iv[AES_BLOCK_SIZE]; 854 855 if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) 856 len = round_down(len, AES_BLOCK_SIZE); 857 858 if (unlikely(len == 0)) 859 return; 860 861 if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) 862 return; 863 864 len -= AES_BLOCK_SIZE; 865 dst += len; 866 src += len; 867 memcpy(next_iv, src, AES_BLOCK_SIZE); 868 for (;;) { 869 aes_decrypt(key, dst, src); 870 if (len == 0) 871 break; 872 src -= AES_BLOCK_SIZE; 873 crypto_xor(dst, src, AES_BLOCK_SIZE); 874 dst -= AES_BLOCK_SIZE; 875 len -= AES_BLOCK_SIZE; 876 } 877 crypto_xor(dst, iv, AES_BLOCK_SIZE); 878 memcpy(iv, next_iv, AES_BLOCK_SIZE); 879 } 880 EXPORT_SYMBOL_GPL(aes_cbc_decrypt); 881 882 /* 883 * Hooks for optimized AES-CBC-CTS implementations, overridable by the 884 * architecture. They are called with len > AES_BLOCK_SIZE. Returning false 885 * causes the fallback implementation to be used instead. The fallback 886 * implementation still uses the arch-optimized AES-CBC code if available, but 887 * direct implementation of AES-CBC-CTS is helpful on short messages. 888 */ 889 #ifndef aes_cbc_cts_encrypt_arch 890 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 891 u8 iv[AES_BLOCK_SIZE], 892 const struct aes_enckey *key) 893 { 894 return false; 895 } 896 #endif 897 #ifndef aes_cbc_cts_decrypt_arch 898 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 899 u8 iv[AES_BLOCK_SIZE], 900 const struct aes_key *key) 901 { 902 return false; 903 } 904 #endif 905 906 void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, 907 u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 908 { 909 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 910 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 911 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 912 size_t pn_len = len - pn_offset; 913 u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 914 u8 *pad; 915 916 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 917 return; 918 919 if (len == AES_BLOCK_SIZE) { 920 aes_cbc_encrypt(dst, src, len, iv, key); 921 return; 922 } 923 if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) 924 return; 925 926 /* CBC-encrypt all blocks except the last. */ 927 aes_cbc_encrypt(dst, src, pn_offset, iv, key); 928 929 /* 930 * Compute C[n] and C[n - 1]. 931 * 932 * Careful: src may equal dst (i.e., the encryption can be in-place), so 933 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 934 */ 935 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 936 memcpy(tmp, pad, AES_BLOCK_SIZE); 937 crypto_xor(tmp, &src[pn_offset], pn_len); 938 memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ 939 aes_encrypt(key, pad, tmp); /* C[n - 1] */ 940 941 memzero_explicit(tmp, sizeof(tmp)); 942 } 943 EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); 944 945 void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, 946 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) 947 { 948 /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ 949 size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); 950 /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ 951 size_t pn_len = len - pn_offset; 952 u8 *pad; 953 954 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 955 return; 956 957 if (len == AES_BLOCK_SIZE) { 958 aes_cbc_decrypt(dst, src, len, iv, key); 959 return; 960 } 961 if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) 962 return; 963 964 /* Compute P[0]..P[n - 2]. */ 965 aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); 966 967 /* 968 * Compute P[n] and P[n - 1]. 969 * 970 * Careful: src may equal dst (i.e., the decryption can be in-place), so 971 * src[pn_offset..] can't be read after dst[pn_offset..] is written. 972 * 973 * To avoid needing a temporary buffer, do a "redundant" XOR to recover 974 * src[pn_offset..] from dst[pn_offset..] after the latter is written. 975 */ 976 pad = &dst[pn_offset - AES_BLOCK_SIZE]; 977 aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); 978 crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, 979 pn_len); /* P[n] */ 980 crypto_xor(pad, &dst[pn_offset], pn_len); 981 aes_decrypt(key, pad, pad); 982 crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ 983 } 984 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); 985 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 986 987 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 988 /* 989 * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the 990 * architecture. They are called with any len >= 0. Returning false causes the 991 * fallback implementation to be used instead. 992 */ 993 #ifndef aes_ctr_arch 994 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 995 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 996 { 997 return false; 998 } 999 #endif 1000 #ifndef aes_xctr_arch 1001 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1002 const u8 iv[AES_BLOCK_SIZE], 1003 const struct aes_enckey *key) 1004 { 1005 return false; 1006 } 1007 #endif 1008 1009 static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) 1010 { 1011 /* 1012 * 255 times out of 256 the first iteration is enough, so unroll the 1013 * first iteration as a micro-optimization. 1014 */ 1015 if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) 1016 return; 1017 for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { 1018 if (++ctr[i] != 0) 1019 break; 1020 } 1021 } 1022 1023 void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], 1024 aes_encrypt_arg key) 1025 { 1026 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1027 1028 if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) 1029 return; 1030 1031 /* Handle the full blocks. */ 1032 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1033 aes_encrypt(key, keystream, ctr); 1034 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1035 inc_be128_ctr(ctr); 1036 dst += AES_BLOCK_SIZE; 1037 src += AES_BLOCK_SIZE; 1038 } 1039 /* Handle any partial block at the end. */ 1040 if (len) { 1041 aes_encrypt(key, keystream, ctr); 1042 crypto_xor_cpy(dst, src, keystream, len); 1043 /* Counter is incremented even with just a partial block. */ 1044 inc_be128_ctr(ctr); 1045 } 1046 memzero_explicit(keystream, sizeof(keystream)); 1047 } 1048 EXPORT_SYMBOL_GPL(aes_ctr); 1049 1050 void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, 1051 const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) 1052 { 1053 const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); 1054 __le64 aes_input[2]; 1055 u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1056 1057 if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) 1058 return; 1059 1060 aes_input[1] = get_unaligned((const __le64 *)&iv[8]); 1061 /* Handle the full blocks. */ 1062 for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { 1063 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1064 aes_encrypt(key, keystream, (const u8 *)aes_input); 1065 crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); 1066 dst += AES_BLOCK_SIZE; 1067 src += AES_BLOCK_SIZE; 1068 } 1069 /* Handle any partial block at the end. */ 1070 if (len) { 1071 /* Counter is incremented even with just a partial block. */ 1072 aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); 1073 aes_encrypt(key, keystream, (const u8 *)aes_input); 1074 crypto_xor_cpy(dst, src, keystream, len); 1075 } 1076 memzero_explicit(keystream, sizeof(keystream)); 1077 memzero_explicit(aes_input, sizeof(aes_input)); 1078 } 1079 EXPORT_SYMBOL_GPL(aes_xctr); 1080 #endif /* CONFIG_CRYPTO_LIB_AES_CTR */ 1081 1082 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 1083 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, 1084 size_t key_len, int flags) 1085 { 1086 int err; 1087 1088 err = __xts_verify_key(in_key, key_len, flags); 1089 if (unlikely(err)) 1090 goto out_zeroize; 1091 /* First half of XTS key is the main key */ 1092 err = aes_preparekey(&key->main_key, in_key, key_len / 2); 1093 if (unlikely(err)) 1094 goto out_zeroize; 1095 /* Second half of XTS key is the tweak key */ 1096 err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], 1097 key_len / 2); 1098 if (unlikely(err)) 1099 goto out_zeroize; 1100 return 0; 1101 1102 out_zeroize: 1103 memzero_explicit(key, sizeof(*key)); 1104 return err; 1105 } 1106 EXPORT_SYMBOL_GPL(aes_xts_preparekey); 1107 1108 /* 1109 * Hooks for optimized AES-XTS implementations, overridable by the architecture. 1110 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, 1111 * they aren't expected to handle ciphertext stealing or empty inputs. 1112 * Returning false causes the fallback implementation to be used instead. 1113 * 1114 * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole 1115 * numbers of blocks anyway, with len >= 512. So there's no need to heavily 1116 * optimize ciphertext stealing for short messages.) 1117 */ 1118 #ifndef aes_xts_encrypt_arch 1119 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 1120 u8 tweak[AES_BLOCK_SIZE], 1121 const struct aes_xts_key *key, bool cont) 1122 { 1123 return false; 1124 } 1125 #endif 1126 #ifndef aes_xts_decrypt_arch 1127 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 1128 u8 tweak[AES_BLOCK_SIZE], 1129 const struct aes_xts_key *key, bool cont) 1130 { 1131 return false; 1132 } 1133 #endif 1134 1135 static noinline void aes_xts_crypt_nocts_blockbyblock( 1136 u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 1137 const struct aes_xts_key *key, bool cont, bool enc) 1138 { 1139 le128 t; 1140 1141 if (cont) 1142 memcpy(&t, tweak, sizeof(t)); 1143 else 1144 aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); 1145 do { 1146 crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); 1147 if (enc) 1148 aes_encrypt(&key->main_key, dst, dst); 1149 else 1150 aes_decrypt(&key->main_key, dst, dst); 1151 crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); 1152 gf128mul_x_ble(&t, &t); 1153 dst += AES_BLOCK_SIZE; 1154 src += AES_BLOCK_SIZE; 1155 len -= AES_BLOCK_SIZE; 1156 } while (len); 1157 memcpy(tweak, &t, sizeof(t)); 1158 memzero_explicit(&t, sizeof(t)); 1159 } 1160 1161 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1162 static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, 1163 size_t len, 1164 u8 tweak[AES_BLOCK_SIZE], 1165 const struct aes_xts_key *key, 1166 bool cont) 1167 { 1168 if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) 1169 return; 1170 1171 /* 1172 * For the fallback, just go block-by-block. It could be implemented on 1173 * top of AES-ECB, which could be significantly faster than this if the 1174 * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS 1175 * performance is important enough that it needs to be (and has been) 1176 * implemented directly by every non-obsolete arch anyway. 1177 */ 1178 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1179 /* enc= */ true); 1180 } 1181 1182 /* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ 1183 static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, 1184 size_t len, 1185 u8 tweak[AES_BLOCK_SIZE], 1186 const struct aes_xts_key *key, 1187 bool cont) 1188 { 1189 if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) 1190 return; 1191 1192 /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ 1193 aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, 1194 /* enc= */ false); 1195 } 1196 1197 static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, 1198 u8 tweak[AES_BLOCK_SIZE], 1199 const struct aes_xts_key *key, 1200 bool cont) 1201 { 1202 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1203 size_t nocts_len = round_down(len, AES_BLOCK_SIZE); 1204 u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); 1205 1206 /* Encrypt all full blocks. */ 1207 aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1208 dst += nocts_len - AES_BLOCK_SIZE; 1209 src += nocts_len - AES_BLOCK_SIZE; 1210 1211 /* 1212 * Swap the partial block with the first 'partial_len' bytes of the 1213 * encrypted last full block. Note that a temporary buffer is needed to 1214 * support in-place encryption. 1215 */ 1216 memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); 1217 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1218 memcpy(dst, tmp_block, partial_len); 1219 1220 /* Encrypt the last full block again. */ 1221 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1222 aes_encrypt(&key->main_key, dst, dst); 1223 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1224 memzero_explicit(tmp_block, sizeof(tmp_block)); 1225 } 1226 1227 static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, 1228 u8 tweak[AES_BLOCK_SIZE], 1229 const struct aes_xts_key *key, 1230 bool cont) 1231 { 1232 size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ 1233 size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; 1234 union { 1235 u8 block[AES_BLOCK_SIZE]; 1236 le128 tweak; 1237 } tmp __aligned(__alignof__(long)); 1238 1239 /* 1240 * Decrypt all blocks except the last full block and the partial block. 1241 * The last full block has to be handled specially because decryption 1242 * ciphertext stealing uses the last two tweaks in reverse order. 1243 * 1244 * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() 1245 * doesn't handle (so that the length doesn't get checked redundantly in 1246 * the fast path). So handle that case specially as well. 1247 */ 1248 if (nocts_len) 1249 aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); 1250 else if (!cont) 1251 aes_encrypt(&key->tweak_key, tweak, tweak); 1252 dst += nocts_len; 1253 src += nocts_len; 1254 1255 /* Copy the tweak, advance it again, then decrypt last full block. */ 1256 memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); 1257 gf128mul_x_ble(&tmp.tweak, &tmp.tweak); 1258 crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); 1259 aes_decrypt(&key->main_key, dst, dst); 1260 crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); 1261 1262 /* 1263 * Swap the partial block with the first 'partial_len' bytes of the 1264 * decrypted last full block. Note that a temporary buffer is needed to 1265 * support in-place decryption. 1266 */ 1267 memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); 1268 memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); 1269 memcpy(dst, tmp.block, partial_len); 1270 1271 /* Decrypt the last full block again. */ 1272 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1273 aes_decrypt(&key->main_key, dst, dst); 1274 crypto_xor(dst, tweak, AES_BLOCK_SIZE); 1275 memzero_explicit(&tmp, sizeof(tmp)); 1276 } 1277 1278 void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, 1279 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1280 bool cont) 1281 { 1282 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1283 return; 1284 1285 if (unlikely(len % AES_BLOCK_SIZE)) { 1286 aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); 1287 return; 1288 } 1289 1290 aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); 1291 } 1292 EXPORT_SYMBOL_GPL(aes_xts_encrypt); 1293 1294 void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, 1295 u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, 1296 bool cont) 1297 { 1298 if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) 1299 return; 1300 1301 if (unlikely(len % AES_BLOCK_SIZE)) { 1302 aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); 1303 return; 1304 } 1305 1306 aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); 1307 } 1308 EXPORT_SYMBOL_GPL(aes_xts_decrypt); 1309 #endif /* CONFIG_CRYPTO_LIB_AES_XTS */ 1310 1311 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) 1312 /* 1313 * Hooks for optimized AES-GCM implementations, overridable by the architecture. 1314 * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't 1315 * expected to handle empty inputs or partial blocks, as those cases are handled 1316 * by non-arch-specific code instead. 1317 * 1318 * The GHASH accumulator is provided in POLYVAL format. The counter is provided 1319 * in big endian format, and it's read-only, as the caller handles updating it. 1320 * 1321 * Returning false causes the fallback implementation to be used instead. 1322 * 1323 * These hooks are used only for en/decrypted data. For the associated data the 1324 * GHASH functions are called instead, so those should be implemented too. 1325 */ 1326 #ifndef aes_gcm_encrypt_update_arch 1327 static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1328 struct polyval_elem *ghash_acc, 1329 const __be32 ctr32[4], 1330 const struct aes_enckey *aes_key, 1331 const struct ghash_key *ghash_key) 1332 { 1333 return false; 1334 } 1335 #endif 1336 #ifndef aes_gcm_decrypt_update_arch 1337 static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, 1338 struct polyval_elem *ghash_acc, 1339 const __be32 ctr32[4], 1340 const struct aes_enckey *aes_key, 1341 const struct ghash_key *ghash_key) 1342 { 1343 return false; 1344 } 1345 #endif 1346 1347 int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, 1348 size_t key_len, size_t authtag_len) 1349 { 1350 u8 h[AES_BLOCK_SIZE] = { 0 }; 1351 int err; 1352 1353 err = crypto_gcm_check_authsize(authtag_len); 1354 if (unlikely(err)) 1355 return err; 1356 1357 err = aes_prepareenckey(&key->aes, in_key, key_len); 1358 if (unlikely(err)) 1359 return err; 1360 1361 aes_encrypt(&key->aes, h, h); 1362 ghash_preparekey(&key->ghash, h); 1363 1364 key->authtag_len = authtag_len; 1365 1366 memzero_explicit(h, sizeof(h)); 1367 return 0; 1368 } 1369 EXPORT_SYMBOL_GPL(aes_gcm_preparekey); 1370 1371 void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], 1372 const struct aes_gcm_key *key) 1373 { 1374 ctx->key = key; 1375 ctx->ad_len = 0; 1376 ctx->data_len = 0; 1377 ghash_init(&ctx->ghash, &key->ghash); 1378 memset(ctx->keystream, 0, sizeof(ctx->keystream)); 1379 1380 memcpy(ctx->ctr32, nonce, 12); 1381 ctx->ctr32[3] = cpu_to_be32(1); 1382 1383 aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); 1384 ctx->ctr32[3] = cpu_to_be32(2); 1385 } 1386 EXPORT_SYMBOL_GPL(aes_gcm_init); 1387 1388 void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) 1389 { 1390 WARN_ON_ONCE(ctx->data_len != 0); 1391 if (len) { 1392 ghash_update(&ctx->ghash, ad, len); 1393 ctx->ad_len += len; 1394 } 1395 } 1396 EXPORT_SYMBOL_GPL(aes_gcm_auth_update); 1397 1398 static const u8 gcm_zeroes[AES_BLOCK_SIZE]; 1399 1400 static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) 1401 { 1402 if (len % AES_BLOCK_SIZE) 1403 ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); 1404 } 1405 1406 static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, 1407 u8 *dst, const u8 *src, 1408 size_t len, bool enc) 1409 { 1410 size_t partial_len, n; 1411 1412 if (unlikely(len == 0)) 1413 return; 1414 1415 partial_len = ctx->data_len % AES_BLOCK_SIZE; 1416 if (ctx->data_len == 0) 1417 ghash_pad(&ctx->ghash, ctx->ad_len); 1418 ctx->data_len += len; 1419 1420 if (unlikely(partial_len != 0)) { 1421 /* 1422 * The previous call ended on a non-block-aligned data_len, so 1423 * continue using a previously-generated keystream block. 1424 */ 1425 n = min(len, AES_BLOCK_SIZE - partial_len); 1426 if (enc) { 1427 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1428 n); 1429 ghash_update(&ctx->ghash, dst, n); 1430 } else { 1431 ghash_update(&ctx->ghash, src, n); 1432 crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], 1433 n); 1434 } 1435 dst += n; 1436 src += n; 1437 len -= n; 1438 } 1439 1440 if (len >= AES_BLOCK_SIZE) { 1441 n = round_down(len, AES_BLOCK_SIZE); 1442 if (enc) { 1443 if (likely(aes_gcm_encrypt_update_arch( 1444 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1445 &ctx->key->aes, &ctx->key->ghash))) { 1446 be32_add_cpu(&ctx->ctr32[3], 1447 n / AES_BLOCK_SIZE); 1448 } else { 1449 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1450 ghash_update(&ctx->ghash, dst, n); 1451 } 1452 } else { 1453 if (likely(aes_gcm_decrypt_update_arch( 1454 dst, src, n, &ctx->ghash.acc, ctx->ctr32, 1455 &ctx->key->aes, &ctx->key->ghash))) { 1456 be32_add_cpu(&ctx->ctr32[3], 1457 n / AES_BLOCK_SIZE); 1458 } else { 1459 ghash_update(&ctx->ghash, src, n); 1460 aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); 1461 } 1462 } 1463 dst += n; 1464 src += n; 1465 len -= n; 1466 } 1467 1468 if (len != 0) { 1469 /* 1470 * Ending on a non-block aligned data_len. Generate the next 1471 * keystream block, use the needed portion of it, and leave it 1472 * cached in ctx->keystream in case this isn't the final call. 1473 */ 1474 aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); 1475 be32_add_cpu(&ctx->ctr32[3], 1); 1476 if (enc) { 1477 crypto_xor_cpy(dst, src, ctx->keystream, len); 1478 ghash_update(&ctx->ghash, dst, len); 1479 } else { 1480 ghash_update(&ctx->ghash, src, len); 1481 crypto_xor_cpy(dst, src, ctx->keystream, len); 1482 } 1483 } 1484 } 1485 1486 void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1487 size_t len) 1488 { 1489 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); 1490 } 1491 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); 1492 1493 void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, 1494 size_t len) 1495 { 1496 aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); 1497 } 1498 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); 1499 1500 /* Maximum AES-GCM associated data length in bytes */ 1501 #define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) 1502 /* Maximum AES-GCM en/decrypted data length in bytes */ 1503 #define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) 1504 1505 void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) 1506 { 1507 __be64 tail[2]; 1508 1509 WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); 1510 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); 1511 1512 ghash_pad(&ctx->ghash, 1513 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1514 1515 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1516 tail[1] = cpu_to_be64(ctx->data_len * 8); 1517 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1518 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1519 1520 crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1521 memzero_explicit(ctx, sizeof(*ctx)); 1522 } 1523 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); 1524 1525 int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) 1526 { 1527 __be64 tail[2]; 1528 int err; 1529 1530 if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || 1531 WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { 1532 err = -EBADMSG; 1533 goto out; 1534 } 1535 1536 ghash_pad(&ctx->ghash, 1537 ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); 1538 1539 tail[0] = cpu_to_be64(ctx->ad_len * 8); 1540 tail[1] = cpu_to_be64(ctx->data_len * 8); 1541 ghash_update(&ctx->ghash, (const u8 *)tail, 16); 1542 ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ 1543 crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); 1544 err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? 1545 -EBADMSG : 1546 0; 1547 out: 1548 memzero_explicit(ctx, sizeof(*ctx)); 1549 return err; 1550 } 1551 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); 1552 1553 void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, 1554 const u8 *ad, size_t ad_len, const u8 nonce[12], 1555 const struct aes_gcm_key *key) 1556 { 1557 struct aes_gcm_ctx ctx; 1558 1559 aes_gcm_init(&ctx, nonce, key); 1560 aes_gcm_auth_update(&ctx, ad, ad_len); 1561 aes_gcm_encrypt_update(&ctx, dst, src, data_len); 1562 aes_gcm_encrypt_final(&ctx, authtag); 1563 } 1564 EXPORT_SYMBOL_GPL(aes_gcm_encrypt); 1565 1566 int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, 1567 const u8 *ad, size_t ad_len, const u8 nonce[12], 1568 const struct aes_gcm_key *key) 1569 { 1570 struct aes_gcm_ctx ctx; 1571 int err; 1572 1573 aes_gcm_init(&ctx, nonce, key); 1574 aes_gcm_auth_update(&ctx, ad, ad_len); 1575 aes_gcm_decrypt_update(&ctx, dst, src, data_len); 1576 err = aes_gcm_decrypt_final(&ctx, authtag); 1577 if (unlikely(err) && data_len) { 1578 /* 1579 * Clear the inauthentic decrypted data so that callers won't 1580 * receive it even if they fail to correctly handle errors. 1581 */ 1582 memset(dst, 0, data_len); 1583 } 1584 return err; 1585 } 1586 EXPORT_SYMBOL_GPL(aes_gcm_decrypt); 1587 1588 #endif /* CONFIG_CRYPTO_LIB_AES_GCM */ 1589 1590 static int __init aes_mod_init(void) 1591 { 1592 #ifdef aes_mod_init_arch 1593 aes_mod_init_arch(); 1594 #endif 1595 if (fips_enabled) 1596 aes_cmac_fips_test(); 1597 return 0; 1598 } 1599 subsys_initcall(aes_mod_init); 1600 1601 static void __exit aes_mod_exit(void) 1602 { 1603 } 1604 module_exit(aes_mod_exit); 1605 1606 MODULE_DESCRIPTION("AES block cipher"); 1607 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 1608 MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>"); 1609 MODULE_LICENSE("GPL v2"); 1610