1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * AES using the RISC-V vector crypto extensions 4 * 5 * Copyright (C) 2023 VRULL GmbH 6 * Copyright (C) 2023 SiFive, Inc. 7 * Copyright 2024 Google LLC 8 */ 9 10 #include <asm/simd.h> 11 #include <asm/vector.h> 12 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned); 14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned_zvkb); 15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_xts); 16 17 /* The assembly code assumes the following offsets. */ 18 static_assert(offsetof(struct aes_enckey, len) == 0); 19 static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16); 20 static_assert(offsetof(struct aes_key, len) == 0); 21 static_assert(offsetof(struct aes_key, k.rndkeys) == 16); 22 23 void aes_encrypt_zvkned(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], 24 const u8 in[AES_BLOCK_SIZE]); 25 void aes_decrypt_zvkned(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], 26 const u8 in[AES_BLOCK_SIZE]); 27 28 static void aes_preparekey_arch(union aes_enckey_arch *k, 29 union aes_invkey_arch *inv_k, 30 const u8 *in_key, int key_len, int nrounds) 31 { 32 aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 33 in_key, key_len); 34 } 35 36 static void aes_encrypt_arch(const struct aes_enckey *key, 37 u8 out[AES_BLOCK_SIZE], 38 const u8 in[AES_BLOCK_SIZE]) 39 { 40 if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) { 41 kernel_vector_begin(); 42 aes_encrypt_zvkned(key, out, in); 43 kernel_vector_end(); 44 } else { 45 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 46 } 47 } 48 49 static void aes_decrypt_arch(const struct aes_key *key, 50 u8 out[AES_BLOCK_SIZE], 51 const u8 in[AES_BLOCK_SIZE]) 52 { 53 /* 54 * Note that the Zvkned code uses the standard round keys, while the 55 * fallback uses the inverse round keys. Thus both must be present. 56 */ 57 if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) { 58 kernel_vector_begin(); 59 aes_decrypt_zvkned(key, out, in); 60 kernel_vector_end(); 61 } else { 62 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, 63 out, in); 64 } 65 } 66 67 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 68 void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len, 69 const struct aes_enckey *key); 70 void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len, 71 const struct aes_key *key); 72 73 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 74 #define aes_ecb_encrypt_arch aes_ecb_encrypt_arch 75 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 76 const struct aes_enckey *key) 77 { 78 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 79 return false; 80 kernel_vector_begin(); 81 aes_ecb_encrypt_zvkned(dst, src, len, key); 82 kernel_vector_end(); 83 return true; 84 } 85 86 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 87 #define aes_ecb_decrypt_arch aes_ecb_decrypt_arch 88 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 89 const struct aes_key *key) 90 { 91 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 92 return false; 93 kernel_vector_begin(); 94 aes_ecb_decrypt_zvkned(dst, src, len, key); 95 kernel_vector_end(); 96 return true; 97 } 98 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 99 100 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 101 void aes_cbc_encrypt_zvkned(u8 *dst, const u8 *src, size_t len, 102 u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key); 103 void aes_cbc_decrypt_zvkned(u8 *dst, const u8 *src, size_t len, 104 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key); 105 void aes_cbc_cts_crypt_zvkned(u8 *dst, const u8 *src, size_t len, 106 const u8 iv[AES_BLOCK_SIZE], 107 aes_encrypt_arg key, bool enc); 108 109 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 110 #define aes_cbc_encrypt_arch aes_cbc_encrypt_arch 111 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 112 u8 iv[AES_BLOCK_SIZE], 113 const struct aes_enckey *key) 114 { 115 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 116 return false; 117 kernel_vector_begin(); 118 aes_cbc_encrypt_zvkned(dst, src, len, iv, key); 119 kernel_vector_end(); 120 return true; 121 } 122 123 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 124 #define aes_cbc_decrypt_arch aes_cbc_decrypt_arch 125 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 126 u8 iv[AES_BLOCK_SIZE], 127 const struct aes_key *key) 128 { 129 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 130 return false; 131 kernel_vector_begin(); 132 aes_cbc_decrypt_zvkned(dst, src, len, iv, key); 133 kernel_vector_end(); 134 return true; 135 } 136 137 /* len can be any value greater than AES_BLOCK_SIZE here. */ 138 #define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch 139 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 140 u8 iv[AES_BLOCK_SIZE], 141 const struct aes_enckey *key) 142 { 143 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 144 return false; 145 146 kernel_vector_begin(); 147 aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, true); 148 kernel_vector_end(); 149 return true; 150 } 151 152 /* len can be any value greater than AES_BLOCK_SIZE here. */ 153 #define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch 154 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 155 u8 iv[AES_BLOCK_SIZE], 156 const struct aes_key *key) 157 { 158 if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd())) 159 return false; 160 161 kernel_vector_begin(); 162 aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, false); 163 kernel_vector_end(); 164 return true; 165 } 166 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 167 168 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) 169 void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, size_t len, u8 iv[16], 170 const struct aes_enckey *key); 171 172 static void aes_ctr_riscv(u8 *dst, const u8 *src, size_t len, 173 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 174 { 175 u32 ctr32 = get_unaligned_be32(&ctr[12]); 176 u32 nblocks; 177 size_t part1_len; 178 179 /* This assumes len <= U32_MAX * AES_BLOCK_SIZE. */ 180 nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE); 181 ctr32 += nblocks; 182 183 if (likely(ctr32 >= nblocks)) { 184 /* The low 32 bits of the counter won't overflow. */ 185 aes_ctr32_crypt_zvkned_zvkb(dst, src, len, ctr, key); 186 } else { 187 /* 188 * The low 32 bits of the counter will overflow. The 189 * assembly doesn't handle this case, so split the 190 * operation into two at the point where the overflow 191 * will occur. After the first part, add the carry bit. 192 */ 193 part1_len = 194 min(len, (size_t)(nblocks - ctr32) * AES_BLOCK_SIZE); 195 aes_ctr32_crypt_zvkned_zvkb(dst, src, part1_len, ctr, key); 196 for (int i = AES_BLOCK_SIZE - 5; i >= 0; i--) { 197 if (++ctr[i] != 0) 198 break; 199 } 200 if (part1_len < len) 201 aes_ctr32_crypt_zvkned_zvkb(dst + part1_len, 202 src + part1_len, 203 len - part1_len, ctr, key); 204 } 205 } 206 207 #define aes_ctr_arch aes_ctr_arch 208 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 209 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 210 { 211 if (!static_branch_likely(&have_zvkned_zvkb) || 212 unlikely(!may_use_simd())) 213 return false; 214 kernel_vector_begin(); 215 while (len) { 216 /* 217 * Split super long messages into chunks that each need at most 218 * 1 carry bit out of the low 32-bit word of the counter. Up to 219 * U32_MAX * AES_BLOCK_SIZE bytes could be allowed per chunk, 220 * but use a lower limit to guard against other errors. 221 */ 222 size_t n = min(len, round_down(S32_MAX, 32 * AES_BLOCK_SIZE)); 223 224 aes_ctr_riscv(dst, src, n, ctr, key); 225 dst += n; 226 src += n; 227 len -= n; 228 } 229 kernel_vector_end(); 230 return true; 231 } 232 #endif /* CONFIG_CRYPTO_LIB_AES_CTR */ 233 234 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 235 void aes_xts_encrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len, 236 u8 tweak[AES_BLOCK_SIZE], 237 const struct aes_key *key); 238 void aes_xts_decrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len, 239 u8 tweak[AES_BLOCK_SIZE], 240 const struct aes_key *key); 241 242 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 243 static __always_inline bool 244 aes_xts_crypt_riscv(u8 *dst, const u8 *src, size_t len, 245 u8 tweak[AES_BLOCK_SIZE], 246 const struct aes_xts_key *key, bool cont, bool enc) 247 { 248 if (!static_branch_likely(&have_xts) || unlikely(!may_use_simd())) 249 return false; 250 kernel_vector_begin(); 251 if (!cont) 252 aes_encrypt_zvkned(&key->tweak_key, tweak, tweak); 253 if (enc) 254 aes_xts_encrypt_zvkned_zvbb_zvkg(dst, src, len, tweak, 255 &key->main_key); 256 else 257 aes_xts_decrypt_zvkned_zvbb_zvkg(dst, src, len, tweak, 258 &key->main_key); 259 kernel_vector_end(); 260 return true; 261 } 262 263 #define aes_xts_encrypt_arch aes_xts_encrypt_arch 264 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 265 u8 tweak[AES_BLOCK_SIZE], 266 const struct aes_xts_key *key, bool cont) 267 { 268 return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, true); 269 } 270 271 #define aes_xts_decrypt_arch aes_xts_decrypt_arch 272 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 273 u8 tweak[AES_BLOCK_SIZE], 274 const struct aes_xts_key *key, bool cont) 275 { 276 return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, false); 277 } 278 #endif /* CONFIG_CRYPTO_LIB_AES_XTS */ 279 280 #define aes_mod_init_arch aes_mod_init_arch 281 static void aes_mod_init_arch(void) 282 { 283 if (riscv_isa_extension_available(NULL, ZVKNED) && 284 riscv_vector_vlen() >= 128) { 285 static_branch_enable(&have_zvkned); 286 if (riscv_isa_extension_available(NULL, ZVKB)) 287 static_branch_enable(&have_zvkned_zvkb); 288 if (riscv_isa_extension_available(NULL, ZVBB) && 289 riscv_isa_extension_available(NULL, ZVKG) && 290 riscv_vector_vlen() < 2048 /* Implementation limitation */) 291 static_branch_enable(&have_xts); 292 } 293 } 294