1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 14 * Use is subject to license terms. 15 */ 16 17 #ifndef _COMMON_CRYPTO_MODES_H 18 #define _COMMON_CRYPTO_MODES_H 19 20 #ifdef __cplusplus 21 extern "C" { 22 #endif 23 24 #include <sys/zfs_context.h> 25 #include <sys/crypto/common.h> 26 #include <sys/crypto/impl.h> 27 28 /* 29 * Does the build chain support all instructions needed for the GCM assembler 30 * routines. AVX support should imply AES-NI and PCLMULQDQ, but make sure 31 * anyhow. 32 */ 33 #if defined(__x86_64__) && HAVE_SIMD(AVX) && \ 34 HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ) 35 #define CAN_USE_GCM_ASM (HAVE_SIMD(VAES) && HAVE_SIMD(VPCLMULQDQ) ? 2 : 1) 36 extern boolean_t gcm_avx_can_use_movbe; 37 #endif 38 39 #define CCM_MODE 0x00000010 40 #define GCM_MODE 0x00000020 41 42 /* 43 * cc_keysched: Pointer to key schedule. 44 * 45 * cc_keysched_len: Length of the key schedule. 46 * 47 * cc_remainder: This is for residual data, i.e. data that can't 48 * be processed because there are too few bytes. 49 * Must wait until more data arrives. 50 * 51 * cc_remainder_len: Number of bytes in cc_remainder. 52 * 53 * cc_iv: Scratch buffer that sometimes contains the IV. 54 * 55 * cc_lastp: Pointer to previous block of ciphertext. 56 * 57 * cc_copy_to: Pointer to where encrypted residual data needs 58 * to be copied. 59 * 60 * cc_flags: PROVIDER_OWNS_KEY_SCHEDULE 61 * When a context is freed, it is necessary 62 * to know whether the key schedule was allocated 63 * by the caller, or internally, e.g. an init routine. 64 * If allocated by the latter, then it needs to be freed. 65 * 66 * CCM_MODE 67 */ 68 struct common_ctx { 69 void *cc_keysched; 70 size_t cc_keysched_len; 71 uint64_t cc_iv[2]; 72 uint64_t cc_remainder[2]; 73 size_t cc_remainder_len; 74 uint8_t *cc_lastp; 75 uint8_t *cc_copy_to; 76 uint32_t cc_flags; 77 }; 78 79 typedef struct common_ctx common_ctx_t; 80 81 /* 82 * 83 * ccm_mac_len: Stores length of the MAC in CCM mode. 84 * ccm_mac_buf: Stores the intermediate value for MAC in CCM encrypt. 85 * In CCM decrypt, stores the input MAC value. 86 * ccm_data_len: Length of the plaintext for CCM mode encrypt, or 87 * length of the ciphertext for CCM mode decrypt. 88 * ccm_processed_data_len: 89 * Length of processed plaintext in CCM mode encrypt, 90 * or length of processed ciphertext for CCM mode decrypt. 91 * ccm_processed_mac_len: 92 * Length of MAC data accumulated in CCM mode decrypt. 93 * 94 * ccm_pt_buf: Only used in CCM mode decrypt. It stores the 95 * decrypted plaintext to be returned when 96 * MAC verification succeeds in decrypt_final. 97 * Memory for this should be allocated in the AES module. 98 * 99 */ 100 typedef struct ccm_ctx { 101 struct common_ctx ccm_common; 102 uint32_t ccm_tmp[4]; 103 size_t ccm_mac_len; 104 uint64_t ccm_mac_buf[2]; 105 size_t ccm_data_len; 106 size_t ccm_processed_data_len; 107 size_t ccm_processed_mac_len; 108 uint8_t *ccm_pt_buf; 109 uint64_t ccm_mac_input_buf[2]; 110 uint64_t ccm_counter_mask; 111 } ccm_ctx_t; 112 113 #define ccm_keysched ccm_common.cc_keysched 114 #define ccm_keysched_len ccm_common.cc_keysched_len 115 #define ccm_cb ccm_common.cc_iv 116 #define ccm_remainder ccm_common.cc_remainder 117 #define ccm_remainder_len ccm_common.cc_remainder_len 118 #define ccm_lastp ccm_common.cc_lastp 119 #define ccm_copy_to ccm_common.cc_copy_to 120 #define ccm_flags ccm_common.cc_flags 121 122 #ifdef CAN_USE_GCM_ASM 123 typedef enum gcm_impl { 124 GCM_IMPL_GENERIC = 0, 125 GCM_IMPL_AVX, 126 GCM_IMPL_AVX2, 127 GCM_IMPL_MAX, 128 } gcm_impl; 129 #endif 130 131 /* 132 * gcm_tag_len: Length of authentication tag. 133 * 134 * gcm_ghash: Stores output from the GHASH function. 135 * 136 * gcm_processed_data_len: 137 * Length of processed plaintext (encrypt) or 138 * length of processed ciphertext (decrypt). 139 * 140 * gcm_pt_buf: Stores the decrypted plaintext returned by 141 * decrypt_final when the computed authentication 142 * tag matches the user supplied tag. 143 * 144 * gcm_pt_buf_len: Length of the plaintext buffer. 145 * 146 * gcm_H: Subkey. 147 * 148 * gcm_Htable: Pre-computed and pre-shifted H, H^2, ... H^6 for the 149 * Karatsuba Algorithm in host byte order. 150 * 151 * gcm_J0: Pre-counter block generated from the IV. 152 * 153 * gcm_len_a_len_c: 64-bit representations of the bit lengths of 154 * AAD and ciphertext. 155 */ 156 typedef struct gcm_ctx { 157 struct common_ctx gcm_common; 158 size_t gcm_tag_len; 159 size_t gcm_processed_data_len; 160 size_t gcm_pt_buf_len; 161 uint32_t gcm_tmp[4]; 162 /* 163 * The offset of gcm_Htable relative to gcm_ghash, (32), is hard coded 164 * in aesni-gcm-x86_64.S, so please don't change (or adjust there). 165 */ 166 uint64_t gcm_ghash[2]; 167 uint64_t gcm_H[2]; 168 #ifdef CAN_USE_GCM_ASM 169 uint64_t *gcm_Htable; 170 size_t gcm_htab_len; 171 #endif 172 uint64_t gcm_J0[2]; 173 uint64_t gcm_len_a_len_c[2]; 174 uint8_t *gcm_pt_buf; 175 #ifdef CAN_USE_GCM_ASM 176 enum gcm_impl impl; 177 #endif 178 } gcm_ctx_t; 179 180 #define gcm_keysched gcm_common.cc_keysched 181 #define gcm_keysched_len gcm_common.cc_keysched_len 182 #define gcm_cb gcm_common.cc_iv 183 #define gcm_remainder gcm_common.cc_remainder 184 #define gcm_remainder_len gcm_common.cc_remainder_len 185 #define gcm_lastp gcm_common.cc_lastp 186 #define gcm_copy_to gcm_common.cc_copy_to 187 #define gcm_flags gcm_common.cc_flags 188 189 void gcm_clear_ctx(gcm_ctx_t *ctx); 190 191 typedef struct aes_ctx { 192 union { 193 ccm_ctx_t acu_ccm; 194 gcm_ctx_t acu_gcm; 195 } acu; 196 } aes_ctx_t; 197 198 #define ac_flags acu.acu_ccm.ccm_common.cc_flags 199 #define ac_remainder_len acu.acu_ccm.ccm_common.cc_remainder_len 200 #define ac_keysched acu.acu_ccm.ccm_common.cc_keysched 201 #define ac_keysched_len acu.acu_ccm.ccm_common.cc_keysched_len 202 #define ac_iv acu.acu_ccm.ccm_common.cc_iv 203 #define ac_lastp acu.acu_ccm.ccm_common.cc_lastp 204 #define ac_pt_buf acu.acu_ccm.ccm_pt_buf 205 #define ac_mac_len acu.acu_ccm.ccm_mac_len 206 #define ac_data_len acu.acu_ccm.ccm_data_len 207 #define ac_processed_mac_len acu.acu_ccm.ccm_processed_mac_len 208 #define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len 209 #define ac_tag_len acu.acu_gcm.gcm_tag_len 210 211 extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, 212 crypto_data_t *, size_t, 213 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 214 void (*copy_block)(uint8_t *, uint8_t *), 215 void (*xor_block)(uint8_t *, uint8_t *)); 216 217 extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, 218 crypto_data_t *, size_t, 219 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 220 void (*copy_block)(uint8_t *, uint8_t *), 221 void (*xor_block)(uint8_t *, uint8_t *)); 222 223 extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, 224 crypto_data_t *, size_t, 225 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 226 void (*copy_block)(uint8_t *, uint8_t *), 227 void (*xor_block)(uint8_t *, uint8_t *)); 228 229 extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, 230 crypto_data_t *, size_t, 231 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 232 void (*copy_block)(uint8_t *, uint8_t *), 233 void (*xor_block)(uint8_t *, uint8_t *)); 234 235 int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, 236 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 237 void (*xor_block)(uint8_t *, uint8_t *)); 238 239 int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, 240 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 241 void (*copy_block)(uint8_t *, uint8_t *), 242 void (*xor_block)(uint8_t *, uint8_t *)); 243 244 extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, 245 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 246 void (*copy_block)(uint8_t *, uint8_t *), 247 void (*xor_block)(uint8_t *, uint8_t *)); 248 249 extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, 250 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 251 void (*xor_block)(uint8_t *, uint8_t *)); 252 253 extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t, 254 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 255 void (*xor_block)(uint8_t *, uint8_t *)); 256 257 extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t, 258 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 259 void (*copy_block)(uint8_t *, uint8_t *), 260 void (*xor_block)(uint8_t *, uint8_t *)); 261 262 extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *, 263 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); 264 265 extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *); 266 267 extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *); 268 extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *, 269 uint8_t **, size_t *, uint8_t **, size_t); 270 271 extern void *ccm_alloc_ctx(int); 272 extern void *gcm_alloc_ctx(int); 273 extern void crypto_free_mode_ctx(void *); 274 275 #ifdef __cplusplus 276 } 277 #endif 278 279 #endif /* _COMMON_CRYPTO_MODES_H */ 280