1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _COMMON_CRYPTO_MODES_H 28 #define _COMMON_CRYPTO_MODES_H 29 30 #ifdef __cplusplus 31 extern "C" { 32 #endif 33 34 #include <sys/zfs_context.h> 35 #include <sys/crypto/common.h> 36 #include <sys/crypto/impl.h> 37 38 /* 39 * Does the build chain support all instructions needed for the GCM assembler 40 * routines. AVX support should imply AES-NI and PCLMULQDQ, but make sure 41 * anyhow. 42 */ 43 #if defined(__x86_64__) && defined(HAVE_AVX) && \ 44 defined(HAVE_AES) && defined(HAVE_PCLMULQDQ) 45 #define CAN_USE_GCM_ASM (HAVE_VAES && HAVE_VPCLMULQDQ ? 2 : 1) 46 extern boolean_t gcm_avx_can_use_movbe; 47 #endif 48 49 #define CCM_MODE 0x00000010 50 #define GCM_MODE 0x00000020 51 52 /* 53 * cc_keysched: Pointer to key schedule. 54 * 55 * cc_keysched_len: Length of the key schedule. 56 * 57 * cc_remainder: This is for residual data, i.e. data that can't 58 * be processed because there are too few bytes. 59 * Must wait until more data arrives. 60 * 61 * cc_remainder_len: Number of bytes in cc_remainder. 62 * 63 * cc_iv: Scratch buffer that sometimes contains the IV. 64 * 65 * cc_lastp: Pointer to previous block of ciphertext. 66 * 67 * cc_copy_to: Pointer to where encrypted residual data needs 68 * to be copied. 69 * 70 * cc_flags: PROVIDER_OWNS_KEY_SCHEDULE 71 * When a context is freed, it is necessary 72 * to know whether the key schedule was allocated 73 * by the caller, or internally, e.g. an init routine. 74 * If allocated by the latter, then it needs to be freed. 75 * 76 * CCM_MODE 77 */ 78 struct common_ctx { 79 void *cc_keysched; 80 size_t cc_keysched_len; 81 uint64_t cc_iv[2]; 82 uint64_t cc_remainder[2]; 83 size_t cc_remainder_len; 84 uint8_t *cc_lastp; 85 uint8_t *cc_copy_to; 86 uint32_t cc_flags; 87 }; 88 89 typedef struct common_ctx common_ctx_t; 90 91 /* 92 * 93 * ccm_mac_len: Stores length of the MAC in CCM mode. 94 * ccm_mac_buf: Stores the intermediate value for MAC in CCM encrypt. 95 * In CCM decrypt, stores the input MAC value. 96 * ccm_data_len: Length of the plaintext for CCM mode encrypt, or 97 * length of the ciphertext for CCM mode decrypt. 98 * ccm_processed_data_len: 99 * Length of processed plaintext in CCM mode encrypt, 100 * or length of processed ciphertext for CCM mode decrypt. 101 * ccm_processed_mac_len: 102 * Length of MAC data accumulated in CCM mode decrypt. 103 * 104 * ccm_pt_buf: Only used in CCM mode decrypt. It stores the 105 * decrypted plaintext to be returned when 106 * MAC verification succeeds in decrypt_final. 107 * Memory for this should be allocated in the AES module. 108 * 109 */ 110 typedef struct ccm_ctx { 111 struct common_ctx ccm_common; 112 uint32_t ccm_tmp[4]; 113 size_t ccm_mac_len; 114 uint64_t ccm_mac_buf[2]; 115 size_t ccm_data_len; 116 size_t ccm_processed_data_len; 117 size_t ccm_processed_mac_len; 118 uint8_t *ccm_pt_buf; 119 uint64_t ccm_mac_input_buf[2]; 120 uint64_t ccm_counter_mask; 121 } ccm_ctx_t; 122 123 #define ccm_keysched ccm_common.cc_keysched 124 #define ccm_keysched_len ccm_common.cc_keysched_len 125 #define ccm_cb ccm_common.cc_iv 126 #define ccm_remainder ccm_common.cc_remainder 127 #define ccm_remainder_len ccm_common.cc_remainder_len 128 #define ccm_lastp ccm_common.cc_lastp 129 #define ccm_copy_to ccm_common.cc_copy_to 130 #define ccm_flags ccm_common.cc_flags 131 132 #ifdef CAN_USE_GCM_ASM 133 typedef enum gcm_impl { 134 GCM_IMPL_GENERIC = 0, 135 GCM_IMPL_AVX, 136 GCM_IMPL_AVX2, 137 GCM_IMPL_MAX, 138 } gcm_impl; 139 #endif 140 141 /* 142 * gcm_tag_len: Length of authentication tag. 143 * 144 * gcm_ghash: Stores output from the GHASH function. 145 * 146 * gcm_processed_data_len: 147 * Length of processed plaintext (encrypt) or 148 * length of processed ciphertext (decrypt). 149 * 150 * gcm_pt_buf: Stores the decrypted plaintext returned by 151 * decrypt_final when the computed authentication 152 * tag matches the user supplied tag. 153 * 154 * gcm_pt_buf_len: Length of the plaintext buffer. 155 * 156 * gcm_H: Subkey. 157 * 158 * gcm_Htable: Pre-computed and pre-shifted H, H^2, ... H^6 for the 159 * Karatsuba Algorithm in host byte order. 160 * 161 * gcm_J0: Pre-counter block generated from the IV. 162 * 163 * gcm_len_a_len_c: 64-bit representations of the bit lengths of 164 * AAD and ciphertext. 165 */ 166 typedef struct gcm_ctx { 167 struct common_ctx gcm_common; 168 size_t gcm_tag_len; 169 size_t gcm_processed_data_len; 170 size_t gcm_pt_buf_len; 171 uint32_t gcm_tmp[4]; 172 /* 173 * The offset of gcm_Htable relative to gcm_ghash, (32), is hard coded 174 * in aesni-gcm-x86_64.S, so please don't change (or adjust there). 175 */ 176 uint64_t gcm_ghash[2]; 177 uint64_t gcm_H[2]; 178 #ifdef CAN_USE_GCM_ASM 179 uint64_t *gcm_Htable; 180 size_t gcm_htab_len; 181 #endif 182 uint64_t gcm_J0[2]; 183 uint64_t gcm_len_a_len_c[2]; 184 uint8_t *gcm_pt_buf; 185 #ifdef CAN_USE_GCM_ASM 186 enum gcm_impl impl; 187 #endif 188 } gcm_ctx_t; 189 190 #define gcm_keysched gcm_common.cc_keysched 191 #define gcm_keysched_len gcm_common.cc_keysched_len 192 #define gcm_cb gcm_common.cc_iv 193 #define gcm_remainder gcm_common.cc_remainder 194 #define gcm_remainder_len gcm_common.cc_remainder_len 195 #define gcm_lastp gcm_common.cc_lastp 196 #define gcm_copy_to gcm_common.cc_copy_to 197 #define gcm_flags gcm_common.cc_flags 198 199 void gcm_clear_ctx(gcm_ctx_t *ctx); 200 201 typedef struct aes_ctx { 202 union { 203 ccm_ctx_t acu_ccm; 204 gcm_ctx_t acu_gcm; 205 } acu; 206 } aes_ctx_t; 207 208 #define ac_flags acu.acu_ccm.ccm_common.cc_flags 209 #define ac_remainder_len acu.acu_ccm.ccm_common.cc_remainder_len 210 #define ac_keysched acu.acu_ccm.ccm_common.cc_keysched 211 #define ac_keysched_len acu.acu_ccm.ccm_common.cc_keysched_len 212 #define ac_iv acu.acu_ccm.ccm_common.cc_iv 213 #define ac_lastp acu.acu_ccm.ccm_common.cc_lastp 214 #define ac_pt_buf acu.acu_ccm.ccm_pt_buf 215 #define ac_mac_len acu.acu_ccm.ccm_mac_len 216 #define ac_data_len acu.acu_ccm.ccm_data_len 217 #define ac_processed_mac_len acu.acu_ccm.ccm_processed_mac_len 218 #define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len 219 #define ac_tag_len acu.acu_gcm.gcm_tag_len 220 221 extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, 222 crypto_data_t *, size_t, 223 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 224 void (*copy_block)(uint8_t *, uint8_t *), 225 void (*xor_block)(uint8_t *, uint8_t *)); 226 227 extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, 228 crypto_data_t *, size_t, 229 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 230 void (*copy_block)(uint8_t *, uint8_t *), 231 void (*xor_block)(uint8_t *, uint8_t *)); 232 233 extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, 234 crypto_data_t *, size_t, 235 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 236 void (*copy_block)(uint8_t *, uint8_t *), 237 void (*xor_block)(uint8_t *, uint8_t *)); 238 239 extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, 240 crypto_data_t *, size_t, 241 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 242 void (*copy_block)(uint8_t *, uint8_t *), 243 void (*xor_block)(uint8_t *, uint8_t *)); 244 245 int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, 246 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 247 void (*xor_block)(uint8_t *, uint8_t *)); 248 249 int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, 250 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 251 void (*copy_block)(uint8_t *, uint8_t *), 252 void (*xor_block)(uint8_t *, uint8_t *)); 253 254 extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, 255 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 256 void (*copy_block)(uint8_t *, uint8_t *), 257 void (*xor_block)(uint8_t *, uint8_t *)); 258 259 extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, 260 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 261 void (*xor_block)(uint8_t *, uint8_t *)); 262 263 extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t, 264 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 265 void (*xor_block)(uint8_t *, uint8_t *)); 266 267 extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t, 268 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 269 void (*copy_block)(uint8_t *, uint8_t *), 270 void (*xor_block)(uint8_t *, uint8_t *)); 271 272 extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *, 273 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); 274 275 extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *); 276 277 extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *); 278 extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *, 279 uint8_t **, size_t *, uint8_t **, size_t); 280 281 extern void *ccm_alloc_ctx(int); 282 extern void *gcm_alloc_ctx(int); 283 extern void crypto_free_mode_ctx(void *); 284 285 #ifdef __cplusplus 286 } 287 #endif 288 289 #endif /* _COMMON_CRYPTO_MODES_H */ 290