1 /* 2 * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. 3 * Copyright (c) 2021, Intel Corporation. All Rights Reserved. 4 * 5 * Licensed under the Apache License 2.0 (the "License"). You may not use 6 * this file except in compliance with the License. You can obtain a copy 7 * in the file LICENSE in the source distribution or at 8 * https://www.openssl.org/source/license.html 9 */ 10 11 /* 12 * This file contains an AES-GCM wrapper implementation from OpenSSL, using 13 * VAES extensions. It was ported from cipher_aes_gcm_hw_vaes_avx512.inc. 14 */ 15 16 #include <sys/endian.h> 17 #include <sys/systm.h> 18 19 #include <crypto/openssl/ossl.h> 20 #include <crypto/openssl/ossl_aes_gcm.h> 21 #include <crypto/openssl/ossl_cipher.h> 22 23 #include <opencrypto/cryptodev.h> 24 25 _Static_assert( 26 sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context), 27 "ossl_gcm_context too large"); 28 29 void aesni_set_encrypt_key(const void *key, int bits, void *ctx); 30 31 static void 32 gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) 33 { 34 KASSERT(keylen == 128 || keylen == 192 || keylen == 256, 35 ("%s: invalid key length %zu", __func__, keylen)); 36 37 memset(&ctx->gcm, 0, sizeof(ctx->gcm)); 38 memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); 39 aesni_set_encrypt_key(key, keylen, &ctx->aes_ks); 40 ctx->ops->init(ctx, key, keylen); 41 } 42 43 static void 44 gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) 45 { 46 (void)ctx->ops->finish(ctx, NULL, 0); 47 memcpy(tag, ctx->gcm.Xi.c, len); 48 } 49 50 void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx); 51 void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); 52 void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, 53 const unsigned char *iv, size_t ivlen); 54 void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, 55 size_t len); 56 void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx, 57 unsigned int *pblocklen, const unsigned char *in, size_t len, 58 unsigned char *out); 59 void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx, 60 unsigned int *pblocklen, const unsigned char *in, size_t len, 61 unsigned char *out); 62 void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); 63 64 static void 65 gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen) 66 { 67 ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm); 68 } 69 70 static void 71 gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv, 72 size_t len) 73 { 74 KASSERT(len == AES_GCM_IV_LEN, 75 ("%s: invalid IV length %zu", __func__, len)); 76 77 ctx->gcm.Yi.u[0] = 0; /* Current counter */ 78 ctx->gcm.Yi.u[1] = 0; 79 ctx->gcm.Xi.u[0] = 0; /* AAD hash */ 80 ctx->gcm.Xi.u[1] = 0; 81 ctx->gcm.len.u[0] = 0; /* AAD length */ 82 ctx->gcm.len.u[1] = 0; /* Message length */ 83 ctx->gcm.ares = 0; 84 ctx->gcm.mres = 0; 85 86 ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len); 87 } 88 89 static int 90 gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad, 91 size_t len) 92 { 93 uint64_t alen = ctx->gcm.len.u[0]; 94 size_t lenblks; 95 unsigned int ares; 96 97 /* Bad sequence: call of AAD update after message processing */ 98 if (ctx->gcm.len.u[1]) 99 return -2; 100 101 alen += len; 102 /* AAD is limited by 2^64 bits, thus 2^61 bytes */ 103 if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) 104 return -1; 105 ctx->gcm.len.u[0] = alen; 106 107 ares = ctx->gcm.ares; 108 /* Partial AAD block left from previous AAD update calls */ 109 if (ares > 0) { 110 /* 111 * Fill partial block buffer till full block 112 * (note, the hash is stored reflected) 113 */ 114 while (ares > 0 && len > 0) { 115 ctx->gcm.Xi.c[15 - ares] ^= *(aad++); 116 --len; 117 ares = (ares + 1) % AES_BLOCK_LEN; 118 } 119 /* Full block gathered */ 120 if (ares == 0) { 121 ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); 122 } else { /* no more AAD */ 123 ctx->gcm.ares = ares; 124 return 0; 125 } 126 } 127 128 /* Bulk AAD processing */ 129 lenblks = len & ((size_t)(-AES_BLOCK_LEN)); 130 if (lenblks > 0) { 131 ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks); 132 aad += lenblks; 133 len -= lenblks; 134 } 135 136 /* Add remaining AAD to the hash (note, the hash is stored reflected) */ 137 if (len > 0) { 138 ares = (unsigned int)len; 139 for (size_t i = 0; i < len; ++i) 140 ctx->gcm.Xi.c[15 - i] ^= aad[i]; 141 } 142 143 ctx->gcm.ares = ares; 144 145 return 0; 146 } 147 148 static int 149 _gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, 150 unsigned char *out, size_t len, bool encrypt) 151 { 152 uint64_t mlen = ctx->gcm.len.u[1]; 153 154 mlen += len; 155 if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) 156 return -1; 157 158 ctx->gcm.len.u[1] = mlen; 159 160 /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ 161 if (ctx->gcm.ares > 0) { 162 ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); 163 ctx->gcm.ares = 0; 164 } 165 166 if (encrypt) { 167 ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, 168 in, len, out); 169 } else { 170 ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, 171 in, len, out); 172 } 173 174 return 0; 175 } 176 177 static int 178 gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, 179 unsigned char *out, size_t len) 180 { 181 return _gcm_encrypt_avx512(ctx, in, out, len, true); 182 } 183 184 static int 185 gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, 186 unsigned char *out, size_t len) 187 { 188 return _gcm_encrypt_avx512(ctx, in, out, len, false); 189 } 190 191 static int 192 gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag, 193 size_t len) 194 { 195 unsigned int *res = &ctx->gcm.mres; 196 197 /* Finalize AAD processing */ 198 if (ctx->gcm.ares > 0) 199 res = &ctx->gcm.ares; 200 201 ossl_aes_gcm_finalize_avx512(ctx, *res); 202 203 ctx->gcm.ares = ctx->gcm.mres = 0; 204 205 if (tag != NULL) 206 return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); 207 return 0; 208 } 209 210 static const struct ossl_aes_gcm_ops gcm_ops_avx512 = { 211 .init = gcm_init_avx512, 212 .setiv = gcm_setiv_avx512, 213 .aad = gcm_aad_avx512, 214 .encrypt = gcm_encrypt_avx512, 215 .decrypt = gcm_decrypt_avx512, 216 .finish = gcm_finish_avx512, 217 .tag = gcm_tag, 218 }; 219 220 int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx); 221 222 int 223 ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, 224 void *_ctx) 225 { 226 struct ossl_gcm_context *ctx; 227 228 ctx = _ctx; 229 ctx->ops = &gcm_ops_avx512; 230 gcm_init(ctx, key, klen); 231 return (0); 232 } 233