xref: /freebsd/sys/crypto/aesni/aesni_ghash.c (revision 95ee2897e98f5d444f26ed2334cc7c439f9c16c6)

/*-
 * Copyright (c) 2014 The FreeBSD Foundation
 * All rights reserved.
 *
 * This software was developed by John-Mark Gurney under
 * the sponsorship of the FreeBSD Foundation and
 * Rubicon Communications, LLC (Netgate).
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1.  Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 * 2.  Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *
 */

/*
 * Figure 5, 8 and 12 are copied from the Intel white paper:
 * Intel® Carry-Less Multiplication Instruction and its Usage for
 * Computing the GCM Mode
 *
 * and as such are:
 * Copyright © 2010 Intel Corporation.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *   * Neither the name of Intel Corporation nor the
 *     names of its contributors may be used to endorse or promote products
 *     derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifdef _KERNEL
#include <crypto/aesni/aesni.h>
#include <crypto/aesni/aesni_os.h>
#else
#include <stdint.h>
#endif

#include <wmmintrin.h>
#include <emmintrin.h>
#include <smmintrin.h>

static inline int
m128icmp(__m128i a, __m128i b)
{
	__m128i cmp;

	cmp = _mm_cmpeq_epi32(a, b);

	return _mm_movemask_epi8(cmp) == 0xffff;
}

#ifdef __i386__
static inline __m128i
_mm_insert_epi64(__m128i a, int64_t b, const int ndx)
{

	if (!ndx) {
		a = _mm_insert_epi32(a, b, 0);
		a = _mm_insert_epi32(a, b >> 32, 1);
	} else {
		a = _mm_insert_epi32(a, b, 2);
		a = _mm_insert_epi32(a, b >> 32, 3);
	}

	return a;
}
#endif

/* some code from carry-less-multiplication-instruction-in-gcm-mode-paper.pdf */

/* Figure 5. Code Sample - Performing Ghash Using Algorithms 1 and 5 (C) */
static void
gfmul(__m128i a, __m128i b, __m128i *res)
{
	__m128i tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9;

	tmp3 = _mm_clmulepi64_si128(a, b, 0x00);
	tmp4 = _mm_clmulepi64_si128(a, b, 0x10);
	tmp5 = _mm_clmulepi64_si128(a, b, 0x01);
	tmp6 = _mm_clmulepi64_si128(a, b, 0x11);

	tmp4 = _mm_xor_si128(tmp4, tmp5);
	tmp5 = _mm_slli_si128(tmp4, 8);
	tmp4 = _mm_srli_si128(tmp4, 8);
	tmp3 = _mm_xor_si128(tmp3, tmp5);
	tmp6 = _mm_xor_si128(tmp6, tmp4);

	tmp7 = _mm_srli_epi32(tmp3, 31);
	tmp8 = _mm_srli_epi32(tmp6, 31);
	tmp3 = _mm_slli_epi32(tmp3, 1);
	tmp6 = _mm_slli_epi32(tmp6, 1);

	tmp9 = _mm_srli_si128(tmp7, 12);
	tmp8 = _mm_slli_si128(tmp8, 4);
	tmp7 = _mm_slli_si128(tmp7, 4);
	tmp3 = _mm_or_si128(tmp3, tmp7);
	tmp6 = _mm_or_si128(tmp6, tmp8);
	tmp6 = _mm_or_si128(tmp6, tmp9);

	tmp7 = _mm_slli_epi32(tmp3, 31);
	tmp8 = _mm_slli_epi32(tmp3, 30);
	tmp9 = _mm_slli_epi32(tmp3, 25);

	tmp7 = _mm_xor_si128(tmp7, tmp8);
	tmp7 = _mm_xor_si128(tmp7, tmp9);
	tmp8 = _mm_srli_si128(tmp7, 4);
	tmp7 = _mm_slli_si128(tmp7, 12);
	tmp3 = _mm_xor_si128(tmp3, tmp7);

	tmp2 = _mm_srli_epi32(tmp3, 1);
	tmp4 = _mm_srli_epi32(tmp3, 2);
	tmp5 = _mm_srli_epi32(tmp3, 7);
	tmp2 = _mm_xor_si128(tmp2, tmp4);
	tmp2 = _mm_xor_si128(tmp2, tmp5);
	tmp2 = _mm_xor_si128(tmp2, tmp8);
	tmp3 = _mm_xor_si128(tmp3, tmp2);
	tmp6 = _mm_xor_si128(tmp6, tmp3);

	*res = tmp6;
}

/*
 * Figure 8. Code Sample - Performing Ghash Using an Aggregated Reduction
 * Method */
static void
reduce4(__m128i H1, __m128i H2, __m128i H3, __m128i H4,
    __m128i X1, __m128i X2, __m128i X3, __m128i X4, __m128i *res)
{
	/*algorithm by Krzysztof Jankowski, Pierre Laurent - Intel*/
	__m128i H1_X1_lo, H1_X1_hi, H2_X2_lo, H2_X2_hi, H3_X3_lo,
	    H3_X3_hi, H4_X4_lo, H4_X4_hi, lo, hi;
	__m128i tmp0, tmp1, tmp2, tmp3;
	__m128i tmp4, tmp5, tmp6, tmp7;
	__m128i tmp8, tmp9;

	H1_X1_lo = _mm_clmulepi64_si128(H1, X1, 0x00);
	H2_X2_lo = _mm_clmulepi64_si128(H2, X2, 0x00);
	H3_X3_lo = _mm_clmulepi64_si128(H3, X3, 0x00);
	H4_X4_lo = _mm_clmulepi64_si128(H4, X4, 0x00);

	lo = _mm_xor_si128(H1_X1_lo, H2_X2_lo);
	lo = _mm_xor_si128(lo, H3_X3_lo);
	lo = _mm_xor_si128(lo, H4_X4_lo);

	H1_X1_hi = _mm_clmulepi64_si128(H1, X1, 0x11);
	H2_X2_hi = _mm_clmulepi64_si128(H2, X2, 0x11);
	H3_X3_hi = _mm_clmulepi64_si128(H3, X3, 0x11);
	H4_X4_hi = _mm_clmulepi64_si128(H4, X4, 0x11);

	hi = _mm_xor_si128(H1_X1_hi, H2_X2_hi);
	hi = _mm_xor_si128(hi, H3_X3_hi);
	hi = _mm_xor_si128(hi, H4_X4_hi);

	tmp0 = _mm_shuffle_epi32(H1, 78);
	tmp4 = _mm_shuffle_epi32(X1, 78);
	tmp0 = _mm_xor_si128(tmp0, H1);
	tmp4 = _mm_xor_si128(tmp4, X1);
	tmp1 = _mm_shuffle_epi32(H2, 78);
	tmp5 = _mm_shuffle_epi32(X2, 78);
	tmp1 = _mm_xor_si128(tmp1, H2);
	tmp5 = _mm_xor_si128(tmp5, X2);
	tmp2 = _mm_shuffle_epi32(H3, 78);
	tmp6 = _mm_shuffle_epi32(X3, 78);
	tmp2 = _mm_xor_si128(tmp2, H3);
	tmp6 = _mm_xor_si128(tmp6, X3);
	tmp3 = _mm_shuffle_epi32(H4, 78);
	tmp7 = _mm_shuffle_epi32(X4, 78);
	tmp3 = _mm_xor_si128(tmp3, H4);
	tmp7 = _mm_xor_si128(tmp7, X4);

	tmp0 = _mm_clmulepi64_si128(tmp0, tmp4, 0x00);
	tmp1 = _mm_clmulepi64_si128(tmp1, tmp5, 0x00);
	tmp2 = _mm_clmulepi64_si128(tmp2, tmp6, 0x00);
	tmp3 = _mm_clmulepi64_si128(tmp3, tmp7, 0x00);

	tmp0 = _mm_xor_si128(tmp0, lo);
	tmp0 = _mm_xor_si128(tmp0, hi);
	tmp0 = _mm_xor_si128(tmp1, tmp0);
	tmp0 = _mm_xor_si128(tmp2, tmp0);
	tmp0 = _mm_xor_si128(tmp3, tmp0);

	tmp4 = _mm_slli_si128(tmp0, 8);
	tmp0 = _mm_srli_si128(tmp0, 8);

	lo = _mm_xor_si128(tmp4, lo);
	hi = _mm_xor_si128(tmp0, hi);

	tmp3 = lo;
	tmp6 = hi;

	tmp7 = _mm_srli_epi32(tmp3, 31);
	tmp8 = _mm_srli_epi32(tmp6, 31);
	tmp3 = _mm_slli_epi32(tmp3, 1);
	tmp6 = _mm_slli_epi32(tmp6, 1);

	tmp9 = _mm_srli_si128(tmp7, 12);
	tmp8 = _mm_slli_si128(tmp8, 4);
	tmp7 = _mm_slli_si128(tmp7, 4);
	tmp3 = _mm_or_si128(tmp3, tmp7);
	tmp6 = _mm_or_si128(tmp6, tmp8);
	tmp6 = _mm_or_si128(tmp6, tmp9);

	tmp7 = _mm_slli_epi32(tmp3, 31);
	tmp8 = _mm_slli_epi32(tmp3, 30);
	tmp9 = _mm_slli_epi32(tmp3, 25);

	tmp7 = _mm_xor_si128(tmp7, tmp8);
	tmp7 = _mm_xor_si128(tmp7, tmp9);
	tmp8 = _mm_srli_si128(tmp7, 4);
	tmp7 = _mm_slli_si128(tmp7, 12);
	tmp3 = _mm_xor_si128(tmp3, tmp7);

	tmp2 = _mm_srli_epi32(tmp3, 1);
	tmp4 = _mm_srli_epi32(tmp3, 2);
	tmp5 = _mm_srli_epi32(tmp3, 7);
	tmp2 = _mm_xor_si128(tmp2, tmp4);
	tmp2 = _mm_xor_si128(tmp2, tmp5);
	tmp2 = _mm_xor_si128(tmp2, tmp8);
	tmp3 = _mm_xor_si128(tmp3, tmp2);
	tmp6 = _mm_xor_si128(tmp6, tmp3);

	*res = tmp6;
}

/*
 * Figure 12. AES-GCM: Processing Four Blocks in Parallel with Aggregated
 * Every Four Blocks
 */
/*
 * per NIST SP-800-38D, 5.2.1.1, len(p) <= 2^39-256 (in bits), or
 * 2^32-256*8*16 bytes.
 */
void
AES_GCM_encrypt(const unsigned char *in, unsigned char *out,
	const unsigned char *addt, const unsigned char *ivec,
	unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
	const unsigned char *key, int nr)
{
	int i, j ,k;
	__m128i tmp1, tmp2, tmp3, tmp4;
	__m128i tmp5, tmp6, tmp7, tmp8;
	__m128i H, H2, H3, H4, Y, T;
	const __m128i *KEY = (const __m128i *)key;
	__m128i ctr1, ctr2, ctr3, ctr4;
	__m128i ctr5, ctr6, ctr7, ctr8;
	__m128i last_block = _mm_setzero_si128();
	__m128i ONE = _mm_set_epi32(0, 1, 0, 0);
	__m128i EIGHT = _mm_set_epi32(0, 8, 0, 0);
	__m128i BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,
	    7);
	__m128i BSWAP_MASK = _mm_set_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,
	    15);
	__m128i X = _mm_setzero_si128();

	if (ibytes == 96/8) {
		Y = _mm_loadu_si128((const __m128i *)ivec);
		Y = _mm_insert_epi32(Y, 0x1000000, 3);
		/*(Compute E[ZERO, KS] and E[Y0, KS] together*/
		tmp1 = _mm_xor_si128(X, KEY[0]);
		tmp2 = _mm_xor_si128(Y, KEY[0]);
		for (j=1; j < nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j]);

			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp2 = _mm_aesenc_si128(tmp2, KEY[nr-1]);

		H = _mm_aesenclast_si128(tmp1, KEY[nr]);
		T = _mm_aesenclast_si128(tmp2, KEY[nr]);

		H = _mm_shuffle_epi8(H, BSWAP_MASK);
	} else {
		tmp1 = _mm_xor_si128(X, KEY[0]);
		for (j=1; j <nr; j++)
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
		H = _mm_aesenclast_si128(tmp1, KEY[nr]);

		H = _mm_shuffle_epi8(H, BSWAP_MASK);
		Y = _mm_setzero_si128();

		for (i=0; i < ibytes/16; i++) {
			tmp1 = _mm_loadu_si128(&((const __m128i *)ivec)[i]);
			tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
			Y = _mm_xor_si128(Y, tmp1);
			gfmul(Y, H, &Y);
		}
		if (ibytes%16) {
			for (j=0; j < ibytes%16; j++)
				((unsigned char*)&last_block)[j] = ivec[i*16+j];
			tmp1 = last_block;
			tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
			Y = _mm_xor_si128(Y, tmp1);
			gfmul(Y, H, &Y);
		}
		tmp1 = _mm_insert_epi64(tmp1, (uint64_t)ibytes*8, 0);
		tmp1 = _mm_insert_epi64(tmp1, 0, 1);

		Y = _mm_xor_si128(Y, tmp1);
		gfmul(Y, H, &Y);
		Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /*Compute E(K, Y0)*/
		tmp1 = _mm_xor_si128(Y, KEY[0]);
		for (j=1; j < nr; j++)
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
		T = _mm_aesenclast_si128(tmp1, KEY[nr]);
	}

	gfmul(H,H,&H2);
	gfmul(H,H2,&H3);
	gfmul(H,H3,&H4);

	for (i=0; i<abytes/16/4; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)addt)[i*4]);
		tmp2 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+1]);
		tmp3 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+2]);
		tmp4 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+3]);

		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
		tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
		tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);
		tmp1 = _mm_xor_si128(X, tmp1);

		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);
	}
	for (i=i*4; i<abytes/16; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)addt)[i]);
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X,tmp1);
		gfmul(X, H, &X);
	}
	if (abytes%16) {
		last_block = _mm_setzero_si128();
		for (j=0; j<abytes%16; j++)
			((unsigned char*)&last_block)[j] = addt[i*16+j];
		tmp1 = last_block;
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X =_mm_xor_si128(X,tmp1);
		gfmul(X,H,&X);
	}

	ctr1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
	ctr1 = _mm_add_epi64(ctr1, ONE);
	ctr2 = _mm_add_epi64(ctr1, ONE);
	ctr3 = _mm_add_epi64(ctr2, ONE);
	ctr4 = _mm_add_epi64(ctr3, ONE);
	ctr5 = _mm_add_epi64(ctr4, ONE);
	ctr6 = _mm_add_epi64(ctr5, ONE);
	ctr7 = _mm_add_epi64(ctr6, ONE);
	ctr8 = _mm_add_epi64(ctr7, ONE);

	for (i=0; i<nbytes/16/8; i++) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		tmp2 = _mm_shuffle_epi8(ctr2, BSWAP_EPI64);
		tmp3 = _mm_shuffle_epi8(ctr3, BSWAP_EPI64);
		tmp4 = _mm_shuffle_epi8(ctr4, BSWAP_EPI64);
		tmp5 = _mm_shuffle_epi8(ctr5, BSWAP_EPI64);
		tmp6 = _mm_shuffle_epi8(ctr6, BSWAP_EPI64);
		tmp7 = _mm_shuffle_epi8(ctr7, BSWAP_EPI64);
		tmp8 = _mm_shuffle_epi8(ctr8, BSWAP_EPI64);

		ctr1 = _mm_add_epi64(ctr1, EIGHT);
		ctr2 = _mm_add_epi64(ctr2, EIGHT);
		ctr3 = _mm_add_epi64(ctr3, EIGHT);
		ctr4 = _mm_add_epi64(ctr4, EIGHT);
		ctr5 = _mm_add_epi64(ctr5, EIGHT);
		ctr6 = _mm_add_epi64(ctr6, EIGHT);
		ctr7 = _mm_add_epi64(ctr7, EIGHT);
		ctr8 = _mm_add_epi64(ctr8, EIGHT);

		tmp1 =_mm_xor_si128(tmp1, KEY[0]);
		tmp2 =_mm_xor_si128(tmp2, KEY[0]);
		tmp3 =_mm_xor_si128(tmp3, KEY[0]);
		tmp4 =_mm_xor_si128(tmp4, KEY[0]);
		tmp5 =_mm_xor_si128(tmp5, KEY[0]);
		tmp6 =_mm_xor_si128(tmp6, KEY[0]);
		tmp7 =_mm_xor_si128(tmp7, KEY[0]);
		tmp8 =_mm_xor_si128(tmp8, KEY[0]);

		for (j=1; j<nr; j++) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j]);
			tmp3 = _mm_aesenc_si128(tmp3, KEY[j]);
			tmp4 = _mm_aesenc_si128(tmp4, KEY[j]);
			tmp5 = _mm_aesenc_si128(tmp5, KEY[j]);
			tmp6 = _mm_aesenc_si128(tmp6, KEY[j]);
			tmp7 = _mm_aesenc_si128(tmp7, KEY[j]);
			tmp8 = _mm_aesenc_si128(tmp8, KEY[j]);
		}
		tmp1 =_mm_aesenclast_si128(tmp1, KEY[nr]);
		tmp2 =_mm_aesenclast_si128(tmp2, KEY[nr]);
		tmp3 =_mm_aesenclast_si128(tmp3, KEY[nr]);
		tmp4 =_mm_aesenclast_si128(tmp4, KEY[nr]);
		tmp5 =_mm_aesenclast_si128(tmp5, KEY[nr]);
		tmp6 =_mm_aesenclast_si128(tmp6, KEY[nr]);
		tmp7 =_mm_aesenclast_si128(tmp7, KEY[nr]);
		tmp8 =_mm_aesenclast_si128(tmp8, KEY[nr]);

		tmp1 = _mm_xor_si128(tmp1,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+0]));
		tmp2 = _mm_xor_si128(tmp2,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+1]));
		tmp3 = _mm_xor_si128(tmp3,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+2]));
		tmp4 = _mm_xor_si128(tmp4,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+3]));
		tmp5 = _mm_xor_si128(tmp5,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+4]));
		tmp6 = _mm_xor_si128(tmp6,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+5]));
		tmp7 = _mm_xor_si128(tmp7,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+6]));
		tmp8 = _mm_xor_si128(tmp8,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+7]));

		_mm_storeu_si128(&((__m128i*)out)[i*8+0], tmp1);
		_mm_storeu_si128(&((__m128i*)out)[i*8+1], tmp2);
		_mm_storeu_si128(&((__m128i*)out)[i*8+2], tmp3);
		_mm_storeu_si128(&((__m128i*)out)[i*8+3], tmp4);
		_mm_storeu_si128(&((__m128i*)out)[i*8+4], tmp5);
		_mm_storeu_si128(&((__m128i*)out)[i*8+5], tmp6);
		_mm_storeu_si128(&((__m128i*)out)[i*8+6], tmp7);
		_mm_storeu_si128(&((__m128i*)out)[i*8+7], tmp8);

		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
		tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
		tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);
		tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_MASK);
		tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_MASK);
		tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_MASK);
		tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_MASK);

		tmp1 = _mm_xor_si128(X, tmp1);

		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);

		tmp5 = _mm_xor_si128(X, tmp5);
		reduce4(H, H2, H3, H4, tmp8, tmp7, tmp6, tmp5, &X);
	}
	for (k=i*8; k<nbytes/16; k++) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		ctr1 = _mm_add_epi64(ctr1, ONE);
		tmp1 = _mm_xor_si128(tmp1, KEY[0]);
		for (j=1; j<nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp1 = _mm_aesenclast_si128(tmp1, KEY[nr]);
		tmp1 = _mm_xor_si128(tmp1,
		    _mm_loadu_si128(&((const __m128i *)in)[k]));
		_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X, tmp1);
		gfmul(X,H,&X);
	}
	//If remains one incomplete block
	if (nbytes%16) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		tmp1 = _mm_xor_si128(tmp1, KEY[0]);
		for (j=1; j<nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp1 = _mm_aesenclast_si128(tmp1, KEY[nr]);
		last_block = _mm_setzero_si128();
		memcpy(&last_block, &((const __m128i *)in)[k],
		    nbytes % 16);
		last_block = _mm_xor_si128(last_block, tmp1);
		for (j=0; j<nbytes%16; j++)
			out[k*16+j] = ((unsigned char*)&last_block)[j];
		for ((void)j; j<16; j++)
			((unsigned char*)&last_block)[j] = 0;
		tmp1 = last_block;
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X, tmp1);
		gfmul(X, H, &X);
	}
	tmp1 = _mm_insert_epi64(tmp1, (uint64_t)nbytes*8, 0);
	tmp1 = _mm_insert_epi64(tmp1, (uint64_t)abytes*8, 1);

	X = _mm_xor_si128(X, tmp1);
	gfmul(X,H,&X);
	X = _mm_shuffle_epi8(X, BSWAP_MASK);
	T = _mm_xor_si128(X, T);
	_mm_storeu_si128((__m128i*)tag, T);
}

/* My modification of _encrypt to be _decrypt */
int
AES_GCM_decrypt(const unsigned char *in, unsigned char *out,
	const unsigned char *addt, const unsigned char *ivec,
	const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
	const unsigned char *key, int nr)
{
	int i, j ,k;
	__m128i tmp1, tmp2, tmp3, tmp4;
	__m128i tmp5, tmp6, tmp7, tmp8;
	__m128i H, H2, H3, H4, Y, T;
	const __m128i *KEY = (const __m128i *)key;
	__m128i ctr1, ctr2, ctr3, ctr4;
	__m128i ctr5, ctr6, ctr7, ctr8;
	__m128i last_block = _mm_setzero_si128();
	__m128i ONE = _mm_set_epi32(0, 1, 0, 0);
	__m128i EIGHT = _mm_set_epi32(0, 8, 0, 0);
	__m128i BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,
	    7);
	__m128i BSWAP_MASK = _mm_set_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,
	    15);
	__m128i X = _mm_setzero_si128();

	if (ibytes == 96/8) {
		Y = _mm_loadu_si128((const __m128i *)ivec);
		Y = _mm_insert_epi32(Y, 0x1000000, 3);
		/*(Compute E[ZERO, KS] and E[Y0, KS] together*/
		tmp1 = _mm_xor_si128(X, KEY[0]);
		tmp2 = _mm_xor_si128(Y, KEY[0]);
		for (j=1; j < nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j]);

			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp2 = _mm_aesenc_si128(tmp2, KEY[nr-1]);

		H = _mm_aesenclast_si128(tmp1, KEY[nr]);
		T = _mm_aesenclast_si128(tmp2, KEY[nr]);

		H = _mm_shuffle_epi8(H, BSWAP_MASK);
	} else {
		tmp1 = _mm_xor_si128(X, KEY[0]);
		for (j=1; j <nr; j++)
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
		H = _mm_aesenclast_si128(tmp1, KEY[nr]);

		H = _mm_shuffle_epi8(H, BSWAP_MASK);
		Y = _mm_setzero_si128();

		for (i=0; i < ibytes/16; i++) {
			tmp1 = _mm_loadu_si128(&((const __m128i *)ivec)[i]);
			tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
			Y = _mm_xor_si128(Y, tmp1);
			gfmul(Y, H, &Y);
		}
		if (ibytes%16) {
			for (j=0; j < ibytes%16; j++)
				((unsigned char*)&last_block)[j] = ivec[i*16+j];
			tmp1 = last_block;
			tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
			Y = _mm_xor_si128(Y, tmp1);
			gfmul(Y, H, &Y);
		}
		tmp1 = _mm_insert_epi64(tmp1, (uint64_t)ibytes*8, 0);
		tmp1 = _mm_insert_epi64(tmp1, 0, 1);

		Y = _mm_xor_si128(Y, tmp1);
		gfmul(Y, H, &Y);
		Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /*Compute E(K, Y0)*/
		tmp1 = _mm_xor_si128(Y, KEY[0]);
		for (j=1; j < nr; j++)
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
		T = _mm_aesenclast_si128(tmp1, KEY[nr]);
	}

	gfmul(H,H,&H2);
	gfmul(H,H2,&H3);
	gfmul(H,H3,&H4);

	for (i=0; i<abytes/16/4; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)addt)[i*4]);
		tmp2 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+1]);
		tmp3 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+2]);
		tmp4 = _mm_loadu_si128(&((const __m128i *)addt)[i*4+3]);

		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
		tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
		tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);

		tmp1 = _mm_xor_si128(X, tmp1);

		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);
	}
	for (i=i*4; i<abytes/16; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)addt)[i]);
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X,tmp1);
		gfmul(X, H, &X);
	}
	if (abytes%16) {
		last_block = _mm_setzero_si128();
		for (j=0; j<abytes%16; j++)
			((unsigned char*)&last_block)[j] = addt[i*16+j];
		tmp1 = last_block;
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X =_mm_xor_si128(X,tmp1);
		gfmul(X,H,&X);
	}

	/* This is where we validate the cipher text before decrypt */
	for (i = 0; i<nbytes/16/4; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)in)[i*4]);
		tmp2 = _mm_loadu_si128(&((const __m128i *)in)[i*4+1]);
		tmp3 = _mm_loadu_si128(&((const __m128i *)in)[i*4+2]);
		tmp4 = _mm_loadu_si128(&((const __m128i *)in)[i*4+3]);

		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
		tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
		tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);

		tmp1 = _mm_xor_si128(X, tmp1);

		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);
	}
	for (i = i*4; i<nbytes/16; i++) {
		tmp1 = _mm_loadu_si128(&((const __m128i *)in)[i]);
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X, tmp1);
		gfmul(X,H,&X);
	}
	if (nbytes%16) {
		last_block = _mm_setzero_si128();
		for (j=0; j<nbytes%16; j++)
			((unsigned char*)&last_block)[j] = in[i*16+j];
		tmp1 = last_block;
		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		X = _mm_xor_si128(X, tmp1);
		gfmul(X, H, &X);
	}

	tmp1 = _mm_insert_epi64(tmp1, (uint64_t)nbytes*8, 0);
	tmp1 = _mm_insert_epi64(tmp1, (uint64_t)abytes*8, 1);

	X = _mm_xor_si128(X, tmp1);
	gfmul(X,H,&X);
	X = _mm_shuffle_epi8(X, BSWAP_MASK);
	T = _mm_xor_si128(X, T);

	if (!m128icmp(T, _mm_loadu_si128((const __m128i*)tag)))
		return 0; //in case the authentication failed

	ctr1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
	ctr1 = _mm_add_epi64(ctr1, ONE);
	ctr2 = _mm_add_epi64(ctr1, ONE);
	ctr3 = _mm_add_epi64(ctr2, ONE);
	ctr4 = _mm_add_epi64(ctr3, ONE);
	ctr5 = _mm_add_epi64(ctr4, ONE);
	ctr6 = _mm_add_epi64(ctr5, ONE);
	ctr7 = _mm_add_epi64(ctr6, ONE);
	ctr8 = _mm_add_epi64(ctr7, ONE);

	for (i=0; i<nbytes/16/8; i++) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		tmp2 = _mm_shuffle_epi8(ctr2, BSWAP_EPI64);
		tmp3 = _mm_shuffle_epi8(ctr3, BSWAP_EPI64);
		tmp4 = _mm_shuffle_epi8(ctr4, BSWAP_EPI64);
		tmp5 = _mm_shuffle_epi8(ctr5, BSWAP_EPI64);
		tmp6 = _mm_shuffle_epi8(ctr6, BSWAP_EPI64);
		tmp7 = _mm_shuffle_epi8(ctr7, BSWAP_EPI64);
		tmp8 = _mm_shuffle_epi8(ctr8, BSWAP_EPI64);

		ctr1 = _mm_add_epi64(ctr1, EIGHT);
		ctr2 = _mm_add_epi64(ctr2, EIGHT);
		ctr3 = _mm_add_epi64(ctr3, EIGHT);
		ctr4 = _mm_add_epi64(ctr4, EIGHT);
		ctr5 = _mm_add_epi64(ctr5, EIGHT);
		ctr6 = _mm_add_epi64(ctr6, EIGHT);
		ctr7 = _mm_add_epi64(ctr7, EIGHT);
		ctr8 = _mm_add_epi64(ctr8, EIGHT);

		tmp1 =_mm_xor_si128(tmp1, KEY[0]);
		tmp2 =_mm_xor_si128(tmp2, KEY[0]);
		tmp3 =_mm_xor_si128(tmp3, KEY[0]);
		tmp4 =_mm_xor_si128(tmp4, KEY[0]);
		tmp5 =_mm_xor_si128(tmp5, KEY[0]);
		tmp6 =_mm_xor_si128(tmp6, KEY[0]);
		tmp7 =_mm_xor_si128(tmp7, KEY[0]);
		tmp8 =_mm_xor_si128(tmp8, KEY[0]);

		for (j=1; j<nr; j++) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp2 = _mm_aesenc_si128(tmp2, KEY[j]);
			tmp3 = _mm_aesenc_si128(tmp3, KEY[j]);
			tmp4 = _mm_aesenc_si128(tmp4, KEY[j]);
			tmp5 = _mm_aesenc_si128(tmp5, KEY[j]);
			tmp6 = _mm_aesenc_si128(tmp6, KEY[j]);
			tmp7 = _mm_aesenc_si128(tmp7, KEY[j]);
			tmp8 = _mm_aesenc_si128(tmp8, KEY[j]);
		}
		tmp1 =_mm_aesenclast_si128(tmp1, KEY[nr]);
		tmp2 =_mm_aesenclast_si128(tmp2, KEY[nr]);
		tmp3 =_mm_aesenclast_si128(tmp3, KEY[nr]);
		tmp4 =_mm_aesenclast_si128(tmp4, KEY[nr]);
		tmp5 =_mm_aesenclast_si128(tmp5, KEY[nr]);
		tmp6 =_mm_aesenclast_si128(tmp6, KEY[nr]);
		tmp7 =_mm_aesenclast_si128(tmp7, KEY[nr]);
		tmp8 =_mm_aesenclast_si128(tmp8, KEY[nr]);

		tmp1 = _mm_xor_si128(tmp1,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+0]));
		tmp2 = _mm_xor_si128(tmp2,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+1]));
		tmp3 = _mm_xor_si128(tmp3,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+2]));
		tmp4 = _mm_xor_si128(tmp4,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+3]));
		tmp5 = _mm_xor_si128(tmp5,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+4]));
		tmp6 = _mm_xor_si128(tmp6,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+5]));
		tmp7 = _mm_xor_si128(tmp7,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+6]));
		tmp8 = _mm_xor_si128(tmp8,
		    _mm_loadu_si128(&((const __m128i *)in)[i*8+7]));

		_mm_storeu_si128(&((__m128i*)out)[i*8+0], tmp1);
		_mm_storeu_si128(&((__m128i*)out)[i*8+1], tmp2);
		_mm_storeu_si128(&((__m128i*)out)[i*8+2], tmp3);
		_mm_storeu_si128(&((__m128i*)out)[i*8+3], tmp4);
		_mm_storeu_si128(&((__m128i*)out)[i*8+4], tmp5);
		_mm_storeu_si128(&((__m128i*)out)[i*8+5], tmp6);
		_mm_storeu_si128(&((__m128i*)out)[i*8+6], tmp7);
		_mm_storeu_si128(&((__m128i*)out)[i*8+7], tmp8);

		tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
		tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
		tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
		tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);
		tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_MASK);
		tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_MASK);
		tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_MASK);
		tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_MASK);
	}
	for (k=i*8; k<nbytes/16; k++) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		ctr1 = _mm_add_epi64(ctr1, ONE);
		tmp1 = _mm_xor_si128(tmp1, KEY[0]);
		for (j=1; j<nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp1 = _mm_aesenclast_si128(tmp1, KEY[nr]);
		tmp1 = _mm_xor_si128(tmp1,
		    _mm_loadu_si128(&((const __m128i *)in)[k]));
		_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
	}
	//If remains one incomplete block
	if (nbytes%16) {
		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
		tmp1 = _mm_xor_si128(tmp1, KEY[0]);
		for (j=1; j<nr-1; j+=2) {
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j]);
			tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]);
		}
		tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]);
		tmp1 = _mm_aesenclast_si128(tmp1, KEY[nr]);
		last_block = _mm_setzero_si128();
		memcpy(&last_block, &((const __m128i *)in)[k], nbytes%16);
		tmp1 = _mm_xor_si128(tmp1, last_block);
		last_block = tmp1;
		for (j=0; j<nbytes%16; j++)
			out[k*16+j] = ((unsigned char*)&last_block)[j];
	}
	return 1; //when sucessfull returns 1
}