67 #include <crypto/aesni/aesni.h>
68 #include <crypto/aesni/aesni_os.h>
70 #include <stdint.h>
73 #include <wmmintrin.h>
74 #include <emmintrin.h>
75 #include <smmintrin.h>
189 	tmp0 = _mm_shuffle_epi32(H1, 78);  in reduce4()
190 	tmp4 = _mm_shuffle_epi32(X1, 78);  in reduce4()
193 	tmp1 = _mm_shuffle_epi32(H2, 78);  in reduce4()
194 	tmp5 = _mm_shuffle_epi32(X2, 78);  in reduce4()
197 	tmp2 = _mm_shuffle_epi32(H3, 78);  in reduce4()
198 	tmp6 = _mm_shuffle_epi32(X3, 78);  in reduce4()
201 	tmp3 = _mm_shuffle_epi32(H4, 78);  in reduce4()
202 	tmp7 = _mm_shuffle_epi32(X4, 78);  in reduce4()
277 	__m128i H, H2, H3, H4, Y, T;  in AES_GCM_encrypt()  local
306 		H = _mm_aesenclast_si128(tmp1, KEY[nr]);  in AES_GCM_encrypt()
309 		H = _mm_shuffle_epi8(H, BSWAP_MASK);  in AES_GCM_encrypt()
314 		H = _mm_aesenclast_si128(tmp1, KEY[nr]);  in AES_GCM_encrypt()
316 		H = _mm_shuffle_epi8(H, BSWAP_MASK);  in AES_GCM_encrypt()
323 			gfmul(Y, H, &Y);  in AES_GCM_encrypt()
331 			gfmul(Y, H, &Y);  in AES_GCM_encrypt()
337 		gfmul(Y, H, &Y);  in AES_GCM_encrypt()
345 	gfmul(H,H,&H2);  in AES_GCM_encrypt()
346 	gfmul(H,H2,&H3);  in AES_GCM_encrypt()
347 	gfmul(H,H3,&H4);  in AES_GCM_encrypt()
361 		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);  in AES_GCM_encrypt()
367 		gfmul(X, H, &X);  in AES_GCM_encrypt()
376 		gfmul(X,H,&X);  in AES_GCM_encrypt()
473 		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);  in AES_GCM_encrypt()
476 		reduce4(H, H2, H3, H4, tmp8, tmp7, tmp6, tmp5, &X);  in AES_GCM_encrypt()
493 		gfmul(X,H,&X);  in AES_GCM_encrypt()
516 		gfmul(X, H, &X);  in AES_GCM_encrypt()
522 	gfmul(X,H,&X);  in AES_GCM_encrypt()
538 	__m128i H, H2, H3, H4, Y, T;  in AES_GCM_decrypt()  local
567 		H = _mm_aesenclast_si128(tmp1, KEY[nr]);  in AES_GCM_decrypt()
570 		H = _mm_shuffle_epi8(H, BSWAP_MASK);  in AES_GCM_decrypt()
575 		H = _mm_aesenclast_si128(tmp1, KEY[nr]);  in AES_GCM_decrypt()
577 		H = _mm_shuffle_epi8(H, BSWAP_MASK);  in AES_GCM_decrypt()
584 			gfmul(Y, H, &Y);  in AES_GCM_decrypt()
592 			gfmul(Y, H, &Y);  in AES_GCM_decrypt()
598 		gfmul(Y, H, &Y);  in AES_GCM_decrypt()
606 	gfmul(H,H,&H2);  in AES_GCM_decrypt()
607 	gfmul(H,H2,&H3);  in AES_GCM_decrypt()
608 	gfmul(H,H3,&H4);  in AES_GCM_decrypt()
623 		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);  in AES_GCM_decrypt()
629 		gfmul(X, H, &X);  in AES_GCM_decrypt()
638 		gfmul(X,H,&X);  in AES_GCM_decrypt()
655 		reduce4(H, H2, H3, H4, tmp4, tmp3, tmp2, tmp1, &X);  in AES_GCM_decrypt()
661 		gfmul(X,H,&X);  in AES_GCM_decrypt()
670 		gfmul(X, H, &X);  in AES_GCM_decrypt()
677 	gfmul(X,H,&X);  in AES_GCM_decrypt()