1/* 2 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 3 * 4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11#include <linux/linkage.h> 12#include <asm/assembler.h> 13 14 SHASH .req v0 15 SHASH2 .req v1 16 T1 .req v2 17 T2 .req v3 18 MASK .req v4 19 XL .req v5 20 XM .req v6 21 XH .req v7 22 IN1 .req v7 23 24 .text 25 .arch armv8-a+crypto 26 27 /* 28 * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 29 * struct ghash_key const *k, const char *head) 30 */ 31ENTRY(pmull_ghash_update) 32 ld1 {SHASH.2d}, [x3] 33 ld1 {XL.2d}, [x1] 34 movi MASK.16b, #0xe1 35 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 36 shl MASK.2d, MASK.2d, #57 37 eor SHASH2.16b, SHASH2.16b, SHASH.16b 38 39 /* do the head block first, if supplied */ 40 cbz x4, 0f 41 ld1 {T1.2d}, [x4] 42 b 1f 43 440: ld1 {T1.2d}, [x2], #16 45 sub w0, w0, #1 46 471: /* multiply XL by SHASH in GF(2^128) */ 48CPU_LE( rev64 T1.16b, T1.16b ) 49 50 ext T2.16b, XL.16b, XL.16b, #8 51 ext IN1.16b, T1.16b, T1.16b, #8 52 eor T1.16b, T1.16b, T2.16b 53 eor XL.16b, XL.16b, IN1.16b 54 55 pmull2 XH.1q, SHASH.2d, XL.2d // a1 * b1 56 eor T1.16b, T1.16b, XL.16b 57 pmull XL.1q, SHASH.1d, XL.1d // a0 * b0 58 pmull XM.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0) 59 60 ext T1.16b, XL.16b, XH.16b, #8 61 eor T2.16b, XL.16b, XH.16b 62 eor XM.16b, XM.16b, T1.16b 63 eor XM.16b, XM.16b, T2.16b 64 pmull T2.1q, XL.1d, MASK.1d 65 66 mov XH.d[0], XM.d[1] 67 mov XM.d[1], XL.d[0] 68 69 eor XL.16b, XM.16b, T2.16b 70 ext T2.16b, XL.16b, XL.16b, #8 71 pmull XL.1q, XL.1d, MASK.1d 72 eor T2.16b, T2.16b, XH.16b 73 eor XL.16b, XL.16b, T2.16b 74 75 cbnz w0, 0b 76 77 st1 {XL.2d}, [x1] 78 ret 79ENDPROC(pmull_ghash_update) 80