/* SPDX-License-Identifier: GPL-2.0 */ #include #include #include #include #include #define STATE0 %v0 #define STATE1 %v1 #define STATE2 %v2 #define STATE3 %v3 #define COPY0 %v4 #define COPY1 %v5 #define COPY2 %v6 #define COPY3 %v7 #define BEPERM %v19 #define TMP0 %v20 #define TMP1 %v21 #define TMP2 %v22 #define TMP3 %v23 .section .rodata .balign 32 SYM_DATA_START_LOCAL(chacha20_constants) .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap SYM_DATA_END(chacha20_constants) .text /* * s390 ChaCha20 implementation meant for vDSO. Produces a given positive * number of blocks of output with nonce 0, taking an input key and 8-bytes * counter. Does not spill to the stack. * * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, * const uint8_t *key, * uint32_t *counter, * size_t nblocks) */ SYM_FUNC_START(__arch_chacha20_blocks_nostack) CFI_STARTPROC larl %r1,chacha20_constants /* COPY0 = "expand 32-byte k" */ VL COPY0,0,,%r1 /* BEPERM = byte selectors for VPERM */ ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148) /* COPY1,COPY2 = key */ VLM COPY1,COPY2,0,%r3 /* COPY3 = counter || zero nonce */ lg %r3,0(%r4) VZERO COPY3 VLVGG COPY3,%r3,0 lghi %r1,0 .Lblock: VLR STATE0,COPY0 VLR STATE1,COPY1 VLR STATE2,COPY2 VLR STATE3,COPY3 lghi %r0,10 .Ldoubleround: /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ VAF STATE0,STATE0,STATE1 VX STATE3,STATE3,STATE0 VERLLF STATE3,STATE3,16 /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ VAF STATE2,STATE2,STATE3 VX STATE1,STATE1,STATE2 VERLLF STATE1,STATE1,12 /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ VAF STATE0,STATE0,STATE1 VX STATE3,STATE3,STATE0 VERLLF STATE3,STATE3,8 /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ VAF STATE2,STATE2,STATE3 VX STATE1,STATE1,STATE2 VERLLF STATE1,STATE1,7 /* STATE1[0,1,2,3] = STATE1[1,2,3,0] */ VSLDB STATE1,STATE1,STATE1,4 /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ VSLDB STATE2,STATE2,STATE2,8 /* STATE3[0,1,2,3] = STATE3[3,0,1,2] */ VSLDB STATE3,STATE3,STATE3,12 /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ VAF STATE0,STATE0,STATE1 VX STATE3,STATE3,STATE0 VERLLF STATE3,STATE3,16 /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ VAF STATE2,STATE2,STATE3 VX STATE1,STATE1,STATE2 VERLLF STATE1,STATE1,12 /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ VAF STATE0,STATE0,STATE1 VX STATE3,STATE3,STATE0 VERLLF STATE3,STATE3,8 /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ VAF STATE2,STATE2,STATE3 VX STATE1,STATE1,STATE2 VERLLF STATE1,STATE1,7 /* STATE1[0,1,2,3] = STATE1[3,0,1,2] */ VSLDB STATE1,STATE1,STATE1,12 /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ VSLDB STATE2,STATE2,STATE2,8 /* STATE3[0,1,2,3] = STATE3[1,2,3,0] */ VSLDB STATE3,STATE3,STATE3,4 brctg %r0,.Ldoubleround /* OUTPUT0 = STATE0 + COPY0 */ VAF STATE0,STATE0,COPY0 /* OUTPUT1 = STATE1 + COPY1 */ VAF STATE1,STATE1,COPY1 /* OUTPUT2 = STATE2 + COPY2 */ VAF STATE2,STATE2,COPY2 /* OUTPUT3 = STATE3 + COPY3 */ VAF STATE3,STATE3,COPY3 ALTERNATIVE \ __stringify( \ /* Convert STATE to little endian and store to OUTPUT */\ VPERM TMP0,STATE0,STATE0,BEPERM; \ VPERM TMP1,STATE1,STATE1,BEPERM; \ VPERM TMP2,STATE2,STATE2,BEPERM; \ VPERM TMP3,STATE3,STATE3,BEPERM; \ VSTM TMP0,TMP3,0,%r2), \ __stringify( \ /* 32 bit wise little endian store to OUTPUT */ \ VSTBRF STATE0,0,,%r2; \ VSTBRF STATE1,16,,%r2; \ VSTBRF STATE2,32,,%r2; \ VSTBRF STATE3,48,,%r2; \ brcl 0,0), \ ALT_FACILITY(148) /* ++COPY3.COUNTER */ /* alsih %r3,1 */ .insn rilu,0xcc0a00000000,%r3,1 alcr %r3,%r1 VLVGG COPY3,%r3,0 /* OUTPUT += 64, --NBLOCKS */ aghi %r2,64 brctg %r5,.Lblock /* COUNTER = COPY3.COUNTER */ stg %r3,0(%r4) /* Zero out potentially sensitive regs */ VZERO STATE0 VZERO STATE1 VZERO STATE2 VZERO STATE3 VZERO COPY1 VZERO COPY2 /* Early exit if TMP0-TMP3 have not been used */ ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148) VZERO TMP0 VZERO TMP1 VZERO TMP2 VZERO TMP3 br %r14 CFI_ENDPROC SYM_FUNC_END(__arch_chacha20_blocks_nostack)