1#! /usr/bin/env perl 2# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the Apache License 2.0 (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9 10# ==================================================================== 11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 12# project. The module is, however, dual licensed under OpenSSL and 13# CRYPTOGAMS licenses depending on where you obtain it. For further 14# details see http://www.openssl.org/~appro/cryptogams/. 15# ==================================================================== 16 17# September 2010. 18# 19# The module implements "4-bit" GCM GHASH function and underlying 20# single multiplication operation in GF(2^128). "4-bit" means that it 21# uses 256 bytes per-key table [+128 bytes shared table]. Performance 22# was measured to be ~18 cycles per processed byte on z10, which is 23# almost 40% better than gcc-generated code. It should be noted that 24# 18 cycles is worse result than expected: loop is scheduled for 12 25# and the result should be close to 12. In the lack of instruction- 26# level profiling data it's impossible to tell why... 27 28# November 2010. 29# 30# Adapt for -m31 build. If kernel supports what's called "highgprs" 31# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit 32# instructions and achieve "64-bit" performance even in 31-bit legacy 33# application context. The feature is not specific to any particular 34# processor, as long as it's "z-CPU". Latter implies that the code 35# remains z/Architecture specific. On z990 it was measured to perform 36# 2.8x better than 32-bit code generated by gcc 4.3. 37 38# March 2011. 39# 40# Support for hardware KIMD-GHASH is verified to produce correct 41# result and therefore is engaged. On z196 it was measured to process 42# 8KB buffer ~7 faster than software implementation. It's not as 43# impressive for smaller buffer sizes and for smallest 16-bytes buffer 44# it's actually almost 2 times slower. Which is the reason why 45# KIMD-GHASH is not used in gcm_gmult_4bit. 46 47# $output is the last argument if it looks like a file (it has an extension) 48# $flavour is the first argument if it doesn't look like a file 49$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; 50$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; 51 52if ($flavour =~ /3[12]/) { 53 $SIZE_T=4; 54 $g=""; 55} else { 56 $SIZE_T=8; 57 $g="g"; 58} 59 60$output and open STDOUT,">$output"; 61 62$softonly=0; 63 64$Zhi="%r0"; 65$Zlo="%r1"; 66 67$Xi="%r2"; # argument block 68$Htbl="%r3"; 69$inp="%r4"; 70$len="%r5"; 71 72$rem0="%r6"; # variables 73$rem1="%r7"; 74$nlo="%r8"; 75$nhi="%r9"; 76$xi="%r10"; 77$cnt="%r11"; 78$tmp="%r12"; 79$x78="%r13"; 80$rem_4bit="%r14"; 81 82$sp="%r15"; 83 84$code.=<<___; 85#include "s390x_arch.h" 86 87.text 88 89.globl gcm_gmult_4bit 90.align 32 91gcm_gmult_4bit: 92___ 93$code.=<<___ if(!$softonly && 0); # hardware is slow for single block... 94 larl %r1,OPENSSL_s390xcap_P 95 lghi %r0,0 96 lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities 97 # vector 98 tmhh %r1,0x4000 # check for function 65 99 jz .Lsoft_gmult 100 stg %r0,16($sp) # arrange 16 bytes of zero input 101 stg %r0,24($sp) 102 lghi %r0,S390X_GHASH # function 65 103 la %r1,0($Xi) # H lies right after Xi in gcm128_context 104 la $inp,16($sp) 105 lghi $len,16 106 .long 0xb93e0004 # kimd %r0,$inp 107 brc 1,.-4 # pay attention to "partial completion" 108 br %r14 109.align 32 110.Lsoft_gmult: 111___ 112$code.=<<___; 113 stm${g} %r6,%r14,6*$SIZE_T($sp) 114 115 aghi $Xi,-1 116 lghi $len,1 117 lghi $x78,`0xf<<3` 118 larl $rem_4bit,rem_4bit 119 120 lg $Zlo,8+1($Xi) # Xi 121 j .Lgmult_shortcut 122.type gcm_gmult_4bit,\@function 123.size gcm_gmult_4bit,(.-gcm_gmult_4bit) 124 125.globl gcm_ghash_4bit 126.align 32 127gcm_ghash_4bit: 128___ 129$code.=<<___ if(!$softonly); 130 larl %r1,OPENSSL_s390xcap_P 131 lg %r0,S390X_KIMD+8(%r1) # load second word of kimd capabilities 132 # vector 133 tmhh %r0,0x4000 # check for function 65 134 jz .Lsoft_ghash 135 lghi %r0,S390X_GHASH # function 65 136 la %r1,0($Xi) # H lies right after Xi in gcm128_context 137 .long 0xb93e0004 # kimd %r0,$inp 138 brc 1,.-4 # pay attention to "partial completion" 139 br %r14 140.align 32 141.Lsoft_ghash: 142___ 143$code.=<<___ if ($flavour =~ /3[12]/); 144 llgfr $len,$len 145___ 146$code.=<<___; 147 stm${g} %r6,%r14,6*$SIZE_T($sp) 148 149 aghi $Xi,-1 150 srlg $len,$len,4 151 lghi $x78,`0xf<<3` 152 larl $rem_4bit,rem_4bit 153 154 lg $Zlo,8+1($Xi) # Xi 155 lg $Zhi,0+1($Xi) 156 lghi $tmp,0 157.Louter: 158 xg $Zhi,0($inp) # Xi ^= inp 159 xg $Zlo,8($inp) 160 xgr $Zhi,$tmp 161 stg $Zlo,8+1($Xi) 162 stg $Zhi,0+1($Xi) 163 164.Lgmult_shortcut: 165 lghi $tmp,0xf0 166 sllg $nlo,$Zlo,4 167 srlg $xi,$Zlo,8 # extract second byte 168 ngr $nlo,$tmp 169 lgr $nhi,$Zlo 170 lghi $cnt,14 171 ngr $nhi,$tmp 172 173 lg $Zlo,8($nlo,$Htbl) 174 lg $Zhi,0($nlo,$Htbl) 175 176 sllg $nlo,$xi,4 177 sllg $rem0,$Zlo,3 178 ngr $nlo,$tmp 179 ngr $rem0,$x78 180 ngr $xi,$tmp 181 182 sllg $tmp,$Zhi,60 183 srlg $Zlo,$Zlo,4 184 srlg $Zhi,$Zhi,4 185 xg $Zlo,8($nhi,$Htbl) 186 xg $Zhi,0($nhi,$Htbl) 187 lgr $nhi,$xi 188 sllg $rem1,$Zlo,3 189 xgr $Zlo,$tmp 190 ngr $rem1,$x78 191 sllg $tmp,$Zhi,60 192 j .Lghash_inner 193.align 16 194.Lghash_inner: 195 srlg $Zlo,$Zlo,4 196 srlg $Zhi,$Zhi,4 197 xg $Zlo,8($nlo,$Htbl) 198 llgc $xi,0($cnt,$Xi) 199 xg $Zhi,0($nlo,$Htbl) 200 sllg $nlo,$xi,4 201 xg $Zhi,0($rem0,$rem_4bit) 202 nill $nlo,0xf0 203 sllg $rem0,$Zlo,3 204 xgr $Zlo,$tmp 205 ngr $rem0,$x78 206 nill $xi,0xf0 207 208 sllg $tmp,$Zhi,60 209 srlg $Zlo,$Zlo,4 210 srlg $Zhi,$Zhi,4 211 xg $Zlo,8($nhi,$Htbl) 212 xg $Zhi,0($nhi,$Htbl) 213 lgr $nhi,$xi 214 xg $Zhi,0($rem1,$rem_4bit) 215 sllg $rem1,$Zlo,3 216 xgr $Zlo,$tmp 217 ngr $rem1,$x78 218 sllg $tmp,$Zhi,60 219 brct $cnt,.Lghash_inner 220 221 srlg $Zlo,$Zlo,4 222 srlg $Zhi,$Zhi,4 223 xg $Zlo,8($nlo,$Htbl) 224 xg $Zhi,0($nlo,$Htbl) 225 sllg $xi,$Zlo,3 226 xg $Zhi,0($rem0,$rem_4bit) 227 xgr $Zlo,$tmp 228 ngr $xi,$x78 229 230 sllg $tmp,$Zhi,60 231 srlg $Zlo,$Zlo,4 232 srlg $Zhi,$Zhi,4 233 xg $Zlo,8($nhi,$Htbl) 234 xg $Zhi,0($nhi,$Htbl) 235 xgr $Zlo,$tmp 236 xg $Zhi,0($rem1,$rem_4bit) 237 238 lg $tmp,0($xi,$rem_4bit) 239 la $inp,16($inp) 240 sllg $tmp,$tmp,4 # correct last rem_4bit[rem] 241 brctg $len,.Louter 242 243 xgr $Zhi,$tmp 244 stg $Zlo,8+1($Xi) 245 stg $Zhi,0+1($Xi) 246 lm${g} %r6,%r14,6*$SIZE_T($sp) 247 br %r14 248.type gcm_ghash_4bit,\@function 249.size gcm_ghash_4bit,(.-gcm_ghash_4bit) 250 251.align 64 252rem_4bit: 253 .long `0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0 254 .long `0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0 255 .long `0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0 256 .long `0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0 257.type rem_4bit,\@object 258.size rem_4bit,(.-rem_4bit) 259.string "GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>" 260___ 261 262$code =~ s/\`([^\`]*)\`/eval $1/gem; 263print $code; 264close STDOUT or die "error closing STDOUT: $!"; 265