1 /* 2 * ChaCha and HChaCha functions (ARM64 optimized) 3 * 4 * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * Based on: 11 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code 12 * 13 * Copyright (C) 2015 Martin Willi 14 * 15 * This program is free software; you can redistribute it and/or modify 16 * it under the terms of the GNU General Public License as published by 17 * the Free Software Foundation; either version 2 of the License, or 18 * (at your option) any later version. 19 */ 20 21 #include <crypto/internal/simd.h> 22 #include <linux/jump_label.h> 23 #include <linux/kernel.h> 24 25 #include <asm/hwcap.h> 26 #include <asm/neon.h> 27 #include <asm/simd.h> 28 29 asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, 30 u8 *dst, const u8 *src, int nrounds); 31 asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, 32 u8 *dst, const u8 *src, 33 int nrounds, int bytes); 34 asmlinkage void hchacha_block_neon(const struct chacha_state *state, 35 u32 out[HCHACHA_OUT_WORDS], int nrounds); 36 37 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 38 39 static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, 40 int bytes, int nrounds) 41 { 42 while (bytes > 0) { 43 int l = min(bytes, CHACHA_BLOCK_SIZE * 5); 44 45 if (l <= CHACHA_BLOCK_SIZE) { 46 u8 buf[CHACHA_BLOCK_SIZE]; 47 48 memcpy(buf, src, l); 49 chacha_block_xor_neon(state, buf, buf, nrounds); 50 memcpy(dst, buf, l); 51 state->x[12] += 1; 52 break; 53 } 54 chacha_4block_xor_neon(state, dst, src, nrounds, l); 55 bytes -= l; 56 src += l; 57 dst += l; 58 state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); 59 } 60 } 61 62 static void hchacha_block_arch(const struct chacha_state *state, 63 u32 out[HCHACHA_OUT_WORDS], int nrounds) 64 { 65 if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { 66 hchacha_block_generic(state, out, nrounds); 67 } else { 68 kernel_neon_begin(); 69 hchacha_block_neon(state, out, nrounds); 70 kernel_neon_end(); 71 } 72 } 73 74 static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, 75 const u8 *src, unsigned int bytes, int nrounds) 76 { 77 if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE || 78 !crypto_simd_usable()) 79 return chacha_crypt_generic(state, dst, src, bytes, nrounds); 80 81 do { 82 unsigned int todo = min_t(unsigned int, bytes, SZ_4K); 83 84 kernel_neon_begin(); 85 chacha_doneon(state, dst, src, todo, nrounds); 86 kernel_neon_end(); 87 88 bytes -= todo; 89 src += todo; 90 dst += todo; 91 } while (bytes); 92 } 93 94 #define chacha_mod_init_arch chacha_mod_init_arch 95 static void chacha_mod_init_arch(void) 96 { 97 if (cpu_have_named_feature(ASIMD)) 98 static_branch_enable(&have_neon); 99 } 100