1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 *
5 * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This
6 * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been
7 * manually reworked for use in kernel space.
8 */
9
10 #include <asm/hwcap.h>
11 #include <asm/neon.h>
12 #include <asm/simd.h>
13 #include <crypto/internal/simd.h>
14 #include <linux/types.h>
15 #include <linux/jump_label.h>
16
17 asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE],
18 const u8 secret[CURVE25519_KEY_SIZE],
19 const u8 basepoint[CURVE25519_KEY_SIZE]);
20
21 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
22
curve25519_arch(u8 out[CURVE25519_KEY_SIZE],const u8 scalar[CURVE25519_KEY_SIZE],const u8 point[CURVE25519_KEY_SIZE])23 static void curve25519_arch(u8 out[CURVE25519_KEY_SIZE],
24 const u8 scalar[CURVE25519_KEY_SIZE],
25 const u8 point[CURVE25519_KEY_SIZE])
26 {
27 if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
28 kernel_neon_begin();
29 curve25519_neon(out, scalar, point);
30 kernel_neon_end();
31 } else {
32 curve25519_generic(out, scalar, point);
33 }
34 }
35
curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],const u8 secret[CURVE25519_KEY_SIZE])36 static void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
37 const u8 secret[CURVE25519_KEY_SIZE])
38 {
39 curve25519_arch(pub, secret, curve25519_base_point);
40 }
41
42 #define curve25519_mod_init_arch curve25519_mod_init_arch
curve25519_mod_init_arch(void)43 static void curve25519_mod_init_arch(void)
44 {
45 if (elf_hwcap & HWCAP_NEON)
46 static_branch_enable(&have_neon);
47 }
48