1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2024- IBM Corp. 4 * 5 * X25519 scalar multiplication with 51 bits limbs for PPC64le. 6 * Based on RFC7748 and AArch64 optimized implementation for X25519 7 * - Algorithm 1 Scalar multiplication of a variable point 8 */ 9 10 #include <crypto/curve25519.h> 11 #include <crypto/internal/kpp.h> 12 13 #include <linux/types.h> 14 #include <linux/jump_label.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/scatterlist.h> 18 19 #include <linux/cpufeature.h> 20 #include <linux/processor.h> 21 22 typedef uint64_t fe51[5]; 23 24 asmlinkage void x25519_fe51_mul(fe51 h, const fe51 f, const fe51 g); 25 asmlinkage void x25519_fe51_sqr(fe51 h, const fe51 f); 26 asmlinkage void x25519_fe51_mul121666(fe51 h, fe51 f); 27 asmlinkage void x25519_fe51_sqr_times(fe51 h, const fe51 f, int n); 28 asmlinkage void x25519_fe51_frombytes(fe51 h, const uint8_t *s); 29 asmlinkage void x25519_fe51_tobytes(uint8_t *s, const fe51 h); 30 asmlinkage void x25519_cswap(fe51 p, fe51 q, unsigned int bit); 31 32 #define fmul x25519_fe51_mul 33 #define fsqr x25519_fe51_sqr 34 #define fmul121666 x25519_fe51_mul121666 35 #define fe51_tobytes x25519_fe51_tobytes 36 37 static void fadd(fe51 h, const fe51 f, const fe51 g) 38 { 39 h[0] = f[0] + g[0]; 40 h[1] = f[1] + g[1]; 41 h[2] = f[2] + g[2]; 42 h[3] = f[3] + g[3]; 43 h[4] = f[4] + g[4]; 44 } 45 46 /* 47 * Prime = 2 ** 255 - 19, 255 bits 48 * (0x7fffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffed) 49 * 50 * Prime in 5 51-bit limbs 51 */ 52 static fe51 prime51 = { 0x7ffffffffffed, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff}; 53 54 static void fsub(fe51 h, const fe51 f, const fe51 g) 55 { 56 h[0] = (f[0] + ((prime51[0] * 2))) - g[0]; 57 h[1] = (f[1] + ((prime51[1] * 2))) - g[1]; 58 h[2] = (f[2] + ((prime51[2] * 2))) - g[2]; 59 h[3] = (f[3] + ((prime51[3] * 2))) - g[3]; 60 h[4] = (f[4] + ((prime51[4] * 2))) - g[4]; 61 } 62 63 static void fe51_frombytes(fe51 h, const uint8_t *s) 64 { 65 /* 66 * Make sure 64-bit aligned. 67 */ 68 unsigned char sbuf[32+8]; 69 unsigned char *sb = PTR_ALIGN((void *)sbuf, 8); 70 71 memcpy(sb, s, 32); 72 x25519_fe51_frombytes(h, sb); 73 } 74 75 static void finv(fe51 o, const fe51 i) 76 { 77 fe51 a0, b, c, t00; 78 79 fsqr(a0, i); 80 x25519_fe51_sqr_times(t00, a0, 2); 81 82 fmul(b, t00, i); 83 fmul(a0, b, a0); 84 85 fsqr(t00, a0); 86 87 fmul(b, t00, b); 88 x25519_fe51_sqr_times(t00, b, 5); 89 90 fmul(b, t00, b); 91 x25519_fe51_sqr_times(t00, b, 10); 92 93 fmul(c, t00, b); 94 x25519_fe51_sqr_times(t00, c, 20); 95 96 fmul(t00, t00, c); 97 x25519_fe51_sqr_times(t00, t00, 10); 98 99 fmul(b, t00, b); 100 x25519_fe51_sqr_times(t00, b, 50); 101 102 fmul(c, t00, b); 103 x25519_fe51_sqr_times(t00, c, 100); 104 105 fmul(t00, t00, c); 106 x25519_fe51_sqr_times(t00, t00, 50); 107 108 fmul(t00, t00, b); 109 x25519_fe51_sqr_times(t00, t00, 5); 110 111 fmul(o, t00, a0); 112 } 113 114 static void curve25519_fe51(uint8_t out[32], const uint8_t scalar[32], 115 const uint8_t point[32]) 116 { 117 fe51 x1, x2, z2, x3, z3; 118 uint8_t s[32]; 119 unsigned int swap = 0; 120 int i; 121 122 memcpy(s, scalar, 32); 123 s[0] &= 0xf8; 124 s[31] &= 0x7f; 125 s[31] |= 0x40; 126 fe51_frombytes(x1, point); 127 128 z2[0] = z2[1] = z2[2] = z2[3] = z2[4] = 0; 129 x3[0] = x1[0]; 130 x3[1] = x1[1]; 131 x3[2] = x1[2]; 132 x3[3] = x1[3]; 133 x3[4] = x1[4]; 134 135 x2[0] = z3[0] = 1; 136 x2[1] = z3[1] = 0; 137 x2[2] = z3[2] = 0; 138 x2[3] = z3[3] = 0; 139 x2[4] = z3[4] = 0; 140 141 for (i = 254; i >= 0; --i) { 142 unsigned int k_t = 1 & (s[i / 8] >> (i & 7)); 143 fe51 a, b, c, d, e; 144 fe51 da, cb, aa, bb; 145 fe51 dacb_p, dacb_m; 146 147 swap ^= k_t; 148 x25519_cswap(x2, x3, swap); 149 x25519_cswap(z2, z3, swap); 150 swap = k_t; 151 152 fsub(b, x2, z2); // B = x_2 - z_2 153 fadd(a, x2, z2); // A = x_2 + z_2 154 fsub(d, x3, z3); // D = x_3 - z_3 155 fadd(c, x3, z3); // C = x_3 + z_3 156 157 fsqr(bb, b); // BB = B^2 158 fsqr(aa, a); // AA = A^2 159 fmul(da, d, a); // DA = D * A 160 fmul(cb, c, b); // CB = C * B 161 162 fsub(e, aa, bb); // E = AA - BB 163 fmul(x2, aa, bb); // x2 = AA * BB 164 fadd(dacb_p, da, cb); // DA + CB 165 fsub(dacb_m, da, cb); // DA - CB 166 167 fmul121666(z3, e); // 121666 * E 168 fsqr(z2, dacb_m); // (DA - CB)^2 169 fsqr(x3, dacb_p); // x3 = (DA + CB)^2 170 fadd(b, bb, z3); // BB + 121666 * E 171 fmul(z3, x1, z2); // z3 = x1 * (DA - CB)^2 172 fmul(z2, e, b); // z2 = e * (BB + (DA + CB)^2) 173 } 174 175 finv(z2, z2); 176 fmul(x2, x2, z2); 177 fe51_tobytes(out, x2); 178 } 179 180 void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], 181 const u8 secret[CURVE25519_KEY_SIZE], 182 const u8 basepoint[CURVE25519_KEY_SIZE]) 183 { 184 curve25519_fe51(mypublic, secret, basepoint); 185 } 186 EXPORT_SYMBOL(curve25519_arch); 187 188 void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], 189 const u8 secret[CURVE25519_KEY_SIZE]) 190 { 191 curve25519_fe51(pub, secret, curve25519_base_point); 192 } 193 EXPORT_SYMBOL(curve25519_base_arch); 194 195 static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, 196 unsigned int len) 197 { 198 u8 *secret = kpp_tfm_ctx(tfm); 199 200 if (!len) 201 curve25519_generate_secret(secret); 202 else if (len == CURVE25519_KEY_SIZE && 203 crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) 204 memcpy(secret, buf, CURVE25519_KEY_SIZE); 205 else 206 return -EINVAL; 207 return 0; 208 } 209 210 static int curve25519_generate_public_key(struct kpp_request *req) 211 { 212 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 213 const u8 *secret = kpp_tfm_ctx(tfm); 214 u8 buf[CURVE25519_KEY_SIZE]; 215 int copied, nbytes; 216 217 if (req->src) 218 return -EINVAL; 219 220 curve25519_base_arch(buf, secret); 221 222 /* might want less than we've got */ 223 nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len); 224 copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, 225 nbytes), 226 buf, nbytes); 227 if (copied != nbytes) 228 return -EINVAL; 229 return 0; 230 } 231 232 static int curve25519_compute_shared_secret(struct kpp_request *req) 233 { 234 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 235 const u8 *secret = kpp_tfm_ctx(tfm); 236 u8 public_key[CURVE25519_KEY_SIZE]; 237 u8 buf[CURVE25519_KEY_SIZE]; 238 int copied, nbytes; 239 240 if (!req->src) 241 return -EINVAL; 242 243 copied = sg_copy_to_buffer(req->src, 244 sg_nents_for_len(req->src, 245 CURVE25519_KEY_SIZE), 246 public_key, CURVE25519_KEY_SIZE); 247 if (copied != CURVE25519_KEY_SIZE) 248 return -EINVAL; 249 250 curve25519_arch(buf, secret, public_key); 251 252 /* might want less than we've got */ 253 nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len); 254 copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, 255 nbytes), 256 buf, nbytes); 257 if (copied != nbytes) 258 return -EINVAL; 259 return 0; 260 } 261 262 static unsigned int curve25519_max_size(struct crypto_kpp *tfm) 263 { 264 return CURVE25519_KEY_SIZE; 265 } 266 267 static struct kpp_alg curve25519_alg = { 268 .base.cra_name = "curve25519", 269 .base.cra_driver_name = "curve25519-ppc64le", 270 .base.cra_priority = 200, 271 .base.cra_module = THIS_MODULE, 272 .base.cra_ctxsize = CURVE25519_KEY_SIZE, 273 274 .set_secret = curve25519_set_secret, 275 .generate_public_key = curve25519_generate_public_key, 276 .compute_shared_secret = curve25519_compute_shared_secret, 277 .max_size = curve25519_max_size, 278 }; 279 280 281 static int __init curve25519_mod_init(void) 282 { 283 return IS_REACHABLE(CONFIG_CRYPTO_KPP) ? 284 crypto_register_kpp(&curve25519_alg) : 0; 285 } 286 287 static void __exit curve25519_mod_exit(void) 288 { 289 if (IS_REACHABLE(CONFIG_CRYPTO_KPP)) 290 crypto_unregister_kpp(&curve25519_alg); 291 } 292 293 module_init(curve25519_mod_init); 294 module_exit(curve25519_mod_exit); 295 296 MODULE_ALIAS_CRYPTO("curve25519"); 297 MODULE_ALIAS_CRYPTO("curve25519-ppc64le"); 298 MODULE_DESCRIPTION("PPC64le Curve25519 scalar multiplication with 51 bits limbs"); 299 MODULE_LICENSE("GPL v2"); 300 MODULE_AUTHOR("Danny Tsen <dtsen@us.ibm.com>"); 301