1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 3 * 4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/crypto.h> 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/mm.h> 13 #include <linux/types.h> 14 #include <crypto/algapi.h> 15 #include <crypto/internal/des.h> 16 #include <crypto/internal/skcipher.h> 17 18 #include <asm/fpumacro.h> 19 #include <asm/opcodes.h> 20 #include <asm/pstate.h> 21 #include <asm/elf.h> 22 23 struct des_sparc64_ctx { 24 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 25 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 26 }; 27 28 struct des3_ede_sparc64_ctx { 29 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 30 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 31 }; 32 33 static void encrypt_to_decrypt(u64 *d, const u64 *e) 34 { 35 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 36 int i; 37 38 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 39 *d++ = *s--; 40 } 41 42 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 43 44 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 45 unsigned int keylen) 46 { 47 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 48 int err; 49 50 /* Even though we have special instructions for key expansion, 51 * we call des_verify_key() so that we don't have to write our own 52 * weak key detection code. 53 */ 54 err = crypto_des_verify_key(tfm, key); 55 if (err) 56 return err; 57 58 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 59 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 60 61 return 0; 62 } 63 64 static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, 65 unsigned int keylen) 66 { 67 return des_set_key(crypto_skcipher_tfm(tfm), key, keylen); 68 } 69 70 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 71 u64 *output); 72 73 static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 74 { 75 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 76 const u64 *K = ctx->encrypt_expkey; 77 78 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 79 } 80 81 static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 82 { 83 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 84 const u64 *K = ctx->decrypt_expkey; 85 86 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 87 } 88 89 extern void des_sparc64_load_keys(const u64 *key); 90 91 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 92 unsigned int len); 93 94 static int __ecb_crypt(struct skcipher_request *req, bool encrypt) 95 { 96 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 97 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 98 struct skcipher_walk walk; 99 unsigned int nbytes; 100 int err; 101 102 err = skcipher_walk_virt(&walk, req, true); 103 if (err) 104 return err; 105 106 if (encrypt) 107 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 108 else 109 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 110 while ((nbytes = walk.nbytes) != 0) { 111 des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr, 112 round_down(nbytes, DES_BLOCK_SIZE)); 113 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 114 } 115 fprs_write(0); 116 return err; 117 } 118 119 static int ecb_encrypt(struct skcipher_request *req) 120 { 121 return __ecb_crypt(req, true); 122 } 123 124 static int ecb_decrypt(struct skcipher_request *req) 125 { 126 return __ecb_crypt(req, false); 127 } 128 129 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 130 unsigned int len, u64 *iv); 131 132 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 133 unsigned int len, u64 *iv); 134 135 static int __cbc_crypt(struct skcipher_request *req, bool encrypt) 136 { 137 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 138 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 139 struct skcipher_walk walk; 140 unsigned int nbytes; 141 int err; 142 143 err = skcipher_walk_virt(&walk, req, true); 144 if (err) 145 return err; 146 147 if (encrypt) 148 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 149 else 150 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 151 while ((nbytes = walk.nbytes) != 0) { 152 if (encrypt) 153 des_sparc64_cbc_encrypt(walk.src.virt.addr, 154 walk.dst.virt.addr, 155 round_down(nbytes, 156 DES_BLOCK_SIZE), 157 walk.iv); 158 else 159 des_sparc64_cbc_decrypt(walk.src.virt.addr, 160 walk.dst.virt.addr, 161 round_down(nbytes, 162 DES_BLOCK_SIZE), 163 walk.iv); 164 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 165 } 166 fprs_write(0); 167 return err; 168 } 169 170 static int cbc_encrypt(struct skcipher_request *req) 171 { 172 return __cbc_crypt(req, true); 173 } 174 175 static int cbc_decrypt(struct skcipher_request *req) 176 { 177 return __cbc_crypt(req, false); 178 } 179 180 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 181 unsigned int keylen) 182 { 183 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 184 u64 k1[DES_EXPKEY_WORDS / 2]; 185 u64 k2[DES_EXPKEY_WORDS / 2]; 186 u64 k3[DES_EXPKEY_WORDS / 2]; 187 int err; 188 189 err = crypto_des3_ede_verify_key(tfm, key); 190 if (err) 191 return err; 192 193 des_sparc64_key_expand((const u32 *)key, k1); 194 key += DES_KEY_SIZE; 195 des_sparc64_key_expand((const u32 *)key, k2); 196 key += DES_KEY_SIZE; 197 des_sparc64_key_expand((const u32 *)key, k3); 198 199 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 200 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 201 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 202 &k3[0], sizeof(k3)); 203 204 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 205 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 206 &k2[0], sizeof(k2)); 207 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 208 &k1[0]); 209 210 return 0; 211 } 212 213 static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, 214 unsigned int keylen) 215 { 216 return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen); 217 } 218 219 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 220 u64 *output); 221 222 static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 223 { 224 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 225 const u64 *K = ctx->encrypt_expkey; 226 227 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 228 } 229 230 static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 231 { 232 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 233 const u64 *K = ctx->decrypt_expkey; 234 235 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 236 } 237 238 extern void des3_ede_sparc64_load_keys(const u64 *key); 239 240 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 241 u64 *output, unsigned int len); 242 243 static int __ecb3_crypt(struct skcipher_request *req, bool encrypt) 244 { 245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 246 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 247 struct skcipher_walk walk; 248 const u64 *K; 249 unsigned int nbytes; 250 int err; 251 252 err = skcipher_walk_virt(&walk, req, true); 253 if (err) 254 return err; 255 256 if (encrypt) 257 K = &ctx->encrypt_expkey[0]; 258 else 259 K = &ctx->decrypt_expkey[0]; 260 des3_ede_sparc64_load_keys(K); 261 while ((nbytes = walk.nbytes) != 0) { 262 des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr, 263 walk.dst.virt.addr, 264 round_down(nbytes, DES_BLOCK_SIZE)); 265 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 266 } 267 fprs_write(0); 268 return err; 269 } 270 271 static int ecb3_encrypt(struct skcipher_request *req) 272 { 273 return __ecb3_crypt(req, true); 274 } 275 276 static int ecb3_decrypt(struct skcipher_request *req) 277 { 278 return __ecb3_crypt(req, false); 279 } 280 281 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 282 u64 *output, unsigned int len, 283 u64 *iv); 284 285 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 286 u64 *output, unsigned int len, 287 u64 *iv); 288 289 static int __cbc3_crypt(struct skcipher_request *req, bool encrypt) 290 { 291 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 292 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 293 struct skcipher_walk walk; 294 const u64 *K; 295 unsigned int nbytes; 296 int err; 297 298 err = skcipher_walk_virt(&walk, req, true); 299 if (err) 300 return err; 301 302 if (encrypt) 303 K = &ctx->encrypt_expkey[0]; 304 else 305 K = &ctx->decrypt_expkey[0]; 306 des3_ede_sparc64_load_keys(K); 307 while ((nbytes = walk.nbytes) != 0) { 308 if (encrypt) 309 des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr, 310 walk.dst.virt.addr, 311 round_down(nbytes, 312 DES_BLOCK_SIZE), 313 walk.iv); 314 else 315 des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr, 316 walk.dst.virt.addr, 317 round_down(nbytes, 318 DES_BLOCK_SIZE), 319 walk.iv); 320 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 321 } 322 fprs_write(0); 323 return err; 324 } 325 326 static int cbc3_encrypt(struct skcipher_request *req) 327 { 328 return __cbc3_crypt(req, true); 329 } 330 331 static int cbc3_decrypt(struct skcipher_request *req) 332 { 333 return __cbc3_crypt(req, false); 334 } 335 336 static struct crypto_alg cipher_algs[] = { 337 { 338 .cra_name = "des", 339 .cra_driver_name = "des-sparc64", 340 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 341 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 342 .cra_blocksize = DES_BLOCK_SIZE, 343 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 344 .cra_alignmask = 7, 345 .cra_module = THIS_MODULE, 346 .cra_u = { 347 .cipher = { 348 .cia_min_keysize = DES_KEY_SIZE, 349 .cia_max_keysize = DES_KEY_SIZE, 350 .cia_setkey = des_set_key, 351 .cia_encrypt = sparc_des_encrypt, 352 .cia_decrypt = sparc_des_decrypt 353 } 354 } 355 }, { 356 .cra_name = "des3_ede", 357 .cra_driver_name = "des3_ede-sparc64", 358 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 359 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 360 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 361 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 362 .cra_alignmask = 7, 363 .cra_module = THIS_MODULE, 364 .cra_u = { 365 .cipher = { 366 .cia_min_keysize = DES3_EDE_KEY_SIZE, 367 .cia_max_keysize = DES3_EDE_KEY_SIZE, 368 .cia_setkey = des3_ede_set_key, 369 .cia_encrypt = sparc_des3_ede_encrypt, 370 .cia_decrypt = sparc_des3_ede_decrypt 371 } 372 } 373 } 374 }; 375 376 static struct skcipher_alg skcipher_algs[] = { 377 { 378 .base.cra_name = "ecb(des)", 379 .base.cra_driver_name = "ecb-des-sparc64", 380 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 381 .base.cra_blocksize = DES_BLOCK_SIZE, 382 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), 383 .base.cra_alignmask = 7, 384 .base.cra_module = THIS_MODULE, 385 .min_keysize = DES_KEY_SIZE, 386 .max_keysize = DES_KEY_SIZE, 387 .setkey = des_set_key_skcipher, 388 .encrypt = ecb_encrypt, 389 .decrypt = ecb_decrypt, 390 }, { 391 .base.cra_name = "cbc(des)", 392 .base.cra_driver_name = "cbc-des-sparc64", 393 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 394 .base.cra_blocksize = DES_BLOCK_SIZE, 395 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), 396 .base.cra_alignmask = 7, 397 .base.cra_module = THIS_MODULE, 398 .min_keysize = DES_KEY_SIZE, 399 .max_keysize = DES_KEY_SIZE, 400 .ivsize = DES_BLOCK_SIZE, 401 .setkey = des_set_key_skcipher, 402 .encrypt = cbc_encrypt, 403 .decrypt = cbc_decrypt, 404 }, { 405 .base.cra_name = "ecb(des3_ede)", 406 .base.cra_driver_name = "ecb-des3_ede-sparc64", 407 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 408 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 409 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 410 .base.cra_alignmask = 7, 411 .base.cra_module = THIS_MODULE, 412 .min_keysize = DES3_EDE_KEY_SIZE, 413 .max_keysize = DES3_EDE_KEY_SIZE, 414 .setkey = des3_ede_set_key_skcipher, 415 .encrypt = ecb3_encrypt, 416 .decrypt = ecb3_decrypt, 417 }, { 418 .base.cra_name = "cbc(des3_ede)", 419 .base.cra_driver_name = "cbc-des3_ede-sparc64", 420 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 421 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 422 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 423 .base.cra_alignmask = 7, 424 .base.cra_module = THIS_MODULE, 425 .min_keysize = DES3_EDE_KEY_SIZE, 426 .max_keysize = DES3_EDE_KEY_SIZE, 427 .ivsize = DES3_EDE_BLOCK_SIZE, 428 .setkey = des3_ede_set_key_skcipher, 429 .encrypt = cbc3_encrypt, 430 .decrypt = cbc3_decrypt, 431 } 432 }; 433 434 static bool __init sparc64_has_des_opcode(void) 435 { 436 unsigned long cfr; 437 438 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 439 return false; 440 441 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 442 if (!(cfr & CFR_DES)) 443 return false; 444 445 return true; 446 } 447 448 static int __init des_sparc64_mod_init(void) 449 { 450 int err; 451 452 if (!sparc64_has_des_opcode()) { 453 pr_info("sparc64 des opcodes not available.\n"); 454 return -ENODEV; 455 } 456 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 457 err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 458 if (err) 459 return err; 460 err = crypto_register_skciphers(skcipher_algs, 461 ARRAY_SIZE(skcipher_algs)); 462 if (err) 463 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 464 return err; 465 } 466 467 static void __exit des_sparc64_mod_fini(void) 468 { 469 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 470 crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 471 } 472 473 module_init(des_sparc64_mod_init); 474 module_exit(des_sparc64_mod_fini); 475 476 MODULE_LICENSE("GPL"); 477 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 478 479 MODULE_ALIAS_CRYPTO("des"); 480 MODULE_ALIAS_CRYPTO("des3_ede"); 481 482 #include "crop_devid.c" 483