1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 SA2UL crypto accelerator driver 4 * 5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com 6 * 7 * Authors: Keerthy 8 * Vitaly Andrianov 9 * Tero Kristo 10 */ 11 #include <linux/bitfield.h> 12 #include <linux/clk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/dmapool.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 23 #include <crypto/aes.h> 24 #include <crypto/authenc.h> 25 #include <crypto/utils.h> 26 #include <crypto/des.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/hash.h> 29 #include <crypto/internal/skcipher.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/sha1.h> 32 #include <crypto/sha2.h> 33 34 #include "sa2ul.h" 35 36 /* Byte offset for key in encryption security context */ 37 #define SC_ENC_KEY_OFFSET (1 + 27 + 4) 38 /* Byte offset for Aux-1 in encryption security context */ 39 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) 40 41 #define SA_CMDL_UPD_ENC 0x0001 42 #define SA_CMDL_UPD_AUTH 0x0002 43 #define SA_CMDL_UPD_ENC_IV 0x0004 44 #define SA_CMDL_UPD_AUTH_IV 0x0008 45 #define SA_CMDL_UPD_AUX_KEY 0x0010 46 47 #define SA_AUTH_SUBKEY_LEN 16 48 #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF 49 #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 50 51 #define MODE_CONTROL_BYTES 27 52 #define SA_HASH_PROCESSING 0 53 #define SA_CRYPTO_PROCESSING 0 54 #define SA_UPLOAD_HASH_TO_TLR BIT(6) 55 56 #define SA_SW0_FLAGS_MASK 0xF0000 57 #define SA_SW0_CMDL_INFO_MASK 0x1F00000 58 #define SA_SW0_CMDL_PRESENT BIT(4) 59 #define SA_SW0_ENG_ID_MASK 0x3E000000 60 #define SA_SW0_DEST_INFO_PRESENT BIT(30) 61 #define SA_SW2_EGRESS_LENGTH 0xFF000000 62 #define SA_BASIC_HASH 0x10 63 64 #define SHA256_DIGEST_WORDS 8 65 /* Make 32-bit word from 4 bytes */ 66 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ 67 ((b2) << 8) | (b3)) 68 69 /* size of SCCTL structure in bytes */ 70 #define SA_SCCTL_SZ 16 71 72 /* Max Authentication tag size */ 73 #define SA_MAX_AUTH_TAG_SZ 64 74 75 enum sa_algo_id { 76 SA_ALG_CBC_AES = 0, 77 SA_ALG_EBC_AES, 78 SA_ALG_CBC_DES3, 79 SA_ALG_ECB_DES3, 80 SA_ALG_SHA1, 81 SA_ALG_SHA256, 82 SA_ALG_SHA512, 83 SA_ALG_AUTHENC_SHA1_AES, 84 SA_ALG_AUTHENC_SHA256_AES, 85 }; 86 87 struct sa_match_data { 88 u8 priv; 89 u8 priv_id; 90 u32 supported_algos; 91 }; 92 93 static struct device *sa_k3_dev; 94 95 /** 96 * struct sa_cmdl_cfg - Command label configuration descriptor 97 * @aalg: authentication algorithm ID 98 * @enc_eng_id: Encryption Engine ID supported by the SA hardware 99 * @auth_eng_id: Authentication Engine ID 100 * @iv_size: Initialization Vector size 101 * @akey: Authentication key 102 * @akey_len: Authentication key length 103 * @enc: True, if this is an encode request 104 */ 105 struct sa_cmdl_cfg { 106 int aalg; 107 u8 enc_eng_id; 108 u8 auth_eng_id; 109 u8 iv_size; 110 const u8 *akey; 111 u16 akey_len; 112 bool enc; 113 }; 114 115 /** 116 * struct algo_data - Crypto algorithm specific data 117 * @enc_eng: Encryption engine info structure 118 * @auth_eng: Authentication engine info structure 119 * @auth_ctrl: Authentication control word 120 * @hash_size: Size of digest 121 * @iv_idx: iv index in psdata 122 * @iv_out_size: iv out size 123 * @ealg_id: Encryption Algorithm ID 124 * @aalg_id: Authentication algorithm ID 125 * @mci_enc: Mode Control Instruction for Encryption algorithm 126 * @mci_dec: Mode Control Instruction for Decryption 127 * @inv_key: Whether the encryption algorithm demands key inversion 128 * @ctx: Pointer to the algorithm context 129 * @keyed_mac: Whether the authentication algorithm has key 130 * @prep_iopad: Function pointer to generate intermediate ipad/opad 131 */ 132 struct algo_data { 133 struct sa_eng_info enc_eng; 134 struct sa_eng_info auth_eng; 135 u8 auth_ctrl; 136 u8 hash_size; 137 u8 iv_idx; 138 u8 iv_out_size; 139 u8 ealg_id; 140 u8 aalg_id; 141 u8 *mci_enc; 142 u8 *mci_dec; 143 bool inv_key; 144 struct sa_tfm_ctx *ctx; 145 bool keyed_mac; 146 void (*prep_iopad)(struct algo_data *algo, const u8 *key, 147 u16 key_sz, __be32 *ipad, __be32 *opad); 148 }; 149 150 /** 151 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms 152 * @type: Type of the crypto algorithm. 153 * @alg: Union of crypto algorithm definitions. 154 * @registered: Flag indicating if the crypto algorithm is already registered 155 */ 156 struct sa_alg_tmpl { 157 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */ 158 union { 159 struct skcipher_alg skcipher; 160 struct ahash_alg ahash; 161 struct aead_alg aead; 162 } alg; 163 bool registered; 164 }; 165 166 /** 167 * struct sa_mapped_sg: scatterlist information for tx and rx 168 * @mapped: Set to true if the @sgt is mapped 169 * @dir: mapping direction used for @sgt 170 * @split_sg: Set if the sg is split and needs to be freed up 171 * @static_sg: Static scatterlist entry for overriding data 172 * @sgt: scatterlist table for DMA API use 173 */ 174 struct sa_mapped_sg { 175 bool mapped; 176 enum dma_data_direction dir; 177 struct scatterlist static_sg; 178 struct scatterlist *split_sg; 179 struct sg_table sgt; 180 }; 181 /** 182 * struct sa_rx_data: RX Packet miscellaneous data place holder 183 * @req: crypto request data pointer 184 * @ddev: pointer to the DMA device 185 * @tx_in: dma_async_tx_descriptor pointer for rx channel 186 * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping 187 * @enc: Flag indicating either encryption or decryption 188 * @enc_iv_size: Initialisation vector size 189 * @iv_idx: Initialisation vector index 190 */ 191 struct sa_rx_data { 192 void *req; 193 struct device *ddev; 194 struct dma_async_tx_descriptor *tx_in; 195 struct sa_mapped_sg mapped_sg[2]; 196 u8 enc; 197 u8 enc_iv_size; 198 u8 iv_idx; 199 }; 200 201 /** 202 * struct sa_req: SA request definition 203 * @dev: device for the request 204 * @size: total data to the xmitted via DMA 205 * @enc_offset: offset of cipher data 206 * @enc_size: data to be passed to cipher engine 207 * @enc_iv: cipher IV 208 * @auth_offset: offset of the authentication data 209 * @auth_size: size of the authentication data 210 * @auth_iv: authentication IV 211 * @type: algorithm type for the request 212 * @cmdl: command label pointer 213 * @base: pointer to the base request 214 * @ctx: pointer to the algorithm context data 215 * @enc: true if this is an encode request 216 * @src: source data 217 * @dst: destination data 218 * @callback: DMA callback for the request 219 * @mdata_size: metadata size passed to DMA 220 */ 221 struct sa_req { 222 struct device *dev; 223 u16 size; 224 u8 enc_offset; 225 u16 enc_size; 226 u8 *enc_iv; 227 u8 auth_offset; 228 u16 auth_size; 229 u8 *auth_iv; 230 u32 type; 231 u32 *cmdl; 232 struct crypto_async_request *base; 233 struct sa_tfm_ctx *ctx; 234 bool enc; 235 struct scatterlist *src; 236 struct scatterlist *dst; 237 dma_async_tx_callback callback; 238 u16 mdata_size; 239 }; 240 241 /* 242 * Mode Control Instructions for various Key lengths 128, 192, 256 243 * For CBC (Cipher Block Chaining) mode for encryption 244 */ 245 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { 246 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 247 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 248 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 249 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 250 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 251 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 252 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 253 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 254 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 255 }; 256 257 /* 258 * Mode Control Instructions for various Key lengths 128, 192, 256 259 * For CBC (Cipher Block Chaining) mode for decryption 260 */ 261 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { 262 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 263 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 264 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 265 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 266 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 267 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 268 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 269 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 270 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 271 }; 272 273 /* 274 * Mode Control Instructions for various Key lengths 128, 192, 256 275 * For CBC (Cipher Block Chaining) mode for encryption 276 */ 277 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { 278 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 279 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 280 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 281 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 282 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 283 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 284 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 285 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 286 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 287 }; 288 289 /* 290 * Mode Control Instructions for various Key lengths 128, 192, 256 291 * For CBC (Cipher Block Chaining) mode for decryption 292 */ 293 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { 294 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 295 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 296 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 297 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 298 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 299 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 300 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 301 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 302 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 303 }; 304 305 /* 306 * Mode Control Instructions for various Key lengths 128, 192, 256 307 * For ECB (Electronic Code Book) mode for encryption 308 */ 309 static u8 mci_ecb_enc_array[3][27] = { 310 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 311 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 312 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 313 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 314 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 315 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 316 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 317 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 318 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 319 }; 320 321 /* 322 * Mode Control Instructions for various Key lengths 128, 192, 256 323 * For ECB (Electronic Code Book) mode for decryption 324 */ 325 static u8 mci_ecb_dec_array[3][27] = { 326 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 327 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 328 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 329 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 330 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 331 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 332 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 333 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 334 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 335 }; 336 337 /* 338 * Mode Control Instructions for DES algorithm 339 * For CBC (Cipher Block Chaining) mode and ECB mode 340 * encryption and for decryption respectively 341 */ 342 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { 343 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, 344 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 345 0x00, 0x00, 0x00, 346 }; 347 348 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { 349 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, 350 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 351 0x00, 0x00, 0x00, 352 }; 353 354 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { 355 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 356 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 357 0x00, 0x00, 0x00, 358 }; 359 360 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { 361 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 362 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 363 0x00, 0x00, 0x00, 364 }; 365 366 /* 367 * Perform 16 byte or 128 bit swizzling 368 * The SA2UL Expects the security context to 369 * be in little Endian and the bus width is 128 bits or 16 bytes 370 * Hence swap 16 bytes at a time from higher to lower address 371 */ 372 static void sa_swiz_128(u8 *in, u16 len) 373 { 374 u8 data[16]; 375 int i, j; 376 377 for (i = 0; i < len; i += 16) { 378 memcpy(data, &in[i], 16); 379 for (j = 0; j < 16; j++) 380 in[i + j] = data[15 - j]; 381 } 382 } 383 384 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ 385 static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) 386 { 387 int i; 388 389 for (i = 0; i < key_sz; i++) 390 k_ipad[i] = key[i] ^ 0x36; 391 392 /* Instead of XOR with 0 */ 393 for (; i < SHA1_BLOCK_SIZE; i++) 394 k_ipad[i] = 0x36; 395 } 396 397 static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) 398 { 399 int i; 400 401 for (i = 0; i < key_sz; i++) 402 k_opad[i] = key[i] ^ 0x5c; 403 404 /* Instead of XOR with 0 */ 405 for (; i < SHA1_BLOCK_SIZE; i++) 406 k_opad[i] = 0x5c; 407 } 408 409 static void sa_export_shash(void *state, struct shash_desc *hash, 410 int digest_size, __be32 *out) 411 { 412 struct sha1_state *sha1; 413 struct sha256_state *sha256; 414 u32 *result; 415 416 switch (digest_size) { 417 case SHA1_DIGEST_SIZE: 418 sha1 = state; 419 result = sha1->state; 420 break; 421 case SHA256_DIGEST_SIZE: 422 sha256 = state; 423 result = sha256->state; 424 break; 425 default: 426 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, 427 digest_size); 428 return; 429 } 430 431 crypto_shash_export(hash, state); 432 433 cpu_to_be32_array(out, result, digest_size / 4); 434 } 435 436 static void sa_prepare_iopads(struct algo_data *data, const u8 *key, 437 u16 key_sz, __be32 *ipad, __be32 *opad) 438 { 439 SHASH_DESC_ON_STACK(shash, data->ctx->shash); 440 int block_size = crypto_shash_blocksize(data->ctx->shash); 441 int digest_size = crypto_shash_digestsize(data->ctx->shash); 442 union { 443 struct sha1_state sha1; 444 struct sha256_state sha256; 445 u8 k_pad[SHA1_BLOCK_SIZE]; 446 } sha; 447 448 shash->tfm = data->ctx->shash; 449 450 prepare_kipad(sha.k_pad, key, key_sz); 451 452 crypto_shash_init(shash); 453 crypto_shash_update(shash, sha.k_pad, block_size); 454 sa_export_shash(&sha, shash, digest_size, ipad); 455 456 prepare_kopad(sha.k_pad, key, key_sz); 457 458 crypto_shash_init(shash); 459 crypto_shash_update(shash, sha.k_pad, block_size); 460 461 sa_export_shash(&sha, shash, digest_size, opad); 462 463 memzero_explicit(&sha, sizeof(sha)); 464 } 465 466 /* Derive the inverse key used in AES-CBC decryption operation */ 467 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) 468 { 469 struct crypto_aes_ctx ctx; 470 int key_pos; 471 472 if (aes_expandkey(&ctx, key, key_sz)) { 473 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 474 return -EINVAL; 475 } 476 477 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ 478 if (key_sz == AES_KEYSIZE_192) { 479 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; 480 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; 481 } 482 483 /* Based crypto_aes_expand_key logic */ 484 switch (key_sz) { 485 case AES_KEYSIZE_128: 486 case AES_KEYSIZE_192: 487 key_pos = key_sz + 24; 488 break; 489 490 case AES_KEYSIZE_256: 491 key_pos = key_sz + 24 - 4; 492 break; 493 494 default: 495 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 496 return -EINVAL; 497 } 498 499 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); 500 return 0; 501 } 502 503 /* Set Security context for the encryption engine */ 504 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, 505 u8 enc, u8 *sc_buf) 506 { 507 const u8 *mci = NULL; 508 509 /* Set Encryption mode selector to crypto processing */ 510 sc_buf[0] = SA_CRYPTO_PROCESSING; 511 512 if (enc) 513 mci = ad->mci_enc; 514 else 515 mci = ad->mci_dec; 516 /* Set the mode control instructions in security context */ 517 if (mci) 518 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); 519 520 /* For AES-CBC decryption get the inverse key */ 521 if (ad->inv_key && !enc) { 522 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) 523 return -EINVAL; 524 /* For all other cases: key is used */ 525 } else { 526 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); 527 } 528 529 return 0; 530 } 531 532 /* Set Security context for the authentication engine */ 533 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, 534 u8 *sc_buf) 535 { 536 __be32 *ipad = (void *)(sc_buf + 32); 537 __be32 *opad = (void *)(sc_buf + 64); 538 539 /* Set Authentication mode selector to hash processing */ 540 sc_buf[0] = SA_HASH_PROCESSING; 541 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ 542 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; 543 sc_buf[1] |= ad->auth_ctrl; 544 545 /* Copy the keys or ipad/opad */ 546 if (ad->keyed_mac) 547 ad->prep_iopad(ad, key, key_sz, ipad, opad); 548 else { 549 /* basic hash */ 550 sc_buf[1] |= SA_BASIC_HASH; 551 } 552 } 553 554 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) 555 { 556 int j; 557 558 for (j = 0; j < ((size16) ? 4 : 2); j++) { 559 *out = cpu_to_be32(*((u32 *)iv)); 560 iv += 4; 561 out++; 562 } 563 } 564 565 /* Format general command label */ 566 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, 567 struct sa_cmdl_upd_info *upd_info) 568 { 569 u8 enc_offset = 0, auth_offset = 0, total = 0; 570 u8 enc_next_eng = SA_ENG_ID_OUTPORT2; 571 u8 auth_next_eng = SA_ENG_ID_OUTPORT2; 572 u32 *word_ptr = (u32 *)cmdl; 573 int i; 574 575 /* Clear the command label */ 576 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); 577 578 /* Initialize the command update structure */ 579 memzero_explicit(upd_info, sizeof(*upd_info)); 580 581 if (cfg->enc_eng_id && cfg->auth_eng_id) { 582 if (cfg->enc) { 583 auth_offset = SA_CMDL_HEADER_SIZE_BYTES; 584 enc_next_eng = cfg->auth_eng_id; 585 586 if (cfg->iv_size) 587 auth_offset += cfg->iv_size; 588 } else { 589 enc_offset = SA_CMDL_HEADER_SIZE_BYTES; 590 auth_next_eng = cfg->enc_eng_id; 591 } 592 } 593 594 if (cfg->enc_eng_id) { 595 upd_info->flags |= SA_CMDL_UPD_ENC; 596 upd_info->enc_size.index = enc_offset >> 2; 597 upd_info->enc_offset.index = upd_info->enc_size.index + 1; 598 /* Encryption command label */ 599 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; 600 601 /* Encryption modes requiring IV */ 602 if (cfg->iv_size) { 603 upd_info->flags |= SA_CMDL_UPD_ENC_IV; 604 upd_info->enc_iv.index = 605 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; 606 upd_info->enc_iv.size = cfg->iv_size; 607 608 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 609 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 610 611 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = 612 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); 613 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 614 } else { 615 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 616 SA_CMDL_HEADER_SIZE_BYTES; 617 total += SA_CMDL_HEADER_SIZE_BYTES; 618 } 619 } 620 621 if (cfg->auth_eng_id) { 622 upd_info->flags |= SA_CMDL_UPD_AUTH; 623 upd_info->auth_size.index = auth_offset >> 2; 624 upd_info->auth_offset.index = upd_info->auth_size.index + 1; 625 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; 626 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = 627 SA_CMDL_HEADER_SIZE_BYTES; 628 total += SA_CMDL_HEADER_SIZE_BYTES; 629 } 630 631 total = roundup(total, 8); 632 633 for (i = 0; i < total / 4; i++) 634 word_ptr[i] = swab32(word_ptr[i]); 635 636 return total; 637 } 638 639 /* Update Command label */ 640 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, 641 struct sa_cmdl_upd_info *upd_info) 642 { 643 int i = 0, j; 644 645 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { 646 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 647 cmdl[upd_info->enc_size.index] |= req->enc_size; 648 cmdl[upd_info->enc_offset.index] &= 649 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 650 cmdl[upd_info->enc_offset.index] |= 651 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, 652 req->enc_offset); 653 654 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { 655 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; 656 u32 *enc_iv = (u32 *)req->enc_iv; 657 658 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { 659 data[j] = cpu_to_be32(*enc_iv); 660 enc_iv++; 661 } 662 } 663 } 664 665 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { 666 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 667 cmdl[upd_info->auth_size.index] |= req->auth_size; 668 cmdl[upd_info->auth_offset.index] &= 669 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 670 cmdl[upd_info->auth_offset.index] |= 671 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, 672 req->auth_offset); 673 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { 674 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], 675 req->auth_iv, 676 (upd_info->auth_iv.size > 8)); 677 } 678 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { 679 int offset = (req->auth_size & 0xF) ? 4 : 0; 680 681 memcpy(&cmdl[upd_info->aux_key_info.index], 682 &upd_info->aux_key[offset], 16); 683 } 684 } 685 } 686 687 /* Format SWINFO words to be sent to SA */ 688 static 689 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, 690 u8 cmdl_present, u8 cmdl_offset, u8 flags, 691 u8 hash_size, u32 *swinfo) 692 { 693 swinfo[0] = sc_id; 694 swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags); 695 if (likely(cmdl_present)) 696 swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK, 697 cmdl_offset | SA_SW0_CMDL_PRESENT); 698 swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id); 699 700 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; 701 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); 702 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); 703 swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size); 704 } 705 706 /* Dump the security context */ 707 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) 708 { 709 #ifdef DEBUG 710 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); 711 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 712 16, 1, buf, SA_CTX_MAX_SZ, false); 713 #endif 714 } 715 716 static 717 int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data, 718 const u8 *enc_key, u16 enc_key_sz, 719 const u8 *auth_key, u16 auth_key_sz, 720 struct algo_data *ad, u8 enc, u32 *swinfo) 721 { 722 int enc_sc_offset = 0; 723 int auth_sc_offset = 0; 724 u8 *sc_buf = ctx->sc; 725 u16 sc_id = ctx->sc_id; 726 u8 first_engine = 0; 727 728 memzero_explicit(sc_buf, SA_CTX_MAX_SZ); 729 730 if (ad->auth_eng.eng_id) { 731 if (enc) 732 first_engine = ad->enc_eng.eng_id; 733 else 734 first_engine = ad->auth_eng.eng_id; 735 736 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 737 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; 738 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; 739 if (!ad->hash_size) 740 return -EINVAL; 741 ad->hash_size = roundup(ad->hash_size, 8); 742 743 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { 744 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 745 first_engine = ad->enc_eng.eng_id; 746 sc_buf[1] = SA_SCCTL_FE_ENC; 747 ad->hash_size = ad->iv_out_size; 748 } 749 750 /* SCCTL Owner info: 0=host, 1=CP_ACE */ 751 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; 752 memcpy(&sc_buf[2], &sc_id, 2); 753 sc_buf[4] = 0x0; 754 sc_buf[5] = match_data->priv_id; 755 sc_buf[6] = match_data->priv; 756 sc_buf[7] = 0x0; 757 758 /* Prepare context for encryption engine */ 759 if (ad->enc_eng.sc_size) { 760 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, 761 &sc_buf[enc_sc_offset])) 762 return -EINVAL; 763 } 764 765 /* Prepare context for authentication engine */ 766 if (ad->auth_eng.sc_size) 767 sa_set_sc_auth(ad, auth_key, auth_key_sz, 768 &sc_buf[auth_sc_offset]); 769 770 /* Set the ownership of context to CP_ACE */ 771 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; 772 773 /* swizzle the security context */ 774 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); 775 776 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, 777 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); 778 779 sa_dump_sc(sc_buf, ctx->sc_phys); 780 781 return 0; 782 } 783 784 /* Free the per direction context memory */ 785 static void sa_free_ctx_info(struct sa_ctx_info *ctx, 786 struct sa_crypto_data *data) 787 { 788 unsigned long bn; 789 790 bn = ctx->sc_id - data->sc_id_start; 791 spin_lock(&data->scid_lock); 792 __clear_bit(bn, data->ctx_bm); 793 data->sc_id--; 794 spin_unlock(&data->scid_lock); 795 796 if (ctx->sc) { 797 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); 798 ctx->sc = NULL; 799 } 800 } 801 802 static int sa_init_ctx_info(struct sa_ctx_info *ctx, 803 struct sa_crypto_data *data) 804 { 805 unsigned long bn; 806 int err; 807 808 spin_lock(&data->scid_lock); 809 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); 810 __set_bit(bn, data->ctx_bm); 811 data->sc_id++; 812 spin_unlock(&data->scid_lock); 813 814 ctx->sc_id = (u16)(data->sc_id_start + bn); 815 816 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); 817 if (!ctx->sc) { 818 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); 819 err = -ENOMEM; 820 goto scid_rollback; 821 } 822 823 return 0; 824 825 scid_rollback: 826 spin_lock(&data->scid_lock); 827 __clear_bit(bn, data->ctx_bm); 828 data->sc_id--; 829 spin_unlock(&data->scid_lock); 830 831 return err; 832 } 833 834 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) 835 { 836 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 837 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 838 839 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 840 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 841 ctx->dec.sc_id, &ctx->dec.sc_phys); 842 843 sa_free_ctx_info(&ctx->enc, data); 844 sa_free_ctx_info(&ctx->dec, data); 845 846 crypto_free_skcipher(ctx->fallback.skcipher); 847 } 848 849 static int sa_cipher_cra_init(struct crypto_skcipher *tfm) 850 { 851 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 852 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 853 const char *name = crypto_tfm_alg_name(&tfm->base); 854 struct crypto_skcipher *child; 855 int ret; 856 857 memzero_explicit(ctx, sizeof(*ctx)); 858 ctx->dev_data = data; 859 860 ret = sa_init_ctx_info(&ctx->enc, data); 861 if (ret) 862 return ret; 863 ret = sa_init_ctx_info(&ctx->dec, data); 864 if (ret) { 865 sa_free_ctx_info(&ctx->enc, data); 866 return ret; 867 } 868 869 child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 870 871 if (IS_ERR(child)) { 872 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); 873 return PTR_ERR(child); 874 } 875 876 ctx->fallback.skcipher = child; 877 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + 878 sizeof(struct skcipher_request)); 879 880 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 881 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 882 ctx->dec.sc_id, &ctx->dec.sc_phys); 883 return 0; 884 } 885 886 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 887 unsigned int keylen, struct algo_data *ad) 888 { 889 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 890 struct crypto_skcipher *child = ctx->fallback.skcipher; 891 int cmdl_len; 892 struct sa_cmdl_cfg cfg; 893 int ret; 894 895 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 896 keylen != AES_KEYSIZE_256) 897 return -EINVAL; 898 899 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 900 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 901 902 memzero_explicit(&cfg, sizeof(cfg)); 903 cfg.enc_eng_id = ad->enc_eng.eng_id; 904 cfg.iv_size = crypto_skcipher_ivsize(tfm); 905 906 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 907 crypto_skcipher_set_flags(child, tfm->base.crt_flags & 908 CRYPTO_TFM_REQ_MASK); 909 ret = crypto_skcipher_setkey(child, key, keylen); 910 if (ret) 911 return ret; 912 913 /* Setup Encryption Security Context & Command label template */ 914 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0, 915 ad, 1, &ctx->enc.epib[1])) 916 goto badkey; 917 918 cmdl_len = sa_format_cmdl_gen(&cfg, 919 (u8 *)ctx->enc.cmdl, 920 &ctx->enc.cmdl_upd_info); 921 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 922 goto badkey; 923 924 ctx->enc.cmdl_size = cmdl_len; 925 926 /* Setup Decryption Security Context & Command label template */ 927 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0, 928 ad, 0, &ctx->dec.epib[1])) 929 goto badkey; 930 931 cfg.enc_eng_id = ad->enc_eng.eng_id; 932 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 933 &ctx->dec.cmdl_upd_info); 934 935 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 936 goto badkey; 937 938 ctx->dec.cmdl_size = cmdl_len; 939 ctx->iv_idx = ad->iv_idx; 940 941 return 0; 942 943 badkey: 944 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 945 return -EINVAL; 946 } 947 948 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 949 unsigned int keylen) 950 { 951 struct algo_data ad = { 0 }; 952 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 953 int key_idx = (keylen >> 3) - 2; 954 955 if (key_idx >= 3) 956 return -EINVAL; 957 958 ad.mci_enc = mci_cbc_enc_array[key_idx]; 959 ad.mci_dec = mci_cbc_dec_array[key_idx]; 960 ad.inv_key = true; 961 ad.ealg_id = SA_EALG_ID_AES_CBC; 962 ad.iv_idx = 4; 963 ad.iv_out_size = 16; 964 965 return sa_cipher_setkey(tfm, key, keylen, &ad); 966 } 967 968 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 969 unsigned int keylen) 970 { 971 struct algo_data ad = { 0 }; 972 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 973 int key_idx = (keylen >> 3) - 2; 974 975 if (key_idx >= 3) 976 return -EINVAL; 977 978 ad.mci_enc = mci_ecb_enc_array[key_idx]; 979 ad.mci_dec = mci_ecb_dec_array[key_idx]; 980 ad.inv_key = true; 981 ad.ealg_id = SA_EALG_ID_AES_ECB; 982 983 return sa_cipher_setkey(tfm, key, keylen, &ad); 984 } 985 986 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 987 unsigned int keylen) 988 { 989 struct algo_data ad = { 0 }; 990 991 ad.mci_enc = mci_cbc_3des_enc_array; 992 ad.mci_dec = mci_cbc_3des_dec_array; 993 ad.ealg_id = SA_EALG_ID_3DES_CBC; 994 ad.iv_idx = 6; 995 ad.iv_out_size = 8; 996 997 return sa_cipher_setkey(tfm, key, keylen, &ad); 998 } 999 1000 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 1001 unsigned int keylen) 1002 { 1003 struct algo_data ad = { 0 }; 1004 1005 ad.mci_enc = mci_ecb_3des_enc_array; 1006 ad.mci_dec = mci_ecb_3des_dec_array; 1007 1008 return sa_cipher_setkey(tfm, key, keylen, &ad); 1009 } 1010 1011 static void sa_sync_from_device(struct sa_rx_data *rxd) 1012 { 1013 struct sg_table *sgt; 1014 1015 if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) 1016 sgt = &rxd->mapped_sg[0].sgt; 1017 else 1018 sgt = &rxd->mapped_sg[1].sgt; 1019 1020 dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); 1021 } 1022 1023 static void sa_free_sa_rx_data(struct sa_rx_data *rxd) 1024 { 1025 int i; 1026 1027 for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { 1028 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; 1029 1030 if (mapped_sg->mapped) { 1031 dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, 1032 mapped_sg->dir, 0); 1033 kfree(mapped_sg->split_sg); 1034 } 1035 } 1036 1037 kfree(rxd); 1038 } 1039 1040 static void sa_aes_dma_in_callback(void *data) 1041 { 1042 struct sa_rx_data *rxd = data; 1043 struct skcipher_request *req; 1044 u32 *result; 1045 __be32 *mdptr; 1046 size_t ml, pl; 1047 int i; 1048 1049 sa_sync_from_device(rxd); 1050 req = container_of(rxd->req, struct skcipher_request, base); 1051 1052 if (req->iv) { 1053 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, 1054 &ml); 1055 result = (u32 *)req->iv; 1056 1057 for (i = 0; i < (rxd->enc_iv_size / 4); i++) 1058 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); 1059 } 1060 1061 sa_free_sa_rx_data(rxd); 1062 1063 skcipher_request_complete(req, 0); 1064 } 1065 1066 static void 1067 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) 1068 { 1069 u32 *out, *in; 1070 int i; 1071 1072 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) 1073 *out++ = *in++; 1074 1075 mdptr[4] = (0xFFFF << 16); 1076 for (out = &mdptr[5], in = psdata, i = 0; 1077 i < pslen / sizeof(u32); i++) 1078 *out++ = *in++; 1079 } 1080 1081 static int sa_run(struct sa_req *req) 1082 { 1083 struct sa_rx_data *rxd; 1084 gfp_t gfp_flags; 1085 u32 cmdl[SA_MAX_CMDL_WORDS]; 1086 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); 1087 struct device *ddev; 1088 struct dma_chan *dma_rx; 1089 int sg_nents, src_nents, dst_nents; 1090 struct scatterlist *src, *dst; 1091 size_t pl, ml, split_size; 1092 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; 1093 int ret; 1094 struct dma_async_tx_descriptor *tx_out; 1095 u32 *mdptr; 1096 bool diff_dst; 1097 enum dma_data_direction dir_src; 1098 struct sa_mapped_sg *mapped_sg; 1099 1100 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 1101 GFP_KERNEL : GFP_ATOMIC; 1102 1103 rxd = kzalloc_obj(*rxd, gfp_flags); 1104 if (!rxd) 1105 return -ENOMEM; 1106 1107 if (req->src != req->dst) { 1108 diff_dst = true; 1109 dir_src = DMA_TO_DEVICE; 1110 } else { 1111 diff_dst = false; 1112 dir_src = DMA_BIDIRECTIONAL; 1113 } 1114 1115 /* 1116 * SA2UL has an interesting feature where the receive DMA channel 1117 * is selected based on the data passed to the engine. Within the 1118 * transition range, there is also a space where it is impossible 1119 * to determine where the data will end up, and this should be 1120 * avoided. This will be handled by the SW fallback mechanism by 1121 * the individual algorithm implementations. 1122 */ 1123 if (req->size >= 256) 1124 dma_rx = pdata->dma_rx2; 1125 else 1126 dma_rx = pdata->dma_rx1; 1127 1128 ddev = dmaengine_get_dma_device(pdata->dma_tx); 1129 rxd->ddev = ddev; 1130 1131 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); 1132 1133 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); 1134 1135 if (req->type != CRYPTO_ALG_TYPE_AHASH) { 1136 if (req->enc) 1137 req->type |= 1138 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); 1139 else 1140 req->type |= 1141 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); 1142 } 1143 1144 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; 1145 1146 /* 1147 * Map the packets, first we check if the data fits into a single 1148 * sg entry and use that if possible. If it does not fit, we check 1149 * if we need to do sg_split to align the scatterlist data on the 1150 * actual data size being processed by the crypto engine. 1151 */ 1152 src = req->src; 1153 sg_nents = sg_nents_for_len(src, req->size); 1154 1155 split_size = req->size; 1156 1157 mapped_sg = &rxd->mapped_sg[0]; 1158 if (sg_nents == 1 && split_size <= req->src->length) { 1159 src = &mapped_sg->static_sg; 1160 src_nents = 1; 1161 sg_init_table(src, 1); 1162 sg_set_page(src, sg_page(req->src), split_size, 1163 req->src->offset); 1164 1165 mapped_sg->sgt.sgl = src; 1166 mapped_sg->sgt.orig_nents = src_nents; 1167 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1168 if (ret) { 1169 kfree(rxd); 1170 return ret; 1171 } 1172 1173 mapped_sg->dir = dir_src; 1174 mapped_sg->mapped = true; 1175 } else { 1176 mapped_sg->sgt.sgl = req->src; 1177 mapped_sg->sgt.orig_nents = sg_nents; 1178 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1179 if (ret) { 1180 kfree(rxd); 1181 return ret; 1182 } 1183 1184 mapped_sg->dir = dir_src; 1185 mapped_sg->mapped = true; 1186 1187 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, 1188 &split_size, &src, &src_nents, gfp_flags); 1189 if (ret) { 1190 src_nents = mapped_sg->sgt.nents; 1191 src = mapped_sg->sgt.sgl; 1192 } else { 1193 mapped_sg->split_sg = src; 1194 } 1195 } 1196 1197 dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); 1198 1199 if (!diff_dst) { 1200 dst_nents = src_nents; 1201 dst = src; 1202 } else { 1203 dst_nents = sg_nents_for_len(req->dst, req->size); 1204 mapped_sg = &rxd->mapped_sg[1]; 1205 1206 if (dst_nents == 1 && split_size <= req->dst->length) { 1207 dst = &mapped_sg->static_sg; 1208 dst_nents = 1; 1209 sg_init_table(dst, 1); 1210 sg_set_page(dst, sg_page(req->dst), split_size, 1211 req->dst->offset); 1212 1213 mapped_sg->sgt.sgl = dst; 1214 mapped_sg->sgt.orig_nents = dst_nents; 1215 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1216 DMA_FROM_DEVICE, 0); 1217 if (ret) 1218 goto err_cleanup; 1219 1220 mapped_sg->dir = DMA_FROM_DEVICE; 1221 mapped_sg->mapped = true; 1222 } else { 1223 mapped_sg->sgt.sgl = req->dst; 1224 mapped_sg->sgt.orig_nents = dst_nents; 1225 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1226 DMA_FROM_DEVICE, 0); 1227 if (ret) 1228 goto err_cleanup; 1229 1230 mapped_sg->dir = DMA_FROM_DEVICE; 1231 mapped_sg->mapped = true; 1232 1233 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 1234 0, 1, &split_size, &dst, &dst_nents, 1235 gfp_flags); 1236 if (ret) { 1237 dst_nents = mapped_sg->sgt.nents; 1238 dst = mapped_sg->sgt.sgl; 1239 } else { 1240 mapped_sg->split_sg = dst; 1241 } 1242 } 1243 } 1244 1245 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, 1246 DMA_DEV_TO_MEM, 1247 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1248 if (!rxd->tx_in) { 1249 dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); 1250 ret = -EINVAL; 1251 goto err_cleanup; 1252 } 1253 1254 rxd->req = (void *)req->base; 1255 rxd->enc = req->enc; 1256 rxd->iv_idx = req->ctx->iv_idx; 1257 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; 1258 rxd->tx_in->callback = req->callback; 1259 rxd->tx_in->callback_param = rxd; 1260 1261 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, 1262 src_nents, DMA_MEM_TO_DEV, 1263 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1264 1265 if (!tx_out) { 1266 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); 1267 ret = -EINVAL; 1268 goto err_cleanup; 1269 } 1270 1271 /* 1272 * Prepare metadata for DMA engine. This essentially describes the 1273 * crypto algorithm to be used, data sizes, different keys etc. 1274 */ 1275 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); 1276 1277 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * 1278 sizeof(u32))), cmdl, sizeof(sa_ctx->epib), 1279 sa_ctx->epib); 1280 1281 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); 1282 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); 1283 1284 dmaengine_submit(tx_out); 1285 dmaengine_submit(rxd->tx_in); 1286 1287 dma_async_issue_pending(dma_rx); 1288 dma_async_issue_pending(pdata->dma_tx); 1289 1290 return -EINPROGRESS; 1291 1292 err_cleanup: 1293 sa_free_sa_rx_data(rxd); 1294 1295 return ret; 1296 } 1297 1298 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) 1299 { 1300 struct sa_tfm_ctx *ctx = 1301 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1302 struct crypto_alg *alg = req->base.tfm->__crt_alg; 1303 struct sa_req sa_req = { 0 }; 1304 1305 if (!req->cryptlen) 1306 return 0; 1307 1308 if (req->cryptlen % alg->cra_blocksize) 1309 return -EINVAL; 1310 1311 /* Use SW fallback if the data size is not supported */ 1312 if (req->cryptlen > SA_MAX_DATA_SZ || 1313 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && 1314 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { 1315 struct skcipher_request *subreq = skcipher_request_ctx(req); 1316 1317 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); 1318 skcipher_request_set_callback(subreq, req->base.flags, 1319 req->base.complete, 1320 req->base.data); 1321 skcipher_request_set_crypt(subreq, req->src, req->dst, 1322 req->cryptlen, req->iv); 1323 if (enc) 1324 return crypto_skcipher_encrypt(subreq); 1325 else 1326 return crypto_skcipher_decrypt(subreq); 1327 } 1328 1329 sa_req.size = req->cryptlen; 1330 sa_req.enc_size = req->cryptlen; 1331 sa_req.src = req->src; 1332 sa_req.dst = req->dst; 1333 sa_req.enc_iv = iv; 1334 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; 1335 sa_req.enc = enc; 1336 sa_req.callback = sa_aes_dma_in_callback; 1337 sa_req.mdata_size = 44; 1338 sa_req.base = &req->base; 1339 sa_req.ctx = ctx; 1340 1341 return sa_run(&sa_req); 1342 } 1343 1344 static int sa_encrypt(struct skcipher_request *req) 1345 { 1346 return sa_cipher_run(req, req->iv, 1); 1347 } 1348 1349 static int sa_decrypt(struct skcipher_request *req) 1350 { 1351 return sa_cipher_run(req, req->iv, 0); 1352 } 1353 1354 static void sa_sha_dma_in_callback(void *data) 1355 { 1356 struct sa_rx_data *rxd = data; 1357 struct ahash_request *req; 1358 struct crypto_ahash *tfm; 1359 unsigned int authsize; 1360 int i; 1361 size_t ml, pl; 1362 u32 *result; 1363 __be32 *mdptr; 1364 1365 sa_sync_from_device(rxd); 1366 req = container_of(rxd->req, struct ahash_request, base); 1367 tfm = crypto_ahash_reqtfm(req); 1368 authsize = crypto_ahash_digestsize(tfm); 1369 1370 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1371 result = (u32 *)req->result; 1372 1373 for (i = 0; i < (authsize / 4); i++) 1374 result[i] = be32_to_cpu(mdptr[i + 4]); 1375 1376 sa_free_sa_rx_data(rxd); 1377 1378 ahash_request_complete(req, 0); 1379 } 1380 1381 static int zero_message_process(struct ahash_request *req) 1382 { 1383 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1384 int sa_digest_size = crypto_ahash_digestsize(tfm); 1385 1386 switch (sa_digest_size) { 1387 case SHA1_DIGEST_SIZE: 1388 memcpy(req->result, sha1_zero_message_hash, sa_digest_size); 1389 break; 1390 case SHA256_DIGEST_SIZE: 1391 memcpy(req->result, sha256_zero_message_hash, sa_digest_size); 1392 break; 1393 case SHA512_DIGEST_SIZE: 1394 memcpy(req->result, sha512_zero_message_hash, sa_digest_size); 1395 break; 1396 default: 1397 return -EINVAL; 1398 } 1399 1400 return 0; 1401 } 1402 1403 static int sa_sha_run(struct ahash_request *req) 1404 { 1405 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 1406 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1407 struct sa_req sa_req = { 0 }; 1408 size_t auth_len; 1409 1410 auth_len = req->nbytes; 1411 1412 if (!auth_len) 1413 return zero_message_process(req); 1414 1415 if (auth_len > SA_MAX_DATA_SZ || 1416 (auth_len >= SA_UNSAFE_DATA_SZ_MIN && 1417 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { 1418 struct ahash_request *subreq = &rctx->fallback_req; 1419 int ret; 1420 1421 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1422 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1423 ahash_request_set_crypt(subreq, req->src, req->result, auth_len); 1424 1425 ret = crypto_ahash_digest(subreq); 1426 1427 return ret; 1428 } 1429 1430 sa_req.size = auth_len; 1431 sa_req.auth_size = auth_len; 1432 sa_req.src = req->src; 1433 sa_req.dst = req->src; 1434 sa_req.enc = true; 1435 sa_req.type = CRYPTO_ALG_TYPE_AHASH; 1436 sa_req.callback = sa_sha_dma_in_callback; 1437 sa_req.mdata_size = 28; 1438 sa_req.ctx = ctx; 1439 sa_req.base = &req->base; 1440 1441 return sa_run(&sa_req); 1442 } 1443 1444 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) 1445 { 1446 int bs = crypto_shash_blocksize(ctx->shash); 1447 int cmdl_len; 1448 struct sa_cmdl_cfg cfg; 1449 1450 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1451 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1452 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1453 1454 memset(ctx->authkey, 0, bs); 1455 memset(&cfg, 0, sizeof(cfg)); 1456 cfg.aalg = ad->aalg_id; 1457 cfg.enc_eng_id = ad->enc_eng.eng_id; 1458 cfg.auth_eng_id = ad->auth_eng.eng_id; 1459 cfg.iv_size = 0; 1460 cfg.akey = NULL; 1461 cfg.akey_len = 0; 1462 1463 ctx->dev_data = dev_get_drvdata(sa_k3_dev); 1464 /* Setup Encryption Security Context & Command label template */ 1465 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0, 1466 ad, 0, &ctx->enc.epib[1])) 1467 goto badkey; 1468 1469 cmdl_len = sa_format_cmdl_gen(&cfg, 1470 (u8 *)ctx->enc.cmdl, 1471 &ctx->enc.cmdl_upd_info); 1472 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1473 goto badkey; 1474 1475 ctx->enc.cmdl_size = cmdl_len; 1476 1477 return 0; 1478 1479 badkey: 1480 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 1481 return -EINVAL; 1482 } 1483 1484 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 1485 { 1486 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1487 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1488 int ret; 1489 1490 memset(ctx, 0, sizeof(*ctx)); 1491 ctx->dev_data = data; 1492 ret = sa_init_ctx_info(&ctx->enc, data); 1493 if (ret) 1494 return ret; 1495 1496 if (alg_base) { 1497 ctx->shash = crypto_alloc_shash(alg_base, 0, 0); 1498 if (IS_ERR(ctx->shash)) { 1499 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", 1500 alg_base); 1501 return PTR_ERR(ctx->shash); 1502 } 1503 /* for fallback */ 1504 ctx->fallback.ahash = 1505 crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); 1506 if (IS_ERR(ctx->fallback.ahash)) { 1507 dev_err(ctx->dev_data->dev, 1508 "Could not load fallback driver\n"); 1509 return PTR_ERR(ctx->fallback.ahash); 1510 } 1511 } 1512 1513 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1514 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1515 ctx->dec.sc_id, &ctx->dec.sc_phys); 1516 1517 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1518 sizeof(struct sa_sha_req_ctx) + 1519 crypto_ahash_reqsize(ctx->fallback.ahash)); 1520 1521 return 0; 1522 } 1523 1524 static int sa_sha_digest(struct ahash_request *req) 1525 { 1526 return sa_sha_run(req); 1527 } 1528 1529 static int sa_sha_init(struct ahash_request *req) 1530 { 1531 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1532 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1533 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1534 1535 dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", 1536 crypto_ahash_digestsize(tfm), rctx); 1537 1538 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1539 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1540 ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); 1541 1542 return crypto_ahash_init(&rctx->fallback_req); 1543 } 1544 1545 static int sa_sha_update(struct ahash_request *req) 1546 { 1547 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1548 1549 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1550 ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); 1551 1552 return crypto_ahash_update(&rctx->fallback_req); 1553 } 1554 1555 static int sa_sha_final(struct ahash_request *req) 1556 { 1557 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1558 1559 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1560 ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); 1561 1562 return crypto_ahash_final(&rctx->fallback_req); 1563 } 1564 1565 static int sa_sha_finup(struct ahash_request *req) 1566 { 1567 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1568 1569 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1570 ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); 1571 1572 return crypto_ahash_finup(&rctx->fallback_req); 1573 } 1574 1575 static int sa_sha_import(struct ahash_request *req, const void *in) 1576 { 1577 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1578 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1579 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1580 1581 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1582 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1583 1584 return crypto_ahash_import(&rctx->fallback_req, in); 1585 } 1586 1587 static int sa_sha_export(struct ahash_request *req, void *out) 1588 { 1589 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1590 struct ahash_request *subreq = &rctx->fallback_req; 1591 1592 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1593 1594 return crypto_ahash_export(subreq, out); 1595 } 1596 1597 static int sa_sha1_cra_init(struct crypto_tfm *tfm) 1598 { 1599 struct algo_data ad = { 0 }; 1600 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1601 1602 sa_sha_cra_init_alg(tfm, "sha1"); 1603 1604 ad.aalg_id = SA_AALG_ID_SHA1; 1605 ad.hash_size = SHA1_DIGEST_SIZE; 1606 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1607 1608 sa_sha_setup(ctx, &ad); 1609 1610 return 0; 1611 } 1612 1613 static int sa_sha256_cra_init(struct crypto_tfm *tfm) 1614 { 1615 struct algo_data ad = { 0 }; 1616 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1617 1618 sa_sha_cra_init_alg(tfm, "sha256"); 1619 1620 ad.aalg_id = SA_AALG_ID_SHA2_256; 1621 ad.hash_size = SHA256_DIGEST_SIZE; 1622 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1623 1624 sa_sha_setup(ctx, &ad); 1625 1626 return 0; 1627 } 1628 1629 static int sa_sha512_cra_init(struct crypto_tfm *tfm) 1630 { 1631 struct algo_data ad = { 0 }; 1632 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1633 1634 sa_sha_cra_init_alg(tfm, "sha512"); 1635 1636 ad.aalg_id = SA_AALG_ID_SHA2_512; 1637 ad.hash_size = SHA512_DIGEST_SIZE; 1638 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; 1639 1640 sa_sha_setup(ctx, &ad); 1641 1642 return 0; 1643 } 1644 1645 static void sa_sha_cra_exit(struct crypto_tfm *tfm) 1646 { 1647 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1648 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1649 1650 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1651 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1652 ctx->dec.sc_id, &ctx->dec.sc_phys); 1653 1654 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) 1655 sa_free_ctx_info(&ctx->enc, data); 1656 1657 crypto_free_shash(ctx->shash); 1658 crypto_free_ahash(ctx->fallback.ahash); 1659 } 1660 1661 static void sa_aead_dma_in_callback(void *data) 1662 { 1663 struct sa_rx_data *rxd = data; 1664 struct aead_request *req; 1665 struct crypto_aead *tfm; 1666 unsigned int start; 1667 unsigned int authsize; 1668 u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; 1669 size_t pl, ml; 1670 int i; 1671 int err = 0; 1672 u32 *mdptr; 1673 1674 sa_sync_from_device(rxd); 1675 req = container_of(rxd->req, struct aead_request, base); 1676 tfm = crypto_aead_reqtfm(req); 1677 start = req->assoclen + req->cryptlen; 1678 authsize = crypto_aead_authsize(tfm); 1679 1680 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1681 for (i = 0; i < (authsize / 4); i++) 1682 mdptr[i + 4] = swab32(mdptr[i + 4]); 1683 1684 if (rxd->enc) { 1685 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1686 1); 1687 } else { 1688 start -= authsize; 1689 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 1690 0); 1691 1692 err = crypto_memneq(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; 1693 } 1694 1695 sa_free_sa_rx_data(rxd); 1696 1697 aead_request_complete(req, err); 1698 } 1699 1700 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, 1701 const char *fallback) 1702 { 1703 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1704 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1705 int ret; 1706 1707 memzero_explicit(ctx, sizeof(*ctx)); 1708 ctx->dev_data = data; 1709 1710 ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK); 1711 if (IS_ERR(ctx->shash)) { 1712 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash); 1713 return PTR_ERR(ctx->shash); 1714 } 1715 1716 ctx->fallback.aead = crypto_alloc_aead(fallback, 0, 1717 CRYPTO_ALG_NEED_FALLBACK); 1718 1719 if (IS_ERR(ctx->fallback.aead)) { 1720 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", 1721 fallback); 1722 return PTR_ERR(ctx->fallback.aead); 1723 } 1724 1725 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + 1726 crypto_aead_reqsize(ctx->fallback.aead)); 1727 1728 ret = sa_init_ctx_info(&ctx->enc, data); 1729 if (ret) 1730 return ret; 1731 1732 ret = sa_init_ctx_info(&ctx->dec, data); 1733 if (ret) { 1734 sa_free_ctx_info(&ctx->enc, data); 1735 return ret; 1736 } 1737 1738 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1739 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1740 ctx->dec.sc_id, &ctx->dec.sc_phys); 1741 1742 return ret; 1743 } 1744 1745 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) 1746 { 1747 return sa_cra_init_aead(tfm, "sha1", 1748 "authenc(hmac(sha1),cbc(aes))"); 1749 } 1750 1751 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) 1752 { 1753 return sa_cra_init_aead(tfm, "sha256", 1754 "authenc(hmac(sha256),cbc(aes))"); 1755 } 1756 1757 static void sa_exit_tfm_aead(struct crypto_aead *tfm) 1758 { 1759 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1760 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1761 1762 crypto_free_shash(ctx->shash); 1763 crypto_free_aead(ctx->fallback.aead); 1764 1765 sa_free_ctx_info(&ctx->enc, data); 1766 sa_free_ctx_info(&ctx->dec, data); 1767 } 1768 1769 /* AEAD algorithm configuration interface function */ 1770 static int sa_aead_setkey(struct crypto_aead *authenc, 1771 const u8 *key, unsigned int keylen, 1772 struct algo_data *ad) 1773 { 1774 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); 1775 struct crypto_authenc_keys keys; 1776 int cmdl_len; 1777 struct sa_cmdl_cfg cfg; 1778 int key_idx; 1779 1780 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1781 return -EINVAL; 1782 1783 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 1784 key_idx = (keys.enckeylen >> 3) - 2; 1785 if (key_idx >= 3) 1786 return -EINVAL; 1787 1788 ad->ctx = ctx; 1789 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 1790 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1791 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1792 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1793 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; 1794 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; 1795 ad->inv_key = true; 1796 ad->keyed_mac = true; 1797 ad->ealg_id = SA_EALG_ID_AES_CBC; 1798 ad->prep_iopad = sa_prepare_iopads; 1799 1800 memset(&cfg, 0, sizeof(cfg)); 1801 cfg.enc = true; 1802 cfg.aalg = ad->aalg_id; 1803 cfg.enc_eng_id = ad->enc_eng.eng_id; 1804 cfg.auth_eng_id = ad->auth_eng.eng_id; 1805 cfg.iv_size = crypto_aead_ivsize(authenc); 1806 cfg.akey = keys.authkey; 1807 cfg.akey_len = keys.authkeylen; 1808 1809 /* Setup Encryption Security Context & Command label template */ 1810 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey, 1811 keys.enckeylen, keys.authkey, keys.authkeylen, 1812 ad, 1, &ctx->enc.epib[1])) 1813 return -EINVAL; 1814 1815 cmdl_len = sa_format_cmdl_gen(&cfg, 1816 (u8 *)ctx->enc.cmdl, 1817 &ctx->enc.cmdl_upd_info); 1818 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1819 return -EINVAL; 1820 1821 ctx->enc.cmdl_size = cmdl_len; 1822 1823 /* Setup Decryption Security Context & Command label template */ 1824 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey, 1825 keys.enckeylen, keys.authkey, keys.authkeylen, 1826 ad, 0, &ctx->dec.epib[1])) 1827 return -EINVAL; 1828 1829 cfg.enc = false; 1830 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 1831 &ctx->dec.cmdl_upd_info); 1832 1833 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1834 return -EINVAL; 1835 1836 ctx->dec.cmdl_size = cmdl_len; 1837 1838 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); 1839 crypto_aead_set_flags(ctx->fallback.aead, 1840 crypto_aead_get_flags(authenc) & 1841 CRYPTO_TFM_REQ_MASK); 1842 1843 return crypto_aead_setkey(ctx->fallback.aead, key, keylen); 1844 } 1845 1846 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1847 { 1848 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); 1849 1850 return crypto_aead_setauthsize(ctx->fallback.aead, authsize); 1851 } 1852 1853 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, 1854 const u8 *key, unsigned int keylen) 1855 { 1856 struct algo_data ad = { 0 }; 1857 1858 ad.ealg_id = SA_EALG_ID_AES_CBC; 1859 ad.aalg_id = SA_AALG_ID_HMAC_SHA1; 1860 ad.hash_size = SHA1_DIGEST_SIZE; 1861 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1862 1863 return sa_aead_setkey(authenc, key, keylen, &ad); 1864 } 1865 1866 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, 1867 const u8 *key, unsigned int keylen) 1868 { 1869 struct algo_data ad = { 0 }; 1870 1871 ad.ealg_id = SA_EALG_ID_AES_CBC; 1872 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; 1873 ad.hash_size = SHA256_DIGEST_SIZE; 1874 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1875 1876 return sa_aead_setkey(authenc, key, keylen, &ad); 1877 } 1878 1879 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) 1880 { 1881 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1882 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1883 struct sa_req sa_req = { 0 }; 1884 size_t auth_size, enc_size; 1885 1886 enc_size = req->cryptlen; 1887 auth_size = req->assoclen + req->cryptlen; 1888 1889 if (!enc) { 1890 enc_size -= crypto_aead_authsize(tfm); 1891 auth_size -= crypto_aead_authsize(tfm); 1892 } 1893 1894 if (auth_size > SA_MAX_DATA_SZ || 1895 (auth_size >= SA_UNSAFE_DATA_SZ_MIN && 1896 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { 1897 struct aead_request *subreq = aead_request_ctx(req); 1898 int ret; 1899 1900 aead_request_set_tfm(subreq, ctx->fallback.aead); 1901 aead_request_set_callback(subreq, req->base.flags, 1902 req->base.complete, req->base.data); 1903 aead_request_set_crypt(subreq, req->src, req->dst, 1904 req->cryptlen, req->iv); 1905 aead_request_set_ad(subreq, req->assoclen); 1906 1907 ret = enc ? crypto_aead_encrypt(subreq) : 1908 crypto_aead_decrypt(subreq); 1909 return ret; 1910 } 1911 1912 sa_req.enc_offset = req->assoclen; 1913 sa_req.enc_size = enc_size; 1914 sa_req.auth_size = auth_size; 1915 sa_req.size = auth_size; 1916 sa_req.enc_iv = iv; 1917 sa_req.type = CRYPTO_ALG_TYPE_AEAD; 1918 sa_req.enc = enc; 1919 sa_req.callback = sa_aead_dma_in_callback; 1920 sa_req.mdata_size = 52; 1921 sa_req.base = &req->base; 1922 sa_req.ctx = ctx; 1923 sa_req.src = req->src; 1924 sa_req.dst = req->dst; 1925 1926 return sa_run(&sa_req); 1927 } 1928 1929 /* AEAD algorithm encrypt interface function */ 1930 static int sa_aead_encrypt(struct aead_request *req) 1931 { 1932 return sa_aead_run(req, req->iv, 1); 1933 } 1934 1935 /* AEAD algorithm decrypt interface function */ 1936 static int sa_aead_decrypt(struct aead_request *req) 1937 { 1938 return sa_aead_run(req, req->iv, 0); 1939 } 1940 1941 static struct sa_alg_tmpl sa_algs[] = { 1942 [SA_ALG_CBC_AES] = { 1943 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1944 .alg.skcipher = { 1945 .base.cra_name = "cbc(aes)", 1946 .base.cra_driver_name = "cbc-aes-sa2ul", 1947 .base.cra_priority = 30000, 1948 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1949 CRYPTO_ALG_KERN_DRIVER_ONLY | 1950 CRYPTO_ALG_ASYNC | 1951 CRYPTO_ALG_NEED_FALLBACK, 1952 .base.cra_blocksize = AES_BLOCK_SIZE, 1953 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1954 .base.cra_module = THIS_MODULE, 1955 .init = sa_cipher_cra_init, 1956 .exit = sa_cipher_cra_exit, 1957 .min_keysize = AES_MIN_KEY_SIZE, 1958 .max_keysize = AES_MAX_KEY_SIZE, 1959 .ivsize = AES_BLOCK_SIZE, 1960 .setkey = sa_aes_cbc_setkey, 1961 .encrypt = sa_encrypt, 1962 .decrypt = sa_decrypt, 1963 } 1964 }, 1965 [SA_ALG_EBC_AES] = { 1966 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1967 .alg.skcipher = { 1968 .base.cra_name = "ecb(aes)", 1969 .base.cra_driver_name = "ecb-aes-sa2ul", 1970 .base.cra_priority = 30000, 1971 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1972 CRYPTO_ALG_KERN_DRIVER_ONLY | 1973 CRYPTO_ALG_ASYNC | 1974 CRYPTO_ALG_NEED_FALLBACK, 1975 .base.cra_blocksize = AES_BLOCK_SIZE, 1976 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1977 .base.cra_module = THIS_MODULE, 1978 .init = sa_cipher_cra_init, 1979 .exit = sa_cipher_cra_exit, 1980 .min_keysize = AES_MIN_KEY_SIZE, 1981 .max_keysize = AES_MAX_KEY_SIZE, 1982 .setkey = sa_aes_ecb_setkey, 1983 .encrypt = sa_encrypt, 1984 .decrypt = sa_decrypt, 1985 } 1986 }, 1987 [SA_ALG_CBC_DES3] = { 1988 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1989 .alg.skcipher = { 1990 .base.cra_name = "cbc(des3_ede)", 1991 .base.cra_driver_name = "cbc-des3-sa2ul", 1992 .base.cra_priority = 30000, 1993 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1994 CRYPTO_ALG_KERN_DRIVER_ONLY | 1995 CRYPTO_ALG_ASYNC | 1996 CRYPTO_ALG_NEED_FALLBACK, 1997 .base.cra_blocksize = DES_BLOCK_SIZE, 1998 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1999 .base.cra_module = THIS_MODULE, 2000 .init = sa_cipher_cra_init, 2001 .exit = sa_cipher_cra_exit, 2002 .min_keysize = 3 * DES_KEY_SIZE, 2003 .max_keysize = 3 * DES_KEY_SIZE, 2004 .ivsize = DES_BLOCK_SIZE, 2005 .setkey = sa_3des_cbc_setkey, 2006 .encrypt = sa_encrypt, 2007 .decrypt = sa_decrypt, 2008 } 2009 }, 2010 [SA_ALG_ECB_DES3] = { 2011 .type = CRYPTO_ALG_TYPE_SKCIPHER, 2012 .alg.skcipher = { 2013 .base.cra_name = "ecb(des3_ede)", 2014 .base.cra_driver_name = "ecb-des3-sa2ul", 2015 .base.cra_priority = 30000, 2016 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2017 CRYPTO_ALG_KERN_DRIVER_ONLY | 2018 CRYPTO_ALG_ASYNC | 2019 CRYPTO_ALG_NEED_FALLBACK, 2020 .base.cra_blocksize = DES_BLOCK_SIZE, 2021 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 2022 .base.cra_module = THIS_MODULE, 2023 .init = sa_cipher_cra_init, 2024 .exit = sa_cipher_cra_exit, 2025 .min_keysize = 3 * DES_KEY_SIZE, 2026 .max_keysize = 3 * DES_KEY_SIZE, 2027 .setkey = sa_3des_ecb_setkey, 2028 .encrypt = sa_encrypt, 2029 .decrypt = sa_decrypt, 2030 } 2031 }, 2032 [SA_ALG_SHA1] = { 2033 .type = CRYPTO_ALG_TYPE_AHASH, 2034 .alg.ahash = { 2035 .halg.base = { 2036 .cra_name = "sha1", 2037 .cra_driver_name = "sha1-sa2ul", 2038 .cra_priority = 400, 2039 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2040 CRYPTO_ALG_ASYNC | 2041 CRYPTO_ALG_KERN_DRIVER_ONLY | 2042 CRYPTO_ALG_NEED_FALLBACK, 2043 .cra_blocksize = SHA1_BLOCK_SIZE, 2044 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2045 .cra_module = THIS_MODULE, 2046 .cra_init = sa_sha1_cra_init, 2047 .cra_exit = sa_sha_cra_exit, 2048 }, 2049 .halg.digestsize = SHA1_DIGEST_SIZE, 2050 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2051 sizeof(struct sha1_state), 2052 .init = sa_sha_init, 2053 .update = sa_sha_update, 2054 .final = sa_sha_final, 2055 .finup = sa_sha_finup, 2056 .digest = sa_sha_digest, 2057 .export = sa_sha_export, 2058 .import = sa_sha_import, 2059 }, 2060 }, 2061 [SA_ALG_SHA256] = { 2062 .type = CRYPTO_ALG_TYPE_AHASH, 2063 .alg.ahash = { 2064 .halg.base = { 2065 .cra_name = "sha256", 2066 .cra_driver_name = "sha256-sa2ul", 2067 .cra_priority = 400, 2068 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2069 CRYPTO_ALG_ASYNC | 2070 CRYPTO_ALG_KERN_DRIVER_ONLY | 2071 CRYPTO_ALG_NEED_FALLBACK, 2072 .cra_blocksize = SHA256_BLOCK_SIZE, 2073 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2074 .cra_module = THIS_MODULE, 2075 .cra_init = sa_sha256_cra_init, 2076 .cra_exit = sa_sha_cra_exit, 2077 }, 2078 .halg.digestsize = SHA256_DIGEST_SIZE, 2079 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2080 sizeof(struct sha256_state), 2081 .init = sa_sha_init, 2082 .update = sa_sha_update, 2083 .final = sa_sha_final, 2084 .finup = sa_sha_finup, 2085 .digest = sa_sha_digest, 2086 .export = sa_sha_export, 2087 .import = sa_sha_import, 2088 }, 2089 }, 2090 [SA_ALG_SHA512] = { 2091 .type = CRYPTO_ALG_TYPE_AHASH, 2092 .alg.ahash = { 2093 .halg.base = { 2094 .cra_name = "sha512", 2095 .cra_driver_name = "sha512-sa2ul", 2096 .cra_priority = 400, 2097 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2098 CRYPTO_ALG_ASYNC | 2099 CRYPTO_ALG_KERN_DRIVER_ONLY | 2100 CRYPTO_ALG_NEED_FALLBACK, 2101 .cra_blocksize = SHA512_BLOCK_SIZE, 2102 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2103 .cra_module = THIS_MODULE, 2104 .cra_init = sa_sha512_cra_init, 2105 .cra_exit = sa_sha_cra_exit, 2106 }, 2107 .halg.digestsize = SHA512_DIGEST_SIZE, 2108 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2109 sizeof(struct sha512_state), 2110 .init = sa_sha_init, 2111 .update = sa_sha_update, 2112 .final = sa_sha_final, 2113 .finup = sa_sha_finup, 2114 .digest = sa_sha_digest, 2115 .export = sa_sha_export, 2116 .import = sa_sha_import, 2117 }, 2118 }, 2119 [SA_ALG_AUTHENC_SHA1_AES] = { 2120 .type = CRYPTO_ALG_TYPE_AEAD, 2121 .alg.aead = { 2122 .base = { 2123 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2124 .cra_driver_name = 2125 "authenc(hmac(sha1),cbc(aes))-sa2ul", 2126 .cra_blocksize = AES_BLOCK_SIZE, 2127 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2128 CRYPTO_ALG_KERN_DRIVER_ONLY | 2129 CRYPTO_ALG_ASYNC | 2130 CRYPTO_ALG_NEED_FALLBACK, 2131 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2132 .cra_module = THIS_MODULE, 2133 .cra_priority = 3000, 2134 }, 2135 .ivsize = AES_BLOCK_SIZE, 2136 .maxauthsize = SHA1_DIGEST_SIZE, 2137 2138 .init = sa_cra_init_aead_sha1, 2139 .exit = sa_exit_tfm_aead, 2140 .setkey = sa_aead_cbc_sha1_setkey, 2141 .setauthsize = sa_aead_setauthsize, 2142 .encrypt = sa_aead_encrypt, 2143 .decrypt = sa_aead_decrypt, 2144 }, 2145 }, 2146 [SA_ALG_AUTHENC_SHA256_AES] = { 2147 .type = CRYPTO_ALG_TYPE_AEAD, 2148 .alg.aead = { 2149 .base = { 2150 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2151 .cra_driver_name = 2152 "authenc(hmac(sha256),cbc(aes))-sa2ul", 2153 .cra_blocksize = AES_BLOCK_SIZE, 2154 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2155 CRYPTO_ALG_KERN_DRIVER_ONLY | 2156 CRYPTO_ALG_ASYNC | 2157 CRYPTO_ALG_NEED_FALLBACK, 2158 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2159 .cra_module = THIS_MODULE, 2160 .cra_alignmask = 0, 2161 .cra_priority = 3000, 2162 }, 2163 .ivsize = AES_BLOCK_SIZE, 2164 .maxauthsize = SHA256_DIGEST_SIZE, 2165 2166 .init = sa_cra_init_aead_sha256, 2167 .exit = sa_exit_tfm_aead, 2168 .setkey = sa_aead_cbc_sha256_setkey, 2169 .setauthsize = sa_aead_setauthsize, 2170 .encrypt = sa_aead_encrypt, 2171 .decrypt = sa_aead_decrypt, 2172 }, 2173 }, 2174 }; 2175 2176 /* Register the algorithms in crypto framework */ 2177 static void sa_register_algos(struct sa_crypto_data *dev_data) 2178 { 2179 const struct sa_match_data *match_data = dev_data->match_data; 2180 struct device *dev = dev_data->dev; 2181 char *alg_name; 2182 u32 type; 2183 int i, err; 2184 2185 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2186 /* Skip unsupported algos */ 2187 if (!(match_data->supported_algos & BIT(i))) 2188 continue; 2189 2190 type = sa_algs[i].type; 2191 if (type == CRYPTO_ALG_TYPE_SKCIPHER) { 2192 alg_name = sa_algs[i].alg.skcipher.base.cra_name; 2193 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); 2194 } else if (type == CRYPTO_ALG_TYPE_AHASH) { 2195 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; 2196 err = crypto_register_ahash(&sa_algs[i].alg.ahash); 2197 } else if (type == CRYPTO_ALG_TYPE_AEAD) { 2198 alg_name = sa_algs[i].alg.aead.base.cra_name; 2199 err = crypto_register_aead(&sa_algs[i].alg.aead); 2200 } else { 2201 dev_err(dev, 2202 "un-supported crypto algorithm (%d)", 2203 sa_algs[i].type); 2204 continue; 2205 } 2206 2207 if (err) 2208 dev_err(dev, "Failed to register '%s'\n", alg_name); 2209 else 2210 sa_algs[i].registered = true; 2211 } 2212 } 2213 2214 /* Unregister the algorithms in crypto framework */ 2215 static void sa_unregister_algos(const struct device *dev) 2216 { 2217 u32 type; 2218 int i; 2219 2220 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2221 type = sa_algs[i].type; 2222 if (!sa_algs[i].registered) 2223 continue; 2224 if (type == CRYPTO_ALG_TYPE_SKCIPHER) 2225 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); 2226 else if (type == CRYPTO_ALG_TYPE_AHASH) 2227 crypto_unregister_ahash(&sa_algs[i].alg.ahash); 2228 else if (type == CRYPTO_ALG_TYPE_AEAD) 2229 crypto_unregister_aead(&sa_algs[i].alg.aead); 2230 2231 sa_algs[i].registered = false; 2232 } 2233 } 2234 2235 static int sa_init_mem(struct sa_crypto_data *dev_data) 2236 { 2237 struct device *dev = &dev_data->pdev->dev; 2238 /* Setup dma pool for security context buffers */ 2239 dev_data->sc_pool = dma_pool_create("keystone-sc", dev, 2240 SA_CTX_MAX_SZ, 64, 0); 2241 if (!dev_data->sc_pool) { 2242 dev_err(dev, "Failed to create dma pool"); 2243 return -ENOMEM; 2244 } 2245 2246 return 0; 2247 } 2248 2249 static int sa_dma_init(struct sa_crypto_data *dd) 2250 { 2251 int ret; 2252 struct dma_slave_config cfg; 2253 2254 dd->dma_rx1 = NULL; 2255 dd->dma_tx = NULL; 2256 dd->dma_rx2 = NULL; 2257 2258 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); 2259 if (ret) 2260 return ret; 2261 2262 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); 2263 if (IS_ERR(dd->dma_rx1)) 2264 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), 2265 "Unable to request rx1 DMA channel\n"); 2266 2267 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); 2268 if (IS_ERR(dd->dma_rx2)) { 2269 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), 2270 "Unable to request rx2 DMA channel\n"); 2271 goto err_dma_rx2; 2272 } 2273 2274 dd->dma_tx = dma_request_chan(dd->dev, "tx"); 2275 if (IS_ERR(dd->dma_tx)) { 2276 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), 2277 "Unable to request tx DMA channel\n"); 2278 goto err_dma_tx; 2279 } 2280 2281 memzero_explicit(&cfg, sizeof(cfg)); 2282 2283 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2284 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2285 cfg.src_maxburst = 4; 2286 cfg.dst_maxburst = 4; 2287 2288 ret = dmaengine_slave_config(dd->dma_rx1, &cfg); 2289 if (ret) { 2290 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2291 ret); 2292 goto err_dma_config; 2293 } 2294 2295 ret = dmaengine_slave_config(dd->dma_rx2, &cfg); 2296 if (ret) { 2297 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2298 ret); 2299 goto err_dma_config; 2300 } 2301 2302 ret = dmaengine_slave_config(dd->dma_tx, &cfg); 2303 if (ret) { 2304 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", 2305 ret); 2306 goto err_dma_config; 2307 } 2308 2309 return 0; 2310 2311 err_dma_config: 2312 dma_release_channel(dd->dma_tx); 2313 err_dma_tx: 2314 dma_release_channel(dd->dma_rx2); 2315 err_dma_rx2: 2316 dma_release_channel(dd->dma_rx1); 2317 2318 return ret; 2319 } 2320 2321 static int sa_link_child(struct device *dev, void *data) 2322 { 2323 struct device *parent = data; 2324 2325 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); 2326 2327 return 0; 2328 } 2329 2330 static struct sa_match_data am654_match_data = { 2331 .priv = 1, 2332 .priv_id = 1, 2333 .supported_algos = BIT(SA_ALG_CBC_AES) | 2334 BIT(SA_ALG_EBC_AES) | 2335 BIT(SA_ALG_CBC_DES3) | 2336 BIT(SA_ALG_ECB_DES3) | 2337 BIT(SA_ALG_SHA1) | 2338 BIT(SA_ALG_SHA256) | 2339 BIT(SA_ALG_SHA512) | 2340 BIT(SA_ALG_AUTHENC_SHA1_AES) | 2341 BIT(SA_ALG_AUTHENC_SHA256_AES), 2342 }; 2343 2344 static struct sa_match_data am64_match_data = { 2345 .priv = 0, 2346 .priv_id = 0, 2347 .supported_algos = BIT(SA_ALG_CBC_AES) | 2348 BIT(SA_ALG_EBC_AES) | 2349 BIT(SA_ALG_SHA256) | 2350 BIT(SA_ALG_SHA512) | 2351 BIT(SA_ALG_AUTHENC_SHA256_AES), 2352 }; 2353 2354 static const struct of_device_id of_match[] = { 2355 { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, }, 2356 { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, }, 2357 { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, }, 2358 { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, }, 2359 {}, 2360 }; 2361 MODULE_DEVICE_TABLE(of, of_match); 2362 2363 static int sa_ul_probe(struct platform_device *pdev) 2364 { 2365 struct device *dev = &pdev->dev; 2366 struct device_node *node = dev->of_node; 2367 static void __iomem *saul_base; 2368 struct sa_crypto_data *dev_data; 2369 u32 status, val; 2370 int ret; 2371 2372 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); 2373 if (!dev_data) 2374 return -ENOMEM; 2375 2376 dev_data->match_data = of_device_get_match_data(dev); 2377 if (!dev_data->match_data) 2378 return -ENODEV; 2379 2380 saul_base = devm_platform_ioremap_resource(pdev, 0); 2381 if (IS_ERR(saul_base)) 2382 return PTR_ERR(saul_base); 2383 2384 sa_k3_dev = dev; 2385 dev_data->dev = dev; 2386 dev_data->pdev = pdev; 2387 dev_data->base = saul_base; 2388 platform_set_drvdata(pdev, dev_data); 2389 dev_set_drvdata(sa_k3_dev, dev_data); 2390 2391 pm_runtime_enable(dev); 2392 ret = pm_runtime_resume_and_get(dev); 2393 if (ret < 0) { 2394 dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret); 2395 pm_runtime_disable(dev); 2396 return ret; 2397 } 2398 2399 ret = sa_init_mem(dev_data); 2400 if (ret) 2401 goto disable_pm; 2402 2403 ret = sa_dma_init(dev_data); 2404 if (ret) 2405 goto destroy_dma_pool; 2406 2407 spin_lock_init(&dev_data->scid_lock); 2408 2409 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | 2410 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | 2411 SA_EEC_TRNG_EN; 2412 status = readl_relaxed(saul_base + SA_ENGINE_STATUS); 2413 /* Only enable engines if all are not already enabled */ 2414 if (val & ~status) 2415 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); 2416 2417 sa_register_algos(dev_data); 2418 2419 ret = of_platform_populate(node, NULL, NULL, dev); 2420 if (ret) 2421 goto release_dma; 2422 2423 device_for_each_child(dev, dev, sa_link_child); 2424 2425 return 0; 2426 2427 release_dma: 2428 sa_unregister_algos(dev); 2429 2430 dma_release_channel(dev_data->dma_rx2); 2431 dma_release_channel(dev_data->dma_rx1); 2432 dma_release_channel(dev_data->dma_tx); 2433 2434 destroy_dma_pool: 2435 dma_pool_destroy(dev_data->sc_pool); 2436 2437 disable_pm: 2438 pm_runtime_put_sync(dev); 2439 pm_runtime_disable(dev); 2440 2441 return ret; 2442 } 2443 2444 static void sa_ul_remove(struct platform_device *pdev) 2445 { 2446 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); 2447 2448 of_platform_depopulate(&pdev->dev); 2449 2450 sa_unregister_algos(&pdev->dev); 2451 2452 dma_release_channel(dev_data->dma_rx2); 2453 dma_release_channel(dev_data->dma_rx1); 2454 dma_release_channel(dev_data->dma_tx); 2455 2456 dma_pool_destroy(dev_data->sc_pool); 2457 2458 platform_set_drvdata(pdev, NULL); 2459 2460 pm_runtime_put_sync(&pdev->dev); 2461 pm_runtime_disable(&pdev->dev); 2462 } 2463 2464 static struct platform_driver sa_ul_driver = { 2465 .probe = sa_ul_probe, 2466 .remove = sa_ul_remove, 2467 .driver = { 2468 .name = "saul-crypto", 2469 .of_match_table = of_match, 2470 }, 2471 }; 2472 module_platform_driver(sa_ul_driver); 2473 MODULE_DESCRIPTION("K3 SA2UL crypto accelerator driver"); 2474 MODULE_LICENSE("GPL v2"); 2475