1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 SA2UL crypto accelerator driver 4 * 5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com 6 * 7 * Authors: Keerthy 8 * Vitaly Andrianov 9 * Tero Kristo 10 */ 11 #include <linux/bitfield.h> 12 #include <linux/clk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/dmapool.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 23 #include <crypto/aes.h> 24 #include <crypto/authenc.h> 25 #include <crypto/utils.h> 26 #include <crypto/des.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/hash.h> 29 #include <crypto/internal/skcipher.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/sha1.h> 32 #include <crypto/sha2.h> 33 34 #include "sa2ul.h" 35 36 /* Byte offset for key in encryption security context */ 37 #define SC_ENC_KEY_OFFSET (1 + 27 + 4) 38 /* Byte offset for Aux-1 in encryption security context */ 39 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) 40 41 #define SA_CMDL_UPD_ENC 0x0001 42 #define SA_CMDL_UPD_AUTH 0x0002 43 #define SA_CMDL_UPD_ENC_IV 0x0004 44 #define SA_CMDL_UPD_AUTH_IV 0x0008 45 #define SA_CMDL_UPD_AUX_KEY 0x0010 46 47 #define SA_AUTH_SUBKEY_LEN 16 48 #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF 49 #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 50 51 #define MODE_CONTROL_BYTES 27 52 #define SA_HASH_PROCESSING 0 53 #define SA_CRYPTO_PROCESSING 0 54 #define SA_UPLOAD_HASH_TO_TLR BIT(6) 55 56 #define SA_SW0_FLAGS_MASK 0xF0000 57 #define SA_SW0_CMDL_INFO_MASK 0x1F00000 58 #define SA_SW0_CMDL_PRESENT BIT(4) 59 #define SA_SW0_ENG_ID_MASK 0x3E000000 60 #define SA_SW0_DEST_INFO_PRESENT BIT(30) 61 #define SA_SW2_EGRESS_LENGTH 0xFF000000 62 #define SA_BASIC_HASH 0x10 63 64 #define SHA256_DIGEST_WORDS 8 65 /* Make 32-bit word from 4 bytes */ 66 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ 67 ((b2) << 8) | (b3)) 68 69 /* size of SCCTL structure in bytes */ 70 #define SA_SCCTL_SZ 16 71 72 /* Max Authentication tag size */ 73 #define SA_MAX_AUTH_TAG_SZ 64 74 75 enum sa_algo_id { 76 SA_ALG_CBC_AES = 0, 77 SA_ALG_EBC_AES, 78 SA_ALG_CBC_DES3, 79 SA_ALG_ECB_DES3, 80 SA_ALG_SHA1, 81 SA_ALG_SHA256, 82 SA_ALG_SHA512, 83 SA_ALG_AUTHENC_SHA1_AES, 84 SA_ALG_AUTHENC_SHA256_AES, 85 }; 86 87 struct sa_match_data { 88 u8 priv; 89 u8 priv_id; 90 u32 supported_algos; 91 }; 92 93 static struct device *sa_k3_dev; 94 95 /** 96 * struct sa_cmdl_cfg - Command label configuration descriptor 97 * @aalg: authentication algorithm ID 98 * @enc_eng_id: Encryption Engine ID supported by the SA hardware 99 * @auth_eng_id: Authentication Engine ID 100 * @iv_size: Initialization Vector size 101 * @akey: Authentication key 102 * @akey_len: Authentication key length 103 * @enc: True, if this is an encode request 104 */ 105 struct sa_cmdl_cfg { 106 int aalg; 107 u8 enc_eng_id; 108 u8 auth_eng_id; 109 u8 iv_size; 110 const u8 *akey; 111 u16 akey_len; 112 bool enc; 113 }; 114 115 /** 116 * struct algo_data - Crypto algorithm specific data 117 * @enc_eng: Encryption engine info structure 118 * @auth_eng: Authentication engine info structure 119 * @auth_ctrl: Authentication control word 120 * @hash_size: Size of digest 121 * @iv_idx: iv index in psdata 122 * @iv_out_size: iv out size 123 * @ealg_id: Encryption Algorithm ID 124 * @aalg_id: Authentication algorithm ID 125 * @mci_enc: Mode Control Instruction for Encryption algorithm 126 * @mci_dec: Mode Control Instruction for Decryption 127 * @inv_key: Whether the encryption algorithm demands key inversion 128 * @keyed_mac: Whether the authentication algorithm has key 129 */ 130 struct algo_data { 131 struct sa_eng_info enc_eng; 132 struct sa_eng_info auth_eng; 133 u8 auth_ctrl; 134 u8 hash_size; 135 u8 iv_idx; 136 u8 iv_out_size; 137 u8 ealg_id; 138 u8 aalg_id; 139 u8 *mci_enc; 140 u8 *mci_dec; 141 bool inv_key; 142 bool keyed_mac; 143 }; 144 145 /** 146 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms 147 * @type: Type of the crypto algorithm. 148 * @alg: Union of crypto algorithm definitions. 149 * @registered: Flag indicating if the crypto algorithm is already registered 150 */ 151 struct sa_alg_tmpl { 152 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */ 153 union { 154 struct skcipher_alg skcipher; 155 struct ahash_alg ahash; 156 struct aead_alg aead; 157 } alg; 158 bool registered; 159 }; 160 161 /** 162 * struct sa_mapped_sg: scatterlist information for tx and rx 163 * @mapped: Set to true if the @sgt is mapped 164 * @dir: mapping direction used for @sgt 165 * @split_sg: Set if the sg is split and needs to be freed up 166 * @static_sg: Static scatterlist entry for overriding data 167 * @sgt: scatterlist table for DMA API use 168 */ 169 struct sa_mapped_sg { 170 bool mapped; 171 enum dma_data_direction dir; 172 struct scatterlist static_sg; 173 struct scatterlist *split_sg; 174 struct sg_table sgt; 175 }; 176 /** 177 * struct sa_rx_data: RX Packet miscellaneous data place holder 178 * @req: crypto request data pointer 179 * @ddev: pointer to the DMA device 180 * @tx_in: dma_async_tx_descriptor pointer for rx channel 181 * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping 182 * @enc: Flag indicating either encryption or decryption 183 * @enc_iv_size: Initialisation vector size 184 * @iv_idx: Initialisation vector index 185 */ 186 struct sa_rx_data { 187 void *req; 188 struct device *ddev; 189 struct dma_async_tx_descriptor *tx_in; 190 struct sa_mapped_sg mapped_sg[2]; 191 u8 enc; 192 u8 enc_iv_size; 193 u8 iv_idx; 194 }; 195 196 /** 197 * struct sa_req: SA request definition 198 * @dev: device for the request 199 * @size: total data to the xmitted via DMA 200 * @enc_offset: offset of cipher data 201 * @enc_size: data to be passed to cipher engine 202 * @enc_iv: cipher IV 203 * @auth_offset: offset of the authentication data 204 * @auth_size: size of the authentication data 205 * @auth_iv: authentication IV 206 * @type: algorithm type for the request 207 * @cmdl: command label pointer 208 * @base: pointer to the base request 209 * @ctx: pointer to the algorithm context data 210 * @enc: true if this is an encode request 211 * @src: source data 212 * @dst: destination data 213 * @callback: DMA callback for the request 214 * @mdata_size: metadata size passed to DMA 215 */ 216 struct sa_req { 217 struct device *dev; 218 u16 size; 219 u8 enc_offset; 220 u16 enc_size; 221 u8 *enc_iv; 222 u8 auth_offset; 223 u16 auth_size; 224 u8 *auth_iv; 225 u32 type; 226 u32 *cmdl; 227 struct crypto_async_request *base; 228 struct sa_tfm_ctx *ctx; 229 bool enc; 230 struct scatterlist *src; 231 struct scatterlist *dst; 232 dma_async_tx_callback callback; 233 u16 mdata_size; 234 }; 235 236 /* 237 * Mode Control Instructions for various Key lengths 128, 192, 256 238 * For CBC (Cipher Block Chaining) mode for encryption 239 */ 240 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { 241 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 242 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 243 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 244 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 245 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 246 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 247 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 248 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 249 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 250 }; 251 252 /* 253 * Mode Control Instructions for various Key lengths 128, 192, 256 254 * For CBC (Cipher Block Chaining) mode for decryption 255 */ 256 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { 257 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 258 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 259 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 260 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 261 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 262 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 263 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 264 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 265 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 266 }; 267 268 /* 269 * Mode Control Instructions for various Key lengths 128, 192, 256 270 * For CBC (Cipher Block Chaining) mode for encryption 271 */ 272 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { 273 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 274 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 275 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 276 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 277 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 278 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 279 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 280 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 281 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 282 }; 283 284 /* 285 * Mode Control Instructions for various Key lengths 128, 192, 256 286 * For CBC (Cipher Block Chaining) mode for decryption 287 */ 288 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { 289 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 290 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 291 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 292 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 293 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 294 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 295 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 296 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 297 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 298 }; 299 300 /* 301 * Mode Control Instructions for various Key lengths 128, 192, 256 302 * For ECB (Electronic Code Book) mode for encryption 303 */ 304 static u8 mci_ecb_enc_array[3][27] = { 305 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 306 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 307 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 308 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 309 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 310 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 311 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 312 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 313 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 314 }; 315 316 /* 317 * Mode Control Instructions for various Key lengths 128, 192, 256 318 * For ECB (Electronic Code Book) mode for decryption 319 */ 320 static u8 mci_ecb_dec_array[3][27] = { 321 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 322 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 323 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 324 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 325 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 326 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 327 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 328 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 329 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 330 }; 331 332 /* 333 * Mode Control Instructions for DES algorithm 334 * For CBC (Cipher Block Chaining) mode and ECB mode 335 * encryption and for decryption respectively 336 */ 337 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { 338 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, 339 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 340 0x00, 0x00, 0x00, 341 }; 342 343 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { 344 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, 345 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 346 0x00, 0x00, 0x00, 347 }; 348 349 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { 350 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 351 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 352 0x00, 0x00, 0x00, 353 }; 354 355 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { 356 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 357 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 358 0x00, 0x00, 0x00, 359 }; 360 361 /* 362 * Perform 16 byte or 128 bit swizzling 363 * The SA2UL Expects the security context to 364 * be in little Endian and the bus width is 128 bits or 16 bytes 365 * Hence swap 16 bytes at a time from higher to lower address 366 */ 367 static void sa_swiz_128(u8 *in, u16 len) 368 { 369 u8 data[16]; 370 int i, j; 371 372 for (i = 0; i < len; i += 16) { 373 memcpy(data, &in[i], 16); 374 for (j = 0; j < 16; j++) 375 in[i + j] = data[15 - j]; 376 } 377 } 378 379 static void sa_prepare_iopads(struct algo_data *data, const u8 *key, u16 key_sz, 380 __be32 *ipad, __be32 *opad) 381 { 382 union { 383 struct hmac_sha1_key sha1; 384 struct hmac_sha256_key sha256; 385 } k; 386 387 switch (data->aalg_id) { 388 case SA_AALG_ID_HMAC_SHA1: 389 hmac_sha1_preparekey(&k.sha1, key, key_sz); 390 cpu_to_be32_array(ipad, k.sha1.istate.h, 391 ARRAY_SIZE(k.sha1.istate.h)); 392 cpu_to_be32_array(opad, k.sha1.ostate.h, 393 ARRAY_SIZE(k.sha1.ostate.h)); 394 break; 395 case SA_AALG_ID_HMAC_SHA2_256: 396 hmac_sha256_preparekey(&k.sha256, key, key_sz); 397 cpu_to_be32_array(ipad, k.sha256.key.istate.h, 398 ARRAY_SIZE(k.sha256.key.istate.h)); 399 cpu_to_be32_array(opad, k.sha256.key.ostate.h, 400 ARRAY_SIZE(k.sha256.key.ostate.h)); 401 break; 402 default: 403 dev_err(sa_k3_dev, "%s: bad aalg_id=%d\n", __func__, 404 data->aalg_id); 405 return; 406 } 407 408 memzero_explicit(&k, sizeof(k)); 409 } 410 411 /* Derive the inverse key used in AES-CBC decryption operation */ 412 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) 413 { 414 struct crypto_aes_ctx ctx __cleanup(aes_zeroize_ctx); 415 int key_pos; 416 417 if (aes_expandkey(&ctx, key, key_sz)) { 418 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 419 return -EINVAL; 420 } 421 422 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ 423 if (key_sz == AES_KEYSIZE_192) { 424 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; 425 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; 426 } 427 428 /* Based crypto_aes_expand_key logic */ 429 switch (key_sz) { 430 case AES_KEYSIZE_128: 431 case AES_KEYSIZE_192: 432 key_pos = key_sz + 24; 433 break; 434 435 case AES_KEYSIZE_256: 436 key_pos = key_sz + 24 - 4; 437 break; 438 439 default: 440 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 441 return -EINVAL; 442 } 443 444 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); 445 return 0; 446 } 447 448 /* Set Security context for the encryption engine */ 449 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, 450 u8 enc, u8 *sc_buf) 451 { 452 const u8 *mci = NULL; 453 454 /* Set Encryption mode selector to crypto processing */ 455 sc_buf[0] = SA_CRYPTO_PROCESSING; 456 457 if (enc) 458 mci = ad->mci_enc; 459 else 460 mci = ad->mci_dec; 461 /* Set the mode control instructions in security context */ 462 if (mci) 463 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); 464 465 /* For AES-CBC decryption get the inverse key */ 466 if (ad->inv_key && !enc) { 467 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) 468 return -EINVAL; 469 /* For all other cases: key is used */ 470 } else { 471 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); 472 } 473 474 return 0; 475 } 476 477 /* Set Security context for the authentication engine */ 478 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, 479 u8 *sc_buf) 480 { 481 __be32 *ipad = (void *)(sc_buf + 32); 482 __be32 *opad = (void *)(sc_buf + 64); 483 484 /* Set Authentication mode selector to hash processing */ 485 sc_buf[0] = SA_HASH_PROCESSING; 486 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ 487 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; 488 sc_buf[1] |= ad->auth_ctrl; 489 490 /* Copy the keys or ipad/opad */ 491 if (ad->keyed_mac) 492 sa_prepare_iopads(ad, key, key_sz, ipad, opad); 493 else { 494 /* basic hash */ 495 sc_buf[1] |= SA_BASIC_HASH; 496 } 497 } 498 499 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) 500 { 501 int j; 502 503 for (j = 0; j < ((size16) ? 4 : 2); j++) { 504 *out = cpu_to_be32(*((u32 *)iv)); 505 iv += 4; 506 out++; 507 } 508 } 509 510 /* Format general command label */ 511 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, 512 struct sa_cmdl_upd_info *upd_info) 513 { 514 u8 enc_offset = 0, auth_offset = 0, total = 0; 515 u8 enc_next_eng = SA_ENG_ID_OUTPORT2; 516 u8 auth_next_eng = SA_ENG_ID_OUTPORT2; 517 u32 *word_ptr = (u32 *)cmdl; 518 int i; 519 520 /* Clear the command label */ 521 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); 522 523 /* Initialize the command update structure */ 524 memzero_explicit(upd_info, sizeof(*upd_info)); 525 526 if (cfg->enc_eng_id && cfg->auth_eng_id) { 527 if (cfg->enc) { 528 auth_offset = SA_CMDL_HEADER_SIZE_BYTES; 529 enc_next_eng = cfg->auth_eng_id; 530 531 if (cfg->iv_size) 532 auth_offset += cfg->iv_size; 533 } else { 534 enc_offset = SA_CMDL_HEADER_SIZE_BYTES; 535 auth_next_eng = cfg->enc_eng_id; 536 } 537 } 538 539 if (cfg->enc_eng_id) { 540 upd_info->flags |= SA_CMDL_UPD_ENC; 541 upd_info->enc_size.index = enc_offset >> 2; 542 upd_info->enc_offset.index = upd_info->enc_size.index + 1; 543 /* Encryption command label */ 544 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; 545 546 /* Encryption modes requiring IV */ 547 if (cfg->iv_size) { 548 upd_info->flags |= SA_CMDL_UPD_ENC_IV; 549 upd_info->enc_iv.index = 550 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; 551 upd_info->enc_iv.size = cfg->iv_size; 552 553 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 554 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 555 556 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = 557 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); 558 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 559 } else { 560 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 561 SA_CMDL_HEADER_SIZE_BYTES; 562 total += SA_CMDL_HEADER_SIZE_BYTES; 563 } 564 } 565 566 if (cfg->auth_eng_id) { 567 upd_info->flags |= SA_CMDL_UPD_AUTH; 568 upd_info->auth_size.index = auth_offset >> 2; 569 upd_info->auth_offset.index = upd_info->auth_size.index + 1; 570 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; 571 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = 572 SA_CMDL_HEADER_SIZE_BYTES; 573 total += SA_CMDL_HEADER_SIZE_BYTES; 574 } 575 576 total = roundup(total, 8); 577 578 for (i = 0; i < total / 4; i++) 579 word_ptr[i] = swab32(word_ptr[i]); 580 581 return total; 582 } 583 584 /* Update Command label */ 585 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, 586 struct sa_cmdl_upd_info *upd_info) 587 { 588 int i = 0, j; 589 590 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { 591 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 592 cmdl[upd_info->enc_size.index] |= req->enc_size; 593 cmdl[upd_info->enc_offset.index] &= 594 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 595 cmdl[upd_info->enc_offset.index] |= 596 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, 597 req->enc_offset); 598 599 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { 600 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; 601 u32 *enc_iv = (u32 *)req->enc_iv; 602 603 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { 604 data[j] = cpu_to_be32(*enc_iv); 605 enc_iv++; 606 } 607 } 608 } 609 610 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { 611 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 612 cmdl[upd_info->auth_size.index] |= req->auth_size; 613 cmdl[upd_info->auth_offset.index] &= 614 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 615 cmdl[upd_info->auth_offset.index] |= 616 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, 617 req->auth_offset); 618 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { 619 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], 620 req->auth_iv, 621 (upd_info->auth_iv.size > 8)); 622 } 623 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { 624 int offset = (req->auth_size & 0xF) ? 4 : 0; 625 626 memcpy(&cmdl[upd_info->aux_key_info.index], 627 &upd_info->aux_key[offset], 16); 628 } 629 } 630 } 631 632 /* Format SWINFO words to be sent to SA */ 633 static 634 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, 635 u8 cmdl_present, u8 cmdl_offset, u8 flags, 636 u8 hash_size, u32 *swinfo) 637 { 638 swinfo[0] = sc_id; 639 swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags); 640 if (likely(cmdl_present)) 641 swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK, 642 cmdl_offset | SA_SW0_CMDL_PRESENT); 643 swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id); 644 645 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; 646 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); 647 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); 648 swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size); 649 } 650 651 /* Dump the security context */ 652 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) 653 { 654 #ifdef DEBUG 655 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); 656 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 657 16, 1, buf, SA_CTX_MAX_SZ, false); 658 #endif 659 } 660 661 static 662 int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data, 663 const u8 *enc_key, u16 enc_key_sz, 664 const u8 *auth_key, u16 auth_key_sz, 665 struct algo_data *ad, u8 enc, u32 *swinfo) 666 { 667 int enc_sc_offset = 0; 668 int auth_sc_offset = 0; 669 u8 *sc_buf = ctx->sc; 670 u16 sc_id = ctx->sc_id; 671 u8 first_engine = 0; 672 673 memzero_explicit(sc_buf, SA_CTX_MAX_SZ); 674 675 if (ad->auth_eng.eng_id) { 676 if (enc) 677 first_engine = ad->enc_eng.eng_id; 678 else 679 first_engine = ad->auth_eng.eng_id; 680 681 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 682 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; 683 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; 684 if (!ad->hash_size) 685 return -EINVAL; 686 ad->hash_size = roundup(ad->hash_size, 8); 687 688 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { 689 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 690 first_engine = ad->enc_eng.eng_id; 691 sc_buf[1] = SA_SCCTL_FE_ENC; 692 ad->hash_size = ad->iv_out_size; 693 } 694 695 /* SCCTL Owner info: 0=host, 1=CP_ACE */ 696 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; 697 memcpy(&sc_buf[2], &sc_id, 2); 698 sc_buf[4] = 0x0; 699 sc_buf[5] = match_data->priv_id; 700 sc_buf[6] = match_data->priv; 701 sc_buf[7] = 0x0; 702 703 /* Prepare context for encryption engine */ 704 if (ad->enc_eng.sc_size) { 705 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, 706 &sc_buf[enc_sc_offset])) 707 return -EINVAL; 708 } 709 710 /* Prepare context for authentication engine */ 711 if (ad->auth_eng.sc_size) 712 sa_set_sc_auth(ad, auth_key, auth_key_sz, 713 &sc_buf[auth_sc_offset]); 714 715 /* Set the ownership of context to CP_ACE */ 716 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; 717 718 /* swizzle the security context */ 719 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); 720 721 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, 722 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); 723 724 sa_dump_sc(sc_buf, ctx->sc_phys); 725 726 return 0; 727 } 728 729 /* Free the per direction context memory */ 730 static void sa_free_ctx_info(struct sa_ctx_info *ctx, 731 struct sa_crypto_data *data) 732 { 733 unsigned long bn; 734 735 bn = ctx->sc_id - data->sc_id_start; 736 spin_lock(&data->scid_lock); 737 __clear_bit(bn, data->ctx_bm); 738 data->sc_id--; 739 spin_unlock(&data->scid_lock); 740 741 if (ctx->sc) { 742 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); 743 ctx->sc = NULL; 744 } 745 } 746 747 static int sa_init_ctx_info(struct sa_ctx_info *ctx, 748 struct sa_crypto_data *data) 749 { 750 unsigned long bn; 751 int err; 752 753 spin_lock(&data->scid_lock); 754 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); 755 __set_bit(bn, data->ctx_bm); 756 data->sc_id++; 757 spin_unlock(&data->scid_lock); 758 759 ctx->sc_id = (u16)(data->sc_id_start + bn); 760 761 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); 762 if (!ctx->sc) { 763 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); 764 err = -ENOMEM; 765 goto scid_rollback; 766 } 767 768 return 0; 769 770 scid_rollback: 771 spin_lock(&data->scid_lock); 772 __clear_bit(bn, data->ctx_bm); 773 data->sc_id--; 774 spin_unlock(&data->scid_lock); 775 776 return err; 777 } 778 779 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) 780 { 781 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 782 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 783 784 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 785 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 786 ctx->dec.sc_id, &ctx->dec.sc_phys); 787 788 sa_free_ctx_info(&ctx->enc, data); 789 sa_free_ctx_info(&ctx->dec, data); 790 791 crypto_free_skcipher(ctx->fallback.skcipher); 792 } 793 794 static int sa_cipher_cra_init(struct crypto_skcipher *tfm) 795 { 796 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 797 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 798 const char *name = crypto_tfm_alg_name(&tfm->base); 799 struct crypto_skcipher *child; 800 int ret; 801 802 memzero_explicit(ctx, sizeof(*ctx)); 803 ctx->dev_data = data; 804 805 ret = sa_init_ctx_info(&ctx->enc, data); 806 if (ret) 807 return ret; 808 ret = sa_init_ctx_info(&ctx->dec, data); 809 if (ret) { 810 sa_free_ctx_info(&ctx->enc, data); 811 return ret; 812 } 813 814 child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 815 816 if (IS_ERR(child)) { 817 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); 818 return PTR_ERR(child); 819 } 820 821 ctx->fallback.skcipher = child; 822 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + 823 sizeof(struct skcipher_request)); 824 825 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 826 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 827 ctx->dec.sc_id, &ctx->dec.sc_phys); 828 return 0; 829 } 830 831 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 832 unsigned int keylen, struct algo_data *ad) 833 { 834 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 835 struct crypto_skcipher *child = ctx->fallback.skcipher; 836 int cmdl_len; 837 struct sa_cmdl_cfg cfg; 838 int ret; 839 840 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 841 keylen != AES_KEYSIZE_256) 842 return -EINVAL; 843 844 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 845 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 846 847 memzero_explicit(&cfg, sizeof(cfg)); 848 cfg.enc_eng_id = ad->enc_eng.eng_id; 849 cfg.iv_size = crypto_skcipher_ivsize(tfm); 850 851 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 852 crypto_skcipher_set_flags(child, tfm->base.crt_flags & 853 CRYPTO_TFM_REQ_MASK); 854 ret = crypto_skcipher_setkey(child, key, keylen); 855 if (ret) 856 return ret; 857 858 /* Setup Encryption Security Context & Command label template */ 859 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0, 860 ad, 1, &ctx->enc.epib[1])) 861 goto badkey; 862 863 cmdl_len = sa_format_cmdl_gen(&cfg, 864 (u8 *)ctx->enc.cmdl, 865 &ctx->enc.cmdl_upd_info); 866 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 867 goto badkey; 868 869 ctx->enc.cmdl_size = cmdl_len; 870 871 /* Setup Decryption Security Context & Command label template */ 872 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0, 873 ad, 0, &ctx->dec.epib[1])) 874 goto badkey; 875 876 cfg.enc_eng_id = ad->enc_eng.eng_id; 877 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 878 &ctx->dec.cmdl_upd_info); 879 880 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 881 goto badkey; 882 883 ctx->dec.cmdl_size = cmdl_len; 884 ctx->iv_idx = ad->iv_idx; 885 886 return 0; 887 888 badkey: 889 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 890 return -EINVAL; 891 } 892 893 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 894 unsigned int keylen) 895 { 896 struct algo_data ad = { 0 }; 897 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 898 int key_idx = (keylen >> 3) - 2; 899 900 if (key_idx >= 3) 901 return -EINVAL; 902 903 ad.mci_enc = mci_cbc_enc_array[key_idx]; 904 ad.mci_dec = mci_cbc_dec_array[key_idx]; 905 ad.inv_key = true; 906 ad.ealg_id = SA_EALG_ID_AES_CBC; 907 ad.iv_idx = 4; 908 ad.iv_out_size = 16; 909 910 return sa_cipher_setkey(tfm, key, keylen, &ad); 911 } 912 913 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 914 unsigned int keylen) 915 { 916 struct algo_data ad = { 0 }; 917 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 918 int key_idx = (keylen >> 3) - 2; 919 920 if (key_idx >= 3) 921 return -EINVAL; 922 923 ad.mci_enc = mci_ecb_enc_array[key_idx]; 924 ad.mci_dec = mci_ecb_dec_array[key_idx]; 925 ad.inv_key = true; 926 ad.ealg_id = SA_EALG_ID_AES_ECB; 927 928 return sa_cipher_setkey(tfm, key, keylen, &ad); 929 } 930 931 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 932 unsigned int keylen) 933 { 934 struct algo_data ad = { 0 }; 935 936 ad.mci_enc = mci_cbc_3des_enc_array; 937 ad.mci_dec = mci_cbc_3des_dec_array; 938 ad.ealg_id = SA_EALG_ID_3DES_CBC; 939 ad.iv_idx = 6; 940 ad.iv_out_size = 8; 941 942 return sa_cipher_setkey(tfm, key, keylen, &ad); 943 } 944 945 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 946 unsigned int keylen) 947 { 948 struct algo_data ad = { 0 }; 949 950 ad.mci_enc = mci_ecb_3des_enc_array; 951 ad.mci_dec = mci_ecb_3des_dec_array; 952 953 return sa_cipher_setkey(tfm, key, keylen, &ad); 954 } 955 956 static void sa_sync_from_device(struct sa_rx_data *rxd) 957 { 958 struct sg_table *sgt; 959 960 if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) 961 sgt = &rxd->mapped_sg[0].sgt; 962 else 963 sgt = &rxd->mapped_sg[1].sgt; 964 965 dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); 966 } 967 968 static void sa_free_sa_rx_data(struct sa_rx_data *rxd) 969 { 970 int i; 971 972 for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { 973 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; 974 975 if (mapped_sg->mapped) { 976 dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, 977 mapped_sg->dir, 0); 978 kfree(mapped_sg->split_sg); 979 } 980 } 981 982 kfree(rxd); 983 } 984 985 static void sa_aes_dma_in_callback(void *data) 986 { 987 struct sa_rx_data *rxd = data; 988 struct skcipher_request *req; 989 u32 *result; 990 __be32 *mdptr; 991 size_t ml, pl; 992 int i; 993 994 sa_sync_from_device(rxd); 995 req = container_of(rxd->req, struct skcipher_request, base); 996 997 if (req->iv) { 998 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, 999 &ml); 1000 result = (u32 *)req->iv; 1001 1002 for (i = 0; i < (rxd->enc_iv_size / 4); i++) 1003 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); 1004 } 1005 1006 sa_free_sa_rx_data(rxd); 1007 1008 skcipher_request_complete(req, 0); 1009 } 1010 1011 static void 1012 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) 1013 { 1014 u32 *out, *in; 1015 int i; 1016 1017 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) 1018 *out++ = *in++; 1019 1020 mdptr[4] = (0xFFFF << 16); 1021 for (out = &mdptr[5], in = psdata, i = 0; 1022 i < pslen / sizeof(u32); i++) 1023 *out++ = *in++; 1024 } 1025 1026 static int sa_run(struct sa_req *req) 1027 { 1028 struct sa_rx_data *rxd; 1029 gfp_t gfp_flags; 1030 u32 cmdl[SA_MAX_CMDL_WORDS]; 1031 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); 1032 struct device *ddev; 1033 struct dma_chan *dma_rx; 1034 int sg_nents, src_nents, dst_nents; 1035 struct scatterlist *src, *dst; 1036 size_t pl, ml, split_size; 1037 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; 1038 int ret; 1039 struct dma_async_tx_descriptor *tx_out; 1040 u32 *mdptr; 1041 bool diff_dst; 1042 enum dma_data_direction dir_src; 1043 struct sa_mapped_sg *mapped_sg; 1044 1045 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 1046 GFP_KERNEL : GFP_ATOMIC; 1047 1048 rxd = kzalloc_obj(*rxd, gfp_flags); 1049 if (!rxd) 1050 return -ENOMEM; 1051 1052 if (req->src != req->dst) { 1053 diff_dst = true; 1054 dir_src = DMA_TO_DEVICE; 1055 } else { 1056 diff_dst = false; 1057 dir_src = DMA_BIDIRECTIONAL; 1058 } 1059 1060 /* 1061 * SA2UL has an interesting feature where the receive DMA channel 1062 * is selected based on the data passed to the engine. Within the 1063 * transition range, there is also a space where it is impossible 1064 * to determine where the data will end up, and this should be 1065 * avoided. This will be handled by the SW fallback mechanism by 1066 * the individual algorithm implementations. 1067 */ 1068 if (req->size >= 256) 1069 dma_rx = pdata->dma_rx2; 1070 else 1071 dma_rx = pdata->dma_rx1; 1072 1073 ddev = dmaengine_get_dma_device(pdata->dma_tx); 1074 rxd->ddev = ddev; 1075 1076 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); 1077 1078 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); 1079 1080 if (req->type != CRYPTO_ALG_TYPE_AHASH) { 1081 if (req->enc) 1082 req->type |= 1083 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); 1084 else 1085 req->type |= 1086 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); 1087 } 1088 1089 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; 1090 1091 /* 1092 * Map the packets, first we check if the data fits into a single 1093 * sg entry and use that if possible. If it does not fit, we check 1094 * if we need to do sg_split to align the scatterlist data on the 1095 * actual data size being processed by the crypto engine. 1096 */ 1097 src = req->src; 1098 sg_nents = sg_nents_for_len(src, req->size); 1099 1100 split_size = req->size; 1101 1102 mapped_sg = &rxd->mapped_sg[0]; 1103 if (sg_nents == 1 && split_size <= req->src->length) { 1104 src = &mapped_sg->static_sg; 1105 src_nents = 1; 1106 sg_init_table(src, 1); 1107 sg_set_page(src, sg_page(req->src), split_size, 1108 req->src->offset); 1109 1110 mapped_sg->sgt.sgl = src; 1111 mapped_sg->sgt.orig_nents = src_nents; 1112 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1113 if (ret) { 1114 kfree(rxd); 1115 return ret; 1116 } 1117 1118 mapped_sg->dir = dir_src; 1119 mapped_sg->mapped = true; 1120 } else { 1121 mapped_sg->sgt.sgl = req->src; 1122 mapped_sg->sgt.orig_nents = sg_nents; 1123 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1124 if (ret) { 1125 kfree(rxd); 1126 return ret; 1127 } 1128 1129 mapped_sg->dir = dir_src; 1130 mapped_sg->mapped = true; 1131 1132 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, 1133 &split_size, &src, &src_nents, gfp_flags); 1134 if (ret) { 1135 src_nents = mapped_sg->sgt.nents; 1136 src = mapped_sg->sgt.sgl; 1137 } else { 1138 mapped_sg->split_sg = src; 1139 } 1140 } 1141 1142 dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); 1143 1144 if (!diff_dst) { 1145 dst_nents = src_nents; 1146 dst = src; 1147 } else { 1148 dst_nents = sg_nents_for_len(req->dst, req->size); 1149 mapped_sg = &rxd->mapped_sg[1]; 1150 1151 if (dst_nents == 1 && split_size <= req->dst->length) { 1152 dst = &mapped_sg->static_sg; 1153 dst_nents = 1; 1154 sg_init_table(dst, 1); 1155 sg_set_page(dst, sg_page(req->dst), split_size, 1156 req->dst->offset); 1157 1158 mapped_sg->sgt.sgl = dst; 1159 mapped_sg->sgt.orig_nents = dst_nents; 1160 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1161 DMA_FROM_DEVICE, 0); 1162 if (ret) 1163 goto err_cleanup; 1164 1165 mapped_sg->dir = DMA_FROM_DEVICE; 1166 mapped_sg->mapped = true; 1167 } else { 1168 mapped_sg->sgt.sgl = req->dst; 1169 mapped_sg->sgt.orig_nents = dst_nents; 1170 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1171 DMA_FROM_DEVICE, 0); 1172 if (ret) 1173 goto err_cleanup; 1174 1175 mapped_sg->dir = DMA_FROM_DEVICE; 1176 mapped_sg->mapped = true; 1177 1178 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 1179 0, 1, &split_size, &dst, &dst_nents, 1180 gfp_flags); 1181 if (ret) { 1182 dst_nents = mapped_sg->sgt.nents; 1183 dst = mapped_sg->sgt.sgl; 1184 } else { 1185 mapped_sg->split_sg = dst; 1186 } 1187 } 1188 } 1189 1190 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, 1191 DMA_DEV_TO_MEM, 1192 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1193 if (!rxd->tx_in) { 1194 dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); 1195 ret = -EINVAL; 1196 goto err_cleanup; 1197 } 1198 1199 rxd->req = (void *)req->base; 1200 rxd->enc = req->enc; 1201 rxd->iv_idx = req->ctx->iv_idx; 1202 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; 1203 rxd->tx_in->callback = req->callback; 1204 rxd->tx_in->callback_param = rxd; 1205 1206 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, 1207 src_nents, DMA_MEM_TO_DEV, 1208 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1209 1210 if (!tx_out) { 1211 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); 1212 ret = -EINVAL; 1213 goto err_cleanup; 1214 } 1215 1216 /* 1217 * Prepare metadata for DMA engine. This essentially describes the 1218 * crypto algorithm to be used, data sizes, different keys etc. 1219 */ 1220 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); 1221 1222 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * 1223 sizeof(u32))), cmdl, sizeof(sa_ctx->epib), 1224 sa_ctx->epib); 1225 1226 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); 1227 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); 1228 1229 dmaengine_submit(tx_out); 1230 dmaengine_submit(rxd->tx_in); 1231 1232 dma_async_issue_pending(dma_rx); 1233 dma_async_issue_pending(pdata->dma_tx); 1234 1235 return -EINPROGRESS; 1236 1237 err_cleanup: 1238 sa_free_sa_rx_data(rxd); 1239 1240 return ret; 1241 } 1242 1243 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) 1244 { 1245 struct sa_tfm_ctx *ctx = 1246 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1247 struct crypto_alg *alg = req->base.tfm->__crt_alg; 1248 struct sa_req sa_req = { 0 }; 1249 1250 if (!req->cryptlen) 1251 return 0; 1252 1253 if (req->cryptlen % alg->cra_blocksize) 1254 return -EINVAL; 1255 1256 /* Use SW fallback if the data size is not supported */ 1257 if (req->cryptlen > SA_MAX_DATA_SZ || 1258 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && 1259 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { 1260 struct skcipher_request *subreq = skcipher_request_ctx(req); 1261 1262 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); 1263 skcipher_request_set_callback(subreq, req->base.flags, 1264 req->base.complete, 1265 req->base.data); 1266 skcipher_request_set_crypt(subreq, req->src, req->dst, 1267 req->cryptlen, req->iv); 1268 if (enc) 1269 return crypto_skcipher_encrypt(subreq); 1270 else 1271 return crypto_skcipher_decrypt(subreq); 1272 } 1273 1274 sa_req.size = req->cryptlen; 1275 sa_req.enc_size = req->cryptlen; 1276 sa_req.src = req->src; 1277 sa_req.dst = req->dst; 1278 sa_req.enc_iv = iv; 1279 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; 1280 sa_req.enc = enc; 1281 sa_req.callback = sa_aes_dma_in_callback; 1282 sa_req.mdata_size = 44; 1283 sa_req.base = &req->base; 1284 sa_req.ctx = ctx; 1285 1286 return sa_run(&sa_req); 1287 } 1288 1289 static int sa_encrypt(struct skcipher_request *req) 1290 { 1291 return sa_cipher_run(req, req->iv, 1); 1292 } 1293 1294 static int sa_decrypt(struct skcipher_request *req) 1295 { 1296 return sa_cipher_run(req, req->iv, 0); 1297 } 1298 1299 static void sa_sha_dma_in_callback(void *data) 1300 { 1301 struct sa_rx_data *rxd = data; 1302 struct ahash_request *req; 1303 struct crypto_ahash *tfm; 1304 unsigned int authsize; 1305 int i; 1306 size_t ml, pl; 1307 u32 *result; 1308 __be32 *mdptr; 1309 1310 sa_sync_from_device(rxd); 1311 req = container_of(rxd->req, struct ahash_request, base); 1312 tfm = crypto_ahash_reqtfm(req); 1313 authsize = crypto_ahash_digestsize(tfm); 1314 1315 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1316 result = (u32 *)req->result; 1317 1318 for (i = 0; i < (authsize / 4); i++) 1319 result[i] = be32_to_cpu(mdptr[i + 4]); 1320 1321 sa_free_sa_rx_data(rxd); 1322 1323 ahash_request_complete(req, 0); 1324 } 1325 1326 static int zero_message_process(struct ahash_request *req) 1327 { 1328 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1329 int sa_digest_size = crypto_ahash_digestsize(tfm); 1330 1331 switch (sa_digest_size) { 1332 case SHA1_DIGEST_SIZE: 1333 memcpy(req->result, sha1_zero_message_hash, sa_digest_size); 1334 break; 1335 case SHA256_DIGEST_SIZE: 1336 memcpy(req->result, sha256_zero_message_hash, sa_digest_size); 1337 break; 1338 case SHA512_DIGEST_SIZE: 1339 memcpy(req->result, sha512_zero_message_hash, sa_digest_size); 1340 break; 1341 default: 1342 return -EINVAL; 1343 } 1344 1345 return 0; 1346 } 1347 1348 static int sa_sha_run(struct ahash_request *req) 1349 { 1350 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 1351 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1352 struct sa_req sa_req = { 0 }; 1353 size_t auth_len; 1354 1355 auth_len = req->nbytes; 1356 1357 if (!auth_len) 1358 return zero_message_process(req); 1359 1360 if (auth_len > SA_MAX_DATA_SZ || 1361 (auth_len >= SA_UNSAFE_DATA_SZ_MIN && 1362 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { 1363 struct ahash_request *subreq = &rctx->fallback_req; 1364 int ret; 1365 1366 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1367 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1368 ahash_request_set_crypt(subreq, req->src, req->result, auth_len); 1369 1370 ret = crypto_ahash_digest(subreq); 1371 1372 return ret; 1373 } 1374 1375 sa_req.size = auth_len; 1376 sa_req.auth_size = auth_len; 1377 sa_req.src = req->src; 1378 sa_req.dst = req->src; 1379 sa_req.enc = true; 1380 sa_req.type = CRYPTO_ALG_TYPE_AHASH; 1381 sa_req.callback = sa_sha_dma_in_callback; 1382 sa_req.mdata_size = 28; 1383 sa_req.ctx = ctx; 1384 sa_req.base = &req->base; 1385 1386 return sa_run(&sa_req); 1387 } 1388 1389 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) 1390 { 1391 int cmdl_len; 1392 struct sa_cmdl_cfg cfg; 1393 1394 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1395 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1396 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1397 1398 memset(&cfg, 0, sizeof(cfg)); 1399 cfg.aalg = ad->aalg_id; 1400 cfg.enc_eng_id = ad->enc_eng.eng_id; 1401 cfg.auth_eng_id = ad->auth_eng.eng_id; 1402 cfg.iv_size = 0; 1403 cfg.akey = NULL; 1404 cfg.akey_len = 0; 1405 1406 ctx->dev_data = dev_get_drvdata(sa_k3_dev); 1407 /* Setup Encryption Security Context & Command label template */ 1408 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0, 1409 ad, 0, &ctx->enc.epib[1])) 1410 goto badkey; 1411 1412 cmdl_len = sa_format_cmdl_gen(&cfg, 1413 (u8 *)ctx->enc.cmdl, 1414 &ctx->enc.cmdl_upd_info); 1415 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1416 goto badkey; 1417 1418 ctx->enc.cmdl_size = cmdl_len; 1419 1420 return 0; 1421 1422 badkey: 1423 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 1424 return -EINVAL; 1425 } 1426 1427 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 1428 { 1429 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1430 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1431 int ret; 1432 1433 memset(ctx, 0, sizeof(*ctx)); 1434 ctx->dev_data = data; 1435 ret = sa_init_ctx_info(&ctx->enc, data); 1436 if (ret) 1437 return ret; 1438 1439 if (alg_base) { 1440 /* for fallback */ 1441 ctx->fallback.ahash = 1442 crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); 1443 if (IS_ERR(ctx->fallback.ahash)) { 1444 dev_err(ctx->dev_data->dev, 1445 "Could not load fallback driver\n"); 1446 return PTR_ERR(ctx->fallback.ahash); 1447 } 1448 } 1449 1450 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1451 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1452 ctx->dec.sc_id, &ctx->dec.sc_phys); 1453 1454 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1455 sizeof(struct sa_sha_req_ctx) + 1456 crypto_ahash_reqsize(ctx->fallback.ahash)); 1457 1458 return 0; 1459 } 1460 1461 static int sa_sha_digest(struct ahash_request *req) 1462 { 1463 return sa_sha_run(req); 1464 } 1465 1466 static int sa_sha_init(struct ahash_request *req) 1467 { 1468 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1469 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1470 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1471 1472 dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", 1473 crypto_ahash_digestsize(tfm), rctx); 1474 1475 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1476 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1477 ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); 1478 1479 return crypto_ahash_init(&rctx->fallback_req); 1480 } 1481 1482 static int sa_sha_update(struct ahash_request *req) 1483 { 1484 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1485 1486 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1487 ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); 1488 1489 return crypto_ahash_update(&rctx->fallback_req); 1490 } 1491 1492 static int sa_sha_final(struct ahash_request *req) 1493 { 1494 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1495 1496 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1497 ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); 1498 1499 return crypto_ahash_final(&rctx->fallback_req); 1500 } 1501 1502 static int sa_sha_finup(struct ahash_request *req) 1503 { 1504 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1505 1506 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1507 ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); 1508 1509 return crypto_ahash_finup(&rctx->fallback_req); 1510 } 1511 1512 static int sa_sha_import(struct ahash_request *req, const void *in) 1513 { 1514 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1515 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1516 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1517 1518 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1519 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1520 1521 return crypto_ahash_import(&rctx->fallback_req, in); 1522 } 1523 1524 static int sa_sha_export(struct ahash_request *req, void *out) 1525 { 1526 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1527 struct ahash_request *subreq = &rctx->fallback_req; 1528 1529 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 1530 1531 return crypto_ahash_export(subreq, out); 1532 } 1533 1534 static int sa_sha1_cra_init(struct crypto_tfm *tfm) 1535 { 1536 struct algo_data ad = { 0 }; 1537 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1538 1539 sa_sha_cra_init_alg(tfm, "sha1"); 1540 1541 ad.aalg_id = SA_AALG_ID_SHA1; 1542 ad.hash_size = SHA1_DIGEST_SIZE; 1543 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1544 1545 sa_sha_setup(ctx, &ad); 1546 1547 return 0; 1548 } 1549 1550 static int sa_sha256_cra_init(struct crypto_tfm *tfm) 1551 { 1552 struct algo_data ad = { 0 }; 1553 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1554 1555 sa_sha_cra_init_alg(tfm, "sha256"); 1556 1557 ad.aalg_id = SA_AALG_ID_SHA2_256; 1558 ad.hash_size = SHA256_DIGEST_SIZE; 1559 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1560 1561 sa_sha_setup(ctx, &ad); 1562 1563 return 0; 1564 } 1565 1566 static int sa_sha512_cra_init(struct crypto_tfm *tfm) 1567 { 1568 struct algo_data ad = { 0 }; 1569 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1570 1571 sa_sha_cra_init_alg(tfm, "sha512"); 1572 1573 ad.aalg_id = SA_AALG_ID_SHA2_512; 1574 ad.hash_size = SHA512_DIGEST_SIZE; 1575 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; 1576 1577 sa_sha_setup(ctx, &ad); 1578 1579 return 0; 1580 } 1581 1582 static void sa_sha_cra_exit(struct crypto_tfm *tfm) 1583 { 1584 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1585 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1586 1587 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1588 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1589 ctx->dec.sc_id, &ctx->dec.sc_phys); 1590 1591 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) 1592 sa_free_ctx_info(&ctx->enc, data); 1593 1594 crypto_free_ahash(ctx->fallback.ahash); 1595 } 1596 1597 static void sa_aead_dma_in_callback(void *data) 1598 { 1599 struct sa_rx_data *rxd = data; 1600 struct aead_request *req; 1601 struct crypto_aead *tfm; 1602 unsigned int start; 1603 unsigned int authsize; 1604 u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; 1605 size_t pl, ml; 1606 int i; 1607 int err = 0; 1608 u32 *mdptr; 1609 1610 sa_sync_from_device(rxd); 1611 req = container_of(rxd->req, struct aead_request, base); 1612 tfm = crypto_aead_reqtfm(req); 1613 start = req->assoclen + req->cryptlen; 1614 authsize = crypto_aead_authsize(tfm); 1615 1616 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1617 for (i = 0; i < (authsize / 4); i++) 1618 mdptr[i + 4] = swab32(mdptr[i + 4]); 1619 1620 if (rxd->enc) { 1621 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1622 1); 1623 } else { 1624 start -= authsize; 1625 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 1626 0); 1627 1628 err = crypto_memneq(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; 1629 } 1630 1631 sa_free_sa_rx_data(rxd); 1632 1633 aead_request_complete(req, err); 1634 } 1635 1636 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *fallback) 1637 { 1638 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1639 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1640 int ret; 1641 1642 memzero_explicit(ctx, sizeof(*ctx)); 1643 ctx->dev_data = data; 1644 1645 ctx->fallback.aead = crypto_alloc_aead(fallback, 0, 1646 CRYPTO_ALG_NEED_FALLBACK); 1647 1648 if (IS_ERR(ctx->fallback.aead)) { 1649 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", 1650 fallback); 1651 return PTR_ERR(ctx->fallback.aead); 1652 } 1653 1654 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + 1655 crypto_aead_reqsize(ctx->fallback.aead)); 1656 1657 ret = sa_init_ctx_info(&ctx->enc, data); 1658 if (ret) 1659 return ret; 1660 1661 ret = sa_init_ctx_info(&ctx->dec, data); 1662 if (ret) { 1663 sa_free_ctx_info(&ctx->enc, data); 1664 return ret; 1665 } 1666 1667 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1668 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1669 ctx->dec.sc_id, &ctx->dec.sc_phys); 1670 1671 return ret; 1672 } 1673 1674 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) 1675 { 1676 return sa_cra_init_aead(tfm, "authenc(hmac(sha1),cbc(aes))"); 1677 } 1678 1679 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) 1680 { 1681 return sa_cra_init_aead(tfm, "authenc(hmac(sha256),cbc(aes))"); 1682 } 1683 1684 static void sa_exit_tfm_aead(struct crypto_aead *tfm) 1685 { 1686 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1687 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1688 1689 crypto_free_aead(ctx->fallback.aead); 1690 1691 sa_free_ctx_info(&ctx->enc, data); 1692 sa_free_ctx_info(&ctx->dec, data); 1693 } 1694 1695 /* AEAD algorithm configuration interface function */ 1696 static int sa_aead_setkey(struct crypto_aead *authenc, 1697 const u8 *key, unsigned int keylen, 1698 struct algo_data *ad) 1699 { 1700 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); 1701 struct crypto_authenc_keys keys; 1702 int cmdl_len; 1703 struct sa_cmdl_cfg cfg; 1704 int key_idx; 1705 1706 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1707 return -EINVAL; 1708 1709 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 1710 key_idx = (keys.enckeylen >> 3) - 2; 1711 if (key_idx >= 3) 1712 return -EINVAL; 1713 1714 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 1715 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1716 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1717 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1718 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; 1719 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; 1720 ad->inv_key = true; 1721 ad->keyed_mac = true; 1722 ad->ealg_id = SA_EALG_ID_AES_CBC; 1723 1724 memset(&cfg, 0, sizeof(cfg)); 1725 cfg.enc = true; 1726 cfg.aalg = ad->aalg_id; 1727 cfg.enc_eng_id = ad->enc_eng.eng_id; 1728 cfg.auth_eng_id = ad->auth_eng.eng_id; 1729 cfg.iv_size = crypto_aead_ivsize(authenc); 1730 cfg.akey = keys.authkey; 1731 cfg.akey_len = keys.authkeylen; 1732 1733 /* Setup Encryption Security Context & Command label template */ 1734 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey, 1735 keys.enckeylen, keys.authkey, keys.authkeylen, 1736 ad, 1, &ctx->enc.epib[1])) 1737 return -EINVAL; 1738 1739 cmdl_len = sa_format_cmdl_gen(&cfg, 1740 (u8 *)ctx->enc.cmdl, 1741 &ctx->enc.cmdl_upd_info); 1742 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1743 return -EINVAL; 1744 1745 ctx->enc.cmdl_size = cmdl_len; 1746 1747 /* Setup Decryption Security Context & Command label template */ 1748 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey, 1749 keys.enckeylen, keys.authkey, keys.authkeylen, 1750 ad, 0, &ctx->dec.epib[1])) 1751 return -EINVAL; 1752 1753 cfg.enc = false; 1754 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 1755 &ctx->dec.cmdl_upd_info); 1756 1757 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1758 return -EINVAL; 1759 1760 ctx->dec.cmdl_size = cmdl_len; 1761 1762 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); 1763 crypto_aead_set_flags(ctx->fallback.aead, 1764 crypto_aead_get_flags(authenc) & 1765 CRYPTO_TFM_REQ_MASK); 1766 1767 return crypto_aead_setkey(ctx->fallback.aead, key, keylen); 1768 } 1769 1770 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1771 { 1772 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); 1773 1774 return crypto_aead_setauthsize(ctx->fallback.aead, authsize); 1775 } 1776 1777 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, 1778 const u8 *key, unsigned int keylen) 1779 { 1780 struct algo_data ad = { 0 }; 1781 1782 ad.ealg_id = SA_EALG_ID_AES_CBC; 1783 ad.aalg_id = SA_AALG_ID_HMAC_SHA1; 1784 ad.hash_size = SHA1_DIGEST_SIZE; 1785 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1786 1787 return sa_aead_setkey(authenc, key, keylen, &ad); 1788 } 1789 1790 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, 1791 const u8 *key, unsigned int keylen) 1792 { 1793 struct algo_data ad = { 0 }; 1794 1795 ad.ealg_id = SA_EALG_ID_AES_CBC; 1796 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; 1797 ad.hash_size = SHA256_DIGEST_SIZE; 1798 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1799 1800 return sa_aead_setkey(authenc, key, keylen, &ad); 1801 } 1802 1803 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) 1804 { 1805 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1806 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1807 struct sa_req sa_req = { 0 }; 1808 size_t auth_size, enc_size; 1809 1810 enc_size = req->cryptlen; 1811 auth_size = req->assoclen + req->cryptlen; 1812 1813 if (!enc) { 1814 enc_size -= crypto_aead_authsize(tfm); 1815 auth_size -= crypto_aead_authsize(tfm); 1816 } 1817 1818 if (auth_size > SA_MAX_DATA_SZ || 1819 (auth_size >= SA_UNSAFE_DATA_SZ_MIN && 1820 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { 1821 struct aead_request *subreq = aead_request_ctx(req); 1822 int ret; 1823 1824 aead_request_set_tfm(subreq, ctx->fallback.aead); 1825 aead_request_set_callback(subreq, req->base.flags, 1826 req->base.complete, req->base.data); 1827 aead_request_set_crypt(subreq, req->src, req->dst, 1828 req->cryptlen, req->iv); 1829 aead_request_set_ad(subreq, req->assoclen); 1830 1831 ret = enc ? crypto_aead_encrypt(subreq) : 1832 crypto_aead_decrypt(subreq); 1833 return ret; 1834 } 1835 1836 sa_req.enc_offset = req->assoclen; 1837 sa_req.enc_size = enc_size; 1838 sa_req.auth_size = auth_size; 1839 sa_req.size = auth_size; 1840 sa_req.enc_iv = iv; 1841 sa_req.type = CRYPTO_ALG_TYPE_AEAD; 1842 sa_req.enc = enc; 1843 sa_req.callback = sa_aead_dma_in_callback; 1844 sa_req.mdata_size = 52; 1845 sa_req.base = &req->base; 1846 sa_req.ctx = ctx; 1847 sa_req.src = req->src; 1848 sa_req.dst = req->dst; 1849 1850 return sa_run(&sa_req); 1851 } 1852 1853 /* AEAD algorithm encrypt interface function */ 1854 static int sa_aead_encrypt(struct aead_request *req) 1855 { 1856 return sa_aead_run(req, req->iv, 1); 1857 } 1858 1859 /* AEAD algorithm decrypt interface function */ 1860 static int sa_aead_decrypt(struct aead_request *req) 1861 { 1862 return sa_aead_run(req, req->iv, 0); 1863 } 1864 1865 static struct sa_alg_tmpl sa_algs[] = { 1866 [SA_ALG_CBC_AES] = { 1867 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1868 .alg.skcipher = { 1869 .base.cra_name = "cbc(aes)", 1870 .base.cra_driver_name = "cbc-aes-sa2ul", 1871 .base.cra_priority = 30000, 1872 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1873 CRYPTO_ALG_KERN_DRIVER_ONLY | 1874 CRYPTO_ALG_ASYNC | 1875 CRYPTO_ALG_NEED_FALLBACK, 1876 .base.cra_blocksize = AES_BLOCK_SIZE, 1877 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1878 .base.cra_module = THIS_MODULE, 1879 .init = sa_cipher_cra_init, 1880 .exit = sa_cipher_cra_exit, 1881 .min_keysize = AES_MIN_KEY_SIZE, 1882 .max_keysize = AES_MAX_KEY_SIZE, 1883 .ivsize = AES_BLOCK_SIZE, 1884 .setkey = sa_aes_cbc_setkey, 1885 .encrypt = sa_encrypt, 1886 .decrypt = sa_decrypt, 1887 } 1888 }, 1889 [SA_ALG_EBC_AES] = { 1890 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1891 .alg.skcipher = { 1892 .base.cra_name = "ecb(aes)", 1893 .base.cra_driver_name = "ecb-aes-sa2ul", 1894 .base.cra_priority = 30000, 1895 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1896 CRYPTO_ALG_KERN_DRIVER_ONLY | 1897 CRYPTO_ALG_ASYNC | 1898 CRYPTO_ALG_NEED_FALLBACK, 1899 .base.cra_blocksize = AES_BLOCK_SIZE, 1900 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1901 .base.cra_module = THIS_MODULE, 1902 .init = sa_cipher_cra_init, 1903 .exit = sa_cipher_cra_exit, 1904 .min_keysize = AES_MIN_KEY_SIZE, 1905 .max_keysize = AES_MAX_KEY_SIZE, 1906 .setkey = sa_aes_ecb_setkey, 1907 .encrypt = sa_encrypt, 1908 .decrypt = sa_decrypt, 1909 } 1910 }, 1911 [SA_ALG_CBC_DES3] = { 1912 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1913 .alg.skcipher = { 1914 .base.cra_name = "cbc(des3_ede)", 1915 .base.cra_driver_name = "cbc-des3-sa2ul", 1916 .base.cra_priority = 30000, 1917 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1918 CRYPTO_ALG_KERN_DRIVER_ONLY | 1919 CRYPTO_ALG_ASYNC | 1920 CRYPTO_ALG_NEED_FALLBACK, 1921 .base.cra_blocksize = DES_BLOCK_SIZE, 1922 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1923 .base.cra_module = THIS_MODULE, 1924 .init = sa_cipher_cra_init, 1925 .exit = sa_cipher_cra_exit, 1926 .min_keysize = 3 * DES_KEY_SIZE, 1927 .max_keysize = 3 * DES_KEY_SIZE, 1928 .ivsize = DES_BLOCK_SIZE, 1929 .setkey = sa_3des_cbc_setkey, 1930 .encrypt = sa_encrypt, 1931 .decrypt = sa_decrypt, 1932 } 1933 }, 1934 [SA_ALG_ECB_DES3] = { 1935 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1936 .alg.skcipher = { 1937 .base.cra_name = "ecb(des3_ede)", 1938 .base.cra_driver_name = "ecb-des3-sa2ul", 1939 .base.cra_priority = 30000, 1940 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1941 CRYPTO_ALG_KERN_DRIVER_ONLY | 1942 CRYPTO_ALG_ASYNC | 1943 CRYPTO_ALG_NEED_FALLBACK, 1944 .base.cra_blocksize = DES_BLOCK_SIZE, 1945 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1946 .base.cra_module = THIS_MODULE, 1947 .init = sa_cipher_cra_init, 1948 .exit = sa_cipher_cra_exit, 1949 .min_keysize = 3 * DES_KEY_SIZE, 1950 .max_keysize = 3 * DES_KEY_SIZE, 1951 .setkey = sa_3des_ecb_setkey, 1952 .encrypt = sa_encrypt, 1953 .decrypt = sa_decrypt, 1954 } 1955 }, 1956 [SA_ALG_SHA1] = { 1957 .type = CRYPTO_ALG_TYPE_AHASH, 1958 .alg.ahash = { 1959 .halg.base = { 1960 .cra_name = "sha1", 1961 .cra_driver_name = "sha1-sa2ul", 1962 .cra_priority = 400, 1963 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1964 CRYPTO_ALG_ASYNC | 1965 CRYPTO_ALG_KERN_DRIVER_ONLY | 1966 CRYPTO_ALG_NEED_FALLBACK, 1967 .cra_blocksize = SHA1_BLOCK_SIZE, 1968 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 1969 .cra_module = THIS_MODULE, 1970 .cra_init = sa_sha1_cra_init, 1971 .cra_exit = sa_sha_cra_exit, 1972 }, 1973 .halg.digestsize = SHA1_DIGEST_SIZE, 1974 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 1975 sizeof(struct sha1_state), 1976 .init = sa_sha_init, 1977 .update = sa_sha_update, 1978 .final = sa_sha_final, 1979 .finup = sa_sha_finup, 1980 .digest = sa_sha_digest, 1981 .export = sa_sha_export, 1982 .import = sa_sha_import, 1983 }, 1984 }, 1985 [SA_ALG_SHA256] = { 1986 .type = CRYPTO_ALG_TYPE_AHASH, 1987 .alg.ahash = { 1988 .halg.base = { 1989 .cra_name = "sha256", 1990 .cra_driver_name = "sha256-sa2ul", 1991 .cra_priority = 400, 1992 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1993 CRYPTO_ALG_ASYNC | 1994 CRYPTO_ALG_KERN_DRIVER_ONLY | 1995 CRYPTO_ALG_NEED_FALLBACK, 1996 .cra_blocksize = SHA256_BLOCK_SIZE, 1997 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 1998 .cra_module = THIS_MODULE, 1999 .cra_init = sa_sha256_cra_init, 2000 .cra_exit = sa_sha_cra_exit, 2001 }, 2002 .halg.digestsize = SHA256_DIGEST_SIZE, 2003 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2004 sizeof(struct sha256_state), 2005 .init = sa_sha_init, 2006 .update = sa_sha_update, 2007 .final = sa_sha_final, 2008 .finup = sa_sha_finup, 2009 .digest = sa_sha_digest, 2010 .export = sa_sha_export, 2011 .import = sa_sha_import, 2012 }, 2013 }, 2014 [SA_ALG_SHA512] = { 2015 .type = CRYPTO_ALG_TYPE_AHASH, 2016 .alg.ahash = { 2017 .halg.base = { 2018 .cra_name = "sha512", 2019 .cra_driver_name = "sha512-sa2ul", 2020 .cra_priority = 400, 2021 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2022 CRYPTO_ALG_ASYNC | 2023 CRYPTO_ALG_KERN_DRIVER_ONLY | 2024 CRYPTO_ALG_NEED_FALLBACK, 2025 .cra_blocksize = SHA512_BLOCK_SIZE, 2026 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2027 .cra_module = THIS_MODULE, 2028 .cra_init = sa_sha512_cra_init, 2029 .cra_exit = sa_sha_cra_exit, 2030 }, 2031 .halg.digestsize = SHA512_DIGEST_SIZE, 2032 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2033 sizeof(struct sha512_state), 2034 .init = sa_sha_init, 2035 .update = sa_sha_update, 2036 .final = sa_sha_final, 2037 .finup = sa_sha_finup, 2038 .digest = sa_sha_digest, 2039 .export = sa_sha_export, 2040 .import = sa_sha_import, 2041 }, 2042 }, 2043 [SA_ALG_AUTHENC_SHA1_AES] = { 2044 .type = CRYPTO_ALG_TYPE_AEAD, 2045 .alg.aead = { 2046 .base = { 2047 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2048 .cra_driver_name = 2049 "authenc(hmac(sha1),cbc(aes))-sa2ul", 2050 .cra_blocksize = AES_BLOCK_SIZE, 2051 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2052 CRYPTO_ALG_KERN_DRIVER_ONLY | 2053 CRYPTO_ALG_ASYNC | 2054 CRYPTO_ALG_NEED_FALLBACK, 2055 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2056 .cra_module = THIS_MODULE, 2057 .cra_priority = 3000, 2058 }, 2059 .ivsize = AES_BLOCK_SIZE, 2060 .maxauthsize = SHA1_DIGEST_SIZE, 2061 2062 .init = sa_cra_init_aead_sha1, 2063 .exit = sa_exit_tfm_aead, 2064 .setkey = sa_aead_cbc_sha1_setkey, 2065 .setauthsize = sa_aead_setauthsize, 2066 .encrypt = sa_aead_encrypt, 2067 .decrypt = sa_aead_decrypt, 2068 }, 2069 }, 2070 [SA_ALG_AUTHENC_SHA256_AES] = { 2071 .type = CRYPTO_ALG_TYPE_AEAD, 2072 .alg.aead = { 2073 .base = { 2074 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2075 .cra_driver_name = 2076 "authenc(hmac(sha256),cbc(aes))-sa2ul", 2077 .cra_blocksize = AES_BLOCK_SIZE, 2078 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2079 CRYPTO_ALG_KERN_DRIVER_ONLY | 2080 CRYPTO_ALG_ASYNC | 2081 CRYPTO_ALG_NEED_FALLBACK, 2082 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2083 .cra_module = THIS_MODULE, 2084 .cra_alignmask = 0, 2085 .cra_priority = 3000, 2086 }, 2087 .ivsize = AES_BLOCK_SIZE, 2088 .maxauthsize = SHA256_DIGEST_SIZE, 2089 2090 .init = sa_cra_init_aead_sha256, 2091 .exit = sa_exit_tfm_aead, 2092 .setkey = sa_aead_cbc_sha256_setkey, 2093 .setauthsize = sa_aead_setauthsize, 2094 .encrypt = sa_aead_encrypt, 2095 .decrypt = sa_aead_decrypt, 2096 }, 2097 }, 2098 }; 2099 2100 /* Register the algorithms in crypto framework */ 2101 static void sa_register_algos(struct sa_crypto_data *dev_data) 2102 { 2103 const struct sa_match_data *match_data = dev_data->match_data; 2104 struct device *dev = dev_data->dev; 2105 char *alg_name; 2106 u32 type; 2107 int i, err; 2108 2109 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2110 /* Skip unsupported algos */ 2111 if (!(match_data->supported_algos & BIT(i))) 2112 continue; 2113 2114 type = sa_algs[i].type; 2115 if (type == CRYPTO_ALG_TYPE_SKCIPHER) { 2116 alg_name = sa_algs[i].alg.skcipher.base.cra_name; 2117 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); 2118 } else if (type == CRYPTO_ALG_TYPE_AHASH) { 2119 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; 2120 err = crypto_register_ahash(&sa_algs[i].alg.ahash); 2121 } else if (type == CRYPTO_ALG_TYPE_AEAD) { 2122 alg_name = sa_algs[i].alg.aead.base.cra_name; 2123 err = crypto_register_aead(&sa_algs[i].alg.aead); 2124 } else { 2125 dev_err(dev, 2126 "un-supported crypto algorithm (%d)", 2127 sa_algs[i].type); 2128 continue; 2129 } 2130 2131 if (err) 2132 dev_err(dev, "Failed to register '%s'\n", alg_name); 2133 else 2134 sa_algs[i].registered = true; 2135 } 2136 } 2137 2138 /* Unregister the algorithms in crypto framework */ 2139 static void sa_unregister_algos(const struct device *dev) 2140 { 2141 u32 type; 2142 int i; 2143 2144 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2145 type = sa_algs[i].type; 2146 if (!sa_algs[i].registered) 2147 continue; 2148 if (type == CRYPTO_ALG_TYPE_SKCIPHER) 2149 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); 2150 else if (type == CRYPTO_ALG_TYPE_AHASH) 2151 crypto_unregister_ahash(&sa_algs[i].alg.ahash); 2152 else if (type == CRYPTO_ALG_TYPE_AEAD) 2153 crypto_unregister_aead(&sa_algs[i].alg.aead); 2154 2155 sa_algs[i].registered = false; 2156 } 2157 } 2158 2159 static int sa_init_mem(struct sa_crypto_data *dev_data) 2160 { 2161 struct device *dev = &dev_data->pdev->dev; 2162 /* Setup dma pool for security context buffers */ 2163 dev_data->sc_pool = dma_pool_create("keystone-sc", dev, 2164 SA_CTX_MAX_SZ, 64, 0); 2165 if (!dev_data->sc_pool) { 2166 dev_err(dev, "Failed to create dma pool"); 2167 return -ENOMEM; 2168 } 2169 2170 return 0; 2171 } 2172 2173 static int sa_dma_init(struct sa_crypto_data *dd) 2174 { 2175 int ret; 2176 struct dma_slave_config cfg; 2177 2178 dd->dma_rx1 = NULL; 2179 dd->dma_tx = NULL; 2180 dd->dma_rx2 = NULL; 2181 2182 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); 2183 if (ret) 2184 return ret; 2185 2186 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); 2187 if (IS_ERR(dd->dma_rx1)) 2188 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), 2189 "Unable to request rx1 DMA channel\n"); 2190 2191 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); 2192 if (IS_ERR(dd->dma_rx2)) { 2193 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), 2194 "Unable to request rx2 DMA channel\n"); 2195 goto err_dma_rx2; 2196 } 2197 2198 dd->dma_tx = dma_request_chan(dd->dev, "tx"); 2199 if (IS_ERR(dd->dma_tx)) { 2200 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), 2201 "Unable to request tx DMA channel\n"); 2202 goto err_dma_tx; 2203 } 2204 2205 memzero_explicit(&cfg, sizeof(cfg)); 2206 2207 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2208 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2209 cfg.src_maxburst = 4; 2210 cfg.dst_maxburst = 4; 2211 2212 ret = dmaengine_slave_config(dd->dma_rx1, &cfg); 2213 if (ret) { 2214 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2215 ret); 2216 goto err_dma_config; 2217 } 2218 2219 ret = dmaengine_slave_config(dd->dma_rx2, &cfg); 2220 if (ret) { 2221 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2222 ret); 2223 goto err_dma_config; 2224 } 2225 2226 ret = dmaengine_slave_config(dd->dma_tx, &cfg); 2227 if (ret) { 2228 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", 2229 ret); 2230 goto err_dma_config; 2231 } 2232 2233 return 0; 2234 2235 err_dma_config: 2236 dma_release_channel(dd->dma_tx); 2237 err_dma_tx: 2238 dma_release_channel(dd->dma_rx2); 2239 err_dma_rx2: 2240 dma_release_channel(dd->dma_rx1); 2241 2242 return ret; 2243 } 2244 2245 static int sa_link_child(struct device *dev, void *data) 2246 { 2247 struct device *parent = data; 2248 2249 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); 2250 2251 return 0; 2252 } 2253 2254 static struct sa_match_data am654_match_data = { 2255 .priv = 1, 2256 .priv_id = 1, 2257 .supported_algos = BIT(SA_ALG_CBC_AES) | 2258 BIT(SA_ALG_EBC_AES) | 2259 BIT(SA_ALG_CBC_DES3) | 2260 BIT(SA_ALG_ECB_DES3) | 2261 BIT(SA_ALG_SHA1) | 2262 BIT(SA_ALG_SHA256) | 2263 BIT(SA_ALG_SHA512) | 2264 BIT(SA_ALG_AUTHENC_SHA1_AES) | 2265 BIT(SA_ALG_AUTHENC_SHA256_AES), 2266 }; 2267 2268 static struct sa_match_data am64_match_data = { 2269 .priv = 0, 2270 .priv_id = 0, 2271 .supported_algos = BIT(SA_ALG_CBC_AES) | 2272 BIT(SA_ALG_EBC_AES) | 2273 BIT(SA_ALG_SHA256) | 2274 BIT(SA_ALG_SHA512) | 2275 BIT(SA_ALG_AUTHENC_SHA256_AES), 2276 }; 2277 2278 static const struct of_device_id of_match[] = { 2279 { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, }, 2280 { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, }, 2281 { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, }, 2282 { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, }, 2283 {}, 2284 }; 2285 MODULE_DEVICE_TABLE(of, of_match); 2286 2287 static int sa_ul_probe(struct platform_device *pdev) 2288 { 2289 struct device *dev = &pdev->dev; 2290 struct device_node *node = dev->of_node; 2291 static void __iomem *saul_base; 2292 struct sa_crypto_data *dev_data; 2293 u32 status, val; 2294 int ret; 2295 2296 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); 2297 if (!dev_data) 2298 return -ENOMEM; 2299 2300 dev_data->match_data = of_device_get_match_data(dev); 2301 if (!dev_data->match_data) 2302 return -ENODEV; 2303 2304 saul_base = devm_platform_ioremap_resource(pdev, 0); 2305 if (IS_ERR(saul_base)) 2306 return PTR_ERR(saul_base); 2307 2308 sa_k3_dev = dev; 2309 dev_data->dev = dev; 2310 dev_data->pdev = pdev; 2311 dev_data->base = saul_base; 2312 platform_set_drvdata(pdev, dev_data); 2313 dev_set_drvdata(sa_k3_dev, dev_data); 2314 2315 pm_runtime_enable(dev); 2316 ret = pm_runtime_resume_and_get(dev); 2317 if (ret < 0) { 2318 dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret); 2319 pm_runtime_disable(dev); 2320 return ret; 2321 } 2322 2323 ret = sa_init_mem(dev_data); 2324 if (ret) 2325 goto disable_pm; 2326 2327 ret = sa_dma_init(dev_data); 2328 if (ret) 2329 goto destroy_dma_pool; 2330 2331 spin_lock_init(&dev_data->scid_lock); 2332 2333 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | 2334 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | 2335 SA_EEC_TRNG_EN; 2336 status = readl_relaxed(saul_base + SA_ENGINE_STATUS); 2337 /* Only enable engines if all are not already enabled */ 2338 if (val & ~status) 2339 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); 2340 2341 sa_register_algos(dev_data); 2342 2343 ret = of_platform_populate(node, NULL, NULL, dev); 2344 if (ret) 2345 goto release_dma; 2346 2347 device_for_each_child(dev, dev, sa_link_child); 2348 2349 return 0; 2350 2351 release_dma: 2352 sa_unregister_algos(dev); 2353 2354 dma_release_channel(dev_data->dma_rx2); 2355 dma_release_channel(dev_data->dma_rx1); 2356 dma_release_channel(dev_data->dma_tx); 2357 2358 destroy_dma_pool: 2359 dma_pool_destroy(dev_data->sc_pool); 2360 2361 disable_pm: 2362 pm_runtime_put_sync(dev); 2363 pm_runtime_disable(dev); 2364 2365 return ret; 2366 } 2367 2368 static void sa_ul_remove(struct platform_device *pdev) 2369 { 2370 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); 2371 2372 of_platform_depopulate(&pdev->dev); 2373 2374 sa_unregister_algos(&pdev->dev); 2375 2376 dma_release_channel(dev_data->dma_rx2); 2377 dma_release_channel(dev_data->dma_rx1); 2378 dma_release_channel(dev_data->dma_tx); 2379 2380 dma_pool_destroy(dev_data->sc_pool); 2381 2382 platform_set_drvdata(pdev, NULL); 2383 2384 pm_runtime_put_sync(&pdev->dev); 2385 pm_runtime_disable(&pdev->dev); 2386 } 2387 2388 static struct platform_driver sa_ul_driver = { 2389 .probe = sa_ul_probe, 2390 .remove = sa_ul_remove, 2391 .driver = { 2392 .name = "saul-crypto", 2393 .of_match_table = of_match, 2394 }, 2395 }; 2396 module_platform_driver(sa_ul_driver); 2397 MODULE_DESCRIPTION("K3 SA2UL crypto accelerator driver"); 2398 MODULE_LICENSE("GPL v2"); 2399