1 /* 2 * Copyright 2016 Broadcom 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License, version 2, as 6 * published by the Free Software Foundation (the "GPL"). 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License version 2 (GPLv2) for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * version 2 (GPLv2) along with this source code. 15 */ 16 17 #ifndef _CIPHER_H 18 #define _CIPHER_H 19 20 #include <linux/atomic.h> 21 #include <linux/mailbox/brcm-message.h> 22 #include <linux/mailbox_client.h> 23 #include <crypto/aes.h> 24 #include <crypto/internal/hash.h> 25 #include <crypto/aead.h> 26 #include <crypto/sha.h> 27 #include <crypto/sha3.h> 28 29 #include "spu.h" 30 #include "spum.h" 31 #include "spu2.h" 32 33 /* Driver supports up to MAX_SPUS SPU blocks */ 34 #define MAX_SPUS 16 35 36 #define ARC4_MIN_KEY_SIZE 1 37 #define ARC4_MAX_KEY_SIZE 256 38 #define ARC4_BLOCK_SIZE 1 39 #define ARC4_STATE_SIZE 4 40 41 #define CCM_AES_IV_SIZE 16 42 #define GCM_AES_IV_SIZE 12 43 #define GCM_ESP_IV_SIZE 8 44 #define CCM_ESP_IV_SIZE 8 45 #define RFC4543_ICV_SIZE 16 46 47 #define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE 48 #define MAX_IV_SIZE AES_BLOCK_SIZE 49 #define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE 50 #define MAX_ASSOC_SIZE 512 51 52 /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */ 53 #define GCM_ESP_SALT_SIZE 4 54 #define CCM_ESP_SALT_SIZE 3 55 #define MAX_SALT_SIZE GCM_ESP_SALT_SIZE 56 #define GCM_ESP_SALT_OFFSET 0 57 #define CCM_ESP_SALT_OFFSET 1 58 59 #define GCM_ESP_DIGESTSIZE 16 60 61 #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 62 63 /* 64 * Maximum number of bytes from a non-final hash request that can be deferred 65 * until more data is available. With new crypto API framework, this 66 * can be no more than one block of data. 67 */ 68 #define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE 69 70 /* Force at least 4-byte alignment of all SPU message fields */ 71 #define SPU_MSG_ALIGN 4 72 73 /* Number of times to resend mailbox message if mb queue is full */ 74 #define SPU_MB_RETRY_MAX 1000 75 76 /* op_counts[] indexes */ 77 enum op_type { 78 SPU_OP_CIPHER, 79 SPU_OP_HASH, 80 SPU_OP_HMAC, 81 SPU_OP_AEAD, 82 SPU_OP_NUM 83 }; 84 85 enum spu_spu_type { 86 SPU_TYPE_SPUM, 87 SPU_TYPE_SPU2, 88 }; 89 90 /* 91 * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus, 92 * respectively. 93 */ 94 enum spu_spu_subtype { 95 SPU_SUBTYPE_SPUM_NS2, 96 SPU_SUBTYPE_SPUM_NSP, 97 SPU_SUBTYPE_SPU2_V1, 98 SPU_SUBTYPE_SPU2_V2 99 }; 100 101 struct spu_type_subtype { 102 enum spu_spu_type type; 103 enum spu_spu_subtype subtype; 104 }; 105 106 struct cipher_op { 107 enum spu_cipher_alg alg; 108 enum spu_cipher_mode mode; 109 }; 110 111 struct auth_op { 112 enum hash_alg alg; 113 enum hash_mode mode; 114 }; 115 116 struct iproc_alg_s { 117 u32 type; 118 union { 119 struct crypto_alg crypto; 120 struct ahash_alg hash; 121 struct aead_alg aead; 122 } alg; 123 struct cipher_op cipher_info; 124 struct auth_op auth_info; 125 bool auth_first; 126 bool registered; 127 }; 128 129 /* 130 * Buffers for a SPU request/reply message pair. All part of one structure to 131 * allow a single alloc per request. 132 */ 133 struct spu_msg_buf { 134 /* Request message fragments */ 135 136 /* 137 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC, 138 * and BD header. For SPU2, holds FMD, OMD. 139 */ 140 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 141 142 /* IV or counter. Size to include salt. Also used for XTS tweek. */ 143 u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; 144 145 /* Hash digest. request and response. */ 146 u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)]; 147 148 /* SPU request message padding */ 149 u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; 150 151 /* SPU-M request message STATUS field */ 152 u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)]; 153 154 /* Response message fragments */ 155 156 /* SPU response message header */ 157 u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 158 159 /* SPU response message STATUS field padding */ 160 u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)]; 161 162 /* SPU response message STATUS field */ 163 u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)]; 164 165 union { 166 /* Buffers only used for ablkcipher */ 167 struct { 168 /* 169 * Field used for either SUPDT when RC4 is used 170 * -OR- tweak value when XTS/AES is used 171 */ 172 u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)]; 173 } c; 174 175 /* Buffers only used for aead */ 176 struct { 177 /* SPU response pad for GCM data */ 178 u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; 179 180 /* SPU request msg padding for GCM AAD */ 181 u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; 182 183 /* SPU response data to be discarded */ 184 u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE, 185 SPU_MSG_ALIGN)]; 186 } a; 187 }; 188 }; 189 190 struct iproc_ctx_s { 191 u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; 192 unsigned int enckeylen; 193 194 u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; 195 unsigned int authkeylen; 196 197 u8 salt[MAX_SALT_SIZE]; 198 unsigned int salt_len; 199 unsigned int salt_offset; 200 u8 iv[MAX_IV_SIZE]; 201 202 unsigned int digestsize; 203 204 struct iproc_alg_s *alg; 205 bool is_esp; 206 207 struct cipher_op cipher; 208 enum spu_cipher_type cipher_type; 209 210 struct auth_op auth; 211 bool auth_first; 212 213 /* 214 * The maximum length in bytes of the payload in a SPU message for this 215 * context. For SPU-M, the payload is the combination of AAD and data. 216 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF 217 * indicates that there is no limit to the length of the SPU message 218 * payload. 219 */ 220 unsigned int max_payload; 221 222 struct crypto_aead *fallback_cipher; 223 224 /* auth_type is determined during processing of request */ 225 226 u8 ipad[MAX_HASH_BLOCK_SIZE]; 227 u8 opad[MAX_HASH_BLOCK_SIZE]; 228 229 /* 230 * Buffer to hold SPU message header template. Template is created at 231 * setkey time for ablkcipher requests, since most of the fields in the 232 * header are known at that time. At request time, just fill in a few 233 * missing pieces related to length of data in the request and IVs, etc. 234 */ 235 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 236 237 /* Length of SPU request header */ 238 u16 spu_req_hdr_len; 239 240 /* Expected length of SPU response header */ 241 u16 spu_resp_hdr_len; 242 243 /* 244 * shash descriptor - needed to perform incremental hashing in 245 * in software, when hw doesn't support it. 246 */ 247 struct shash_desc *shash; 248 249 bool is_rfc4543; /* RFC 4543 style of GMAC */ 250 }; 251 252 /* state from iproc_reqctx_s necessary for hash state export/import */ 253 struct spu_hash_export_s { 254 unsigned int total_todo; 255 unsigned int total_sent; 256 u8 hash_carry[HASH_CARRY_MAX]; 257 unsigned int hash_carry_len; 258 u8 incr_hash[MAX_DIGEST_SIZE]; 259 bool is_sw_hmac; 260 }; 261 262 struct iproc_reqctx_s { 263 /* general context */ 264 struct crypto_async_request *parent; 265 266 /* only valid after enqueue() */ 267 struct iproc_ctx_s *ctx; 268 269 u8 chan_idx; /* Mailbox channel to be used to submit this request */ 270 271 /* total todo, rx'd, and sent for this request */ 272 unsigned int total_todo; 273 unsigned int total_received; /* only valid for ablkcipher */ 274 unsigned int total_sent; 275 276 /* 277 * num bytes sent to hw from the src sg in this request. This can differ 278 * from total_sent for incremental hashing. total_sent includes previous 279 * init() and update() data. src_sent does not. 280 */ 281 unsigned int src_sent; 282 283 /* 284 * For AEAD requests, start of associated data. This will typically 285 * point to the beginning of the src scatterlist from the request, 286 * since assoc data is at the beginning of the src scatterlist rather 287 * than in its own sg. 288 */ 289 struct scatterlist *assoc; 290 291 /* 292 * scatterlist entry and offset to start of data for next chunk. Crypto 293 * API src scatterlist for AEAD starts with AAD, if present. For first 294 * chunk, src_sg is sg entry at beginning of input data (after AAD). 295 * src_skip begins at the offset in that sg entry where data begins. 296 */ 297 struct scatterlist *src_sg; 298 int src_nents; /* Number of src entries with data */ 299 u32 src_skip; /* bytes of current sg entry already used */ 300 301 /* 302 * Same for destination. For AEAD, if there is AAD, output data must 303 * be written at offset following AAD. 304 */ 305 struct scatterlist *dst_sg; 306 int dst_nents; /* Number of dst entries with data */ 307 u32 dst_skip; /* bytes of current sg entry already written */ 308 309 /* Mailbox message used to send this request to PDC driver */ 310 struct brcm_message mb_mssg; 311 312 bool bd_suppress; /* suppress BD field in SPU response? */ 313 314 /* cipher context */ 315 bool is_encrypt; 316 317 /* 318 * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA 319 * buffer for AEAD requests. So allocate as DMAable memory. If IV 320 * concatenated with salt, includes the salt. 321 */ 322 u8 *iv_ctr; 323 /* Length of IV or counter, in bytes */ 324 unsigned int iv_ctr_len; 325 326 /* 327 * Hash requests can be of any size, whether initial, update, or final. 328 * A non-final request must be submitted to the SPU as an integral 329 * number of blocks. This may leave data at the end of the request 330 * that is not a full block. Since the request is non-final, it cannot 331 * be padded. So, we write the remainder to this hash_carry buffer and 332 * hold it until the next request arrives. The carry data is then 333 * submitted at the beginning of the data in the next SPU msg. 334 * hash_carry_len is the number of bytes currently in hash_carry. These 335 * fields are only used for ahash requests. 336 */ 337 u8 hash_carry[HASH_CARRY_MAX]; 338 unsigned int hash_carry_len; 339 unsigned int is_final; /* is this the final for the hash op? */ 340 341 /* 342 * Digest from incremental hash is saved here to include in next hash 343 * operation. Cannot be stored in req->result for truncated hashes, 344 * since result may be sized for final digest. Cannot be saved in 345 * msg_buf because that gets deleted between incremental hash ops 346 * and is not saved as part of export(). 347 */ 348 u8 incr_hash[MAX_DIGEST_SIZE]; 349 350 /* hmac context */ 351 bool is_sw_hmac; 352 353 /* aead context */ 354 struct crypto_tfm *old_tfm; 355 crypto_completion_t old_complete; 356 void *old_data; 357 358 gfp_t gfp; 359 360 /* Buffers used to build SPU request and response messages */ 361 struct spu_msg_buf msg_buf; 362 }; 363 364 /* 365 * Structure encapsulates a set of function pointers specific to the type of 366 * SPU hardware running. These functions handling creation and parsing of 367 * SPU request messages and SPU response messages. Includes hardware-specific 368 * values read from device tree. 369 */ 370 struct spu_hw { 371 void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len); 372 u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg, 373 enum spu_cipher_mode cipher_mode, 374 unsigned int blocksize); 375 u32 (*spu_payload_length)(u8 *spu_hdr); 376 u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len, 377 bool is_hash); 378 u16 (*spu_hash_pad_len)(enum hash_alg hash_alg, 379 enum hash_mode hash_mode, u32 chunksize, 380 u16 hash_block_size); 381 u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode, 382 unsigned int data_size); 383 u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode, 384 unsigned int assoc_len, 385 unsigned int iv_len, bool is_encrypt); 386 u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode, 387 u16 iv_len); 388 enum hash_type (*spu_hash_type)(u32 src_sent); 389 u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg, 390 enum hash_type); 391 u32 (*spu_create_request)(u8 *spu_hdr, 392 struct spu_request_opts *req_opts, 393 struct spu_cipher_parms *cipher_parms, 394 struct spu_hash_parms *hash_parms, 395 struct spu_aead_parms *aead_parms, 396 unsigned int data_size); 397 u16 (*spu_cipher_req_init)(u8 *spu_hdr, 398 struct spu_cipher_parms *cipher_parms); 399 void (*spu_cipher_req_finish)(u8 *spu_hdr, 400 u16 spu_req_hdr_len, 401 unsigned int is_inbound, 402 struct spu_cipher_parms *cipher_parms, 403 bool update_key, 404 unsigned int data_size); 405 void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, 406 u32 hash_pad_len, enum hash_alg auth_alg, 407 enum hash_mode auth_mode, 408 unsigned int total_sent, u32 status_padding); 409 u8 (*spu_xts_tweak_in_payload)(void); 410 u8 (*spu_tx_status_len)(void); 411 u8 (*spu_rx_status_len)(void); 412 int (*spu_status_process)(u8 *statp); 413 void (*spu_ccm_update_iv)(unsigned int digestsize, 414 struct spu_cipher_parms *cipher_parms, 415 unsigned int assoclen, unsigned int chunksize, 416 bool is_encrypt, bool is_esp); 417 u32 (*spu_wordalign_padlen)(u32 data_size); 418 419 /* The base virtual address of the SPU hw registers */ 420 void __iomem *reg_vbase[MAX_SPUS]; 421 422 /* Version of the SPU hardware */ 423 enum spu_spu_type spu_type; 424 425 /* Sub-version of the SPU hardware */ 426 enum spu_spu_subtype spu_subtype; 427 428 /* The number of SPUs on this platform */ 429 u32 num_spu; 430 }; 431 432 struct device_private { 433 struct platform_device *pdev[MAX_SPUS]; 434 435 struct spu_hw spu; 436 437 atomic_t session_count; /* number of streams active */ 438 atomic_t stream_count; /* monotonic counter for streamID's */ 439 440 /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */ 441 u8 bcm_hdr_len; 442 443 /* The index of the channel to use for the next crypto request */ 444 atomic_t next_chan; 445 446 struct dentry *debugfs_dir; 447 struct dentry *debugfs_stats; 448 449 /* Number of request bytes processed and result bytes returned */ 450 atomic64_t bytes_in; 451 atomic64_t bytes_out; 452 453 /* Number of operations of each type */ 454 atomic_t op_counts[SPU_OP_NUM]; 455 456 atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST]; 457 atomic_t hash_cnt[HASH_ALG_LAST]; 458 atomic_t hmac_cnt[HASH_ALG_LAST]; 459 atomic_t aead_cnt[AEAD_TYPE_LAST]; 460 461 /* Number of calls to setkey() for each operation type */ 462 atomic_t setkey_cnt[SPU_OP_NUM]; 463 464 /* Number of times request was resubmitted because mb was full */ 465 atomic_t mb_no_spc; 466 467 /* Number of mailbox send failures */ 468 atomic_t mb_send_fail; 469 470 /* Number of ICV check failures for AEAD messages */ 471 atomic_t bad_icv; 472 473 struct mbox_client mcl[MAX_SPUS]; 474 /* Array of mailbox channel pointers, one for each channel */ 475 struct mbox_chan *mbox[MAX_SPUS]; 476 477 /* Driver initialized */ 478 bool inited; 479 }; 480 481 extern struct device_private iproc_priv; 482 483 #endif 484