1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/device.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/interrupt.h> 9 #include <linux/string.h> 10 #include <crypto/internal/hash.h> 11 12 #include "common.h" 13 #include "core.h" 14 #include "sha.h" 15 16 struct qce_sha_saved_state { 17 u8 pending_buf[QCE_SHA_MAX_BLOCKSIZE]; 18 u8 partial_digest[QCE_SHA_MAX_DIGESTSIZE]; 19 __be32 byte_count[2]; 20 unsigned int pending_buflen; 21 unsigned int flags; 22 u64 count; 23 bool first_blk; 24 }; 25 26 static LIST_HEAD(ahash_algs); 27 28 static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { 29 SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, 30 SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 31 }; 32 33 static void qce_ahash_done(void *data) 34 { 35 struct crypto_async_request *async_req = data; 36 struct ahash_request *req = ahash_request_cast(async_req); 37 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 38 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 39 struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); 40 struct qce_device *qce = tmpl->qce; 41 struct qce_result_dump *result = qce->dma.result_buf; 42 unsigned int digestsize = crypto_ahash_digestsize(ahash); 43 int error; 44 u32 status; 45 46 error = qce_dma_terminate_all(&qce->dma); 47 if (error) 48 dev_dbg(qce->dev, "ahash dma termination error (%d)\n", error); 49 50 dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 51 dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); 52 53 memcpy(rctx->digest, result->auth_iv, digestsize); 54 if (req->result && rctx->last_blk) 55 memcpy(req->result, result->auth_iv, digestsize); 56 57 rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); 58 rctx->byte_count[1] = cpu_to_be32(result->auth_byte_count[1]); 59 60 error = qce_check_status(qce, &status); 61 if (error < 0) 62 dev_dbg(qce->dev, "ahash operation error (%x)\n", status); 63 64 req->src = rctx->src_orig; 65 req->nbytes = rctx->nbytes_orig; 66 rctx->last_blk = false; 67 rctx->first_blk = false; 68 69 qce->async_req_done(tmpl->qce, error); 70 } 71 72 static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) 73 { 74 struct ahash_request *req = ahash_request_cast(async_req); 75 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 76 struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm); 77 struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); 78 struct qce_device *qce = tmpl->qce; 79 unsigned long flags = rctx->flags; 80 int ret; 81 82 if (IS_SHA_HMAC(flags)) { 83 rctx->authkey = ctx->authkey; 84 rctx->authklen = QCE_SHA_HMAC_KEY_SIZE; 85 } else if (IS_CMAC(flags)) { 86 rctx->authkey = ctx->authkey; 87 rctx->authklen = AES_KEYSIZE_128; 88 } 89 90 rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); 91 if (rctx->src_nents < 0) { 92 dev_err(qce->dev, "Invalid numbers of src SG.\n"); 93 return rctx->src_nents; 94 } 95 96 ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 97 if (!ret) 98 return -EIO; 99 100 sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); 101 102 ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); 103 if (!ret) { 104 ret = -EIO; 105 goto error_unmap_src; 106 } 107 108 ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents, 109 &rctx->result_sg, 1, qce_ahash_done, async_req); 110 if (ret) 111 goto error_unmap_dst; 112 113 qce_dma_issue_pending(&qce->dma); 114 115 ret = qce_start(async_req, tmpl->crypto_alg_type); 116 if (ret) 117 goto error_terminate; 118 119 return 0; 120 121 error_terminate: 122 qce_dma_terminate_all(&qce->dma); 123 error_unmap_dst: 124 dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); 125 error_unmap_src: 126 dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 127 return ret; 128 } 129 130 static int qce_ahash_init(struct ahash_request *req) 131 { 132 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 133 struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); 134 const u32 *std_iv = tmpl->std_iv; 135 136 memset(rctx, 0, sizeof(*rctx)); 137 rctx->first_blk = true; 138 rctx->last_blk = false; 139 rctx->flags = tmpl->alg_flags; 140 memcpy(rctx->digest, std_iv, sizeof(rctx->digest)); 141 142 return 0; 143 } 144 145 static int qce_ahash_export(struct ahash_request *req, void *out) 146 { 147 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 148 struct qce_sha_saved_state *export_state = out; 149 150 memcpy(export_state->pending_buf, rctx->buf, rctx->buflen); 151 memcpy(export_state->partial_digest, rctx->digest, sizeof(rctx->digest)); 152 export_state->byte_count[0] = rctx->byte_count[0]; 153 export_state->byte_count[1] = rctx->byte_count[1]; 154 export_state->pending_buflen = rctx->buflen; 155 export_state->count = rctx->count; 156 export_state->first_blk = rctx->first_blk; 157 export_state->flags = rctx->flags; 158 159 return 0; 160 } 161 162 static int qce_ahash_import(struct ahash_request *req, const void *in) 163 { 164 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 165 const struct qce_sha_saved_state *import_state = in; 166 167 memset(rctx, 0, sizeof(*rctx)); 168 rctx->count = import_state->count; 169 rctx->buflen = import_state->pending_buflen; 170 rctx->first_blk = import_state->first_blk; 171 rctx->flags = import_state->flags; 172 rctx->byte_count[0] = import_state->byte_count[0]; 173 rctx->byte_count[1] = import_state->byte_count[1]; 174 memcpy(rctx->buf, import_state->pending_buf, rctx->buflen); 175 memcpy(rctx->digest, import_state->partial_digest, sizeof(rctx->digest)); 176 177 return 0; 178 } 179 180 static int qce_ahash_update(struct ahash_request *req) 181 { 182 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 183 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 184 struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); 185 struct qce_device *qce = tmpl->qce; 186 unsigned int total; 187 unsigned int hash_later; 188 unsigned int blocksize; 189 190 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 191 rctx->count += req->nbytes; 192 193 /* check for buffer from previous updates and append it */ 194 total = req->nbytes + rctx->buflen; 195 196 if (total <= blocksize) { 197 scatterwalk_map_and_copy(rctx->buf + rctx->buflen, req->src, 198 0, req->nbytes, 0); 199 rctx->buflen += req->nbytes; 200 return 0; 201 } 202 203 /* save the original req structure fields */ 204 rctx->src_orig = req->src; 205 rctx->nbytes_orig = req->nbytes; 206 207 /* 208 * if we have data from previous update copy them on buffer. The old 209 * data will be combined with current request bytes. 210 */ 211 if (rctx->buflen) 212 memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); 213 214 /* calculate how many bytes will be hashed later */ 215 hash_later = total % blocksize; 216 217 /* 218 * At this point, there is more than one block size of data. If 219 * the available data to transfer is exactly a multiple of block 220 * size, save the last block to be transferred in qce_ahash_final 221 * (with the last block bit set) if this is indeed the end of data 222 * stream. If not this saved block will be transferred as part of 223 * next update. If this block is not held back and if this is 224 * indeed the end of data stream, the digest obtained will be wrong 225 * since qce_ahash_final will see that rctx->buflen is 0 and return 226 * doing nothing which in turn means that a digest will not be 227 * copied to the destination result buffer. qce_ahash_final cannot 228 * be made to alter this behavior and allowed to proceed if 229 * rctx->buflen is 0 because the crypto engine BAM does not allow 230 * for zero length transfers. 231 */ 232 if (!hash_later) 233 hash_later = blocksize; 234 235 scatterwalk_map_and_copy(rctx->buf, req->src, req->nbytes - hash_later, 236 hash_later, 0); 237 238 if (rctx->buflen) { 239 sg_init_table(rctx->sg, 2); 240 sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); 241 sg_chain(rctx->sg, 2, req->src); 242 req->src = rctx->sg; 243 } 244 245 /* hash only complete blocks */ 246 req->nbytes = total - hash_later; 247 rctx->buflen = hash_later; 248 249 return qce->async_req_enqueue(tmpl->qce, &req->base); 250 } 251 252 static int qce_ahash_final(struct ahash_request *req) 253 { 254 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 255 struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); 256 struct qce_device *qce = tmpl->qce; 257 258 if (!rctx->buflen) { 259 if (tmpl->hash_zero) 260 memcpy(req->result, tmpl->hash_zero, 261 tmpl->alg.ahash.halg.digestsize); 262 return 0; 263 } 264 265 rctx->last_blk = true; 266 267 rctx->src_orig = req->src; 268 rctx->nbytes_orig = req->nbytes; 269 270 memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); 271 sg_init_one(rctx->sg, rctx->tmpbuf, rctx->buflen); 272 273 req->src = rctx->sg; 274 req->nbytes = rctx->buflen; 275 276 return qce->async_req_enqueue(tmpl->qce, &req->base); 277 } 278 279 static int qce_ahash_digest(struct ahash_request *req) 280 { 281 struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); 282 struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); 283 struct qce_device *qce = tmpl->qce; 284 int ret; 285 286 ret = qce_ahash_init(req); 287 if (ret) 288 return ret; 289 290 rctx->src_orig = req->src; 291 rctx->nbytes_orig = req->nbytes; 292 rctx->first_blk = true; 293 rctx->last_blk = true; 294 295 if (!rctx->nbytes_orig) { 296 if (tmpl->hash_zero) 297 memcpy(req->result, tmpl->hash_zero, 298 tmpl->alg.ahash.halg.digestsize); 299 return 0; 300 } 301 302 return qce->async_req_enqueue(tmpl->qce, &req->base); 303 } 304 305 static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, 306 unsigned int keylen) 307 { 308 unsigned int digestsize = crypto_ahash_digestsize(tfm); 309 struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base); 310 struct crypto_wait wait; 311 struct ahash_request *req; 312 struct scatterlist sg; 313 unsigned int blocksize; 314 struct crypto_ahash *ahash_tfm; 315 u8 *buf; 316 int ret; 317 const char *alg_name; 318 319 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 320 memset(ctx->authkey, 0, sizeof(ctx->authkey)); 321 322 if (keylen <= blocksize) { 323 memcpy(ctx->authkey, key, keylen); 324 return 0; 325 } 326 327 if (digestsize == SHA256_DIGEST_SIZE) 328 alg_name = "sha256-qce"; 329 else 330 return -EINVAL; 331 332 ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); 333 if (IS_ERR(ahash_tfm)) 334 return PTR_ERR(ahash_tfm); 335 336 req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); 337 if (!req) { 338 ret = -ENOMEM; 339 goto err_free_ahash; 340 } 341 342 crypto_init_wait(&wait); 343 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 344 crypto_req_done, &wait); 345 crypto_ahash_clear_flags(ahash_tfm, ~0); 346 347 buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL); 348 if (!buf) { 349 ret = -ENOMEM; 350 goto err_free_req; 351 } 352 353 memcpy(buf, key, keylen); 354 sg_init_one(&sg, buf, keylen); 355 ahash_request_set_crypt(req, &sg, ctx->authkey, keylen); 356 357 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 358 359 kfree(buf); 360 err_free_req: 361 ahash_request_free(req); 362 err_free_ahash: 363 crypto_free_ahash(ahash_tfm); 364 return ret; 365 } 366 367 static int qce_ahash_cra_init(struct crypto_tfm *tfm) 368 { 369 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 370 struct qce_sha_ctx *ctx = crypto_tfm_ctx(tfm); 371 372 crypto_ahash_set_reqsize_dma(ahash, sizeof(struct qce_sha_reqctx)); 373 memset(ctx, 0, sizeof(*ctx)); 374 return 0; 375 } 376 377 struct qce_ahash_def { 378 unsigned long flags; 379 const char *name; 380 const char *drv_name; 381 unsigned int digestsize; 382 unsigned int blocksize; 383 unsigned int statesize; 384 const u32 *std_iv; 385 }; 386 387 static const struct qce_ahash_def ahash_def[] = { 388 { 389 .flags = QCE_HASH_SHA256, 390 .name = "sha256", 391 .drv_name = "sha256-qce", 392 .digestsize = SHA256_DIGEST_SIZE, 393 .blocksize = SHA256_BLOCK_SIZE, 394 .statesize = sizeof(struct qce_sha_saved_state), 395 .std_iv = std_iv_sha256, 396 }, 397 { 398 .flags = QCE_HASH_SHA256_HMAC, 399 .name = "hmac(sha256)", 400 .drv_name = "hmac-sha256-qce", 401 .digestsize = SHA256_DIGEST_SIZE, 402 .blocksize = SHA256_BLOCK_SIZE, 403 .statesize = sizeof(struct qce_sha_saved_state), 404 .std_iv = std_iv_sha256, 405 }, 406 }; 407 408 static int qce_ahash_register_one(const struct qce_ahash_def *def, 409 struct qce_device *qce) 410 { 411 struct qce_alg_template *tmpl; 412 struct ahash_alg *alg; 413 struct crypto_alg *base; 414 int ret; 415 416 tmpl = kzalloc_obj(*tmpl); 417 if (!tmpl) 418 return -ENOMEM; 419 420 tmpl->std_iv = def->std_iv; 421 422 alg = &tmpl->alg.ahash; 423 alg->init = qce_ahash_init; 424 alg->update = qce_ahash_update; 425 alg->final = qce_ahash_final; 426 alg->digest = qce_ahash_digest; 427 alg->export = qce_ahash_export; 428 alg->import = qce_ahash_import; 429 if (IS_SHA_HMAC(def->flags)) 430 alg->setkey = qce_ahash_hmac_setkey; 431 alg->halg.digestsize = def->digestsize; 432 alg->halg.statesize = def->statesize; 433 434 if (IS_SHA256(def->flags)) 435 tmpl->hash_zero = sha256_zero_message_hash; 436 437 base = &alg->halg.base; 438 base->cra_blocksize = def->blocksize; 439 base->cra_priority = 175; 440 base->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 441 base->cra_ctxsize = sizeof(struct qce_sha_ctx); 442 base->cra_alignmask = 0; 443 base->cra_module = THIS_MODULE; 444 base->cra_init = qce_ahash_cra_init; 445 446 strscpy(base->cra_name, def->name); 447 strscpy(base->cra_driver_name, def->drv_name); 448 449 INIT_LIST_HEAD(&tmpl->entry); 450 tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AHASH; 451 tmpl->alg_flags = def->flags; 452 tmpl->qce = qce; 453 454 ret = crypto_register_ahash(alg); 455 if (ret) { 456 dev_err(qce->dev, "%s registration failed\n", base->cra_name); 457 kfree(tmpl); 458 return ret; 459 } 460 461 list_add_tail(&tmpl->entry, &ahash_algs); 462 dev_dbg(qce->dev, "%s is registered\n", base->cra_name); 463 return 0; 464 } 465 466 static void qce_ahash_unregister(struct qce_device *qce) 467 { 468 struct qce_alg_template *tmpl, *n; 469 470 list_for_each_entry_safe(tmpl, n, &ahash_algs, entry) { 471 crypto_unregister_ahash(&tmpl->alg.ahash); 472 list_del(&tmpl->entry); 473 kfree(tmpl); 474 } 475 } 476 477 static int qce_ahash_register(struct qce_device *qce) 478 { 479 int ret, i; 480 481 for (i = 0; i < ARRAY_SIZE(ahash_def); i++) { 482 ret = qce_ahash_register_one(&ahash_def[i], qce); 483 if (ret) 484 goto err; 485 } 486 487 return 0; 488 err: 489 qce_ahash_unregister(qce); 490 return ret; 491 } 492 493 const struct qce_algo_ops ahash_ops = { 494 .type = CRYPTO_ALG_TYPE_AHASH, 495 .register_algs = qce_ahash_register, 496 .unregister_algs = qce_ahash_unregister, 497 .async_req_handle = qce_ahash_async_req_handle, 498 }; 499