1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Provide TDMA helper functions used by cipher and hash algorithm 4 * implementations. 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 * Author: Arnaud Ebalard <arno@natisbad.org> 8 * 9 * This work is based on an initial version written by 10 * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > 11 */ 12 13 #include "cesa.h" 14 15 bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, 16 struct mv_cesa_sg_dma_iter *sgiter, 17 unsigned int len) 18 { 19 if (!sgiter->sg) 20 return false; 21 22 sgiter->op_offset += len; 23 sgiter->offset += len; 24 if (sgiter->offset == sg_dma_len(sgiter->sg)) { 25 if (sg_is_last(sgiter->sg)) 26 return false; 27 sgiter->offset = 0; 28 sgiter->sg = sg_next(sgiter->sg); 29 } 30 31 if (sgiter->op_offset == iter->op_len) 32 return false; 33 34 return true; 35 } 36 37 void mv_cesa_dma_step(struct mv_cesa_req *dreq) 38 { 39 struct mv_cesa_engine *engine = dreq->engine; 40 41 spin_lock_bh(&engine->lock); 42 if (engine->chain_sw.first == dreq->chain.first) { 43 engine->chain_sw.first = NULL; 44 engine->chain_sw.last = NULL; 45 } 46 engine->chain_hw.first = dreq->chain.first; 47 engine->chain_hw.last = dreq->chain.last; 48 spin_unlock_bh(&engine->lock); 49 50 writel_relaxed(0, engine->regs + CESA_SA_CFG); 51 52 mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); 53 writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | 54 CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, 55 engine->regs + CESA_TDMA_CONTROL); 56 57 writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | 58 CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, 59 engine->regs + CESA_SA_CFG); 60 writel_relaxed(dreq->chain.first->cur_dma, 61 engine->regs + CESA_TDMA_NEXT_ADDR); 62 WARN_ON(readl(engine->regs + CESA_SA_CMD) & 63 CESA_SA_CMD_EN_CESA_SA_ACCL0); 64 writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); 65 } 66 67 void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) 68 { 69 struct mv_cesa_tdma_desc *tdma; 70 71 for (tdma = dreq->chain.first; tdma;) { 72 struct mv_cesa_tdma_desc *old_tdma = tdma; 73 u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; 74 75 if (type == CESA_TDMA_OP) 76 dma_pool_free(cesa_dev->dma->op_pool, tdma->op, 77 le32_to_cpu(tdma->src)); 78 79 tdma = tdma->next; 80 dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, 81 old_tdma->cur_dma); 82 } 83 84 dreq->chain.first = NULL; 85 dreq->chain.last = NULL; 86 } 87 88 void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, 89 struct mv_cesa_engine *engine) 90 { 91 struct mv_cesa_tdma_desc *tdma; 92 93 for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { 94 if (tdma->flags & CESA_TDMA_DST_IN_SRAM) 95 tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma); 96 97 if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) 98 tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma); 99 100 if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) 101 mv_cesa_adjust_op(engine, tdma->op); 102 } 103 } 104 105 void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, 106 struct mv_cesa_req *dreq) 107 { 108 struct mv_cesa_tdma_desc *last = engine->chain_sw.last; 109 110 /* 111 * Break the DMA chain if the request being queued needs the IV 112 * regs to be set before lauching the request. 113 */ 114 if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) 115 engine->chain_sw.first = dreq->chain.first; 116 else { 117 last->next = dreq->chain.first; 118 last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); 119 } 120 last = dreq->chain.last; 121 engine->chain_sw.last = last; 122 /* 123 * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on 124 * the last element of the current chain. 125 */ 126 if (last->flags & CESA_TDMA_BREAK_CHAIN) { 127 engine->chain_sw.first = NULL; 128 engine->chain_sw.last = NULL; 129 } 130 } 131 132 int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) 133 { 134 struct crypto_async_request *req = NULL; 135 struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; 136 dma_addr_t tdma_cur; 137 int res = 0; 138 139 tdma_cur = readl(engine->regs + CESA_TDMA_CUR); 140 141 for (tdma = engine->chain_hw.first; tdma; tdma = next) { 142 spin_lock_bh(&engine->lock); 143 next = tdma->next; 144 spin_unlock_bh(&engine->lock); 145 146 if (tdma->flags & CESA_TDMA_END_OF_REQ) { 147 struct crypto_async_request *backlog = NULL; 148 struct mv_cesa_ctx *ctx; 149 u32 current_status; 150 151 spin_lock_bh(&engine->lock); 152 /* 153 * if req is NULL, this means we're processing the 154 * request in engine->req. 155 */ 156 if (!req) 157 req = engine->req; 158 else 159 req = mv_cesa_dequeue_req_locked(engine, 160 &backlog); 161 162 /* Re-chaining to the next request */ 163 engine->chain_hw.first = tdma->next; 164 tdma->next = NULL; 165 166 /* If this is the last request, clear the chain */ 167 if (engine->chain_hw.first == NULL) 168 engine->chain_hw.last = NULL; 169 spin_unlock_bh(&engine->lock); 170 171 ctx = crypto_tfm_ctx(req->tfm); 172 current_status = (tdma->cur_dma == tdma_cur) ? 173 status : CESA_SA_INT_ACC0_IDMA_DONE; 174 res = ctx->ops->process(req, current_status); 175 ctx->ops->complete(req); 176 177 if (res == 0) 178 mv_cesa_engine_enqueue_complete_request(engine, 179 req); 180 181 if (backlog) 182 crypto_request_complete(backlog, -EINPROGRESS); 183 } 184 185 if (res || tdma->cur_dma == tdma_cur) 186 break; 187 } 188 189 /* 190 * Save the last request in error to engine->req, so that the core 191 * knows which request was faulty 192 */ 193 if (res) { 194 spin_lock_bh(&engine->lock); 195 engine->req = req; 196 spin_unlock_bh(&engine->lock); 197 } 198 199 return res; 200 } 201 202 static struct mv_cesa_tdma_desc * 203 mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) 204 { 205 struct mv_cesa_tdma_desc *new_tdma = NULL; 206 dma_addr_t dma_handle; 207 208 new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags, 209 &dma_handle); 210 if (!new_tdma) 211 return ERR_PTR(-ENOMEM); 212 213 new_tdma->cur_dma = dma_handle; 214 if (chain->last) { 215 chain->last->next_dma = cpu_to_le32(dma_handle); 216 chain->last->next = new_tdma; 217 } else { 218 chain->first = new_tdma; 219 } 220 221 chain->last = new_tdma; 222 223 return new_tdma; 224 } 225 226 int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, 227 u32 size, u32 flags, gfp_t gfp_flags) 228 { 229 struct mv_cesa_tdma_desc *tdma, *op_desc; 230 231 tdma = mv_cesa_dma_add_desc(chain, gfp_flags); 232 if (IS_ERR(tdma)) 233 return PTR_ERR(tdma); 234 235 /* We re-use an existing op_desc object to retrieve the context 236 * and result instead of allocating a new one. 237 * There is at least one object of this type in a CESA crypto 238 * req, just pick the first one in the chain. 239 */ 240 for (op_desc = chain->first; op_desc; op_desc = op_desc->next) { 241 u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK; 242 243 if (type == CESA_TDMA_OP) 244 break; 245 } 246 247 if (!op_desc) 248 return -EIO; 249 250 tdma->byte_cnt = cpu_to_le32(size | BIT(31)); 251 tdma->src_dma = src; 252 tdma->dst_dma = op_desc->src_dma; 253 tdma->op = op_desc->op; 254 255 flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); 256 tdma->flags = flags | CESA_TDMA_RESULT; 257 return 0; 258 } 259 260 struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, 261 const struct mv_cesa_op_ctx *op_templ, 262 bool skip_ctx, 263 gfp_t flags) 264 { 265 struct mv_cesa_tdma_desc *tdma; 266 struct mv_cesa_op_ctx *op; 267 dma_addr_t dma_handle; 268 unsigned int size; 269 270 tdma = mv_cesa_dma_add_desc(chain, flags); 271 if (IS_ERR(tdma)) 272 return ERR_CAST(tdma); 273 274 op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle); 275 if (!op) 276 return ERR_PTR(-ENOMEM); 277 278 *op = *op_templ; 279 280 size = skip_ctx ? sizeof(op->desc) : sizeof(*op); 281 282 tdma = chain->last; 283 tdma->op = op; 284 tdma->byte_cnt = cpu_to_le32(size | BIT(31)); 285 tdma->src = cpu_to_le32(dma_handle); 286 tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET; 287 tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; 288 289 return op; 290 } 291 292 int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, 293 dma_addr_t dst, dma_addr_t src, u32 size, 294 u32 flags, gfp_t gfp_flags) 295 { 296 struct mv_cesa_tdma_desc *tdma; 297 298 tdma = mv_cesa_dma_add_desc(chain, gfp_flags); 299 if (IS_ERR(tdma)) 300 return PTR_ERR(tdma); 301 302 tdma->byte_cnt = cpu_to_le32(size | BIT(31)); 303 tdma->src_dma = src; 304 tdma->dst_dma = dst; 305 306 flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); 307 tdma->flags = flags | CESA_TDMA_DATA; 308 309 return 0; 310 } 311 312 int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) 313 { 314 struct mv_cesa_tdma_desc *tdma; 315 316 tdma = mv_cesa_dma_add_desc(chain, flags); 317 return PTR_ERR_OR_ZERO(tdma); 318 } 319 320 int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) 321 { 322 struct mv_cesa_tdma_desc *tdma; 323 324 tdma = mv_cesa_dma_add_desc(chain, flags); 325 if (IS_ERR(tdma)) 326 return PTR_ERR(tdma); 327 328 tdma->byte_cnt = cpu_to_le32(BIT(31)); 329 330 return 0; 331 } 332 333 int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, 334 struct mv_cesa_dma_iter *dma_iter, 335 struct mv_cesa_sg_dma_iter *sgiter, 336 gfp_t gfp_flags) 337 { 338 u32 flags = sgiter->dir == DMA_TO_DEVICE ? 339 CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM; 340 unsigned int len; 341 342 do { 343 dma_addr_t dst, src; 344 int ret; 345 346 len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter); 347 if (sgiter->dir == DMA_TO_DEVICE) { 348 dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; 349 src = sg_dma_address(sgiter->sg) + sgiter->offset; 350 } else { 351 dst = sg_dma_address(sgiter->sg) + sgiter->offset; 352 src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; 353 } 354 355 ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len, 356 flags, gfp_flags); 357 if (ret) 358 return ret; 359 360 } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len)); 361 362 return 0; 363 } 364 365 size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, 366 struct scatterlist *sgl, unsigned int nents, 367 unsigned int sram_off, size_t buflen, off_t skip, 368 bool to_sram) 369 { 370 unsigned int sg_flags = SG_MITER_ATOMIC; 371 struct sg_mapping_iter miter; 372 unsigned int offset = 0; 373 374 if (to_sram) 375 sg_flags |= SG_MITER_FROM_SG; 376 else 377 sg_flags |= SG_MITER_TO_SG; 378 379 sg_miter_start(&miter, sgl, nents, sg_flags); 380 381 if (!sg_miter_skip(&miter, skip)) 382 return 0; 383 384 while ((offset < buflen) && sg_miter_next(&miter)) { 385 unsigned int len; 386 387 len = min(miter.length, buflen - offset); 388 389 if (to_sram) { 390 if (engine->pool) 391 memcpy(engine->sram_pool + sram_off + offset, 392 miter.addr, len); 393 else 394 memcpy_toio(engine->sram + sram_off + offset, 395 miter.addr, len); 396 } else { 397 if (engine->pool) 398 memcpy(miter.addr, 399 engine->sram_pool + sram_off + offset, 400 len); 401 else 402 memcpy_fromio(miter.addr, 403 engine->sram + sram_off + offset, 404 len); 405 } 406 407 offset += len; 408 } 409 410 sg_miter_stop(&miter); 411 412 return offset; 413 } 414