1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include "otx2_cptvf.h" 5 #include "otx2_cpt_common.h" 6 7 /* Default timeout when waiting for free pending entry in us */ 8 #define CPT_PENTRY_TIMEOUT 1000 9 #define CPT_PENTRY_STEP 50 10 11 /* Default threshold for stopping and resuming sender requests */ 12 #define CPT_IQ_STOP_MARGIN 128 13 #define CPT_IQ_RESUME_MARGIN 512 14 15 /* Default command timeout in seconds */ 16 #define CPT_COMMAND_TIMEOUT 4 17 #define CPT_TIME_IN_RESET_COUNT 5 18 19 static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, 20 struct otx2_cpt_req_info *req) 21 { 22 int i; 23 24 pr_debug("Gather list size %d\n", req->in_cnt); 25 for (i = 0; i < req->in_cnt; i++) { 26 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, 27 req->in[i].size, req->in[i].vptr, 28 req->in[i].dma_addr); 29 pr_debug("Buffer hexdump (%d bytes)\n", 30 req->in[i].size); 31 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 32 req->in[i].vptr, req->in[i].size, false); 33 } 34 pr_debug("Scatter list size %d\n", req->out_cnt); 35 for (i = 0; i < req->out_cnt; i++) { 36 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, 37 req->out[i].size, req->out[i].vptr, 38 req->out[i].dma_addr); 39 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); 40 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 41 req->out[i].vptr, req->out[i].size, false); 42 } 43 } 44 45 static inline struct otx2_cpt_pending_entry *get_free_pending_entry( 46 struct otx2_cpt_pending_queue *q, 47 int qlen) 48 { 49 struct otx2_cpt_pending_entry *ent = NULL; 50 51 ent = &q->head[q->rear]; 52 if (unlikely(ent->busy)) 53 return NULL; 54 55 q->rear++; 56 if (unlikely(q->rear == qlen)) 57 q->rear = 0; 58 59 return ent; 60 } 61 62 static inline u32 modulo_inc(u32 index, u32 length, u32 inc) 63 { 64 if (WARN_ON(inc > length)) 65 inc = length; 66 67 index += inc; 68 if (unlikely(index >= length)) 69 index -= length; 70 71 return index; 72 } 73 74 static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) 75 { 76 pentry->completion_addr = NULL; 77 pentry->info = NULL; 78 pentry->callback = NULL; 79 pentry->areq = NULL; 80 pentry->resume_sender = false; 81 pentry->busy = false; 82 } 83 84 static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 85 struct otx2_cpt_pending_queue *pqueue, 86 struct otx2_cptlf_info *lf) 87 { 88 struct otx2_cptvf_request *cpt_req = &req->req; 89 struct otx2_cpt_pending_entry *pentry = NULL; 90 union otx2_cpt_ctrl_info *ctrl = &req->ctrl; 91 struct otx2_cpt_inst_info *info = NULL; 92 union otx2_cpt_res_s *result = NULL; 93 struct otx2_cpt_iq_command iq_cmd; 94 union otx2_cpt_inst_s cptinst; 95 int retry, ret = 0; 96 u8 resume_sender; 97 gfp_t gfp; 98 99 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : 100 GFP_ATOMIC; 101 if (unlikely(!otx2_cptlf_started(lf->lfs))) 102 return -ENODEV; 103 104 info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp); 105 if (unlikely(!info)) { 106 dev_err(&pdev->dev, "Setting up cpt inst info failed"); 107 return -ENOMEM; 108 } 109 cpt_req->dlen = info->dlen; 110 111 result = info->completion_addr; 112 result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; 113 114 spin_lock_bh(&pqueue->lock); 115 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 116 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; 117 while (unlikely(!pentry) && retry--) { 118 spin_unlock_bh(&pqueue->lock); 119 udelay(CPT_PENTRY_STEP); 120 spin_lock_bh(&pqueue->lock); 121 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 122 } 123 124 if (unlikely(!pentry)) { 125 ret = -ENOSPC; 126 goto destroy_info; 127 } 128 129 /* 130 * Check if we are close to filling in entire pending queue, 131 * if so then tell the sender to stop/sleep by returning -EBUSY 132 * We do it only for context which can sleep (GFP_KERNEL) 133 */ 134 if (gfp == GFP_KERNEL && 135 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { 136 pentry->resume_sender = true; 137 } else 138 pentry->resume_sender = false; 139 resume_sender = pentry->resume_sender; 140 pqueue->pending_count++; 141 142 pentry->completion_addr = info->completion_addr; 143 pentry->info = info; 144 pentry->callback = req->callback; 145 pentry->areq = req->areq; 146 pentry->busy = true; 147 info->pentry = pentry; 148 info->time_in = jiffies; 149 info->req = req; 150 151 /* Fill in the command */ 152 iq_cmd.cmd.u = 0; 153 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); 154 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); 155 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); 156 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); 157 158 /* 64-bit swap for microcode data reads, not needed for addresses*/ 159 cpu_to_be64s(&iq_cmd.cmd.u); 160 iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60; 161 iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60; 162 iq_cmd.cptr.s.cptr = cpt_req->cptr_dma; 163 iq_cmd.cptr.s.grp = ctrl->s.grp; 164 165 /* Fill in the CPT_INST_S type command for HW interpretation */ 166 otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); 167 168 /* Print debug info if enabled */ 169 otx2_cpt_dump_sg_list(pdev, req); 170 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); 171 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); 172 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); 173 print_hex_dump_debug("", 0, 16, 1, info->in_buffer, 174 cpt_req->dlen, false); 175 176 /* Send CPT command */ 177 lf->lfs->ops->send_cmd(&cptinst, 1, lf); 178 179 /* 180 * We allocate and prepare pending queue entry in critical section 181 * together with submitting CPT instruction to CPT instruction queue 182 * to make sure that order of CPT requests is the same in both 183 * pending and instruction queues 184 */ 185 spin_unlock_bh(&pqueue->lock); 186 187 ret = resume_sender ? -EBUSY : -EINPROGRESS; 188 return ret; 189 190 destroy_info: 191 spin_unlock_bh(&pqueue->lock); 192 otx2_cpt_info_destroy(pdev, info); 193 return ret; 194 } 195 196 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 197 int cpu_num) 198 { 199 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); 200 struct otx2_cptlfs_info *lfs = &cptvf->lfs; 201 202 return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, 203 &lfs->lf[cpu_num]); 204 } 205 206 static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, 207 union otx2_cpt_res_s *cpt_status, 208 struct otx2_cpt_inst_info *info, 209 u32 *res_code) 210 { 211 u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); 212 u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); 213 struct pci_dev *pdev = lfs->pdev; 214 215 switch (ccode) { 216 case OTX2_CPT_COMP_E_FAULT: 217 dev_err(&pdev->dev, 218 "Request failed with DMA fault\n"); 219 otx2_cpt_dump_sg_list(pdev, info->req); 220 break; 221 222 case OTX2_CPT_COMP_E_HWERR: 223 dev_err(&pdev->dev, 224 "Request failed with hardware error\n"); 225 otx2_cpt_dump_sg_list(pdev, info->req); 226 break; 227 228 case OTX2_CPT_COMP_E_INSTERR: 229 dev_err(&pdev->dev, 230 "Request failed with instruction error\n"); 231 otx2_cpt_dump_sg_list(pdev, info->req); 232 break; 233 234 case OTX2_CPT_COMP_E_NOTDONE: 235 /* check for timeout */ 236 if (time_after_eq(jiffies, info->time_in + 237 CPT_COMMAND_TIMEOUT * HZ)) 238 dev_warn(&pdev->dev, 239 "Request timed out 0x%p", info->req); 240 else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { 241 info->time_in = jiffies; 242 info->extra_time++; 243 } 244 return 1; 245 246 case OTX2_CPT_COMP_E_GOOD: 247 case OTX2_CPT_COMP_E_WARN: 248 /* 249 * Check microcode completion code, it is only valid 250 * when completion code is CPT_COMP_E::GOOD 251 */ 252 if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { 253 /* 254 * If requested hmac is truncated and ucode returns 255 * s/g write length error then we report success 256 * because ucode writes as many bytes of calculated 257 * hmac as available in gather buffer and reports 258 * s/g write length error if number of bytes in gather 259 * buffer is less than full hmac size. 260 */ 261 if (info->req->is_trunc_hmac && 262 uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { 263 *res_code = 0; 264 break; 265 } 266 267 dev_err(&pdev->dev, 268 "Request failed with software error code 0x%x\n", 269 cpt_status->s.uc_compcode); 270 otx2_cpt_dump_sg_list(pdev, info->req); 271 break; 272 } 273 /* Request has been processed with success */ 274 *res_code = 0; 275 break; 276 277 default: 278 dev_err(&pdev->dev, 279 "Request returned invalid status %d\n", ccode); 280 break; 281 } 282 return 0; 283 } 284 285 static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, 286 struct otx2_cpt_pending_queue *pqueue) 287 { 288 struct otx2_cpt_pending_entry *resume_pentry = NULL; 289 void (*callback)(int status, void *arg, void *req); 290 struct otx2_cpt_pending_entry *pentry = NULL; 291 union otx2_cpt_res_s *cpt_status = NULL; 292 struct otx2_cpt_inst_info *info = NULL; 293 struct otx2_cpt_req_info *req = NULL; 294 struct crypto_async_request *areq; 295 struct pci_dev *pdev = lfs->pdev; 296 u32 res_code, resume_index; 297 298 while (1) { 299 spin_lock_bh(&pqueue->lock); 300 pentry = &pqueue->head[pqueue->front]; 301 302 if (WARN_ON(!pentry)) { 303 spin_unlock_bh(&pqueue->lock); 304 break; 305 } 306 307 res_code = -EINVAL; 308 if (unlikely(!pentry->busy)) { 309 spin_unlock_bh(&pqueue->lock); 310 break; 311 } 312 313 if (unlikely(!pentry->callback)) { 314 dev_err(&pdev->dev, "Callback NULL\n"); 315 goto process_pentry; 316 } 317 318 info = pentry->info; 319 if (unlikely(!info)) { 320 dev_err(&pdev->dev, "Pending entry post arg NULL\n"); 321 goto process_pentry; 322 } 323 324 req = info->req; 325 if (unlikely(!req)) { 326 dev_err(&pdev->dev, "Request NULL\n"); 327 goto process_pentry; 328 } 329 330 cpt_status = pentry->completion_addr; 331 if (unlikely(!cpt_status)) { 332 dev_err(&pdev->dev, "Completion address NULL\n"); 333 goto process_pentry; 334 } 335 336 if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) { 337 spin_unlock_bh(&pqueue->lock); 338 return; 339 } 340 info->pdev = pdev; 341 342 process_pentry: 343 /* 344 * Check if we should inform sending side to resume 345 * We do it CPT_IQ_RESUME_MARGIN elements in advance before 346 * pending queue becomes empty 347 */ 348 resume_index = modulo_inc(pqueue->front, pqueue->qlen, 349 CPT_IQ_RESUME_MARGIN); 350 resume_pentry = &pqueue->head[resume_index]; 351 if (resume_pentry && 352 resume_pentry->resume_sender) { 353 resume_pentry->resume_sender = false; 354 callback = resume_pentry->callback; 355 areq = resume_pentry->areq; 356 357 if (callback) { 358 spin_unlock_bh(&pqueue->lock); 359 360 /* 361 * EINPROGRESS is an indication for sending 362 * side that it can resume sending requests 363 */ 364 callback(-EINPROGRESS, areq, info); 365 spin_lock_bh(&pqueue->lock); 366 } 367 } 368 369 callback = pentry->callback; 370 areq = pentry->areq; 371 free_pentry(pentry); 372 373 pqueue->pending_count--; 374 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); 375 spin_unlock_bh(&pqueue->lock); 376 377 /* 378 * Call callback after current pending entry has been 379 * processed, we don't do it if the callback pointer is 380 * invalid. 381 */ 382 if (callback) 383 callback(res_code, areq, info); 384 } 385 } 386 387 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) 388 { 389 process_pending_queue(wqe->lfs, 390 &wqe->lfs->lf[wqe->lf_num].pqueue); 391 } 392 393 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) 394 { 395 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); 396 397 return cptvf->lfs.kcrypto_eng_grp_num; 398 } 399