Lines Matching +full:cmdq +full:- +full:sync
1 // SPDX-License-Identifier: GPL-2.0
24 * 0x00 - Success
26 * 0x43 - ERR_GC_DATA_LEN_INVALID
28 * less than 16 bytes for AES-XTS and AES-CTS.
29 * 0x45 - ERR_GC_CTX_LEN_INVALID
31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
33 * AES/DES-CBC mode encryption.
34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
42 * Invalid flag options in AES-CCM IV.
48 index = index + count - max; in incr_index()
57 struct nitrox_device *ndev = sr->ndev; in softreq_unmap_sgbufs()
61 dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), in softreq_unmap_sgbufs()
63 dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, in softreq_unmap_sgbufs()
65 kfree(sr->in.sgcomp); in softreq_unmap_sgbufs()
66 sr->in.sg = NULL; in softreq_unmap_sgbufs()
67 sr->in.sgmap_cnt = 0; in softreq_unmap_sgbufs()
69 dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), in softreq_unmap_sgbufs()
71 dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, in softreq_unmap_sgbufs()
73 kfree(sr->out.sgcomp); in softreq_unmap_sgbufs()
74 sr->out.sg = NULL; in softreq_unmap_sgbufs()
75 sr->out.sgmap_cnt = 0; in softreq_unmap_sgbufs()
85 * create_sg_component - create SG componets for N5 device.
93 * --------------------------------------
95 * |-------------------------------------
97 * --------------------------------------
99 * --------------------------------------
101 * --------------------------------------
103 * --------------------------------------
110 struct nitrox_device *ndev = sr->ndev; in create_sg_component()
121 sgcomp = kzalloc(sz_comp, sr->gfp); in create_sg_component()
123 return -ENOMEM; in create_sg_component()
125 sgtbl->sgcomp = sgcomp; in create_sg_component()
127 sg = sgtbl->sg; in create_sg_component()
137 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); in create_sg_component()
139 kfree(sgtbl->sgcomp); in create_sg_component()
140 sgtbl->sgcomp = NULL; in create_sg_component()
141 return -ENOMEM; in create_sg_component()
144 sgtbl->sgcomp_dma = dma; in create_sg_component()
145 sgtbl->sgcomp_len = sz_comp; in create_sg_component()
151 * dma_map_inbufs - DMA map input sglist and creates sglist component
161 struct device *dev = DEV(sr->ndev); in dma_map_inbufs()
165 nents = dma_map_sg(dev, req->src, sg_nents(req->src), in dma_map_inbufs()
168 return -EINVAL; in dma_map_inbufs()
170 for_each_sg(req->src, sg, nents, i) in dma_map_inbufs()
171 sr->in.total_bytes += sg_dma_len(sg); in dma_map_inbufs()
173 sr->in.sg = req->src; in dma_map_inbufs()
174 sr->in.sgmap_cnt = nents; in dma_map_inbufs()
175 ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); in dma_map_inbufs()
182 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); in dma_map_inbufs()
183 sr->in.sgmap_cnt = 0; in dma_map_inbufs()
190 struct device *dev = DEV(sr->ndev); in dma_map_outbufs()
193 nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), in dma_map_outbufs()
196 return -EINVAL; in dma_map_outbufs()
198 sr->out.sg = req->dst; in dma_map_outbufs()
199 sr->out.sgmap_cnt = nents; in dma_map_outbufs()
200 ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); in dma_map_outbufs()
207 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); in dma_map_outbufs()
208 sr->out.sgmap_cnt = 0; in dma_map_outbufs()
209 sr->out.sg = NULL; in dma_map_outbufs()
230 struct nitrox_cmdq *cmdq) in backlog_list_add() argument
232 INIT_LIST_HEAD(&sr->backlog); in backlog_list_add()
234 spin_lock_bh(&cmdq->backlog_qlock); in backlog_list_add()
235 list_add_tail(&sr->backlog, &cmdq->backlog_head); in backlog_list_add()
236 atomic_inc(&cmdq->backlog_count); in backlog_list_add()
237 atomic_set(&sr->status, REQ_BACKLOG); in backlog_list_add()
238 spin_unlock_bh(&cmdq->backlog_qlock); in backlog_list_add()
242 struct nitrox_cmdq *cmdq) in response_list_add() argument
244 INIT_LIST_HEAD(&sr->response); in response_list_add()
246 spin_lock_bh(&cmdq->resp_qlock); in response_list_add()
247 list_add_tail(&sr->response, &cmdq->response_head); in response_list_add()
248 spin_unlock_bh(&cmdq->resp_qlock); in response_list_add()
252 struct nitrox_cmdq *cmdq) in response_list_del() argument
254 spin_lock_bh(&cmdq->resp_qlock); in response_list_del()
255 list_del(&sr->response); in response_list_del()
256 spin_unlock_bh(&cmdq->resp_qlock); in response_list_del()
260 get_first_response_entry(struct nitrox_cmdq *cmdq) in get_first_response_entry() argument
262 return list_first_entry_or_null(&cmdq->response_head, in get_first_response_entry()
266 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) in cmdq_full() argument
268 if (atomic_inc_return(&cmdq->pending_count) > qlen) { in cmdq_full()
269 atomic_dec(&cmdq->pending_count); in cmdq_full()
270 /* sync with other cpus */ in cmdq_full()
274 /* sync with other cpus */ in cmdq_full()
280 * post_se_instr - Post SE instruction to Packet Input ring
282 * @cmdq: Command queue structure
288 struct nitrox_cmdq *cmdq) in post_se_instr() argument
290 struct nitrox_device *ndev = sr->ndev; in post_se_instr()
294 spin_lock_bh(&cmdq->cmd_qlock); in post_se_instr()
296 idx = cmdq->write_idx; in post_se_instr()
298 ent = cmdq->base + (idx * cmdq->instr_size); in post_se_instr()
299 memcpy(ent, &sr->instr, cmdq->instr_size); in post_se_instr()
301 atomic_set(&sr->status, REQ_POSTED); in post_se_instr()
302 response_list_add(sr, cmdq); in post_se_instr()
303 sr->tstamp = jiffies; in post_se_instr()
308 writeq(1, cmdq->dbell_csr_addr); in post_se_instr()
310 cmdq->write_idx = incr_index(idx, 1, ndev->qlen); in post_se_instr()
312 spin_unlock_bh(&cmdq->cmd_qlock); in post_se_instr()
315 atomic64_inc(&ndev->stats.posted); in post_se_instr()
318 static int post_backlog_cmds(struct nitrox_cmdq *cmdq) in post_backlog_cmds() argument
320 struct nitrox_device *ndev = cmdq->ndev; in post_backlog_cmds()
324 if (!atomic_read(&cmdq->backlog_count)) in post_backlog_cmds()
327 spin_lock_bh(&cmdq->backlog_qlock); in post_backlog_cmds()
329 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { in post_backlog_cmds()
331 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { in post_backlog_cmds()
332 ret = -ENOSPC; in post_backlog_cmds()
336 list_del(&sr->backlog); in post_backlog_cmds()
337 atomic_dec(&cmdq->backlog_count); in post_backlog_cmds()
338 /* sync with other cpus */ in post_backlog_cmds()
342 post_se_instr(sr, cmdq); in post_backlog_cmds()
344 spin_unlock_bh(&cmdq->backlog_qlock); in post_backlog_cmds()
351 struct nitrox_cmdq *cmdq = sr->cmdq; in nitrox_enqueue_request() local
352 struct nitrox_device *ndev = sr->ndev; in nitrox_enqueue_request()
355 post_backlog_cmds(cmdq); in nitrox_enqueue_request()
357 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { in nitrox_enqueue_request()
358 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { in nitrox_enqueue_request()
360 atomic64_inc(&ndev->stats.dropped); in nitrox_enqueue_request()
361 return -ENOSPC; in nitrox_enqueue_request()
364 backlog_list_add(sr, cmdq); in nitrox_enqueue_request()
365 return -EINPROGRESS; in nitrox_enqueue_request()
367 post_se_instr(sr, cmdq); in nitrox_enqueue_request()
369 return -EINPROGRESS; in nitrox_enqueue_request()
373 * nitrox_process_se_request - Send request to SE core
391 return -ENODEV; in nitrox_process_se_request()
393 sr = kzalloc(sizeof(*sr), req->gfp); in nitrox_process_se_request()
395 return -ENOMEM; in nitrox_process_se_request()
397 sr->ndev = ndev; in nitrox_process_se_request()
398 sr->flags = req->flags; in nitrox_process_se_request()
399 sr->gfp = req->gfp; in nitrox_process_se_request()
400 sr->callback = callback; in nitrox_process_se_request()
401 sr->cb_arg = cb_arg; in nitrox_process_se_request()
403 atomic_set(&sr->status, REQ_NOT_POSTED); in nitrox_process_se_request()
405 sr->resp.orh = req->orh; in nitrox_process_se_request()
406 sr->resp.completion = req->comp; in nitrox_process_se_request()
415 if (req->ctx_handle) { in nitrox_process_se_request()
419 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; in nitrox_process_se_request()
420 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); in nitrox_process_se_request()
421 ctx_handle = hdr->ctx_dma; in nitrox_process_se_request()
425 qno = smp_processor_id() % ndev->nr_queues; in nitrox_process_se_request()
427 sr->cmdq = &ndev->pkt_inq[qno]; in nitrox_process_se_request()
430 * 64-Byte Instruction Format in nitrox_process_se_request()
432 * ---------------------- in nitrox_process_se_request()
434 * ---------------------- in nitrox_process_se_request()
436 * ---------------------- in nitrox_process_se_request()
438 * ---------------------- in nitrox_process_se_request()
440 * ---------------------- in nitrox_process_se_request()
442 * ---------------------- in nitrox_process_se_request()
447 sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); in nitrox_process_se_request()
450 sr->instr.ih.value = 0; in nitrox_process_se_request()
451 sr->instr.ih.s.g = 1; in nitrox_process_se_request()
452 sr->instr.ih.s.gsz = sr->in.sgmap_cnt; in nitrox_process_se_request()
453 sr->instr.ih.s.ssz = sr->out.sgmap_cnt; in nitrox_process_se_request()
454 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); in nitrox_process_se_request()
455 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; in nitrox_process_se_request()
456 sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); in nitrox_process_se_request()
459 sr->instr.irh.value[0] = 0; in nitrox_process_se_request()
460 sr->instr.irh.s.uddl = MIN_UDD_LEN; in nitrox_process_se_request()
461 /* context length in 64-bit words */ in nitrox_process_se_request()
462 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); in nitrox_process_se_request()
464 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; in nitrox_process_se_request()
465 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; in nitrox_process_se_request()
466 sr->instr.irh.s.arg = req->ctrl.s.arg; in nitrox_process_se_request()
467 sr->instr.irh.s.opcode = req->opcode; in nitrox_process_se_request()
468 sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); in nitrox_process_se_request()
471 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); in nitrox_process_se_request()
474 sr->instr.slc.value[0] = 0; in nitrox_process_se_request()
475 sr->instr.slc.s.ssz = sr->out.sgmap_cnt; in nitrox_process_se_request()
476 sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); in nitrox_process_se_request()
479 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); in nitrox_process_se_request()
486 sr->instr.fdata[0] = *((u64 *)&req->gph); in nitrox_process_se_request()
487 sr->instr.fdata[1] = 0; in nitrox_process_se_request()
490 if (ret == -ENOSPC) in nitrox_process_se_request()
507 struct nitrox_cmdq *cmdq; in backlog_qflush_work() local
509 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); in backlog_qflush_work()
510 post_backlog_cmds(cmdq); in backlog_qflush_work()
515 u64 orh = READ_ONCE(*sr->resp.orh); in sr_completed()
521 while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { in sr_completed()
532 * process_response_list - process completed requests
533 * @cmdq: Command queue structure
537 static void process_response_list(struct nitrox_cmdq *cmdq) in process_response_list() argument
539 struct nitrox_device *ndev = cmdq->ndev; in process_response_list()
546 budget = atomic_read(&cmdq->pending_count); in process_response_list()
549 sr = get_first_response_entry(cmdq); in process_response_list()
553 if (atomic_read(&sr->status) != REQ_POSTED) in process_response_list()
559 if (!cmd_timeout(sr->tstamp, ndev->timeout)) in process_response_list()
563 READ_ONCE(*sr->resp.orh)); in process_response_list()
565 atomic_dec(&cmdq->pending_count); in process_response_list()
566 atomic64_inc(&ndev->stats.completed); in process_response_list()
567 /* sync with other cpus */ in process_response_list()
570 response_list_del(sr, cmdq); in process_response_list()
572 err = READ_ONCE(*sr->resp.orh) & 0xff; in process_response_list()
573 callback = sr->callback; in process_response_list()
574 cb_arg = sr->cb_arg; in process_response_list()
584 * pkt_slc_resp_tasklet - post processing of SE responses
589 struct nitrox_cmdq *cmdq = qvec->cmdq; in pkt_slc_resp_tasklet() local
593 slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); in pkt_slc_resp_tasklet()
597 process_response_list(cmdq); in pkt_slc_resp_tasklet()
601 * MSI-X interrupt generates if Completion count > Threshold in pkt_slc_resp_tasklet()
603 writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); in pkt_slc_resp_tasklet()
605 if (atomic_read(&cmdq->backlog_count)) in pkt_slc_resp_tasklet()
606 schedule_work(&cmdq->backlog_qflush); in pkt_slc_resp_tasklet()