1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3
4 #include "otx2_cptvf.h"
5 #include "otx2_cpt_common.h"
6
7 /* Default timeout when waiting for free pending entry in us */
8 #define CPT_PENTRY_TIMEOUT 1000
9 #define CPT_PENTRY_STEP 50
10
11 /* Default threshold for stopping and resuming sender requests */
12 #define CPT_IQ_STOP_MARGIN 128
13 #define CPT_IQ_RESUME_MARGIN 512
14
15 /* Default command timeout in seconds */
16 #define CPT_COMMAND_TIMEOUT 4
17 #define CPT_TIME_IN_RESET_COUNT 5
18
otx2_cpt_dump_sg_list(struct pci_dev * pdev,struct otx2_cpt_req_info * req)19 static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,
20 struct otx2_cpt_req_info *req)
21 {
22 int i;
23
24 pr_debug("Gather list size %d\n", req->in_cnt);
25 for (i = 0; i < req->in_cnt; i++) {
26 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
27 req->in[i].size, req->in[i].vptr,
28 req->in[i].dma_addr);
29 pr_debug("Buffer hexdump (%d bytes)\n",
30 req->in[i].size);
31 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
32 req->in[i].vptr, req->in[i].size, false);
33 }
34 pr_debug("Scatter list size %d\n", req->out_cnt);
35 for (i = 0; i < req->out_cnt; i++) {
36 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
37 req->out[i].size, req->out[i].vptr,
38 req->out[i].dma_addr);
39 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
40 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
41 req->out[i].vptr, req->out[i].size, false);
42 }
43 }
44
get_free_pending_entry(struct otx2_cpt_pending_queue * q,int qlen)45 static inline struct otx2_cpt_pending_entry *get_free_pending_entry(
46 struct otx2_cpt_pending_queue *q,
47 int qlen)
48 {
49 struct otx2_cpt_pending_entry *ent = NULL;
50
51 ent = &q->head[q->rear];
52 if (unlikely(ent->busy))
53 return NULL;
54
55 q->rear++;
56 if (unlikely(q->rear == qlen))
57 q->rear = 0;
58
59 return ent;
60 }
61
modulo_inc(u32 index,u32 length,u32 inc)62 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
63 {
64 if (WARN_ON(inc > length))
65 inc = length;
66
67 index += inc;
68 if (unlikely(index >= length))
69 index -= length;
70
71 return index;
72 }
73
free_pentry(struct otx2_cpt_pending_entry * pentry)74 static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)
75 {
76 pentry->completion_addr = NULL;
77 pentry->info = NULL;
78 pentry->callback = NULL;
79 pentry->areq = NULL;
80 pentry->resume_sender = false;
81 pentry->busy = false;
82 }
83
process_request(struct pci_dev * pdev,struct otx2_cpt_req_info * req,struct otx2_cpt_pending_queue * pqueue,struct otx2_cptlf_info * lf)84 static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
85 struct otx2_cpt_pending_queue *pqueue,
86 struct otx2_cptlf_info *lf)
87 {
88 struct otx2_cptvf_request *cpt_req = &req->req;
89 struct otx2_cpt_pending_entry *pentry = NULL;
90 union otx2_cpt_ctrl_info *ctrl = &req->ctrl;
91 struct otx2_cpt_inst_info *info = NULL;
92 union otx2_cpt_res_s *result = NULL;
93 struct otx2_cpt_iq_command iq_cmd;
94 union otx2_cpt_inst_s cptinst;
95 int retry, ret = 0;
96 u8 resume_sender;
97 gfp_t gfp;
98
99 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
100 GFP_ATOMIC;
101 if (unlikely(!otx2_cptlf_started(lf->lfs)))
102 return -ENODEV;
103
104 info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp);
105 if (unlikely(!info)) {
106 dev_err(&pdev->dev, "Setting up cpt inst info failed");
107 return -ENOMEM;
108 }
109 cpt_req->dlen = info->dlen;
110
111 result = info->completion_addr;
112 result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
113
114 spin_lock_bh(&pqueue->lock);
115 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
116 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
117 while (unlikely(!pentry) && retry--) {
118 spin_unlock_bh(&pqueue->lock);
119 udelay(CPT_PENTRY_STEP);
120 spin_lock_bh(&pqueue->lock);
121 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
122 }
123
124 if (unlikely(!pentry)) {
125 ret = -ENOSPC;
126 goto destroy_info;
127 }
128
129 /*
130 * Check if we are close to filling in entire pending queue,
131 * if so then tell the sender to stop/sleep by returning -EBUSY
132 * We do it only for context which can sleep (GFP_KERNEL)
133 */
134 if (gfp == GFP_KERNEL &&
135 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
136 pentry->resume_sender = true;
137 } else
138 pentry->resume_sender = false;
139 resume_sender = pentry->resume_sender;
140 pqueue->pending_count++;
141
142 pentry->completion_addr = info->completion_addr;
143 pentry->info = info;
144 pentry->callback = req->callback;
145 pentry->areq = req->areq;
146 pentry->busy = true;
147 info->pentry = pentry;
148 info->time_in = jiffies;
149 info->req = req;
150
151 /* Fill in the command */
152 iq_cmd.cmd.u = 0;
153 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
154 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
155 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
156 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
157
158 /* 64-bit swap for microcode data reads, not needed for addresses*/
159 cpu_to_be64s(&iq_cmd.cmd.u);
160 iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;
161 iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;
162 iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;
163 iq_cmd.cptr.s.grp = ctrl->s.grp;
164
165 /* Fill in the CPT_INST_S type command for HW interpretation */
166 otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);
167
168 /* Print debug info if enabled */
169 otx2_cpt_dump_sg_list(pdev, req);
170 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);
171 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);
172 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
173 print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
174 cpt_req->dlen, false);
175
176 /* Send CPT command */
177 lf->lfs->ops->send_cmd(&cptinst, 1, lf);
178
179 /*
180 * We allocate and prepare pending queue entry in critical section
181 * together with submitting CPT instruction to CPT instruction queue
182 * to make sure that order of CPT requests is the same in both
183 * pending and instruction queues
184 */
185 spin_unlock_bh(&pqueue->lock);
186
187 ret = resume_sender ? -EBUSY : -EINPROGRESS;
188 return ret;
189
190 destroy_info:
191 spin_unlock_bh(&pqueue->lock);
192 otx2_cpt_info_destroy(pdev, info);
193 return ret;
194 }
195
otx2_cpt_do_request(struct pci_dev * pdev,struct otx2_cpt_req_info * req,int cpu_num)196 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
197 int cpu_num)
198 {
199 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
200 struct otx2_cptlfs_info *lfs = &cptvf->lfs;
201
202 return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,
203 &lfs->lf[cpu_num]);
204 }
205
cpt_process_ccode(struct otx2_cptlfs_info * lfs,union otx2_cpt_res_s * cpt_status,struct otx2_cpt_inst_info * info,u32 * res_code)206 static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
207 union otx2_cpt_res_s *cpt_status,
208 struct otx2_cpt_inst_info *info,
209 u32 *res_code)
210 {
211 u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
212 u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
213 struct pci_dev *pdev = lfs->pdev;
214
215 switch (ccode) {
216 case OTX2_CPT_COMP_E_FAULT:
217 dev_err(&pdev->dev,
218 "Request failed with DMA fault\n");
219 otx2_cpt_dump_sg_list(pdev, info->req);
220 break;
221
222 case OTX2_CPT_COMP_E_HWERR:
223 dev_err(&pdev->dev,
224 "Request failed with hardware error\n");
225 otx2_cpt_dump_sg_list(pdev, info->req);
226 break;
227
228 case OTX2_CPT_COMP_E_INSTERR:
229 dev_err(&pdev->dev,
230 "Request failed with instruction error\n");
231 otx2_cpt_dump_sg_list(pdev, info->req);
232 break;
233
234 case OTX2_CPT_COMP_E_NOTDONE:
235 /* check for timeout */
236 if (time_after_eq(jiffies, info->time_in +
237 CPT_COMMAND_TIMEOUT * HZ))
238 dev_warn(&pdev->dev,
239 "Request timed out 0x%p", info->req);
240 else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {
241 info->time_in = jiffies;
242 info->extra_time++;
243 }
244 return 1;
245
246 case OTX2_CPT_COMP_E_GOOD:
247 case OTX2_CPT_COMP_E_WARN:
248 /*
249 * Check microcode completion code, it is only valid
250 * when completion code is CPT_COMP_E::GOOD
251 */
252 if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {
253 /*
254 * If requested hmac is truncated and ucode returns
255 * s/g write length error then we report success
256 * because ucode writes as many bytes of calculated
257 * hmac as available in gather buffer and reports
258 * s/g write length error if number of bytes in gather
259 * buffer is less than full hmac size.
260 */
261 if (info->req->is_trunc_hmac &&
262 uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {
263 *res_code = 0;
264 break;
265 }
266
267 dev_err(&pdev->dev,
268 "Request failed with software error code 0x%x\n",
269 cpt_status->s.uc_compcode);
270 otx2_cpt_dump_sg_list(pdev, info->req);
271 break;
272 }
273 /* Request has been processed with success */
274 *res_code = 0;
275 break;
276
277 default:
278 dev_err(&pdev->dev,
279 "Request returned invalid status %d\n", ccode);
280 break;
281 }
282 return 0;
283 }
284
process_pending_queue(struct otx2_cptlfs_info * lfs,struct otx2_cpt_pending_queue * pqueue)285 static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
286 struct otx2_cpt_pending_queue *pqueue)
287 {
288 struct otx2_cpt_pending_entry *resume_pentry = NULL;
289 void (*callback)(int status, void *arg, void *req);
290 struct otx2_cpt_pending_entry *pentry = NULL;
291 union otx2_cpt_res_s *cpt_status = NULL;
292 struct otx2_cpt_inst_info *info = NULL;
293 struct otx2_cpt_req_info *req = NULL;
294 struct crypto_async_request *areq;
295 struct pci_dev *pdev = lfs->pdev;
296 u32 res_code, resume_index;
297
298 while (1) {
299 spin_lock_bh(&pqueue->lock);
300 pentry = &pqueue->head[pqueue->front];
301
302 if (WARN_ON(!pentry)) {
303 spin_unlock_bh(&pqueue->lock);
304 break;
305 }
306
307 res_code = -EINVAL;
308 if (unlikely(!pentry->busy)) {
309 spin_unlock_bh(&pqueue->lock);
310 break;
311 }
312
313 if (unlikely(!pentry->callback)) {
314 dev_err(&pdev->dev, "Callback NULL\n");
315 goto process_pentry;
316 }
317
318 info = pentry->info;
319 if (unlikely(!info)) {
320 dev_err(&pdev->dev, "Pending entry post arg NULL\n");
321 goto process_pentry;
322 }
323
324 req = info->req;
325 if (unlikely(!req)) {
326 dev_err(&pdev->dev, "Request NULL\n");
327 goto process_pentry;
328 }
329
330 cpt_status = pentry->completion_addr;
331 if (unlikely(!cpt_status)) {
332 dev_err(&pdev->dev, "Completion address NULL\n");
333 goto process_pentry;
334 }
335
336 if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
337 spin_unlock_bh(&pqueue->lock);
338 return;
339 }
340 info->pdev = pdev;
341
342 process_pentry:
343 /*
344 * Check if we should inform sending side to resume
345 * We do it CPT_IQ_RESUME_MARGIN elements in advance before
346 * pending queue becomes empty
347 */
348 resume_index = modulo_inc(pqueue->front, pqueue->qlen,
349 CPT_IQ_RESUME_MARGIN);
350 resume_pentry = &pqueue->head[resume_index];
351 if (resume_pentry &&
352 resume_pentry->resume_sender) {
353 resume_pentry->resume_sender = false;
354 callback = resume_pentry->callback;
355 areq = resume_pentry->areq;
356
357 if (callback) {
358 spin_unlock_bh(&pqueue->lock);
359
360 /*
361 * EINPROGRESS is an indication for sending
362 * side that it can resume sending requests
363 */
364 callback(-EINPROGRESS, areq, info);
365 spin_lock_bh(&pqueue->lock);
366 }
367 }
368
369 callback = pentry->callback;
370 areq = pentry->areq;
371 free_pentry(pentry);
372
373 pqueue->pending_count--;
374 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
375 spin_unlock_bh(&pqueue->lock);
376
377 /*
378 * Call callback after current pending entry has been
379 * processed, we don't do it if the callback pointer is
380 * invalid.
381 */
382 if (callback)
383 callback(res_code, areq, info);
384 }
385 }
386
otx2_cpt_post_process(struct otx2_cptlf_wqe * wqe)387 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
388 {
389 process_pending_queue(wqe->lfs,
390 &wqe->lfs->lf[wqe->lf_num].pqueue);
391 }
392
otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev * pdev)393 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
394 {
395 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
396
397 return cptvf->lfs.kcrypto_eng_grp_num;
398 }
399