1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3
4 #define pr_fmt(fmt) "QAT: " fmt
5
6 #include <linux/align.h>
7 #include <linux/bitops.h>
8 #include <linux/slab.h>
9 #include <linux/ctype.h>
10 #include <linux/kernel.h>
11 #include <linux/delay.h>
12 #include <linux/pci_ids.h>
13 #include <linux/wordpart.h>
14 #include "adf_accel_devices.h"
15 #include "adf_common_drv.h"
16 #include "icp_qat_uclo.h"
17 #include "icp_qat_hal.h"
18 #include "icp_qat_fw_loader_handle.h"
19
20 #define UWORD_CPYBUF_SIZE 1024U
21 #define INVLD_UWORD 0xffffffffffull
22 #define PID_MINOR_REV 0xf
23 #define PID_MAJOR_REV (0xf << 4)
24
qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle * obj_handle,unsigned int ae,unsigned int image_num)25 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
26 unsigned int ae, unsigned int image_num)
27 {
28 struct icp_qat_uclo_aedata *ae_data;
29 struct icp_qat_uclo_encapme *encap_image;
30 struct icp_qat_uclo_page *page = NULL;
31 struct icp_qat_uclo_aeslice *ae_slice = NULL;
32
33 ae_data = &obj_handle->ae_data[ae];
34 encap_image = &obj_handle->ae_uimage[image_num];
35 ae_slice = &ae_data->ae_slices[ae_data->slice_num];
36 ae_slice->encap_image = encap_image;
37
38 if (encap_image->img_ptr) {
39 ae_slice->ctx_mask_assigned =
40 encap_image->img_ptr->ctx_assigned;
41 ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
42 } else {
43 ae_slice->ctx_mask_assigned = 0;
44 }
45 ae_slice->region = kzalloc_obj(*ae_slice->region);
46 if (!ae_slice->region)
47 return -ENOMEM;
48 ae_slice->page = kzalloc_obj(*ae_slice->page);
49 if (!ae_slice->page)
50 goto out_err;
51 page = ae_slice->page;
52 page->encap_page = encap_image->page;
53 ae_slice->page->region = ae_slice->region;
54 ae_data->slice_num++;
55 return 0;
56 out_err:
57 kfree(ae_slice->region);
58 ae_slice->region = NULL;
59 return -ENOMEM;
60 }
61
qat_uclo_free_ae_data(struct icp_qat_uclo_aedata * ae_data)62 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
63 {
64 unsigned int i;
65
66 if (!ae_data) {
67 pr_err("bad argument, ae_data is NULL\n");
68 return -EINVAL;
69 }
70
71 for (i = 0; i < ae_data->slice_num; i++) {
72 kfree(ae_data->ae_slices[i].region);
73 ae_data->ae_slices[i].region = NULL;
74 kfree(ae_data->ae_slices[i].page);
75 ae_data->ae_slices[i].page = NULL;
76 }
77 return 0;
78 }
79
qat_uclo_get_string(struct icp_qat_uof_strtable * str_table,unsigned int str_offset)80 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
81 unsigned int str_offset)
82 {
83 if (!str_table->table_len || str_offset > str_table->table_len)
84 return NULL;
85 return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
86 }
87
qat_uclo_check_uof_format(struct icp_qat_uof_filehdr * hdr)88 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
89 {
90 int maj = hdr->maj_ver & 0xff;
91 int min = hdr->min_ver & 0xff;
92
93 if (hdr->file_id != ICP_QAT_UOF_FID) {
94 pr_err("Invalid header 0x%x\n", hdr->file_id);
95 return -EINVAL;
96 }
97 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
98 pr_err("bad UOF version, major 0x%x, minor 0x%x\n", maj, min);
99 return -EINVAL;
100 }
101 return 0;
102 }
103
qat_uclo_check_suof_format(struct icp_qat_suof_filehdr * suof_hdr)104 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr)
105 {
106 int maj = suof_hdr->maj_ver & 0xff;
107 int min = suof_hdr->min_ver & 0xff;
108
109 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
110 pr_err("invalid header 0x%x\n", suof_hdr->file_id);
111 return -EINVAL;
112 }
113 if (suof_hdr->fw_type != 0) {
114 pr_err("unsupported firmware type\n");
115 return -EINVAL;
116 }
117 if (suof_hdr->num_chunks <= 0x1) {
118 pr_err("SUOF chunk amount is incorrect\n");
119 return -EINVAL;
120 }
121 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
122 pr_err("bad SUOF version, major 0x%x, minor 0x%x\n", maj, min);
123 return -EINVAL;
124 }
125 return 0;
126 }
127
qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle * handle,unsigned int addr,unsigned int * val,unsigned int num_in_bytes)128 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
129 unsigned int addr, unsigned int *val,
130 unsigned int num_in_bytes)
131 {
132 unsigned int outval;
133 unsigned char *ptr = (unsigned char *)val;
134
135 while (num_in_bytes) {
136 memcpy(&outval, ptr, 4);
137 SRAM_WRITE(handle, addr, outval);
138 num_in_bytes -= 4;
139 ptr += 4;
140 addr += 4;
141 }
142 }
143
qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int addr,unsigned int * val,unsigned int num_in_bytes)144 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
145 unsigned char ae, unsigned int addr,
146 unsigned int *val,
147 unsigned int num_in_bytes)
148 {
149 unsigned int outval;
150 unsigned char *ptr = (unsigned char *)val;
151
152 addr >>= 0x2; /* convert to uword address */
153
154 while (num_in_bytes) {
155 memcpy(&outval, ptr, 4);
156 qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
157 num_in_bytes -= 4;
158 ptr += 4;
159 }
160 }
161
qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle * handle,unsigned char ae,struct icp_qat_uof_batch_init * umem_init_header)162 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
163 unsigned char ae,
164 struct icp_qat_uof_batch_init
165 *umem_init_header)
166 {
167 struct icp_qat_uof_batch_init *umem_init;
168
169 if (!umem_init_header)
170 return;
171 umem_init = umem_init_header->next;
172 while (umem_init) {
173 unsigned int addr, *value, size;
174
175 ae = umem_init->ae;
176 addr = umem_init->addr;
177 value = umem_init->value;
178 size = umem_init->size;
179 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
180 umem_init = umem_init->next;
181 }
182 }
183
184 static void
qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_batch_init ** base)185 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
186 struct icp_qat_uof_batch_init **base)
187 {
188 struct icp_qat_uof_batch_init *umem_init;
189
190 umem_init = *base;
191 while (umem_init) {
192 struct icp_qat_uof_batch_init *pre;
193
194 pre = umem_init;
195 umem_init = umem_init->next;
196 kfree(pre);
197 }
198 *base = NULL;
199 }
200
qat_uclo_parse_num(char * str,unsigned int * num)201 static int qat_uclo_parse_num(char *str, unsigned int *num)
202 {
203 unsigned long long ae;
204 char *end;
205
206 ae = simple_strtoull(str, &end, 10);
207 if (ae > UINT_MAX || str == end || (end - str) > 19)
208 return -EINVAL;
209 *num = (unsigned int)ae;
210 return 0;
211 }
212
qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_initmem * init_mem,unsigned int size_range,unsigned int * ae)213 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
214 struct icp_qat_uof_initmem *init_mem,
215 unsigned int size_range, unsigned int *ae)
216 {
217 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
218 char *str;
219
220 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
221 pr_err("initmem is out of range");
222 return -EINVAL;
223 }
224 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
225 pr_err("Memory scope for init_mem error\n");
226 return -EINVAL;
227 }
228 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
229 if (!str) {
230 pr_err("AE name assigned in UOF init table is NULL\n");
231 return -EINVAL;
232 }
233 if (qat_uclo_parse_num(str, ae)) {
234 pr_err("Parse num for AE number failed\n");
235 return -EINVAL;
236 }
237 if (*ae >= ICP_QAT_UCLO_MAX_AE) {
238 pr_err("ae %d out of range\n", *ae);
239 return -EINVAL;
240 }
241 return 0;
242 }
243
qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_initmem * init_mem,unsigned int ae,struct icp_qat_uof_batch_init ** init_tab_base)244 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle
245 *handle, struct icp_qat_uof_initmem
246 *init_mem, unsigned int ae,
247 struct icp_qat_uof_batch_init
248 **init_tab_base)
249 {
250 struct icp_qat_uof_batch_init *init_header, *tail;
251 struct icp_qat_uof_batch_init *mem_init, *tail_old;
252 struct icp_qat_uof_memvar_attr *mem_val_attr;
253 unsigned int i, flag = 0;
254
255 mem_val_attr =
256 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
257 sizeof(struct icp_qat_uof_initmem));
258
259 init_header = *init_tab_base;
260 if (!init_header) {
261 init_header = kzalloc_obj(*init_header);
262 if (!init_header)
263 return -ENOMEM;
264 init_header->size = 1;
265 *init_tab_base = init_header;
266 flag = 1;
267 }
268 tail_old = init_header;
269 while (tail_old->next)
270 tail_old = tail_old->next;
271 tail = tail_old;
272 for (i = 0; i < init_mem->val_attr_num; i++) {
273 mem_init = kzalloc_obj(*mem_init);
274 if (!mem_init)
275 goto out_err;
276 mem_init->ae = ae;
277 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
278 mem_init->value = &mem_val_attr->value;
279 mem_init->size = 4;
280 mem_init->next = NULL;
281 tail->next = mem_init;
282 tail = mem_init;
283 init_header->size += qat_hal_get_ins_num();
284 mem_val_attr++;
285 }
286 return 0;
287 out_err:
288 /* Do not free the list head unless we allocated it. */
289 tail_old = tail_old->next;
290 if (flag) {
291 kfree(*init_tab_base);
292 *init_tab_base = NULL;
293 }
294
295 while (tail_old) {
296 mem_init = tail_old->next;
297 kfree(tail_old);
298 tail_old = mem_init;
299 }
300 return -ENOMEM;
301 }
302
qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_initmem * init_mem)303 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
304 struct icp_qat_uof_initmem *init_mem)
305 {
306 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
307 unsigned int ae;
308
309 if (qat_uclo_fetch_initmem_ae(handle, init_mem,
310 handle->chip_info->lm_size, &ae))
311 return -EINVAL;
312 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
313 &obj_handle->lm_init_tab[ae]))
314 return -EINVAL;
315 return 0;
316 }
317
qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_initmem * init_mem)318 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
319 struct icp_qat_uof_initmem *init_mem)
320 {
321 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
322 unsigned int ae, ustore_size, uaddr, i;
323 struct icp_qat_uclo_aedata *aed;
324
325 ustore_size = obj_handle->ustore_phy_size;
326 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
327 return -EINVAL;
328 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
329 &obj_handle->umem_init_tab[ae]))
330 return -EINVAL;
331 /* set the highest ustore address referenced */
332 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
333 aed = &obj_handle->ae_data[ae];
334 for (i = 0; i < aed->slice_num; i++) {
335 if (aed->ae_slices[i].encap_image->uwords_num < uaddr)
336 aed->ae_slices[i].encap_image->uwords_num = uaddr;
337 }
338 return 0;
339 }
340
qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_initmem * init_mem)341 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
342 struct icp_qat_uof_initmem *init_mem)
343 {
344 switch (init_mem->region) {
345 case ICP_QAT_UOF_LMEM_REGION:
346 if (qat_uclo_init_lmem_seg(handle, init_mem))
347 return -EINVAL;
348 break;
349 case ICP_QAT_UOF_UMEM_REGION:
350 if (qat_uclo_init_umem_seg(handle, init_mem))
351 return -EINVAL;
352 break;
353 default:
354 pr_err("initmem region error. region type=0x%x\n", init_mem->region);
355 return -EINVAL;
356 }
357 return 0;
358 }
359
qat_uclo_init_ustore(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uclo_encapme * image)360 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
361 struct icp_qat_uclo_encapme *image)
362 {
363 unsigned int i;
364 struct icp_qat_uclo_encap_page *page;
365 struct icp_qat_uof_image *uof_image;
366 unsigned char ae;
367 unsigned int ustore_size;
368 unsigned int patt_pos;
369 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
370 unsigned long ae_mask = handle->hal_handle->ae_mask;
371 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
372 u64 *fill_data;
373
374 uof_image = image->img_ptr;
375 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64),
376 GFP_KERNEL);
377 if (!fill_data)
378 return -ENOMEM;
379 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++)
380 memcpy(&fill_data[i], &uof_image->fill_pattern,
381 sizeof(u64));
382 page = image->page;
383
384 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
385 unsigned long ae_assigned = uof_image->ae_assigned;
386
387 if (!test_bit(ae, &ae_assigned))
388 continue;
389
390 if (!test_bit(ae, &cfg_ae_mask))
391 continue;
392
393 ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
394 patt_pos = page->beg_addr_p + page->micro_words_num;
395
396 qat_hal_wr_uwords(handle, (unsigned char)ae, 0,
397 page->beg_addr_p, &fill_data[0]);
398 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos,
399 ustore_size - patt_pos + 1,
400 &fill_data[page->beg_addr_p]);
401 }
402 kfree(fill_data);
403 return 0;
404 }
405
qat_uclo_init_memory(struct icp_qat_fw_loader_handle * handle)406 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
407 {
408 int i, ae;
409 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
410 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
411 unsigned long ae_mask = handle->hal_handle->ae_mask;
412
413 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
414 if (initmem->num_in_bytes) {
415 if (qat_uclo_init_ae_memory(handle, initmem))
416 return -EINVAL;
417 }
418 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)(
419 (uintptr_t)initmem +
420 sizeof(struct icp_qat_uof_initmem)) +
421 (sizeof(struct icp_qat_uof_memvar_attr) *
422 initmem->val_attr_num));
423 }
424
425 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
426 if (qat_hal_batch_wr_lm(handle, ae,
427 obj_handle->lm_init_tab[ae])) {
428 pr_err("fail to batch init lmem for AE %d\n", ae);
429 return -EINVAL;
430 }
431 qat_uclo_cleanup_batch_init_list(handle,
432 &obj_handle->lm_init_tab[ae]);
433 qat_uclo_batch_wr_umem(handle, ae,
434 obj_handle->umem_init_tab[ae]);
435 qat_uclo_cleanup_batch_init_list(handle,
436 &obj_handle->
437 umem_init_tab[ae]);
438 }
439 return 0;
440 }
441
qat_uclo_find_chunk(struct icp_qat_uof_objhdr * obj_hdr,char * chunk_id,void * cur)442 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
443 char *chunk_id, void *cur)
444 {
445 int i;
446 struct icp_qat_uof_chunkhdr *chunk_hdr =
447 (struct icp_qat_uof_chunkhdr *)
448 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr));
449
450 for (i = 0; i < obj_hdr->num_chunks; i++) {
451 if ((cur < (void *)&chunk_hdr[i]) &&
452 !strncmp(chunk_hdr[i].chunk_id, chunk_id,
453 ICP_QAT_UOF_OBJID_LEN)) {
454 return &chunk_hdr[i];
455 }
456 }
457 return NULL;
458 }
459
qat_uclo_calc_checksum(unsigned int reg,int ch)460 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch)
461 {
462 int i;
463 unsigned int topbit = 1 << 0xF;
464 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
465
466 reg ^= inbyte << 0x8;
467 for (i = 0; i < 0x8; i++) {
468 if (reg & topbit)
469 reg = (reg << 1) ^ 0x1021;
470 else
471 reg <<= 1;
472 }
473 return reg & 0xFFFF;
474 }
475
qat_uclo_calc_str_checksum(char * ptr,int num)476 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num)
477 {
478 unsigned int chksum = 0;
479
480 if (ptr)
481 while (num--)
482 chksum = qat_uclo_calc_checksum(chksum, *ptr++);
483 return chksum;
484 }
485
486 static struct icp_qat_uclo_objhdr *
qat_uclo_map_chunk(char * buf,struct icp_qat_uof_filehdr * file_hdr,char * chunk_id)487 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr,
488 char *chunk_id)
489 {
490 struct icp_qat_uof_filechunkhdr *file_chunk;
491 struct icp_qat_uclo_objhdr *obj_hdr;
492 char *chunk;
493 int i;
494
495 file_chunk = (struct icp_qat_uof_filechunkhdr *)
496 (buf + sizeof(struct icp_qat_uof_filehdr));
497 for (i = 0; i < file_hdr->num_chunks; i++) {
498 if (!strncmp(file_chunk->chunk_id, chunk_id,
499 ICP_QAT_UOF_OBJID_LEN)) {
500 chunk = buf + file_chunk->offset;
501 if (file_chunk->checksum != qat_uclo_calc_str_checksum(
502 chunk, file_chunk->size))
503 break;
504 obj_hdr = kzalloc_obj(*obj_hdr);
505 if (!obj_hdr)
506 break;
507 obj_hdr->file_buff = chunk;
508 obj_hdr->checksum = file_chunk->checksum;
509 obj_hdr->size = file_chunk->size;
510 return obj_hdr;
511 }
512 file_chunk++;
513 }
514 return NULL;
515 }
516
517 static int
qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj * encap_uof_obj,struct icp_qat_uof_image * image)518 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
519 struct icp_qat_uof_image *image)
520 {
521 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
522 struct icp_qat_uof_objtable *neigh_reg_tab;
523 struct icp_qat_uof_code_page *code_page;
524
525 code_page = (struct icp_qat_uof_code_page *)
526 ((char *)image + sizeof(struct icp_qat_uof_image));
527 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
528 code_page->uc_var_tab_offset);
529 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
530 code_page->imp_var_tab_offset);
531 imp_expr_tab = (struct icp_qat_uof_objtable *)
532 (encap_uof_obj->beg_uof +
533 code_page->imp_expr_tab_offset);
534 if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
535 imp_expr_tab->entry_num) {
536 pr_err("UOF can't contain imported variable to be parsed\n");
537 return -EINVAL;
538 }
539 neigh_reg_tab = (struct icp_qat_uof_objtable *)
540 (encap_uof_obj->beg_uof +
541 code_page->neigh_reg_tab_offset);
542 if (neigh_reg_tab->entry_num) {
543 pr_err("UOF can't contain neighbor register table\n");
544 return -EINVAL;
545 }
546 if (image->numpages > 1) {
547 pr_err("UOF can't contain multiple pages\n");
548 return -EINVAL;
549 }
550 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) {
551 pr_err("UOF can't use shared control store feature\n");
552 return -EFAULT;
553 }
554 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
555 pr_err("UOF can't use reloadable feature\n");
556 return -EFAULT;
557 }
558 return 0;
559 }
560
qat_uclo_map_image_page(struct icp_qat_uof_encap_obj * encap_uof_obj,struct icp_qat_uof_image * img,struct icp_qat_uclo_encap_page * page)561 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj
562 *encap_uof_obj,
563 struct icp_qat_uof_image *img,
564 struct icp_qat_uclo_encap_page *page)
565 {
566 struct icp_qat_uof_code_page *code_page;
567 struct icp_qat_uof_code_area *code_area;
568 struct icp_qat_uof_objtable *uword_block_tab;
569 struct icp_qat_uof_uword_block *uwblock;
570 int i;
571
572 code_page = (struct icp_qat_uof_code_page *)
573 ((char *)img + sizeof(struct icp_qat_uof_image));
574 page->def_page = code_page->def_page;
575 page->page_region = code_page->page_region;
576 page->beg_addr_v = code_page->beg_addr_v;
577 page->beg_addr_p = code_page->beg_addr_p;
578 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
579 code_page->code_area_offset);
580 page->micro_words_num = code_area->micro_words_num;
581 uword_block_tab = (struct icp_qat_uof_objtable *)
582 (encap_uof_obj->beg_uof +
583 code_area->uword_block_tab);
584 page->uwblock_num = uword_block_tab->entry_num;
585 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab +
586 sizeof(struct icp_qat_uof_objtable));
587 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
588 for (i = 0; i < uword_block_tab->entry_num; i++)
589 page->uwblock[i].micro_words =
590 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
591 }
592
qat_uclo_map_uimage(struct icp_qat_uclo_objhandle * obj_handle,struct icp_qat_uclo_encapme * ae_uimage,int max_image)593 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
594 struct icp_qat_uclo_encapme *ae_uimage,
595 int max_image)
596 {
597 int i, j;
598 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
599 struct icp_qat_uof_image *image;
600 struct icp_qat_uof_objtable *ae_regtab;
601 struct icp_qat_uof_objtable *init_reg_sym_tab;
602 struct icp_qat_uof_objtable *sbreak_tab;
603 struct icp_qat_uof_encap_obj *encap_uof_obj =
604 &obj_handle->encap_uof_obj;
605
606 for (j = 0; j < max_image; j++) {
607 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
608 ICP_QAT_UOF_IMAG, chunk_hdr);
609 if (!chunk_hdr)
610 break;
611 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
612 chunk_hdr->offset);
613 ae_regtab = (struct icp_qat_uof_objtable *)
614 (image->reg_tab_offset +
615 obj_handle->obj_hdr->file_buff);
616 ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
617 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)
618 (((char *)ae_regtab) +
619 sizeof(struct icp_qat_uof_objtable));
620 init_reg_sym_tab = (struct icp_qat_uof_objtable *)
621 (image->init_reg_sym_tab +
622 obj_handle->obj_hdr->file_buff);
623 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
624 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)
625 (((char *)init_reg_sym_tab) +
626 sizeof(struct icp_qat_uof_objtable));
627 sbreak_tab = (struct icp_qat_uof_objtable *)
628 (image->sbreak_tab + obj_handle->obj_hdr->file_buff);
629 ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
630 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)
631 (((char *)sbreak_tab) +
632 sizeof(struct icp_qat_uof_objtable));
633 ae_uimage[j].img_ptr = image;
634 if (qat_uclo_check_image_compat(encap_uof_obj, image))
635 goto out_err;
636 ae_uimage[j].page =
637 kzalloc_obj(struct icp_qat_uclo_encap_page);
638 if (!ae_uimage[j].page)
639 goto out_err;
640 qat_uclo_map_image_page(encap_uof_obj, image,
641 ae_uimage[j].page);
642 }
643 return j;
644 out_err:
645 for (i = 0; i < j; i++)
646 kfree(ae_uimage[i].page);
647 return 0;
648 }
649
qat_uclo_map_ae(struct icp_qat_fw_loader_handle * handle,int max_ae)650 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
651 {
652 int i, ae;
653 int mflag = 0;
654 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
655 unsigned long ae_mask = handle->hal_handle->ae_mask;
656 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
657
658 for_each_set_bit(ae, &ae_mask, max_ae) {
659 if (!test_bit(ae, &cfg_ae_mask))
660 continue;
661
662 for (i = 0; i < obj_handle->uimage_num; i++) {
663 unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned;
664
665 if (!test_bit(ae, &ae_assigned))
666 continue;
667 mflag = 1;
668 if (qat_uclo_init_ae_data(obj_handle, ae, i))
669 return -EINVAL;
670 }
671 }
672 if (!mflag) {
673 pr_err("uimage uses AE not set\n");
674 return -EINVAL;
675 }
676 return 0;
677 }
678
679 static struct icp_qat_uof_strtable *
qat_uclo_map_str_table(struct icp_qat_uclo_objhdr * obj_hdr,char * tab_name,struct icp_qat_uof_strtable * str_table)680 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
681 char *tab_name, struct icp_qat_uof_strtable *str_table)
682 {
683 struct icp_qat_uof_chunkhdr *chunk_hdr;
684
685 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)
686 obj_hdr->file_buff, tab_name, NULL);
687 if (chunk_hdr) {
688 int hdr_size;
689
690 memcpy(&str_table->table_len, obj_hdr->file_buff +
691 chunk_hdr->offset, sizeof(str_table->table_len));
692 hdr_size = (char *)&str_table->strings - (char *)str_table;
693 str_table->strings = (uintptr_t)obj_hdr->file_buff +
694 chunk_hdr->offset + hdr_size;
695 return str_table;
696 }
697 return NULL;
698 }
699
700 static void
qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj * encap_uof_obj,struct icp_qat_uclo_init_mem_table * init_mem_tab)701 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
702 struct icp_qat_uclo_init_mem_table *init_mem_tab)
703 {
704 struct icp_qat_uof_chunkhdr *chunk_hdr;
705
706 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
707 ICP_QAT_UOF_IMEM, NULL);
708 if (chunk_hdr) {
709 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof +
710 chunk_hdr->offset, sizeof(unsigned int));
711 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)
712 (encap_uof_obj->beg_uof + chunk_hdr->offset +
713 sizeof(unsigned int));
714 }
715 }
716
717 static unsigned int
qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle * handle)718 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
719 {
720 switch (handle->pci_dev->device) {
721 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
722 return ICP_QAT_AC_895XCC_DEV_TYPE;
723 case PCI_DEVICE_ID_INTEL_QAT_C62X:
724 return ICP_QAT_AC_C62X_DEV_TYPE;
725 case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
726 return ICP_QAT_AC_C3XXX_DEV_TYPE;
727 case PCI_DEVICE_ID_INTEL_QAT_4XXX:
728 case PCI_DEVICE_ID_INTEL_QAT_401XX:
729 case PCI_DEVICE_ID_INTEL_QAT_402XX:
730 case PCI_DEVICE_ID_INTEL_QAT_420XX:
731 return ICP_QAT_AC_4XXX_A_DEV_TYPE;
732 case PCI_DEVICE_ID_INTEL_QAT_6XXX:
733 return ICP_QAT_AC_6XXX_DEV_TYPE;
734 default:
735 pr_err("unsupported device 0x%x\n", handle->pci_dev->device);
736 return 0;
737 }
738 }
739
qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle * obj_handle)740 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
741 {
742 unsigned int maj_ver, prod_type = obj_handle->prod_type;
743
744 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
745 pr_err("UOF type 0x%x doesn't match with platform 0x%x\n",
746 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
747 prod_type);
748 return -EINVAL;
749 }
750 maj_ver = obj_handle->prod_rev & 0xff;
751 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver ||
752 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) {
753 pr_err("UOF majVer 0x%x out of range\n", maj_ver);
754 return -EINVAL;
755 }
756 return 0;
757 }
758
qat_uclo_init_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_addr,unsigned int value)759 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
760 unsigned char ae, unsigned char ctx_mask,
761 enum icp_qat_uof_regtype reg_type,
762 unsigned short reg_addr, unsigned int value)
763 {
764 switch (reg_type) {
765 case ICP_GPA_ABS:
766 case ICP_GPB_ABS:
767 ctx_mask = 0;
768 fallthrough;
769 case ICP_GPA_REL:
770 case ICP_GPB_REL:
771 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type,
772 reg_addr, value);
773 case ICP_SR_ABS:
774 case ICP_DR_ABS:
775 case ICP_SR_RD_ABS:
776 case ICP_DR_RD_ABS:
777 ctx_mask = 0;
778 fallthrough;
779 case ICP_SR_REL:
780 case ICP_DR_REL:
781 case ICP_SR_RD_REL:
782 case ICP_DR_RD_REL:
783 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type,
784 reg_addr, value);
785 case ICP_SR_WR_ABS:
786 case ICP_DR_WR_ABS:
787 ctx_mask = 0;
788 fallthrough;
789 case ICP_SR_WR_REL:
790 case ICP_DR_WR_REL:
791 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type,
792 reg_addr, value);
793 case ICP_NEIGH_REL:
794 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
795 default:
796 pr_err("UOF uses not supported reg type 0x%x\n", reg_type);
797 return -EFAULT;
798 }
799 return 0;
800 }
801
qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle * handle,unsigned int ae,struct icp_qat_uclo_encapme * encap_ae)802 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
803 unsigned int ae,
804 struct icp_qat_uclo_encapme *encap_ae)
805 {
806 unsigned int i;
807 unsigned char ctx_mask;
808 struct icp_qat_uof_init_regsym *init_regsym;
809
810 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
811 ICP_QAT_UCLO_MAX_CTX)
812 ctx_mask = 0xff;
813 else
814 ctx_mask = 0x55;
815
816 for (i = 0; i < encap_ae->init_regsym_num; i++) {
817 unsigned int exp_res;
818
819 init_regsym = &encap_ae->init_regsym[i];
820 exp_res = init_regsym->value;
821 switch (init_regsym->init_type) {
822 case ICP_QAT_UOF_INIT_REG:
823 qat_uclo_init_reg(handle, ae, ctx_mask,
824 (enum icp_qat_uof_regtype)
825 init_regsym->reg_type,
826 (unsigned short)init_regsym->reg_addr,
827 exp_res);
828 break;
829 case ICP_QAT_UOF_INIT_REG_CTX:
830 /* check if ctx is appropriate for the ctxMode */
831 if (!((1 << init_regsym->ctx) & ctx_mask)) {
832 pr_err("invalid ctx num = 0x%x\n", init_regsym->ctx);
833 return -EINVAL;
834 }
835 qat_uclo_init_reg(handle, ae,
836 (unsigned char)
837 (1 << init_regsym->ctx),
838 (enum icp_qat_uof_regtype)
839 init_regsym->reg_type,
840 (unsigned short)init_regsym->reg_addr,
841 exp_res);
842 break;
843 case ICP_QAT_UOF_INIT_EXPR:
844 pr_err("INIT_EXPR feature not supported\n");
845 return -EINVAL;
846 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
847 pr_err("INIT_EXPR_ENDIAN_SWAP feature not supported\n");
848 return -EINVAL;
849 default:
850 break;
851 }
852 }
853 return 0;
854 }
855
qat_uclo_init_globals(struct icp_qat_fw_loader_handle * handle)856 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
857 {
858 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
859 unsigned long ae_mask = handle->hal_handle->ae_mask;
860 struct icp_qat_uclo_aedata *aed;
861 unsigned int s, ae;
862
863 if (obj_handle->global_inited)
864 return 0;
865 if (obj_handle->init_mem_tab.entry_num) {
866 if (qat_uclo_init_memory(handle)) {
867 pr_err("initialize memory failed\n");
868 return -EINVAL;
869 }
870 }
871
872 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
873 aed = &obj_handle->ae_data[ae];
874 for (s = 0; s < aed->slice_num; s++) {
875 if (!aed->ae_slices[s].encap_image)
876 continue;
877 if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image))
878 return -EINVAL;
879 }
880 }
881 obj_handle->global_inited = 1;
882 return 0;
883 }
884
qat_hal_set_modes(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uclo_objhandle * obj_handle,unsigned char ae,struct icp_qat_uof_image * uof_image)885 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle,
886 struct icp_qat_uclo_objhandle *obj_handle,
887 unsigned char ae,
888 struct icp_qat_uof_image *uof_image)
889 {
890 unsigned char mode;
891 int ret;
892
893 mode = ICP_QAT_CTX_MODE(uof_image->ae_mode);
894 ret = qat_hal_set_ae_ctx_mode(handle, ae, mode);
895 if (ret) {
896 pr_err("qat_hal_set_ae_ctx_mode error\n");
897 return ret;
898 }
899 if (handle->chip_info->nn) {
900 mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
901 ret = qat_hal_set_ae_nn_mode(handle, ae, mode);
902 if (ret) {
903 pr_err("qat_hal_set_ae_nn_mode error\n");
904 return ret;
905 }
906 }
907 mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode);
908 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode);
909 if (ret) {
910 pr_err("qat_hal_set_ae_lm_mode LMEM0 error\n");
911 return ret;
912 }
913 mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode);
914 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode);
915 if (ret) {
916 pr_err("qat_hal_set_ae_lm_mode LMEM1 error\n");
917 return ret;
918 }
919 if (handle->chip_info->lm2lm3) {
920 mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode);
921 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode);
922 if (ret) {
923 pr_err("qat_hal_set_ae_lm_mode LMEM2 error\n");
924 return ret;
925 }
926 mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode);
927 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode);
928 if (ret) {
929 pr_err("qat_hal_set_ae_lm_mode LMEM3 error\n");
930 return ret;
931 }
932 mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode);
933 qat_hal_set_ae_tindex_mode(handle, ae, mode);
934 }
935 return 0;
936 }
937
qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle * handle)938 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
939 {
940 struct icp_qat_uof_image *uof_image;
941 struct icp_qat_uclo_aedata *ae_data;
942 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
943 unsigned long ae_mask = handle->hal_handle->ae_mask;
944 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
945 unsigned char ae, s;
946 int error;
947
948 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
949 if (!test_bit(ae, &cfg_ae_mask))
950 continue;
951
952 ae_data = &obj_handle->ae_data[ae];
953 for (s = 0; s < min_t(unsigned int, ae_data->slice_num,
954 ICP_QAT_UCLO_MAX_CTX); s++) {
955 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
956 continue;
957 uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
958 error = qat_hal_set_modes(handle, obj_handle, ae,
959 uof_image);
960 if (error)
961 return error;
962 }
963 }
964 return 0;
965 }
966
qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle * handle)967 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
968 {
969 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
970 struct icp_qat_uclo_encapme *image;
971 int a;
972
973 for (a = 0; a < obj_handle->uimage_num; a++) {
974 image = &obj_handle->ae_uimage[a];
975 image->uwords_num = image->page->beg_addr_p +
976 image->page->micro_words_num;
977 }
978 }
979
qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle * handle)980 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
981 {
982 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
983 unsigned int ae;
984
985 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
986 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
987 obj_handle->obj_hdr->file_buff;
988 obj_handle->uword_in_bytes = 6;
989 obj_handle->prod_type = qat_uclo_get_dev_type(handle);
990 obj_handle->prod_rev = PID_MAJOR_REV |
991 (PID_MINOR_REV & handle->hal_handle->revision_id);
992 if (qat_uclo_check_uof_compat(obj_handle)) {
993 pr_err("UOF incompatible\n");
994 return -EINVAL;
995 }
996 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64),
997 GFP_KERNEL);
998 if (!obj_handle->uword_buf)
999 return -ENOMEM;
1000 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
1001 if (!obj_handle->obj_hdr->file_buff ||
1002 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
1003 &obj_handle->str_table)) {
1004 pr_err("UOF doesn't have effective images\n");
1005 goto out_err;
1006 }
1007 obj_handle->uimage_num =
1008 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage,
1009 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
1010 if (!obj_handle->uimage_num)
1011 goto out_err;
1012 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
1013 pr_err("Bad object\n");
1014 goto out_check_uof_aemask_err;
1015 }
1016 qat_uclo_init_uword_num(handle);
1017 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
1018 &obj_handle->init_mem_tab);
1019 if (qat_uclo_set_ae_mode(handle))
1020 goto out_check_uof_aemask_err;
1021 return 0;
1022 out_check_uof_aemask_err:
1023 for (ae = 0; ae < obj_handle->uimage_num; ae++)
1024 kfree(obj_handle->ae_uimage[ae].page);
1025 out_err:
1026 kfree(obj_handle->uword_buf);
1027 return -EFAULT;
1028 }
1029
qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle * handle)1030 static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle)
1031 {
1032 if (handle->chip_info->dual_sign)
1033 return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN;
1034
1035 return ICP_QAT_AE_IMG_OFFSET(handle);
1036 }
1037
qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle * handle)1038 static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle)
1039 {
1040 if (handle->chip_info->dual_sign)
1041 return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN + ICP_QAT_DUALSIGN_MISC_INFO_LEN;
1042
1043 return ICP_QAT_AE_IMG_OFFSET(handle);
1044 }
1045
qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle * handle,void * img_ptr)1046 static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr)
1047 {
1048 struct icp_qat_css_hdr *hdr = img_ptr;
1049 char *fw_hdr = img_ptr;
1050 unsigned int offset;
1051
1052 if (handle->chip_info->dual_sign) {
1053 offset = qat_uclo_simg_hdr2sign_len(handle) + ICP_QAT_DUALSIGN_FW_TYPE_LEN;
1054 return *(fw_hdr + offset);
1055 }
1056
1057 return hdr->fw_type;
1058 }
1059
qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle * handle,struct icp_qat_suof_filehdr * suof_ptr,int suof_size)1060 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1061 struct icp_qat_suof_filehdr *suof_ptr,
1062 int suof_size)
1063 {
1064 unsigned int check_sum = 0;
1065 unsigned int min_ver_offset = 0;
1066 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1067
1068 suof_handle->file_id = ICP_QAT_SUOF_FID;
1069 suof_handle->suof_buf = (char *)suof_ptr;
1070 suof_handle->suof_size = suof_size;
1071 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr,
1072 min_ver);
1073 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver,
1074 min_ver_offset);
1075 if (check_sum != suof_ptr->check_sum) {
1076 pr_err("incorrect SUOF checksum\n");
1077 return -EINVAL;
1078 }
1079 suof_handle->check_sum = suof_ptr->check_sum;
1080 suof_handle->min_ver = suof_ptr->min_ver;
1081 suof_handle->maj_ver = suof_ptr->maj_ver;
1082 suof_handle->fw_type = suof_ptr->fw_type;
1083 return 0;
1084 }
1085
qat_uclo_map_simg(struct icp_qat_fw_loader_handle * handle,struct icp_qat_suof_img_hdr * suof_img_hdr,struct icp_qat_suof_chunk_hdr * suof_chunk_hdr)1086 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle,
1087 struct icp_qat_suof_img_hdr *suof_img_hdr,
1088 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1089 {
1090 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1091 unsigned int offset = qat_uclo_simg_hdr2cont_len(handle);
1092 struct icp_qat_suof_objhdr *suof_objhdr;
1093 struct icp_qat_simg_ae_mode *ae_mode;
1094
1095 suof_img_hdr->simg_buf = (suof_handle->suof_buf +
1096 suof_chunk_hdr->offset +
1097 sizeof(*suof_objhdr));
1098 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)
1099 (suof_handle->suof_buf +
1100 suof_chunk_hdr->offset))->img_length;
1101
1102 suof_img_hdr->css_header = suof_img_hdr->simg_buf;
1103 suof_img_hdr->css_simg = suof_img_hdr->css_header + offset;
1104
1105 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
1106 suof_img_hdr->ae_mask = ae_mode->ae_mask;
1107 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
1108 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
1109 suof_img_hdr->fw_type = ae_mode->fw_type;
1110 }
1111
1112 static void
qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle * suof_handle,struct icp_qat_suof_chunk_hdr * suof_chunk_hdr)1113 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
1114 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1115 {
1116 char **sym_str = (char **)&suof_handle->sym_str;
1117 unsigned int *sym_size = &suof_handle->sym_size;
1118 struct icp_qat_suof_strtable *str_table_obj;
1119
1120 *sym_size = *(unsigned int *)(uintptr_t)
1121 (suof_chunk_hdr->offset + suof_handle->suof_buf);
1122 *sym_str = (char *)(uintptr_t)
1123 (suof_handle->suof_buf + suof_chunk_hdr->offset +
1124 sizeof(str_table_obj->tab_length));
1125 }
1126
qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle * handle,struct icp_qat_suof_img_hdr * img_hdr)1127 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
1128 struct icp_qat_suof_img_hdr *img_hdr)
1129 {
1130 struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
1131 unsigned int prod_rev, maj_ver, prod_type;
1132
1133 prod_type = qat_uclo_get_dev_type(handle);
1134 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
1135 prod_rev = PID_MAJOR_REV |
1136 (PID_MINOR_REV & handle->hal_handle->revision_id);
1137 if (img_ae_mode->dev_type != prod_type) {
1138 pr_err("incompatible product type %x\n", img_ae_mode->dev_type);
1139 return -EINVAL;
1140 }
1141 maj_ver = prod_rev & 0xff;
1142 if (maj_ver > img_ae_mode->devmax_ver ||
1143 maj_ver < img_ae_mode->devmin_ver) {
1144 pr_err("incompatible device majver 0x%x\n", maj_ver);
1145 return -EINVAL;
1146 }
1147 return 0;
1148 }
1149
qat_uclo_del_suof(struct icp_qat_fw_loader_handle * handle)1150 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
1151 {
1152 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1153
1154 kfree(sobj_handle->img_table.simg_hdr);
1155 sobj_handle->img_table.simg_hdr = NULL;
1156 kfree(handle->sobj_handle);
1157 handle->sobj_handle = NULL;
1158 }
1159
qat_uclo_tail_img(struct icp_qat_suof_img_hdr * suof_img_hdr,unsigned int img_id,unsigned int num_simgs)1160 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
1161 unsigned int img_id, unsigned int num_simgs)
1162 {
1163 struct icp_qat_suof_img_hdr img_header;
1164
1165 if (img_id != num_simgs - 1) {
1166 memcpy(&img_header, &suof_img_hdr[num_simgs - 1],
1167 sizeof(*suof_img_hdr));
1168 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id],
1169 sizeof(*suof_img_hdr));
1170 memcpy(&suof_img_hdr[img_id], &img_header,
1171 sizeof(*suof_img_hdr));
1172 }
1173 }
1174
qat_uclo_map_suof(struct icp_qat_fw_loader_handle * handle,struct icp_qat_suof_filehdr * suof_ptr,int suof_size)1175 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
1176 struct icp_qat_suof_filehdr *suof_ptr,
1177 int suof_size)
1178 {
1179 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1180 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
1181 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
1182 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
1183 unsigned int i = 0;
1184 struct icp_qat_suof_img_hdr img_header;
1185
1186 if (!suof_ptr || suof_size == 0) {
1187 pr_err("input parameter SUOF pointer/size is NULL\n");
1188 return -EINVAL;
1189 }
1190 if (qat_uclo_check_suof_format(suof_ptr))
1191 return -EINVAL;
1192 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
1193 if (ret)
1194 return ret;
1195 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)
1196 ((uintptr_t)suof_ptr + sizeof(*suof_ptr));
1197
1198 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
1199 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
1200
1201 if (suof_handle->img_table.num_simgs != 0) {
1202 suof_img_hdr = kzalloc_objs(img_header,
1203 suof_handle->img_table.num_simgs);
1204 if (!suof_img_hdr)
1205 return -ENOMEM;
1206 suof_handle->img_table.simg_hdr = suof_img_hdr;
1207
1208 for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
1209 qat_uclo_map_simg(handle, &suof_img_hdr[i],
1210 &suof_chunk_hdr[1 + i]);
1211 ret = qat_uclo_check_simg_compat(handle,
1212 &suof_img_hdr[i]);
1213 if (ret)
1214 return ret;
1215 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask;
1216 if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
1217 ae0_img = i;
1218 }
1219
1220 if (!handle->chip_info->tgroup_share_ustore) {
1221 qat_uclo_tail_img(suof_img_hdr, ae0_img,
1222 suof_handle->img_table.num_simgs);
1223 }
1224 }
1225 return 0;
1226 }
1227
1228 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low)
1229
qat_uclo_auth_fw(struct icp_qat_fw_loader_handle * handle,struct icp_qat_fw_auth_desc * desc)1230 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
1231 struct icp_qat_fw_auth_desc *desc)
1232 {
1233 u32 fcu_sts, retry = 0;
1234 u32 fcu_ctl_csr, fcu_sts_csr;
1235 u32 fcu_dram_hi_csr, fcu_dram_lo_csr;
1236 u64 bus_addr;
1237
1238 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low)
1239 - sizeof(struct icp_qat_auth_chunk);
1240
1241 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1242 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1243 fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi;
1244 fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo;
1245
1246 SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32));
1247 SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr);
1248 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH);
1249
1250 do {
1251 msleep(FW_AUTH_WAIT_PERIOD);
1252 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1253 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
1254 goto auth_fail;
1255 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
1256 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
1257 return 0;
1258 } while (retry++ < FW_AUTH_MAX_RETRY);
1259 auth_fail:
1260 pr_err("authentication error (FCU_STATUS = 0x%x),retry = %d\n",
1261 fcu_sts & FCU_AUTH_STS_MASK, retry);
1262 return -EINVAL;
1263 }
1264
qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle * handle,int imgid)1265 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle,
1266 int imgid)
1267 {
1268 struct icp_qat_suof_handle *sobj_handle;
1269
1270 if (!handle->chip_info->tgroup_share_ustore)
1271 return false;
1272
1273 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
1274 if (handle->hal_handle->admin_ae_mask &
1275 sobj_handle->img_table.simg_hdr[imgid].ae_mask)
1276 return false;
1277
1278 return true;
1279 }
1280
qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle * handle,struct icp_qat_fw_auth_desc * desc)1281 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
1282 struct icp_qat_fw_auth_desc *desc)
1283 {
1284 unsigned long ae_mask = handle->hal_handle->ae_mask;
1285 unsigned long desc_ae_mask = desc->ae_mask;
1286 u32 fcu_sts, ae_broadcast_mask = 0;
1287 u32 fcu_loaded_csr, ae_loaded;
1288 u32 fcu_sts_csr, fcu_ctl_csr;
1289 unsigned int ae, retry = 0;
1290
1291 if (handle->chip_info->tgroup_share_ustore) {
1292 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1293 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1294 fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1295 } else {
1296 pr_err("Chip 0x%x doesn't support broadcast load\n", handle->pci_dev->device);
1297 return -EINVAL;
1298 }
1299
1300 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
1301 if (qat_hal_check_ae_active(handle, (unsigned char)ae)) {
1302 pr_err("Broadcast load failed. AE is not enabled or active.\n");
1303 return -EINVAL;
1304 }
1305
1306 if (test_bit(ae, &desc_ae_mask))
1307 ae_broadcast_mask |= 1 << ae;
1308 }
1309
1310 if (ae_broadcast_mask) {
1311 SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE,
1312 ae_broadcast_mask);
1313
1314 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD);
1315
1316 do {
1317 msleep(FW_AUTH_WAIT_PERIOD);
1318 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1319 fcu_sts &= FCU_AUTH_STS_MASK;
1320
1321 if (fcu_sts == FCU_STS_LOAD_FAIL) {
1322 pr_err("Broadcast load failed: 0x%x)\n", fcu_sts);
1323 return -EINVAL;
1324 } else if (fcu_sts == FCU_STS_LOAD_DONE) {
1325 ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr);
1326 ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos;
1327
1328 if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask)
1329 break;
1330 }
1331 } while (retry++ < FW_AUTH_MAX_RETRY);
1332
1333 if (retry > FW_AUTH_MAX_RETRY) {
1334 pr_err("broadcast load failed timeout %d\n", retry);
1335 return -EINVAL;
1336 }
1337 }
1338 return 0;
1339 }
1340
qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle * handle,struct icp_firml_dram_desc * dram_desc,unsigned int size)1341 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
1342 struct icp_firml_dram_desc *dram_desc,
1343 unsigned int size)
1344 {
1345 void *vptr;
1346 dma_addr_t ptr;
1347
1348 vptr = dma_alloc_coherent(&handle->pci_dev->dev,
1349 size, &ptr, GFP_KERNEL);
1350 if (!vptr)
1351 return -ENOMEM;
1352 dram_desc->dram_base_addr_v = vptr;
1353 dram_desc->dram_bus_addr = ptr;
1354 dram_desc->dram_size = size;
1355 return 0;
1356 }
1357
qat_uclo_simg_free(struct icp_qat_fw_loader_handle * handle,struct icp_firml_dram_desc * dram_desc)1358 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
1359 struct icp_firml_dram_desc *dram_desc)
1360 {
1361 if (handle && dram_desc && dram_desc->dram_base_addr_v) {
1362 dma_free_coherent(&handle->pci_dev->dev,
1363 (size_t)(dram_desc->dram_size),
1364 dram_desc->dram_base_addr_v,
1365 dram_desc->dram_bus_addr);
1366 }
1367
1368 if (dram_desc)
1369 memset(dram_desc, 0, sizeof(*dram_desc));
1370 }
1371
qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle * handle,struct icp_qat_fw_auth_desc ** desc)1372 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle,
1373 struct icp_qat_fw_auth_desc **desc)
1374 {
1375 struct icp_firml_dram_desc dram_desc;
1376
1377 if (*desc) {
1378 dram_desc.dram_base_addr_v = *desc;
1379 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *)
1380 (*desc))->chunk_bus_addr;
1381 dram_desc.dram_size = ((struct icp_qat_auth_chunk *)
1382 (*desc))->chunk_size;
1383 qat_uclo_simg_free(handle, &dram_desc);
1384 }
1385 }
1386
qat_uclo_check_image(struct icp_qat_fw_loader_handle * handle,void * image,unsigned int size,unsigned int fw_type)1387 static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle,
1388 void *image, unsigned int size,
1389 unsigned int fw_type)
1390 {
1391 char *fw_type_name = fw_type ? "MMP" : "AE";
1392 unsigned int css_dword_size = sizeof(u32);
1393 unsigned int header_len, simg_type;
1394 struct icp_qat_css_hdr *css_hdr;
1395
1396 if (handle->chip_info->fw_auth) {
1397 header_len = qat_uclo_simg_hdr2sign_len(handle);
1398 simg_type = qat_uclo_simg_fw_type(handle, image);
1399 css_hdr = image;
1400
1401 if (handle->chip_info->dual_sign) {
1402 if (css_hdr->module_type != ICP_QAT_DUALSIGN_MODULE_TYPE)
1403 goto err;
1404 if (css_hdr->header_len != ICP_QAT_DUALSIGN_HDR_LEN)
1405 goto err;
1406 if (css_hdr->header_ver != ICP_QAT_DUALSIGN_HDR_VER)
1407 goto err;
1408 } else {
1409 if (css_hdr->header_len * css_dword_size != header_len)
1410 goto err;
1411 if (css_hdr->size * css_dword_size != size)
1412 goto err;
1413 if (size <= header_len)
1414 goto err;
1415 }
1416
1417 if (fw_type != simg_type)
1418 goto err;
1419
1420 size -= header_len;
1421 }
1422
1423 if (fw_type == CSS_AE_FIRMWARE) {
1424 if (size < sizeof(struct icp_qat_simg_ae_mode *) +
1425 ICP_QAT_SIMG_AE_INIT_SEQ_LEN)
1426 goto err;
1427 if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)
1428 goto err;
1429 } else if (fw_type == CSS_MMP_FIRMWARE) {
1430 if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN)
1431 goto err;
1432 } else {
1433 pr_err("Unsupported firmware type\n");
1434 return -EINVAL;
1435 }
1436 return 0;
1437
1438 err:
1439 pr_err("Invalid %s firmware image\n", fw_type_name);
1440 return -EINVAL;
1441 }
1442
qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle * handle,char * image,unsigned int size,struct icp_firml_dram_desc * dram_desc,unsigned int fw_type,struct icp_qat_fw_auth_desc ** desc)1443 static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle,
1444 char *image, unsigned int size,
1445 struct icp_firml_dram_desc *dram_desc,
1446 unsigned int fw_type, struct icp_qat_fw_auth_desc **desc)
1447 {
1448 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1449 struct icp_qat_simg_ae_mode *simg_ae_mode;
1450 struct icp_qat_fw_auth_desc *auth_desc;
1451 char *virt_addr, *virt_base;
1452 u64 bus_addr;
1453
1454 virt_base = dram_desc->dram_base_addr_v;
1455 virt_base += sizeof(struct icp_qat_auth_chunk);
1456 bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk);
1457 auth_desc = dram_desc->dram_base_addr_v;
1458 auth_desc->css_hdr_high = upper_32_bits(bus_addr);
1459 auth_desc->css_hdr_low = lower_32_bits(bus_addr);
1460 virt_addr = virt_base;
1461
1462 memcpy(virt_addr, image, sizeof(*css_hdr));
1463 /* pub key */
1464 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
1465 sizeof(*css_hdr);
1466 virt_addr = virt_addr + sizeof(*css_hdr);
1467
1468 auth_desc->fwsk_pub_high = upper_32_bits(bus_addr);
1469 auth_desc->fwsk_pub_low = lower_32_bits(bus_addr);
1470
1471 memcpy(virt_addr, image + sizeof(*css_hdr), ICP_QAT_CSS_FWSK_MODULUS_LEN(handle));
1472 /* padding */
1473 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)),
1474 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle));
1475
1476 /* exponent */
1477 memcpy(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1478 ICP_QAT_CSS_FWSK_PAD_LEN(handle), image + sizeof(*css_hdr) +
1479 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle), sizeof(unsigned int));
1480
1481 /* signature */
1482 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high,
1483 auth_desc->fwsk_pub_low) +
1484 ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1485 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1486 auth_desc->signature_high = upper_32_bits(bus_addr);
1487 auth_desc->signature_low = lower_32_bits(bus_addr);
1488
1489 memcpy(virt_addr, image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1490 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle), ICP_QAT_CSS_SIGNATURE_LEN(handle));
1491
1492 bus_addr = ADD_ADDR(auth_desc->signature_high,
1493 auth_desc->signature_low) +
1494 ICP_QAT_CSS_SIGNATURE_LEN(handle);
1495 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1496
1497 auth_desc->img_high = upper_32_bits(bus_addr);
1498 auth_desc->img_low = lower_32_bits(bus_addr);
1499 auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle);
1500 if (bus_addr + auth_desc->img_len >
1501 dram_desc->dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) {
1502 pr_err("insufficient memory size for authentication data\n");
1503 qat_uclo_simg_free(handle, dram_desc);
1504 return -ENOMEM;
1505 }
1506
1507 memcpy(virt_addr, image + qat_uclo_simg_hdr2sign_len(handle), auth_desc->img_len);
1508 virt_addr = virt_base;
1509 /* AE firmware */
1510 if (fw_type == CSS_AE_FIRMWARE) {
1511 auth_desc->img_ae_mode_data_high = auth_desc->img_high;
1512 auth_desc->img_ae_mode_data_low = auth_desc->img_low;
1513 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
1514 auth_desc->img_ae_mode_data_low) +
1515 sizeof(struct icp_qat_simg_ae_mode);
1516
1517 auth_desc->img_ae_init_data_high = upper_32_bits(bus_addr);
1518 auth_desc->img_ae_init_data_low = lower_32_bits(bus_addr);
1519 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1520 auth_desc->img_ae_insts_high = upper_32_bits(bus_addr);
1521 auth_desc->img_ae_insts_low = lower_32_bits(bus_addr);
1522 virt_addr += sizeof(struct icp_qat_css_hdr);
1523 virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1524 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1525 simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr;
1526 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask;
1527 } else {
1528 auth_desc->img_ae_insts_high = auth_desc->img_high;
1529 auth_desc->img_ae_insts_low = auth_desc->img_low;
1530 }
1531 *desc = auth_desc;
1532 return 0;
1533 }
1534
qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle * handle,char * image,unsigned int size,struct icp_firml_dram_desc * dram_desc,unsigned int fw_type,struct icp_qat_fw_auth_desc ** desc)1535 static int qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle *handle,
1536 char *image, unsigned int size,
1537 struct icp_firml_dram_desc *dram_desc,
1538 unsigned int fw_type,
1539 struct icp_qat_fw_auth_desc **desc)
1540 {
1541 struct icp_qat_simg_ae_mode *simg_ae_mode;
1542 struct icp_qat_fw_auth_desc *auth_desc;
1543 unsigned int chunk_offset, img_offset;
1544 u64 bus_addr, addr;
1545 char *virt_addr;
1546
1547 virt_addr = dram_desc->dram_base_addr_v;
1548 virt_addr += sizeof(struct icp_qat_auth_chunk);
1549 bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk);
1550
1551 auth_desc = dram_desc->dram_base_addr_v;
1552 auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle);
1553 auth_desc->css_hdr_high = upper_32_bits(bus_addr);
1554 auth_desc->css_hdr_low = lower_32_bits(bus_addr);
1555 memcpy(virt_addr, image, ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN);
1556
1557 img_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN;
1558 chunk_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN;
1559
1560 /* RSA pub key */
1561 addr = bus_addr + chunk_offset;
1562 auth_desc->fwsk_pub_high = upper_32_bits(addr);
1563 auth_desc->fwsk_pub_low = lower_32_bits(addr);
1564 memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_MODULUS_LEN(handle));
1565
1566 img_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle);
1567 chunk_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle);
1568 /* RSA padding */
1569 memset(virt_addr + chunk_offset, 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle));
1570
1571 chunk_offset += ICP_QAT_CSS_FWSK_PAD_LEN(handle);
1572 /* RSA exponent */
1573 memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle));
1574
1575 img_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle);
1576 chunk_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle);
1577 /* RSA signature */
1578 addr = bus_addr + chunk_offset;
1579 auth_desc->signature_high = upper_32_bits(addr);
1580 auth_desc->signature_low = lower_32_bits(addr);
1581 memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_SIGNATURE_LEN(handle));
1582
1583 img_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1584 chunk_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1585 /* XMSS pubkey */
1586 addr = bus_addr + chunk_offset;
1587 auth_desc->xmss_pubkey_high = upper_32_bits(addr);
1588 auth_desc->xmss_pubkey_low = lower_32_bits(addr);
1589 memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN);
1590
1591 img_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN;
1592 chunk_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN;
1593 /* XMSS signature */
1594 addr = bus_addr + chunk_offset;
1595 auth_desc->xmss_sig_high = upper_32_bits(addr);
1596 auth_desc->xmss_sig_low = lower_32_bits(addr);
1597 memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_SIG_LEN);
1598
1599 img_offset += ICP_QAT_DUALSIGN_XMSS_SIG_LEN;
1600 chunk_offset += ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN;
1601
1602 if (dram_desc->dram_size < (chunk_offset + auth_desc->img_len)) {
1603 pr_err("auth chunk memory size is not enough to store data\n");
1604 return -ENOMEM;
1605 }
1606
1607 /* Signed data */
1608 addr = bus_addr + chunk_offset;
1609 auth_desc->img_high = upper_32_bits(addr);
1610 auth_desc->img_low = lower_32_bits(addr);
1611 memcpy(virt_addr + chunk_offset, image + img_offset, auth_desc->img_len);
1612
1613 chunk_offset += ICP_QAT_DUALSIGN_MISC_INFO_LEN;
1614 /* AE firmware */
1615 if (fw_type == CSS_AE_FIRMWARE) {
1616 /* AE mode data */
1617 addr = bus_addr + chunk_offset;
1618 auth_desc->img_ae_mode_data_high = upper_32_bits(addr);
1619 auth_desc->img_ae_mode_data_low = lower_32_bits(addr);
1620 simg_ae_mode =
1621 (struct icp_qat_simg_ae_mode *)(virt_addr + chunk_offset);
1622 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask;
1623
1624 chunk_offset += sizeof(struct icp_qat_simg_ae_mode);
1625 /* AE init seq */
1626 addr = bus_addr + chunk_offset;
1627 auth_desc->img_ae_init_data_high = upper_32_bits(addr);
1628 auth_desc->img_ae_init_data_low = lower_32_bits(addr);
1629
1630 chunk_offset += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1631 /* AE instructions */
1632 addr = bus_addr + chunk_offset;
1633 auth_desc->img_ae_insts_high = upper_32_bits(addr);
1634 auth_desc->img_ae_insts_low = lower_32_bits(addr);
1635 } else {
1636 addr = bus_addr + chunk_offset;
1637 auth_desc->img_ae_insts_high = upper_32_bits(addr);
1638 auth_desc->img_ae_insts_low = lower_32_bits(addr);
1639 }
1640 *desc = auth_desc;
1641 return 0;
1642 }
1643
qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle * handle,char * image,unsigned int size,struct icp_qat_fw_auth_desc ** desc)1644 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
1645 char *image, unsigned int size,
1646 struct icp_qat_fw_auth_desc **desc)
1647 {
1648 struct icp_qat_auth_chunk *auth_chunk;
1649 struct icp_firml_dram_desc img_desc;
1650 unsigned int simg_fw_type;
1651 int ret;
1652
1653 ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN);
1654 if (ret)
1655 return ret;
1656
1657 simg_fw_type = qat_uclo_simg_fw_type(handle, image);
1658 auth_chunk = img_desc.dram_base_addr_v;
1659 auth_chunk->chunk_size = img_desc.dram_size;
1660 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr;
1661
1662 if (handle->chip_info->dual_sign)
1663 return qat_uclo_build_auth_desc_dualsign(handle, image, size, &img_desc,
1664 simg_fw_type, desc);
1665
1666 return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc,
1667 simg_fw_type, desc);
1668 }
1669
qat_uclo_load_fw(struct icp_qat_fw_loader_handle * handle,struct icp_qat_fw_auth_desc * desc)1670 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
1671 struct icp_qat_fw_auth_desc *desc)
1672 {
1673 unsigned long ae_mask = handle->hal_handle->ae_mask;
1674 u32 fcu_sts_csr, fcu_ctl_csr;
1675 u32 loaded_aes, loaded_csr;
1676 unsigned int i;
1677 u32 fcu_sts;
1678
1679 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1680 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1681 loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1682
1683 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) {
1684 int retry = 0;
1685
1686 if (!((desc->ae_mask >> i) & 0x1))
1687 continue;
1688 if (qat_hal_check_ae_active(handle, i)) {
1689 pr_err("AE %d is active\n", i);
1690 return -EINVAL;
1691 }
1692 SET_CAP_CSR(handle, fcu_ctl_csr,
1693 (FCU_CTRL_CMD_LOAD |
1694 (1 << FCU_CTRL_BROADCAST_POS) |
1695 (i << FCU_CTRL_AE_POS)));
1696
1697 do {
1698 msleep(FW_AUTH_WAIT_PERIOD);
1699 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1700 if ((fcu_sts & FCU_AUTH_STS_MASK) ==
1701 FCU_STS_LOAD_DONE) {
1702 loaded_aes = GET_CAP_CSR(handle, loaded_csr);
1703 loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos;
1704 if (loaded_aes & (1 << i))
1705 break;
1706 }
1707 } while (retry++ < FW_AUTH_MAX_RETRY);
1708 if (retry > FW_AUTH_MAX_RETRY) {
1709 pr_err("firmware load failed timeout %x\n", retry);
1710 return -EINVAL;
1711 }
1712 }
1713 return 0;
1714 }
1715
qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle * handle,void * addr_ptr,int mem_size)1716 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
1717 void *addr_ptr, int mem_size)
1718 {
1719 struct icp_qat_suof_handle *suof_handle;
1720
1721 suof_handle = kzalloc_obj(*suof_handle);
1722 if (!suof_handle)
1723 return -ENOMEM;
1724 handle->sobj_handle = suof_handle;
1725 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
1726 qat_uclo_del_suof(handle);
1727 pr_err("map SUOF failed\n");
1728 return -EINVAL;
1729 }
1730 return 0;
1731 }
1732
qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle * handle,void * addr_ptr,int mem_size)1733 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
1734 void *addr_ptr, int mem_size)
1735 {
1736 struct icp_qat_fw_auth_desc *desc = NULL;
1737 int status = 0;
1738 int ret;
1739
1740 ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE);
1741 if (ret)
1742 return ret;
1743
1744 if (handle->chip_info->fw_auth) {
1745 status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc);
1746 if (!status)
1747 status = qat_uclo_auth_fw(handle, desc);
1748 qat_uclo_ummap_auth_fw(handle, &desc);
1749 } else {
1750 if (handle->chip_info->mmp_sram_size < mem_size) {
1751 pr_err("MMP size is too large: 0x%x\n", mem_size);
1752 return -EFBIG;
1753 }
1754 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
1755 }
1756 return status;
1757 }
1758
qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle * handle,void * addr_ptr,int mem_size)1759 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
1760 void *addr_ptr, int mem_size)
1761 {
1762 struct icp_qat_uof_filehdr *filehdr;
1763 struct icp_qat_uclo_objhandle *objhdl;
1764
1765 objhdl = kzalloc_obj(*objhdl);
1766 if (!objhdl)
1767 return -ENOMEM;
1768 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL);
1769 if (!objhdl->obj_buf)
1770 goto out_objbuf_err;
1771 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
1772 if (qat_uclo_check_uof_format(filehdr))
1773 goto out_objhdr_err;
1774 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr,
1775 ICP_QAT_UOF_OBJS);
1776 if (!objhdl->obj_hdr) {
1777 pr_err("object file chunk is null\n");
1778 goto out_objhdr_err;
1779 }
1780 handle->obj_handle = objhdl;
1781 if (qat_uclo_parse_uof_obj(handle))
1782 goto out_overlay_obj_err;
1783 return 0;
1784
1785 out_overlay_obj_err:
1786 handle->obj_handle = NULL;
1787 kfree(objhdl->obj_hdr);
1788 out_objhdr_err:
1789 kfree(objhdl->obj_buf);
1790 out_objbuf_err:
1791 kfree(objhdl);
1792 return -ENOMEM;
1793 }
1794
qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle * handle,struct icp_qat_mof_file_hdr * mof_ptr,u32 mof_size)1795 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1796 struct icp_qat_mof_file_hdr *mof_ptr,
1797 u32 mof_size)
1798 {
1799 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1800 unsigned int min_ver_offset;
1801 unsigned int checksum;
1802
1803 mobj_handle->file_id = ICP_QAT_MOF_FID;
1804 mobj_handle->mof_buf = (char *)mof_ptr;
1805 mobj_handle->mof_size = mof_size;
1806
1807 min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr,
1808 min_ver);
1809 checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver,
1810 min_ver_offset);
1811 if (checksum != mof_ptr->checksum) {
1812 pr_err("incorrect MOF checksum\n");
1813 return -EINVAL;
1814 }
1815
1816 mobj_handle->checksum = mof_ptr->checksum;
1817 mobj_handle->min_ver = mof_ptr->min_ver;
1818 mobj_handle->maj_ver = mof_ptr->maj_ver;
1819 return 0;
1820 }
1821
qat_uclo_del_mof(struct icp_qat_fw_loader_handle * handle)1822 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle)
1823 {
1824 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1825
1826 kfree(mobj_handle->obj_table.obj_hdr);
1827 mobj_handle->obj_table.obj_hdr = NULL;
1828 kfree(handle->mobj_handle);
1829 handle->mobj_handle = NULL;
1830 }
1831
qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle * mobj_handle,const char * obj_name,char ** obj_ptr,unsigned int * obj_size)1832 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle,
1833 const char *obj_name, char **obj_ptr,
1834 unsigned int *obj_size)
1835 {
1836 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr;
1837 unsigned int i;
1838
1839 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) {
1840 if (!strncmp(obj_hdr[i].obj_name, obj_name,
1841 ICP_QAT_SUOF_OBJ_NAME_LEN)) {
1842 *obj_ptr = obj_hdr[i].obj_buf;
1843 *obj_size = obj_hdr[i].obj_size;
1844 return 0;
1845 }
1846 }
1847
1848 pr_err("object %s is not found inside MOF\n", obj_name);
1849 return -EINVAL;
1850 }
1851
qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle * mobj_handle,struct icp_qat_mof_objhdr * mobj_hdr,struct icp_qat_mof_obj_chunkhdr * obj_chunkhdr)1852 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle,
1853 struct icp_qat_mof_objhdr *mobj_hdr,
1854 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr)
1855 {
1856 u8 *obj;
1857
1858 if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG,
1859 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1860 obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset;
1861 } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG,
1862 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1863 obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset;
1864 } else {
1865 pr_err("unsupported chunk id\n");
1866 return -EINVAL;
1867 }
1868 mobj_hdr->obj_buf = obj;
1869 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size;
1870 mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str;
1871 return 0;
1872 }
1873
qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle * mobj_handle)1874 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle)
1875 {
1876 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr;
1877 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr;
1878 struct icp_qat_mof_obj_hdr *uobj_hdr;
1879 struct icp_qat_mof_obj_hdr *sobj_hdr;
1880 struct icp_qat_mof_objhdr *mobj_hdr;
1881 unsigned int uobj_chunk_num = 0;
1882 unsigned int sobj_chunk_num = 0;
1883 unsigned int *valid_chunk;
1884 int ret, i;
1885
1886 uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr;
1887 sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr;
1888 if (uobj_hdr)
1889 uobj_chunk_num = uobj_hdr->num_chunks;
1890 if (sobj_hdr)
1891 sobj_chunk_num = sobj_hdr->num_chunks;
1892
1893 mobj_hdr = kzalloc_objs(*mobj_hdr,
1894 size_add(uobj_chunk_num, sobj_chunk_num));
1895 if (!mobj_hdr)
1896 return -ENOMEM;
1897
1898 mobj_handle->obj_table.obj_hdr = mobj_hdr;
1899 valid_chunk = &mobj_handle->obj_table.num_objs;
1900 uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1901 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr));
1902 sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1903 ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr));
1904
1905 /* map uof objects */
1906 for (i = 0; i < uobj_chunk_num; i++) {
1907 ret = qat_uclo_map_obj_from_mof(mobj_handle,
1908 &mobj_hdr[*valid_chunk],
1909 &uobj_chunkhdr[i]);
1910 if (ret)
1911 return ret;
1912 (*valid_chunk)++;
1913 }
1914
1915 /* map suof objects */
1916 for (i = 0; i < sobj_chunk_num; i++) {
1917 ret = qat_uclo_map_obj_from_mof(mobj_handle,
1918 &mobj_hdr[*valid_chunk],
1919 &sobj_chunkhdr[i]);
1920 if (ret)
1921 return ret;
1922 (*valid_chunk)++;
1923 }
1924
1925 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) {
1926 pr_err("inconsistent UOF/SUOF chunk amount\n");
1927 return -EINVAL;
1928 }
1929 return 0;
1930 }
1931
qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle * mobj_handle,struct icp_qat_mof_chunkhdr * mof_chunkhdr)1932 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle,
1933 struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1934 {
1935 char **sym_str = (char **)&mobj_handle->sym_str;
1936 unsigned int *sym_size = &mobj_handle->sym_size;
1937 struct icp_qat_mof_str_table *str_table_obj;
1938
1939 *sym_size = *(unsigned int *)(uintptr_t)
1940 (mof_chunkhdr->offset + mobj_handle->mof_buf);
1941 *sym_str = (char *)(uintptr_t)
1942 (mobj_handle->mof_buf + mof_chunkhdr->offset +
1943 sizeof(str_table_obj->tab_len));
1944 }
1945
qat_uclo_map_mof_chunk(struct icp_qat_mof_handle * mobj_handle,struct icp_qat_mof_chunkhdr * mof_chunkhdr)1946 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle,
1947 struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1948 {
1949 char *chunk_id = mof_chunkhdr->chunk_id;
1950
1951 if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1952 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr);
1953 else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1954 mobj_handle->uobjs_hdr = mobj_handle->mof_buf +
1955 mof_chunkhdr->offset;
1956 else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1957 mobj_handle->sobjs_hdr = mobj_handle->mof_buf +
1958 mof_chunkhdr->offset;
1959 }
1960
qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr * mof_hdr)1961 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr)
1962 {
1963 int maj = mof_hdr->maj_ver & 0xff;
1964 int min = mof_hdr->min_ver & 0xff;
1965
1966 if (mof_hdr->file_id != ICP_QAT_MOF_FID) {
1967 pr_err("invalid header 0x%x\n", mof_hdr->file_id);
1968 return -EINVAL;
1969 }
1970
1971 if (mof_hdr->num_chunks <= 0x1) {
1972 pr_err("MOF chunk amount is incorrect\n");
1973 return -EINVAL;
1974 }
1975 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) {
1976 pr_err("bad MOF version, major 0x%x, minor 0x%x\n", maj, min);
1977 return -EINVAL;
1978 }
1979 return 0;
1980 }
1981
qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle * handle,struct icp_qat_mof_file_hdr * mof_ptr,u32 mof_size,const char * obj_name,char ** obj_ptr,unsigned int * obj_size)1982 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle,
1983 struct icp_qat_mof_file_hdr *mof_ptr,
1984 u32 mof_size, const char *obj_name,
1985 char **obj_ptr, unsigned int *obj_size)
1986 {
1987 struct icp_qat_mof_chunkhdr *mof_chunkhdr;
1988 unsigned int file_id = mof_ptr->file_id;
1989 struct icp_qat_mof_handle *mobj_handle;
1990 unsigned short chunks_num;
1991 unsigned int i;
1992 int ret;
1993
1994 if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) {
1995 if (obj_ptr)
1996 *obj_ptr = (char *)mof_ptr;
1997 if (obj_size)
1998 *obj_size = mof_size;
1999 return 0;
2000 }
2001 if (qat_uclo_check_mof_format(mof_ptr))
2002 return -EINVAL;
2003
2004 mobj_handle = kzalloc_obj(*mobj_handle);
2005 if (!mobj_handle)
2006 return -ENOMEM;
2007
2008 handle->mobj_handle = mobj_handle;
2009 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size);
2010 if (ret)
2011 return ret;
2012
2013 mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr);
2014 chunks_num = mof_ptr->num_chunks;
2015
2016 /* Parse MOF file chunks */
2017 for (i = 0; i < chunks_num; i++)
2018 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]);
2019
2020 /* All sym_objs uobjs and sobjs should be available */
2021 if (!mobj_handle->sym_str ||
2022 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr))
2023 return -EINVAL;
2024
2025 ret = qat_uclo_map_objs_from_mof(mobj_handle);
2026 if (ret)
2027 return ret;
2028
2029 /* Seek specified uof object in MOF */
2030 return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name,
2031 obj_ptr, obj_size);
2032 }
2033
qat_uclo_map_obj(struct icp_qat_fw_loader_handle * handle,void * addr_ptr,u32 mem_size,const char * obj_name)2034 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
2035 void *addr_ptr, u32 mem_size, const char *obj_name)
2036 {
2037 char *obj_addr;
2038 u32 obj_size;
2039 int ret;
2040
2041 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >=
2042 (sizeof(handle->hal_handle->ae_mask) * 8));
2043
2044 if (!handle || !addr_ptr || mem_size < 24)
2045 return -EINVAL;
2046
2047 if (obj_name) {
2048 ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name,
2049 &obj_addr, &obj_size);
2050 if (ret)
2051 return ret;
2052 } else {
2053 obj_addr = addr_ptr;
2054 obj_size = mem_size;
2055 }
2056
2057 return (handle->chip_info->fw_auth) ?
2058 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) :
2059 qat_uclo_map_uof_obj(handle, obj_addr, obj_size);
2060 }
2061
qat_uclo_del_obj(struct icp_qat_fw_loader_handle * handle)2062 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle)
2063 {
2064 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2065 unsigned int a;
2066
2067 if (handle->mobj_handle)
2068 qat_uclo_del_mof(handle);
2069 if (handle->sobj_handle)
2070 qat_uclo_del_suof(handle);
2071 if (!obj_handle)
2072 return;
2073
2074 kfree(obj_handle->uword_buf);
2075 for (a = 0; a < obj_handle->uimage_num; a++)
2076 kfree(obj_handle->ae_uimage[a].page);
2077
2078 for (a = 0; a < handle->hal_handle->ae_max_num; a++)
2079 qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
2080
2081 kfree(obj_handle->obj_hdr);
2082 kfree(obj_handle->obj_buf);
2083 kfree(obj_handle);
2084 handle->obj_handle = NULL;
2085 }
2086
qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle * obj_handle,struct icp_qat_uclo_encap_page * encap_page,u64 * uword,unsigned int addr_p,unsigned int raddr,u64 fill)2087 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
2088 struct icp_qat_uclo_encap_page *encap_page,
2089 u64 *uword, unsigned int addr_p,
2090 unsigned int raddr, u64 fill)
2091 {
2092 unsigned int i, addr;
2093 u64 uwrd = 0;
2094
2095 if (!encap_page) {
2096 *uword = fill;
2097 return;
2098 }
2099 addr = (encap_page->page_region) ? raddr : addr_p;
2100 for (i = 0; i < encap_page->uwblock_num; i++) {
2101 if (addr >= encap_page->uwblock[i].start_addr &&
2102 addr <= encap_page->uwblock[i].start_addr +
2103 encap_page->uwblock[i].words_num - 1) {
2104 addr -= encap_page->uwblock[i].start_addr;
2105 addr *= obj_handle->uword_in_bytes;
2106 memcpy(&uwrd, (void *)(((uintptr_t)
2107 encap_page->uwblock[i].micro_words) + addr),
2108 obj_handle->uword_in_bytes);
2109 uwrd = uwrd & GENMASK_ULL(43, 0);
2110 }
2111 }
2112 *uword = uwrd;
2113 if (*uword == INVLD_UWORD)
2114 *uword = fill;
2115 }
2116
qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uclo_encap_page * encap_page,unsigned int ae)2117 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
2118 struct icp_qat_uclo_encap_page
2119 *encap_page, unsigned int ae)
2120 {
2121 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
2122 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2123 u64 fill_pat;
2124
2125 /* load the page starting at appropriate ustore address */
2126 /* get fill-pattern from an image -- they are all the same */
2127 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern,
2128 sizeof(u64));
2129 uw_physical_addr = encap_page->beg_addr_p;
2130 uw_relative_addr = 0;
2131 words_num = encap_page->micro_words_num;
2132 while (words_num) {
2133 cpylen = min(words_num, UWORD_CPYBUF_SIZE);
2134
2135 /* load the buffer */
2136 for (i = 0; i < cpylen; i++)
2137 qat_uclo_fill_uwords(obj_handle, encap_page,
2138 &obj_handle->uword_buf[i],
2139 uw_physical_addr + i,
2140 uw_relative_addr + i, fill_pat);
2141
2142 /* copy the buffer to ustore */
2143 qat_hal_wr_uwords(handle, (unsigned char)ae,
2144 uw_physical_addr, cpylen,
2145 obj_handle->uword_buf);
2146
2147 uw_physical_addr += cpylen;
2148 uw_relative_addr += cpylen;
2149 words_num -= cpylen;
2150 }
2151 }
2152
qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle * handle,struct icp_qat_uof_image * image)2153 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
2154 struct icp_qat_uof_image *image)
2155 {
2156 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2157 unsigned long ae_mask = handle->hal_handle->ae_mask;
2158 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
2159 unsigned long ae_assigned = image->ae_assigned;
2160 struct icp_qat_uclo_aedata *aed;
2161 unsigned int ctx_mask, s;
2162 struct icp_qat_uclo_page *page;
2163 unsigned char ae;
2164 int ctx;
2165
2166 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
2167 ctx_mask = 0xff;
2168 else
2169 ctx_mask = 0x55;
2170 /* load the default page and set assigned CTX PC
2171 * to the entrypoint address */
2172 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
2173 if (!test_bit(ae, &cfg_ae_mask))
2174 continue;
2175
2176 if (!test_bit(ae, &ae_assigned))
2177 continue;
2178
2179 aed = &obj_handle->ae_data[ae];
2180 /* find the slice to which this image is assigned */
2181 for (s = 0; s < aed->slice_num; s++) {
2182 if (image->ctx_assigned &
2183 aed->ae_slices[s].ctx_mask_assigned)
2184 break;
2185 }
2186 if (s >= aed->slice_num)
2187 continue;
2188 page = aed->ae_slices[s].page;
2189 if (!page->encap_page->def_page)
2190 continue;
2191 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
2192
2193 page = aed->ae_slices[s].page;
2194 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
2195 aed->ae_slices[s].cur_page[ctx] =
2196 (ctx_mask & (1 << ctx)) ? page : NULL;
2197 qat_hal_set_live_ctx(handle, (unsigned char)ae,
2198 image->ctx_assigned);
2199 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned,
2200 image->entry_address);
2201 }
2202 }
2203
qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle * handle)2204 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
2205 {
2206 unsigned int i;
2207 struct icp_qat_fw_auth_desc *desc = NULL;
2208 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
2209 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
2210 int ret;
2211
2212 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
2213 ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf,
2214 simg_hdr[i].simg_len,
2215 CSS_AE_FIRMWARE);
2216 if (ret)
2217 return ret;
2218
2219 if (qat_uclo_map_auth_fw(handle,
2220 (char *)simg_hdr[i].simg_buf,
2221 (unsigned int)
2222 simg_hdr[i].simg_len,
2223 &desc))
2224 goto wr_err;
2225 if (qat_uclo_auth_fw(handle, desc))
2226 goto wr_err;
2227 if (qat_uclo_is_broadcast(handle, i)) {
2228 if (qat_uclo_broadcast_load_fw(handle, desc))
2229 goto wr_err;
2230 } else {
2231 if (qat_uclo_load_fw(handle, desc))
2232 goto wr_err;
2233 }
2234 qat_uclo_ummap_auth_fw(handle, &desc);
2235 }
2236 return 0;
2237 wr_err:
2238 qat_uclo_ummap_auth_fw(handle, &desc);
2239 return -EINVAL;
2240 }
2241
qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle * handle)2242 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
2243 {
2244 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2245 unsigned int i;
2246
2247 if (qat_uclo_init_globals(handle))
2248 return -EINVAL;
2249 for (i = 0; i < obj_handle->uimage_num; i++) {
2250 if (!obj_handle->ae_uimage[i].img_ptr)
2251 return -EINVAL;
2252 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
2253 return -EINVAL;
2254 qat_uclo_wr_uimage_page(handle,
2255 obj_handle->ae_uimage[i].img_ptr);
2256 }
2257 return 0;
2258 }
2259
qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle * handle)2260 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
2261 {
2262 return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) :
2263 qat_uclo_wr_uof_img(handle);
2264 }
2265
qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle * handle,unsigned int cfg_ae_mask)2266 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle,
2267 unsigned int cfg_ae_mask)
2268 {
2269 if (!cfg_ae_mask)
2270 return -EINVAL;
2271
2272 handle->cfg_ae_mask = cfg_ae_mask;
2273 return 0;
2274 }
2275