1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/slab.h> 4 #include <linux/ctype.h> 5 #include <linux/kernel.h> 6 #include <linux/delay.h> 7 #include <linux/pci_ids.h> 8 #include "adf_accel_devices.h" 9 #include "adf_common_drv.h" 10 #include "icp_qat_uclo.h" 11 #include "icp_qat_hal.h" 12 #include "icp_qat_fw_loader_handle.h" 13 14 #define UWORD_CPYBUF_SIZE 1024U 15 #define INVLD_UWORD 0xffffffffffull 16 #define PID_MINOR_REV 0xf 17 #define PID_MAJOR_REV (0xf << 4) 18 19 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, 20 unsigned int ae, unsigned int image_num) 21 { 22 struct icp_qat_uclo_aedata *ae_data; 23 struct icp_qat_uclo_encapme *encap_image; 24 struct icp_qat_uclo_page *page = NULL; 25 struct icp_qat_uclo_aeslice *ae_slice = NULL; 26 27 ae_data = &obj_handle->ae_data[ae]; 28 encap_image = &obj_handle->ae_uimage[image_num]; 29 ae_slice = &ae_data->ae_slices[ae_data->slice_num]; 30 ae_slice->encap_image = encap_image; 31 32 if (encap_image->img_ptr) { 33 ae_slice->ctx_mask_assigned = 34 encap_image->img_ptr->ctx_assigned; 35 ae_data->eff_ustore_size = obj_handle->ustore_phy_size; 36 } else { 37 ae_slice->ctx_mask_assigned = 0; 38 } 39 ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL); 40 if (!ae_slice->region) 41 return -ENOMEM; 42 ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL); 43 if (!ae_slice->page) 44 goto out_err; 45 page = ae_slice->page; 46 page->encap_page = encap_image->page; 47 ae_slice->page->region = ae_slice->region; 48 ae_data->slice_num++; 49 return 0; 50 out_err: 51 kfree(ae_slice->region); 52 ae_slice->region = NULL; 53 return -ENOMEM; 54 } 55 56 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) 57 { 58 unsigned int i; 59 60 if (!ae_data) { 61 pr_err("QAT: bad argument, ae_data is NULL\n"); 62 return -EINVAL; 63 } 64 65 for (i = 0; i < ae_data->slice_num; i++) { 66 kfree(ae_data->ae_slices[i].region); 67 ae_data->ae_slices[i].region = NULL; 68 kfree(ae_data->ae_slices[i].page); 69 ae_data->ae_slices[i].page = NULL; 70 } 71 return 0; 72 } 73 74 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, 75 unsigned int str_offset) 76 { 77 if (!str_table->table_len || str_offset > str_table->table_len) 78 return NULL; 79 return (char *)(((uintptr_t)(str_table->strings)) + str_offset); 80 } 81 82 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) 83 { 84 int maj = hdr->maj_ver & 0xff; 85 int min = hdr->min_ver & 0xff; 86 87 if (hdr->file_id != ICP_QAT_UOF_FID) { 88 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); 89 return -EINVAL; 90 } 91 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { 92 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", 93 maj, min); 94 return -EINVAL; 95 } 96 return 0; 97 } 98 99 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) 100 { 101 int maj = suof_hdr->maj_ver & 0xff; 102 int min = suof_hdr->min_ver & 0xff; 103 104 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { 105 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); 106 return -EINVAL; 107 } 108 if (suof_hdr->fw_type != 0) { 109 pr_err("QAT: unsupported firmware type\n"); 110 return -EINVAL; 111 } 112 if (suof_hdr->num_chunks <= 0x1) { 113 pr_err("QAT: SUOF chunk amount is incorrect\n"); 114 return -EINVAL; 115 } 116 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { 117 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", 118 maj, min); 119 return -EINVAL; 120 } 121 return 0; 122 } 123 124 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, 125 unsigned int addr, unsigned int *val, 126 unsigned int num_in_bytes) 127 { 128 unsigned int outval; 129 unsigned char *ptr = (unsigned char *)val; 130 131 while (num_in_bytes) { 132 memcpy(&outval, ptr, 4); 133 SRAM_WRITE(handle, addr, outval); 134 num_in_bytes -= 4; 135 ptr += 4; 136 addr += 4; 137 } 138 } 139 140 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, 141 unsigned char ae, unsigned int addr, 142 unsigned int *val, 143 unsigned int num_in_bytes) 144 { 145 unsigned int outval; 146 unsigned char *ptr = (unsigned char *)val; 147 148 addr >>= 0x2; /* convert to uword address */ 149 150 while (num_in_bytes) { 151 memcpy(&outval, ptr, 4); 152 qat_hal_wr_umem(handle, ae, addr++, 1, &outval); 153 num_in_bytes -= 4; 154 ptr += 4; 155 } 156 } 157 158 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, 159 unsigned char ae, 160 struct icp_qat_uof_batch_init 161 *umem_init_header) 162 { 163 struct icp_qat_uof_batch_init *umem_init; 164 165 if (!umem_init_header) 166 return; 167 umem_init = umem_init_header->next; 168 while (umem_init) { 169 unsigned int addr, *value, size; 170 171 ae = umem_init->ae; 172 addr = umem_init->addr; 173 value = umem_init->value; 174 size = umem_init->size; 175 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); 176 umem_init = umem_init->next; 177 } 178 } 179 180 static void 181 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, 182 struct icp_qat_uof_batch_init **base) 183 { 184 struct icp_qat_uof_batch_init *umem_init; 185 186 umem_init = *base; 187 while (umem_init) { 188 struct icp_qat_uof_batch_init *pre; 189 190 pre = umem_init; 191 umem_init = umem_init->next; 192 kfree(pre); 193 } 194 *base = NULL; 195 } 196 197 static int qat_uclo_parse_num(char *str, unsigned int *num) 198 { 199 char buf[16] = {0}; 200 unsigned long ae = 0; 201 int i; 202 203 strscpy(buf, str, sizeof(buf)); 204 for (i = 0; i < 16; i++) { 205 if (!isdigit(buf[i])) { 206 buf[i] = '\0'; 207 break; 208 } 209 } 210 if ((kstrtoul(buf, 10, &ae))) 211 return -EFAULT; 212 213 *num = (unsigned int)ae; 214 return 0; 215 } 216 217 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, 218 struct icp_qat_uof_initmem *init_mem, 219 unsigned int size_range, unsigned int *ae) 220 { 221 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 222 char *str; 223 224 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { 225 pr_err("QAT: initmem is out of range"); 226 return -EINVAL; 227 } 228 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { 229 pr_err("QAT: Memory scope for init_mem error\n"); 230 return -EINVAL; 231 } 232 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); 233 if (!str) { 234 pr_err("QAT: AE name assigned in UOF init table is NULL\n"); 235 return -EINVAL; 236 } 237 if (qat_uclo_parse_num(str, ae)) { 238 pr_err("QAT: Parse num for AE number failed\n"); 239 return -EINVAL; 240 } 241 if (*ae >= ICP_QAT_UCLO_MAX_AE) { 242 pr_err("QAT: ae %d out of range\n", *ae); 243 return -EINVAL; 244 } 245 return 0; 246 } 247 248 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle 249 *handle, struct icp_qat_uof_initmem 250 *init_mem, unsigned int ae, 251 struct icp_qat_uof_batch_init 252 **init_tab_base) 253 { 254 struct icp_qat_uof_batch_init *init_header, *tail; 255 struct icp_qat_uof_batch_init *mem_init, *tail_old; 256 struct icp_qat_uof_memvar_attr *mem_val_attr; 257 unsigned int i, flag = 0; 258 259 mem_val_attr = 260 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + 261 sizeof(struct icp_qat_uof_initmem)); 262 263 init_header = *init_tab_base; 264 if (!init_header) { 265 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL); 266 if (!init_header) 267 return -ENOMEM; 268 init_header->size = 1; 269 *init_tab_base = init_header; 270 flag = 1; 271 } 272 tail_old = init_header; 273 while (tail_old->next) 274 tail_old = tail_old->next; 275 tail = tail_old; 276 for (i = 0; i < init_mem->val_attr_num; i++) { 277 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL); 278 if (!mem_init) 279 goto out_err; 280 mem_init->ae = ae; 281 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; 282 mem_init->value = &mem_val_attr->value; 283 mem_init->size = 4; 284 mem_init->next = NULL; 285 tail->next = mem_init; 286 tail = mem_init; 287 init_header->size += qat_hal_get_ins_num(); 288 mem_val_attr++; 289 } 290 return 0; 291 out_err: 292 /* Do not free the list head unless we allocated it. */ 293 tail_old = tail_old->next; 294 if (flag) { 295 kfree(*init_tab_base); 296 *init_tab_base = NULL; 297 } 298 299 while (tail_old) { 300 mem_init = tail_old->next; 301 kfree(tail_old); 302 tail_old = mem_init; 303 } 304 return -ENOMEM; 305 } 306 307 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, 308 struct icp_qat_uof_initmem *init_mem) 309 { 310 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 311 unsigned int ae; 312 313 if (qat_uclo_fetch_initmem_ae(handle, init_mem, 314 handle->chip_info->lm_size, &ae)) 315 return -EINVAL; 316 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 317 &obj_handle->lm_init_tab[ae])) 318 return -EINVAL; 319 return 0; 320 } 321 322 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, 323 struct icp_qat_uof_initmem *init_mem) 324 { 325 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 326 unsigned int ae, ustore_size, uaddr, i; 327 struct icp_qat_uclo_aedata *aed; 328 329 ustore_size = obj_handle->ustore_phy_size; 330 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) 331 return -EINVAL; 332 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 333 &obj_handle->umem_init_tab[ae])) 334 return -EINVAL; 335 /* set the highest ustore address referenced */ 336 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; 337 aed = &obj_handle->ae_data[ae]; 338 for (i = 0; i < aed->slice_num; i++) { 339 if (aed->ae_slices[i].encap_image->uwords_num < uaddr) 340 aed->ae_slices[i].encap_image->uwords_num = uaddr; 341 } 342 return 0; 343 } 344 345 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, 346 struct icp_qat_uof_initmem *init_mem) 347 { 348 switch (init_mem->region) { 349 case ICP_QAT_UOF_LMEM_REGION: 350 if (qat_uclo_init_lmem_seg(handle, init_mem)) 351 return -EINVAL; 352 break; 353 case ICP_QAT_UOF_UMEM_REGION: 354 if (qat_uclo_init_umem_seg(handle, init_mem)) 355 return -EINVAL; 356 break; 357 default: 358 pr_err("QAT: initmem region error. region type=0x%x\n", 359 init_mem->region); 360 return -EINVAL; 361 } 362 return 0; 363 } 364 365 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, 366 struct icp_qat_uclo_encapme *image) 367 { 368 unsigned int i; 369 struct icp_qat_uclo_encap_page *page; 370 struct icp_qat_uof_image *uof_image; 371 unsigned char ae; 372 unsigned int ustore_size; 373 unsigned int patt_pos; 374 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 375 unsigned long ae_mask = handle->hal_handle->ae_mask; 376 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 377 u64 *fill_data; 378 379 uof_image = image->img_ptr; 380 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64), 381 GFP_KERNEL); 382 if (!fill_data) 383 return -ENOMEM; 384 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) 385 memcpy(&fill_data[i], &uof_image->fill_pattern, 386 sizeof(u64)); 387 page = image->page; 388 389 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 390 unsigned long ae_assigned = uof_image->ae_assigned; 391 392 if (!test_bit(ae, &ae_assigned)) 393 continue; 394 395 if (!test_bit(ae, &cfg_ae_mask)) 396 continue; 397 398 ustore_size = obj_handle->ae_data[ae].eff_ustore_size; 399 patt_pos = page->beg_addr_p + page->micro_words_num; 400 401 qat_hal_wr_uwords(handle, (unsigned char)ae, 0, 402 page->beg_addr_p, &fill_data[0]); 403 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, 404 ustore_size - patt_pos + 1, 405 &fill_data[page->beg_addr_p]); 406 } 407 kfree(fill_data); 408 return 0; 409 } 410 411 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) 412 { 413 int i, ae; 414 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 415 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; 416 unsigned long ae_mask = handle->hal_handle->ae_mask; 417 418 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { 419 if (initmem->num_in_bytes) { 420 if (qat_uclo_init_ae_memory(handle, initmem)) 421 return -EINVAL; 422 } 423 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( 424 (uintptr_t)initmem + 425 sizeof(struct icp_qat_uof_initmem)) + 426 (sizeof(struct icp_qat_uof_memvar_attr) * 427 initmem->val_attr_num)); 428 } 429 430 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 431 if (qat_hal_batch_wr_lm(handle, ae, 432 obj_handle->lm_init_tab[ae])) { 433 pr_err("QAT: fail to batch init lmem for AE %d\n", ae); 434 return -EINVAL; 435 } 436 qat_uclo_cleanup_batch_init_list(handle, 437 &obj_handle->lm_init_tab[ae]); 438 qat_uclo_batch_wr_umem(handle, ae, 439 obj_handle->umem_init_tab[ae]); 440 qat_uclo_cleanup_batch_init_list(handle, 441 &obj_handle-> 442 umem_init_tab[ae]); 443 } 444 return 0; 445 } 446 447 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, 448 char *chunk_id, void *cur) 449 { 450 int i; 451 struct icp_qat_uof_chunkhdr *chunk_hdr = 452 (struct icp_qat_uof_chunkhdr *) 453 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); 454 455 for (i = 0; i < obj_hdr->num_chunks; i++) { 456 if ((cur < (void *)&chunk_hdr[i]) && 457 !strncmp(chunk_hdr[i].chunk_id, chunk_id, 458 ICP_QAT_UOF_OBJID_LEN)) { 459 return &chunk_hdr[i]; 460 } 461 } 462 return NULL; 463 } 464 465 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) 466 { 467 int i; 468 unsigned int topbit = 1 << 0xF; 469 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); 470 471 reg ^= inbyte << 0x8; 472 for (i = 0; i < 0x8; i++) { 473 if (reg & topbit) 474 reg = (reg << 1) ^ 0x1021; 475 else 476 reg <<= 1; 477 } 478 return reg & 0xFFFF; 479 } 480 481 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) 482 { 483 unsigned int chksum = 0; 484 485 if (ptr) 486 while (num--) 487 chksum = qat_uclo_calc_checksum(chksum, *ptr++); 488 return chksum; 489 } 490 491 static struct icp_qat_uclo_objhdr * 492 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, 493 char *chunk_id) 494 { 495 struct icp_qat_uof_filechunkhdr *file_chunk; 496 struct icp_qat_uclo_objhdr *obj_hdr; 497 char *chunk; 498 int i; 499 500 file_chunk = (struct icp_qat_uof_filechunkhdr *) 501 (buf + sizeof(struct icp_qat_uof_filehdr)); 502 for (i = 0; i < file_hdr->num_chunks; i++) { 503 if (!strncmp(file_chunk->chunk_id, chunk_id, 504 ICP_QAT_UOF_OBJID_LEN)) { 505 chunk = buf + file_chunk->offset; 506 if (file_chunk->checksum != qat_uclo_calc_str_checksum( 507 chunk, file_chunk->size)) 508 break; 509 obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL); 510 if (!obj_hdr) 511 break; 512 obj_hdr->file_buff = chunk; 513 obj_hdr->checksum = file_chunk->checksum; 514 obj_hdr->size = file_chunk->size; 515 return obj_hdr; 516 } 517 file_chunk++; 518 } 519 return NULL; 520 } 521 522 static int 523 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, 524 struct icp_qat_uof_image *image) 525 { 526 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; 527 struct icp_qat_uof_objtable *neigh_reg_tab; 528 struct icp_qat_uof_code_page *code_page; 529 530 code_page = (struct icp_qat_uof_code_page *) 531 ((char *)image + sizeof(struct icp_qat_uof_image)); 532 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 533 code_page->uc_var_tab_offset); 534 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 535 code_page->imp_var_tab_offset); 536 imp_expr_tab = (struct icp_qat_uof_objtable *) 537 (encap_uof_obj->beg_uof + 538 code_page->imp_expr_tab_offset); 539 if (uc_var_tab->entry_num || imp_var_tab->entry_num || 540 imp_expr_tab->entry_num) { 541 pr_err("QAT: UOF can't contain imported variable to be parsed\n"); 542 return -EINVAL; 543 } 544 neigh_reg_tab = (struct icp_qat_uof_objtable *) 545 (encap_uof_obj->beg_uof + 546 code_page->neigh_reg_tab_offset); 547 if (neigh_reg_tab->entry_num) { 548 pr_err("QAT: UOF can't contain neighbor register table\n"); 549 return -EINVAL; 550 } 551 if (image->numpages > 1) { 552 pr_err("QAT: UOF can't contain multiple pages\n"); 553 return -EINVAL; 554 } 555 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { 556 pr_err("QAT: UOF can't use shared control store feature\n"); 557 return -EFAULT; 558 } 559 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { 560 pr_err("QAT: UOF can't use reloadable feature\n"); 561 return -EFAULT; 562 } 563 return 0; 564 } 565 566 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj 567 *encap_uof_obj, 568 struct icp_qat_uof_image *img, 569 struct icp_qat_uclo_encap_page *page) 570 { 571 struct icp_qat_uof_code_page *code_page; 572 struct icp_qat_uof_code_area *code_area; 573 struct icp_qat_uof_objtable *uword_block_tab; 574 struct icp_qat_uof_uword_block *uwblock; 575 int i; 576 577 code_page = (struct icp_qat_uof_code_page *) 578 ((char *)img + sizeof(struct icp_qat_uof_image)); 579 page->def_page = code_page->def_page; 580 page->page_region = code_page->page_region; 581 page->beg_addr_v = code_page->beg_addr_v; 582 page->beg_addr_p = code_page->beg_addr_p; 583 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + 584 code_page->code_area_offset); 585 page->micro_words_num = code_area->micro_words_num; 586 uword_block_tab = (struct icp_qat_uof_objtable *) 587 (encap_uof_obj->beg_uof + 588 code_area->uword_block_tab); 589 page->uwblock_num = uword_block_tab->entry_num; 590 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + 591 sizeof(struct icp_qat_uof_objtable)); 592 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; 593 for (i = 0; i < uword_block_tab->entry_num; i++) 594 page->uwblock[i].micro_words = 595 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; 596 } 597 598 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, 599 struct icp_qat_uclo_encapme *ae_uimage, 600 int max_image) 601 { 602 int i, j; 603 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; 604 struct icp_qat_uof_image *image; 605 struct icp_qat_uof_objtable *ae_regtab; 606 struct icp_qat_uof_objtable *init_reg_sym_tab; 607 struct icp_qat_uof_objtable *sbreak_tab; 608 struct icp_qat_uof_encap_obj *encap_uof_obj = 609 &obj_handle->encap_uof_obj; 610 611 for (j = 0; j < max_image; j++) { 612 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 613 ICP_QAT_UOF_IMAG, chunk_hdr); 614 if (!chunk_hdr) 615 break; 616 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + 617 chunk_hdr->offset); 618 ae_regtab = (struct icp_qat_uof_objtable *) 619 (image->reg_tab_offset + 620 obj_handle->obj_hdr->file_buff); 621 ae_uimage[j].ae_reg_num = ae_regtab->entry_num; 622 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) 623 (((char *)ae_regtab) + 624 sizeof(struct icp_qat_uof_objtable)); 625 init_reg_sym_tab = (struct icp_qat_uof_objtable *) 626 (image->init_reg_sym_tab + 627 obj_handle->obj_hdr->file_buff); 628 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; 629 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) 630 (((char *)init_reg_sym_tab) + 631 sizeof(struct icp_qat_uof_objtable)); 632 sbreak_tab = (struct icp_qat_uof_objtable *) 633 (image->sbreak_tab + obj_handle->obj_hdr->file_buff); 634 ae_uimage[j].sbreak_num = sbreak_tab->entry_num; 635 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) 636 (((char *)sbreak_tab) + 637 sizeof(struct icp_qat_uof_objtable)); 638 ae_uimage[j].img_ptr = image; 639 if (qat_uclo_check_image_compat(encap_uof_obj, image)) 640 goto out_err; 641 ae_uimage[j].page = 642 kzalloc(sizeof(struct icp_qat_uclo_encap_page), 643 GFP_KERNEL); 644 if (!ae_uimage[j].page) 645 goto out_err; 646 qat_uclo_map_image_page(encap_uof_obj, image, 647 ae_uimage[j].page); 648 } 649 return j; 650 out_err: 651 for (i = 0; i < j; i++) 652 kfree(ae_uimage[i].page); 653 return 0; 654 } 655 656 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) 657 { 658 int i, ae; 659 int mflag = 0; 660 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 661 unsigned long ae_mask = handle->hal_handle->ae_mask; 662 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 663 664 for_each_set_bit(ae, &ae_mask, max_ae) { 665 if (!test_bit(ae, &cfg_ae_mask)) 666 continue; 667 668 for (i = 0; i < obj_handle->uimage_num; i++) { 669 unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; 670 671 if (!test_bit(ae, &ae_assigned)) 672 continue; 673 mflag = 1; 674 if (qat_uclo_init_ae_data(obj_handle, ae, i)) 675 return -EINVAL; 676 } 677 } 678 if (!mflag) { 679 pr_err("QAT: uimage uses AE not set\n"); 680 return -EINVAL; 681 } 682 return 0; 683 } 684 685 static struct icp_qat_uof_strtable * 686 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, 687 char *tab_name, struct icp_qat_uof_strtable *str_table) 688 { 689 struct icp_qat_uof_chunkhdr *chunk_hdr; 690 691 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) 692 obj_hdr->file_buff, tab_name, NULL); 693 if (chunk_hdr) { 694 int hdr_size; 695 696 memcpy(&str_table->table_len, obj_hdr->file_buff + 697 chunk_hdr->offset, sizeof(str_table->table_len)); 698 hdr_size = (char *)&str_table->strings - (char *)str_table; 699 str_table->strings = (uintptr_t)obj_hdr->file_buff + 700 chunk_hdr->offset + hdr_size; 701 return str_table; 702 } 703 return NULL; 704 } 705 706 static void 707 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, 708 struct icp_qat_uclo_init_mem_table *init_mem_tab) 709 { 710 struct icp_qat_uof_chunkhdr *chunk_hdr; 711 712 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 713 ICP_QAT_UOF_IMEM, NULL); 714 if (chunk_hdr) { 715 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + 716 chunk_hdr->offset, sizeof(unsigned int)); 717 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) 718 (encap_uof_obj->beg_uof + chunk_hdr->offset + 719 sizeof(unsigned int)); 720 } 721 } 722 723 static unsigned int 724 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) 725 { 726 switch (handle->pci_dev->device) { 727 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: 728 return ICP_QAT_AC_895XCC_DEV_TYPE; 729 case PCI_DEVICE_ID_INTEL_QAT_C62X: 730 return ICP_QAT_AC_C62X_DEV_TYPE; 731 case PCI_DEVICE_ID_INTEL_QAT_C3XXX: 732 return ICP_QAT_AC_C3XXX_DEV_TYPE; 733 case ADF_4XXX_PCI_DEVICE_ID: 734 case ADF_401XX_PCI_DEVICE_ID: 735 case ADF_402XX_PCI_DEVICE_ID: 736 case ADF_420XX_PCI_DEVICE_ID: 737 return ICP_QAT_AC_4XXX_A_DEV_TYPE; 738 default: 739 pr_err("QAT: unsupported device 0x%x\n", 740 handle->pci_dev->device); 741 return 0; 742 } 743 } 744 745 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) 746 { 747 unsigned int maj_ver, prod_type = obj_handle->prod_type; 748 749 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { 750 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", 751 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, 752 prod_type); 753 return -EINVAL; 754 } 755 maj_ver = obj_handle->prod_rev & 0xff; 756 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || 757 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { 758 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); 759 return -EINVAL; 760 } 761 return 0; 762 } 763 764 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, 765 unsigned char ae, unsigned char ctx_mask, 766 enum icp_qat_uof_regtype reg_type, 767 unsigned short reg_addr, unsigned int value) 768 { 769 switch (reg_type) { 770 case ICP_GPA_ABS: 771 case ICP_GPB_ABS: 772 ctx_mask = 0; 773 fallthrough; 774 case ICP_GPA_REL: 775 case ICP_GPB_REL: 776 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, 777 reg_addr, value); 778 case ICP_SR_ABS: 779 case ICP_DR_ABS: 780 case ICP_SR_RD_ABS: 781 case ICP_DR_RD_ABS: 782 ctx_mask = 0; 783 fallthrough; 784 case ICP_SR_REL: 785 case ICP_DR_REL: 786 case ICP_SR_RD_REL: 787 case ICP_DR_RD_REL: 788 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, 789 reg_addr, value); 790 case ICP_SR_WR_ABS: 791 case ICP_DR_WR_ABS: 792 ctx_mask = 0; 793 fallthrough; 794 case ICP_SR_WR_REL: 795 case ICP_DR_WR_REL: 796 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, 797 reg_addr, value); 798 case ICP_NEIGH_REL: 799 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); 800 default: 801 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); 802 return -EFAULT; 803 } 804 return 0; 805 } 806 807 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, 808 unsigned int ae, 809 struct icp_qat_uclo_encapme *encap_ae) 810 { 811 unsigned int i; 812 unsigned char ctx_mask; 813 struct icp_qat_uof_init_regsym *init_regsym; 814 815 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == 816 ICP_QAT_UCLO_MAX_CTX) 817 ctx_mask = 0xff; 818 else 819 ctx_mask = 0x55; 820 821 for (i = 0; i < encap_ae->init_regsym_num; i++) { 822 unsigned int exp_res; 823 824 init_regsym = &encap_ae->init_regsym[i]; 825 exp_res = init_regsym->value; 826 switch (init_regsym->init_type) { 827 case ICP_QAT_UOF_INIT_REG: 828 qat_uclo_init_reg(handle, ae, ctx_mask, 829 (enum icp_qat_uof_regtype) 830 init_regsym->reg_type, 831 (unsigned short)init_regsym->reg_addr, 832 exp_res); 833 break; 834 case ICP_QAT_UOF_INIT_REG_CTX: 835 /* check if ctx is appropriate for the ctxMode */ 836 if (!((1 << init_regsym->ctx) & ctx_mask)) { 837 pr_err("QAT: invalid ctx num = 0x%x\n", 838 init_regsym->ctx); 839 return -EINVAL; 840 } 841 qat_uclo_init_reg(handle, ae, 842 (unsigned char) 843 (1 << init_regsym->ctx), 844 (enum icp_qat_uof_regtype) 845 init_regsym->reg_type, 846 (unsigned short)init_regsym->reg_addr, 847 exp_res); 848 break; 849 case ICP_QAT_UOF_INIT_EXPR: 850 pr_err("QAT: INIT_EXPR feature not supported\n"); 851 return -EINVAL; 852 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: 853 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); 854 return -EINVAL; 855 default: 856 break; 857 } 858 } 859 return 0; 860 } 861 862 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) 863 { 864 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 865 unsigned long ae_mask = handle->hal_handle->ae_mask; 866 struct icp_qat_uclo_aedata *aed; 867 unsigned int s, ae; 868 869 if (obj_handle->global_inited) 870 return 0; 871 if (obj_handle->init_mem_tab.entry_num) { 872 if (qat_uclo_init_memory(handle)) { 873 pr_err("QAT: initialize memory failed\n"); 874 return -EINVAL; 875 } 876 } 877 878 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 879 aed = &obj_handle->ae_data[ae]; 880 for (s = 0; s < aed->slice_num; s++) { 881 if (!aed->ae_slices[s].encap_image) 882 continue; 883 if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) 884 return -EINVAL; 885 } 886 } 887 obj_handle->global_inited = 1; 888 return 0; 889 } 890 891 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, 892 struct icp_qat_uclo_objhandle *obj_handle, 893 unsigned char ae, 894 struct icp_qat_uof_image *uof_image) 895 { 896 unsigned char mode; 897 int ret; 898 899 mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); 900 ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); 901 if (ret) { 902 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); 903 return ret; 904 } 905 if (handle->chip_info->nn) { 906 mode = ICP_QAT_NN_MODE(uof_image->ae_mode); 907 ret = qat_hal_set_ae_nn_mode(handle, ae, mode); 908 if (ret) { 909 pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); 910 return ret; 911 } 912 } 913 mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); 914 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); 915 if (ret) { 916 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); 917 return ret; 918 } 919 mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); 920 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); 921 if (ret) { 922 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); 923 return ret; 924 } 925 if (handle->chip_info->lm2lm3) { 926 mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); 927 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); 928 if (ret) { 929 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); 930 return ret; 931 } 932 mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); 933 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); 934 if (ret) { 935 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); 936 return ret; 937 } 938 mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); 939 qat_hal_set_ae_tindex_mode(handle, ae, mode); 940 } 941 return 0; 942 } 943 944 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) 945 { 946 struct icp_qat_uof_image *uof_image; 947 struct icp_qat_uclo_aedata *ae_data; 948 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 949 unsigned long ae_mask = handle->hal_handle->ae_mask; 950 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 951 unsigned char ae, s; 952 int error; 953 954 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 955 if (!test_bit(ae, &cfg_ae_mask)) 956 continue; 957 958 ae_data = &obj_handle->ae_data[ae]; 959 for (s = 0; s < min_t(unsigned int, ae_data->slice_num, 960 ICP_QAT_UCLO_MAX_CTX); s++) { 961 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) 962 continue; 963 uof_image = ae_data->ae_slices[s].encap_image->img_ptr; 964 error = qat_hal_set_modes(handle, obj_handle, ae, 965 uof_image); 966 if (error) 967 return error; 968 } 969 } 970 return 0; 971 } 972 973 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) 974 { 975 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 976 struct icp_qat_uclo_encapme *image; 977 int a; 978 979 for (a = 0; a < obj_handle->uimage_num; a++) { 980 image = &obj_handle->ae_uimage[a]; 981 image->uwords_num = image->page->beg_addr_p + 982 image->page->micro_words_num; 983 } 984 } 985 986 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) 987 { 988 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 989 unsigned int ae; 990 991 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; 992 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) 993 obj_handle->obj_hdr->file_buff; 994 obj_handle->uword_in_bytes = 6; 995 obj_handle->prod_type = qat_uclo_get_dev_type(handle); 996 obj_handle->prod_rev = PID_MAJOR_REV | 997 (PID_MINOR_REV & handle->hal_handle->revision_id); 998 if (qat_uclo_check_uof_compat(obj_handle)) { 999 pr_err("QAT: UOF incompatible\n"); 1000 return -EINVAL; 1001 } 1002 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), 1003 GFP_KERNEL); 1004 if (!obj_handle->uword_buf) 1005 return -ENOMEM; 1006 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; 1007 if (!obj_handle->obj_hdr->file_buff || 1008 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, 1009 &obj_handle->str_table)) { 1010 pr_err("QAT: UOF doesn't have effective images\n"); 1011 goto out_err; 1012 } 1013 obj_handle->uimage_num = 1014 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, 1015 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); 1016 if (!obj_handle->uimage_num) 1017 goto out_err; 1018 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { 1019 pr_err("QAT: Bad object\n"); 1020 goto out_check_uof_aemask_err; 1021 } 1022 qat_uclo_init_uword_num(handle); 1023 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, 1024 &obj_handle->init_mem_tab); 1025 if (qat_uclo_set_ae_mode(handle)) 1026 goto out_check_uof_aemask_err; 1027 return 0; 1028 out_check_uof_aemask_err: 1029 for (ae = 0; ae < obj_handle->uimage_num; ae++) 1030 kfree(obj_handle->ae_uimage[ae].page); 1031 out_err: 1032 kfree(obj_handle->uword_buf); 1033 return -EFAULT; 1034 } 1035 1036 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1037 struct icp_qat_suof_filehdr *suof_ptr, 1038 int suof_size) 1039 { 1040 unsigned int check_sum = 0; 1041 unsigned int min_ver_offset = 0; 1042 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1043 1044 suof_handle->file_id = ICP_QAT_SUOF_FID; 1045 suof_handle->suof_buf = (char *)suof_ptr; 1046 suof_handle->suof_size = suof_size; 1047 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, 1048 min_ver); 1049 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, 1050 min_ver_offset); 1051 if (check_sum != suof_ptr->check_sum) { 1052 pr_err("QAT: incorrect SUOF checksum\n"); 1053 return -EINVAL; 1054 } 1055 suof_handle->check_sum = suof_ptr->check_sum; 1056 suof_handle->min_ver = suof_ptr->min_ver; 1057 suof_handle->maj_ver = suof_ptr->maj_ver; 1058 suof_handle->fw_type = suof_ptr->fw_type; 1059 return 0; 1060 } 1061 1062 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, 1063 struct icp_qat_suof_img_hdr *suof_img_hdr, 1064 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1065 { 1066 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1067 struct icp_qat_simg_ae_mode *ae_mode; 1068 struct icp_qat_suof_objhdr *suof_objhdr; 1069 1070 suof_img_hdr->simg_buf = (suof_handle->suof_buf + 1071 suof_chunk_hdr->offset + 1072 sizeof(*suof_objhdr)); 1073 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) 1074 (suof_handle->suof_buf + 1075 suof_chunk_hdr->offset))->img_length; 1076 1077 suof_img_hdr->css_header = suof_img_hdr->simg_buf; 1078 suof_img_hdr->css_key = (suof_img_hdr->css_header + 1079 sizeof(struct icp_qat_css_hdr)); 1080 suof_img_hdr->css_signature = suof_img_hdr->css_key + 1081 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1082 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); 1083 suof_img_hdr->css_simg = suof_img_hdr->css_signature + 1084 ICP_QAT_CSS_SIGNATURE_LEN(handle); 1085 1086 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); 1087 suof_img_hdr->ae_mask = ae_mode->ae_mask; 1088 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; 1089 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; 1090 suof_img_hdr->fw_type = ae_mode->fw_type; 1091 } 1092 1093 static void 1094 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, 1095 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1096 { 1097 char **sym_str = (char **)&suof_handle->sym_str; 1098 unsigned int *sym_size = &suof_handle->sym_size; 1099 struct icp_qat_suof_strtable *str_table_obj; 1100 1101 *sym_size = *(unsigned int *)(uintptr_t) 1102 (suof_chunk_hdr->offset + suof_handle->suof_buf); 1103 *sym_str = (char *)(uintptr_t) 1104 (suof_handle->suof_buf + suof_chunk_hdr->offset + 1105 sizeof(str_table_obj->tab_length)); 1106 } 1107 1108 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, 1109 struct icp_qat_suof_img_hdr *img_hdr) 1110 { 1111 struct icp_qat_simg_ae_mode *img_ae_mode = NULL; 1112 unsigned int prod_rev, maj_ver, prod_type; 1113 1114 prod_type = qat_uclo_get_dev_type(handle); 1115 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; 1116 prod_rev = PID_MAJOR_REV | 1117 (PID_MINOR_REV & handle->hal_handle->revision_id); 1118 if (img_ae_mode->dev_type != prod_type) { 1119 pr_err("QAT: incompatible product type %x\n", 1120 img_ae_mode->dev_type); 1121 return -EINVAL; 1122 } 1123 maj_ver = prod_rev & 0xff; 1124 if (maj_ver > img_ae_mode->devmax_ver || 1125 maj_ver < img_ae_mode->devmin_ver) { 1126 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); 1127 return -EINVAL; 1128 } 1129 return 0; 1130 } 1131 1132 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) 1133 { 1134 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 1135 1136 kfree(sobj_handle->img_table.simg_hdr); 1137 sobj_handle->img_table.simg_hdr = NULL; 1138 kfree(handle->sobj_handle); 1139 handle->sobj_handle = NULL; 1140 } 1141 1142 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, 1143 unsigned int img_id, unsigned int num_simgs) 1144 { 1145 struct icp_qat_suof_img_hdr img_header; 1146 1147 if (img_id != num_simgs - 1) { 1148 memcpy(&img_header, &suof_img_hdr[num_simgs - 1], 1149 sizeof(*suof_img_hdr)); 1150 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], 1151 sizeof(*suof_img_hdr)); 1152 memcpy(&suof_img_hdr[img_id], &img_header, 1153 sizeof(*suof_img_hdr)); 1154 } 1155 } 1156 1157 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, 1158 struct icp_qat_suof_filehdr *suof_ptr, 1159 int suof_size) 1160 { 1161 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1162 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; 1163 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; 1164 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; 1165 unsigned int i = 0; 1166 struct icp_qat_suof_img_hdr img_header; 1167 1168 if (!suof_ptr || suof_size == 0) { 1169 pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); 1170 return -EINVAL; 1171 } 1172 if (qat_uclo_check_suof_format(suof_ptr)) 1173 return -EINVAL; 1174 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); 1175 if (ret) 1176 return ret; 1177 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) 1178 ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); 1179 1180 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); 1181 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; 1182 1183 if (suof_handle->img_table.num_simgs != 0) { 1184 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs, 1185 sizeof(img_header), 1186 GFP_KERNEL); 1187 if (!suof_img_hdr) 1188 return -ENOMEM; 1189 suof_handle->img_table.simg_hdr = suof_img_hdr; 1190 1191 for (i = 0; i < suof_handle->img_table.num_simgs; i++) { 1192 qat_uclo_map_simg(handle, &suof_img_hdr[i], 1193 &suof_chunk_hdr[1 + i]); 1194 ret = qat_uclo_check_simg_compat(handle, 1195 &suof_img_hdr[i]); 1196 if (ret) 1197 return ret; 1198 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; 1199 if ((suof_img_hdr[i].ae_mask & 0x1) != 0) 1200 ae0_img = i; 1201 } 1202 1203 if (!handle->chip_info->tgroup_share_ustore) { 1204 qat_uclo_tail_img(suof_img_hdr, ae0_img, 1205 suof_handle->img_table.num_simgs); 1206 } 1207 } 1208 return 0; 1209 } 1210 1211 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) 1212 #define BITS_IN_DWORD 32 1213 1214 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, 1215 struct icp_qat_fw_auth_desc *desc) 1216 { 1217 u32 fcu_sts, retry = 0; 1218 u32 fcu_ctl_csr, fcu_sts_csr; 1219 u32 fcu_dram_hi_csr, fcu_dram_lo_csr; 1220 u64 bus_addr; 1221 1222 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) 1223 - sizeof(struct icp_qat_auth_chunk); 1224 1225 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1226 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1227 fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; 1228 fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; 1229 1230 SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); 1231 SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); 1232 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); 1233 1234 do { 1235 msleep(FW_AUTH_WAIT_PERIOD); 1236 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1237 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) 1238 goto auth_fail; 1239 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) 1240 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) 1241 return 0; 1242 } while (retry++ < FW_AUTH_MAX_RETRY); 1243 auth_fail: 1244 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", 1245 fcu_sts & FCU_AUTH_STS_MASK, retry); 1246 return -EINVAL; 1247 } 1248 1249 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, 1250 int imgid) 1251 { 1252 struct icp_qat_suof_handle *sobj_handle; 1253 1254 if (!handle->chip_info->tgroup_share_ustore) 1255 return false; 1256 1257 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; 1258 if (handle->hal_handle->admin_ae_mask & 1259 sobj_handle->img_table.simg_hdr[imgid].ae_mask) 1260 return false; 1261 1262 return true; 1263 } 1264 1265 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, 1266 struct icp_qat_fw_auth_desc *desc) 1267 { 1268 unsigned long ae_mask = handle->hal_handle->ae_mask; 1269 unsigned long desc_ae_mask = desc->ae_mask; 1270 u32 fcu_sts, ae_broadcast_mask = 0; 1271 u32 fcu_loaded_csr, ae_loaded; 1272 u32 fcu_sts_csr, fcu_ctl_csr; 1273 unsigned int ae, retry = 0; 1274 1275 if (handle->chip_info->tgroup_share_ustore) { 1276 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1277 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1278 fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1279 } else { 1280 pr_err("Chip 0x%x doesn't support broadcast load\n", 1281 handle->pci_dev->device); 1282 return -EINVAL; 1283 } 1284 1285 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 1286 if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { 1287 pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); 1288 return -EINVAL; 1289 } 1290 1291 if (test_bit(ae, &desc_ae_mask)) 1292 ae_broadcast_mask |= 1 << ae; 1293 } 1294 1295 if (ae_broadcast_mask) { 1296 SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, 1297 ae_broadcast_mask); 1298 1299 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); 1300 1301 do { 1302 msleep(FW_AUTH_WAIT_PERIOD); 1303 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1304 fcu_sts &= FCU_AUTH_STS_MASK; 1305 1306 if (fcu_sts == FCU_STS_LOAD_FAIL) { 1307 pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); 1308 return -EINVAL; 1309 } else if (fcu_sts == FCU_STS_LOAD_DONE) { 1310 ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); 1311 ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; 1312 1313 if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) 1314 break; 1315 } 1316 } while (retry++ < FW_AUTH_MAX_RETRY); 1317 1318 if (retry > FW_AUTH_MAX_RETRY) { 1319 pr_err("QAT: broadcast load failed timeout %d\n", retry); 1320 return -EINVAL; 1321 } 1322 } 1323 return 0; 1324 } 1325 1326 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, 1327 struct icp_firml_dram_desc *dram_desc, 1328 unsigned int size) 1329 { 1330 void *vptr; 1331 dma_addr_t ptr; 1332 1333 vptr = dma_alloc_coherent(&handle->pci_dev->dev, 1334 size, &ptr, GFP_KERNEL); 1335 if (!vptr) 1336 return -ENOMEM; 1337 dram_desc->dram_base_addr_v = vptr; 1338 dram_desc->dram_bus_addr = ptr; 1339 dram_desc->dram_size = size; 1340 return 0; 1341 } 1342 1343 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, 1344 struct icp_firml_dram_desc *dram_desc) 1345 { 1346 if (handle && dram_desc && dram_desc->dram_base_addr_v) { 1347 dma_free_coherent(&handle->pci_dev->dev, 1348 (size_t)(dram_desc->dram_size), 1349 dram_desc->dram_base_addr_v, 1350 dram_desc->dram_bus_addr); 1351 } 1352 1353 if (dram_desc) 1354 memset(dram_desc, 0, sizeof(*dram_desc)); 1355 } 1356 1357 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, 1358 struct icp_qat_fw_auth_desc **desc) 1359 { 1360 struct icp_firml_dram_desc dram_desc; 1361 1362 if (*desc) { 1363 dram_desc.dram_base_addr_v = *desc; 1364 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) 1365 (*desc))->chunk_bus_addr; 1366 dram_desc.dram_size = ((struct icp_qat_auth_chunk *) 1367 (*desc))->chunk_size; 1368 qat_uclo_simg_free(handle, &dram_desc); 1369 } 1370 } 1371 1372 static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, 1373 char *image, unsigned int size, 1374 unsigned int fw_type) 1375 { 1376 char *fw_type_name = fw_type ? "MMP" : "AE"; 1377 unsigned int css_dword_size = sizeof(u32); 1378 1379 if (handle->chip_info->fw_auth) { 1380 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1381 unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); 1382 1383 if ((css_hdr->header_len * css_dword_size) != header_len) 1384 goto err; 1385 if ((css_hdr->size * css_dword_size) != size) 1386 goto err; 1387 if (fw_type != css_hdr->fw_type) 1388 goto err; 1389 if (size <= header_len) 1390 goto err; 1391 size -= header_len; 1392 } 1393 1394 if (fw_type == CSS_AE_FIRMWARE) { 1395 if (size < sizeof(struct icp_qat_simg_ae_mode *) + 1396 ICP_QAT_SIMG_AE_INIT_SEQ_LEN) 1397 goto err; 1398 if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) 1399 goto err; 1400 } else if (fw_type == CSS_MMP_FIRMWARE) { 1401 if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) 1402 goto err; 1403 } else { 1404 pr_err("QAT: Unsupported firmware type\n"); 1405 return -EINVAL; 1406 } 1407 return 0; 1408 1409 err: 1410 pr_err("QAT: Invalid %s firmware image\n", fw_type_name); 1411 return -EINVAL; 1412 } 1413 1414 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, 1415 char *image, unsigned int size, 1416 struct icp_qat_fw_auth_desc **desc) 1417 { 1418 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1419 struct icp_qat_fw_auth_desc *auth_desc; 1420 struct icp_qat_auth_chunk *auth_chunk; 1421 u64 virt_addr, bus_addr, virt_base; 1422 unsigned int length, simg_offset = sizeof(*auth_chunk); 1423 struct icp_qat_simg_ae_mode *simg_ae_mode; 1424 struct icp_firml_dram_desc img_desc; 1425 1426 if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { 1427 pr_err("QAT: error, input image size overflow %d\n", size); 1428 return -EINVAL; 1429 } 1430 length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? 1431 ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : 1432 size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; 1433 if (qat_uclo_simg_alloc(handle, &img_desc, length)) { 1434 pr_err("QAT: error, allocate continuous dram fail\n"); 1435 return -ENOMEM; 1436 } 1437 1438 auth_chunk = img_desc.dram_base_addr_v; 1439 auth_chunk->chunk_size = img_desc.dram_size; 1440 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; 1441 virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; 1442 bus_addr = img_desc.dram_bus_addr + simg_offset; 1443 auth_desc = img_desc.dram_base_addr_v; 1444 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1445 auth_desc->css_hdr_low = (unsigned int)bus_addr; 1446 virt_addr = virt_base; 1447 1448 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); 1449 /* pub key */ 1450 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + 1451 sizeof(*css_hdr); 1452 virt_addr = virt_addr + sizeof(*css_hdr); 1453 1454 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1455 auth_desc->fwsk_pub_low = (unsigned int)bus_addr; 1456 1457 memcpy((void *)(uintptr_t)virt_addr, 1458 (void *)(image + sizeof(*css_hdr)), 1459 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); 1460 /* padding */ 1461 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1462 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); 1463 1464 /* exponent */ 1465 memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1466 ICP_QAT_CSS_FWSK_PAD_LEN(handle)), 1467 (void *)(image + sizeof(*css_hdr) + 1468 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1469 sizeof(unsigned int)); 1470 1471 /* signature */ 1472 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, 1473 auth_desc->fwsk_pub_low) + 1474 ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1475 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1476 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1477 auth_desc->signature_low = (unsigned int)bus_addr; 1478 1479 memcpy((void *)(uintptr_t)virt_addr, 1480 (void *)(image + sizeof(*css_hdr) + 1481 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1482 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), 1483 ICP_QAT_CSS_SIGNATURE_LEN(handle)); 1484 1485 bus_addr = ADD_ADDR(auth_desc->signature_high, 1486 auth_desc->signature_low) + 1487 ICP_QAT_CSS_SIGNATURE_LEN(handle); 1488 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1489 1490 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1491 auth_desc->img_low = (unsigned int)bus_addr; 1492 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); 1493 memcpy((void *)(uintptr_t)virt_addr, 1494 (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), 1495 auth_desc->img_len); 1496 virt_addr = virt_base; 1497 /* AE firmware */ 1498 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == 1499 CSS_AE_FIRMWARE) { 1500 auth_desc->img_ae_mode_data_high = auth_desc->img_high; 1501 auth_desc->img_ae_mode_data_low = auth_desc->img_low; 1502 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, 1503 auth_desc->img_ae_mode_data_low) + 1504 sizeof(struct icp_qat_simg_ae_mode); 1505 1506 auth_desc->img_ae_init_data_high = (unsigned int) 1507 (bus_addr >> BITS_IN_DWORD); 1508 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; 1509 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; 1510 auth_desc->img_ae_insts_high = (unsigned int) 1511 (bus_addr >> BITS_IN_DWORD); 1512 auth_desc->img_ae_insts_low = (unsigned int)bus_addr; 1513 virt_addr += sizeof(struct icp_qat_css_hdr); 1514 virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1515 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1516 simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; 1517 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; 1518 } else { 1519 auth_desc->img_ae_insts_high = auth_desc->img_high; 1520 auth_desc->img_ae_insts_low = auth_desc->img_low; 1521 } 1522 *desc = auth_desc; 1523 return 0; 1524 } 1525 1526 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, 1527 struct icp_qat_fw_auth_desc *desc) 1528 { 1529 unsigned long ae_mask = handle->hal_handle->ae_mask; 1530 u32 fcu_sts_csr, fcu_ctl_csr; 1531 u32 loaded_aes, loaded_csr; 1532 unsigned int i; 1533 u32 fcu_sts; 1534 1535 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1536 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1537 loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1538 1539 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { 1540 int retry = 0; 1541 1542 if (!((desc->ae_mask >> i) & 0x1)) 1543 continue; 1544 if (qat_hal_check_ae_active(handle, i)) { 1545 pr_err("QAT: AE %d is active\n", i); 1546 return -EINVAL; 1547 } 1548 SET_CAP_CSR(handle, fcu_ctl_csr, 1549 (FCU_CTRL_CMD_LOAD | 1550 (1 << FCU_CTRL_BROADCAST_POS) | 1551 (i << FCU_CTRL_AE_POS))); 1552 1553 do { 1554 msleep(FW_AUTH_WAIT_PERIOD); 1555 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1556 if ((fcu_sts & FCU_AUTH_STS_MASK) == 1557 FCU_STS_LOAD_DONE) { 1558 loaded_aes = GET_CAP_CSR(handle, loaded_csr); 1559 loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; 1560 if (loaded_aes & (1 << i)) 1561 break; 1562 } 1563 } while (retry++ < FW_AUTH_MAX_RETRY); 1564 if (retry > FW_AUTH_MAX_RETRY) { 1565 pr_err("QAT: firmware load failed timeout %x\n", retry); 1566 return -EINVAL; 1567 } 1568 } 1569 return 0; 1570 } 1571 1572 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, 1573 void *addr_ptr, int mem_size) 1574 { 1575 struct icp_qat_suof_handle *suof_handle; 1576 1577 suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL); 1578 if (!suof_handle) 1579 return -ENOMEM; 1580 handle->sobj_handle = suof_handle; 1581 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { 1582 qat_uclo_del_suof(handle); 1583 pr_err("QAT: map SUOF failed\n"); 1584 return -EINVAL; 1585 } 1586 return 0; 1587 } 1588 1589 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, 1590 void *addr_ptr, int mem_size) 1591 { 1592 struct icp_qat_fw_auth_desc *desc = NULL; 1593 int status = 0; 1594 int ret; 1595 1596 ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); 1597 if (ret) 1598 return ret; 1599 1600 if (handle->chip_info->fw_auth) { 1601 status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); 1602 if (!status) 1603 status = qat_uclo_auth_fw(handle, desc); 1604 qat_uclo_ummap_auth_fw(handle, &desc); 1605 } else { 1606 if (handle->chip_info->mmp_sram_size < mem_size) { 1607 pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); 1608 return -EFBIG; 1609 } 1610 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); 1611 } 1612 return status; 1613 } 1614 1615 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, 1616 void *addr_ptr, int mem_size) 1617 { 1618 struct icp_qat_uof_filehdr *filehdr; 1619 struct icp_qat_uclo_objhandle *objhdl; 1620 1621 objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); 1622 if (!objhdl) 1623 return -ENOMEM; 1624 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); 1625 if (!objhdl->obj_buf) 1626 goto out_objbuf_err; 1627 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; 1628 if (qat_uclo_check_uof_format(filehdr)) 1629 goto out_objhdr_err; 1630 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, 1631 ICP_QAT_UOF_OBJS); 1632 if (!objhdl->obj_hdr) { 1633 pr_err("QAT: object file chunk is null\n"); 1634 goto out_objhdr_err; 1635 } 1636 handle->obj_handle = objhdl; 1637 if (qat_uclo_parse_uof_obj(handle)) 1638 goto out_overlay_obj_err; 1639 return 0; 1640 1641 out_overlay_obj_err: 1642 handle->obj_handle = NULL; 1643 kfree(objhdl->obj_hdr); 1644 out_objhdr_err: 1645 kfree(objhdl->obj_buf); 1646 out_objbuf_err: 1647 kfree(objhdl); 1648 return -ENOMEM; 1649 } 1650 1651 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1652 struct icp_qat_mof_file_hdr *mof_ptr, 1653 u32 mof_size) 1654 { 1655 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1656 unsigned int min_ver_offset; 1657 unsigned int checksum; 1658 1659 mobj_handle->file_id = ICP_QAT_MOF_FID; 1660 mobj_handle->mof_buf = (char *)mof_ptr; 1661 mobj_handle->mof_size = mof_size; 1662 1663 min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, 1664 min_ver); 1665 checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, 1666 min_ver_offset); 1667 if (checksum != mof_ptr->checksum) { 1668 pr_err("QAT: incorrect MOF checksum\n"); 1669 return -EINVAL; 1670 } 1671 1672 mobj_handle->checksum = mof_ptr->checksum; 1673 mobj_handle->min_ver = mof_ptr->min_ver; 1674 mobj_handle->maj_ver = mof_ptr->maj_ver; 1675 return 0; 1676 } 1677 1678 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) 1679 { 1680 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1681 1682 kfree(mobj_handle->obj_table.obj_hdr); 1683 mobj_handle->obj_table.obj_hdr = NULL; 1684 kfree(handle->mobj_handle); 1685 handle->mobj_handle = NULL; 1686 } 1687 1688 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, 1689 const char *obj_name, char **obj_ptr, 1690 unsigned int *obj_size) 1691 { 1692 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; 1693 unsigned int i; 1694 1695 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { 1696 if (!strncmp(obj_hdr[i].obj_name, obj_name, 1697 ICP_QAT_SUOF_OBJ_NAME_LEN)) { 1698 *obj_ptr = obj_hdr[i].obj_buf; 1699 *obj_size = obj_hdr[i].obj_size; 1700 return 0; 1701 } 1702 } 1703 1704 pr_err("QAT: object %s is not found inside MOF\n", obj_name); 1705 return -EINVAL; 1706 } 1707 1708 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, 1709 struct icp_qat_mof_objhdr *mobj_hdr, 1710 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) 1711 { 1712 u8 *obj; 1713 1714 if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, 1715 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1716 obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; 1717 } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, 1718 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1719 obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; 1720 } else { 1721 pr_err("QAT: unsupported chunk id\n"); 1722 return -EINVAL; 1723 } 1724 mobj_hdr->obj_buf = obj; 1725 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; 1726 mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; 1727 return 0; 1728 } 1729 1730 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) 1731 { 1732 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; 1733 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; 1734 struct icp_qat_mof_obj_hdr *uobj_hdr; 1735 struct icp_qat_mof_obj_hdr *sobj_hdr; 1736 struct icp_qat_mof_objhdr *mobj_hdr; 1737 unsigned int uobj_chunk_num = 0; 1738 unsigned int sobj_chunk_num = 0; 1739 unsigned int *valid_chunk; 1740 int ret, i; 1741 1742 uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; 1743 sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; 1744 if (uobj_hdr) 1745 uobj_chunk_num = uobj_hdr->num_chunks; 1746 if (sobj_hdr) 1747 sobj_chunk_num = sobj_hdr->num_chunks; 1748 1749 mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * 1750 sizeof(*mobj_hdr), GFP_KERNEL); 1751 if (!mobj_hdr) 1752 return -ENOMEM; 1753 1754 mobj_handle->obj_table.obj_hdr = mobj_hdr; 1755 valid_chunk = &mobj_handle->obj_table.num_objs; 1756 uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1757 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); 1758 sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1759 ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); 1760 1761 /* map uof objects */ 1762 for (i = 0; i < uobj_chunk_num; i++) { 1763 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1764 &mobj_hdr[*valid_chunk], 1765 &uobj_chunkhdr[i]); 1766 if (ret) 1767 return ret; 1768 (*valid_chunk)++; 1769 } 1770 1771 /* map suof objects */ 1772 for (i = 0; i < sobj_chunk_num; i++) { 1773 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1774 &mobj_hdr[*valid_chunk], 1775 &sobj_chunkhdr[i]); 1776 if (ret) 1777 return ret; 1778 (*valid_chunk)++; 1779 } 1780 1781 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { 1782 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); 1783 return -EINVAL; 1784 } 1785 return 0; 1786 } 1787 1788 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, 1789 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1790 { 1791 char **sym_str = (char **)&mobj_handle->sym_str; 1792 unsigned int *sym_size = &mobj_handle->sym_size; 1793 struct icp_qat_mof_str_table *str_table_obj; 1794 1795 *sym_size = *(unsigned int *)(uintptr_t) 1796 (mof_chunkhdr->offset + mobj_handle->mof_buf); 1797 *sym_str = (char *)(uintptr_t) 1798 (mobj_handle->mof_buf + mof_chunkhdr->offset + 1799 sizeof(str_table_obj->tab_len)); 1800 } 1801 1802 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, 1803 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1804 { 1805 char *chunk_id = mof_chunkhdr->chunk_id; 1806 1807 if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1808 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); 1809 else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1810 mobj_handle->uobjs_hdr = mobj_handle->mof_buf + 1811 mof_chunkhdr->offset; 1812 else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1813 mobj_handle->sobjs_hdr = mobj_handle->mof_buf + 1814 mof_chunkhdr->offset; 1815 } 1816 1817 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) 1818 { 1819 int maj = mof_hdr->maj_ver & 0xff; 1820 int min = mof_hdr->min_ver & 0xff; 1821 1822 if (mof_hdr->file_id != ICP_QAT_MOF_FID) { 1823 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); 1824 return -EINVAL; 1825 } 1826 1827 if (mof_hdr->num_chunks <= 0x1) { 1828 pr_err("QAT: MOF chunk amount is incorrect\n"); 1829 return -EINVAL; 1830 } 1831 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { 1832 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", 1833 maj, min); 1834 return -EINVAL; 1835 } 1836 return 0; 1837 } 1838 1839 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, 1840 struct icp_qat_mof_file_hdr *mof_ptr, 1841 u32 mof_size, const char *obj_name, 1842 char **obj_ptr, unsigned int *obj_size) 1843 { 1844 struct icp_qat_mof_chunkhdr *mof_chunkhdr; 1845 unsigned int file_id = mof_ptr->file_id; 1846 struct icp_qat_mof_handle *mobj_handle; 1847 unsigned short chunks_num; 1848 unsigned int i; 1849 int ret; 1850 1851 if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { 1852 if (obj_ptr) 1853 *obj_ptr = (char *)mof_ptr; 1854 if (obj_size) 1855 *obj_size = mof_size; 1856 return 0; 1857 } 1858 if (qat_uclo_check_mof_format(mof_ptr)) 1859 return -EINVAL; 1860 1861 mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); 1862 if (!mobj_handle) 1863 return -ENOMEM; 1864 1865 handle->mobj_handle = mobj_handle; 1866 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); 1867 if (ret) 1868 return ret; 1869 1870 mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); 1871 chunks_num = mof_ptr->num_chunks; 1872 1873 /* Parse MOF file chunks */ 1874 for (i = 0; i < chunks_num; i++) 1875 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); 1876 1877 /* All sym_objs uobjs and sobjs should be available */ 1878 if (!mobj_handle->sym_str || 1879 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) 1880 return -EINVAL; 1881 1882 ret = qat_uclo_map_objs_from_mof(mobj_handle); 1883 if (ret) 1884 return ret; 1885 1886 /* Seek specified uof object in MOF */ 1887 return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, 1888 obj_ptr, obj_size); 1889 } 1890 1891 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, 1892 void *addr_ptr, u32 mem_size, const char *obj_name) 1893 { 1894 char *obj_addr; 1895 u32 obj_size; 1896 int ret; 1897 1898 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= 1899 (sizeof(handle->hal_handle->ae_mask) * 8)); 1900 1901 if (!handle || !addr_ptr || mem_size < 24) 1902 return -EINVAL; 1903 1904 if (obj_name) { 1905 ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, 1906 &obj_addr, &obj_size); 1907 if (ret) 1908 return ret; 1909 } else { 1910 obj_addr = addr_ptr; 1911 obj_size = mem_size; 1912 } 1913 1914 return (handle->chip_info->fw_auth) ? 1915 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : 1916 qat_uclo_map_uof_obj(handle, obj_addr, obj_size); 1917 } 1918 1919 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) 1920 { 1921 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1922 unsigned int a; 1923 1924 if (handle->mobj_handle) 1925 qat_uclo_del_mof(handle); 1926 if (handle->sobj_handle) 1927 qat_uclo_del_suof(handle); 1928 if (!obj_handle) 1929 return; 1930 1931 kfree(obj_handle->uword_buf); 1932 for (a = 0; a < obj_handle->uimage_num; a++) 1933 kfree(obj_handle->ae_uimage[a].page); 1934 1935 for (a = 0; a < handle->hal_handle->ae_max_num; a++) 1936 qat_uclo_free_ae_data(&obj_handle->ae_data[a]); 1937 1938 kfree(obj_handle->obj_hdr); 1939 kfree(obj_handle->obj_buf); 1940 kfree(obj_handle); 1941 handle->obj_handle = NULL; 1942 } 1943 1944 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, 1945 struct icp_qat_uclo_encap_page *encap_page, 1946 u64 *uword, unsigned int addr_p, 1947 unsigned int raddr, u64 fill) 1948 { 1949 unsigned int i, addr; 1950 u64 uwrd = 0; 1951 1952 if (!encap_page) { 1953 *uword = fill; 1954 return; 1955 } 1956 addr = (encap_page->page_region) ? raddr : addr_p; 1957 for (i = 0; i < encap_page->uwblock_num; i++) { 1958 if (addr >= encap_page->uwblock[i].start_addr && 1959 addr <= encap_page->uwblock[i].start_addr + 1960 encap_page->uwblock[i].words_num - 1) { 1961 addr -= encap_page->uwblock[i].start_addr; 1962 addr *= obj_handle->uword_in_bytes; 1963 memcpy(&uwrd, (void *)(((uintptr_t) 1964 encap_page->uwblock[i].micro_words) + addr), 1965 obj_handle->uword_in_bytes); 1966 uwrd = uwrd & GENMASK_ULL(43, 0); 1967 } 1968 } 1969 *uword = uwrd; 1970 if (*uword == INVLD_UWORD) 1971 *uword = fill; 1972 } 1973 1974 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, 1975 struct icp_qat_uclo_encap_page 1976 *encap_page, unsigned int ae) 1977 { 1978 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; 1979 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1980 u64 fill_pat; 1981 1982 /* load the page starting at appropriate ustore address */ 1983 /* get fill-pattern from an image -- they are all the same */ 1984 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, 1985 sizeof(u64)); 1986 uw_physical_addr = encap_page->beg_addr_p; 1987 uw_relative_addr = 0; 1988 words_num = encap_page->micro_words_num; 1989 while (words_num) { 1990 cpylen = min(words_num, UWORD_CPYBUF_SIZE); 1991 1992 /* load the buffer */ 1993 for (i = 0; i < cpylen; i++) 1994 qat_uclo_fill_uwords(obj_handle, encap_page, 1995 &obj_handle->uword_buf[i], 1996 uw_physical_addr + i, 1997 uw_relative_addr + i, fill_pat); 1998 1999 /* copy the buffer to ustore */ 2000 qat_hal_wr_uwords(handle, (unsigned char)ae, 2001 uw_physical_addr, cpylen, 2002 obj_handle->uword_buf); 2003 2004 uw_physical_addr += cpylen; 2005 uw_relative_addr += cpylen; 2006 words_num -= cpylen; 2007 } 2008 } 2009 2010 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, 2011 struct icp_qat_uof_image *image) 2012 { 2013 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2014 unsigned long ae_mask = handle->hal_handle->ae_mask; 2015 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 2016 unsigned long ae_assigned = image->ae_assigned; 2017 struct icp_qat_uclo_aedata *aed; 2018 unsigned int ctx_mask, s; 2019 struct icp_qat_uclo_page *page; 2020 unsigned char ae; 2021 int ctx; 2022 2023 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) 2024 ctx_mask = 0xff; 2025 else 2026 ctx_mask = 0x55; 2027 /* load the default page and set assigned CTX PC 2028 * to the entrypoint address */ 2029 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 2030 if (!test_bit(ae, &cfg_ae_mask)) 2031 continue; 2032 2033 if (!test_bit(ae, &ae_assigned)) 2034 continue; 2035 2036 aed = &obj_handle->ae_data[ae]; 2037 /* find the slice to which this image is assigned */ 2038 for (s = 0; s < aed->slice_num; s++) { 2039 if (image->ctx_assigned & 2040 aed->ae_slices[s].ctx_mask_assigned) 2041 break; 2042 } 2043 if (s >= aed->slice_num) 2044 continue; 2045 page = aed->ae_slices[s].page; 2046 if (!page->encap_page->def_page) 2047 continue; 2048 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); 2049 2050 page = aed->ae_slices[s].page; 2051 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) 2052 aed->ae_slices[s].cur_page[ctx] = 2053 (ctx_mask & (1 << ctx)) ? page : NULL; 2054 qat_hal_set_live_ctx(handle, (unsigned char)ae, 2055 image->ctx_assigned); 2056 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, 2057 image->entry_address); 2058 } 2059 } 2060 2061 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) 2062 { 2063 unsigned int i; 2064 struct icp_qat_fw_auth_desc *desc = NULL; 2065 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 2066 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; 2067 int ret; 2068 2069 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { 2070 ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, 2071 simg_hdr[i].simg_len, 2072 CSS_AE_FIRMWARE); 2073 if (ret) 2074 return ret; 2075 2076 if (qat_uclo_map_auth_fw(handle, 2077 (char *)simg_hdr[i].simg_buf, 2078 (unsigned int) 2079 simg_hdr[i].simg_len, 2080 &desc)) 2081 goto wr_err; 2082 if (qat_uclo_auth_fw(handle, desc)) 2083 goto wr_err; 2084 if (qat_uclo_is_broadcast(handle, i)) { 2085 if (qat_uclo_broadcast_load_fw(handle, desc)) 2086 goto wr_err; 2087 } else { 2088 if (qat_uclo_load_fw(handle, desc)) 2089 goto wr_err; 2090 } 2091 qat_uclo_ummap_auth_fw(handle, &desc); 2092 } 2093 return 0; 2094 wr_err: 2095 qat_uclo_ummap_auth_fw(handle, &desc); 2096 return -EINVAL; 2097 } 2098 2099 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) 2100 { 2101 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2102 unsigned int i; 2103 2104 if (qat_uclo_init_globals(handle)) 2105 return -EINVAL; 2106 for (i = 0; i < obj_handle->uimage_num; i++) { 2107 if (!obj_handle->ae_uimage[i].img_ptr) 2108 return -EINVAL; 2109 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) 2110 return -EINVAL; 2111 qat_uclo_wr_uimage_page(handle, 2112 obj_handle->ae_uimage[i].img_ptr); 2113 } 2114 return 0; 2115 } 2116 2117 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) 2118 { 2119 return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : 2120 qat_uclo_wr_uof_img(handle); 2121 } 2122 2123 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, 2124 unsigned int cfg_ae_mask) 2125 { 2126 if (!cfg_ae_mask) 2127 return -EINVAL; 2128 2129 handle->cfg_ae_mask = cfg_ae_mask; 2130 return 0; 2131 } 2132