1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_transport_access_macros.h" 12 #include "adf_transport_internal.h" 13 #include <sys/ctype.h> 14 #include <sys/kernel.h> 15 #include <linux/delay.h> 16 #include "adf_accel_devices.h" 17 #include "adf_common_drv.h" 18 #include "icp_qat_uclo.h" 19 #include "icp_qat_hal.h" 20 #include "icp_qat_fw_loader_handle.h" 21 22 #define UWORD_CPYBUF_SIZE 1024 23 #define INVLD_UWORD 0xffffffffffull 24 #define PID_MINOR_REV 0xf 25 #define PID_MAJOR_REV (0xf << 4) 26 #define MAX_UINT32_VAL 0xfffffffful 27 28 static int 29 qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, 30 unsigned int ae, 31 unsigned int image_num) 32 { 33 struct icp_qat_uclo_aedata *ae_data; 34 struct icp_qat_uclo_encapme *encap_image; 35 struct icp_qat_uclo_page *page = NULL; 36 struct icp_qat_uclo_aeslice *ae_slice = NULL; 37 38 ae_data = &obj_handle->ae_data[ae]; 39 encap_image = &obj_handle->ae_uimage[image_num]; 40 ae_slice = &ae_data->ae_slices[ae_data->slice_num]; 41 ae_slice->encap_image = encap_image; 42 43 if (encap_image->img_ptr) { 44 ae_slice->ctx_mask_assigned = 45 encap_image->img_ptr->ctx_assigned; 46 ae_data->shareable_ustore = 47 ICP_QAT_SHARED_USTORE_MODE(encap_image->img_ptr->ae_mode); 48 if (obj_handle->prod_type == ICP_QAT_AC_4XXX_A_DEV_TYPE) 49 ae_data->eff_ustore_size = obj_handle->ustore_phy_size; 50 else { 51 ae_data->eff_ustore_size = ae_data->shareable_ustore ? 52 (obj_handle->ustore_phy_size << 1) : 53 obj_handle->ustore_phy_size; 54 } 55 } else { 56 ae_slice->ctx_mask_assigned = 0; 57 } 58 ae_slice->region = 59 malloc(sizeof(*ae_slice->region), M_QAT, M_WAITOK | M_ZERO); 60 ae_slice->page = 61 malloc(sizeof(*ae_slice->page), M_QAT, M_WAITOK | M_ZERO); 62 page = ae_slice->page; 63 page->encap_page = encap_image->page; 64 ae_slice->page->region = ae_slice->region; 65 ae_data->slice_num++; 66 return 0; 67 } 68 69 static int 70 qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) 71 { 72 unsigned int i; 73 74 if (!ae_data) { 75 pr_err("QAT: bad argument, ae_data is NULL\n "); 76 return EINVAL; 77 } 78 79 for (i = 0; i < ae_data->slice_num; i++) { 80 free(ae_data->ae_slices[i].region, M_QAT); 81 ae_data->ae_slices[i].region = NULL; 82 free(ae_data->ae_slices[i].page, M_QAT); 83 ae_data->ae_slices[i].page = NULL; 84 } 85 return 0; 86 } 87 88 static char * 89 qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, 90 unsigned int str_offset) 91 { 92 if (!str_table->table_len || str_offset > str_table->table_len) 93 return NULL; 94 return (char *)(((uintptr_t)(str_table->strings)) + str_offset); 95 } 96 97 static int 98 qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) 99 { 100 int maj = hdr->maj_ver & 0xff; 101 int min = hdr->min_ver & 0xff; 102 103 if (hdr->file_id != ICP_QAT_UOF_FID) { 104 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); 105 return EINVAL; 106 } 107 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { 108 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", 109 maj, 110 min); 111 return EINVAL; 112 } 113 return 0; 114 } 115 116 static int 117 qat_uclo_check_suof_format(const struct icp_qat_suof_filehdr *suof_hdr) 118 { 119 int maj = suof_hdr->maj_ver & 0xff; 120 int min = suof_hdr->min_ver & 0xff; 121 122 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { 123 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); 124 return EINVAL; 125 } 126 if (suof_hdr->fw_type != 0) { 127 pr_err("QAT: unsupported firmware type\n"); 128 return EINVAL; 129 } 130 if (suof_hdr->num_chunks <= 0x1) { 131 pr_err("QAT: SUOF chunk amount is incorrect\n"); 132 return EINVAL; 133 } 134 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { 135 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", 136 maj, 137 min); 138 return EINVAL; 139 } 140 return 0; 141 } 142 143 static int 144 qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, 145 unsigned int addr, 146 const unsigned int *val, 147 unsigned int num_in_bytes) 148 { 149 unsigned int outval; 150 const unsigned char *ptr = (const unsigned char *)val; 151 152 if (num_in_bytes > handle->hal_sram_size) { 153 pr_err("QAT: error, mmp size overflow %d\n", num_in_bytes); 154 return EINVAL; 155 } 156 while (num_in_bytes) { 157 memcpy(&outval, ptr, 4); 158 SRAM_WRITE(handle, addr, outval); 159 num_in_bytes -= 4; 160 ptr += 4; 161 addr += 4; 162 } 163 return 0; 164 } 165 166 static void 167 qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, 168 unsigned char ae, 169 unsigned int addr, 170 unsigned int *val, 171 unsigned int num_in_bytes) 172 { 173 unsigned int outval; 174 unsigned char *ptr = (unsigned char *)val; 175 176 addr >>= 0x2; /* convert to uword address */ 177 178 while (num_in_bytes) { 179 memcpy(&outval, ptr, 4); 180 qat_hal_wr_umem(handle, ae, addr++, 1, &outval); 181 num_in_bytes -= 4; 182 ptr += 4; 183 } 184 } 185 186 static void 187 qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, 188 unsigned char ae, 189 struct icp_qat_uof_batch_init *umem_init_header) 190 { 191 struct icp_qat_uof_batch_init *umem_init; 192 193 if (!umem_init_header) 194 return; 195 umem_init = umem_init_header->next; 196 while (umem_init) { 197 unsigned int addr, *value, size; 198 199 ae = umem_init->ae; 200 addr = umem_init->addr; 201 value = umem_init->value; 202 size = umem_init->size; 203 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); 204 umem_init = umem_init->next; 205 } 206 } 207 208 static void 209 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, 210 struct icp_qat_uof_batch_init **base) 211 { 212 struct icp_qat_uof_batch_init *umem_init; 213 214 umem_init = *base; 215 while (umem_init) { 216 struct icp_qat_uof_batch_init *pre; 217 218 pre = umem_init; 219 umem_init = umem_init->next; 220 free(pre, M_QAT); 221 } 222 *base = NULL; 223 } 224 225 static int 226 qat_uclo_parse_num(char *str, unsigned int *num) 227 { 228 char buf[16] = { 0 }; 229 unsigned long ae = 0; 230 int i; 231 232 strncpy(buf, str, 15); 233 for (i = 0; i < 16; i++) { 234 if (!isdigit(buf[i])) { 235 buf[i] = '\0'; 236 break; 237 } 238 } 239 if ((compat_strtoul(buf, 10, &ae))) 240 return EFAULT; 241 242 if (ae > MAX_UINT32_VAL) 243 return EFAULT; 244 245 *num = (unsigned int)ae; 246 return 0; 247 } 248 249 static int 250 qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, 251 struct icp_qat_uof_initmem *init_mem, 252 unsigned int size_range, 253 unsigned int *ae) 254 { 255 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 256 char *str; 257 258 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { 259 pr_err("QAT: initmem is out of range"); 260 return EINVAL; 261 } 262 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { 263 pr_err("QAT: Memory scope for init_mem error\n"); 264 return EINVAL; 265 } 266 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); 267 if (!str) { 268 pr_err("QAT: AE name assigned in UOF init table is NULL\n"); 269 return EINVAL; 270 } 271 if (qat_uclo_parse_num(str, ae)) { 272 pr_err("QAT: Parse num for AE number failed\n"); 273 return EINVAL; 274 } 275 if (*ae >= ICP_QAT_UCLO_MAX_AE) { 276 pr_err("QAT: ae %d out of range\n", *ae); 277 return EINVAL; 278 } 279 return 0; 280 } 281 282 static int 283 qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle *handle, 284 struct icp_qat_uof_initmem *init_mem, 285 unsigned int ae, 286 struct icp_qat_uof_batch_init **init_tab_base) 287 { 288 struct icp_qat_uof_batch_init *init_header, *tail; 289 struct icp_qat_uof_batch_init *mem_init, *tail_old; 290 struct icp_qat_uof_memvar_attr *mem_val_attr; 291 unsigned int i = 0; 292 293 mem_val_attr = 294 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + 295 sizeof( 296 struct icp_qat_uof_initmem)); 297 298 init_header = *init_tab_base; 299 if (!init_header) { 300 init_header = 301 malloc(sizeof(*init_header), M_QAT, M_WAITOK | M_ZERO); 302 init_header->size = 1; 303 *init_tab_base = init_header; 304 } 305 tail_old = init_header; 306 while (tail_old->next) 307 tail_old = tail_old->next; 308 tail = tail_old; 309 for (i = 0; i < init_mem->val_attr_num; i++) { 310 mem_init = malloc(sizeof(*mem_init), M_QAT, M_WAITOK | M_ZERO); 311 mem_init->ae = ae; 312 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; 313 mem_init->value = &mem_val_attr->value; 314 mem_init->size = 4; 315 mem_init->next = NULL; 316 tail->next = mem_init; 317 tail = mem_init; 318 init_header->size += qat_hal_get_ins_num(); 319 mem_val_attr++; 320 } 321 return 0; 322 } 323 324 static int 325 qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, 326 struct icp_qat_uof_initmem *init_mem) 327 { 328 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 329 unsigned int ae; 330 unsigned int lmem; 331 332 lmem = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))) ? 333 ICP_QAT_UCLO_MAX_LMEM_REG_2X : 334 ICP_QAT_UCLO_MAX_LMEM_REG; 335 336 if (qat_uclo_fetch_initmem_ae(handle, init_mem, lmem, &ae)) 337 return EINVAL; 338 if (qat_uclo_create_batch_init_list( 339 handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) 340 return EINVAL; 341 return 0; 342 } 343 344 static int 345 qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, 346 struct icp_qat_uof_initmem *init_mem) 347 { 348 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 349 unsigned int ae, ustore_size, uaddr, i; 350 struct icp_qat_uclo_aedata *aed; 351 352 ustore_size = obj_handle->ustore_phy_size; 353 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) 354 return EINVAL; 355 if (qat_uclo_create_batch_init_list( 356 handle, init_mem, ae, &obj_handle->umem_init_tab[ae])) 357 return EINVAL; 358 /* set the highest ustore address referenced */ 359 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; 360 aed = &obj_handle->ae_data[ae]; 361 for (i = 0; i < aed->slice_num; i++) { 362 if (aed->ae_slices[i].encap_image->uwords_num < uaddr) 363 aed->ae_slices[i].encap_image->uwords_num = uaddr; 364 } 365 return 0; 366 } 367 368 #define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000 369 static int 370 qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, 371 struct icp_qat_uof_initmem *init_mem) 372 { 373 switch (init_mem->region) { 374 case ICP_QAT_UOF_LMEM_REGION: 375 if (qat_uclo_init_lmem_seg(handle, init_mem)) 376 return EINVAL; 377 break; 378 case ICP_QAT_UOF_UMEM_REGION: 379 if (qat_uclo_init_umem_seg(handle, init_mem)) 380 return EINVAL; 381 break; 382 default: 383 pr_err("QAT: initmem region error. region type=0x%x\n", 384 init_mem->region); 385 return EINVAL; 386 } 387 return 0; 388 } 389 390 static int 391 qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, 392 struct icp_qat_uclo_encapme *image) 393 { 394 unsigned int i; 395 struct icp_qat_uclo_encap_page *page; 396 struct icp_qat_uof_image *uof_image; 397 unsigned char ae = 0; 398 unsigned char neigh_ae; 399 unsigned int ustore_size; 400 unsigned int patt_pos; 401 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 402 uint64_t *fill_data; 403 static unsigned int init[32] = { 0 }; 404 unsigned long ae_mask = handle->hal_handle->ae_mask; 405 406 uof_image = image->img_ptr; 407 /*if shared CS mode, the ustore size should be 2*ustore_phy_size*/ 408 fill_data = malloc(obj_handle->ustore_phy_size * 2 * sizeof(uint64_t), 409 M_QAT, 410 M_WAITOK | M_ZERO); 411 for (i = 0; i < obj_handle->ustore_phy_size * 2; i++) 412 memcpy(&fill_data[i], 413 &uof_image->fill_pattern, 414 sizeof(uint64_t)); 415 page = image->page; 416 417 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 418 { 419 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 420 unsigned long ae_assigned = uof_image->ae_assigned; 421 const bool gen4 = 422 IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))); 423 424 if (!test_bit(ae, &cfg_ae_mask)) 425 continue; 426 427 if (!test_bit(ae, &ae_assigned)) 428 continue; 429 430 if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1) && 431 !gen4) { 432 qat_hal_get_scs_neigh_ae(ae, &neigh_ae); 433 434 if (test_bit(neigh_ae, &ae_assigned)) 435 continue; 436 } 437 438 ustore_size = obj_handle->ae_data[ae].eff_ustore_size; 439 patt_pos = page->beg_addr_p + page->micro_words_num; 440 if (obj_handle->ae_data[ae].shareable_ustore && !gen4) { 441 qat_hal_get_scs_neigh_ae(ae, &neigh_ae); 442 if (init[ae] == 0 && page->beg_addr_p != 0) { 443 qat_hal_wr_coalesce_uwords(handle, 444 (unsigned char)ae, 445 0, 446 page->beg_addr_p, 447 &fill_data[0]); 448 } 449 qat_hal_wr_coalesce_uwords( 450 handle, 451 (unsigned char)ae, 452 patt_pos, 453 ustore_size - patt_pos, 454 &fill_data[page->beg_addr_p]); 455 init[ae] = 1; 456 init[neigh_ae] = 1; 457 } else { 458 if (gen4 && (ae % 4 != 0)) 459 continue; 460 461 qat_hal_wr_uwords(handle, 462 (unsigned char)ae, 463 0, 464 page->beg_addr_p, 465 &fill_data[0]); 466 qat_hal_wr_uwords(handle, 467 (unsigned char)ae, 468 patt_pos, 469 ustore_size - patt_pos + 1, 470 &fill_data[page->beg_addr_p]); 471 } 472 } 473 free(fill_data, M_QAT); 474 return 0; 475 } 476 477 static int 478 qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) 479 { 480 int i; 481 int ae = 0; 482 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 483 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; 484 unsigned long ae_mask = handle->hal_handle->ae_mask; 485 486 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { 487 if (initmem->num_in_bytes) { 488 if (qat_uclo_init_ae_memory(handle, initmem)) 489 return EINVAL; 490 } 491 initmem = 492 (struct icp_qat_uof_initmem 493 *)((uintptr_t)((uintptr_t)initmem + 494 sizeof(struct icp_qat_uof_initmem)) + 495 (sizeof(struct icp_qat_uof_memvar_attr) * 496 initmem->val_attr_num)); 497 } 498 499 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 500 { 501 if (qat_hal_batch_wr_lm(handle, 502 ae, 503 obj_handle->lm_init_tab[ae])) { 504 pr_err("QAT: fail to batch init lmem for AE %d\n", ae); 505 return EINVAL; 506 } 507 qat_uclo_cleanup_batch_init_list(handle, 508 &obj_handle->lm_init_tab[ae]); 509 qat_uclo_batch_wr_umem(handle, 510 ae, 511 obj_handle->umem_init_tab[ae]); 512 qat_uclo_cleanup_batch_init_list( 513 handle, &obj_handle->umem_init_tab[ae]); 514 } 515 return 0; 516 } 517 518 static void * 519 qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, 520 char *chunk_id, 521 void *cur) 522 { 523 int i; 524 struct icp_qat_uof_chunkhdr *chunk_hdr = 525 (struct icp_qat_uof_chunkhdr *)((uintptr_t)obj_hdr + 526 sizeof(struct icp_qat_uof_objhdr)); 527 528 for (i = 0; i < obj_hdr->num_chunks; i++) { 529 if ((cur < (void *)&chunk_hdr[i]) && 530 !strncmp(chunk_hdr[i].chunk_id, 531 chunk_id, 532 ICP_QAT_UOF_OBJID_LEN)) { 533 return &chunk_hdr[i]; 534 } 535 } 536 return NULL; 537 } 538 539 static unsigned int 540 qat_uclo_calc_checksum(unsigned int reg, int ch) 541 { 542 int i; 543 unsigned int topbit = 1 << 0xF; 544 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); 545 546 reg ^= inbyte << 0x8; 547 for (i = 0; i < 0x8; i++) { 548 if (reg & topbit) 549 reg = (reg << 1) ^ 0x1021; 550 else 551 reg <<= 1; 552 } 553 return reg & 0xFFFF; 554 } 555 556 static unsigned int 557 qat_uclo_calc_str_checksum(const char *ptr, int num) 558 { 559 unsigned int chksum = 0; 560 561 if (ptr) 562 while (num--) 563 chksum = qat_uclo_calc_checksum(chksum, *ptr++); 564 return chksum; 565 } 566 567 static struct icp_qat_uclo_objhdr * 568 qat_uclo_map_chunk(char *buf, 569 struct icp_qat_uof_filehdr *file_hdr, 570 char *chunk_id) 571 { 572 struct icp_qat_uof_filechunkhdr *file_chunk; 573 struct icp_qat_uclo_objhdr *obj_hdr; 574 char *chunk; 575 int i; 576 577 file_chunk = (struct icp_qat_uof_filechunkhdr 578 *)(buf + sizeof(struct icp_qat_uof_filehdr)); 579 for (i = 0; i < file_hdr->num_chunks; i++) { 580 if (!strncmp(file_chunk->chunk_id, 581 chunk_id, 582 ICP_QAT_UOF_OBJID_LEN)) { 583 chunk = buf + file_chunk->offset; 584 if (file_chunk->checksum != 585 qat_uclo_calc_str_checksum(chunk, file_chunk->size)) 586 break; 587 obj_hdr = 588 malloc(sizeof(*obj_hdr), M_QAT, M_WAITOK | M_ZERO); 589 obj_hdr->file_buff = chunk; 590 obj_hdr->checksum = file_chunk->checksum; 591 obj_hdr->size = file_chunk->size; 592 return obj_hdr; 593 } 594 file_chunk++; 595 } 596 return NULL; 597 } 598 599 static unsigned int 600 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, 601 struct icp_qat_uof_image *image) 602 { 603 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; 604 struct icp_qat_uof_objtable *neigh_reg_tab; 605 struct icp_qat_uof_code_page *code_page; 606 607 code_page = 608 (struct icp_qat_uof_code_page *)((char *)image + 609 sizeof(struct icp_qat_uof_image)); 610 uc_var_tab = 611 (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 612 code_page->uc_var_tab_offset); 613 imp_var_tab = 614 (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 615 code_page->imp_var_tab_offset); 616 imp_expr_tab = 617 (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 618 code_page->imp_expr_tab_offset); 619 if (uc_var_tab->entry_num || imp_var_tab->entry_num || 620 imp_expr_tab->entry_num) { 621 pr_err("QAT: UOF can't contain imported variable to be parsed"); 622 return EINVAL; 623 } 624 neigh_reg_tab = 625 (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 626 code_page->neigh_reg_tab_offset); 627 if (neigh_reg_tab->entry_num) { 628 pr_err("QAT: UOF can't contain neighbor register table\n"); 629 return EINVAL; 630 } 631 if (image->numpages > 1) { 632 pr_err("QAT: UOF can't contain multiple pages\n"); 633 return EINVAL; 634 } 635 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { 636 pr_err("QAT: UOF can't use reloadable feature\n"); 637 return EFAULT; 638 } 639 return 0; 640 } 641 642 static void 643 qat_uclo_map_image_page(struct icp_qat_uof_encap_obj *encap_uof_obj, 644 struct icp_qat_uof_image *img, 645 struct icp_qat_uclo_encap_page *page) 646 { 647 struct icp_qat_uof_code_page *code_page; 648 struct icp_qat_uof_code_area *code_area; 649 struct icp_qat_uof_objtable *uword_block_tab; 650 struct icp_qat_uof_uword_block *uwblock; 651 int i; 652 653 code_page = 654 (struct icp_qat_uof_code_page *)((char *)img + 655 sizeof(struct icp_qat_uof_image)); 656 page->def_page = code_page->def_page; 657 page->page_region = code_page->page_region; 658 page->beg_addr_v = code_page->beg_addr_v; 659 page->beg_addr_p = code_page->beg_addr_p; 660 code_area = 661 (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + 662 code_page->code_area_offset); 663 page->micro_words_num = code_area->micro_words_num; 664 uword_block_tab = 665 (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 666 code_area->uword_block_tab); 667 page->uwblock_num = uword_block_tab->entry_num; 668 uwblock = (struct icp_qat_uof_uword_block 669 *)((char *)uword_block_tab + 670 sizeof(struct icp_qat_uof_objtable)); 671 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; 672 for (i = 0; i < uword_block_tab->entry_num; i++) 673 page->uwblock[i].micro_words = 674 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; 675 } 676 677 static int 678 qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, 679 struct icp_qat_uclo_encapme *ae_uimage, 680 int max_image) 681 { 682 int i, j; 683 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; 684 struct icp_qat_uof_image *image; 685 struct icp_qat_uof_objtable *ae_regtab; 686 struct icp_qat_uof_objtable *init_reg_sym_tab; 687 struct icp_qat_uof_objtable *sbreak_tab; 688 struct icp_qat_uof_encap_obj *encap_uof_obj = 689 &obj_handle->encap_uof_obj; 690 691 for (j = 0; j < max_image; j++) { 692 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 693 ICP_QAT_UOF_IMAG, 694 chunk_hdr); 695 if (!chunk_hdr) 696 break; 697 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + 698 chunk_hdr->offset); 699 ae_regtab = 700 (struct icp_qat_uof_objtable *)(image->reg_tab_offset + 701 obj_handle->obj_hdr 702 ->file_buff); 703 ae_uimage[j].ae_reg_num = ae_regtab->entry_num; 704 ae_uimage[j].ae_reg = 705 (struct icp_qat_uof_ae_reg 706 *)(((char *)ae_regtab) + 707 sizeof(struct icp_qat_uof_objtable)); 708 init_reg_sym_tab = 709 (struct icp_qat_uof_objtable *)(image->init_reg_sym_tab + 710 obj_handle->obj_hdr 711 ->file_buff); 712 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; 713 ae_uimage[j].init_regsym = 714 (struct icp_qat_uof_init_regsym 715 *)(((char *)init_reg_sym_tab) + 716 sizeof(struct icp_qat_uof_objtable)); 717 sbreak_tab = (struct icp_qat_uof_objtable *)(image->sbreak_tab + 718 obj_handle->obj_hdr 719 ->file_buff); 720 ae_uimage[j].sbreak_num = sbreak_tab->entry_num; 721 ae_uimage[j].sbreak = 722 (struct icp_qat_uof_sbreak 723 *)(((char *)sbreak_tab) + 724 sizeof(struct icp_qat_uof_objtable)); 725 ae_uimage[j].img_ptr = image; 726 if (qat_uclo_check_image_compat(encap_uof_obj, image)) 727 goto out_err; 728 ae_uimage[j].page = 729 malloc(sizeof(struct icp_qat_uclo_encap_page), 730 M_QAT, 731 M_WAITOK | M_ZERO); 732 qat_uclo_map_image_page(encap_uof_obj, 733 image, 734 ae_uimage[j].page); 735 } 736 return j; 737 out_err: 738 for (i = 0; i < j; i++) 739 free(ae_uimage[i].page, M_QAT); 740 return 0; 741 } 742 743 static int 744 UcLo_checkTGroupList2X(struct icp_qat_fw_loader_handle *handle) 745 { 746 int i; 747 unsigned int swAe = 0; 748 unsigned int ii, jj; 749 struct icp_qat_uclo_aedata *ae_data0, *ae_datax; 750 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 751 752 for (i = 0; i < obj_handle->uimage_num; i++) { 753 struct icp_qat_uof_image *image = 754 obj_handle->ae_uimage[i].img_ptr; 755 if (image->numpages > 1) { 756 pr_err( 757 "Only 1 page is allowed in a UOF for CPM2X; We found %d in %s\n", 758 image->numpages, 759 qat_uclo_get_string(&obj_handle->str_table, 760 image->img_name)); 761 return EINVAL; 762 } 763 } 764 765 for (swAe = 0; 766 (swAe < obj_handle->ae_num) && (swAe < ICP_QAT_UCLO_MAX_AE); 767 swAe += AE_TG_NUM_CPM2X) { 768 if (!qat_hal_check_ae_active(handle, swAe)) { 769 continue; 770 } 771 772 for (ii = swAe; ii < (swAe + AE_TG_NUM_CPM2X); ii++) { 773 ae_data0 = &obj_handle->ae_data[ii]; 774 if (ae_data0->slice_num != 1) // not assigned 775 continue; 776 777 for (jj = ii + 1; jj < (swAe + AE_TG_NUM_CPM2X); jj++) { 778 ae_datax = &obj_handle->ae_data[jj]; 779 if (ae_datax->slice_num != 1) // not assigned 780 continue; 781 if (ae_data0->ae_slices[0] 782 .encap_image->img_ptr != 783 ae_datax->ae_slices[0] 784 .encap_image->img_ptr) { 785 pr_err("Only 1 list is allowed in a "); 786 pr_err("Tgroup for CPM2X;\n"); 787 pr_err("ME%d, %d is assigned", ii, jj); 788 pr_err(" different list files\n"); 789 return EINVAL; 790 } 791 } 792 } 793 } 794 795 return 0; 796 } 797 798 static int 799 qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) 800 { 801 int i; 802 int ae = 0; 803 unsigned long ae_mask = handle->hal_handle->ae_mask; 804 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 805 int mflag = 0; 806 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 807 808 for_each_set_bit(ae, &ae_mask, max_ae) 809 { 810 if (!test_bit(ae, &cfg_ae_mask)) 811 continue; 812 813 for (i = 0; i < obj_handle->uimage_num; i++) { 814 unsigned long ae_assigned = 815 obj_handle->ae_uimage[i].img_ptr->ae_assigned; 816 if (!test_bit(ae, &ae_assigned)) 817 continue; 818 mflag = 1; 819 if (qat_uclo_init_ae_data(obj_handle, ae, i)) 820 return EINVAL; 821 } 822 } 823 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 824 if (UcLo_checkTGroupList2X(handle)) { 825 return EINVAL; 826 } 827 } 828 if (!mflag) { 829 pr_err("QAT: uimage uses AE not set"); 830 return EINVAL; 831 } 832 return 0; 833 } 834 835 static struct icp_qat_uof_strtable * 836 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, 837 char *tab_name, 838 struct icp_qat_uof_strtable *str_table) 839 { 840 struct icp_qat_uof_chunkhdr *chunk_hdr; 841 842 chunk_hdr = 843 qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)obj_hdr->file_buff, 844 tab_name, 845 NULL); 846 if (chunk_hdr) { 847 int hdr_size; 848 849 memcpy(&str_table->table_len, 850 obj_hdr->file_buff + chunk_hdr->offset, 851 sizeof(str_table->table_len)); 852 hdr_size = (char *)&str_table->strings - (char *)str_table; 853 str_table->strings = (uintptr_t)obj_hdr->file_buff + 854 chunk_hdr->offset + hdr_size; 855 return str_table; 856 } 857 return NULL; 858 } 859 860 static void 861 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, 862 struct icp_qat_uclo_init_mem_table *init_mem_tab) 863 { 864 struct icp_qat_uof_chunkhdr *chunk_hdr; 865 866 chunk_hdr = 867 qat_uclo_find_chunk(encap_uof_obj->obj_hdr, ICP_QAT_UOF_IMEM, NULL); 868 if (chunk_hdr) { 869 memmove(&init_mem_tab->entry_num, 870 encap_uof_obj->beg_uof + chunk_hdr->offset, 871 sizeof(unsigned int)); 872 init_mem_tab->init_mem = 873 (struct icp_qat_uof_initmem *)(encap_uof_obj->beg_uof + 874 chunk_hdr->offset + 875 sizeof(unsigned int)); 876 } 877 } 878 879 static unsigned int 880 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) 881 { 882 switch (pci_get_device(GET_DEV(handle->accel_dev))) { 883 case ADF_DH895XCC_PCI_DEVICE_ID: 884 return ICP_QAT_AC_895XCC_DEV_TYPE; 885 case ADF_C62X_PCI_DEVICE_ID: 886 return ICP_QAT_AC_C62X_DEV_TYPE; 887 case ADF_C3XXX_PCI_DEVICE_ID: 888 return ICP_QAT_AC_C3XXX_DEV_TYPE; 889 case ADF_200XX_PCI_DEVICE_ID: 890 return ICP_QAT_AC_200XX_DEV_TYPE; 891 case ADF_C4XXX_PCI_DEVICE_ID: 892 return ICP_QAT_AC_C4XXX_DEV_TYPE; 893 case ADF_4XXX_PCI_DEVICE_ID: 894 case ADF_401XX_PCI_DEVICE_ID: 895 return ICP_QAT_AC_4XXX_A_DEV_TYPE; 896 default: 897 pr_err("QAT: unsupported device 0x%x\n", 898 pci_get_device(GET_DEV(handle->accel_dev))); 899 return 0; 900 } 901 } 902 903 static int 904 qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) 905 { 906 unsigned int maj_ver, prod_type = obj_handle->prod_type; 907 908 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { 909 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", 910 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, 911 prod_type); 912 return EINVAL; 913 } 914 maj_ver = obj_handle->prod_rev & 0xff; 915 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || 916 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { 917 pr_err("QAT: UOF maj_ver 0x%x out of range\n", maj_ver); 918 return EINVAL; 919 } 920 return 0; 921 } 922 923 static int 924 qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, 925 unsigned char ae, 926 unsigned char ctx_mask, 927 enum icp_qat_uof_regtype reg_type, 928 unsigned short reg_addr, 929 unsigned int value) 930 { 931 switch (reg_type) { 932 case ICP_GPA_ABS: 933 case ICP_GPB_ABS: 934 ctx_mask = 0; 935 return qat_hal_init_gpr( 936 handle, ae, ctx_mask, reg_type, reg_addr, value); 937 case ICP_GPA_REL: 938 case ICP_GPB_REL: 939 return qat_hal_init_gpr( 940 handle, ae, ctx_mask, reg_type, reg_addr, value); 941 case ICP_SR_ABS: 942 case ICP_DR_ABS: 943 case ICP_SR_RD_ABS: 944 case ICP_DR_RD_ABS: 945 ctx_mask = 0; 946 return qat_hal_init_rd_xfer( 947 handle, ae, ctx_mask, reg_type, reg_addr, value); 948 case ICP_SR_REL: 949 case ICP_DR_REL: 950 case ICP_SR_RD_REL: 951 case ICP_DR_RD_REL: 952 return qat_hal_init_rd_xfer( 953 handle, ae, ctx_mask, reg_type, reg_addr, value); 954 case ICP_SR_WR_ABS: 955 case ICP_DR_WR_ABS: 956 ctx_mask = 0; 957 return qat_hal_init_wr_xfer( 958 handle, ae, ctx_mask, reg_type, reg_addr, value); 959 case ICP_SR_WR_REL: 960 case ICP_DR_WR_REL: 961 return qat_hal_init_wr_xfer( 962 handle, ae, ctx_mask, reg_type, reg_addr, value); 963 case ICP_NEIGH_REL: 964 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); 965 default: 966 pr_err("QAT: UOF uses unsupported reg type 0x%x\n", reg_type); 967 return EFAULT; 968 } 969 return 0; 970 } 971 972 static int 973 qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, 974 unsigned int ae, 975 struct icp_qat_uclo_encapme *encap_ae) 976 { 977 unsigned int i; 978 unsigned char ctx_mask; 979 struct icp_qat_uof_init_regsym *init_regsym; 980 981 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == 982 ICP_QAT_UCLO_MAX_CTX) 983 ctx_mask = 0xff; 984 else 985 ctx_mask = 0x55; 986 987 for (i = 0; i < encap_ae->init_regsym_num; i++) { 988 unsigned int exp_res; 989 990 init_regsym = &encap_ae->init_regsym[i]; 991 exp_res = init_regsym->value; 992 switch (init_regsym->init_type) { 993 case ICP_QAT_UOF_INIT_REG: 994 qat_uclo_init_reg(handle, 995 ae, 996 ctx_mask, 997 (enum icp_qat_uof_regtype) 998 init_regsym->reg_type, 999 (unsigned short)init_regsym->reg_addr, 1000 exp_res); 1001 break; 1002 case ICP_QAT_UOF_INIT_REG_CTX: 1003 /* check if ctx is appropriate for the ctxMode */ 1004 if (!((1 << init_regsym->ctx) & ctx_mask)) { 1005 pr_err("QAT: invalid ctx num = 0x%x\n", 1006 init_regsym->ctx); 1007 return EINVAL; 1008 } 1009 qat_uclo_init_reg( 1010 handle, 1011 ae, 1012 (unsigned char)(1 << init_regsym->ctx), 1013 (enum icp_qat_uof_regtype)init_regsym->reg_type, 1014 (unsigned short)init_regsym->reg_addr, 1015 exp_res); 1016 break; 1017 case ICP_QAT_UOF_INIT_EXPR: 1018 pr_err("QAT: INIT_EXPR feature not supported\n"); 1019 return EINVAL; 1020 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: 1021 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP not supported\n"); 1022 return EINVAL; 1023 default: 1024 break; 1025 } 1026 } 1027 return 0; 1028 } 1029 1030 static int 1031 qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) 1032 { 1033 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1034 unsigned int s; 1035 unsigned int ae = 0; 1036 struct icp_qat_uclo_aedata *aed; 1037 unsigned long ae_mask = handle->hal_handle->ae_mask; 1038 1039 if (obj_handle->global_inited) 1040 return 0; 1041 if (obj_handle->init_mem_tab.entry_num) { 1042 if (qat_uclo_init_memory(handle)) { 1043 pr_err("QAT: initialize memory failed\n"); 1044 return EINVAL; 1045 } 1046 } 1047 1048 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1049 { 1050 aed = &obj_handle->ae_data[ae]; 1051 for (s = 0; s < aed->slice_num; s++) { 1052 if (!aed->ae_slices[s].encap_image) 1053 continue; 1054 if (qat_uclo_init_reg_sym( 1055 handle, ae, aed->ae_slices[s].encap_image)) 1056 return EINVAL; 1057 } 1058 } 1059 obj_handle->global_inited = 1; 1060 return 0; 1061 } 1062 1063 static int 1064 qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, 1065 struct icp_qat_uclo_objhandle *obj_handle, 1066 unsigned char ae, 1067 struct icp_qat_uof_image *uof_image) 1068 { 1069 unsigned char nn_mode; 1070 char ae_mode = 0; 1071 1072 ae_mode = (char)ICP_QAT_CTX_MODE(uof_image->ae_mode); 1073 if (qat_hal_set_ae_ctx_mode(handle, ae, ae_mode)) { 1074 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); 1075 return EFAULT; 1076 } 1077 1078 ae_mode = (char)ICP_QAT_SHARED_USTORE_MODE(uof_image->ae_mode); 1079 qat_hal_set_ae_scs_mode(handle, ae, ae_mode); 1080 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1081 nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); 1082 1083 if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { 1084 pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); 1085 return EFAULT; 1086 } 1087 } 1088 ae_mode = (char)ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); 1089 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, ae_mode)) { 1090 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); 1091 return EFAULT; 1092 } 1093 ae_mode = (char)ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); 1094 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, ae_mode)) { 1095 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); 1096 return EFAULT; 1097 } 1098 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1099 ae_mode = (char)ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); 1100 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, ae_mode)) { 1101 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); 1102 return EFAULT; 1103 } 1104 ae_mode = (char)ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); 1105 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, ae_mode)) { 1106 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); 1107 return EFAULT; 1108 } 1109 ae_mode = (char)ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); 1110 qat_hal_set_ae_tindex_mode(handle, ae, ae_mode); 1111 } 1112 return 0; 1113 } 1114 1115 static int 1116 qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) 1117 { 1118 int error; 1119 unsigned char s; 1120 unsigned char ae = 0; 1121 struct icp_qat_uof_image *uof_image; 1122 struct icp_qat_uclo_aedata *ae_data; 1123 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1124 unsigned long ae_mask = handle->hal_handle->ae_mask; 1125 1126 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1127 { 1128 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 1129 1130 if (!test_bit(ae, &cfg_ae_mask)) 1131 continue; 1132 1133 ae_data = &obj_handle->ae_data[ae]; 1134 for (s = 0; s < min_t(unsigned int, 1135 ae_data->slice_num, 1136 ICP_QAT_UCLO_MAX_CTX); 1137 s++) { 1138 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) 1139 continue; 1140 uof_image = ae_data->ae_slices[s].encap_image->img_ptr; 1141 error = qat_hal_set_modes(handle, 1142 obj_handle, 1143 ae, 1144 uof_image); 1145 if (error) 1146 return error; 1147 } 1148 } 1149 return 0; 1150 } 1151 1152 static void 1153 qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) 1154 { 1155 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1156 struct icp_qat_uclo_encapme *image; 1157 int a; 1158 1159 for (a = 0; a < obj_handle->uimage_num; a++) { 1160 image = &obj_handle->ae_uimage[a]; 1161 image->uwords_num = 1162 image->page->beg_addr_p + image->page->micro_words_num; 1163 } 1164 } 1165 1166 static int 1167 qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) 1168 { 1169 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1170 unsigned int ae; 1171 1172 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; 1173 obj_handle->encap_uof_obj.obj_hdr = 1174 (struct icp_qat_uof_objhdr *)obj_handle->obj_hdr->file_buff; 1175 obj_handle->uword_in_bytes = 6; 1176 obj_handle->prod_type = qat_uclo_get_dev_type(handle); 1177 obj_handle->prod_rev = 1178 PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); 1179 if (qat_uclo_check_uof_compat(obj_handle)) { 1180 pr_err("QAT: UOF incompatible\n"); 1181 return EINVAL; 1182 } 1183 obj_handle->uword_buf = malloc(UWORD_CPYBUF_SIZE * sizeof(uint64_t), 1184 M_QAT, 1185 M_WAITOK | M_ZERO); 1186 obj_handle->ustore_phy_size = 1187 (obj_handle->prod_type == ICP_QAT_AC_C4XXX_DEV_TYPE) ? 0x2000 : 1188 0x4000; 1189 if (!obj_handle->obj_hdr->file_buff || 1190 !qat_uclo_map_str_table(obj_handle->obj_hdr, 1191 ICP_QAT_UOF_STRT, 1192 &obj_handle->str_table)) { 1193 pr_err("QAT: UOF doesn't have effective images\n"); 1194 goto out_err; 1195 } 1196 obj_handle->uimage_num = 1197 qat_uclo_map_uimage(obj_handle, 1198 obj_handle->ae_uimage, 1199 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); 1200 if (!obj_handle->uimage_num) 1201 goto out_err; 1202 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { 1203 pr_err("QAT: Bad object\n"); 1204 goto out_check_uof_aemask_err; 1205 } 1206 qat_uclo_init_uword_num(handle); 1207 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, 1208 &obj_handle->init_mem_tab); 1209 if (qat_uclo_set_ae_mode(handle)) 1210 goto out_check_uof_aemask_err; 1211 return 0; 1212 out_check_uof_aemask_err: 1213 for (ae = 0; ae < obj_handle->uimage_num; ae++) 1214 free(obj_handle->ae_uimage[ae].page, M_QAT); 1215 out_err: 1216 free(obj_handle->uword_buf, M_QAT); 1217 obj_handle->uword_buf = NULL; 1218 return EFAULT; 1219 } 1220 1221 static int 1222 qat_uclo_map_suof_file_hdr(const struct icp_qat_fw_loader_handle *handle, 1223 const struct icp_qat_suof_filehdr *suof_ptr, 1224 int suof_size) 1225 { 1226 unsigned int check_sum = 0; 1227 unsigned int min_ver_offset = 0; 1228 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1229 1230 suof_handle->file_id = ICP_QAT_SUOF_FID; 1231 suof_handle->suof_buf = (const char *)suof_ptr; 1232 suof_handle->suof_size = suof_size; 1233 min_ver_offset = 1234 suof_size - offsetof(struct icp_qat_suof_filehdr, min_ver); 1235 check_sum = qat_uclo_calc_str_checksum((const char *)&suof_ptr->min_ver, 1236 min_ver_offset); 1237 if (check_sum != suof_ptr->check_sum) { 1238 pr_err("QAT: incorrect SUOF checksum\n"); 1239 return EINVAL; 1240 } 1241 suof_handle->check_sum = suof_ptr->check_sum; 1242 suof_handle->min_ver = suof_ptr->min_ver; 1243 suof_handle->maj_ver = suof_ptr->maj_ver; 1244 suof_handle->fw_type = suof_ptr->fw_type; 1245 return 0; 1246 } 1247 1248 static void 1249 qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, 1250 struct icp_qat_suof_img_hdr *suof_img_hdr, 1251 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1252 { 1253 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1254 const struct icp_qat_simg_ae_mode *ae_mode; 1255 struct icp_qat_suof_objhdr *suof_objhdr; 1256 unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); 1257 1258 suof_img_hdr->simg_buf = 1259 (suof_handle->suof_buf + suof_chunk_hdr->offset + 1260 sizeof(*suof_objhdr)); 1261 suof_img_hdr->simg_len = 1262 ((struct icp_qat_suof_objhdr *)(uintptr_t)(suof_handle->suof_buf + 1263 suof_chunk_hdr->offset)) 1264 ->img_length; 1265 1266 suof_img_hdr->css_header = suof_img_hdr->simg_buf; 1267 suof_img_hdr->css_key = 1268 (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); 1269 suof_img_hdr->css_signature = suof_img_hdr->css_key + 1270 ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + 1271 ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id); 1272 suof_img_hdr->css_simg = 1273 suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN(device_id); 1274 1275 ae_mode = (const struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); 1276 suof_img_hdr->ae_mask = ae_mode->ae_mask; 1277 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; 1278 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; 1279 suof_img_hdr->fw_type = ae_mode->fw_type; 1280 } 1281 1282 static void 1283 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, 1284 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1285 { 1286 char **sym_str = (char **)&suof_handle->sym_str; 1287 unsigned int *sym_size = &suof_handle->sym_size; 1288 struct icp_qat_suof_strtable *str_table_obj; 1289 1290 *sym_size = *(unsigned int *)(uintptr_t)(suof_chunk_hdr->offset + 1291 suof_handle->suof_buf); 1292 *sym_str = 1293 (char *)(uintptr_t)(suof_handle->suof_buf + suof_chunk_hdr->offset + 1294 sizeof(str_table_obj->tab_length)); 1295 } 1296 1297 static int 1298 qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, 1299 struct icp_qat_suof_img_hdr *img_hdr) 1300 { 1301 const struct icp_qat_simg_ae_mode *img_ae_mode = NULL; 1302 unsigned int prod_rev, maj_ver, prod_type; 1303 1304 prod_type = qat_uclo_get_dev_type(handle); 1305 img_ae_mode = (const struct icp_qat_simg_ae_mode *)img_hdr->css_simg; 1306 prod_rev = 1307 PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); 1308 if (img_ae_mode->dev_type != prod_type) { 1309 pr_err("QAT: incompatible product type %x\n", 1310 img_ae_mode->dev_type); 1311 return EINVAL; 1312 } 1313 maj_ver = prod_rev & 0xff; 1314 if (maj_ver > img_ae_mode->devmax_ver || 1315 maj_ver < img_ae_mode->devmin_ver) { 1316 pr_err("QAT: incompatible device maj_ver 0x%x\n", maj_ver); 1317 return EINVAL; 1318 } 1319 return 0; 1320 } 1321 1322 static void 1323 qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) 1324 { 1325 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 1326 1327 free(sobj_handle->img_table.simg_hdr, M_QAT); 1328 sobj_handle->img_table.simg_hdr = NULL; 1329 free(handle->sobj_handle, M_QAT); 1330 handle->sobj_handle = NULL; 1331 } 1332 1333 static void 1334 qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, 1335 unsigned int img_id, 1336 unsigned int num_simgs) 1337 { 1338 struct icp_qat_suof_img_hdr img_header; 1339 1340 if ((img_id != num_simgs - 1) && img_id != ICP_QAT_UCLO_MAX_AE) { 1341 memcpy(&img_header, 1342 &suof_img_hdr[num_simgs - 1], 1343 sizeof(*suof_img_hdr)); 1344 memcpy(&suof_img_hdr[num_simgs - 1], 1345 &suof_img_hdr[img_id], 1346 sizeof(*suof_img_hdr)); 1347 memcpy(&suof_img_hdr[img_id], 1348 &img_header, 1349 sizeof(*suof_img_hdr)); 1350 } 1351 } 1352 1353 static int 1354 qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, 1355 const struct icp_qat_suof_filehdr *suof_ptr, 1356 int suof_size) 1357 { 1358 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1359 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; 1360 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; 1361 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE, 1362 aeMax_img = ICP_QAT_UCLO_MAX_AE; 1363 unsigned int i = 0; 1364 struct icp_qat_suof_img_hdr img_header; 1365 1366 if (!suof_ptr || suof_size == 0) { 1367 pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); 1368 return EINVAL; 1369 } 1370 if (qat_uclo_check_suof_format(suof_ptr)) 1371 return EINVAL; 1372 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); 1373 if (ret) 1374 return ret; 1375 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)((uintptr_t)suof_ptr + 1376 sizeof(*suof_ptr)); 1377 1378 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); 1379 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; 1380 1381 if (suof_handle->img_table.num_simgs != 0) { 1382 suof_img_hdr = malloc(suof_handle->img_table.num_simgs * 1383 sizeof(img_header), 1384 M_QAT, 1385 M_WAITOK | M_ZERO); 1386 suof_handle->img_table.simg_hdr = suof_img_hdr; 1387 } 1388 1389 for (i = 0; i < suof_handle->img_table.num_simgs; i++) { 1390 qat_uclo_map_simg(handle, 1391 &suof_img_hdr[i], 1392 &suof_chunk_hdr[1 + i]); 1393 ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]); 1394 if (ret) 1395 return ret; 1396 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; 1397 if ((suof_img_hdr[i].ae_mask & 0x1) != 0) 1398 ae0_img = i; 1399 } 1400 1401 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1402 qat_uclo_tail_img(suof_img_hdr, 1403 ae0_img, 1404 suof_handle->img_table.num_simgs); 1405 } else { 1406 if (suof_handle->img_table.num_simgs == 1) 1407 return 0; 1408 qat_uclo_tail_img(suof_img_hdr, 1409 ae0_img, 1410 suof_handle->img_table.num_simgs - 1); 1411 for (i = 0; i < suof_handle->img_table.num_simgs; i++) { 1412 if ((suof_img_hdr[i].ae_mask & 1413 (0x1 << (handle->hal_handle->ae_max_num - 1))) != 1414 0) { 1415 aeMax_img = i; 1416 break; 1417 } 1418 } 1419 qat_uclo_tail_img(suof_img_hdr, 1420 aeMax_img, 1421 suof_handle->img_table.num_simgs); 1422 } 1423 return 0; 1424 } 1425 1426 #define ADD_ADDR(high, low) ((((uint64_t)high) << 32) + (low)) 1427 #define BITS_IN_DWORD 32 1428 1429 static int 1430 qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, 1431 struct icp_qat_fw_auth_desc *desc) 1432 { 1433 unsigned int fcu_sts, mem_cfg_err, retry = 0; 1434 unsigned int fcu_ctl_csr, fcu_sts_csr; 1435 unsigned int fcu_dram_hi_csr, fcu_dram_lo_csr; 1436 u64 bus_addr; 1437 1438 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - 1439 sizeof(struct icp_qat_auth_chunk); 1440 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1441 fcu_ctl_csr = FCU_CONTROL_C4XXX; 1442 fcu_sts_csr = FCU_STATUS_C4XXX; 1443 fcu_dram_hi_csr = FCU_DRAM_ADDR_HI_C4XXX; 1444 fcu_dram_lo_csr = FCU_DRAM_ADDR_LO_C4XXX; 1445 } else { 1446 fcu_ctl_csr = FCU_CONTROL; 1447 fcu_sts_csr = FCU_STATUS; 1448 fcu_dram_hi_csr = FCU_DRAM_ADDR_HI; 1449 fcu_dram_lo_csr = FCU_DRAM_ADDR_LO; 1450 } 1451 SET_FCU_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); 1452 SET_FCU_CSR(handle, fcu_dram_lo_csr, bus_addr); 1453 SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); 1454 1455 do { 1456 pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); 1457 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); 1458 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) 1459 goto auth_fail; 1460 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) 1461 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) 1462 return 0; 1463 } while (retry++ < FW_AUTH_MAX_RETRY); 1464 auth_fail: 1465 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", 1466 fcu_sts & FCU_AUTH_STS_MASK, 1467 retry); 1468 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 1469 mem_cfg_err = 1470 (GET_FCU_CSR(handle, FCU_STATUS1_C4XXX) & MEM_CFG_ERR_BIT); 1471 if (mem_cfg_err) 1472 pr_err("QAT: MEM_CFG_ERR\n"); 1473 } 1474 return EINVAL; 1475 } 1476 1477 static int 1478 qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, int imgid) 1479 { 1480 struct icp_qat_suof_handle *sobj_handle; 1481 1482 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) 1483 return 0; 1484 1485 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; 1486 if (handle->hal_handle->admin_ae_mask & 1487 sobj_handle->img_table.simg_hdr[imgid].ae_mask) 1488 return 0; 1489 1490 return 1; 1491 } 1492 1493 static int 1494 qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, 1495 struct icp_qat_fw_auth_desc *desc) 1496 { 1497 unsigned int i = 0; 1498 unsigned int fcuSts = 0, fcuAeBroadcastMask = 0; 1499 unsigned int retry = 0; 1500 unsigned int fcuStsCsr = 0; 1501 unsigned int fcuCtlCsr = 0; 1502 unsigned int loadedAes = 0; 1503 unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); 1504 1505 if (IS_QAT_GEN4(device_id)) { 1506 fcuCtlCsr = FCU_CONTROL_4XXX; 1507 fcuStsCsr = FCU_STATUS_4XXX; 1508 } else { 1509 pr_err("Uclo_BroadcastLoadFW only applicable for CPM20\n"); 1510 return EINVAL; 1511 } 1512 1513 for (i = 0; i < ICP_QAT_UCLO_MAX_AE; i++) { 1514 if (!test_bit(i, (unsigned long *)&handle->hal_handle->ae_mask)) 1515 continue; 1516 1517 if (qat_hal_check_ae_active(handle, (unsigned char)i)) { 1518 pr_err( 1519 "Uclo_BroadcastLoadFW error (invalid AE status)\n"); 1520 return EINVAL; 1521 } 1522 1523 if ((desc->ae_mask >> i) & 0x1) { 1524 fcuAeBroadcastMask |= 1 << i; 1525 } 1526 } 1527 1528 if (fcuAeBroadcastMask) { 1529 retry = 0; 1530 SET_FCU_CSR(handle, 1531 FCU_ME_BROADCAST_MASK_TYPE, 1532 fcuAeBroadcastMask); 1533 SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD); 1534 do { 1535 msleep(FW_AUTH_WAIT_PERIOD); 1536 fcuSts = GET_FCU_CSR(handle, fcuStsCsr); 1537 1538 if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_FAIL) { 1539 pr_err( 1540 "Uclo_BroadcastLoadFW fail (fcu_status = 0x%x)\n", 1541 fcuSts & FCU_AUTH_STS_MASK); 1542 return EINVAL; 1543 } else if ((fcuSts & FCU_AUTH_STS_MASK) == 1544 FCU_STS_LOAD_DONE) { 1545 if (IS_QAT_GEN4(device_id)) 1546 loadedAes = 1547 GET_FCU_CSR(handle, 1548 FCU_AE_LOADED_4XXX); 1549 else 1550 loadedAes = 1551 (fcuSts >> FCU_LOADED_AE_POS); 1552 1553 if ((loadedAes & fcuAeBroadcastMask) == 1554 fcuAeBroadcastMask) 1555 break; 1556 } else if ((fcuSts & FCU_AUTH_STS_MASK) == 1557 FCU_STS_VERI_DONE) { 1558 SET_FCU_CSR(handle, 1559 fcuCtlCsr, 1560 FCU_CTRL_CMD_LOAD); 1561 } 1562 } while (retry++ < FW_BROADCAST_MAX_RETRY); 1563 if (retry > FW_BROADCAST_MAX_RETRY) { 1564 pr_err( 1565 "Uclo_BroadcastLoadFW fail(fcu_status = 0x%x),retry = %d\n", 1566 fcuSts & FCU_AUTH_STS_MASK, 1567 retry); 1568 return EINVAL; 1569 } 1570 } 1571 return 0; 1572 } 1573 1574 static int 1575 qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, 1576 struct icp_firml_dram_desc *dram_desc, 1577 unsigned int size) 1578 { 1579 int ret; 1580 1581 ret = bus_dma_mem_create(&dram_desc->dram_mem, 1582 handle->accel_dev->dma_tag, 1583 1, 1584 BUS_SPACE_MAXADDR, 1585 size, 1586 0); 1587 if (ret != 0) 1588 return ret; 1589 dram_desc->dram_base_addr_v = dram_desc->dram_mem.dma_vaddr; 1590 dram_desc->dram_bus_addr = dram_desc->dram_mem.dma_baddr; 1591 dram_desc->dram_size = size; 1592 return 0; 1593 } 1594 1595 static void 1596 qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, 1597 struct icp_firml_dram_desc *dram_desc) 1598 { 1599 if (handle && dram_desc && dram_desc->dram_base_addr_v) 1600 bus_dma_mem_free(&dram_desc->dram_mem); 1601 1602 if (dram_desc) 1603 explicit_bzero(dram_desc, sizeof(*dram_desc)); 1604 } 1605 1606 static int 1607 qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, 1608 const char *image, 1609 unsigned int size, 1610 struct icp_firml_dram_desc *img_desc, 1611 struct icp_qat_fw_auth_desc **desc) 1612 { 1613 const struct icp_qat_css_hdr *css_hdr = 1614 (const struct icp_qat_css_hdr *)image; 1615 struct icp_qat_fw_auth_desc *auth_desc; 1616 struct icp_qat_auth_chunk *auth_chunk; 1617 u64 virt_addr, bus_addr, virt_base; 1618 unsigned int length, simg_offset = sizeof(*auth_chunk); 1619 unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); 1620 1621 if (size > 1622 (ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { 1623 pr_err("QAT: error, input image size overflow %d\n", size); 1624 return EINVAL; 1625 } 1626 length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? 1627 ICP_QAT_CSS_AE_SIMG_LEN(device_id) + simg_offset : 1628 size + ICP_QAT_CSS_FWSK_PAD_LEN(device_id) + simg_offset; 1629 if (qat_uclo_simg_alloc(handle, img_desc, length)) { 1630 pr_err("QAT: error, allocate continuous dram fail\n"); 1631 return -ENOMEM; 1632 } 1633 1634 auth_chunk = img_desc->dram_base_addr_v; 1635 auth_chunk->chunk_size = img_desc->dram_size; 1636 auth_chunk->chunk_bus_addr = img_desc->dram_bus_addr; 1637 virt_base = (uintptr_t)img_desc->dram_base_addr_v + simg_offset; 1638 bus_addr = img_desc->dram_bus_addr + simg_offset; 1639 auth_desc = img_desc->dram_base_addr_v; 1640 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1641 auth_desc->css_hdr_low = (unsigned int)bus_addr; 1642 virt_addr = virt_base; 1643 1644 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); 1645 /* pub key */ 1646 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + 1647 sizeof(*css_hdr); 1648 virt_addr = virt_addr + sizeof(*css_hdr); 1649 1650 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1651 auth_desc->fwsk_pub_low = (unsigned int)bus_addr; 1652 1653 memcpy((void *)(uintptr_t)virt_addr, 1654 (const void *)(image + sizeof(*css_hdr)), 1655 ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)); 1656 /* padding */ 1657 explicit_bzero((void *)(uintptr_t)( 1658 virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), 1659 ICP_QAT_CSS_FWSK_PAD_LEN(device_id)); 1660 1661 /* exponent */ 1662 memcpy((void *)(uintptr_t)(virt_addr + 1663 ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + 1664 ICP_QAT_CSS_FWSK_PAD_LEN(device_id)), 1665 (const void *)(image + sizeof(*css_hdr) + 1666 ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), 1667 sizeof(unsigned int)); 1668 1669 /* signature */ 1670 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + 1671 ICP_QAT_CSS_FWSK_PUB_LEN(device_id); 1672 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(device_id); 1673 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1674 auth_desc->signature_low = (unsigned int)bus_addr; 1675 1676 memcpy((void *)(uintptr_t)virt_addr, 1677 (const void *)(image + sizeof(*css_hdr) + 1678 ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + 1679 ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id)), 1680 ICP_QAT_CSS_SIGNATURE_LEN(device_id)); 1681 1682 bus_addr = 1683 ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + 1684 ICP_QAT_CSS_SIGNATURE_LEN(device_id); 1685 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(device_id); 1686 1687 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); 1688 auth_desc->img_low = (unsigned int)bus_addr; 1689 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(device_id); 1690 memcpy((void *)(uintptr_t)virt_addr, 1691 (const void *)(image + ICP_QAT_AE_IMG_OFFSET(device_id)), 1692 auth_desc->img_len); 1693 virt_addr = virt_base; 1694 /* AE firmware */ 1695 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == 1696 CSS_AE_FIRMWARE) { 1697 auth_desc->img_ae_mode_data_high = auth_desc->img_high; 1698 auth_desc->img_ae_mode_data_low = auth_desc->img_low; 1699 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, 1700 auth_desc->img_ae_mode_data_low) + 1701 sizeof(struct icp_qat_simg_ae_mode); 1702 1703 auth_desc->img_ae_init_data_high = 1704 (unsigned int)(bus_addr >> BITS_IN_DWORD); 1705 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; 1706 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; 1707 auth_desc->img_ae_insts_high = 1708 (unsigned int)(bus_addr >> BITS_IN_DWORD); 1709 auth_desc->img_ae_insts_low = (unsigned int)bus_addr; 1710 virt_addr += sizeof(struct icp_qat_css_hdr) + 1711 ICP_QAT_CSS_FWSK_PUB_LEN(device_id) + 1712 ICP_QAT_CSS_SIGNATURE_LEN(device_id); 1713 auth_desc->ae_mask = 1714 ((struct icp_qat_simg_ae_mode *)virt_addr)->ae_mask & 1715 handle->cfg_ae_mask; 1716 } else { 1717 auth_desc->img_ae_insts_high = auth_desc->img_high; 1718 auth_desc->img_ae_insts_low = auth_desc->img_low; 1719 } 1720 *desc = auth_desc; 1721 return 0; 1722 } 1723 1724 static int 1725 qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, 1726 struct icp_qat_fw_auth_desc *desc) 1727 { 1728 unsigned int i = 0; 1729 unsigned int fcu_sts; 1730 unsigned int fcu_sts_csr, fcu_ctl_csr; 1731 unsigned int loaded_aes = FCU_LOADED_AE_POS; 1732 unsigned long ae_mask = handle->hal_handle->ae_mask; 1733 1734 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1735 fcu_ctl_csr = FCU_CONTROL_C4XXX; 1736 fcu_sts_csr = FCU_STATUS_C4XXX; 1737 1738 } else { 1739 fcu_ctl_csr = FCU_CONTROL; 1740 fcu_sts_csr = FCU_STATUS; 1741 } 1742 1743 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) 1744 { 1745 int retry = 0; 1746 1747 if (!((desc->ae_mask >> i) & 0x1)) 1748 continue; 1749 if (qat_hal_check_ae_active(handle, i)) { 1750 pr_err("QAT: AE %d is active\n", i); 1751 return EINVAL; 1752 } 1753 SET_FCU_CSR(handle, 1754 fcu_ctl_csr, 1755 (FCU_CTRL_CMD_LOAD | 1756 (IS_QAT_GEN4( 1757 pci_get_device(GET_DEV(handle->accel_dev))) ? 1758 (1 << FCU_CTRL_BROADCAST_POS) : 1759 0) | 1760 (i << FCU_CTRL_AE_POS))); 1761 1762 do { 1763 pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); 1764 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); 1765 if ((fcu_sts & FCU_AUTH_STS_MASK) == 1766 FCU_STS_LOAD_DONE) { 1767 loaded_aes = IS_QAT_GEN3_OR_GEN4(pci_get_device( 1768 GET_DEV(handle->accel_dev))) ? 1769 GET_FCU_CSR(handle, FCU_AE_LOADED_C4XXX) : 1770 (fcu_sts >> FCU_LOADED_AE_POS); 1771 if (loaded_aes & (1 << i)) 1772 break; 1773 } 1774 } while (retry++ < FW_AUTH_MAX_RETRY); 1775 if (retry > FW_AUTH_MAX_RETRY) { 1776 pr_err("QAT: firmware load failed timeout %x\n", retry); 1777 return EINVAL; 1778 } 1779 } 1780 return 0; 1781 } 1782 1783 static int 1784 qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, 1785 const void *addr_ptr, 1786 int mem_size) 1787 { 1788 struct icp_qat_suof_handle *suof_handle; 1789 1790 suof_handle = malloc(sizeof(*suof_handle), M_QAT, M_WAITOK | M_ZERO); 1791 handle->sobj_handle = suof_handle; 1792 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { 1793 qat_uclo_del_suof(handle); 1794 pr_err("QAT: map SUOF failed\n"); 1795 return EINVAL; 1796 } 1797 return 0; 1798 } 1799 1800 int 1801 qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, 1802 const void *addr_ptr, 1803 int mem_size) 1804 { 1805 struct icp_qat_fw_auth_desc *desc = NULL; 1806 struct icp_firml_dram_desc img_desc; 1807 int status = 0; 1808 1809 if (handle->fw_auth) { 1810 status = qat_uclo_map_auth_fw( 1811 handle, addr_ptr, mem_size, &img_desc, &desc); 1812 if (!status) 1813 status = qat_uclo_auth_fw(handle, desc); 1814 1815 qat_uclo_simg_free(handle, &img_desc); 1816 } else { 1817 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1818 device_printf( 1819 NULL, "QAT: PKE service is not allowed because "); 1820 device_printf(NULL, "MMP fw will not be loaded for "); 1821 device_printf(NULL, 1822 "device 0x%x", 1823 pci_get_device( 1824 GET_DEV(handle->accel_dev))); 1825 return status; 1826 } 1827 if (pci_get_device(GET_DEV(handle->accel_dev)) == 1828 ADF_C3XXX_PCI_DEVICE_ID) { 1829 pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); 1830 return EINVAL; 1831 } 1832 status = qat_uclo_wr_sram_by_words(handle, 1833 handle->hal_sram_offset, 1834 addr_ptr, 1835 mem_size); 1836 } 1837 return status; 1838 } 1839 1840 static int 1841 qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, 1842 const void *addr_ptr, 1843 int mem_size) 1844 { 1845 struct icp_qat_uof_filehdr *filehdr; 1846 struct icp_qat_uclo_objhandle *objhdl; 1847 1848 objhdl = malloc(sizeof(*objhdl), M_QAT, M_WAITOK | M_ZERO); 1849 objhdl->obj_buf = malloc(mem_size, M_QAT, M_WAITOK); 1850 bcopy(addr_ptr, objhdl->obj_buf, mem_size); 1851 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; 1852 if (qat_uclo_check_uof_format(filehdr)) 1853 goto out_objhdr_err; 1854 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, 1855 filehdr, 1856 ICP_QAT_UOF_OBJS); 1857 if (!objhdl->obj_hdr) { 1858 pr_err("QAT: object file chunk is null\n"); 1859 goto out_objhdr_err; 1860 } 1861 handle->obj_handle = objhdl; 1862 if (qat_uclo_parse_uof_obj(handle)) 1863 goto out_overlay_obj_err; 1864 return 0; 1865 1866 out_overlay_obj_err: 1867 handle->obj_handle = NULL; 1868 free(objhdl->obj_hdr, M_QAT); 1869 out_objhdr_err: 1870 free(objhdl->obj_buf, M_QAT); 1871 free(objhdl, M_QAT); 1872 return ENOMEM; 1873 } 1874 1875 static int 1876 qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1877 const struct icp_qat_mof_file_hdr *mof_ptr, 1878 u32 mof_size) 1879 { 1880 unsigned int checksum = 0; 1881 unsigned int min_ver_offset = 0; 1882 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1883 1884 mobj_handle->file_id = ICP_QAT_MOF_FID; 1885 mobj_handle->mof_buf = (const char *)mof_ptr; 1886 mobj_handle->mof_size = mof_size; 1887 1888 min_ver_offset = 1889 mof_size - offsetof(struct icp_qat_mof_file_hdr, min_ver); 1890 checksum = qat_uclo_calc_str_checksum((const char *)&mof_ptr->min_ver, 1891 min_ver_offset); 1892 if (checksum != mof_ptr->checksum) { 1893 pr_err("QAT: incorrect MOF checksum\n"); 1894 return EINVAL; 1895 } 1896 mobj_handle->checksum = mof_ptr->checksum; 1897 mobj_handle->min_ver = mof_ptr->min_ver; 1898 mobj_handle->maj_ver = mof_ptr->maj_ver; 1899 return 0; 1900 } 1901 1902 void 1903 qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) 1904 { 1905 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1906 1907 free(mobj_handle->obj_table.obj_hdr, M_QAT); 1908 mobj_handle->obj_table.obj_hdr = NULL; 1909 free(handle->mobj_handle, M_QAT); 1910 handle->mobj_handle = NULL; 1911 } 1912 1913 static int 1914 qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, 1915 const char *obj_name, 1916 const char **obj_ptr, 1917 unsigned int *obj_size) 1918 { 1919 unsigned int i; 1920 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; 1921 1922 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { 1923 if (!strncmp(obj_hdr[i].obj_name, 1924 obj_name, 1925 ICP_QAT_SUOF_OBJ_NAME_LEN)) { 1926 *obj_ptr = obj_hdr[i].obj_buf; 1927 *obj_size = obj_hdr[i].obj_size; 1928 break; 1929 } 1930 } 1931 1932 if (i >= mobj_handle->obj_table.num_objs) { 1933 pr_err("QAT: object %s is not found inside MOF\n", obj_name); 1934 return EFAULT; 1935 } 1936 return 0; 1937 } 1938 1939 static int 1940 qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, 1941 struct icp_qat_mof_objhdr *mobj_hdr, 1942 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) 1943 { 1944 if ((strncmp((char *)obj_chunkhdr->chunk_id, 1945 ICP_QAT_UOF_IMAG, 1946 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) { 1947 mobj_hdr->obj_buf = 1948 (const char *)((unsigned long)obj_chunkhdr->offset + 1949 mobj_handle->uobjs_hdr); 1950 } else if ((strncmp((char *)(obj_chunkhdr->chunk_id), 1951 ICP_QAT_SUOF_IMAG, 1952 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) { 1953 mobj_hdr->obj_buf = 1954 (const char *)((unsigned long)obj_chunkhdr->offset + 1955 mobj_handle->sobjs_hdr); 1956 1957 } else { 1958 pr_err("QAT: unsupported chunk id\n"); 1959 return EINVAL; 1960 } 1961 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; 1962 mobj_hdr->obj_name = 1963 (char *)(obj_chunkhdr->name + mobj_handle->sym_str); 1964 return 0; 1965 } 1966 1967 static int 1968 qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) 1969 { 1970 struct icp_qat_mof_objhdr *mof_obj_hdr; 1971 const struct icp_qat_mof_obj_hdr *uobj_hdr; 1972 const struct icp_qat_mof_obj_hdr *sobj_hdr; 1973 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; 1974 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; 1975 unsigned int uobj_chunk_num = 0, sobj_chunk_num = 0; 1976 unsigned int *valid_chunks = 0; 1977 int ret, i; 1978 1979 uobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; 1980 sobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; 1981 if (uobj_hdr) 1982 uobj_chunk_num = uobj_hdr->num_chunks; 1983 if (sobj_hdr) 1984 sobj_chunk_num = sobj_hdr->num_chunks; 1985 1986 mof_obj_hdr = (struct icp_qat_mof_objhdr *) 1987 malloc((uobj_chunk_num + sobj_chunk_num) * sizeof(*mof_obj_hdr), 1988 M_QAT, 1989 M_WAITOK | M_ZERO); 1990 1991 mobj_handle->obj_table.obj_hdr = mof_obj_hdr; 1992 valid_chunks = &mobj_handle->obj_table.num_objs; 1993 uobj_chunkhdr = 1994 (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)uobj_hdr + 1995 sizeof(*uobj_hdr)); 1996 sobj_chunkhdr = 1997 (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)sobj_hdr + 1998 sizeof(*sobj_hdr)); 1999 2000 /* map uof objects */ 2001 for (i = 0; i < uobj_chunk_num; i++) { 2002 ret = qat_uclo_map_obj_from_mof(mobj_handle, 2003 &mof_obj_hdr[*valid_chunks], 2004 &uobj_chunkhdr[i]); 2005 if (ret) 2006 return ret; 2007 (*valid_chunks)++; 2008 } 2009 2010 /* map suof objects */ 2011 for (i = 0; i < sobj_chunk_num; i++) { 2012 ret = qat_uclo_map_obj_from_mof(mobj_handle, 2013 &mof_obj_hdr[*valid_chunks], 2014 &sobj_chunkhdr[i]); 2015 if (ret) 2016 return ret; 2017 (*valid_chunks)++; 2018 } 2019 2020 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunks) { 2021 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); 2022 return EINVAL; 2023 } 2024 return 0; 2025 } 2026 2027 static void 2028 qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, 2029 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 2030 { 2031 char **sym_str = (char **)&mobj_handle->sym_str; 2032 unsigned int *sym_size = &mobj_handle->sym_size; 2033 struct icp_qat_mof_str_table *str_table_obj; 2034 2035 *sym_size = *(unsigned int *)(uintptr_t)(mof_chunkhdr->offset + 2036 mobj_handle->mof_buf); 2037 *sym_str = 2038 (char *)(uintptr_t)(mobj_handle->mof_buf + mof_chunkhdr->offset + 2039 sizeof(str_table_obj->tab_len)); 2040 } 2041 2042 static void 2043 qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, 2044 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 2045 { 2046 if (!strncmp(mof_chunkhdr->chunk_id, 2047 ICP_QAT_MOF_SYM_OBJS, 2048 ICP_QAT_MOF_OBJ_ID_LEN)) 2049 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); 2050 else if (!strncmp(mof_chunkhdr->chunk_id, 2051 ICP_QAT_UOF_OBJS, 2052 ICP_QAT_MOF_OBJ_ID_LEN)) 2053 mobj_handle->uobjs_hdr = 2054 mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset; 2055 else if (!strncmp(mof_chunkhdr->chunk_id, 2056 ICP_QAT_SUOF_OBJS, 2057 ICP_QAT_MOF_OBJ_ID_LEN)) 2058 mobj_handle->sobjs_hdr = 2059 mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset; 2060 } 2061 2062 static int 2063 qat_uclo_check_mof_format(const struct icp_qat_mof_file_hdr *mof_hdr) 2064 { 2065 int maj = mof_hdr->maj_ver & 0xff; 2066 int min = mof_hdr->min_ver & 0xff; 2067 2068 if (mof_hdr->file_id != ICP_QAT_MOF_FID) { 2069 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); 2070 return EINVAL; 2071 } 2072 2073 if (mof_hdr->num_chunks <= 0x1) { 2074 pr_err("QAT: MOF chunk amount is incorrect\n"); 2075 return EINVAL; 2076 } 2077 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { 2078 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", 2079 maj, 2080 min); 2081 return EINVAL; 2082 } 2083 return 0; 2084 } 2085 2086 static int 2087 qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, 2088 const struct icp_qat_mof_file_hdr *mof_ptr, 2089 u32 mof_size, 2090 const char *obj_name, 2091 const char **obj_ptr, 2092 unsigned int *obj_size) 2093 { 2094 struct icp_qat_mof_handle *mobj_handle; 2095 struct icp_qat_mof_chunkhdr *mof_chunkhdr; 2096 unsigned short chunks_num; 2097 int ret; 2098 unsigned int i; 2099 2100 if (mof_ptr->file_id == ICP_QAT_UOF_FID || 2101 mof_ptr->file_id == ICP_QAT_SUOF_FID) { 2102 if (obj_ptr) 2103 *obj_ptr = (const char *)mof_ptr; 2104 if (obj_size) 2105 *obj_size = (unsigned int)mof_size; 2106 return 0; 2107 } 2108 if (qat_uclo_check_mof_format(mof_ptr)) 2109 return EINVAL; 2110 mobj_handle = malloc(sizeof(*mobj_handle), M_QAT, M_WAITOK | M_ZERO); 2111 handle->mobj_handle = mobj_handle; 2112 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); 2113 if (ret) 2114 return ret; 2115 mof_chunkhdr = (struct icp_qat_mof_chunkhdr *)((uintptr_t)mof_ptr + 2116 sizeof(*mof_ptr)); 2117 chunks_num = mof_ptr->num_chunks; 2118 /*Parse MOF file chunks*/ 2119 for (i = 0; i < chunks_num; i++) 2120 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); 2121 /*All sym_objs uobjs and sobjs should be available*/ 2122 if (!mobj_handle->sym_str || 2123 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) 2124 return EINVAL; 2125 ret = qat_uclo_map_objs_from_mof(mobj_handle); 2126 if (ret) 2127 return ret; 2128 /*Seek specified uof object in MOF*/ 2129 ret = qat_uclo_seek_obj_inside_mof(mobj_handle, 2130 obj_name, 2131 obj_ptr, 2132 obj_size); 2133 if (ret) 2134 return ret; 2135 return 0; 2136 } 2137 2138 int 2139 qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, 2140 const void *addr_ptr, 2141 u32 mem_size, 2142 const char *obj_name) 2143 { 2144 const char *obj_addr; 2145 u32 obj_size; 2146 int ret; 2147 2148 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE > 2149 (sizeof(handle->hal_handle->ae_mask) * 8)); 2150 2151 if (!handle || !addr_ptr || mem_size < 24) 2152 return EINVAL; 2153 2154 if (obj_name) { 2155 ret = qat_uclo_map_mof_obj( 2156 handle, addr_ptr, mem_size, obj_name, &obj_addr, &obj_size); 2157 if (ret) 2158 return ret; 2159 } else { 2160 obj_addr = addr_ptr; 2161 obj_size = mem_size; 2162 } 2163 2164 return (handle->fw_auth) ? 2165 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : 2166 qat_uclo_map_uof_obj(handle, obj_addr, obj_size); 2167 } 2168 2169 void 2170 qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) 2171 { 2172 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2173 unsigned int a; 2174 unsigned long ae_mask = handle->hal_handle->ae_mask; 2175 2176 if (handle->mobj_handle) 2177 qat_uclo_del_mof(handle); 2178 if (handle->sobj_handle) 2179 qat_uclo_del_suof(handle); 2180 if (!obj_handle) 2181 return; 2182 2183 free(obj_handle->uword_buf, M_QAT); 2184 for (a = 0; a < obj_handle->uimage_num; a++) 2185 free(obj_handle->ae_uimage[a].page, M_QAT); 2186 2187 for_each_set_bit(a, &ae_mask, handle->hal_handle->ae_max_num) 2188 { 2189 qat_uclo_free_ae_data(&obj_handle->ae_data[a]); 2190 } 2191 2192 free(obj_handle->obj_hdr, M_QAT); 2193 free(obj_handle->obj_buf, M_QAT); 2194 free(obj_handle, M_QAT); 2195 handle->obj_handle = NULL; 2196 } 2197 2198 static void 2199 qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, 2200 struct icp_qat_uclo_encap_page *encap_page, 2201 uint64_t *uword, 2202 unsigned int addr_p, 2203 unsigned int raddr, 2204 uint64_t fill) 2205 { 2206 uint64_t uwrd = 0; 2207 unsigned int i, addr; 2208 2209 if (!encap_page) { 2210 *uword = fill; 2211 return; 2212 } 2213 addr = (encap_page->page_region) ? raddr : addr_p; 2214 for (i = 0; i < encap_page->uwblock_num; i++) { 2215 if (addr >= encap_page->uwblock[i].start_addr && 2216 addr <= encap_page->uwblock[i].start_addr + 2217 encap_page->uwblock[i].words_num - 1) { 2218 addr -= encap_page->uwblock[i].start_addr; 2219 addr *= obj_handle->uword_in_bytes; 2220 memcpy(&uwrd, 2221 (void *)(((uintptr_t)encap_page->uwblock[i] 2222 .micro_words) + 2223 addr), 2224 obj_handle->uword_in_bytes); 2225 uwrd = uwrd & 0xbffffffffffull; 2226 } 2227 } 2228 *uword = uwrd; 2229 if (*uword == INVLD_UWORD) 2230 *uword = fill; 2231 } 2232 2233 static void 2234 qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, 2235 struct icp_qat_uclo_encap_page *encap_page, 2236 unsigned int ae) 2237 { 2238 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; 2239 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2240 uint64_t fill_pat; 2241 2242 /* load the page starting at appropriate ustore address */ 2243 /* get fill-pattern from an image -- they are all the same */ 2244 memcpy(&fill_pat, 2245 obj_handle->ae_uimage[0].img_ptr->fill_pattern, 2246 sizeof(uint64_t)); 2247 uw_physical_addr = encap_page->beg_addr_p; 2248 uw_relative_addr = 0; 2249 words_num = encap_page->micro_words_num; 2250 while (words_num) { 2251 if (words_num < UWORD_CPYBUF_SIZE) 2252 cpylen = words_num; 2253 else 2254 cpylen = UWORD_CPYBUF_SIZE; 2255 2256 /* load the buffer */ 2257 for (i = 0; i < cpylen; i++) 2258 qat_uclo_fill_uwords(obj_handle, 2259 encap_page, 2260 &obj_handle->uword_buf[i], 2261 uw_physical_addr + i, 2262 uw_relative_addr + i, 2263 fill_pat); 2264 2265 if (obj_handle->ae_data[ae].shareable_ustore && 2266 !IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) 2267 /* copy the buffer to ustore */ 2268 qat_hal_wr_coalesce_uwords(handle, 2269 (unsigned char)ae, 2270 uw_physical_addr, 2271 cpylen, 2272 obj_handle->uword_buf); 2273 else 2274 /* copy the buffer to ustore */ 2275 qat_hal_wr_uwords(handle, 2276 (unsigned char)ae, 2277 uw_physical_addr, 2278 cpylen, 2279 obj_handle->uword_buf); 2280 uw_physical_addr += cpylen; 2281 uw_relative_addr += cpylen; 2282 words_num -= cpylen; 2283 } 2284 } 2285 2286 static void 2287 qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, 2288 struct icp_qat_uof_image *image) 2289 { 2290 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2291 unsigned int ctx_mask, s; 2292 struct icp_qat_uclo_page *page; 2293 unsigned char ae = 0; 2294 int ctx; 2295 struct icp_qat_uclo_aedata *aed; 2296 unsigned long ae_mask = handle->hal_handle->ae_mask; 2297 2298 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) 2299 ctx_mask = 0xff; 2300 else 2301 ctx_mask = 0x55; 2302 /* load the default page and set assigned CTX PC 2303 * to the entrypoint address 2304 */ 2305 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 2306 { 2307 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 2308 unsigned long ae_assigned = image->ae_assigned; 2309 2310 if (!test_bit(ae, &cfg_ae_mask)) 2311 continue; 2312 2313 if (!test_bit(ae, &ae_assigned)) 2314 continue; 2315 2316 aed = &obj_handle->ae_data[ae]; 2317 /* find the slice to which this image is assigned */ 2318 for (s = 0; s < aed->slice_num; s++) { 2319 if (image->ctx_assigned & 2320 aed->ae_slices[s].ctx_mask_assigned) 2321 break; 2322 } 2323 if (s >= aed->slice_num) 2324 continue; 2325 page = aed->ae_slices[s].page; 2326 if (!page->encap_page->def_page) 2327 continue; 2328 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); 2329 2330 page = aed->ae_slices[s].page; 2331 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) 2332 aed->ae_slices[s].cur_page[ctx] = 2333 (ctx_mask & (1 << ctx)) ? page : NULL; 2334 qat_hal_set_live_ctx(handle, 2335 (unsigned char)ae, 2336 image->ctx_assigned); 2337 qat_hal_set_pc(handle, 2338 (unsigned char)ae, 2339 image->ctx_assigned, 2340 image->entry_address); 2341 } 2342 } 2343 2344 static int 2345 qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) 2346 { 2347 unsigned int i; 2348 struct icp_qat_fw_auth_desc *desc = NULL; 2349 struct icp_firml_dram_desc img_desc; 2350 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 2351 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; 2352 2353 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { 2354 if (qat_uclo_map_auth_fw(handle, 2355 (const char *)simg_hdr[i].simg_buf, 2356 (unsigned int)(simg_hdr[i].simg_len), 2357 &img_desc, 2358 &desc)) 2359 goto wr_err; 2360 if (qat_uclo_auth_fw(handle, desc)) 2361 goto wr_err; 2362 if (qat_uclo_is_broadcast(handle, i)) { 2363 if (qat_uclo_broadcast_load_fw(handle, desc)) 2364 goto wr_err; 2365 } else { 2366 if (qat_uclo_load_fw(handle, desc)) 2367 goto wr_err; 2368 } 2369 qat_uclo_simg_free(handle, &img_desc); 2370 } 2371 2372 return 0; 2373 wr_err: 2374 qat_uclo_simg_free(handle, &img_desc); 2375 return -EINVAL; 2376 } 2377 2378 static int 2379 qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) 2380 { 2381 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2382 unsigned int i; 2383 2384 if (qat_uclo_init_globals(handle)) 2385 return EINVAL; 2386 for (i = 0; i < obj_handle->uimage_num; i++) { 2387 if (!obj_handle->ae_uimage[i].img_ptr) 2388 return EINVAL; 2389 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) 2390 return EINVAL; 2391 qat_uclo_wr_uimage_page(handle, 2392 obj_handle->ae_uimage[i].img_ptr); 2393 } 2394 return 0; 2395 } 2396 2397 int 2398 qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) 2399 { 2400 return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : 2401 qat_uclo_wr_uof_img(handle); 2402 } 2403 2404 int 2405 qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, 2406 unsigned int cfg_ae_mask) 2407 { 2408 if (!cfg_ae_mask) 2409 return EINVAL; 2410 2411 handle->cfg_ae_mask = cfg_ae_mask; 2412 return 0; 2413 } 2414