1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/align.h> 4 #include <linux/bitops.h> 5 #include <linux/slab.h> 6 #include <linux/ctype.h> 7 #include <linux/kernel.h> 8 #include <linux/delay.h> 9 #include <linux/pci_ids.h> 10 #include "adf_accel_devices.h" 11 #include "adf_common_drv.h" 12 #include "icp_qat_uclo.h" 13 #include "icp_qat_hal.h" 14 #include "icp_qat_fw_loader_handle.h" 15 16 #define UWORD_CPYBUF_SIZE 1024U 17 #define INVLD_UWORD 0xffffffffffull 18 #define PID_MINOR_REV 0xf 19 #define PID_MAJOR_REV (0xf << 4) 20 21 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, 22 unsigned int ae, unsigned int image_num) 23 { 24 struct icp_qat_uclo_aedata *ae_data; 25 struct icp_qat_uclo_encapme *encap_image; 26 struct icp_qat_uclo_page *page = NULL; 27 struct icp_qat_uclo_aeslice *ae_slice = NULL; 28 29 ae_data = &obj_handle->ae_data[ae]; 30 encap_image = &obj_handle->ae_uimage[image_num]; 31 ae_slice = &ae_data->ae_slices[ae_data->slice_num]; 32 ae_slice->encap_image = encap_image; 33 34 if (encap_image->img_ptr) { 35 ae_slice->ctx_mask_assigned = 36 encap_image->img_ptr->ctx_assigned; 37 ae_data->eff_ustore_size = obj_handle->ustore_phy_size; 38 } else { 39 ae_slice->ctx_mask_assigned = 0; 40 } 41 ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL); 42 if (!ae_slice->region) 43 return -ENOMEM; 44 ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL); 45 if (!ae_slice->page) 46 goto out_err; 47 page = ae_slice->page; 48 page->encap_page = encap_image->page; 49 ae_slice->page->region = ae_slice->region; 50 ae_data->slice_num++; 51 return 0; 52 out_err: 53 kfree(ae_slice->region); 54 ae_slice->region = NULL; 55 return -ENOMEM; 56 } 57 58 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) 59 { 60 unsigned int i; 61 62 if (!ae_data) { 63 pr_err("QAT: bad argument, ae_data is NULL\n"); 64 return -EINVAL; 65 } 66 67 for (i = 0; i < ae_data->slice_num; i++) { 68 kfree(ae_data->ae_slices[i].region); 69 ae_data->ae_slices[i].region = NULL; 70 kfree(ae_data->ae_slices[i].page); 71 ae_data->ae_slices[i].page = NULL; 72 } 73 return 0; 74 } 75 76 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, 77 unsigned int str_offset) 78 { 79 if (!str_table->table_len || str_offset > str_table->table_len) 80 return NULL; 81 return (char *)(((uintptr_t)(str_table->strings)) + str_offset); 82 } 83 84 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) 85 { 86 int maj = hdr->maj_ver & 0xff; 87 int min = hdr->min_ver & 0xff; 88 89 if (hdr->file_id != ICP_QAT_UOF_FID) { 90 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); 91 return -EINVAL; 92 } 93 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { 94 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", 95 maj, min); 96 return -EINVAL; 97 } 98 return 0; 99 } 100 101 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) 102 { 103 int maj = suof_hdr->maj_ver & 0xff; 104 int min = suof_hdr->min_ver & 0xff; 105 106 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { 107 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); 108 return -EINVAL; 109 } 110 if (suof_hdr->fw_type != 0) { 111 pr_err("QAT: unsupported firmware type\n"); 112 return -EINVAL; 113 } 114 if (suof_hdr->num_chunks <= 0x1) { 115 pr_err("QAT: SUOF chunk amount is incorrect\n"); 116 return -EINVAL; 117 } 118 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { 119 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", 120 maj, min); 121 return -EINVAL; 122 } 123 return 0; 124 } 125 126 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, 127 unsigned int addr, unsigned int *val, 128 unsigned int num_in_bytes) 129 { 130 unsigned int outval; 131 unsigned char *ptr = (unsigned char *)val; 132 133 while (num_in_bytes) { 134 memcpy(&outval, ptr, 4); 135 SRAM_WRITE(handle, addr, outval); 136 num_in_bytes -= 4; 137 ptr += 4; 138 addr += 4; 139 } 140 } 141 142 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, 143 unsigned char ae, unsigned int addr, 144 unsigned int *val, 145 unsigned int num_in_bytes) 146 { 147 unsigned int outval; 148 unsigned char *ptr = (unsigned char *)val; 149 150 addr >>= 0x2; /* convert to uword address */ 151 152 while (num_in_bytes) { 153 memcpy(&outval, ptr, 4); 154 qat_hal_wr_umem(handle, ae, addr++, 1, &outval); 155 num_in_bytes -= 4; 156 ptr += 4; 157 } 158 } 159 160 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, 161 unsigned char ae, 162 struct icp_qat_uof_batch_init 163 *umem_init_header) 164 { 165 struct icp_qat_uof_batch_init *umem_init; 166 167 if (!umem_init_header) 168 return; 169 umem_init = umem_init_header->next; 170 while (umem_init) { 171 unsigned int addr, *value, size; 172 173 ae = umem_init->ae; 174 addr = umem_init->addr; 175 value = umem_init->value; 176 size = umem_init->size; 177 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); 178 umem_init = umem_init->next; 179 } 180 } 181 182 static void 183 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, 184 struct icp_qat_uof_batch_init **base) 185 { 186 struct icp_qat_uof_batch_init *umem_init; 187 188 umem_init = *base; 189 while (umem_init) { 190 struct icp_qat_uof_batch_init *pre; 191 192 pre = umem_init; 193 umem_init = umem_init->next; 194 kfree(pre); 195 } 196 *base = NULL; 197 } 198 199 static int qat_uclo_parse_num(char *str, unsigned int *num) 200 { 201 char buf[16] = {0}; 202 unsigned long ae = 0; 203 int i; 204 205 strscpy(buf, str, sizeof(buf)); 206 for (i = 0; i < 16; i++) { 207 if (!isdigit(buf[i])) { 208 buf[i] = '\0'; 209 break; 210 } 211 } 212 if ((kstrtoul(buf, 10, &ae))) 213 return -EFAULT; 214 215 *num = (unsigned int)ae; 216 return 0; 217 } 218 219 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, 220 struct icp_qat_uof_initmem *init_mem, 221 unsigned int size_range, unsigned int *ae) 222 { 223 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 224 char *str; 225 226 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { 227 pr_err("QAT: initmem is out of range"); 228 return -EINVAL; 229 } 230 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { 231 pr_err("QAT: Memory scope for init_mem error\n"); 232 return -EINVAL; 233 } 234 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); 235 if (!str) { 236 pr_err("QAT: AE name assigned in UOF init table is NULL\n"); 237 return -EINVAL; 238 } 239 if (qat_uclo_parse_num(str, ae)) { 240 pr_err("QAT: Parse num for AE number failed\n"); 241 return -EINVAL; 242 } 243 if (*ae >= ICP_QAT_UCLO_MAX_AE) { 244 pr_err("QAT: ae %d out of range\n", *ae); 245 return -EINVAL; 246 } 247 return 0; 248 } 249 250 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle 251 *handle, struct icp_qat_uof_initmem 252 *init_mem, unsigned int ae, 253 struct icp_qat_uof_batch_init 254 **init_tab_base) 255 { 256 struct icp_qat_uof_batch_init *init_header, *tail; 257 struct icp_qat_uof_batch_init *mem_init, *tail_old; 258 struct icp_qat_uof_memvar_attr *mem_val_attr; 259 unsigned int i, flag = 0; 260 261 mem_val_attr = 262 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + 263 sizeof(struct icp_qat_uof_initmem)); 264 265 init_header = *init_tab_base; 266 if (!init_header) { 267 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL); 268 if (!init_header) 269 return -ENOMEM; 270 init_header->size = 1; 271 *init_tab_base = init_header; 272 flag = 1; 273 } 274 tail_old = init_header; 275 while (tail_old->next) 276 tail_old = tail_old->next; 277 tail = tail_old; 278 for (i = 0; i < init_mem->val_attr_num; i++) { 279 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL); 280 if (!mem_init) 281 goto out_err; 282 mem_init->ae = ae; 283 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; 284 mem_init->value = &mem_val_attr->value; 285 mem_init->size = 4; 286 mem_init->next = NULL; 287 tail->next = mem_init; 288 tail = mem_init; 289 init_header->size += qat_hal_get_ins_num(); 290 mem_val_attr++; 291 } 292 return 0; 293 out_err: 294 /* Do not free the list head unless we allocated it. */ 295 tail_old = tail_old->next; 296 if (flag) { 297 kfree(*init_tab_base); 298 *init_tab_base = NULL; 299 } 300 301 while (tail_old) { 302 mem_init = tail_old->next; 303 kfree(tail_old); 304 tail_old = mem_init; 305 } 306 return -ENOMEM; 307 } 308 309 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, 310 struct icp_qat_uof_initmem *init_mem) 311 { 312 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 313 unsigned int ae; 314 315 if (qat_uclo_fetch_initmem_ae(handle, init_mem, 316 handle->chip_info->lm_size, &ae)) 317 return -EINVAL; 318 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 319 &obj_handle->lm_init_tab[ae])) 320 return -EINVAL; 321 return 0; 322 } 323 324 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, 325 struct icp_qat_uof_initmem *init_mem) 326 { 327 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 328 unsigned int ae, ustore_size, uaddr, i; 329 struct icp_qat_uclo_aedata *aed; 330 331 ustore_size = obj_handle->ustore_phy_size; 332 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) 333 return -EINVAL; 334 if (qat_uclo_create_batch_init_list(handle, init_mem, ae, 335 &obj_handle->umem_init_tab[ae])) 336 return -EINVAL; 337 /* set the highest ustore address referenced */ 338 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; 339 aed = &obj_handle->ae_data[ae]; 340 for (i = 0; i < aed->slice_num; i++) { 341 if (aed->ae_slices[i].encap_image->uwords_num < uaddr) 342 aed->ae_slices[i].encap_image->uwords_num = uaddr; 343 } 344 return 0; 345 } 346 347 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, 348 struct icp_qat_uof_initmem *init_mem) 349 { 350 switch (init_mem->region) { 351 case ICP_QAT_UOF_LMEM_REGION: 352 if (qat_uclo_init_lmem_seg(handle, init_mem)) 353 return -EINVAL; 354 break; 355 case ICP_QAT_UOF_UMEM_REGION: 356 if (qat_uclo_init_umem_seg(handle, init_mem)) 357 return -EINVAL; 358 break; 359 default: 360 pr_err("QAT: initmem region error. region type=0x%x\n", 361 init_mem->region); 362 return -EINVAL; 363 } 364 return 0; 365 } 366 367 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, 368 struct icp_qat_uclo_encapme *image) 369 { 370 unsigned int i; 371 struct icp_qat_uclo_encap_page *page; 372 struct icp_qat_uof_image *uof_image; 373 unsigned char ae; 374 unsigned int ustore_size; 375 unsigned int patt_pos; 376 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 377 unsigned long ae_mask = handle->hal_handle->ae_mask; 378 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 379 u64 *fill_data; 380 381 uof_image = image->img_ptr; 382 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64), 383 GFP_KERNEL); 384 if (!fill_data) 385 return -ENOMEM; 386 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) 387 memcpy(&fill_data[i], &uof_image->fill_pattern, 388 sizeof(u64)); 389 page = image->page; 390 391 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 392 unsigned long ae_assigned = uof_image->ae_assigned; 393 394 if (!test_bit(ae, &ae_assigned)) 395 continue; 396 397 if (!test_bit(ae, &cfg_ae_mask)) 398 continue; 399 400 ustore_size = obj_handle->ae_data[ae].eff_ustore_size; 401 patt_pos = page->beg_addr_p + page->micro_words_num; 402 403 qat_hal_wr_uwords(handle, (unsigned char)ae, 0, 404 page->beg_addr_p, &fill_data[0]); 405 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, 406 ustore_size - patt_pos + 1, 407 &fill_data[page->beg_addr_p]); 408 } 409 kfree(fill_data); 410 return 0; 411 } 412 413 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) 414 { 415 int i, ae; 416 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 417 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; 418 unsigned long ae_mask = handle->hal_handle->ae_mask; 419 420 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { 421 if (initmem->num_in_bytes) { 422 if (qat_uclo_init_ae_memory(handle, initmem)) 423 return -EINVAL; 424 } 425 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( 426 (uintptr_t)initmem + 427 sizeof(struct icp_qat_uof_initmem)) + 428 (sizeof(struct icp_qat_uof_memvar_attr) * 429 initmem->val_attr_num)); 430 } 431 432 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 433 if (qat_hal_batch_wr_lm(handle, ae, 434 obj_handle->lm_init_tab[ae])) { 435 pr_err("QAT: fail to batch init lmem for AE %d\n", ae); 436 return -EINVAL; 437 } 438 qat_uclo_cleanup_batch_init_list(handle, 439 &obj_handle->lm_init_tab[ae]); 440 qat_uclo_batch_wr_umem(handle, ae, 441 obj_handle->umem_init_tab[ae]); 442 qat_uclo_cleanup_batch_init_list(handle, 443 &obj_handle-> 444 umem_init_tab[ae]); 445 } 446 return 0; 447 } 448 449 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, 450 char *chunk_id, void *cur) 451 { 452 int i; 453 struct icp_qat_uof_chunkhdr *chunk_hdr = 454 (struct icp_qat_uof_chunkhdr *) 455 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); 456 457 for (i = 0; i < obj_hdr->num_chunks; i++) { 458 if ((cur < (void *)&chunk_hdr[i]) && 459 !strncmp(chunk_hdr[i].chunk_id, chunk_id, 460 ICP_QAT_UOF_OBJID_LEN)) { 461 return &chunk_hdr[i]; 462 } 463 } 464 return NULL; 465 } 466 467 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) 468 { 469 int i; 470 unsigned int topbit = 1 << 0xF; 471 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); 472 473 reg ^= inbyte << 0x8; 474 for (i = 0; i < 0x8; i++) { 475 if (reg & topbit) 476 reg = (reg << 1) ^ 0x1021; 477 else 478 reg <<= 1; 479 } 480 return reg & 0xFFFF; 481 } 482 483 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) 484 { 485 unsigned int chksum = 0; 486 487 if (ptr) 488 while (num--) 489 chksum = qat_uclo_calc_checksum(chksum, *ptr++); 490 return chksum; 491 } 492 493 static struct icp_qat_uclo_objhdr * 494 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, 495 char *chunk_id) 496 { 497 struct icp_qat_uof_filechunkhdr *file_chunk; 498 struct icp_qat_uclo_objhdr *obj_hdr; 499 char *chunk; 500 int i; 501 502 file_chunk = (struct icp_qat_uof_filechunkhdr *) 503 (buf + sizeof(struct icp_qat_uof_filehdr)); 504 for (i = 0; i < file_hdr->num_chunks; i++) { 505 if (!strncmp(file_chunk->chunk_id, chunk_id, 506 ICP_QAT_UOF_OBJID_LEN)) { 507 chunk = buf + file_chunk->offset; 508 if (file_chunk->checksum != qat_uclo_calc_str_checksum( 509 chunk, file_chunk->size)) 510 break; 511 obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL); 512 if (!obj_hdr) 513 break; 514 obj_hdr->file_buff = chunk; 515 obj_hdr->checksum = file_chunk->checksum; 516 obj_hdr->size = file_chunk->size; 517 return obj_hdr; 518 } 519 file_chunk++; 520 } 521 return NULL; 522 } 523 524 static int 525 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, 526 struct icp_qat_uof_image *image) 527 { 528 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; 529 struct icp_qat_uof_objtable *neigh_reg_tab; 530 struct icp_qat_uof_code_page *code_page; 531 532 code_page = (struct icp_qat_uof_code_page *) 533 ((char *)image + sizeof(struct icp_qat_uof_image)); 534 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 535 code_page->uc_var_tab_offset); 536 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + 537 code_page->imp_var_tab_offset); 538 imp_expr_tab = (struct icp_qat_uof_objtable *) 539 (encap_uof_obj->beg_uof + 540 code_page->imp_expr_tab_offset); 541 if (uc_var_tab->entry_num || imp_var_tab->entry_num || 542 imp_expr_tab->entry_num) { 543 pr_err("QAT: UOF can't contain imported variable to be parsed\n"); 544 return -EINVAL; 545 } 546 neigh_reg_tab = (struct icp_qat_uof_objtable *) 547 (encap_uof_obj->beg_uof + 548 code_page->neigh_reg_tab_offset); 549 if (neigh_reg_tab->entry_num) { 550 pr_err("QAT: UOF can't contain neighbor register table\n"); 551 return -EINVAL; 552 } 553 if (image->numpages > 1) { 554 pr_err("QAT: UOF can't contain multiple pages\n"); 555 return -EINVAL; 556 } 557 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { 558 pr_err("QAT: UOF can't use shared control store feature\n"); 559 return -EFAULT; 560 } 561 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { 562 pr_err("QAT: UOF can't use reloadable feature\n"); 563 return -EFAULT; 564 } 565 return 0; 566 } 567 568 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj 569 *encap_uof_obj, 570 struct icp_qat_uof_image *img, 571 struct icp_qat_uclo_encap_page *page) 572 { 573 struct icp_qat_uof_code_page *code_page; 574 struct icp_qat_uof_code_area *code_area; 575 struct icp_qat_uof_objtable *uword_block_tab; 576 struct icp_qat_uof_uword_block *uwblock; 577 int i; 578 579 code_page = (struct icp_qat_uof_code_page *) 580 ((char *)img + sizeof(struct icp_qat_uof_image)); 581 page->def_page = code_page->def_page; 582 page->page_region = code_page->page_region; 583 page->beg_addr_v = code_page->beg_addr_v; 584 page->beg_addr_p = code_page->beg_addr_p; 585 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + 586 code_page->code_area_offset); 587 page->micro_words_num = code_area->micro_words_num; 588 uword_block_tab = (struct icp_qat_uof_objtable *) 589 (encap_uof_obj->beg_uof + 590 code_area->uword_block_tab); 591 page->uwblock_num = uword_block_tab->entry_num; 592 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + 593 sizeof(struct icp_qat_uof_objtable)); 594 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; 595 for (i = 0; i < uword_block_tab->entry_num; i++) 596 page->uwblock[i].micro_words = 597 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; 598 } 599 600 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, 601 struct icp_qat_uclo_encapme *ae_uimage, 602 int max_image) 603 { 604 int i, j; 605 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; 606 struct icp_qat_uof_image *image; 607 struct icp_qat_uof_objtable *ae_regtab; 608 struct icp_qat_uof_objtable *init_reg_sym_tab; 609 struct icp_qat_uof_objtable *sbreak_tab; 610 struct icp_qat_uof_encap_obj *encap_uof_obj = 611 &obj_handle->encap_uof_obj; 612 613 for (j = 0; j < max_image; j++) { 614 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 615 ICP_QAT_UOF_IMAG, chunk_hdr); 616 if (!chunk_hdr) 617 break; 618 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + 619 chunk_hdr->offset); 620 ae_regtab = (struct icp_qat_uof_objtable *) 621 (image->reg_tab_offset + 622 obj_handle->obj_hdr->file_buff); 623 ae_uimage[j].ae_reg_num = ae_regtab->entry_num; 624 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) 625 (((char *)ae_regtab) + 626 sizeof(struct icp_qat_uof_objtable)); 627 init_reg_sym_tab = (struct icp_qat_uof_objtable *) 628 (image->init_reg_sym_tab + 629 obj_handle->obj_hdr->file_buff); 630 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; 631 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) 632 (((char *)init_reg_sym_tab) + 633 sizeof(struct icp_qat_uof_objtable)); 634 sbreak_tab = (struct icp_qat_uof_objtable *) 635 (image->sbreak_tab + obj_handle->obj_hdr->file_buff); 636 ae_uimage[j].sbreak_num = sbreak_tab->entry_num; 637 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) 638 (((char *)sbreak_tab) + 639 sizeof(struct icp_qat_uof_objtable)); 640 ae_uimage[j].img_ptr = image; 641 if (qat_uclo_check_image_compat(encap_uof_obj, image)) 642 goto out_err; 643 ae_uimage[j].page = 644 kzalloc(sizeof(struct icp_qat_uclo_encap_page), 645 GFP_KERNEL); 646 if (!ae_uimage[j].page) 647 goto out_err; 648 qat_uclo_map_image_page(encap_uof_obj, image, 649 ae_uimage[j].page); 650 } 651 return j; 652 out_err: 653 for (i = 0; i < j; i++) 654 kfree(ae_uimage[i].page); 655 return 0; 656 } 657 658 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) 659 { 660 int i, ae; 661 int mflag = 0; 662 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 663 unsigned long ae_mask = handle->hal_handle->ae_mask; 664 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 665 666 for_each_set_bit(ae, &ae_mask, max_ae) { 667 if (!test_bit(ae, &cfg_ae_mask)) 668 continue; 669 670 for (i = 0; i < obj_handle->uimage_num; i++) { 671 unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; 672 673 if (!test_bit(ae, &ae_assigned)) 674 continue; 675 mflag = 1; 676 if (qat_uclo_init_ae_data(obj_handle, ae, i)) 677 return -EINVAL; 678 } 679 } 680 if (!mflag) { 681 pr_err("QAT: uimage uses AE not set\n"); 682 return -EINVAL; 683 } 684 return 0; 685 } 686 687 static struct icp_qat_uof_strtable * 688 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, 689 char *tab_name, struct icp_qat_uof_strtable *str_table) 690 { 691 struct icp_qat_uof_chunkhdr *chunk_hdr; 692 693 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) 694 obj_hdr->file_buff, tab_name, NULL); 695 if (chunk_hdr) { 696 int hdr_size; 697 698 memcpy(&str_table->table_len, obj_hdr->file_buff + 699 chunk_hdr->offset, sizeof(str_table->table_len)); 700 hdr_size = (char *)&str_table->strings - (char *)str_table; 701 str_table->strings = (uintptr_t)obj_hdr->file_buff + 702 chunk_hdr->offset + hdr_size; 703 return str_table; 704 } 705 return NULL; 706 } 707 708 static void 709 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, 710 struct icp_qat_uclo_init_mem_table *init_mem_tab) 711 { 712 struct icp_qat_uof_chunkhdr *chunk_hdr; 713 714 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, 715 ICP_QAT_UOF_IMEM, NULL); 716 if (chunk_hdr) { 717 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + 718 chunk_hdr->offset, sizeof(unsigned int)); 719 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) 720 (encap_uof_obj->beg_uof + chunk_hdr->offset + 721 sizeof(unsigned int)); 722 } 723 } 724 725 static unsigned int 726 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) 727 { 728 switch (handle->pci_dev->device) { 729 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: 730 return ICP_QAT_AC_895XCC_DEV_TYPE; 731 case PCI_DEVICE_ID_INTEL_QAT_C62X: 732 return ICP_QAT_AC_C62X_DEV_TYPE; 733 case PCI_DEVICE_ID_INTEL_QAT_C3XXX: 734 return ICP_QAT_AC_C3XXX_DEV_TYPE; 735 case PCI_DEVICE_ID_INTEL_QAT_4XXX: 736 case PCI_DEVICE_ID_INTEL_QAT_401XX: 737 case PCI_DEVICE_ID_INTEL_QAT_402XX: 738 case PCI_DEVICE_ID_INTEL_QAT_420XX: 739 return ICP_QAT_AC_4XXX_A_DEV_TYPE; 740 default: 741 pr_err("QAT: unsupported device 0x%x\n", 742 handle->pci_dev->device); 743 return 0; 744 } 745 } 746 747 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) 748 { 749 unsigned int maj_ver, prod_type = obj_handle->prod_type; 750 751 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { 752 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", 753 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, 754 prod_type); 755 return -EINVAL; 756 } 757 maj_ver = obj_handle->prod_rev & 0xff; 758 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || 759 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { 760 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); 761 return -EINVAL; 762 } 763 return 0; 764 } 765 766 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, 767 unsigned char ae, unsigned char ctx_mask, 768 enum icp_qat_uof_regtype reg_type, 769 unsigned short reg_addr, unsigned int value) 770 { 771 switch (reg_type) { 772 case ICP_GPA_ABS: 773 case ICP_GPB_ABS: 774 ctx_mask = 0; 775 fallthrough; 776 case ICP_GPA_REL: 777 case ICP_GPB_REL: 778 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, 779 reg_addr, value); 780 case ICP_SR_ABS: 781 case ICP_DR_ABS: 782 case ICP_SR_RD_ABS: 783 case ICP_DR_RD_ABS: 784 ctx_mask = 0; 785 fallthrough; 786 case ICP_SR_REL: 787 case ICP_DR_REL: 788 case ICP_SR_RD_REL: 789 case ICP_DR_RD_REL: 790 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, 791 reg_addr, value); 792 case ICP_SR_WR_ABS: 793 case ICP_DR_WR_ABS: 794 ctx_mask = 0; 795 fallthrough; 796 case ICP_SR_WR_REL: 797 case ICP_DR_WR_REL: 798 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, 799 reg_addr, value); 800 case ICP_NEIGH_REL: 801 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); 802 default: 803 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); 804 return -EFAULT; 805 } 806 return 0; 807 } 808 809 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, 810 unsigned int ae, 811 struct icp_qat_uclo_encapme *encap_ae) 812 { 813 unsigned int i; 814 unsigned char ctx_mask; 815 struct icp_qat_uof_init_regsym *init_regsym; 816 817 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == 818 ICP_QAT_UCLO_MAX_CTX) 819 ctx_mask = 0xff; 820 else 821 ctx_mask = 0x55; 822 823 for (i = 0; i < encap_ae->init_regsym_num; i++) { 824 unsigned int exp_res; 825 826 init_regsym = &encap_ae->init_regsym[i]; 827 exp_res = init_regsym->value; 828 switch (init_regsym->init_type) { 829 case ICP_QAT_UOF_INIT_REG: 830 qat_uclo_init_reg(handle, ae, ctx_mask, 831 (enum icp_qat_uof_regtype) 832 init_regsym->reg_type, 833 (unsigned short)init_regsym->reg_addr, 834 exp_res); 835 break; 836 case ICP_QAT_UOF_INIT_REG_CTX: 837 /* check if ctx is appropriate for the ctxMode */ 838 if (!((1 << init_regsym->ctx) & ctx_mask)) { 839 pr_err("QAT: invalid ctx num = 0x%x\n", 840 init_regsym->ctx); 841 return -EINVAL; 842 } 843 qat_uclo_init_reg(handle, ae, 844 (unsigned char) 845 (1 << init_regsym->ctx), 846 (enum icp_qat_uof_regtype) 847 init_regsym->reg_type, 848 (unsigned short)init_regsym->reg_addr, 849 exp_res); 850 break; 851 case ICP_QAT_UOF_INIT_EXPR: 852 pr_err("QAT: INIT_EXPR feature not supported\n"); 853 return -EINVAL; 854 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: 855 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); 856 return -EINVAL; 857 default: 858 break; 859 } 860 } 861 return 0; 862 } 863 864 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) 865 { 866 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 867 unsigned long ae_mask = handle->hal_handle->ae_mask; 868 struct icp_qat_uclo_aedata *aed; 869 unsigned int s, ae; 870 871 if (obj_handle->global_inited) 872 return 0; 873 if (obj_handle->init_mem_tab.entry_num) { 874 if (qat_uclo_init_memory(handle)) { 875 pr_err("QAT: initialize memory failed\n"); 876 return -EINVAL; 877 } 878 } 879 880 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 881 aed = &obj_handle->ae_data[ae]; 882 for (s = 0; s < aed->slice_num; s++) { 883 if (!aed->ae_slices[s].encap_image) 884 continue; 885 if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) 886 return -EINVAL; 887 } 888 } 889 obj_handle->global_inited = 1; 890 return 0; 891 } 892 893 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, 894 struct icp_qat_uclo_objhandle *obj_handle, 895 unsigned char ae, 896 struct icp_qat_uof_image *uof_image) 897 { 898 unsigned char mode; 899 int ret; 900 901 mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); 902 ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); 903 if (ret) { 904 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); 905 return ret; 906 } 907 if (handle->chip_info->nn) { 908 mode = ICP_QAT_NN_MODE(uof_image->ae_mode); 909 ret = qat_hal_set_ae_nn_mode(handle, ae, mode); 910 if (ret) { 911 pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); 912 return ret; 913 } 914 } 915 mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); 916 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); 917 if (ret) { 918 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); 919 return ret; 920 } 921 mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); 922 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); 923 if (ret) { 924 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); 925 return ret; 926 } 927 if (handle->chip_info->lm2lm3) { 928 mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); 929 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); 930 if (ret) { 931 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); 932 return ret; 933 } 934 mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); 935 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); 936 if (ret) { 937 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); 938 return ret; 939 } 940 mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); 941 qat_hal_set_ae_tindex_mode(handle, ae, mode); 942 } 943 return 0; 944 } 945 946 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) 947 { 948 struct icp_qat_uof_image *uof_image; 949 struct icp_qat_uclo_aedata *ae_data; 950 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 951 unsigned long ae_mask = handle->hal_handle->ae_mask; 952 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 953 unsigned char ae, s; 954 int error; 955 956 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 957 if (!test_bit(ae, &cfg_ae_mask)) 958 continue; 959 960 ae_data = &obj_handle->ae_data[ae]; 961 for (s = 0; s < min_t(unsigned int, ae_data->slice_num, 962 ICP_QAT_UCLO_MAX_CTX); s++) { 963 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) 964 continue; 965 uof_image = ae_data->ae_slices[s].encap_image->img_ptr; 966 error = qat_hal_set_modes(handle, obj_handle, ae, 967 uof_image); 968 if (error) 969 return error; 970 } 971 } 972 return 0; 973 } 974 975 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) 976 { 977 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 978 struct icp_qat_uclo_encapme *image; 979 int a; 980 981 for (a = 0; a < obj_handle->uimage_num; a++) { 982 image = &obj_handle->ae_uimage[a]; 983 image->uwords_num = image->page->beg_addr_p + 984 image->page->micro_words_num; 985 } 986 } 987 988 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) 989 { 990 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 991 unsigned int ae; 992 993 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; 994 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) 995 obj_handle->obj_hdr->file_buff; 996 obj_handle->uword_in_bytes = 6; 997 obj_handle->prod_type = qat_uclo_get_dev_type(handle); 998 obj_handle->prod_rev = PID_MAJOR_REV | 999 (PID_MINOR_REV & handle->hal_handle->revision_id); 1000 if (qat_uclo_check_uof_compat(obj_handle)) { 1001 pr_err("QAT: UOF incompatible\n"); 1002 return -EINVAL; 1003 } 1004 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), 1005 GFP_KERNEL); 1006 if (!obj_handle->uword_buf) 1007 return -ENOMEM; 1008 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; 1009 if (!obj_handle->obj_hdr->file_buff || 1010 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, 1011 &obj_handle->str_table)) { 1012 pr_err("QAT: UOF doesn't have effective images\n"); 1013 goto out_err; 1014 } 1015 obj_handle->uimage_num = 1016 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, 1017 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); 1018 if (!obj_handle->uimage_num) 1019 goto out_err; 1020 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { 1021 pr_err("QAT: Bad object\n"); 1022 goto out_check_uof_aemask_err; 1023 } 1024 qat_uclo_init_uword_num(handle); 1025 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, 1026 &obj_handle->init_mem_tab); 1027 if (qat_uclo_set_ae_mode(handle)) 1028 goto out_check_uof_aemask_err; 1029 return 0; 1030 out_check_uof_aemask_err: 1031 for (ae = 0; ae < obj_handle->uimage_num; ae++) 1032 kfree(obj_handle->ae_uimage[ae].page); 1033 out_err: 1034 kfree(obj_handle->uword_buf); 1035 return -EFAULT; 1036 } 1037 1038 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1039 struct icp_qat_suof_filehdr *suof_ptr, 1040 int suof_size) 1041 { 1042 unsigned int check_sum = 0; 1043 unsigned int min_ver_offset = 0; 1044 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1045 1046 suof_handle->file_id = ICP_QAT_SUOF_FID; 1047 suof_handle->suof_buf = (char *)suof_ptr; 1048 suof_handle->suof_size = suof_size; 1049 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, 1050 min_ver); 1051 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, 1052 min_ver_offset); 1053 if (check_sum != suof_ptr->check_sum) { 1054 pr_err("QAT: incorrect SUOF checksum\n"); 1055 return -EINVAL; 1056 } 1057 suof_handle->check_sum = suof_ptr->check_sum; 1058 suof_handle->min_ver = suof_ptr->min_ver; 1059 suof_handle->maj_ver = suof_ptr->maj_ver; 1060 suof_handle->fw_type = suof_ptr->fw_type; 1061 return 0; 1062 } 1063 1064 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, 1065 struct icp_qat_suof_img_hdr *suof_img_hdr, 1066 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1067 { 1068 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1069 unsigned int offset = ICP_QAT_AE_IMG_OFFSET(handle); 1070 struct icp_qat_simg_ae_mode *ae_mode; 1071 struct icp_qat_suof_objhdr *suof_objhdr; 1072 1073 suof_img_hdr->simg_buf = (suof_handle->suof_buf + 1074 suof_chunk_hdr->offset + 1075 sizeof(*suof_objhdr)); 1076 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) 1077 (suof_handle->suof_buf + 1078 suof_chunk_hdr->offset))->img_length; 1079 1080 suof_img_hdr->css_header = suof_img_hdr->simg_buf; 1081 suof_img_hdr->css_simg = suof_img_hdr->css_header + offset; 1082 1083 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); 1084 suof_img_hdr->ae_mask = ae_mode->ae_mask; 1085 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; 1086 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; 1087 suof_img_hdr->fw_type = ae_mode->fw_type; 1088 } 1089 1090 static void 1091 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, 1092 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) 1093 { 1094 char **sym_str = (char **)&suof_handle->sym_str; 1095 unsigned int *sym_size = &suof_handle->sym_size; 1096 struct icp_qat_suof_strtable *str_table_obj; 1097 1098 *sym_size = *(unsigned int *)(uintptr_t) 1099 (suof_chunk_hdr->offset + suof_handle->suof_buf); 1100 *sym_str = (char *)(uintptr_t) 1101 (suof_handle->suof_buf + suof_chunk_hdr->offset + 1102 sizeof(str_table_obj->tab_length)); 1103 } 1104 1105 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, 1106 struct icp_qat_suof_img_hdr *img_hdr) 1107 { 1108 struct icp_qat_simg_ae_mode *img_ae_mode = NULL; 1109 unsigned int prod_rev, maj_ver, prod_type; 1110 1111 prod_type = qat_uclo_get_dev_type(handle); 1112 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; 1113 prod_rev = PID_MAJOR_REV | 1114 (PID_MINOR_REV & handle->hal_handle->revision_id); 1115 if (img_ae_mode->dev_type != prod_type) { 1116 pr_err("QAT: incompatible product type %x\n", 1117 img_ae_mode->dev_type); 1118 return -EINVAL; 1119 } 1120 maj_ver = prod_rev & 0xff; 1121 if (maj_ver > img_ae_mode->devmax_ver || 1122 maj_ver < img_ae_mode->devmin_ver) { 1123 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); 1124 return -EINVAL; 1125 } 1126 return 0; 1127 } 1128 1129 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) 1130 { 1131 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 1132 1133 kfree(sobj_handle->img_table.simg_hdr); 1134 sobj_handle->img_table.simg_hdr = NULL; 1135 kfree(handle->sobj_handle); 1136 handle->sobj_handle = NULL; 1137 } 1138 1139 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, 1140 unsigned int img_id, unsigned int num_simgs) 1141 { 1142 struct icp_qat_suof_img_hdr img_header; 1143 1144 if (img_id != num_simgs - 1) { 1145 memcpy(&img_header, &suof_img_hdr[num_simgs - 1], 1146 sizeof(*suof_img_hdr)); 1147 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], 1148 sizeof(*suof_img_hdr)); 1149 memcpy(&suof_img_hdr[img_id], &img_header, 1150 sizeof(*suof_img_hdr)); 1151 } 1152 } 1153 1154 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, 1155 struct icp_qat_suof_filehdr *suof_ptr, 1156 int suof_size) 1157 { 1158 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; 1159 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; 1160 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; 1161 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; 1162 unsigned int i = 0; 1163 struct icp_qat_suof_img_hdr img_header; 1164 1165 if (!suof_ptr || suof_size == 0) { 1166 pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); 1167 return -EINVAL; 1168 } 1169 if (qat_uclo_check_suof_format(suof_ptr)) 1170 return -EINVAL; 1171 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); 1172 if (ret) 1173 return ret; 1174 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) 1175 ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); 1176 1177 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); 1178 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; 1179 1180 if (suof_handle->img_table.num_simgs != 0) { 1181 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs, 1182 sizeof(img_header), 1183 GFP_KERNEL); 1184 if (!suof_img_hdr) 1185 return -ENOMEM; 1186 suof_handle->img_table.simg_hdr = suof_img_hdr; 1187 1188 for (i = 0; i < suof_handle->img_table.num_simgs; i++) { 1189 qat_uclo_map_simg(handle, &suof_img_hdr[i], 1190 &suof_chunk_hdr[1 + i]); 1191 ret = qat_uclo_check_simg_compat(handle, 1192 &suof_img_hdr[i]); 1193 if (ret) 1194 return ret; 1195 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; 1196 if ((suof_img_hdr[i].ae_mask & 0x1) != 0) 1197 ae0_img = i; 1198 } 1199 1200 if (!handle->chip_info->tgroup_share_ustore) { 1201 qat_uclo_tail_img(suof_img_hdr, ae0_img, 1202 suof_handle->img_table.num_simgs); 1203 } 1204 } 1205 return 0; 1206 } 1207 1208 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low) 1209 1210 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, 1211 struct icp_qat_fw_auth_desc *desc) 1212 { 1213 u32 fcu_sts, retry = 0; 1214 u32 fcu_ctl_csr, fcu_sts_csr; 1215 u32 fcu_dram_hi_csr, fcu_dram_lo_csr; 1216 u64 bus_addr; 1217 1218 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) 1219 - sizeof(struct icp_qat_auth_chunk); 1220 1221 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1222 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1223 fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; 1224 fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; 1225 1226 SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32)); 1227 SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); 1228 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); 1229 1230 do { 1231 msleep(FW_AUTH_WAIT_PERIOD); 1232 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1233 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) 1234 goto auth_fail; 1235 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) 1236 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) 1237 return 0; 1238 } while (retry++ < FW_AUTH_MAX_RETRY); 1239 auth_fail: 1240 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", 1241 fcu_sts & FCU_AUTH_STS_MASK, retry); 1242 return -EINVAL; 1243 } 1244 1245 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, 1246 int imgid) 1247 { 1248 struct icp_qat_suof_handle *sobj_handle; 1249 1250 if (!handle->chip_info->tgroup_share_ustore) 1251 return false; 1252 1253 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; 1254 if (handle->hal_handle->admin_ae_mask & 1255 sobj_handle->img_table.simg_hdr[imgid].ae_mask) 1256 return false; 1257 1258 return true; 1259 } 1260 1261 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, 1262 struct icp_qat_fw_auth_desc *desc) 1263 { 1264 unsigned long ae_mask = handle->hal_handle->ae_mask; 1265 unsigned long desc_ae_mask = desc->ae_mask; 1266 u32 fcu_sts, ae_broadcast_mask = 0; 1267 u32 fcu_loaded_csr, ae_loaded; 1268 u32 fcu_sts_csr, fcu_ctl_csr; 1269 unsigned int ae, retry = 0; 1270 1271 if (handle->chip_info->tgroup_share_ustore) { 1272 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1273 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1274 fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1275 } else { 1276 pr_err("Chip 0x%x doesn't support broadcast load\n", 1277 handle->pci_dev->device); 1278 return -EINVAL; 1279 } 1280 1281 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 1282 if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { 1283 pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); 1284 return -EINVAL; 1285 } 1286 1287 if (test_bit(ae, &desc_ae_mask)) 1288 ae_broadcast_mask |= 1 << ae; 1289 } 1290 1291 if (ae_broadcast_mask) { 1292 SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, 1293 ae_broadcast_mask); 1294 1295 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); 1296 1297 do { 1298 msleep(FW_AUTH_WAIT_PERIOD); 1299 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1300 fcu_sts &= FCU_AUTH_STS_MASK; 1301 1302 if (fcu_sts == FCU_STS_LOAD_FAIL) { 1303 pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); 1304 return -EINVAL; 1305 } else if (fcu_sts == FCU_STS_LOAD_DONE) { 1306 ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); 1307 ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; 1308 1309 if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) 1310 break; 1311 } 1312 } while (retry++ < FW_AUTH_MAX_RETRY); 1313 1314 if (retry > FW_AUTH_MAX_RETRY) { 1315 pr_err("QAT: broadcast load failed timeout %d\n", retry); 1316 return -EINVAL; 1317 } 1318 } 1319 return 0; 1320 } 1321 1322 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, 1323 struct icp_firml_dram_desc *dram_desc, 1324 unsigned int size) 1325 { 1326 void *vptr; 1327 dma_addr_t ptr; 1328 1329 vptr = dma_alloc_coherent(&handle->pci_dev->dev, 1330 size, &ptr, GFP_KERNEL); 1331 if (!vptr) 1332 return -ENOMEM; 1333 dram_desc->dram_base_addr_v = vptr; 1334 dram_desc->dram_bus_addr = ptr; 1335 dram_desc->dram_size = size; 1336 return 0; 1337 } 1338 1339 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, 1340 struct icp_firml_dram_desc *dram_desc) 1341 { 1342 if (handle && dram_desc && dram_desc->dram_base_addr_v) { 1343 dma_free_coherent(&handle->pci_dev->dev, 1344 (size_t)(dram_desc->dram_size), 1345 dram_desc->dram_base_addr_v, 1346 dram_desc->dram_bus_addr); 1347 } 1348 1349 if (dram_desc) 1350 memset(dram_desc, 0, sizeof(*dram_desc)); 1351 } 1352 1353 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, 1354 struct icp_qat_fw_auth_desc **desc) 1355 { 1356 struct icp_firml_dram_desc dram_desc; 1357 1358 if (*desc) { 1359 dram_desc.dram_base_addr_v = *desc; 1360 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) 1361 (*desc))->chunk_bus_addr; 1362 dram_desc.dram_size = ((struct icp_qat_auth_chunk *) 1363 (*desc))->chunk_size; 1364 qat_uclo_simg_free(handle, &dram_desc); 1365 } 1366 } 1367 1368 static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, 1369 char *image, unsigned int size, 1370 unsigned int fw_type) 1371 { 1372 char *fw_type_name = fw_type ? "MMP" : "AE"; 1373 unsigned int css_dword_size = sizeof(u32); 1374 1375 if (handle->chip_info->fw_auth) { 1376 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1377 unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); 1378 1379 if ((css_hdr->header_len * css_dword_size) != header_len) 1380 goto err; 1381 if ((css_hdr->size * css_dword_size) != size) 1382 goto err; 1383 if (fw_type != css_hdr->fw_type) 1384 goto err; 1385 if (size <= header_len) 1386 goto err; 1387 size -= header_len; 1388 } 1389 1390 if (fw_type == CSS_AE_FIRMWARE) { 1391 if (size < sizeof(struct icp_qat_simg_ae_mode *) + 1392 ICP_QAT_SIMG_AE_INIT_SEQ_LEN) 1393 goto err; 1394 if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) 1395 goto err; 1396 } else if (fw_type == CSS_MMP_FIRMWARE) { 1397 if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) 1398 goto err; 1399 } else { 1400 pr_err("QAT: Unsupported firmware type\n"); 1401 return -EINVAL; 1402 } 1403 return 0; 1404 1405 err: 1406 pr_err("QAT: Invalid %s firmware image\n", fw_type_name); 1407 return -EINVAL; 1408 } 1409 1410 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, 1411 char *image, unsigned int size, 1412 struct icp_qat_fw_auth_desc **desc) 1413 { 1414 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; 1415 struct icp_qat_fw_auth_desc *auth_desc; 1416 struct icp_qat_auth_chunk *auth_chunk; 1417 u64 virt_addr, bus_addr, virt_base; 1418 unsigned int simg_offset = sizeof(*auth_chunk); 1419 struct icp_qat_simg_ae_mode *simg_ae_mode; 1420 struct icp_firml_dram_desc img_desc; 1421 int ret; 1422 1423 ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); 1424 if (ret) { 1425 pr_err("QAT: error, allocate continuous dram fail\n"); 1426 return ret; 1427 } 1428 1429 if (!IS_ALIGNED(img_desc.dram_size, 8) || !img_desc.dram_bus_addr) { 1430 pr_debug("QAT: invalid address\n"); 1431 qat_uclo_simg_free(handle, &img_desc); 1432 return -EINVAL; 1433 } 1434 1435 auth_chunk = img_desc.dram_base_addr_v; 1436 auth_chunk->chunk_size = img_desc.dram_size; 1437 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; 1438 virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; 1439 bus_addr = img_desc.dram_bus_addr + simg_offset; 1440 auth_desc = img_desc.dram_base_addr_v; 1441 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1442 auth_desc->css_hdr_low = (unsigned int)bus_addr; 1443 virt_addr = virt_base; 1444 1445 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); 1446 /* pub key */ 1447 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + 1448 sizeof(*css_hdr); 1449 virt_addr = virt_addr + sizeof(*css_hdr); 1450 1451 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1452 auth_desc->fwsk_pub_low = (unsigned int)bus_addr; 1453 1454 memcpy((void *)(uintptr_t)virt_addr, 1455 (void *)(image + sizeof(*css_hdr)), 1456 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); 1457 /* padding */ 1458 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1459 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); 1460 1461 /* exponent */ 1462 memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1463 ICP_QAT_CSS_FWSK_PAD_LEN(handle)), 1464 (void *)(image + sizeof(*css_hdr) + 1465 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 1466 sizeof(unsigned int)); 1467 1468 /* signature */ 1469 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, 1470 auth_desc->fwsk_pub_low) + 1471 ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1472 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1473 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1474 auth_desc->signature_low = (unsigned int)bus_addr; 1475 1476 memcpy((void *)(uintptr_t)virt_addr, 1477 (void *)(image + sizeof(*css_hdr) + 1478 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + 1479 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), 1480 ICP_QAT_CSS_SIGNATURE_LEN(handle)); 1481 1482 bus_addr = ADD_ADDR(auth_desc->signature_high, 1483 auth_desc->signature_low) + 1484 ICP_QAT_CSS_SIGNATURE_LEN(handle); 1485 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1486 1487 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1488 auth_desc->img_low = (unsigned int)bus_addr; 1489 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); 1490 if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + 1491 ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { 1492 pr_err("QAT: insufficient memory size for authentication data\n"); 1493 qat_uclo_simg_free(handle, &img_desc); 1494 return -ENOMEM; 1495 } 1496 1497 memcpy((void *)(uintptr_t)virt_addr, 1498 (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), 1499 auth_desc->img_len); 1500 virt_addr = virt_base; 1501 /* AE firmware */ 1502 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == 1503 CSS_AE_FIRMWARE) { 1504 auth_desc->img_ae_mode_data_high = auth_desc->img_high; 1505 auth_desc->img_ae_mode_data_low = auth_desc->img_low; 1506 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, 1507 auth_desc->img_ae_mode_data_low) + 1508 sizeof(struct icp_qat_simg_ae_mode); 1509 1510 auth_desc->img_ae_init_data_high = 1511 (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1512 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; 1513 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; 1514 auth_desc->img_ae_insts_high = 1515 (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); 1516 auth_desc->img_ae_insts_low = (unsigned int)bus_addr; 1517 virt_addr += sizeof(struct icp_qat_css_hdr); 1518 virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); 1519 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); 1520 simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; 1521 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; 1522 } else { 1523 auth_desc->img_ae_insts_high = auth_desc->img_high; 1524 auth_desc->img_ae_insts_low = auth_desc->img_low; 1525 } 1526 *desc = auth_desc; 1527 return 0; 1528 } 1529 1530 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, 1531 struct icp_qat_fw_auth_desc *desc) 1532 { 1533 unsigned long ae_mask = handle->hal_handle->ae_mask; 1534 u32 fcu_sts_csr, fcu_ctl_csr; 1535 u32 loaded_aes, loaded_csr; 1536 unsigned int i; 1537 u32 fcu_sts; 1538 1539 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; 1540 fcu_sts_csr = handle->chip_info->fcu_sts_csr; 1541 loaded_csr = handle->chip_info->fcu_loaded_ae_csr; 1542 1543 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { 1544 int retry = 0; 1545 1546 if (!((desc->ae_mask >> i) & 0x1)) 1547 continue; 1548 if (qat_hal_check_ae_active(handle, i)) { 1549 pr_err("QAT: AE %d is active\n", i); 1550 return -EINVAL; 1551 } 1552 SET_CAP_CSR(handle, fcu_ctl_csr, 1553 (FCU_CTRL_CMD_LOAD | 1554 (1 << FCU_CTRL_BROADCAST_POS) | 1555 (i << FCU_CTRL_AE_POS))); 1556 1557 do { 1558 msleep(FW_AUTH_WAIT_PERIOD); 1559 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); 1560 if ((fcu_sts & FCU_AUTH_STS_MASK) == 1561 FCU_STS_LOAD_DONE) { 1562 loaded_aes = GET_CAP_CSR(handle, loaded_csr); 1563 loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; 1564 if (loaded_aes & (1 << i)) 1565 break; 1566 } 1567 } while (retry++ < FW_AUTH_MAX_RETRY); 1568 if (retry > FW_AUTH_MAX_RETRY) { 1569 pr_err("QAT: firmware load failed timeout %x\n", retry); 1570 return -EINVAL; 1571 } 1572 } 1573 return 0; 1574 } 1575 1576 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, 1577 void *addr_ptr, int mem_size) 1578 { 1579 struct icp_qat_suof_handle *suof_handle; 1580 1581 suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL); 1582 if (!suof_handle) 1583 return -ENOMEM; 1584 handle->sobj_handle = suof_handle; 1585 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { 1586 qat_uclo_del_suof(handle); 1587 pr_err("QAT: map SUOF failed\n"); 1588 return -EINVAL; 1589 } 1590 return 0; 1591 } 1592 1593 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, 1594 void *addr_ptr, int mem_size) 1595 { 1596 struct icp_qat_fw_auth_desc *desc = NULL; 1597 int status = 0; 1598 int ret; 1599 1600 ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); 1601 if (ret) 1602 return ret; 1603 1604 if (handle->chip_info->fw_auth) { 1605 status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); 1606 if (!status) 1607 status = qat_uclo_auth_fw(handle, desc); 1608 qat_uclo_ummap_auth_fw(handle, &desc); 1609 } else { 1610 if (handle->chip_info->mmp_sram_size < mem_size) { 1611 pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); 1612 return -EFBIG; 1613 } 1614 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); 1615 } 1616 return status; 1617 } 1618 1619 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, 1620 void *addr_ptr, int mem_size) 1621 { 1622 struct icp_qat_uof_filehdr *filehdr; 1623 struct icp_qat_uclo_objhandle *objhdl; 1624 1625 objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); 1626 if (!objhdl) 1627 return -ENOMEM; 1628 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); 1629 if (!objhdl->obj_buf) 1630 goto out_objbuf_err; 1631 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; 1632 if (qat_uclo_check_uof_format(filehdr)) 1633 goto out_objhdr_err; 1634 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, 1635 ICP_QAT_UOF_OBJS); 1636 if (!objhdl->obj_hdr) { 1637 pr_err("QAT: object file chunk is null\n"); 1638 goto out_objhdr_err; 1639 } 1640 handle->obj_handle = objhdl; 1641 if (qat_uclo_parse_uof_obj(handle)) 1642 goto out_overlay_obj_err; 1643 return 0; 1644 1645 out_overlay_obj_err: 1646 handle->obj_handle = NULL; 1647 kfree(objhdl->obj_hdr); 1648 out_objhdr_err: 1649 kfree(objhdl->obj_buf); 1650 out_objbuf_err: 1651 kfree(objhdl); 1652 return -ENOMEM; 1653 } 1654 1655 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, 1656 struct icp_qat_mof_file_hdr *mof_ptr, 1657 u32 mof_size) 1658 { 1659 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1660 unsigned int min_ver_offset; 1661 unsigned int checksum; 1662 1663 mobj_handle->file_id = ICP_QAT_MOF_FID; 1664 mobj_handle->mof_buf = (char *)mof_ptr; 1665 mobj_handle->mof_size = mof_size; 1666 1667 min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, 1668 min_ver); 1669 checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, 1670 min_ver_offset); 1671 if (checksum != mof_ptr->checksum) { 1672 pr_err("QAT: incorrect MOF checksum\n"); 1673 return -EINVAL; 1674 } 1675 1676 mobj_handle->checksum = mof_ptr->checksum; 1677 mobj_handle->min_ver = mof_ptr->min_ver; 1678 mobj_handle->maj_ver = mof_ptr->maj_ver; 1679 return 0; 1680 } 1681 1682 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) 1683 { 1684 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; 1685 1686 kfree(mobj_handle->obj_table.obj_hdr); 1687 mobj_handle->obj_table.obj_hdr = NULL; 1688 kfree(handle->mobj_handle); 1689 handle->mobj_handle = NULL; 1690 } 1691 1692 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, 1693 const char *obj_name, char **obj_ptr, 1694 unsigned int *obj_size) 1695 { 1696 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; 1697 unsigned int i; 1698 1699 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { 1700 if (!strncmp(obj_hdr[i].obj_name, obj_name, 1701 ICP_QAT_SUOF_OBJ_NAME_LEN)) { 1702 *obj_ptr = obj_hdr[i].obj_buf; 1703 *obj_size = obj_hdr[i].obj_size; 1704 return 0; 1705 } 1706 } 1707 1708 pr_err("QAT: object %s is not found inside MOF\n", obj_name); 1709 return -EINVAL; 1710 } 1711 1712 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, 1713 struct icp_qat_mof_objhdr *mobj_hdr, 1714 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) 1715 { 1716 u8 *obj; 1717 1718 if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, 1719 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1720 obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; 1721 } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, 1722 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { 1723 obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; 1724 } else { 1725 pr_err("QAT: unsupported chunk id\n"); 1726 return -EINVAL; 1727 } 1728 mobj_hdr->obj_buf = obj; 1729 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; 1730 mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; 1731 return 0; 1732 } 1733 1734 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) 1735 { 1736 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; 1737 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; 1738 struct icp_qat_mof_obj_hdr *uobj_hdr; 1739 struct icp_qat_mof_obj_hdr *sobj_hdr; 1740 struct icp_qat_mof_objhdr *mobj_hdr; 1741 unsigned int uobj_chunk_num = 0; 1742 unsigned int sobj_chunk_num = 0; 1743 unsigned int *valid_chunk; 1744 int ret, i; 1745 1746 uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; 1747 sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; 1748 if (uobj_hdr) 1749 uobj_chunk_num = uobj_hdr->num_chunks; 1750 if (sobj_hdr) 1751 sobj_chunk_num = sobj_hdr->num_chunks; 1752 1753 mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * 1754 sizeof(*mobj_hdr), GFP_KERNEL); 1755 if (!mobj_hdr) 1756 return -ENOMEM; 1757 1758 mobj_handle->obj_table.obj_hdr = mobj_hdr; 1759 valid_chunk = &mobj_handle->obj_table.num_objs; 1760 uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1761 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); 1762 sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) 1763 ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); 1764 1765 /* map uof objects */ 1766 for (i = 0; i < uobj_chunk_num; i++) { 1767 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1768 &mobj_hdr[*valid_chunk], 1769 &uobj_chunkhdr[i]); 1770 if (ret) 1771 return ret; 1772 (*valid_chunk)++; 1773 } 1774 1775 /* map suof objects */ 1776 for (i = 0; i < sobj_chunk_num; i++) { 1777 ret = qat_uclo_map_obj_from_mof(mobj_handle, 1778 &mobj_hdr[*valid_chunk], 1779 &sobj_chunkhdr[i]); 1780 if (ret) 1781 return ret; 1782 (*valid_chunk)++; 1783 } 1784 1785 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { 1786 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); 1787 return -EINVAL; 1788 } 1789 return 0; 1790 } 1791 1792 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, 1793 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1794 { 1795 char **sym_str = (char **)&mobj_handle->sym_str; 1796 unsigned int *sym_size = &mobj_handle->sym_size; 1797 struct icp_qat_mof_str_table *str_table_obj; 1798 1799 *sym_size = *(unsigned int *)(uintptr_t) 1800 (mof_chunkhdr->offset + mobj_handle->mof_buf); 1801 *sym_str = (char *)(uintptr_t) 1802 (mobj_handle->mof_buf + mof_chunkhdr->offset + 1803 sizeof(str_table_obj->tab_len)); 1804 } 1805 1806 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, 1807 struct icp_qat_mof_chunkhdr *mof_chunkhdr) 1808 { 1809 char *chunk_id = mof_chunkhdr->chunk_id; 1810 1811 if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1812 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); 1813 else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1814 mobj_handle->uobjs_hdr = mobj_handle->mof_buf + 1815 mof_chunkhdr->offset; 1816 else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) 1817 mobj_handle->sobjs_hdr = mobj_handle->mof_buf + 1818 mof_chunkhdr->offset; 1819 } 1820 1821 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) 1822 { 1823 int maj = mof_hdr->maj_ver & 0xff; 1824 int min = mof_hdr->min_ver & 0xff; 1825 1826 if (mof_hdr->file_id != ICP_QAT_MOF_FID) { 1827 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); 1828 return -EINVAL; 1829 } 1830 1831 if (mof_hdr->num_chunks <= 0x1) { 1832 pr_err("QAT: MOF chunk amount is incorrect\n"); 1833 return -EINVAL; 1834 } 1835 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { 1836 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", 1837 maj, min); 1838 return -EINVAL; 1839 } 1840 return 0; 1841 } 1842 1843 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, 1844 struct icp_qat_mof_file_hdr *mof_ptr, 1845 u32 mof_size, const char *obj_name, 1846 char **obj_ptr, unsigned int *obj_size) 1847 { 1848 struct icp_qat_mof_chunkhdr *mof_chunkhdr; 1849 unsigned int file_id = mof_ptr->file_id; 1850 struct icp_qat_mof_handle *mobj_handle; 1851 unsigned short chunks_num; 1852 unsigned int i; 1853 int ret; 1854 1855 if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { 1856 if (obj_ptr) 1857 *obj_ptr = (char *)mof_ptr; 1858 if (obj_size) 1859 *obj_size = mof_size; 1860 return 0; 1861 } 1862 if (qat_uclo_check_mof_format(mof_ptr)) 1863 return -EINVAL; 1864 1865 mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); 1866 if (!mobj_handle) 1867 return -ENOMEM; 1868 1869 handle->mobj_handle = mobj_handle; 1870 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); 1871 if (ret) 1872 return ret; 1873 1874 mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); 1875 chunks_num = mof_ptr->num_chunks; 1876 1877 /* Parse MOF file chunks */ 1878 for (i = 0; i < chunks_num; i++) 1879 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); 1880 1881 /* All sym_objs uobjs and sobjs should be available */ 1882 if (!mobj_handle->sym_str || 1883 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) 1884 return -EINVAL; 1885 1886 ret = qat_uclo_map_objs_from_mof(mobj_handle); 1887 if (ret) 1888 return ret; 1889 1890 /* Seek specified uof object in MOF */ 1891 return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, 1892 obj_ptr, obj_size); 1893 } 1894 1895 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, 1896 void *addr_ptr, u32 mem_size, const char *obj_name) 1897 { 1898 char *obj_addr; 1899 u32 obj_size; 1900 int ret; 1901 1902 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= 1903 (sizeof(handle->hal_handle->ae_mask) * 8)); 1904 1905 if (!handle || !addr_ptr || mem_size < 24) 1906 return -EINVAL; 1907 1908 if (obj_name) { 1909 ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, 1910 &obj_addr, &obj_size); 1911 if (ret) 1912 return ret; 1913 } else { 1914 obj_addr = addr_ptr; 1915 obj_size = mem_size; 1916 } 1917 1918 return (handle->chip_info->fw_auth) ? 1919 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : 1920 qat_uclo_map_uof_obj(handle, obj_addr, obj_size); 1921 } 1922 1923 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) 1924 { 1925 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1926 unsigned int a; 1927 1928 if (handle->mobj_handle) 1929 qat_uclo_del_mof(handle); 1930 if (handle->sobj_handle) 1931 qat_uclo_del_suof(handle); 1932 if (!obj_handle) 1933 return; 1934 1935 kfree(obj_handle->uword_buf); 1936 for (a = 0; a < obj_handle->uimage_num; a++) 1937 kfree(obj_handle->ae_uimage[a].page); 1938 1939 for (a = 0; a < handle->hal_handle->ae_max_num; a++) 1940 qat_uclo_free_ae_data(&obj_handle->ae_data[a]); 1941 1942 kfree(obj_handle->obj_hdr); 1943 kfree(obj_handle->obj_buf); 1944 kfree(obj_handle); 1945 handle->obj_handle = NULL; 1946 } 1947 1948 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, 1949 struct icp_qat_uclo_encap_page *encap_page, 1950 u64 *uword, unsigned int addr_p, 1951 unsigned int raddr, u64 fill) 1952 { 1953 unsigned int i, addr; 1954 u64 uwrd = 0; 1955 1956 if (!encap_page) { 1957 *uword = fill; 1958 return; 1959 } 1960 addr = (encap_page->page_region) ? raddr : addr_p; 1961 for (i = 0; i < encap_page->uwblock_num; i++) { 1962 if (addr >= encap_page->uwblock[i].start_addr && 1963 addr <= encap_page->uwblock[i].start_addr + 1964 encap_page->uwblock[i].words_num - 1) { 1965 addr -= encap_page->uwblock[i].start_addr; 1966 addr *= obj_handle->uword_in_bytes; 1967 memcpy(&uwrd, (void *)(((uintptr_t) 1968 encap_page->uwblock[i].micro_words) + addr), 1969 obj_handle->uword_in_bytes); 1970 uwrd = uwrd & GENMASK_ULL(43, 0); 1971 } 1972 } 1973 *uword = uwrd; 1974 if (*uword == INVLD_UWORD) 1975 *uword = fill; 1976 } 1977 1978 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, 1979 struct icp_qat_uclo_encap_page 1980 *encap_page, unsigned int ae) 1981 { 1982 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; 1983 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 1984 u64 fill_pat; 1985 1986 /* load the page starting at appropriate ustore address */ 1987 /* get fill-pattern from an image -- they are all the same */ 1988 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, 1989 sizeof(u64)); 1990 uw_physical_addr = encap_page->beg_addr_p; 1991 uw_relative_addr = 0; 1992 words_num = encap_page->micro_words_num; 1993 while (words_num) { 1994 cpylen = min(words_num, UWORD_CPYBUF_SIZE); 1995 1996 /* load the buffer */ 1997 for (i = 0; i < cpylen; i++) 1998 qat_uclo_fill_uwords(obj_handle, encap_page, 1999 &obj_handle->uword_buf[i], 2000 uw_physical_addr + i, 2001 uw_relative_addr + i, fill_pat); 2002 2003 /* copy the buffer to ustore */ 2004 qat_hal_wr_uwords(handle, (unsigned char)ae, 2005 uw_physical_addr, cpylen, 2006 obj_handle->uword_buf); 2007 2008 uw_physical_addr += cpylen; 2009 uw_relative_addr += cpylen; 2010 words_num -= cpylen; 2011 } 2012 } 2013 2014 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, 2015 struct icp_qat_uof_image *image) 2016 { 2017 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2018 unsigned long ae_mask = handle->hal_handle->ae_mask; 2019 unsigned long cfg_ae_mask = handle->cfg_ae_mask; 2020 unsigned long ae_assigned = image->ae_assigned; 2021 struct icp_qat_uclo_aedata *aed; 2022 unsigned int ctx_mask, s; 2023 struct icp_qat_uclo_page *page; 2024 unsigned char ae; 2025 int ctx; 2026 2027 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) 2028 ctx_mask = 0xff; 2029 else 2030 ctx_mask = 0x55; 2031 /* load the default page and set assigned CTX PC 2032 * to the entrypoint address */ 2033 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { 2034 if (!test_bit(ae, &cfg_ae_mask)) 2035 continue; 2036 2037 if (!test_bit(ae, &ae_assigned)) 2038 continue; 2039 2040 aed = &obj_handle->ae_data[ae]; 2041 /* find the slice to which this image is assigned */ 2042 for (s = 0; s < aed->slice_num; s++) { 2043 if (image->ctx_assigned & 2044 aed->ae_slices[s].ctx_mask_assigned) 2045 break; 2046 } 2047 if (s >= aed->slice_num) 2048 continue; 2049 page = aed->ae_slices[s].page; 2050 if (!page->encap_page->def_page) 2051 continue; 2052 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); 2053 2054 page = aed->ae_slices[s].page; 2055 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) 2056 aed->ae_slices[s].cur_page[ctx] = 2057 (ctx_mask & (1 << ctx)) ? page : NULL; 2058 qat_hal_set_live_ctx(handle, (unsigned char)ae, 2059 image->ctx_assigned); 2060 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, 2061 image->entry_address); 2062 } 2063 } 2064 2065 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) 2066 { 2067 unsigned int i; 2068 struct icp_qat_fw_auth_desc *desc = NULL; 2069 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; 2070 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; 2071 int ret; 2072 2073 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { 2074 ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, 2075 simg_hdr[i].simg_len, 2076 CSS_AE_FIRMWARE); 2077 if (ret) 2078 return ret; 2079 2080 if (qat_uclo_map_auth_fw(handle, 2081 (char *)simg_hdr[i].simg_buf, 2082 (unsigned int) 2083 simg_hdr[i].simg_len, 2084 &desc)) 2085 goto wr_err; 2086 if (qat_uclo_auth_fw(handle, desc)) 2087 goto wr_err; 2088 if (qat_uclo_is_broadcast(handle, i)) { 2089 if (qat_uclo_broadcast_load_fw(handle, desc)) 2090 goto wr_err; 2091 } else { 2092 if (qat_uclo_load_fw(handle, desc)) 2093 goto wr_err; 2094 } 2095 qat_uclo_ummap_auth_fw(handle, &desc); 2096 } 2097 return 0; 2098 wr_err: 2099 qat_uclo_ummap_auth_fw(handle, &desc); 2100 return -EINVAL; 2101 } 2102 2103 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) 2104 { 2105 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; 2106 unsigned int i; 2107 2108 if (qat_uclo_init_globals(handle)) 2109 return -EINVAL; 2110 for (i = 0; i < obj_handle->uimage_num; i++) { 2111 if (!obj_handle->ae_uimage[i].img_ptr) 2112 return -EINVAL; 2113 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) 2114 return -EINVAL; 2115 qat_uclo_wr_uimage_page(handle, 2116 obj_handle->ae_uimage[i].img_ptr); 2117 } 2118 return 0; 2119 } 2120 2121 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) 2122 { 2123 return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : 2124 qat_uclo_wr_uof_img(handle); 2125 } 2126 2127 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, 2128 unsigned int cfg_ae_mask) 2129 { 2130 if (!cfg_ae_mask) 2131 return -EINVAL; 2132 2133 handle->cfg_ae_mask = cfg_ae_mask; 2134 return 0; 2135 } 2136