1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2026 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "osdep.h" 36 #include "irdma_hmc.h" 37 #include "irdma_defs.h" 38 #include "irdma_type.h" 39 #include "irdma_protos.h" 40 41 /** 42 * irdma_find_sd_index_limit - finds segment descriptor index limit 43 * @hmc_info: pointer to the HMC configuration information structure 44 * @type: type of HMC resources we're searching 45 * @idx: starting index for the object 46 * @cnt: number of objects we're trying to create 47 * @sd_idx: pointer to return index of the segment descriptor in question 48 * @sd_limit: pointer to return the maximum number of segment descriptors 49 * 50 * This function calculates the segment descriptor index and index limit 51 * for the resource defined by irdma_hmc_rsrc_type. 52 */ 53 54 static void 55 irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type, 56 u32 idx, u32 cnt, u32 *sd_idx, 57 u32 *sd_limit) 58 { 59 u64 fpm_addr, fpm_limit; 60 61 fpm_addr = hmc_info->hmc_obj[(type)].base + 62 hmc_info->hmc_obj[type].size * idx; 63 fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt; 64 *sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE); 65 *sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE); 66 *sd_limit += 1; 67 } 68 69 /** 70 * irdma_find_pd_index_limit - finds page descriptor index limit 71 * @hmc_info: pointer to the HMC configuration information struct 72 * @type: HMC resource type we're examining 73 * @idx: starting index for the object 74 * @cnt: number of objects we're trying to create 75 * @pd_idx: pointer to return page descriptor index 76 * @pd_limit: pointer to return page descriptor index limit 77 * 78 * Calculates the page descriptor index and index limit for the resource 79 * defined by irdma_hmc_rsrc_type. 80 */ 81 82 static void 83 irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type, 84 u32 idx, u32 cnt, u32 *pd_idx, 85 u32 *pd_limit) 86 { 87 u64 fpm_adr, fpm_limit; 88 89 fpm_adr = hmc_info->hmc_obj[type].base + 90 hmc_info->hmc_obj[type].size * idx; 91 fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt); 92 *pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE); 93 *pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE); 94 *pd_limit += 1; 95 } 96 97 /** 98 * irdma_set_sd_entry - setup entry for sd programming 99 * @pa: physical addr 100 * @idx: sd index 101 * @type: paged or direct sd 102 * @entry: sd entry ptr 103 */ 104 static void 105 irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type, 106 struct irdma_update_sd_entry *entry) 107 { 108 entry->data = pa | 109 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) | 110 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE, 111 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) | 112 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1); 113 114 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | 115 IRDMA_PFHMC_SDCMD_PMSDPARTSEL; 116 } 117 118 /** 119 * irdma_clr_sd_entry - setup entry for sd clear 120 * @idx: sd index 121 * @type: paged or direct sd 122 * @entry: sd entry ptr 123 */ 124 static void 125 irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type, 126 struct irdma_update_sd_entry *entry) 127 { 128 entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) | 129 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE, 130 type == IRDMA_SD_TYPE_PAGED ? 0 : 1); 131 132 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | 133 IRDMA_PFHMC_SDCMD_PMSDPARTSEL; 134 } 135 136 /** 137 * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF 138 * @dev: pointer to our device struct 139 * @sd_idx: segment descriptor index 140 * @pd_idx: page descriptor index 141 */ 142 static inline void 143 irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx, 144 u32 pd_idx) 145 { 146 u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) | 147 FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) | 148 FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx); 149 150 writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]); 151 } 152 153 /** 154 * irdma_hmc_sd_one - setup 1 sd entry for cqp 155 * @dev: pointer to the device structure 156 * @hmc_fn_id: hmc's function id 157 * @pa: physical addr 158 * @sd_idx: sd index 159 * @type: paged or direct sd 160 * @setsd: flag to set or clear sd 161 */ 162 int 163 irdma_hmc_sd_one(struct irdma_sc_dev *dev, u16 hmc_fn_id, u64 pa, u32 sd_idx, 164 enum irdma_sd_entry_type type, bool setsd) 165 { 166 struct irdma_update_sds_info sdinfo; 167 168 sdinfo.cnt = 1; 169 sdinfo.hmc_fn_id = hmc_fn_id; 170 if (setsd) 171 irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry); 172 else 173 irdma_clr_sd_entry(sd_idx, type, sdinfo.entry); 174 return dev->cqp->process_cqp_sds(dev, &sdinfo); 175 } 176 177 /** 178 * irdma_hmc_sd_grp - setup group of sd entries for cqp 179 * @dev: pointer to the device structure 180 * @hmc_info: pointer to the HMC configuration information struct 181 * @sd_index: sd index 182 * @sd_cnt: number of sd entries 183 * @setsd: flag to set or clear sd 184 */ 185 static int 186 irdma_hmc_sd_grp(struct irdma_sc_dev *dev, 187 struct irdma_hmc_info *hmc_info, u32 sd_index, 188 u32 sd_cnt, bool setsd) 189 { 190 struct irdma_hmc_sd_entry *sd_entry; 191 struct irdma_update_sds_info sdinfo = {0}; 192 u64 pa; 193 u32 i; 194 int ret_code = 0; 195 196 sdinfo.hmc_fn_id = hmc_info->hmc_fn_id; 197 for (i = sd_index; i < sd_index + sd_cnt; i++) { 198 sd_entry = &hmc_info->sd_table.sd_entry[i]; 199 if (!sd_entry || (!sd_entry->valid && setsd) || 200 (sd_entry->valid && !setsd)) 201 continue; 202 if (setsd) { 203 pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ? 204 sd_entry->u.pd_table.pd_page_addr.pa : 205 sd_entry->u.bp.addr.pa; 206 irdma_set_sd_entry(pa, i, sd_entry->entry_type, 207 &sdinfo.entry[sdinfo.cnt]); 208 } else { 209 irdma_clr_sd_entry(i, sd_entry->entry_type, 210 &sdinfo.entry[sdinfo.cnt]); 211 } 212 sdinfo.cnt++; 213 if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) { 214 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); 215 if (ret_code) { 216 irdma_debug(dev, IRDMA_DEBUG_HMC, 217 "sd_programming failed err=%d\n", 218 ret_code); 219 return ret_code; 220 } 221 222 sdinfo.cnt = 0; 223 } 224 } 225 if (sdinfo.cnt) 226 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); 227 228 return ret_code; 229 } 230 231 /** 232 * irdma_hmc_finish_add_sd_reg - program sd entries for objects 233 * @dev: pointer to the device structure 234 * @info: create obj info 235 */ 236 static int 237 irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev, 238 struct irdma_hmc_create_obj_info *info) 239 { 240 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) 241 return -EINVAL; 242 243 if ((info->start_idx + info->count) > 244 info->hmc_info->hmc_obj[info->rsrc_type].cnt) 245 return -EINVAL; 246 247 if (!info->add_sd_cnt) 248 return 0; 249 return irdma_hmc_sd_grp(dev, info->hmc_info, 250 info->hmc_info->sd_indexes[0], info->add_sd_cnt, 251 true); 252 } 253 254 /** 255 * irdma_sc_create_hmc_obj - allocate backing store for hmc objects 256 * @dev: pointer to the device structure 257 * @info: pointer to irdma_hmc_create_obj_info struct 258 * 259 * This will allocate memory for PDs and backing pages and populate 260 * the sd and pd entries. 261 */ 262 int 263 irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev, 264 struct irdma_hmc_create_obj_info *info) 265 { 266 struct irdma_hmc_sd_entry *sd_entry; 267 u32 sd_idx, sd_lmt; 268 u32 pd_idx = 0, pd_lmt = 0; 269 u32 pd_idx1 = 0, pd_lmt1 = 0; 270 u32 i, j; 271 bool pd_error = false; 272 int ret_code = 0; 273 274 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 275 irdma_debug(dev, IRDMA_DEBUG_ERR, 276 "invalid hmc obj type %u, start = %u, req cnt %u, cnt = %u\n", 277 info->rsrc_type, info->start_idx, info->count, 278 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 279 280 return -EINVAL; 281 } 282 283 if ((info->start_idx + info->count) > 284 info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 285 irdma_debug(dev, IRDMA_DEBUG_ERR, 286 "error type %u, start = %u, req cnt %u, cnt = %u\n", 287 info->rsrc_type, info->start_idx, info->count, 288 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 289 return -EINVAL; 290 } 291 292 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type, 293 info->start_idx, info->count, &sd_idx, 294 &sd_lmt); 295 if (sd_idx >= info->hmc_info->sd_table.sd_cnt || 296 sd_lmt > info->hmc_info->sd_table.sd_cnt) { 297 return -EINVAL; 298 } 299 300 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type, 301 info->start_idx, info->count, &pd_idx, 302 &pd_lmt); 303 304 for (j = sd_idx; j < sd_lmt; j++) { 305 ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j, 306 info->entry_type, 307 IRDMA_HMC_DIRECT_BP_SIZE); 308 if (ret_code) 309 goto exit_sd_error; 310 311 sd_entry = &info->hmc_info->sd_table.sd_entry[j]; 312 if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED && 313 (dev->hmc_info == info->hmc_info && 314 info->rsrc_type != IRDMA_HMC_IW_PBLE)) { 315 pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT)); 316 pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT); 317 for (i = pd_idx1; i < pd_lmt1; i++) { 318 /* update the pd table entry */ 319 ret_code = irdma_add_pd_table_entry(dev, 320 info->hmc_info, 321 i, NULL); 322 if (ret_code) { 323 pd_error = true; 324 break; 325 } 326 } 327 if (pd_error) { 328 while (i && (i > pd_idx1)) { 329 irdma_remove_pd_bp(dev, info->hmc_info, 330 i - 1); 331 i--; 332 } 333 } 334 } 335 if (sd_entry->valid) 336 continue; 337 338 info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j; 339 info->add_sd_cnt++; 340 sd_entry->valid = true; 341 } 342 return irdma_hmc_finish_add_sd_reg(dev, info); 343 344 exit_sd_error: 345 while (j && (j > sd_idx)) { 346 sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1]; 347 switch (sd_entry->entry_type) { 348 case IRDMA_SD_TYPE_PAGED: 349 pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT); 350 pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT)); 351 for (i = pd_idx1; i < pd_lmt1; i++) 352 irdma_prep_remove_pd_page(info->hmc_info, i); 353 break; 354 case IRDMA_SD_TYPE_DIRECT: 355 irdma_prep_remove_pd_page(info->hmc_info, (j - 1)); 356 break; 357 default: 358 ret_code = -EINVAL; 359 break; 360 } 361 j--; 362 } 363 364 return ret_code; 365 } 366 367 /** 368 * irdma_finish_del_sd_reg - delete sd entries for objects 369 * @dev: pointer to the device structure 370 * @info: dele obj info 371 * @reset: true if called before reset 372 */ 373 static int 374 irdma_finish_del_sd_reg(struct irdma_sc_dev *dev, 375 struct irdma_hmc_del_obj_info *info, 376 bool reset) 377 { 378 struct irdma_hmc_sd_entry *sd_entry; 379 int ret_code = 0; 380 struct irdma_dma_mem *mem; 381 u32 i, sd_idx; 382 383 if (!reset) 384 ret_code = irdma_hmc_sd_grp(dev, info->hmc_info, 385 info->hmc_info->sd_indexes[0], 386 info->del_sd_cnt, false); 387 388 if (ret_code) 389 irdma_debug(dev, IRDMA_DEBUG_HMC, "error cqp sd sd_grp\n"); 390 for (i = 0; i < info->del_sd_cnt; i++) { 391 sd_idx = info->hmc_info->sd_indexes[i]; 392 sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx]; 393 394 mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ? 395 &sd_entry->u.pd_table.pd_page_addr : 396 &sd_entry->u.bp.addr; 397 398 if (!mem || !mem->va) 399 irdma_debug(dev, IRDMA_DEBUG_HMC, "error cqp sd mem\n"); 400 else 401 irdma_free_dma_mem(dev->hw, mem); 402 } 403 404 return ret_code; 405 } 406 407 /** 408 * irdma_sc_del_hmc_obj - remove pe hmc objects 409 * @dev: pointer to the device structure 410 * @info: pointer to irdma_hmc_del_obj_info struct 411 * @reset: true if called before reset 412 * 413 * This will de-populate the SDs and PDs. It frees 414 * the memory for PDS and backing storage. After this function is returned, 415 * caller should deallocate memory allocated previously for 416 * book-keeping information about PDs and backing storage. 417 */ 418 int 419 irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev, 420 struct irdma_hmc_del_obj_info *info, bool reset) 421 { 422 struct irdma_hmc_pd_table *pd_table; 423 u32 sd_idx, sd_lmt; 424 u32 pd_idx, pd_lmt, rel_pd_idx; 425 u32 i, j; 426 int ret_code = 0; 427 428 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 429 irdma_debug(dev, IRDMA_DEBUG_HMC, 430 "error start_idx[%04d] >= [type %04d].cnt[%04d]\n", 431 info->start_idx, info->rsrc_type, 432 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 433 return -EINVAL; 434 } 435 436 if ((info->start_idx + info->count) > 437 info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 438 irdma_debug(dev, IRDMA_DEBUG_HMC, 439 "error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n", 440 info->start_idx, info->count, info->rsrc_type, 441 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 442 return -EINVAL; 443 } 444 445 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type, 446 info->start_idx, info->count, &pd_idx, 447 &pd_lmt); 448 449 for (j = pd_idx; j < pd_lmt; j++) { 450 sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD; 451 452 if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid) 453 continue; 454 455 if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type != 456 IRDMA_SD_TYPE_PAGED) 457 continue; 458 459 rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD; 460 pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 461 if (pd_table->pd_entry && 462 pd_table->pd_entry[rel_pd_idx].valid) { 463 ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j); 464 if (ret_code) { 465 irdma_debug(dev, IRDMA_DEBUG_HMC, 466 "remove_pd_bp error\n"); 467 return ret_code; 468 } 469 } 470 } 471 472 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type, 473 info->start_idx, info->count, &sd_idx, 474 &sd_lmt); 475 if (sd_idx >= info->hmc_info->sd_table.sd_cnt || 476 sd_lmt > info->hmc_info->sd_table.sd_cnt) { 477 irdma_debug(dev, IRDMA_DEBUG_HMC, "invalid sd_idx\n"); 478 return -EINVAL; 479 } 480 481 for (i = sd_idx; i < sd_lmt; i++) { 482 pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table; 483 if (!info->hmc_info->sd_table.sd_entry[i].valid) 484 continue; 485 switch (info->hmc_info->sd_table.sd_entry[i].entry_type) { 486 case IRDMA_SD_TYPE_DIRECT: 487 ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i); 488 if (!ret_code) { 489 info->hmc_info->sd_indexes[info->del_sd_cnt] = 490 (u16)i; 491 info->del_sd_cnt++; 492 } 493 break; 494 case IRDMA_SD_TYPE_PAGED: 495 ret_code = irdma_prep_remove_pd_page(info->hmc_info, i); 496 if (ret_code) 497 break; 498 if (dev->hmc_info != info->hmc_info && 499 info->rsrc_type == IRDMA_HMC_IW_PBLE && 500 pd_table->pd_entry) { 501 kfree(pd_table->pd_entry_virt_mem.va); 502 pd_table->pd_entry = NULL; 503 } 504 info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i; 505 info->del_sd_cnt++; 506 break; 507 default: 508 break; 509 } 510 } 511 return irdma_finish_del_sd_reg(dev, info, reset); 512 } 513 514 /** 515 * irdma_add_sd_table_entry - Adds a segment descriptor to the table 516 * @hw: pointer to our hw struct 517 * @hmc_info: pointer to the HMC configuration information struct 518 * @sd_index: segment descriptor index to manipulate 519 * @type: what type of segment descriptor we're manipulating 520 * @direct_mode_sz: size to alloc in direct mode 521 */ 522 int 523 irdma_add_sd_table_entry(struct irdma_hw *hw, 524 struct irdma_hmc_info *hmc_info, u32 sd_index, 525 enum irdma_sd_entry_type type, u64 direct_mode_sz) 526 { 527 struct irdma_hmc_sd_entry *sd_entry; 528 struct irdma_dma_mem dma_mem; 529 u64 alloc_len; 530 531 sd_entry = &hmc_info->sd_table.sd_entry[sd_index]; 532 if (!sd_entry->valid) { 533 if (type == IRDMA_SD_TYPE_PAGED) 534 alloc_len = IRDMA_HMC_PAGED_BP_SIZE; 535 else 536 alloc_len = direct_mode_sz; 537 538 /* allocate a 4K pd page or 2M backing page */ 539 dma_mem.size = alloc_len; 540 dma_mem.va = irdma_allocate_dma_mem(hw, &dma_mem, dma_mem.size, 541 IRDMA_HMC_PD_BP_BUF_ALIGNMENT); 542 if (!dma_mem.va) 543 return -ENOMEM; 544 if (type == IRDMA_SD_TYPE_PAGED) { 545 struct irdma_virt_mem *vmem = 546 &sd_entry->u.pd_table.pd_entry_virt_mem; 547 548 vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512; 549 vmem->va = kzalloc(vmem->size, GFP_KERNEL); 550 if (!vmem->va) { 551 irdma_free_dma_mem(hw, &dma_mem); 552 return -ENOMEM; 553 } 554 sd_entry->u.pd_table.pd_entry = vmem->va; 555 556 irdma_memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem, 557 sizeof(sd_entry->u.pd_table.pd_page_addr)); 558 } else { 559 irdma_memcpy(&sd_entry->u.bp.addr, &dma_mem, 560 sizeof(sd_entry->u.bp.addr)); 561 562 sd_entry->u.bp.sd_pd_index = sd_index; 563 } 564 565 hmc_info->sd_table.sd_entry[sd_index].entry_type = type; 566 hmc_info->sd_table.use_cnt++; 567 } 568 if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT) 569 sd_entry->u.bp.use_cnt++; 570 571 return 0; 572 } 573 574 /** 575 * irdma_add_pd_table_entry - Adds page descriptor to the specified table 576 * @dev: pointer to our device structure 577 * @hmc_info: pointer to the HMC configuration information structure 578 * @pd_index: which page descriptor index to manipulate 579 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one. 580 * 581 * This function: 582 * 1. Initializes the pd entry 583 * 2. Adds pd_entry in the pd_table 584 * 3. Mark the entry valid in irdma_hmc_pd_entry structure 585 * 4. Initializes the pd_entry's ref count to 1 586 * assumptions: 587 * 1. The memory for pd should be pinned down, physically contiguous and 588 * aligned on 4K boundary and zeroed memory. 589 * 2. It should be 4K in size. 590 */ 591 int 592 irdma_add_pd_table_entry(struct irdma_sc_dev *dev, 593 struct irdma_hmc_info *hmc_info, u32 pd_index, 594 struct irdma_dma_mem *rsrc_pg) 595 { 596 struct irdma_hmc_pd_table *pd_table; 597 struct irdma_hmc_pd_entry *pd_entry; 598 struct irdma_dma_mem mem; 599 struct irdma_dma_mem *page = &mem; 600 u32 sd_idx, rel_pd_idx; 601 u64 *pd_addr; 602 u64 page_desc; 603 604 if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) 605 return -EINVAL; 606 607 sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD); 608 if (hmc_info->sd_table.sd_entry[sd_idx].entry_type != 609 IRDMA_SD_TYPE_PAGED) 610 return 0; 611 612 rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD); 613 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 614 pd_entry = &pd_table->pd_entry[rel_pd_idx]; 615 if (!pd_entry->valid) { 616 if (rsrc_pg) { 617 pd_entry->rsrc_pg = true; 618 page = rsrc_pg; 619 } else { 620 page->size = IRDMA_HMC_PAGED_BP_SIZE; 621 page->va = irdma_allocate_dma_mem(dev->hw, page, 622 page->size, 623 IRDMA_HMC_PD_BP_BUF_ALIGNMENT); 624 if (!page->va) 625 return -ENOMEM; 626 627 pd_entry->rsrc_pg = false; 628 } 629 630 irdma_memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr)); 631 pd_entry->bp.sd_pd_index = pd_index; 632 pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED; 633 page_desc = page->pa | 0x1; 634 pd_addr = pd_table->pd_page_addr.va; 635 pd_addr += rel_pd_idx; 636 irdma_memcpy(pd_addr, &page_desc, sizeof(*pd_addr)); 637 pd_entry->sd_index = sd_idx; 638 pd_entry->valid = true; 639 pd_table->use_cnt++; 640 irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx); 641 } 642 pd_entry->bp.use_cnt++; 643 644 return 0; 645 } 646 647 /** 648 * irdma_remove_pd_bp - remove a backing page from a page descriptor 649 * @dev: pointer to our HW structure 650 * @hmc_info: pointer to the HMC configuration information structure 651 * @idx: the page index 652 * 653 * This function: 654 * 1. Marks the entry in pd table (for paged address mode) or in sd table 655 * (for direct address mode) invalid. 656 * 2. Write to register PMPDINV to invalidate the backing page in FV cache 657 * 3. Decrement the ref count for the pd _entry 658 * assumptions: 659 * 1. Caller can deallocate the memory used by backing storage after this 660 * function returns. 661 */ 662 int 663 irdma_remove_pd_bp(struct irdma_sc_dev *dev, 664 struct irdma_hmc_info *hmc_info, u32 idx) 665 { 666 struct irdma_hmc_pd_entry *pd_entry; 667 struct irdma_hmc_pd_table *pd_table; 668 struct irdma_hmc_sd_entry *sd_entry; 669 u32 sd_idx, rel_pd_idx; 670 struct irdma_dma_mem *mem; 671 u64 *pd_addr; 672 673 sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD; 674 rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD; 675 if (sd_idx >= hmc_info->sd_table.sd_cnt) 676 return -EINVAL; 677 678 sd_entry = &hmc_info->sd_table.sd_entry[sd_idx]; 679 if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED) 680 return -EINVAL; 681 682 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 683 pd_entry = &pd_table->pd_entry[rel_pd_idx]; 684 if (--pd_entry->bp.use_cnt) 685 return 0; 686 687 pd_entry->valid = false; 688 pd_table->use_cnt--; 689 pd_addr = pd_table->pd_page_addr.va; 690 pd_addr += rel_pd_idx; 691 irdma_memset(pd_addr, 0, sizeof(u64)); 692 irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx); 693 694 if (!pd_entry->rsrc_pg) { 695 mem = &pd_entry->bp.addr; 696 if (!mem || !mem->va) 697 return -EINVAL; 698 699 irdma_free_dma_mem(dev->hw, mem); 700 } 701 if (!pd_table->use_cnt) 702 kfree(pd_table->pd_entry_virt_mem.va); 703 704 return 0; 705 } 706 707 /** 708 * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry 709 * @hmc_info: pointer to the HMC configuration information structure 710 * @idx: the page index 711 */ 712 int 713 irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx) 714 { 715 struct irdma_hmc_sd_entry *sd_entry; 716 717 sd_entry = &hmc_info->sd_table.sd_entry[idx]; 718 if (--sd_entry->u.bp.use_cnt) 719 return -EBUSY; 720 721 hmc_info->sd_table.use_cnt--; 722 sd_entry->valid = false; 723 724 return 0; 725 } 726 727 /** 728 * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry. 729 * @hmc_info: pointer to the HMC configuration information structure 730 * @idx: segment descriptor index to find the relevant page descriptor 731 */ 732 int 733 irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx) 734 { 735 struct irdma_hmc_sd_entry *sd_entry; 736 737 sd_entry = &hmc_info->sd_table.sd_entry[idx]; 738 739 if (sd_entry->u.pd_table.use_cnt) 740 return -EBUSY; 741 742 sd_entry->valid = false; 743 hmc_info->sd_table.use_cnt--; 744 745 return 0; 746 } 747