1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include "hns_roce_device.h" 36 #include "hns_roce_hem.h" 37 #include "hns_roce_common.h" 38 39 #define DMA_ADDR_T_SHIFT 12 40 #define BT_BA_SHIFT 32 41 42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 43 { 44 if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) || 45 (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) || 46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) || 47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) || 48 (hr_dev->caps.sccc_hop_num && type == HEM_TYPE_SCCC) || 49 (hr_dev->caps.qpc_timer_hop_num && type == HEM_TYPE_QPC_TIMER) || 50 (hr_dev->caps.cqc_timer_hop_num && type == HEM_TYPE_CQC_TIMER) || 51 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) || 52 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) || 53 (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) || 54 (hr_dev->caps.idx_hop_num && type == HEM_TYPE_IDX)) 55 return true; 56 57 return false; 58 } 59 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); 60 61 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, 62 u32 bt_chunk_num) 63 { 64 int i; 65 66 for (i = 0; i < bt_chunk_num; i++) 67 if (hem[start_idx + i]) 68 return false; 69 70 return true; 71 } 72 73 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) 74 { 75 int i; 76 77 for (i = 0; i < bt_chunk_num; i++) 78 if (bt[start_idx + i]) 79 return false; 80 81 return true; 82 } 83 84 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 85 { 86 if (check_whether_bt_num_3(table_type, hop_num)) 87 return 3; 88 else if (check_whether_bt_num_2(table_type, hop_num)) 89 return 2; 90 else if (check_whether_bt_num_1(table_type, hop_num)) 91 return 1; 92 else 93 return 0; 94 } 95 96 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 97 struct hns_roce_hem_table *table, unsigned long *obj, 98 struct hns_roce_hem_mhop *mhop) 99 { 100 struct device *dev = hr_dev->dev; 101 u32 chunk_ba_num; 102 u32 table_idx; 103 u32 bt_num; 104 u32 chunk_size; 105 106 switch (table->type) { 107 case HEM_TYPE_QPC: 108 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 109 + PAGE_SHIFT); 110 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 111 + PAGE_SHIFT); 112 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 113 mhop->hop_num = hr_dev->caps.qpc_hop_num; 114 break; 115 case HEM_TYPE_MTPT: 116 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 117 + PAGE_SHIFT); 118 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 119 + PAGE_SHIFT); 120 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 121 mhop->hop_num = hr_dev->caps.mpt_hop_num; 122 break; 123 case HEM_TYPE_CQC: 124 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 125 + PAGE_SHIFT); 126 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 127 + PAGE_SHIFT); 128 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 129 mhop->hop_num = hr_dev->caps.cqc_hop_num; 130 break; 131 case HEM_TYPE_SCCC: 132 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 133 + PAGE_SHIFT); 134 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 135 + PAGE_SHIFT); 136 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 137 mhop->hop_num = hr_dev->caps.sccc_hop_num; 138 break; 139 case HEM_TYPE_QPC_TIMER: 140 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 141 + PAGE_SHIFT); 142 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 143 + PAGE_SHIFT); 144 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 145 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 146 break; 147 case HEM_TYPE_CQC_TIMER: 148 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 149 + PAGE_SHIFT); 150 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 151 + PAGE_SHIFT); 152 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 153 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 154 break; 155 case HEM_TYPE_SRQC: 156 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 157 + PAGE_SHIFT); 158 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 159 + PAGE_SHIFT); 160 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 161 mhop->hop_num = hr_dev->caps.srqc_hop_num; 162 break; 163 case HEM_TYPE_MTT: 164 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz 165 + PAGE_SHIFT); 166 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 167 + PAGE_SHIFT); 168 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 169 mhop->hop_num = hr_dev->caps.mtt_hop_num; 170 break; 171 case HEM_TYPE_CQE: 172 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz 173 + PAGE_SHIFT); 174 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 175 + PAGE_SHIFT); 176 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 177 mhop->hop_num = hr_dev->caps.cqe_hop_num; 178 break; 179 case HEM_TYPE_SRQWQE: 180 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz 181 + PAGE_SHIFT); 182 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz 183 + PAGE_SHIFT); 184 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 185 mhop->hop_num = hr_dev->caps.srqwqe_hop_num; 186 break; 187 case HEM_TYPE_IDX: 188 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz 189 + PAGE_SHIFT); 190 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz 191 + PAGE_SHIFT); 192 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 193 mhop->hop_num = hr_dev->caps.idx_hop_num; 194 break; 195 default: 196 dev_err(dev, "Table %d not support multi-hop addressing!\n", 197 table->type); 198 return -EINVAL; 199 } 200 201 if (!obj) 202 return 0; 203 204 /* 205 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 206 * MTT/CQE alloc hem for bt pages. 207 */ 208 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 209 chunk_ba_num = mhop->bt_chunk_size / 8; 210 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 211 mhop->bt_chunk_size; 212 table_idx = (*obj & (table->num_obj - 1)) / 213 (chunk_size / table->obj_size); 214 switch (bt_num) { 215 case 3: 216 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 217 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 218 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 219 break; 220 case 2: 221 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 222 mhop->l0_idx = table_idx / chunk_ba_num; 223 break; 224 case 1: 225 mhop->l0_idx = table_idx; 226 break; 227 default: 228 dev_err(dev, "Table %d not support hop_num = %d!\n", 229 table->type, mhop->hop_num); 230 return -EINVAL; 231 } 232 if (mhop->l0_idx >= mhop->ba_l0_num) 233 mhop->l0_idx %= mhop->ba_l0_num; 234 235 return 0; 236 } 237 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); 238 239 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 240 int npages, 241 unsigned long hem_alloc_size, 242 gfp_t gfp_mask) 243 { 244 struct hns_roce_hem_chunk *chunk = NULL; 245 struct hns_roce_hem *hem; 246 struct scatterlist *mem; 247 int order; 248 void *buf; 249 250 WARN_ON(gfp_mask & __GFP_HIGHMEM); 251 252 hem = kmalloc(sizeof(*hem), 253 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 254 if (!hem) 255 return NULL; 256 257 hem->refcount = 0; 258 INIT_LIST_HEAD(&hem->chunk_list); 259 260 order = get_order(hem_alloc_size); 261 262 while (npages > 0) { 263 if (!chunk) { 264 chunk = kmalloc(sizeof(*chunk), 265 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 266 if (!chunk) 267 goto fail; 268 269 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 270 chunk->npages = 0; 271 chunk->nsg = 0; 272 memset(chunk->buf, 0, sizeof(chunk->buf)); 273 list_add_tail(&chunk->list, &hem->chunk_list); 274 } 275 276 while (1 << order > npages) 277 --order; 278 279 /* 280 * Alloc memory one time. If failed, don't alloc small block 281 * memory, directly return fail. 282 */ 283 mem = &chunk->mem[chunk->npages]; 284 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 285 &sg_dma_address(mem), gfp_mask); 286 if (!buf) 287 goto fail; 288 289 chunk->buf[chunk->npages] = buf; 290 sg_dma_len(mem) = PAGE_SIZE << order; 291 292 ++chunk->npages; 293 ++chunk->nsg; 294 npages -= 1 << order; 295 } 296 297 return hem; 298 299 fail: 300 hns_roce_free_hem(hr_dev, hem); 301 return NULL; 302 } 303 304 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 305 { 306 struct hns_roce_hem_chunk *chunk, *tmp; 307 int i; 308 309 if (!hem) 310 return; 311 312 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 313 for (i = 0; i < chunk->npages; ++i) 314 dma_free_coherent(hr_dev->dev, 315 sg_dma_len(&chunk->mem[i]), 316 chunk->buf[i], 317 sg_dma_address(&chunk->mem[i])); 318 kfree(chunk); 319 } 320 321 kfree(hem); 322 } 323 324 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, 325 struct hns_roce_hem_table *table, unsigned long obj) 326 { 327 spinlock_t *lock = &hr_dev->bt_cmd_lock; 328 struct device *dev = hr_dev->dev; 329 unsigned long end = 0; 330 unsigned long flags; 331 struct hns_roce_hem_iter iter; 332 void __iomem *bt_cmd; 333 u32 bt_cmd_h_val = 0; 334 u32 bt_cmd_val[2]; 335 u32 bt_cmd_l = 0; 336 u64 bt_ba = 0; 337 int ret = 0; 338 339 /* Find the HEM(Hardware Entry Memory) entry */ 340 unsigned long i = (obj & (table->num_obj - 1)) / 341 (table->table_chunk_size / table->obj_size); 342 343 switch (table->type) { 344 case HEM_TYPE_QPC: 345 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 346 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC); 347 break; 348 case HEM_TYPE_MTPT: 349 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 350 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, 351 HEM_TYPE_MTPT); 352 break; 353 case HEM_TYPE_CQC: 354 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 355 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC); 356 break; 357 case HEM_TYPE_SRQC: 358 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 359 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, 360 HEM_TYPE_SRQC); 361 break; 362 default: 363 return ret; 364 } 365 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, 366 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); 367 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); 368 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); 369 370 /* Currently iter only a chunk */ 371 for (hns_roce_hem_first(table->hem[i], &iter); 372 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 373 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; 374 375 spin_lock_irqsave(lock, flags); 376 377 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; 378 379 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; 380 while (1) { 381 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { 382 if (!(time_before(jiffies, end))) { 383 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); 384 spin_unlock_irqrestore(lock, flags); 385 return -EBUSY; 386 } 387 } else { 388 break; 389 } 390 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); 391 } 392 393 bt_cmd_l = (u32)bt_ba; 394 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, 395 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 396 bt_ba >> BT_BA_SHIFT); 397 398 bt_cmd_val[0] = bt_cmd_l; 399 bt_cmd_val[1] = bt_cmd_h_val; 400 hns_roce_write64_k(bt_cmd_val, 401 hr_dev->reg_base + ROCEE_BT_CMD_L_REG); 402 spin_unlock_irqrestore(lock, flags); 403 } 404 405 return ret; 406 } 407 408 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 409 struct hns_roce_hem_table *table, 410 unsigned long obj) 411 { 412 struct device *dev = hr_dev->dev; 413 struct hns_roce_hem_mhop mhop; 414 struct hns_roce_hem_iter iter; 415 u32 buf_chunk_size; 416 u32 bt_chunk_size; 417 u32 chunk_ba_num; 418 u32 hop_num; 419 u32 size; 420 u32 bt_num; 421 u64 hem_idx; 422 u64 bt_l1_idx = 0; 423 u64 bt_l0_idx = 0; 424 u64 bt_ba; 425 unsigned long mhop_obj = obj; 426 int bt_l1_allocated = 0; 427 int bt_l0_allocated = 0; 428 int step_idx; 429 int ret; 430 431 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 432 if (ret) 433 return ret; 434 435 buf_chunk_size = mhop.buf_chunk_size; 436 bt_chunk_size = mhop.bt_chunk_size; 437 hop_num = mhop.hop_num; 438 chunk_ba_num = bt_chunk_size / 8; 439 440 bt_num = hns_roce_get_bt_num(table->type, hop_num); 441 switch (bt_num) { 442 case 3: 443 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 444 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 445 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 446 bt_l0_idx = mhop.l0_idx; 447 break; 448 case 2: 449 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 450 bt_l0_idx = mhop.l0_idx; 451 break; 452 case 1: 453 hem_idx = mhop.l0_idx; 454 break; 455 default: 456 dev_err(dev, "Table %d not support hop_num = %d!\n", 457 table->type, hop_num); 458 return -EINVAL; 459 } 460 461 mutex_lock(&table->mutex); 462 463 if (table->hem[hem_idx]) { 464 ++table->hem[hem_idx]->refcount; 465 goto out; 466 } 467 468 /* alloc L1 BA's chunk */ 469 if ((check_whether_bt_num_3(table->type, hop_num) || 470 check_whether_bt_num_2(table->type, hop_num)) && 471 !table->bt_l0[bt_l0_idx]) { 472 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, 473 &(table->bt_l0_dma_addr[bt_l0_idx]), 474 GFP_KERNEL); 475 if (!table->bt_l0[bt_l0_idx]) { 476 ret = -ENOMEM; 477 goto out; 478 } 479 bt_l0_allocated = 1; 480 481 /* set base address to hardware */ 482 if (table->type < HEM_TYPE_MTT) { 483 step_idx = 0; 484 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 485 ret = -ENODEV; 486 dev_err(dev, "set HEM base address to HW failed!\n"); 487 goto err_dma_alloc_l1; 488 } 489 } 490 } 491 492 /* alloc L2 BA's chunk */ 493 if (check_whether_bt_num_3(table->type, hop_num) && 494 !table->bt_l1[bt_l1_idx]) { 495 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, 496 &(table->bt_l1_dma_addr[bt_l1_idx]), 497 GFP_KERNEL); 498 if (!table->bt_l1[bt_l1_idx]) { 499 ret = -ENOMEM; 500 goto err_dma_alloc_l1; 501 } 502 bt_l1_allocated = 1; 503 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = 504 table->bt_l1_dma_addr[bt_l1_idx]; 505 506 /* set base address to hardware */ 507 step_idx = 1; 508 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 509 ret = -ENODEV; 510 dev_err(dev, "set HEM base address to HW failed!\n"); 511 goto err_alloc_hem_buf; 512 } 513 } 514 515 /* 516 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 517 * alloc bt space chunk for MTT/CQE. 518 */ 519 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; 520 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, 521 size >> PAGE_SHIFT, 522 size, 523 (table->lowmem ? GFP_KERNEL : 524 GFP_HIGHUSER) | __GFP_NOWARN); 525 if (!table->hem[hem_idx]) { 526 ret = -ENOMEM; 527 goto err_alloc_hem_buf; 528 } 529 530 hns_roce_hem_first(table->hem[hem_idx], &iter); 531 bt_ba = hns_roce_hem_addr(&iter); 532 533 if (table->type < HEM_TYPE_MTT) { 534 if (hop_num == 2) { 535 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; 536 step_idx = 2; 537 } else if (hop_num == 1) { 538 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 539 step_idx = 1; 540 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 541 step_idx = 0; 542 } else { 543 ret = -EINVAL; 544 goto err_dma_alloc_l1; 545 } 546 547 /* set HEM base address to hardware */ 548 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 549 ret = -ENODEV; 550 dev_err(dev, "set HEM base address to HW failed!\n"); 551 goto err_alloc_hem_buf; 552 } 553 } else if (hop_num == 2) { 554 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 555 } 556 557 ++table->hem[hem_idx]->refcount; 558 goto out; 559 560 err_alloc_hem_buf: 561 if (bt_l1_allocated) { 562 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], 563 table->bt_l1_dma_addr[bt_l1_idx]); 564 table->bt_l1[bt_l1_idx] = NULL; 565 } 566 567 err_dma_alloc_l1: 568 if (bt_l0_allocated) { 569 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], 570 table->bt_l0_dma_addr[bt_l0_idx]); 571 table->bt_l0[bt_l0_idx] = NULL; 572 } 573 574 out: 575 mutex_unlock(&table->mutex); 576 return ret; 577 } 578 579 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 580 struct hns_roce_hem_table *table, unsigned long obj) 581 { 582 struct device *dev = hr_dev->dev; 583 int ret = 0; 584 unsigned long i; 585 586 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 587 return hns_roce_table_mhop_get(hr_dev, table, obj); 588 589 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / 590 table->obj_size); 591 592 mutex_lock(&table->mutex); 593 594 if (table->hem[i]) { 595 ++table->hem[i]->refcount; 596 goto out; 597 } 598 599 table->hem[i] = hns_roce_alloc_hem(hr_dev, 600 table->table_chunk_size >> PAGE_SHIFT, 601 table->table_chunk_size, 602 (table->lowmem ? GFP_KERNEL : 603 GFP_HIGHUSER) | __GFP_NOWARN); 604 if (!table->hem[i]) { 605 ret = -ENOMEM; 606 goto out; 607 } 608 609 /* Set HEM base address(128K/page, pa) to Hardware */ 610 if (hns_roce_set_hem(hr_dev, table, obj)) { 611 hns_roce_free_hem(hr_dev, table->hem[i]); 612 table->hem[i] = NULL; 613 ret = -ENODEV; 614 dev_err(dev, "set HEM base address to HW failed.\n"); 615 goto out; 616 } 617 618 ++table->hem[i]->refcount; 619 out: 620 mutex_unlock(&table->mutex); 621 return ret; 622 } 623 EXPORT_SYMBOL_GPL(hns_roce_table_get); 624 625 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 626 struct hns_roce_hem_table *table, 627 unsigned long obj, 628 int check_refcount) 629 { 630 struct device *dev = hr_dev->dev; 631 struct hns_roce_hem_mhop mhop; 632 unsigned long mhop_obj = obj; 633 u32 bt_chunk_size; 634 u32 chunk_ba_num; 635 u32 hop_num; 636 u32 start_idx; 637 u32 bt_num; 638 u64 hem_idx; 639 u64 bt_l1_idx = 0; 640 int ret; 641 642 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 643 if (ret) 644 return; 645 646 bt_chunk_size = mhop.bt_chunk_size; 647 hop_num = mhop.hop_num; 648 chunk_ba_num = bt_chunk_size / 8; 649 650 bt_num = hns_roce_get_bt_num(table->type, hop_num); 651 switch (bt_num) { 652 case 3: 653 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 654 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 655 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 656 break; 657 case 2: 658 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 659 break; 660 case 1: 661 hem_idx = mhop.l0_idx; 662 break; 663 default: 664 dev_err(dev, "Table %d not support hop_num = %d!\n", 665 table->type, hop_num); 666 return; 667 } 668 669 mutex_lock(&table->mutex); 670 671 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { 672 mutex_unlock(&table->mutex); 673 return; 674 } 675 676 if (table->type < HEM_TYPE_MTT && hop_num == 1) { 677 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 678 dev_warn(dev, "Clear HEM base address failed.\n"); 679 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { 680 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) 681 dev_warn(dev, "Clear HEM base address failed.\n"); 682 } else if (table->type < HEM_TYPE_MTT && 683 hop_num == HNS_ROCE_HOP_NUM_0) { 684 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 685 dev_warn(dev, "Clear HEM base address failed.\n"); 686 } 687 688 /* 689 * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 690 * free bt space chunk for MTT/CQE. 691 */ 692 hns_roce_free_hem(hr_dev, table->hem[hem_idx]); 693 table->hem[hem_idx] = NULL; 694 695 if (check_whether_bt_num_2(table->type, hop_num)) { 696 start_idx = mhop.l0_idx * chunk_ba_num; 697 if (hns_roce_check_hem_null(table->hem, start_idx, 698 chunk_ba_num)) { 699 if (table->type < HEM_TYPE_MTT && 700 hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 701 dev_warn(dev, "Clear HEM base address failed.\n"); 702 703 dma_free_coherent(dev, bt_chunk_size, 704 table->bt_l0[mhop.l0_idx], 705 table->bt_l0_dma_addr[mhop.l0_idx]); 706 table->bt_l0[mhop.l0_idx] = NULL; 707 } 708 } else if (check_whether_bt_num_3(table->type, hop_num)) { 709 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 710 mhop.l1_idx * chunk_ba_num; 711 if (hns_roce_check_hem_null(table->hem, start_idx, 712 chunk_ba_num)) { 713 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 714 dev_warn(dev, "Clear HEM base address failed.\n"); 715 716 dma_free_coherent(dev, bt_chunk_size, 717 table->bt_l1[bt_l1_idx], 718 table->bt_l1_dma_addr[bt_l1_idx]); 719 table->bt_l1[bt_l1_idx] = NULL; 720 721 start_idx = mhop.l0_idx * chunk_ba_num; 722 if (hns_roce_check_bt_null(table->bt_l1, start_idx, 723 chunk_ba_num)) { 724 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 725 0)) 726 dev_warn(dev, "Clear HEM base address failed.\n"); 727 728 dma_free_coherent(dev, bt_chunk_size, 729 table->bt_l0[mhop.l0_idx], 730 table->bt_l0_dma_addr[mhop.l0_idx]); 731 table->bt_l0[mhop.l0_idx] = NULL; 732 } 733 } 734 } 735 736 mutex_unlock(&table->mutex); 737 } 738 739 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 740 struct hns_roce_hem_table *table, unsigned long obj) 741 { 742 struct device *dev = hr_dev->dev; 743 unsigned long i; 744 745 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 746 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 747 return; 748 } 749 750 i = (obj & (table->num_obj - 1)) / 751 (table->table_chunk_size / table->obj_size); 752 753 mutex_lock(&table->mutex); 754 755 if (--table->hem[i]->refcount == 0) { 756 /* Clear HEM base address */ 757 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 758 dev_warn(dev, "Clear HEM base address failed.\n"); 759 760 hns_roce_free_hem(hr_dev, table->hem[i]); 761 table->hem[i] = NULL; 762 } 763 764 mutex_unlock(&table->mutex); 765 } 766 EXPORT_SYMBOL_GPL(hns_roce_table_put); 767 768 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 769 struct hns_roce_hem_table *table, 770 unsigned long obj, dma_addr_t *dma_handle) 771 { 772 struct hns_roce_hem_chunk *chunk; 773 struct hns_roce_hem_mhop mhop; 774 struct hns_roce_hem *hem; 775 void *addr = NULL; 776 unsigned long mhop_obj = obj; 777 unsigned long obj_per_chunk; 778 unsigned long idx_offset; 779 int offset, dma_offset; 780 int length; 781 int i, j; 782 u32 hem_idx = 0; 783 784 if (!table->lowmem) 785 return NULL; 786 787 mutex_lock(&table->mutex); 788 789 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 790 obj_per_chunk = table->table_chunk_size / table->obj_size; 791 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; 792 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; 793 dma_offset = offset = idx_offset * table->obj_size; 794 } else { 795 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 796 797 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 798 /* mtt mhop */ 799 i = mhop.l0_idx; 800 j = mhop.l1_idx; 801 if (mhop.hop_num == 2) 802 hem_idx = i * (mhop.bt_chunk_size / 8) + j; 803 else if (mhop.hop_num == 1 || 804 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 805 hem_idx = i; 806 807 hem = table->hem[hem_idx]; 808 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % 809 mhop.bt_chunk_size; 810 if (mhop.hop_num == 2) 811 dma_offset = offset = 0; 812 } 813 814 if (!hem) 815 goto out; 816 817 list_for_each_entry(chunk, &hem->chunk_list, list) { 818 for (i = 0; i < chunk->npages; ++i) { 819 length = sg_dma_len(&chunk->mem[i]); 820 if (dma_handle && dma_offset >= 0) { 821 if (length > (u32)dma_offset) 822 *dma_handle = sg_dma_address( 823 &chunk->mem[i]) + dma_offset; 824 dma_offset -= length; 825 } 826 827 if (length > (u32)offset) { 828 addr = chunk->buf[i] + offset; 829 goto out; 830 } 831 offset -= length; 832 } 833 } 834 835 out: 836 mutex_unlock(&table->mutex); 837 return addr; 838 } 839 EXPORT_SYMBOL_GPL(hns_roce_table_find); 840 841 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, 842 struct hns_roce_hem_table *table, 843 unsigned long start, unsigned long end) 844 { 845 struct hns_roce_hem_mhop mhop; 846 unsigned long inc = table->table_chunk_size / table->obj_size; 847 unsigned long i; 848 int ret; 849 850 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 851 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 852 inc = mhop.bt_chunk_size / table->obj_size; 853 } 854 855 /* Allocate MTT entry memory according to chunk(128K) */ 856 for (i = start; i <= end; i += inc) { 857 ret = hns_roce_table_get(hr_dev, table, i); 858 if (ret) 859 goto fail; 860 } 861 862 return 0; 863 864 fail: 865 while (i > start) { 866 i -= inc; 867 hns_roce_table_put(hr_dev, table, i); 868 } 869 return ret; 870 } 871 872 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, 873 struct hns_roce_hem_table *table, 874 unsigned long start, unsigned long end) 875 { 876 struct hns_roce_hem_mhop mhop; 877 unsigned long inc = table->table_chunk_size / table->obj_size; 878 unsigned long i; 879 880 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 881 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 882 inc = mhop.bt_chunk_size / table->obj_size; 883 } 884 885 for (i = start; i <= end; i += inc) 886 hns_roce_table_put(hr_dev, table, i); 887 } 888 889 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 890 struct hns_roce_hem_table *table, u32 type, 891 unsigned long obj_size, unsigned long nobj, 892 int use_lowmem) 893 { 894 struct device *dev = hr_dev->dev; 895 unsigned long obj_per_chunk; 896 unsigned long num_hem; 897 898 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 899 table->table_chunk_size = hr_dev->caps.chunk_sz; 900 obj_per_chunk = table->table_chunk_size / obj_size; 901 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 902 903 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 904 if (!table->hem) 905 return -ENOMEM; 906 } else { 907 unsigned long buf_chunk_size; 908 unsigned long bt_chunk_size; 909 unsigned long bt_chunk_num; 910 unsigned long num_bt_l0 = 0; 911 u32 hop_num; 912 913 switch (type) { 914 case HEM_TYPE_QPC: 915 buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 916 + PAGE_SHIFT); 917 bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 918 + PAGE_SHIFT); 919 num_bt_l0 = hr_dev->caps.qpc_bt_num; 920 hop_num = hr_dev->caps.qpc_hop_num; 921 break; 922 case HEM_TYPE_MTPT: 923 buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 924 + PAGE_SHIFT); 925 bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 926 + PAGE_SHIFT); 927 num_bt_l0 = hr_dev->caps.mpt_bt_num; 928 hop_num = hr_dev->caps.mpt_hop_num; 929 break; 930 case HEM_TYPE_CQC: 931 buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 932 + PAGE_SHIFT); 933 bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 934 + PAGE_SHIFT); 935 num_bt_l0 = hr_dev->caps.cqc_bt_num; 936 hop_num = hr_dev->caps.cqc_hop_num; 937 break; 938 case HEM_TYPE_SCCC: 939 buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 940 + PAGE_SHIFT); 941 bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 942 + PAGE_SHIFT); 943 num_bt_l0 = hr_dev->caps.sccc_bt_num; 944 hop_num = hr_dev->caps.sccc_hop_num; 945 break; 946 case HEM_TYPE_QPC_TIMER: 947 buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 948 + PAGE_SHIFT); 949 bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 950 + PAGE_SHIFT); 951 num_bt_l0 = hr_dev->caps.qpc_timer_bt_num; 952 hop_num = hr_dev->caps.qpc_timer_hop_num; 953 break; 954 case HEM_TYPE_CQC_TIMER: 955 buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 956 + PAGE_SHIFT); 957 bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 958 + PAGE_SHIFT); 959 num_bt_l0 = hr_dev->caps.cqc_timer_bt_num; 960 hop_num = hr_dev->caps.cqc_timer_hop_num; 961 break; 962 case HEM_TYPE_SRQC: 963 buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 964 + PAGE_SHIFT); 965 bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 966 + PAGE_SHIFT); 967 num_bt_l0 = hr_dev->caps.srqc_bt_num; 968 hop_num = hr_dev->caps.srqc_hop_num; 969 break; 970 case HEM_TYPE_MTT: 971 buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 972 + PAGE_SHIFT); 973 bt_chunk_size = buf_chunk_size; 974 hop_num = hr_dev->caps.mtt_hop_num; 975 break; 976 case HEM_TYPE_CQE: 977 buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 978 + PAGE_SHIFT); 979 bt_chunk_size = buf_chunk_size; 980 hop_num = hr_dev->caps.cqe_hop_num; 981 break; 982 case HEM_TYPE_SRQWQE: 983 buf_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz 984 + PAGE_SHIFT); 985 bt_chunk_size = buf_chunk_size; 986 hop_num = hr_dev->caps.srqwqe_hop_num; 987 break; 988 case HEM_TYPE_IDX: 989 buf_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz 990 + PAGE_SHIFT); 991 bt_chunk_size = buf_chunk_size; 992 hop_num = hr_dev->caps.idx_hop_num; 993 break; 994 default: 995 dev_err(dev, 996 "Table %d not support to init hem table here!\n", 997 type); 998 return -EINVAL; 999 } 1000 obj_per_chunk = buf_chunk_size / obj_size; 1001 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 1002 bt_chunk_num = bt_chunk_size / 8; 1003 if (type >= HEM_TYPE_MTT) 1004 num_bt_l0 = bt_chunk_num; 1005 1006 table->hem = kcalloc(num_hem, sizeof(*table->hem), 1007 GFP_KERNEL); 1008 if (!table->hem) 1009 goto err_kcalloc_hem_buf; 1010 1011 if (check_whether_bt_num_3(type, hop_num)) { 1012 unsigned long num_bt_l1; 1013 1014 num_bt_l1 = (num_hem + bt_chunk_num - 1) / 1015 bt_chunk_num; 1016 table->bt_l1 = kcalloc(num_bt_l1, 1017 sizeof(*table->bt_l1), 1018 GFP_KERNEL); 1019 if (!table->bt_l1) 1020 goto err_kcalloc_bt_l1; 1021 1022 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 1023 sizeof(*table->bt_l1_dma_addr), 1024 GFP_KERNEL); 1025 1026 if (!table->bt_l1_dma_addr) 1027 goto err_kcalloc_l1_dma; 1028 } 1029 1030 if (check_whether_bt_num_2(type, hop_num) || 1031 check_whether_bt_num_3(type, hop_num)) { 1032 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 1033 GFP_KERNEL); 1034 if (!table->bt_l0) 1035 goto err_kcalloc_bt_l0; 1036 1037 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 1038 sizeof(*table->bt_l0_dma_addr), 1039 GFP_KERNEL); 1040 if (!table->bt_l0_dma_addr) 1041 goto err_kcalloc_l0_dma; 1042 } 1043 } 1044 1045 table->type = type; 1046 table->num_hem = num_hem; 1047 table->num_obj = nobj; 1048 table->obj_size = obj_size; 1049 table->lowmem = use_lowmem; 1050 mutex_init(&table->mutex); 1051 1052 return 0; 1053 1054 err_kcalloc_l0_dma: 1055 kfree(table->bt_l0); 1056 table->bt_l0 = NULL; 1057 1058 err_kcalloc_bt_l0: 1059 kfree(table->bt_l1_dma_addr); 1060 table->bt_l1_dma_addr = NULL; 1061 1062 err_kcalloc_l1_dma: 1063 kfree(table->bt_l1); 1064 table->bt_l1 = NULL; 1065 1066 err_kcalloc_bt_l1: 1067 kfree(table->hem); 1068 table->hem = NULL; 1069 1070 err_kcalloc_hem_buf: 1071 return -ENOMEM; 1072 } 1073 1074 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 1075 struct hns_roce_hem_table *table) 1076 { 1077 struct hns_roce_hem_mhop mhop; 1078 u32 buf_chunk_size; 1079 int i; 1080 u64 obj; 1081 1082 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 1083 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 1084 mhop.bt_chunk_size; 1085 1086 for (i = 0; i < table->num_hem; ++i) { 1087 obj = i * buf_chunk_size / table->obj_size; 1088 if (table->hem[i]) 1089 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 1090 } 1091 1092 kfree(table->hem); 1093 table->hem = NULL; 1094 kfree(table->bt_l1); 1095 table->bt_l1 = NULL; 1096 kfree(table->bt_l1_dma_addr); 1097 table->bt_l1_dma_addr = NULL; 1098 kfree(table->bt_l0); 1099 table->bt_l0 = NULL; 1100 kfree(table->bt_l0_dma_addr); 1101 table->bt_l0_dma_addr = NULL; 1102 } 1103 1104 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 1105 struct hns_roce_hem_table *table) 1106 { 1107 struct device *dev = hr_dev->dev; 1108 unsigned long i; 1109 1110 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 1111 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 1112 return; 1113 } 1114 1115 for (i = 0; i < table->num_hem; ++i) 1116 if (table->hem[i]) { 1117 if (hr_dev->hw->clear_hem(hr_dev, table, 1118 i * table->table_chunk_size / table->obj_size, 0)) 1119 dev_err(dev, "Clear HEM base address failed.\n"); 1120 1121 hns_roce_free_hem(hr_dev, table->hem[i]); 1122 } 1123 1124 kfree(table->hem); 1125 } 1126 1127 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 1128 { 1129 if ((hr_dev->caps.num_idx_segs)) 1130 hns_roce_cleanup_hem_table(hr_dev, 1131 &hr_dev->mr_table.mtt_idx_table); 1132 if (hr_dev->caps.num_srqwqe_segs) 1133 hns_roce_cleanup_hem_table(hr_dev, 1134 &hr_dev->mr_table.mtt_srqwqe_table); 1135 if (hr_dev->caps.srqc_entry_sz) 1136 hns_roce_cleanup_hem_table(hr_dev, 1137 &hr_dev->srq_table.table); 1138 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 1139 if (hr_dev->caps.qpc_timer_entry_sz) 1140 hns_roce_cleanup_hem_table(hr_dev, 1141 &hr_dev->qpc_timer_table); 1142 if (hr_dev->caps.cqc_timer_entry_sz) 1143 hns_roce_cleanup_hem_table(hr_dev, 1144 &hr_dev->cqc_timer_table); 1145 if (hr_dev->caps.sccc_entry_sz) 1146 hns_roce_cleanup_hem_table(hr_dev, 1147 &hr_dev->qp_table.sccc_table); 1148 if (hr_dev->caps.trrl_entry_sz) 1149 hns_roce_cleanup_hem_table(hr_dev, 1150 &hr_dev->qp_table.trrl_table); 1151 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 1152 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 1153 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1154 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 1155 hns_roce_cleanup_hem_table(hr_dev, 1156 &hr_dev->mr_table.mtt_cqe_table); 1157 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); 1158 } 1159