1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include "hns_roce_device.h" 35 #include "hns_roce_hem.h" 36 #include "hns_roce_common.h" 37 38 #define HEM_INDEX_BUF BIT(0) 39 #define HEM_INDEX_L0 BIT(1) 40 #define HEM_INDEX_L1 BIT(2) 41 struct hns_roce_hem_index { 42 u64 buf; 43 u64 l0; 44 u64 l1; 45 u32 inited; /* indicate which index is available */ 46 }; 47 48 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 49 { 50 int hop_num = 0; 51 52 switch (type) { 53 case HEM_TYPE_QPC: 54 hop_num = hr_dev->caps.qpc_hop_num; 55 break; 56 case HEM_TYPE_MTPT: 57 hop_num = hr_dev->caps.mpt_hop_num; 58 break; 59 case HEM_TYPE_CQC: 60 hop_num = hr_dev->caps.cqc_hop_num; 61 break; 62 case HEM_TYPE_SRQC: 63 hop_num = hr_dev->caps.srqc_hop_num; 64 break; 65 case HEM_TYPE_SCCC: 66 hop_num = hr_dev->caps.sccc_hop_num; 67 break; 68 case HEM_TYPE_QPC_TIMER: 69 hop_num = hr_dev->caps.qpc_timer_hop_num; 70 break; 71 case HEM_TYPE_CQC_TIMER: 72 hop_num = hr_dev->caps.cqc_timer_hop_num; 73 break; 74 case HEM_TYPE_GMV: 75 hop_num = hr_dev->caps.gmv_hop_num; 76 break; 77 default: 78 return false; 79 } 80 81 return hop_num; 82 } 83 84 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, 85 u32 bt_chunk_num, u64 hem_max_num) 86 { 87 u64 start_idx = round_down(hem_idx, bt_chunk_num); 88 u64 check_max_num = start_idx + bt_chunk_num; 89 u64 i; 90 91 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) 92 if (i != hem_idx && hem[i]) 93 return false; 94 95 return true; 96 } 97 98 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) 99 { 100 u64 start_idx = round_down(ba_idx, bt_chunk_num); 101 int i; 102 103 for (i = 0; i < bt_chunk_num; i++) 104 if (i != ba_idx && bt[start_idx + i]) 105 return false; 106 107 return true; 108 } 109 110 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 111 { 112 if (check_whether_bt_num_3(table_type, hop_num)) 113 return 3; 114 else if (check_whether_bt_num_2(table_type, hop_num)) 115 return 2; 116 else if (check_whether_bt_num_1(table_type, hop_num)) 117 return 1; 118 else 119 return 0; 120 } 121 122 static int get_hem_table_config(struct hns_roce_dev *hr_dev, 123 struct hns_roce_hem_mhop *mhop, 124 u32 type) 125 { 126 struct device *dev = hr_dev->dev; 127 128 switch (type) { 129 case HEM_TYPE_QPC: 130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 131 + PAGE_SHIFT); 132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 133 + PAGE_SHIFT); 134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 135 mhop->hop_num = hr_dev->caps.qpc_hop_num; 136 break; 137 case HEM_TYPE_MTPT: 138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 139 + PAGE_SHIFT); 140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 141 + PAGE_SHIFT); 142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 143 mhop->hop_num = hr_dev->caps.mpt_hop_num; 144 break; 145 case HEM_TYPE_CQC: 146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 147 + PAGE_SHIFT); 148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 149 + PAGE_SHIFT); 150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 151 mhop->hop_num = hr_dev->caps.cqc_hop_num; 152 break; 153 case HEM_TYPE_SCCC: 154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 155 + PAGE_SHIFT); 156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 157 + PAGE_SHIFT); 158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 159 mhop->hop_num = hr_dev->caps.sccc_hop_num; 160 break; 161 case HEM_TYPE_QPC_TIMER: 162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 163 + PAGE_SHIFT); 164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 165 + PAGE_SHIFT); 166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 168 break; 169 case HEM_TYPE_CQC_TIMER: 170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 171 + PAGE_SHIFT); 172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 173 + PAGE_SHIFT); 174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 176 break; 177 case HEM_TYPE_SRQC: 178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 179 + PAGE_SHIFT); 180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 181 + PAGE_SHIFT); 182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 183 mhop->hop_num = hr_dev->caps.srqc_hop_num; 184 break; 185 case HEM_TYPE_GMV: 186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + 187 PAGE_SHIFT); 188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + 189 PAGE_SHIFT); 190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; 191 mhop->hop_num = hr_dev->caps.gmv_hop_num; 192 break; 193 default: 194 dev_err(dev, "table %u not support multi-hop addressing!\n", 195 type); 196 return -EINVAL; 197 } 198 199 return 0; 200 } 201 202 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 203 struct hns_roce_hem_table *table, unsigned long *obj, 204 struct hns_roce_hem_mhop *mhop) 205 { 206 struct device *dev = hr_dev->dev; 207 u32 chunk_ba_num; 208 u32 chunk_size; 209 u32 table_idx; 210 u32 bt_num; 211 212 if (get_hem_table_config(hr_dev, mhop, table->type)) 213 return -EINVAL; 214 215 if (!obj) 216 return 0; 217 218 /* 219 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 220 * MTT/CQE alloc hem for bt pages. 221 */ 222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 223 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 225 mhop->bt_chunk_size; 226 table_idx = *obj / (chunk_size / table->obj_size); 227 switch (bt_num) { 228 case 3: 229 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 230 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 231 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 232 break; 233 case 2: 234 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 235 mhop->l0_idx = table_idx / chunk_ba_num; 236 break; 237 case 1: 238 mhop->l0_idx = table_idx; 239 break; 240 default: 241 dev_err(dev, "table %u not support hop_num = %u!\n", 242 table->type, mhop->hop_num); 243 return -EINVAL; 244 } 245 if (mhop->l0_idx >= mhop->ba_l0_num) 246 mhop->l0_idx %= mhop->ba_l0_num; 247 248 return 0; 249 } 250 251 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 252 unsigned long hem_alloc_size, 253 gfp_t gfp_mask) 254 { 255 struct hns_roce_hem *hem; 256 int order; 257 void *buf; 258 259 WARN_ON(gfp_mask & __GFP_HIGHMEM); 260 261 order = get_order(hem_alloc_size); 262 if (PAGE_SIZE << order != hem_alloc_size) { 263 dev_err(hr_dev->dev, "invalid hem_alloc_size: %lu!\n", 264 hem_alloc_size); 265 return NULL; 266 } 267 268 hem = kmalloc(sizeof(*hem), 269 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 270 if (!hem) 271 return NULL; 272 273 buf = dma_alloc_coherent(hr_dev->dev, hem_alloc_size, 274 &hem->dma, gfp_mask); 275 if (!buf) 276 goto fail; 277 278 hem->buf = buf; 279 hem->size = hem_alloc_size; 280 281 return hem; 282 283 fail: 284 hns_roce_free_hem(hr_dev, hem); 285 return NULL; 286 } 287 288 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 289 { 290 if (!hem) 291 return; 292 293 dma_free_coherent(hr_dev->dev, hem->size, hem->buf, hem->dma); 294 295 kfree(hem); 296 } 297 298 static int calc_hem_config(struct hns_roce_dev *hr_dev, 299 struct hns_roce_hem_table *table, unsigned long obj, 300 struct hns_roce_hem_mhop *mhop, 301 struct hns_roce_hem_index *index) 302 { 303 struct ib_device *ibdev = &hr_dev->ib_dev; 304 unsigned long mhop_obj = obj; 305 u32 l0_idx, l1_idx, l2_idx; 306 u32 chunk_ba_num; 307 u32 bt_num; 308 int ret; 309 310 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); 311 if (ret) 312 return ret; 313 314 l0_idx = mhop->l0_idx; 315 l1_idx = mhop->l1_idx; 316 l2_idx = mhop->l2_idx; 317 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 318 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 319 switch (bt_num) { 320 case 3: 321 index->l1 = l0_idx * chunk_ba_num + l1_idx; 322 index->l0 = l0_idx; 323 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + 324 l1_idx * chunk_ba_num + l2_idx; 325 break; 326 case 2: 327 index->l0 = l0_idx; 328 index->buf = l0_idx * chunk_ba_num + l1_idx; 329 break; 330 case 1: 331 index->buf = l0_idx; 332 break; 333 default: 334 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n", 335 table->type, mhop->hop_num); 336 return -EINVAL; 337 } 338 339 if (unlikely(index->buf >= table->num_hem)) { 340 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n", 341 table->type, index->buf, table->num_hem); 342 return -EINVAL; 343 } 344 345 return 0; 346 } 347 348 static void free_mhop_hem(struct hns_roce_dev *hr_dev, 349 struct hns_roce_hem_table *table, 350 struct hns_roce_hem_mhop *mhop, 351 struct hns_roce_hem_index *index) 352 { 353 u32 bt_size = mhop->bt_chunk_size; 354 struct device *dev = hr_dev->dev; 355 356 if (index->inited & HEM_INDEX_BUF) { 357 hns_roce_free_hem(hr_dev, table->hem[index->buf]); 358 table->hem[index->buf] = NULL; 359 } 360 361 if (index->inited & HEM_INDEX_L1) { 362 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], 363 table->bt_l1_dma_addr[index->l1]); 364 table->bt_l1[index->l1] = NULL; 365 } 366 367 if (index->inited & HEM_INDEX_L0) { 368 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], 369 table->bt_l0_dma_addr[index->l0]); 370 table->bt_l0[index->l0] = NULL; 371 } 372 } 373 374 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, 375 struct hns_roce_hem_table *table, 376 struct hns_roce_hem_mhop *mhop, 377 struct hns_roce_hem_index *index) 378 { 379 u32 bt_size = mhop->bt_chunk_size; 380 struct device *dev = hr_dev->dev; 381 gfp_t flag; 382 u64 bt_ba; 383 u32 size; 384 int ret; 385 386 /* alloc L1 BA's chunk */ 387 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || 388 check_whether_bt_num_2(table->type, mhop->hop_num)) && 389 !table->bt_l0[index->l0]) { 390 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, 391 &table->bt_l0_dma_addr[index->l0], 392 GFP_KERNEL); 393 if (!table->bt_l0[index->l0]) { 394 ret = -ENOMEM; 395 goto out; 396 } 397 index->inited |= HEM_INDEX_L0; 398 } 399 400 /* alloc L2 BA's chunk */ 401 if (check_whether_bt_num_3(table->type, mhop->hop_num) && 402 !table->bt_l1[index->l1]) { 403 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, 404 &table->bt_l1_dma_addr[index->l1], 405 GFP_KERNEL); 406 if (!table->bt_l1[index->l1]) { 407 ret = -ENOMEM; 408 goto err_alloc_hem; 409 } 410 index->inited |= HEM_INDEX_L1; 411 *(table->bt_l0[index->l0] + mhop->l1_idx) = 412 table->bt_l1_dma_addr[index->l1]; 413 } 414 415 /* 416 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 417 * alloc bt space chunk for MTT/CQE. 418 */ 419 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; 420 flag = GFP_KERNEL | __GFP_NOWARN; 421 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag); 422 if (!table->hem[index->buf]) { 423 ret = -ENOMEM; 424 goto err_alloc_hem; 425 } 426 427 index->inited |= HEM_INDEX_BUF; 428 bt_ba = table->hem[index->buf]->dma; 429 430 if (table->type < HEM_TYPE_MTT) { 431 if (mhop->hop_num == 2) 432 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; 433 else if (mhop->hop_num == 1) 434 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 435 } else if (mhop->hop_num == 2) { 436 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 437 } 438 439 return 0; 440 err_alloc_hem: 441 free_mhop_hem(hr_dev, table, mhop, index); 442 out: 443 return ret; 444 } 445 446 static int set_mhop_hem(struct hns_roce_dev *hr_dev, 447 struct hns_roce_hem_table *table, unsigned long obj, 448 struct hns_roce_hem_mhop *mhop, 449 struct hns_roce_hem_index *index) 450 { 451 struct ib_device *ibdev = &hr_dev->ib_dev; 452 u32 step_idx; 453 int ret = 0; 454 455 if (index->inited & HEM_INDEX_L0) { 456 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); 457 if (ret) { 458 ibdev_err(ibdev, "set HEM step 0 failed!\n"); 459 goto out; 460 } 461 } 462 463 if (index->inited & HEM_INDEX_L1) { 464 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); 465 if (ret) { 466 ibdev_err(ibdev, "set HEM step 1 failed!\n"); 467 goto out; 468 } 469 } 470 471 if (index->inited & HEM_INDEX_BUF) { 472 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) 473 step_idx = 0; 474 else 475 step_idx = mhop->hop_num; 476 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); 477 if (ret) 478 ibdev_err(ibdev, "set HEM step last failed!\n"); 479 } 480 out: 481 return ret; 482 } 483 484 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 485 struct hns_roce_hem_table *table, 486 unsigned long obj) 487 { 488 struct ib_device *ibdev = &hr_dev->ib_dev; 489 struct hns_roce_hem_index index = {}; 490 struct hns_roce_hem_mhop mhop = {}; 491 int ret; 492 493 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 494 if (ret) { 495 ibdev_err(ibdev, "calc hem config failed!\n"); 496 return ret; 497 } 498 499 mutex_lock(&table->mutex); 500 if (table->hem[index.buf]) { 501 refcount_inc(&table->hem[index.buf]->refcount); 502 goto out; 503 } 504 505 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); 506 if (ret) { 507 ibdev_err(ibdev, "alloc mhop hem failed!\n"); 508 goto out; 509 } 510 511 /* set HEM base address to hardware */ 512 if (table->type < HEM_TYPE_MTT) { 513 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); 514 if (ret) { 515 ibdev_err(ibdev, "set HEM address to HW failed!\n"); 516 goto err_alloc; 517 } 518 } 519 520 refcount_set(&table->hem[index.buf]->refcount, 1); 521 goto out; 522 523 err_alloc: 524 free_mhop_hem(hr_dev, table, &mhop, &index); 525 out: 526 mutex_unlock(&table->mutex); 527 return ret; 528 } 529 530 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 531 struct hns_roce_hem_table *table, unsigned long obj) 532 { 533 struct device *dev = hr_dev->dev; 534 unsigned long i; 535 int ret = 0; 536 537 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 538 return hns_roce_table_mhop_get(hr_dev, table, obj); 539 540 i = obj / (table->table_chunk_size / table->obj_size); 541 542 mutex_lock(&table->mutex); 543 544 if (table->hem[i]) { 545 refcount_inc(&table->hem[i]->refcount); 546 goto out; 547 } 548 549 table->hem[i] = hns_roce_alloc_hem(hr_dev, 550 table->table_chunk_size, 551 GFP_KERNEL | __GFP_NOWARN); 552 if (!table->hem[i]) { 553 ret = -ENOMEM; 554 goto out; 555 } 556 557 /* Set HEM base address(128K/page, pa) to Hardware */ 558 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); 559 if (ret) { 560 hns_roce_free_hem(hr_dev, table->hem[i]); 561 table->hem[i] = NULL; 562 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", 563 ret); 564 goto out; 565 } 566 567 refcount_set(&table->hem[i]->refcount, 1); 568 out: 569 mutex_unlock(&table->mutex); 570 return ret; 571 } 572 573 static void clear_mhop_hem(struct hns_roce_dev *hr_dev, 574 struct hns_roce_hem_table *table, unsigned long obj, 575 struct hns_roce_hem_mhop *mhop, 576 struct hns_roce_hem_index *index) 577 { 578 struct ib_device *ibdev = &hr_dev->ib_dev; 579 u32 hop_num = mhop->hop_num; 580 u32 chunk_ba_num; 581 u32 step_idx; 582 int ret; 583 584 index->inited = HEM_INDEX_BUF; 585 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 586 if (check_whether_bt_num_2(table->type, hop_num)) { 587 if (hns_roce_check_hem_null(table->hem, index->buf, 588 chunk_ba_num, table->num_hem)) 589 index->inited |= HEM_INDEX_L0; 590 } else if (check_whether_bt_num_3(table->type, hop_num)) { 591 if (hns_roce_check_hem_null(table->hem, index->buf, 592 chunk_ba_num, table->num_hem)) { 593 index->inited |= HEM_INDEX_L1; 594 if (hns_roce_check_bt_null(table->bt_l1, index->l1, 595 chunk_ba_num)) 596 index->inited |= HEM_INDEX_L0; 597 } 598 } 599 600 if (table->type < HEM_TYPE_MTT) { 601 if (hop_num == HNS_ROCE_HOP_NUM_0) 602 step_idx = 0; 603 else 604 step_idx = hop_num; 605 606 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx); 607 if (ret) 608 ibdev_warn(ibdev, "failed to clear hop%u HEM, ret = %d.\n", 609 hop_num, ret); 610 611 if (index->inited & HEM_INDEX_L1) { 612 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1); 613 if (ret) 614 ibdev_warn(ibdev, "failed to clear HEM step 1, ret = %d.\n", 615 ret); 616 } 617 618 if (index->inited & HEM_INDEX_L0) { 619 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); 620 if (ret) 621 ibdev_warn(ibdev, "failed to clear HEM step 0, ret = %d.\n", 622 ret); 623 } 624 } 625 } 626 627 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 628 struct hns_roce_hem_table *table, 629 unsigned long obj, 630 int check_refcount) 631 { 632 struct ib_device *ibdev = &hr_dev->ib_dev; 633 struct hns_roce_hem_index index = {}; 634 struct hns_roce_hem_mhop mhop = {}; 635 int ret; 636 637 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 638 if (ret) { 639 ibdev_err(ibdev, "calc hem config failed!\n"); 640 return; 641 } 642 643 if (!check_refcount) 644 mutex_lock(&table->mutex); 645 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, 646 &table->mutex)) 647 return; 648 649 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); 650 free_mhop_hem(hr_dev, table, &mhop, &index); 651 652 mutex_unlock(&table->mutex); 653 } 654 655 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 656 struct hns_roce_hem_table *table, unsigned long obj) 657 { 658 struct device *dev = hr_dev->dev; 659 unsigned long i; 660 int ret; 661 662 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 663 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 664 return; 665 } 666 667 i = obj / (table->table_chunk_size / table->obj_size); 668 669 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, 670 &table->mutex)) 671 return; 672 673 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); 674 if (ret) 675 dev_warn(dev, "failed to clear HEM base address, ret = %d.\n", 676 ret); 677 678 hns_roce_free_hem(hr_dev, table->hem[i]); 679 table->hem[i] = NULL; 680 681 mutex_unlock(&table->mutex); 682 } 683 684 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 685 struct hns_roce_hem_table *table, 686 unsigned long obj, dma_addr_t *dma_handle) 687 { 688 struct hns_roce_hem_mhop mhop; 689 struct hns_roce_hem *hem; 690 unsigned long mhop_obj = obj; 691 unsigned long obj_per_chunk; 692 unsigned long idx_offset; 693 int offset, dma_offset; 694 void *addr = NULL; 695 u32 hem_idx = 0; 696 int i, j; 697 698 mutex_lock(&table->mutex); 699 700 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 701 obj_per_chunk = table->table_chunk_size / table->obj_size; 702 hem = table->hem[obj / obj_per_chunk]; 703 idx_offset = obj % obj_per_chunk; 704 dma_offset = offset = idx_offset * table->obj_size; 705 } else { 706 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 707 708 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) 709 goto out; 710 /* mtt mhop */ 711 i = mhop.l0_idx; 712 j = mhop.l1_idx; 713 if (mhop.hop_num == 2) 714 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; 715 else if (mhop.hop_num == 1 || 716 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 717 hem_idx = i; 718 719 hem = table->hem[hem_idx]; 720 dma_offset = offset = obj * seg_size % mhop.bt_chunk_size; 721 if (mhop.hop_num == 2) 722 dma_offset = offset = 0; 723 } 724 725 if (!hem) 726 goto out; 727 728 *dma_handle = hem->dma + dma_offset; 729 addr = hem->buf + offset; 730 731 out: 732 mutex_unlock(&table->mutex); 733 return addr; 734 } 735 736 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 737 struct hns_roce_hem_table *table, u32 type, 738 unsigned long obj_size, unsigned long nobj) 739 { 740 unsigned long obj_per_chunk; 741 unsigned long num_hem; 742 743 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 744 table->table_chunk_size = hr_dev->caps.chunk_sz; 745 obj_per_chunk = table->table_chunk_size / obj_size; 746 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); 747 748 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 749 if (!table->hem) 750 return -ENOMEM; 751 } else { 752 struct hns_roce_hem_mhop mhop = {}; 753 unsigned long buf_chunk_size; 754 unsigned long bt_chunk_size; 755 unsigned long bt_chunk_num; 756 unsigned long num_bt_l0; 757 u32 hop_num; 758 759 if (get_hem_table_config(hr_dev, &mhop, type)) 760 return -EINVAL; 761 762 buf_chunk_size = mhop.buf_chunk_size; 763 bt_chunk_size = mhop.bt_chunk_size; 764 num_bt_l0 = mhop.ba_l0_num; 765 hop_num = mhop.hop_num; 766 767 obj_per_chunk = buf_chunk_size / obj_size; 768 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); 769 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; 770 771 if (type >= HEM_TYPE_MTT) 772 num_bt_l0 = bt_chunk_num; 773 774 table->hem = kcalloc(num_hem, sizeof(*table->hem), 775 GFP_KERNEL); 776 if (!table->hem) 777 goto err_kcalloc_hem_buf; 778 779 if (check_whether_bt_num_3(type, hop_num)) { 780 unsigned long num_bt_l1; 781 782 num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num); 783 table->bt_l1 = kcalloc(num_bt_l1, 784 sizeof(*table->bt_l1), 785 GFP_KERNEL); 786 if (!table->bt_l1) 787 goto err_kcalloc_bt_l1; 788 789 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 790 sizeof(*table->bt_l1_dma_addr), 791 GFP_KERNEL); 792 793 if (!table->bt_l1_dma_addr) 794 goto err_kcalloc_l1_dma; 795 } 796 797 if (check_whether_bt_num_2(type, hop_num) || 798 check_whether_bt_num_3(type, hop_num)) { 799 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 800 GFP_KERNEL); 801 if (!table->bt_l0) 802 goto err_kcalloc_bt_l0; 803 804 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 805 sizeof(*table->bt_l0_dma_addr), 806 GFP_KERNEL); 807 if (!table->bt_l0_dma_addr) 808 goto err_kcalloc_l0_dma; 809 } 810 } 811 812 table->type = type; 813 table->num_hem = num_hem; 814 table->obj_size = obj_size; 815 mutex_init(&table->mutex); 816 817 return 0; 818 819 err_kcalloc_l0_dma: 820 kfree(table->bt_l0); 821 table->bt_l0 = NULL; 822 823 err_kcalloc_bt_l0: 824 kfree(table->bt_l1_dma_addr); 825 table->bt_l1_dma_addr = NULL; 826 827 err_kcalloc_l1_dma: 828 kfree(table->bt_l1); 829 table->bt_l1 = NULL; 830 831 err_kcalloc_bt_l1: 832 kfree(table->hem); 833 table->hem = NULL; 834 835 err_kcalloc_hem_buf: 836 return -ENOMEM; 837 } 838 839 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 840 struct hns_roce_hem_table *table) 841 { 842 struct hns_roce_hem_mhop mhop; 843 u32 buf_chunk_size; 844 u64 obj; 845 int i; 846 847 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 848 return; 849 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 850 mhop.bt_chunk_size; 851 852 for (i = 0; i < table->num_hem; ++i) { 853 obj = i * buf_chunk_size / table->obj_size; 854 if (table->hem[i]) 855 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 856 } 857 858 kfree(table->hem); 859 table->hem = NULL; 860 kfree(table->bt_l1); 861 table->bt_l1 = NULL; 862 kfree(table->bt_l1_dma_addr); 863 table->bt_l1_dma_addr = NULL; 864 kfree(table->bt_l0); 865 table->bt_l0 = NULL; 866 kfree(table->bt_l0_dma_addr); 867 table->bt_l0_dma_addr = NULL; 868 } 869 870 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 871 struct hns_roce_hem_table *table) 872 { 873 struct device *dev = hr_dev->dev; 874 unsigned long i; 875 int obj; 876 int ret; 877 878 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 879 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 880 return; 881 } 882 883 for (i = 0; i < table->num_hem; ++i) 884 if (table->hem[i]) { 885 obj = i * table->table_chunk_size / table->obj_size; 886 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); 887 if (ret) 888 dev_err(dev, "clear HEM base address failed, ret = %d.\n", 889 ret); 890 891 hns_roce_free_hem(hr_dev, table->hem[i]); 892 } 893 894 kfree(table->hem); 895 } 896 897 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 898 { 899 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) 900 hns_roce_cleanup_hem_table(hr_dev, 901 &hr_dev->srq_table.table); 902 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 903 if (hr_dev->caps.qpc_timer_entry_sz) 904 hns_roce_cleanup_hem_table(hr_dev, 905 &hr_dev->qpc_timer_table); 906 if (hr_dev->caps.cqc_timer_entry_sz) 907 hns_roce_cleanup_hem_table(hr_dev, 908 &hr_dev->cqc_timer_table); 909 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 910 hns_roce_cleanup_hem_table(hr_dev, 911 &hr_dev->qp_table.sccc_table); 912 if (hr_dev->caps.trrl_entry_sz) 913 hns_roce_cleanup_hem_table(hr_dev, 914 &hr_dev->qp_table.trrl_table); 915 916 if (hr_dev->caps.gmv_entry_sz) 917 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); 918 919 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 920 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 921 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 922 } 923 924 struct hns_roce_hem_item { 925 struct list_head list; /* link all hems in the same bt level */ 926 struct list_head sibling; /* link all hems in last hop for mtt */ 927 void *addr; 928 dma_addr_t dma_addr; 929 size_t count; /* max ba numbers */ 930 int start; /* start buf offset in this hem */ 931 int end; /* end buf offset in this hem */ 932 }; 933 934 /* All HEM items are linked in a tree structure */ 935 struct hns_roce_hem_head { 936 struct list_head branch[HNS_ROCE_MAX_BT_REGION]; 937 struct list_head root; 938 struct list_head leaf; 939 }; 940 941 static struct hns_roce_hem_item * 942 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, 943 bool exist_bt) 944 { 945 struct hns_roce_hem_item *hem; 946 947 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 948 if (!hem) 949 return NULL; 950 951 if (exist_bt) { 952 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, 953 &hem->dma_addr, GFP_KERNEL); 954 if (!hem->addr) { 955 kfree(hem); 956 return NULL; 957 } 958 } 959 960 hem->count = count; 961 hem->start = start; 962 hem->end = end; 963 INIT_LIST_HEAD(&hem->list); 964 INIT_LIST_HEAD(&hem->sibling); 965 966 return hem; 967 } 968 969 static void hem_list_free_item(struct hns_roce_dev *hr_dev, 970 struct hns_roce_hem_item *hem, bool exist_bt) 971 { 972 if (exist_bt) 973 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, 974 hem->addr, hem->dma_addr); 975 kfree(hem); 976 } 977 978 static void hem_list_free_all(struct hns_roce_dev *hr_dev, 979 struct list_head *head, bool exist_bt) 980 { 981 struct hns_roce_hem_item *hem, *temp_hem; 982 983 list_for_each_entry_safe(hem, temp_hem, head, list) { 984 list_del(&hem->list); 985 hem_list_free_item(hr_dev, hem, exist_bt); 986 } 987 } 988 989 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, 990 u64 table_addr) 991 { 992 *(u64 *)(base_addr) = table_addr; 993 } 994 995 /* assign L0 table address to hem from root bt */ 996 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 997 struct hns_roce_hem_item *hem, void *cpu_addr, 998 u64 phy_addr) 999 { 1000 hem->addr = cpu_addr; 1001 hem->dma_addr = (dma_addr_t)phy_addr; 1002 } 1003 1004 static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem, 1005 int offset) 1006 { 1007 return (hem->start <= offset && offset <= hem->end); 1008 } 1009 1010 static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1011 int page_offset) 1012 { 1013 struct hns_roce_hem_item *hem, *temp_hem; 1014 struct hns_roce_hem_item *found = NULL; 1015 1016 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1017 if (hem_list_page_is_in_range(hem, page_offset)) { 1018 found = hem; 1019 break; 1020 } 1021 } 1022 1023 return found; 1024 } 1025 1026 static bool hem_list_is_bottom_bt(int hopnum, int bt_level) 1027 { 1028 /* 1029 * hopnum base address table levels 1030 * 0 L0(buf) 1031 * 1 L0 -> buf 1032 * 2 L0 -> L1 -> buf 1033 * 3 L0 -> L1 -> L2 -> buf 1034 */ 1035 return bt_level >= (hopnum ? hopnum - 1 : hopnum); 1036 } 1037 1038 /* 1039 * calc base address entries num 1040 * @hopnum: num of mutihop addressing 1041 * @bt_level: base address table level 1042 * @unit: ba entries per bt page 1043 */ 1044 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) 1045 { 1046 u32 step; 1047 int max; 1048 int i; 1049 1050 if (hopnum <= bt_level) 1051 return 0; 1052 /* 1053 * hopnum bt_level range 1054 * 1 0 unit 1055 * ------------ 1056 * 2 0 unit * unit 1057 * 2 1 unit 1058 * ------------ 1059 * 3 0 unit * unit * unit 1060 * 3 1 unit * unit 1061 * 3 2 unit 1062 */ 1063 step = 1; 1064 max = hopnum - bt_level; 1065 for (i = 0; i < max; i++) 1066 step = step * unit; 1067 1068 return step; 1069 } 1070 1071 /* 1072 * calc the root ba entries which could cover all regions 1073 * @regions: buf region array 1074 * @region_cnt: array size of @regions 1075 * @unit: ba entries per bt page 1076 */ 1077 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, 1078 int region_cnt, int unit) 1079 { 1080 struct hns_roce_buf_region *r; 1081 int total = 0; 1082 int step; 1083 int i; 1084 1085 for (i = 0; i < region_cnt; i++) { 1086 r = (struct hns_roce_buf_region *)®ions[i]; 1087 if (r->hopnum > 1) { 1088 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1089 if (step > 0) 1090 total += (r->count + step - 1) / step; 1091 } else { 1092 total += r->count; 1093 } 1094 } 1095 1096 return total; 1097 } 1098 1099 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, 1100 const struct hns_roce_buf_region *r, int unit, 1101 int offset, struct list_head *mid_bt, 1102 struct list_head *btm_bt) 1103 { 1104 struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1105 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1106 struct hns_roce_hem_item *cur, *pre; 1107 const int hopnum = r->hopnum; 1108 int start_aligned; 1109 int distance; 1110 int ret = 0; 1111 int max_ofs; 1112 int level; 1113 u32 step; 1114 int end; 1115 1116 if (hopnum <= 1) 1117 return 0; 1118 1119 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { 1120 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); 1121 return -EINVAL; 1122 } 1123 1124 if (offset < r->offset) { 1125 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", 1126 offset, r->offset); 1127 return -EINVAL; 1128 } 1129 1130 distance = offset - r->offset; 1131 max_ofs = r->offset + r->count - 1; 1132 for (level = 0; level < hopnum; level++) 1133 INIT_LIST_HEAD(&temp_list[level]); 1134 1135 /* config L1 bt to last bt and link them to corresponding parent */ 1136 for (level = 1; level < hopnum; level++) { 1137 cur = hem_list_search_item(&mid_bt[level], offset); 1138 if (cur) { 1139 hem_ptrs[level] = cur; 1140 continue; 1141 } 1142 1143 step = hem_list_calc_ba_range(hopnum, level, unit); 1144 if (step < 1) { 1145 ret = -EINVAL; 1146 goto err_exit; 1147 } 1148 1149 start_aligned = (distance / step) * step + r->offset; 1150 end = min_t(int, start_aligned + step - 1, max_ofs); 1151 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, 1152 true); 1153 if (!cur) { 1154 ret = -ENOMEM; 1155 goto err_exit; 1156 } 1157 hem_ptrs[level] = cur; 1158 list_add(&cur->list, &temp_list[level]); 1159 if (hem_list_is_bottom_bt(hopnum, level)) 1160 list_add(&cur->sibling, &temp_list[0]); 1161 1162 /* link bt to parent bt */ 1163 if (level > 1) { 1164 pre = hem_ptrs[level - 1]; 1165 step = (cur->start - pre->start) / step * BA_BYTE_LEN; 1166 hem_list_link_bt(hr_dev, pre->addr + step, 1167 cur->dma_addr); 1168 } 1169 } 1170 1171 list_splice(&temp_list[0], btm_bt); 1172 for (level = 1; level < hopnum; level++) 1173 list_splice(&temp_list[level], &mid_bt[level]); 1174 1175 return 0; 1176 1177 err_exit: 1178 for (level = 1; level < hopnum; level++) 1179 hem_list_free_all(hr_dev, &temp_list[level], true); 1180 1181 return ret; 1182 } 1183 1184 static struct hns_roce_hem_item * 1185 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, 1186 const struct hns_roce_buf_region *regions, int region_cnt) 1187 { 1188 const struct hns_roce_buf_region *r; 1189 struct hns_roce_hem_item *hem; 1190 int ba_num; 1191 int offset; 1192 1193 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); 1194 if (ba_num < 1) 1195 return ERR_PTR(-ENOMEM); 1196 1197 if (ba_num > unit) 1198 return ERR_PTR(-ENOBUFS); 1199 1200 offset = regions[0].offset; 1201 /* indicate to last region */ 1202 r = ®ions[region_cnt - 1]; 1203 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1204 ba_num, true); 1205 if (!hem) 1206 return ERR_PTR(-ENOMEM); 1207 1208 *max_ba_num = ba_num; 1209 1210 return hem; 1211 } 1212 1213 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1214 u64 phy_base, const struct hns_roce_buf_region *r, 1215 struct list_head *branch_head, 1216 struct list_head *leaf_head) 1217 { 1218 struct hns_roce_hem_item *hem; 1219 1220 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, 1221 r->count, false); 1222 if (!hem) 1223 return -ENOMEM; 1224 1225 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1226 list_add(&hem->list, branch_head); 1227 list_add(&hem->sibling, leaf_head); 1228 1229 return r->count; 1230 } 1231 1232 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1233 int unit, const struct hns_roce_buf_region *r, 1234 const struct list_head *branch_head) 1235 { 1236 struct hns_roce_hem_item *hem, *temp_hem; 1237 int total = 0; 1238 int offset; 1239 int step; 1240 1241 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1242 if (step < 1) 1243 return -EINVAL; 1244 1245 /* if exist mid bt, link L1 to L0 */ 1246 list_for_each_entry_safe(hem, temp_hem, branch_head, list) { 1247 offset = (hem->start - r->offset) / step * BA_BYTE_LEN; 1248 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); 1249 total++; 1250 } 1251 1252 return total; 1253 } 1254 1255 static int 1256 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, 1257 int unit, int max_ba_num, struct hns_roce_hem_head *head, 1258 const struct hns_roce_buf_region *regions, int region_cnt) 1259 { 1260 const struct hns_roce_buf_region *r; 1261 struct hns_roce_hem_item *root_hem; 1262 void *cpu_base; 1263 u64 phy_base; 1264 int i, total; 1265 int ret; 1266 1267 root_hem = list_first_entry(&head->root, 1268 struct hns_roce_hem_item, list); 1269 if (!root_hem) 1270 return -ENOMEM; 1271 1272 total = 0; 1273 for (i = 0; i < region_cnt && total < max_ba_num; i++) { 1274 r = ®ions[i]; 1275 if (!r->count) 1276 continue; 1277 1278 /* all regions's mid[x][0] shared the root_bt's trunk */ 1279 cpu_base = root_hem->addr + total * BA_BYTE_LEN; 1280 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; 1281 1282 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1283 * which's address share to all regions. 1284 */ 1285 if (hem_list_is_bottom_bt(r->hopnum, 0)) 1286 ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r, 1287 &head->branch[i], &head->leaf); 1288 else 1289 ret = setup_middle_bt(hr_dev, cpu_base, unit, r, 1290 &hem_list->mid_bt[i][1]); 1291 1292 if (ret < 0) 1293 return ret; 1294 1295 total += ret; 1296 } 1297 1298 list_splice(&head->leaf, &hem_list->btm_bt); 1299 list_splice(&head->root, &hem_list->root_bt); 1300 for (i = 0; i < region_cnt; i++) 1301 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); 1302 1303 return 0; 1304 } 1305 1306 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1307 struct hns_roce_hem_list *hem_list, int unit, 1308 const struct hns_roce_buf_region *regions, 1309 int region_cnt) 1310 { 1311 struct hns_roce_hem_item *root_hem; 1312 struct hns_roce_hem_head head; 1313 int max_ba_num; 1314 int ret; 1315 int i; 1316 1317 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); 1318 if (root_hem) 1319 return 0; 1320 1321 max_ba_num = 0; 1322 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, 1323 region_cnt); 1324 if (IS_ERR(root_hem)) 1325 return PTR_ERR(root_hem); 1326 1327 /* List head for storing all allocated HEM items */ 1328 INIT_LIST_HEAD(&head.root); 1329 INIT_LIST_HEAD(&head.leaf); 1330 for (i = 0; i < region_cnt; i++) 1331 INIT_LIST_HEAD(&head.branch[i]); 1332 1333 hem_list->root_ba = root_hem->dma_addr; 1334 list_add(&root_hem->list, &head.root); 1335 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, 1336 region_cnt); 1337 if (ret) { 1338 for (i = 0; i < region_cnt; i++) 1339 hem_list_free_all(hr_dev, &head.branch[i], false); 1340 1341 hem_list_free_all(hr_dev, &head.root, true); 1342 } 1343 1344 return ret; 1345 } 1346 1347 /* construct the base address table and link them by address hop config */ 1348 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, 1349 struct hns_roce_hem_list *hem_list, 1350 const struct hns_roce_buf_region *regions, 1351 int region_cnt, unsigned int bt_pg_shift) 1352 { 1353 const struct hns_roce_buf_region *r; 1354 int ofs, end; 1355 int unit; 1356 int ret; 1357 int i; 1358 1359 if (region_cnt > HNS_ROCE_MAX_BT_REGION) { 1360 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", 1361 region_cnt); 1362 return -EINVAL; 1363 } 1364 1365 unit = (1 << bt_pg_shift) / BA_BYTE_LEN; 1366 for (i = 0; i < region_cnt; i++) { 1367 r = ®ions[i]; 1368 if (!r->count) 1369 continue; 1370 1371 end = r->offset + r->count; 1372 for (ofs = r->offset; ofs < end; ofs += unit) { 1373 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, 1374 hem_list->mid_bt[i], 1375 &hem_list->btm_bt); 1376 if (ret) { 1377 dev_err(hr_dev->dev, 1378 "alloc hem trunk fail ret = %d!\n", ret); 1379 goto err_alloc; 1380 } 1381 } 1382 } 1383 1384 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, 1385 region_cnt); 1386 if (ret) 1387 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); 1388 else 1389 return 0; 1390 1391 err_alloc: 1392 hns_roce_hem_list_release(hr_dev, hem_list); 1393 1394 return ret; 1395 } 1396 1397 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, 1398 struct hns_roce_hem_list *hem_list) 1399 { 1400 int i, j; 1401 1402 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1403 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1404 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], 1405 j != 0); 1406 1407 hem_list_free_all(hr_dev, &hem_list->root_bt, true); 1408 INIT_LIST_HEAD(&hem_list->btm_bt); 1409 hem_list->root_ba = 0; 1410 } 1411 1412 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) 1413 { 1414 int i, j; 1415 1416 INIT_LIST_HEAD(&hem_list->root_bt); 1417 INIT_LIST_HEAD(&hem_list->btm_bt); 1418 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1419 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1420 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); 1421 } 1422 1423 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, 1424 struct hns_roce_hem_list *hem_list, 1425 int offset, int *mtt_cnt) 1426 { 1427 struct list_head *head = &hem_list->btm_bt; 1428 struct hns_roce_hem_item *hem, *temp_hem; 1429 void *cpu_base = NULL; 1430 int nr = 0; 1431 1432 list_for_each_entry_safe(hem, temp_hem, head, sibling) { 1433 if (hem_list_page_is_in_range(hem, offset)) { 1434 nr = offset - hem->start; 1435 cpu_base = hem->addr + nr * BA_BYTE_LEN; 1436 nr = hem->end + 1 - offset; 1437 break; 1438 } 1439 } 1440 1441 if (mtt_cnt) 1442 *mtt_cnt = nr; 1443 1444 return cpu_base; 1445 } 1446