1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include "hns_roce_device.h" 36 #include "hns_roce_hem.h" 37 #include "hns_roce_common.h" 38 39 #define HEM_INDEX_BUF BIT(0) 40 #define HEM_INDEX_L0 BIT(1) 41 #define HEM_INDEX_L1 BIT(2) 42 struct hns_roce_hem_index { 43 u64 buf; 44 u64 l0; 45 u64 l1; 46 u32 inited; /* indicate which index is available */ 47 }; 48 49 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 50 { 51 int hop_num = 0; 52 53 switch (type) { 54 case HEM_TYPE_QPC: 55 hop_num = hr_dev->caps.qpc_hop_num; 56 break; 57 case HEM_TYPE_MTPT: 58 hop_num = hr_dev->caps.mpt_hop_num; 59 break; 60 case HEM_TYPE_CQC: 61 hop_num = hr_dev->caps.cqc_hop_num; 62 break; 63 case HEM_TYPE_SRQC: 64 hop_num = hr_dev->caps.srqc_hop_num; 65 break; 66 case HEM_TYPE_SCCC: 67 hop_num = hr_dev->caps.sccc_hop_num; 68 break; 69 case HEM_TYPE_QPC_TIMER: 70 hop_num = hr_dev->caps.qpc_timer_hop_num; 71 break; 72 case HEM_TYPE_CQC_TIMER: 73 hop_num = hr_dev->caps.cqc_timer_hop_num; 74 break; 75 case HEM_TYPE_GMV: 76 hop_num = hr_dev->caps.gmv_hop_num; 77 break; 78 default: 79 return false; 80 } 81 82 return hop_num ? true : false; 83 } 84 85 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, 86 u32 bt_chunk_num, u64 hem_max_num) 87 { 88 u64 start_idx = round_down(hem_idx, bt_chunk_num); 89 u64 check_max_num = start_idx + bt_chunk_num; 90 u64 i; 91 92 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) 93 if (i != hem_idx && hem[i]) 94 return false; 95 96 return true; 97 } 98 99 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) 100 { 101 u64 start_idx = round_down(ba_idx, bt_chunk_num); 102 int i; 103 104 for (i = 0; i < bt_chunk_num; i++) 105 if (i != ba_idx && bt[start_idx + i]) 106 return false; 107 108 return true; 109 } 110 111 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 112 { 113 if (check_whether_bt_num_3(table_type, hop_num)) 114 return 3; 115 else if (check_whether_bt_num_2(table_type, hop_num)) 116 return 2; 117 else if (check_whether_bt_num_1(table_type, hop_num)) 118 return 1; 119 else 120 return 0; 121 } 122 123 static int get_hem_table_config(struct hns_roce_dev *hr_dev, 124 struct hns_roce_hem_mhop *mhop, 125 u32 type) 126 { 127 struct device *dev = hr_dev->dev; 128 129 switch (type) { 130 case HEM_TYPE_QPC: 131 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 132 + PAGE_SHIFT); 133 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 134 + PAGE_SHIFT); 135 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 136 mhop->hop_num = hr_dev->caps.qpc_hop_num; 137 break; 138 case HEM_TYPE_MTPT: 139 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 140 + PAGE_SHIFT); 141 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 142 + PAGE_SHIFT); 143 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 144 mhop->hop_num = hr_dev->caps.mpt_hop_num; 145 break; 146 case HEM_TYPE_CQC: 147 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 148 + PAGE_SHIFT); 149 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 150 + PAGE_SHIFT); 151 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 152 mhop->hop_num = hr_dev->caps.cqc_hop_num; 153 break; 154 case HEM_TYPE_SCCC: 155 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 156 + PAGE_SHIFT); 157 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 158 + PAGE_SHIFT); 159 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 160 mhop->hop_num = hr_dev->caps.sccc_hop_num; 161 break; 162 case HEM_TYPE_QPC_TIMER: 163 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 164 + PAGE_SHIFT); 165 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 166 + PAGE_SHIFT); 167 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 168 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 169 break; 170 case HEM_TYPE_CQC_TIMER: 171 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 172 + PAGE_SHIFT); 173 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 174 + PAGE_SHIFT); 175 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 176 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 177 break; 178 case HEM_TYPE_SRQC: 179 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 180 + PAGE_SHIFT); 181 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 182 + PAGE_SHIFT); 183 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 184 mhop->hop_num = hr_dev->caps.srqc_hop_num; 185 break; 186 case HEM_TYPE_GMV: 187 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + 188 PAGE_SHIFT); 189 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + 190 PAGE_SHIFT); 191 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; 192 mhop->hop_num = hr_dev->caps.gmv_hop_num; 193 break; 194 default: 195 dev_err(dev, "table %u not support multi-hop addressing!\n", 196 type); 197 return -EINVAL; 198 } 199 200 return 0; 201 } 202 203 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 204 struct hns_roce_hem_table *table, unsigned long *obj, 205 struct hns_roce_hem_mhop *mhop) 206 { 207 struct device *dev = hr_dev->dev; 208 u32 chunk_ba_num; 209 u32 chunk_size; 210 u32 table_idx; 211 u32 bt_num; 212 213 if (get_hem_table_config(hr_dev, mhop, table->type)) 214 return -EINVAL; 215 216 if (!obj) 217 return 0; 218 219 /* 220 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 221 * MTT/CQE alloc hem for bt pages. 222 */ 223 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 224 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 225 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 226 mhop->bt_chunk_size; 227 table_idx = *obj / (chunk_size / table->obj_size); 228 switch (bt_num) { 229 case 3: 230 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 231 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 232 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 233 break; 234 case 2: 235 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 236 mhop->l0_idx = table_idx / chunk_ba_num; 237 break; 238 case 1: 239 mhop->l0_idx = table_idx; 240 break; 241 default: 242 dev_err(dev, "table %u not support hop_num = %u!\n", 243 table->type, mhop->hop_num); 244 return -EINVAL; 245 } 246 if (mhop->l0_idx >= mhop->ba_l0_num) 247 mhop->l0_idx %= mhop->ba_l0_num; 248 249 return 0; 250 } 251 252 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 253 int npages, 254 unsigned long hem_alloc_size, 255 gfp_t gfp_mask) 256 { 257 struct hns_roce_hem_chunk *chunk = NULL; 258 struct hns_roce_hem *hem; 259 struct scatterlist *mem; 260 int order; 261 void *buf; 262 263 WARN_ON(gfp_mask & __GFP_HIGHMEM); 264 265 hem = kmalloc(sizeof(*hem), 266 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 267 if (!hem) 268 return NULL; 269 270 INIT_LIST_HEAD(&hem->chunk_list); 271 272 order = get_order(hem_alloc_size); 273 274 while (npages > 0) { 275 if (!chunk) { 276 chunk = kmalloc(sizeof(*chunk), 277 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 278 if (!chunk) 279 goto fail; 280 281 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 282 chunk->npages = 0; 283 chunk->nsg = 0; 284 memset(chunk->buf, 0, sizeof(chunk->buf)); 285 list_add_tail(&chunk->list, &hem->chunk_list); 286 } 287 288 while (1 << order > npages) 289 --order; 290 291 /* 292 * Alloc memory one time. If failed, don't alloc small block 293 * memory, directly return fail. 294 */ 295 mem = &chunk->mem[chunk->npages]; 296 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 297 &sg_dma_address(mem), gfp_mask); 298 if (!buf) 299 goto fail; 300 301 chunk->buf[chunk->npages] = buf; 302 sg_dma_len(mem) = PAGE_SIZE << order; 303 304 ++chunk->npages; 305 ++chunk->nsg; 306 npages -= 1 << order; 307 } 308 309 return hem; 310 311 fail: 312 hns_roce_free_hem(hr_dev, hem); 313 return NULL; 314 } 315 316 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 317 { 318 struct hns_roce_hem_chunk *chunk, *tmp; 319 int i; 320 321 if (!hem) 322 return; 323 324 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 325 for (i = 0; i < chunk->npages; ++i) 326 dma_free_coherent(hr_dev->dev, 327 sg_dma_len(&chunk->mem[i]), 328 chunk->buf[i], 329 sg_dma_address(&chunk->mem[i])); 330 kfree(chunk); 331 } 332 333 kfree(hem); 334 } 335 336 static int calc_hem_config(struct hns_roce_dev *hr_dev, 337 struct hns_roce_hem_table *table, unsigned long obj, 338 struct hns_roce_hem_mhop *mhop, 339 struct hns_roce_hem_index *index) 340 { 341 struct ib_device *ibdev = &hr_dev->ib_dev; 342 unsigned long mhop_obj = obj; 343 u32 l0_idx, l1_idx, l2_idx; 344 u32 chunk_ba_num; 345 u32 bt_num; 346 int ret; 347 348 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); 349 if (ret) 350 return ret; 351 352 l0_idx = mhop->l0_idx; 353 l1_idx = mhop->l1_idx; 354 l2_idx = mhop->l2_idx; 355 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 356 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 357 switch (bt_num) { 358 case 3: 359 index->l1 = l0_idx * chunk_ba_num + l1_idx; 360 index->l0 = l0_idx; 361 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + 362 l1_idx * chunk_ba_num + l2_idx; 363 break; 364 case 2: 365 index->l0 = l0_idx; 366 index->buf = l0_idx * chunk_ba_num + l1_idx; 367 break; 368 case 1: 369 index->buf = l0_idx; 370 break; 371 default: 372 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n", 373 table->type, mhop->hop_num); 374 return -EINVAL; 375 } 376 377 if (unlikely(index->buf >= table->num_hem)) { 378 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n", 379 table->type, index->buf, table->num_hem); 380 return -EINVAL; 381 } 382 383 return 0; 384 } 385 386 static void free_mhop_hem(struct hns_roce_dev *hr_dev, 387 struct hns_roce_hem_table *table, 388 struct hns_roce_hem_mhop *mhop, 389 struct hns_roce_hem_index *index) 390 { 391 u32 bt_size = mhop->bt_chunk_size; 392 struct device *dev = hr_dev->dev; 393 394 if (index->inited & HEM_INDEX_BUF) { 395 hns_roce_free_hem(hr_dev, table->hem[index->buf]); 396 table->hem[index->buf] = NULL; 397 } 398 399 if (index->inited & HEM_INDEX_L1) { 400 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], 401 table->bt_l1_dma_addr[index->l1]); 402 table->bt_l1[index->l1] = NULL; 403 } 404 405 if (index->inited & HEM_INDEX_L0) { 406 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], 407 table->bt_l0_dma_addr[index->l0]); 408 table->bt_l0[index->l0] = NULL; 409 } 410 } 411 412 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, 413 struct hns_roce_hem_table *table, 414 struct hns_roce_hem_mhop *mhop, 415 struct hns_roce_hem_index *index) 416 { 417 u32 bt_size = mhop->bt_chunk_size; 418 struct device *dev = hr_dev->dev; 419 struct hns_roce_hem_iter iter; 420 gfp_t flag; 421 u64 bt_ba; 422 u32 size; 423 int ret; 424 425 /* alloc L1 BA's chunk */ 426 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || 427 check_whether_bt_num_2(table->type, mhop->hop_num)) && 428 !table->bt_l0[index->l0]) { 429 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, 430 &table->bt_l0_dma_addr[index->l0], 431 GFP_KERNEL); 432 if (!table->bt_l0[index->l0]) { 433 ret = -ENOMEM; 434 goto out; 435 } 436 index->inited |= HEM_INDEX_L0; 437 } 438 439 /* alloc L2 BA's chunk */ 440 if (check_whether_bt_num_3(table->type, mhop->hop_num) && 441 !table->bt_l1[index->l1]) { 442 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, 443 &table->bt_l1_dma_addr[index->l1], 444 GFP_KERNEL); 445 if (!table->bt_l1[index->l1]) { 446 ret = -ENOMEM; 447 goto err_alloc_hem; 448 } 449 index->inited |= HEM_INDEX_L1; 450 *(table->bt_l0[index->l0] + mhop->l1_idx) = 451 table->bt_l1_dma_addr[index->l1]; 452 } 453 454 /* 455 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 456 * alloc bt space chunk for MTT/CQE. 457 */ 458 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; 459 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; 460 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, 461 size, flag); 462 if (!table->hem[index->buf]) { 463 ret = -ENOMEM; 464 goto err_alloc_hem; 465 } 466 467 index->inited |= HEM_INDEX_BUF; 468 hns_roce_hem_first(table->hem[index->buf], &iter); 469 bt_ba = hns_roce_hem_addr(&iter); 470 if (table->type < HEM_TYPE_MTT) { 471 if (mhop->hop_num == 2) 472 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; 473 else if (mhop->hop_num == 1) 474 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 475 } else if (mhop->hop_num == 2) { 476 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 477 } 478 479 return 0; 480 err_alloc_hem: 481 free_mhop_hem(hr_dev, table, mhop, index); 482 out: 483 return ret; 484 } 485 486 static int set_mhop_hem(struct hns_roce_dev *hr_dev, 487 struct hns_roce_hem_table *table, unsigned long obj, 488 struct hns_roce_hem_mhop *mhop, 489 struct hns_roce_hem_index *index) 490 { 491 struct ib_device *ibdev = &hr_dev->ib_dev; 492 int step_idx; 493 int ret = 0; 494 495 if (index->inited & HEM_INDEX_L0) { 496 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); 497 if (ret) { 498 ibdev_err(ibdev, "set HEM step 0 failed!\n"); 499 goto out; 500 } 501 } 502 503 if (index->inited & HEM_INDEX_L1) { 504 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); 505 if (ret) { 506 ibdev_err(ibdev, "set HEM step 1 failed!\n"); 507 goto out; 508 } 509 } 510 511 if (index->inited & HEM_INDEX_BUF) { 512 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) 513 step_idx = 0; 514 else 515 step_idx = mhop->hop_num; 516 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); 517 if (ret) 518 ibdev_err(ibdev, "set HEM step last failed!\n"); 519 } 520 out: 521 return ret; 522 } 523 524 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 525 struct hns_roce_hem_table *table, 526 unsigned long obj) 527 { 528 struct ib_device *ibdev = &hr_dev->ib_dev; 529 struct hns_roce_hem_index index = {}; 530 struct hns_roce_hem_mhop mhop = {}; 531 int ret; 532 533 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 534 if (ret) { 535 ibdev_err(ibdev, "calc hem config failed!\n"); 536 return ret; 537 } 538 539 mutex_lock(&table->mutex); 540 if (table->hem[index.buf]) { 541 refcount_inc(&table->hem[index.buf]->refcount); 542 goto out; 543 } 544 545 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); 546 if (ret) { 547 ibdev_err(ibdev, "alloc mhop hem failed!\n"); 548 goto out; 549 } 550 551 /* set HEM base address to hardware */ 552 if (table->type < HEM_TYPE_MTT) { 553 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); 554 if (ret) { 555 ibdev_err(ibdev, "set HEM address to HW failed!\n"); 556 goto err_alloc; 557 } 558 } 559 560 refcount_set(&table->hem[index.buf]->refcount, 1); 561 goto out; 562 563 err_alloc: 564 free_mhop_hem(hr_dev, table, &mhop, &index); 565 out: 566 mutex_unlock(&table->mutex); 567 return ret; 568 } 569 570 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 571 struct hns_roce_hem_table *table, unsigned long obj) 572 { 573 struct device *dev = hr_dev->dev; 574 unsigned long i; 575 int ret = 0; 576 577 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 578 return hns_roce_table_mhop_get(hr_dev, table, obj); 579 580 i = obj / (table->table_chunk_size / table->obj_size); 581 582 mutex_lock(&table->mutex); 583 584 if (table->hem[i]) { 585 refcount_inc(&table->hem[i]->refcount); 586 goto out; 587 } 588 589 table->hem[i] = hns_roce_alloc_hem(hr_dev, 590 table->table_chunk_size >> PAGE_SHIFT, 591 table->table_chunk_size, 592 (table->lowmem ? GFP_KERNEL : 593 GFP_HIGHUSER) | __GFP_NOWARN); 594 if (!table->hem[i]) { 595 ret = -ENOMEM; 596 goto out; 597 } 598 599 /* Set HEM base address(128K/page, pa) to Hardware */ 600 if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) { 601 hns_roce_free_hem(hr_dev, table->hem[i]); 602 table->hem[i] = NULL; 603 ret = -ENODEV; 604 dev_err(dev, "set HEM base address to HW failed.\n"); 605 goto out; 606 } 607 608 refcount_set(&table->hem[i]->refcount, 1); 609 out: 610 mutex_unlock(&table->mutex); 611 return ret; 612 } 613 614 static void clear_mhop_hem(struct hns_roce_dev *hr_dev, 615 struct hns_roce_hem_table *table, unsigned long obj, 616 struct hns_roce_hem_mhop *mhop, 617 struct hns_roce_hem_index *index) 618 { 619 struct ib_device *ibdev = &hr_dev->ib_dev; 620 u32 hop_num = mhop->hop_num; 621 u32 chunk_ba_num; 622 int step_idx; 623 624 index->inited = HEM_INDEX_BUF; 625 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 626 if (check_whether_bt_num_2(table->type, hop_num)) { 627 if (hns_roce_check_hem_null(table->hem, index->buf, 628 chunk_ba_num, table->num_hem)) 629 index->inited |= HEM_INDEX_L0; 630 } else if (check_whether_bt_num_3(table->type, hop_num)) { 631 if (hns_roce_check_hem_null(table->hem, index->buf, 632 chunk_ba_num, table->num_hem)) { 633 index->inited |= HEM_INDEX_L1; 634 if (hns_roce_check_bt_null(table->bt_l1, index->l1, 635 chunk_ba_num)) 636 index->inited |= HEM_INDEX_L0; 637 } 638 } 639 640 if (table->type < HEM_TYPE_MTT) { 641 if (hop_num == HNS_ROCE_HOP_NUM_0) 642 step_idx = 0; 643 else 644 step_idx = hop_num; 645 646 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) 647 ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num); 648 649 if (index->inited & HEM_INDEX_L1) 650 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 651 ibdev_warn(ibdev, "failed to clear HEM step 1.\n"); 652 653 if (index->inited & HEM_INDEX_L0) 654 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 655 ibdev_warn(ibdev, "failed to clear HEM step 0.\n"); 656 } 657 } 658 659 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 660 struct hns_roce_hem_table *table, 661 unsigned long obj, 662 int check_refcount) 663 { 664 struct ib_device *ibdev = &hr_dev->ib_dev; 665 struct hns_roce_hem_index index = {}; 666 struct hns_roce_hem_mhop mhop = {}; 667 int ret; 668 669 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 670 if (ret) { 671 ibdev_err(ibdev, "calc hem config failed!\n"); 672 return; 673 } 674 675 if (!check_refcount) 676 mutex_lock(&table->mutex); 677 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, 678 &table->mutex)) 679 return; 680 681 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); 682 free_mhop_hem(hr_dev, table, &mhop, &index); 683 684 mutex_unlock(&table->mutex); 685 } 686 687 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 688 struct hns_roce_hem_table *table, unsigned long obj) 689 { 690 struct device *dev = hr_dev->dev; 691 unsigned long i; 692 693 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 694 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 695 return; 696 } 697 698 i = obj / (table->table_chunk_size / table->obj_size); 699 700 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, 701 &table->mutex)) 702 return; 703 704 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) 705 dev_warn(dev, "failed to clear HEM base address.\n"); 706 707 hns_roce_free_hem(hr_dev, table->hem[i]); 708 table->hem[i] = NULL; 709 710 mutex_unlock(&table->mutex); 711 } 712 713 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 714 struct hns_roce_hem_table *table, 715 unsigned long obj, dma_addr_t *dma_handle) 716 { 717 struct hns_roce_hem_chunk *chunk; 718 struct hns_roce_hem_mhop mhop; 719 struct hns_roce_hem *hem; 720 unsigned long mhop_obj = obj; 721 unsigned long obj_per_chunk; 722 unsigned long idx_offset; 723 int offset, dma_offset; 724 void *addr = NULL; 725 u32 hem_idx = 0; 726 int length; 727 int i, j; 728 729 if (!table->lowmem) 730 return NULL; 731 732 mutex_lock(&table->mutex); 733 734 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 735 obj_per_chunk = table->table_chunk_size / table->obj_size; 736 hem = table->hem[obj / obj_per_chunk]; 737 idx_offset = obj % obj_per_chunk; 738 dma_offset = offset = idx_offset * table->obj_size; 739 } else { 740 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 741 742 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) 743 goto out; 744 /* mtt mhop */ 745 i = mhop.l0_idx; 746 j = mhop.l1_idx; 747 if (mhop.hop_num == 2) 748 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; 749 else if (mhop.hop_num == 1 || 750 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 751 hem_idx = i; 752 753 hem = table->hem[hem_idx]; 754 dma_offset = offset = obj * seg_size % mhop.bt_chunk_size; 755 if (mhop.hop_num == 2) 756 dma_offset = offset = 0; 757 } 758 759 if (!hem) 760 goto out; 761 762 list_for_each_entry(chunk, &hem->chunk_list, list) { 763 for (i = 0; i < chunk->npages; ++i) { 764 length = sg_dma_len(&chunk->mem[i]); 765 if (dma_handle && dma_offset >= 0) { 766 if (length > (u32)dma_offset) 767 *dma_handle = sg_dma_address( 768 &chunk->mem[i]) + dma_offset; 769 dma_offset -= length; 770 } 771 772 if (length > (u32)offset) { 773 addr = chunk->buf[i] + offset; 774 goto out; 775 } 776 offset -= length; 777 } 778 } 779 780 out: 781 mutex_unlock(&table->mutex); 782 return addr; 783 } 784 785 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 786 struct hns_roce_hem_table *table, u32 type, 787 unsigned long obj_size, unsigned long nobj, 788 int use_lowmem) 789 { 790 unsigned long obj_per_chunk; 791 unsigned long num_hem; 792 793 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 794 table->table_chunk_size = hr_dev->caps.chunk_sz; 795 obj_per_chunk = table->table_chunk_size / obj_size; 796 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); 797 798 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 799 if (!table->hem) 800 return -ENOMEM; 801 } else { 802 struct hns_roce_hem_mhop mhop = {}; 803 unsigned long buf_chunk_size; 804 unsigned long bt_chunk_size; 805 unsigned long bt_chunk_num; 806 unsigned long num_bt_l0; 807 u32 hop_num; 808 809 if (get_hem_table_config(hr_dev, &mhop, type)) 810 return -EINVAL; 811 812 buf_chunk_size = mhop.buf_chunk_size; 813 bt_chunk_size = mhop.bt_chunk_size; 814 num_bt_l0 = mhop.ba_l0_num; 815 hop_num = mhop.hop_num; 816 817 obj_per_chunk = buf_chunk_size / obj_size; 818 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); 819 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; 820 821 if (type >= HEM_TYPE_MTT) 822 num_bt_l0 = bt_chunk_num; 823 824 table->hem = kcalloc(num_hem, sizeof(*table->hem), 825 GFP_KERNEL); 826 if (!table->hem) 827 goto err_kcalloc_hem_buf; 828 829 if (check_whether_bt_num_3(type, hop_num)) { 830 unsigned long num_bt_l1; 831 832 num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num); 833 table->bt_l1 = kcalloc(num_bt_l1, 834 sizeof(*table->bt_l1), 835 GFP_KERNEL); 836 if (!table->bt_l1) 837 goto err_kcalloc_bt_l1; 838 839 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 840 sizeof(*table->bt_l1_dma_addr), 841 GFP_KERNEL); 842 843 if (!table->bt_l1_dma_addr) 844 goto err_kcalloc_l1_dma; 845 } 846 847 if (check_whether_bt_num_2(type, hop_num) || 848 check_whether_bt_num_3(type, hop_num)) { 849 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 850 GFP_KERNEL); 851 if (!table->bt_l0) 852 goto err_kcalloc_bt_l0; 853 854 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 855 sizeof(*table->bt_l0_dma_addr), 856 GFP_KERNEL); 857 if (!table->bt_l0_dma_addr) 858 goto err_kcalloc_l0_dma; 859 } 860 } 861 862 table->type = type; 863 table->num_hem = num_hem; 864 table->obj_size = obj_size; 865 table->lowmem = use_lowmem; 866 mutex_init(&table->mutex); 867 868 return 0; 869 870 err_kcalloc_l0_dma: 871 kfree(table->bt_l0); 872 table->bt_l0 = NULL; 873 874 err_kcalloc_bt_l0: 875 kfree(table->bt_l1_dma_addr); 876 table->bt_l1_dma_addr = NULL; 877 878 err_kcalloc_l1_dma: 879 kfree(table->bt_l1); 880 table->bt_l1 = NULL; 881 882 err_kcalloc_bt_l1: 883 kfree(table->hem); 884 table->hem = NULL; 885 886 err_kcalloc_hem_buf: 887 return -ENOMEM; 888 } 889 890 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 891 struct hns_roce_hem_table *table) 892 { 893 struct hns_roce_hem_mhop mhop; 894 u32 buf_chunk_size; 895 u64 obj; 896 int i; 897 898 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 899 return; 900 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 901 mhop.bt_chunk_size; 902 903 for (i = 0; i < table->num_hem; ++i) { 904 obj = i * buf_chunk_size / table->obj_size; 905 if (table->hem[i]) 906 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 907 } 908 909 kfree(table->hem); 910 table->hem = NULL; 911 kfree(table->bt_l1); 912 table->bt_l1 = NULL; 913 kfree(table->bt_l1_dma_addr); 914 table->bt_l1_dma_addr = NULL; 915 kfree(table->bt_l0); 916 table->bt_l0 = NULL; 917 kfree(table->bt_l0_dma_addr); 918 table->bt_l0_dma_addr = NULL; 919 } 920 921 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 922 struct hns_roce_hem_table *table) 923 { 924 struct device *dev = hr_dev->dev; 925 unsigned long i; 926 927 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 928 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 929 return; 930 } 931 932 for (i = 0; i < table->num_hem; ++i) 933 if (table->hem[i]) { 934 if (hr_dev->hw->clear_hem(hr_dev, table, 935 i * table->table_chunk_size / table->obj_size, 0)) 936 dev_err(dev, "Clear HEM base address failed.\n"); 937 938 hns_roce_free_hem(hr_dev, table->hem[i]); 939 } 940 941 kfree(table->hem); 942 } 943 944 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 945 { 946 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) 947 hns_roce_cleanup_hem_table(hr_dev, 948 &hr_dev->srq_table.table); 949 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 950 if (hr_dev->caps.qpc_timer_entry_sz) 951 hns_roce_cleanup_hem_table(hr_dev, 952 &hr_dev->qpc_timer_table); 953 if (hr_dev->caps.cqc_timer_entry_sz) 954 hns_roce_cleanup_hem_table(hr_dev, 955 &hr_dev->cqc_timer_table); 956 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 957 hns_roce_cleanup_hem_table(hr_dev, 958 &hr_dev->qp_table.sccc_table); 959 if (hr_dev->caps.trrl_entry_sz) 960 hns_roce_cleanup_hem_table(hr_dev, 961 &hr_dev->qp_table.trrl_table); 962 963 if (hr_dev->caps.gmv_entry_sz) 964 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); 965 966 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 967 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 968 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 969 } 970 971 struct hns_roce_hem_item { 972 struct list_head list; /* link all hems in the same bt level */ 973 struct list_head sibling; /* link all hems in last hop for mtt */ 974 void *addr; 975 dma_addr_t dma_addr; 976 size_t count; /* max ba numbers */ 977 int start; /* start buf offset in this hem */ 978 int end; /* end buf offset in this hem */ 979 }; 980 981 /* All HEM items are linked in a tree structure */ 982 struct hns_roce_hem_head { 983 struct list_head branch[HNS_ROCE_MAX_BT_REGION]; 984 struct list_head root; 985 struct list_head leaf; 986 }; 987 988 static struct hns_roce_hem_item * 989 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, 990 bool exist_bt, int bt_level) 991 { 992 struct hns_roce_hem_item *hem; 993 994 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 995 if (!hem) 996 return NULL; 997 998 if (exist_bt) { 999 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, 1000 &hem->dma_addr, GFP_KERNEL); 1001 if (!hem->addr) { 1002 kfree(hem); 1003 return NULL; 1004 } 1005 } 1006 1007 hem->count = count; 1008 hem->start = start; 1009 hem->end = end; 1010 INIT_LIST_HEAD(&hem->list); 1011 INIT_LIST_HEAD(&hem->sibling); 1012 1013 return hem; 1014 } 1015 1016 static void hem_list_free_item(struct hns_roce_dev *hr_dev, 1017 struct hns_roce_hem_item *hem, bool exist_bt) 1018 { 1019 if (exist_bt) 1020 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, 1021 hem->addr, hem->dma_addr); 1022 kfree(hem); 1023 } 1024 1025 static void hem_list_free_all(struct hns_roce_dev *hr_dev, 1026 struct list_head *head, bool exist_bt) 1027 { 1028 struct hns_roce_hem_item *hem, *temp_hem; 1029 1030 list_for_each_entry_safe(hem, temp_hem, head, list) { 1031 list_del(&hem->list); 1032 hem_list_free_item(hr_dev, hem, exist_bt); 1033 } 1034 } 1035 1036 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, 1037 u64 table_addr) 1038 { 1039 *(u64 *)(base_addr) = table_addr; 1040 } 1041 1042 /* assign L0 table address to hem from root bt */ 1043 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 1044 struct hns_roce_hem_item *hem, void *cpu_addr, 1045 u64 phy_addr) 1046 { 1047 hem->addr = cpu_addr; 1048 hem->dma_addr = (dma_addr_t)phy_addr; 1049 } 1050 1051 static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem, 1052 int offset) 1053 { 1054 return (hem->start <= offset && offset <= hem->end); 1055 } 1056 1057 static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1058 int page_offset) 1059 { 1060 struct hns_roce_hem_item *hem, *temp_hem; 1061 struct hns_roce_hem_item *found = NULL; 1062 1063 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1064 if (hem_list_page_is_in_range(hem, page_offset)) { 1065 found = hem; 1066 break; 1067 } 1068 } 1069 1070 return found; 1071 } 1072 1073 static bool hem_list_is_bottom_bt(int hopnum, int bt_level) 1074 { 1075 /* 1076 * hopnum base address table levels 1077 * 0 L0(buf) 1078 * 1 L0 -> buf 1079 * 2 L0 -> L1 -> buf 1080 * 3 L0 -> L1 -> L2 -> buf 1081 */ 1082 return bt_level >= (hopnum ? hopnum - 1 : hopnum); 1083 } 1084 1085 /* 1086 * calc base address entries num 1087 * @hopnum: num of mutihop addressing 1088 * @bt_level: base address table level 1089 * @unit: ba entries per bt page 1090 */ 1091 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) 1092 { 1093 u32 step; 1094 int max; 1095 int i; 1096 1097 if (hopnum <= bt_level) 1098 return 0; 1099 /* 1100 * hopnum bt_level range 1101 * 1 0 unit 1102 * ------------ 1103 * 2 0 unit * unit 1104 * 2 1 unit 1105 * ------------ 1106 * 3 0 unit * unit * unit 1107 * 3 1 unit * unit 1108 * 3 2 unit 1109 */ 1110 step = 1; 1111 max = hopnum - bt_level; 1112 for (i = 0; i < max; i++) 1113 step = step * unit; 1114 1115 return step; 1116 } 1117 1118 /* 1119 * calc the root ba entries which could cover all regions 1120 * @regions: buf region array 1121 * @region_cnt: array size of @regions 1122 * @unit: ba entries per bt page 1123 */ 1124 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, 1125 int region_cnt, int unit) 1126 { 1127 struct hns_roce_buf_region *r; 1128 int total = 0; 1129 int step; 1130 int i; 1131 1132 for (i = 0; i < region_cnt; i++) { 1133 r = (struct hns_roce_buf_region *)®ions[i]; 1134 if (r->hopnum > 1) { 1135 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1136 if (step > 0) 1137 total += (r->count + step - 1) / step; 1138 } else { 1139 total += r->count; 1140 } 1141 } 1142 1143 return total; 1144 } 1145 1146 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, 1147 const struct hns_roce_buf_region *r, int unit, 1148 int offset, struct list_head *mid_bt, 1149 struct list_head *btm_bt) 1150 { 1151 struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1152 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1153 struct hns_roce_hem_item *cur, *pre; 1154 const int hopnum = r->hopnum; 1155 int start_aligned; 1156 int distance; 1157 int ret = 0; 1158 int max_ofs; 1159 int level; 1160 u32 step; 1161 int end; 1162 1163 if (hopnum <= 1) 1164 return 0; 1165 1166 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { 1167 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); 1168 return -EINVAL; 1169 } 1170 1171 if (offset < r->offset) { 1172 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", 1173 offset, r->offset); 1174 return -EINVAL; 1175 } 1176 1177 distance = offset - r->offset; 1178 max_ofs = r->offset + r->count - 1; 1179 for (level = 0; level < hopnum; level++) 1180 INIT_LIST_HEAD(&temp_list[level]); 1181 1182 /* config L1 bt to last bt and link them to corresponding parent */ 1183 for (level = 1; level < hopnum; level++) { 1184 cur = hem_list_search_item(&mid_bt[level], offset); 1185 if (cur) { 1186 hem_ptrs[level] = cur; 1187 continue; 1188 } 1189 1190 step = hem_list_calc_ba_range(hopnum, level, unit); 1191 if (step < 1) { 1192 ret = -EINVAL; 1193 goto err_exit; 1194 } 1195 1196 start_aligned = (distance / step) * step + r->offset; 1197 end = min_t(int, start_aligned + step - 1, max_ofs); 1198 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, 1199 true, level); 1200 if (!cur) { 1201 ret = -ENOMEM; 1202 goto err_exit; 1203 } 1204 hem_ptrs[level] = cur; 1205 list_add(&cur->list, &temp_list[level]); 1206 if (hem_list_is_bottom_bt(hopnum, level)) 1207 list_add(&cur->sibling, &temp_list[0]); 1208 1209 /* link bt to parent bt */ 1210 if (level > 1) { 1211 pre = hem_ptrs[level - 1]; 1212 step = (cur->start - pre->start) / step * BA_BYTE_LEN; 1213 hem_list_link_bt(hr_dev, pre->addr + step, 1214 cur->dma_addr); 1215 } 1216 } 1217 1218 list_splice(&temp_list[0], btm_bt); 1219 for (level = 1; level < hopnum; level++) 1220 list_splice(&temp_list[level], &mid_bt[level]); 1221 1222 return 0; 1223 1224 err_exit: 1225 for (level = 1; level < hopnum; level++) 1226 hem_list_free_all(hr_dev, &temp_list[level], true); 1227 1228 return ret; 1229 } 1230 1231 static struct hns_roce_hem_item * 1232 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, 1233 const struct hns_roce_buf_region *regions, int region_cnt) 1234 { 1235 const struct hns_roce_buf_region *r; 1236 struct hns_roce_hem_item *hem; 1237 int ba_num; 1238 int offset; 1239 1240 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); 1241 if (ba_num < 1) 1242 return ERR_PTR(-ENOMEM); 1243 1244 if (ba_num > unit) 1245 return ERR_PTR(-ENOBUFS); 1246 1247 offset = regions[0].offset; 1248 /* indicate to last region */ 1249 r = ®ions[region_cnt - 1]; 1250 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1251 ba_num, true, 0); 1252 if (!hem) 1253 return ERR_PTR(-ENOMEM); 1254 1255 *max_ba_num = ba_num; 1256 1257 return hem; 1258 } 1259 1260 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1261 u64 phy_base, const struct hns_roce_buf_region *r, 1262 struct list_head *branch_head, 1263 struct list_head *leaf_head) 1264 { 1265 struct hns_roce_hem_item *hem; 1266 1267 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, 1268 r->count, false, 0); 1269 if (!hem) 1270 return -ENOMEM; 1271 1272 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1273 list_add(&hem->list, branch_head); 1274 list_add(&hem->sibling, leaf_head); 1275 1276 return r->count; 1277 } 1278 1279 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1280 int unit, const struct hns_roce_buf_region *r, 1281 const struct list_head *branch_head) 1282 { 1283 struct hns_roce_hem_item *hem, *temp_hem; 1284 int total = 0; 1285 int offset; 1286 int step; 1287 1288 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1289 if (step < 1) 1290 return -EINVAL; 1291 1292 /* if exist mid bt, link L1 to L0 */ 1293 list_for_each_entry_safe(hem, temp_hem, branch_head, list) { 1294 offset = (hem->start - r->offset) / step * BA_BYTE_LEN; 1295 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); 1296 total++; 1297 } 1298 1299 return total; 1300 } 1301 1302 static int 1303 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, 1304 int unit, int max_ba_num, struct hns_roce_hem_head *head, 1305 const struct hns_roce_buf_region *regions, int region_cnt) 1306 { 1307 const struct hns_roce_buf_region *r; 1308 struct hns_roce_hem_item *root_hem; 1309 void *cpu_base; 1310 u64 phy_base; 1311 int i, total; 1312 int ret; 1313 1314 root_hem = list_first_entry(&head->root, 1315 struct hns_roce_hem_item, list); 1316 if (!root_hem) 1317 return -ENOMEM; 1318 1319 total = 0; 1320 for (i = 0; i < region_cnt && total < max_ba_num; i++) { 1321 r = ®ions[i]; 1322 if (!r->count) 1323 continue; 1324 1325 /* all regions's mid[x][0] shared the root_bt's trunk */ 1326 cpu_base = root_hem->addr + total * BA_BYTE_LEN; 1327 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; 1328 1329 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1330 * which's address share to all regions. 1331 */ 1332 if (hem_list_is_bottom_bt(r->hopnum, 0)) 1333 ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r, 1334 &head->branch[i], &head->leaf); 1335 else 1336 ret = setup_middle_bt(hr_dev, cpu_base, unit, r, 1337 &hem_list->mid_bt[i][1]); 1338 1339 if (ret < 0) 1340 return ret; 1341 1342 total += ret; 1343 } 1344 1345 list_splice(&head->leaf, &hem_list->btm_bt); 1346 list_splice(&head->root, &hem_list->root_bt); 1347 for (i = 0; i < region_cnt; i++) 1348 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); 1349 1350 return 0; 1351 } 1352 1353 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1354 struct hns_roce_hem_list *hem_list, int unit, 1355 const struct hns_roce_buf_region *regions, 1356 int region_cnt) 1357 { 1358 struct hns_roce_hem_item *root_hem; 1359 struct hns_roce_hem_head head; 1360 int max_ba_num; 1361 int ret; 1362 int i; 1363 1364 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); 1365 if (root_hem) 1366 return 0; 1367 1368 max_ba_num = 0; 1369 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, 1370 region_cnt); 1371 if (IS_ERR(root_hem)) 1372 return PTR_ERR(root_hem); 1373 1374 /* List head for storing all allocated HEM items */ 1375 INIT_LIST_HEAD(&head.root); 1376 INIT_LIST_HEAD(&head.leaf); 1377 for (i = 0; i < region_cnt; i++) 1378 INIT_LIST_HEAD(&head.branch[i]); 1379 1380 hem_list->root_ba = root_hem->dma_addr; 1381 list_add(&root_hem->list, &head.root); 1382 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, 1383 region_cnt); 1384 if (ret) { 1385 for (i = 0; i < region_cnt; i++) 1386 hem_list_free_all(hr_dev, &head.branch[i], false); 1387 1388 hem_list_free_all(hr_dev, &head.root, true); 1389 } 1390 1391 return ret; 1392 } 1393 1394 /* construct the base address table and link them by address hop config */ 1395 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, 1396 struct hns_roce_hem_list *hem_list, 1397 const struct hns_roce_buf_region *regions, 1398 int region_cnt, unsigned int bt_pg_shift) 1399 { 1400 const struct hns_roce_buf_region *r; 1401 int ofs, end; 1402 int unit; 1403 int ret; 1404 int i; 1405 1406 if (region_cnt > HNS_ROCE_MAX_BT_REGION) { 1407 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", 1408 region_cnt); 1409 return -EINVAL; 1410 } 1411 1412 unit = (1 << bt_pg_shift) / BA_BYTE_LEN; 1413 for (i = 0; i < region_cnt; i++) { 1414 r = ®ions[i]; 1415 if (!r->count) 1416 continue; 1417 1418 end = r->offset + r->count; 1419 for (ofs = r->offset; ofs < end; ofs += unit) { 1420 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, 1421 hem_list->mid_bt[i], 1422 &hem_list->btm_bt); 1423 if (ret) { 1424 dev_err(hr_dev->dev, 1425 "alloc hem trunk fail ret=%d!\n", ret); 1426 goto err_alloc; 1427 } 1428 } 1429 } 1430 1431 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, 1432 region_cnt); 1433 if (ret) 1434 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); 1435 else 1436 return 0; 1437 1438 err_alloc: 1439 hns_roce_hem_list_release(hr_dev, hem_list); 1440 1441 return ret; 1442 } 1443 1444 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, 1445 struct hns_roce_hem_list *hem_list) 1446 { 1447 int i, j; 1448 1449 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1450 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1451 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], 1452 j != 0); 1453 1454 hem_list_free_all(hr_dev, &hem_list->root_bt, true); 1455 INIT_LIST_HEAD(&hem_list->btm_bt); 1456 hem_list->root_ba = 0; 1457 } 1458 1459 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) 1460 { 1461 int i, j; 1462 1463 INIT_LIST_HEAD(&hem_list->root_bt); 1464 INIT_LIST_HEAD(&hem_list->btm_bt); 1465 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1466 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1467 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); 1468 } 1469 1470 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, 1471 struct hns_roce_hem_list *hem_list, 1472 int offset, int *mtt_cnt, u64 *phy_addr) 1473 { 1474 struct list_head *head = &hem_list->btm_bt; 1475 struct hns_roce_hem_item *hem, *temp_hem; 1476 void *cpu_base = NULL; 1477 u64 phy_base = 0; 1478 int nr = 0; 1479 1480 list_for_each_entry_safe(hem, temp_hem, head, sibling) { 1481 if (hem_list_page_is_in_range(hem, offset)) { 1482 nr = offset - hem->start; 1483 cpu_base = hem->addr + nr * BA_BYTE_LEN; 1484 phy_base = hem->dma_addr + nr * BA_BYTE_LEN; 1485 nr = hem->end + 1 - offset; 1486 break; 1487 } 1488 } 1489 1490 if (mtt_cnt) 1491 *mtt_cnt = nr; 1492 1493 if (phy_addr) 1494 *phy_addr = phy_base; 1495 1496 return cpu_base; 1497 } 1498