1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include "hns_roce_device.h" 36 #include "hns_roce_hem.h" 37 #include "hns_roce_common.h" 38 39 #define DMA_ADDR_T_SHIFT 12 40 #define BT_BA_SHIFT 32 41 42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 43 { 44 int hop_num = 0; 45 46 switch (type) { 47 case HEM_TYPE_QPC: 48 hop_num = hr_dev->caps.qpc_hop_num; 49 break; 50 case HEM_TYPE_MTPT: 51 hop_num = hr_dev->caps.mpt_hop_num; 52 break; 53 case HEM_TYPE_CQC: 54 hop_num = hr_dev->caps.cqc_hop_num; 55 break; 56 case HEM_TYPE_SRQC: 57 hop_num = hr_dev->caps.srqc_hop_num; 58 break; 59 case HEM_TYPE_SCCC: 60 hop_num = hr_dev->caps.sccc_hop_num; 61 break; 62 case HEM_TYPE_QPC_TIMER: 63 hop_num = hr_dev->caps.qpc_timer_hop_num; 64 break; 65 case HEM_TYPE_CQC_TIMER: 66 hop_num = hr_dev->caps.cqc_timer_hop_num; 67 break; 68 case HEM_TYPE_CQE: 69 hop_num = hr_dev->caps.cqe_hop_num; 70 break; 71 case HEM_TYPE_MTT: 72 hop_num = hr_dev->caps.mtt_hop_num; 73 break; 74 case HEM_TYPE_SRQWQE: 75 hop_num = hr_dev->caps.srqwqe_hop_num; 76 break; 77 case HEM_TYPE_IDX: 78 hop_num = hr_dev->caps.idx_hop_num; 79 break; 80 default: 81 return false; 82 } 83 84 return hop_num ? true : false; 85 } 86 87 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, 88 u32 bt_chunk_num, u64 hem_max_num) 89 { 90 u64 check_max_num = start_idx + bt_chunk_num; 91 u64 i; 92 93 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) 94 if (hem[i]) 95 return false; 96 97 return true; 98 } 99 100 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) 101 { 102 int i; 103 104 for (i = 0; i < bt_chunk_num; i++) 105 if (bt[start_idx + i]) 106 return false; 107 108 return true; 109 } 110 111 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 112 { 113 if (check_whether_bt_num_3(table_type, hop_num)) 114 return 3; 115 else if (check_whether_bt_num_2(table_type, hop_num)) 116 return 2; 117 else if (check_whether_bt_num_1(table_type, hop_num)) 118 return 1; 119 else 120 return 0; 121 } 122 123 static int get_hem_table_config(struct hns_roce_dev *hr_dev, 124 struct hns_roce_hem_mhop *mhop, 125 u32 type) 126 { 127 struct device *dev = hr_dev->dev; 128 129 switch (type) { 130 case HEM_TYPE_QPC: 131 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 132 + PAGE_SHIFT); 133 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 134 + PAGE_SHIFT); 135 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 136 mhop->hop_num = hr_dev->caps.qpc_hop_num; 137 break; 138 case HEM_TYPE_MTPT: 139 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 140 + PAGE_SHIFT); 141 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 142 + PAGE_SHIFT); 143 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 144 mhop->hop_num = hr_dev->caps.mpt_hop_num; 145 break; 146 case HEM_TYPE_CQC: 147 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 148 + PAGE_SHIFT); 149 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 150 + PAGE_SHIFT); 151 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 152 mhop->hop_num = hr_dev->caps.cqc_hop_num; 153 break; 154 case HEM_TYPE_SCCC: 155 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 156 + PAGE_SHIFT); 157 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 158 + PAGE_SHIFT); 159 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 160 mhop->hop_num = hr_dev->caps.sccc_hop_num; 161 break; 162 case HEM_TYPE_QPC_TIMER: 163 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 164 + PAGE_SHIFT); 165 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 166 + PAGE_SHIFT); 167 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 168 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 169 break; 170 case HEM_TYPE_CQC_TIMER: 171 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 172 + PAGE_SHIFT); 173 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 174 + PAGE_SHIFT); 175 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 176 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 177 break; 178 case HEM_TYPE_SRQC: 179 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 180 + PAGE_SHIFT); 181 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 182 + PAGE_SHIFT); 183 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 184 mhop->hop_num = hr_dev->caps.srqc_hop_num; 185 break; 186 case HEM_TYPE_MTT: 187 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz 188 + PAGE_SHIFT); 189 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 190 + PAGE_SHIFT); 191 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 192 mhop->hop_num = hr_dev->caps.mtt_hop_num; 193 break; 194 case HEM_TYPE_CQE: 195 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz 196 + PAGE_SHIFT); 197 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 198 + PAGE_SHIFT); 199 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 200 mhop->hop_num = hr_dev->caps.cqe_hop_num; 201 break; 202 case HEM_TYPE_SRQWQE: 203 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz 204 + PAGE_SHIFT); 205 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz 206 + PAGE_SHIFT); 207 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 208 mhop->hop_num = hr_dev->caps.srqwqe_hop_num; 209 break; 210 case HEM_TYPE_IDX: 211 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz 212 + PAGE_SHIFT); 213 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz 214 + PAGE_SHIFT); 215 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 216 mhop->hop_num = hr_dev->caps.idx_hop_num; 217 break; 218 default: 219 dev_err(dev, "Table %d not support multi-hop addressing!\n", 220 type); 221 return -EINVAL; 222 } 223 224 return 0; 225 } 226 227 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 228 struct hns_roce_hem_table *table, unsigned long *obj, 229 struct hns_roce_hem_mhop *mhop) 230 { 231 struct device *dev = hr_dev->dev; 232 u32 chunk_ba_num; 233 u32 table_idx; 234 u32 bt_num; 235 u32 chunk_size; 236 237 if (get_hem_table_config(hr_dev, mhop, table->type)) 238 return -EINVAL; 239 240 if (!obj) 241 return 0; 242 243 /* 244 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 245 * MTT/CQE alloc hem for bt pages. 246 */ 247 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 248 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 249 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 250 mhop->bt_chunk_size; 251 table_idx = (*obj & (table->num_obj - 1)) / 252 (chunk_size / table->obj_size); 253 switch (bt_num) { 254 case 3: 255 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 256 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 257 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 258 break; 259 case 2: 260 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 261 mhop->l0_idx = table_idx / chunk_ba_num; 262 break; 263 case 1: 264 mhop->l0_idx = table_idx; 265 break; 266 default: 267 dev_err(dev, "Table %d not support hop_num = %d!\n", 268 table->type, mhop->hop_num); 269 return -EINVAL; 270 } 271 if (mhop->l0_idx >= mhop->ba_l0_num) 272 mhop->l0_idx %= mhop->ba_l0_num; 273 274 return 0; 275 } 276 277 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 278 int npages, 279 unsigned long hem_alloc_size, 280 gfp_t gfp_mask) 281 { 282 struct hns_roce_hem_chunk *chunk = NULL; 283 struct hns_roce_hem *hem; 284 struct scatterlist *mem; 285 int order; 286 void *buf; 287 288 WARN_ON(gfp_mask & __GFP_HIGHMEM); 289 290 hem = kmalloc(sizeof(*hem), 291 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 292 if (!hem) 293 return NULL; 294 295 hem->refcount = 0; 296 INIT_LIST_HEAD(&hem->chunk_list); 297 298 order = get_order(hem_alloc_size); 299 300 while (npages > 0) { 301 if (!chunk) { 302 chunk = kmalloc(sizeof(*chunk), 303 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 304 if (!chunk) 305 goto fail; 306 307 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 308 chunk->npages = 0; 309 chunk->nsg = 0; 310 memset(chunk->buf, 0, sizeof(chunk->buf)); 311 list_add_tail(&chunk->list, &hem->chunk_list); 312 } 313 314 while (1 << order > npages) 315 --order; 316 317 /* 318 * Alloc memory one time. If failed, don't alloc small block 319 * memory, directly return fail. 320 */ 321 mem = &chunk->mem[chunk->npages]; 322 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 323 &sg_dma_address(mem), gfp_mask); 324 if (!buf) 325 goto fail; 326 327 chunk->buf[chunk->npages] = buf; 328 sg_dma_len(mem) = PAGE_SIZE << order; 329 330 ++chunk->npages; 331 ++chunk->nsg; 332 npages -= 1 << order; 333 } 334 335 return hem; 336 337 fail: 338 hns_roce_free_hem(hr_dev, hem); 339 return NULL; 340 } 341 342 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 343 { 344 struct hns_roce_hem_chunk *chunk, *tmp; 345 int i; 346 347 if (!hem) 348 return; 349 350 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 351 for (i = 0; i < chunk->npages; ++i) 352 dma_free_coherent(hr_dev->dev, 353 sg_dma_len(&chunk->mem[i]), 354 chunk->buf[i], 355 sg_dma_address(&chunk->mem[i])); 356 kfree(chunk); 357 } 358 359 kfree(hem); 360 } 361 362 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, 363 struct hns_roce_hem_table *table, unsigned long obj) 364 { 365 spinlock_t *lock = &hr_dev->bt_cmd_lock; 366 struct device *dev = hr_dev->dev; 367 long end; 368 unsigned long flags; 369 struct hns_roce_hem_iter iter; 370 void __iomem *bt_cmd; 371 __le32 bt_cmd_val[2]; 372 __le32 bt_cmd_h = 0; 373 __le32 bt_cmd_l = 0; 374 u64 bt_ba = 0; 375 int ret = 0; 376 377 /* Find the HEM(Hardware Entry Memory) entry */ 378 unsigned long i = (obj & (table->num_obj - 1)) / 379 (table->table_chunk_size / table->obj_size); 380 381 switch (table->type) { 382 case HEM_TYPE_QPC: 383 case HEM_TYPE_MTPT: 384 case HEM_TYPE_CQC: 385 case HEM_TYPE_SRQC: 386 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 387 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); 388 break; 389 default: 390 return ret; 391 } 392 393 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, 394 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); 395 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); 396 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); 397 398 /* Currently iter only a chunk */ 399 for (hns_roce_hem_first(table->hem[i], &iter); 400 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 401 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; 402 403 spin_lock_irqsave(lock, flags); 404 405 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; 406 407 end = HW_SYNC_TIMEOUT_MSECS; 408 while (end > 0) { 409 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) 410 break; 411 412 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); 413 end -= HW_SYNC_SLEEP_TIME_INTERVAL; 414 } 415 416 if (end <= 0) { 417 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); 418 spin_unlock_irqrestore(lock, flags); 419 return -EBUSY; 420 } 421 422 bt_cmd_l = cpu_to_le32(bt_ba); 423 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, 424 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 425 bt_ba >> BT_BA_SHIFT); 426 427 bt_cmd_val[0] = bt_cmd_l; 428 bt_cmd_val[1] = bt_cmd_h; 429 hns_roce_write64_k(bt_cmd_val, 430 hr_dev->reg_base + ROCEE_BT_CMD_L_REG); 431 spin_unlock_irqrestore(lock, flags); 432 } 433 434 return ret; 435 } 436 437 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 438 struct hns_roce_hem_table *table, 439 unsigned long obj) 440 { 441 struct device *dev = hr_dev->dev; 442 struct hns_roce_hem_mhop mhop; 443 struct hns_roce_hem_iter iter; 444 u32 buf_chunk_size; 445 u32 bt_chunk_size; 446 u32 chunk_ba_num; 447 u32 hop_num; 448 u32 size; 449 u32 bt_num; 450 u64 hem_idx; 451 u64 bt_l1_idx = 0; 452 u64 bt_l0_idx = 0; 453 u64 bt_ba; 454 unsigned long mhop_obj = obj; 455 int bt_l1_allocated = 0; 456 int bt_l0_allocated = 0; 457 int step_idx; 458 int ret; 459 460 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 461 if (ret) 462 return ret; 463 464 buf_chunk_size = mhop.buf_chunk_size; 465 bt_chunk_size = mhop.bt_chunk_size; 466 hop_num = mhop.hop_num; 467 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; 468 469 bt_num = hns_roce_get_bt_num(table->type, hop_num); 470 switch (bt_num) { 471 case 3: 472 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 473 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 474 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 475 bt_l0_idx = mhop.l0_idx; 476 break; 477 case 2: 478 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 479 bt_l0_idx = mhop.l0_idx; 480 break; 481 case 1: 482 hem_idx = mhop.l0_idx; 483 break; 484 default: 485 dev_err(dev, "Table %d not support hop_num = %d!\n", 486 table->type, hop_num); 487 return -EINVAL; 488 } 489 490 if (unlikely(hem_idx >= table->num_hem)) { 491 dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n", 492 table->type, hem_idx, table->num_hem); 493 return -EINVAL; 494 } 495 496 mutex_lock(&table->mutex); 497 498 if (table->hem[hem_idx]) { 499 ++table->hem[hem_idx]->refcount; 500 goto out; 501 } 502 503 /* alloc L1 BA's chunk */ 504 if ((check_whether_bt_num_3(table->type, hop_num) || 505 check_whether_bt_num_2(table->type, hop_num)) && 506 !table->bt_l0[bt_l0_idx]) { 507 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, 508 &(table->bt_l0_dma_addr[bt_l0_idx]), 509 GFP_KERNEL); 510 if (!table->bt_l0[bt_l0_idx]) { 511 ret = -ENOMEM; 512 goto out; 513 } 514 bt_l0_allocated = 1; 515 516 /* set base address to hardware */ 517 if (table->type < HEM_TYPE_MTT) { 518 step_idx = 0; 519 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 520 ret = -ENODEV; 521 dev_err(dev, "set HEM base address to HW failed!\n"); 522 goto err_dma_alloc_l1; 523 } 524 } 525 } 526 527 /* alloc L2 BA's chunk */ 528 if (check_whether_bt_num_3(table->type, hop_num) && 529 !table->bt_l1[bt_l1_idx]) { 530 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, 531 &(table->bt_l1_dma_addr[bt_l1_idx]), 532 GFP_KERNEL); 533 if (!table->bt_l1[bt_l1_idx]) { 534 ret = -ENOMEM; 535 goto err_dma_alloc_l1; 536 } 537 bt_l1_allocated = 1; 538 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = 539 table->bt_l1_dma_addr[bt_l1_idx]; 540 541 /* set base address to hardware */ 542 step_idx = 1; 543 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 544 ret = -ENODEV; 545 dev_err(dev, "set HEM base address to HW failed!\n"); 546 goto err_alloc_hem_buf; 547 } 548 } 549 550 /* 551 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 552 * alloc bt space chunk for MTT/CQE. 553 */ 554 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; 555 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, 556 size >> PAGE_SHIFT, 557 size, 558 (table->lowmem ? GFP_KERNEL : 559 GFP_HIGHUSER) | __GFP_NOWARN); 560 if (!table->hem[hem_idx]) { 561 ret = -ENOMEM; 562 goto err_alloc_hem_buf; 563 } 564 565 hns_roce_hem_first(table->hem[hem_idx], &iter); 566 bt_ba = hns_roce_hem_addr(&iter); 567 568 if (table->type < HEM_TYPE_MTT) { 569 if (hop_num == 2) { 570 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; 571 step_idx = 2; 572 } else if (hop_num == 1) { 573 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 574 step_idx = 1; 575 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 576 step_idx = 0; 577 } else { 578 ret = -EINVAL; 579 goto err_dma_alloc_l1; 580 } 581 582 /* set HEM base address to hardware */ 583 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 584 ret = -ENODEV; 585 dev_err(dev, "set HEM base address to HW failed!\n"); 586 goto err_alloc_hem_buf; 587 } 588 } else if (hop_num == 2) { 589 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 590 } 591 592 ++table->hem[hem_idx]->refcount; 593 goto out; 594 595 err_alloc_hem_buf: 596 if (bt_l1_allocated) { 597 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], 598 table->bt_l1_dma_addr[bt_l1_idx]); 599 table->bt_l1[bt_l1_idx] = NULL; 600 } 601 602 err_dma_alloc_l1: 603 if (bt_l0_allocated) { 604 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], 605 table->bt_l0_dma_addr[bt_l0_idx]); 606 table->bt_l0[bt_l0_idx] = NULL; 607 } 608 609 out: 610 mutex_unlock(&table->mutex); 611 return ret; 612 } 613 614 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 615 struct hns_roce_hem_table *table, unsigned long obj) 616 { 617 struct device *dev = hr_dev->dev; 618 int ret = 0; 619 unsigned long i; 620 621 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 622 return hns_roce_table_mhop_get(hr_dev, table, obj); 623 624 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / 625 table->obj_size); 626 627 mutex_lock(&table->mutex); 628 629 if (table->hem[i]) { 630 ++table->hem[i]->refcount; 631 goto out; 632 } 633 634 table->hem[i] = hns_roce_alloc_hem(hr_dev, 635 table->table_chunk_size >> PAGE_SHIFT, 636 table->table_chunk_size, 637 (table->lowmem ? GFP_KERNEL : 638 GFP_HIGHUSER) | __GFP_NOWARN); 639 if (!table->hem[i]) { 640 ret = -ENOMEM; 641 goto out; 642 } 643 644 /* Set HEM base address(128K/page, pa) to Hardware */ 645 if (hns_roce_set_hem(hr_dev, table, obj)) { 646 hns_roce_free_hem(hr_dev, table->hem[i]); 647 table->hem[i] = NULL; 648 ret = -ENODEV; 649 dev_err(dev, "set HEM base address to HW failed.\n"); 650 goto out; 651 } 652 653 ++table->hem[i]->refcount; 654 out: 655 mutex_unlock(&table->mutex); 656 return ret; 657 } 658 659 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 660 struct hns_roce_hem_table *table, 661 unsigned long obj, 662 int check_refcount) 663 { 664 struct device *dev = hr_dev->dev; 665 struct hns_roce_hem_mhop mhop; 666 unsigned long mhop_obj = obj; 667 u32 bt_chunk_size; 668 u32 chunk_ba_num; 669 u32 hop_num; 670 u32 start_idx; 671 u32 bt_num; 672 u64 hem_idx; 673 u64 bt_l1_idx = 0; 674 int ret; 675 676 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 677 if (ret) 678 return; 679 680 bt_chunk_size = mhop.bt_chunk_size; 681 hop_num = mhop.hop_num; 682 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; 683 684 bt_num = hns_roce_get_bt_num(table->type, hop_num); 685 switch (bt_num) { 686 case 3: 687 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 688 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 689 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 690 break; 691 case 2: 692 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 693 break; 694 case 1: 695 hem_idx = mhop.l0_idx; 696 break; 697 default: 698 dev_err(dev, "Table %d not support hop_num = %d!\n", 699 table->type, hop_num); 700 return; 701 } 702 703 mutex_lock(&table->mutex); 704 705 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { 706 mutex_unlock(&table->mutex); 707 return; 708 } 709 710 if (table->type < HEM_TYPE_MTT && hop_num == 1) { 711 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 712 dev_warn(dev, "Clear HEM base address failed.\n"); 713 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { 714 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) 715 dev_warn(dev, "Clear HEM base address failed.\n"); 716 } else if (table->type < HEM_TYPE_MTT && 717 hop_num == HNS_ROCE_HOP_NUM_0) { 718 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 719 dev_warn(dev, "Clear HEM base address failed.\n"); 720 } 721 722 /* 723 * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 724 * free bt space chunk for MTT/CQE. 725 */ 726 hns_roce_free_hem(hr_dev, table->hem[hem_idx]); 727 table->hem[hem_idx] = NULL; 728 729 if (check_whether_bt_num_2(table->type, hop_num)) { 730 start_idx = mhop.l0_idx * chunk_ba_num; 731 if (hns_roce_check_hem_null(table->hem, start_idx, 732 chunk_ba_num, table->num_hem)) { 733 if (table->type < HEM_TYPE_MTT && 734 hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 735 dev_warn(dev, "Clear HEM base address failed.\n"); 736 737 dma_free_coherent(dev, bt_chunk_size, 738 table->bt_l0[mhop.l0_idx], 739 table->bt_l0_dma_addr[mhop.l0_idx]); 740 table->bt_l0[mhop.l0_idx] = NULL; 741 } 742 } else if (check_whether_bt_num_3(table->type, hop_num)) { 743 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 744 mhop.l1_idx * chunk_ba_num; 745 if (hns_roce_check_hem_null(table->hem, start_idx, 746 chunk_ba_num, table->num_hem)) { 747 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 748 dev_warn(dev, "Clear HEM base address failed.\n"); 749 750 dma_free_coherent(dev, bt_chunk_size, 751 table->bt_l1[bt_l1_idx], 752 table->bt_l1_dma_addr[bt_l1_idx]); 753 table->bt_l1[bt_l1_idx] = NULL; 754 755 start_idx = mhop.l0_idx * chunk_ba_num; 756 if (hns_roce_check_bt_null(table->bt_l1, start_idx, 757 chunk_ba_num)) { 758 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 759 0)) 760 dev_warn(dev, "Clear HEM base address failed.\n"); 761 762 dma_free_coherent(dev, bt_chunk_size, 763 table->bt_l0[mhop.l0_idx], 764 table->bt_l0_dma_addr[mhop.l0_idx]); 765 table->bt_l0[mhop.l0_idx] = NULL; 766 } 767 } 768 } 769 770 mutex_unlock(&table->mutex); 771 } 772 773 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 774 struct hns_roce_hem_table *table, unsigned long obj) 775 { 776 struct device *dev = hr_dev->dev; 777 unsigned long i; 778 779 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 780 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 781 return; 782 } 783 784 i = (obj & (table->num_obj - 1)) / 785 (table->table_chunk_size / table->obj_size); 786 787 mutex_lock(&table->mutex); 788 789 if (--table->hem[i]->refcount == 0) { 790 /* Clear HEM base address */ 791 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 792 dev_warn(dev, "Clear HEM base address failed.\n"); 793 794 hns_roce_free_hem(hr_dev, table->hem[i]); 795 table->hem[i] = NULL; 796 } 797 798 mutex_unlock(&table->mutex); 799 } 800 801 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 802 struct hns_roce_hem_table *table, 803 unsigned long obj, dma_addr_t *dma_handle) 804 { 805 struct hns_roce_hem_chunk *chunk; 806 struct hns_roce_hem_mhop mhop; 807 struct hns_roce_hem *hem; 808 void *addr = NULL; 809 unsigned long mhop_obj = obj; 810 unsigned long obj_per_chunk; 811 unsigned long idx_offset; 812 int offset, dma_offset; 813 int length; 814 int i, j; 815 u32 hem_idx = 0; 816 817 if (!table->lowmem) 818 return NULL; 819 820 mutex_lock(&table->mutex); 821 822 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 823 obj_per_chunk = table->table_chunk_size / table->obj_size; 824 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; 825 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; 826 dma_offset = offset = idx_offset * table->obj_size; 827 } else { 828 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 829 830 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) 831 goto out; 832 /* mtt mhop */ 833 i = mhop.l0_idx; 834 j = mhop.l1_idx; 835 if (mhop.hop_num == 2) 836 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; 837 else if (mhop.hop_num == 1 || 838 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 839 hem_idx = i; 840 841 hem = table->hem[hem_idx]; 842 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % 843 mhop.bt_chunk_size; 844 if (mhop.hop_num == 2) 845 dma_offset = offset = 0; 846 } 847 848 if (!hem) 849 goto out; 850 851 list_for_each_entry(chunk, &hem->chunk_list, list) { 852 for (i = 0; i < chunk->npages; ++i) { 853 length = sg_dma_len(&chunk->mem[i]); 854 if (dma_handle && dma_offset >= 0) { 855 if (length > (u32)dma_offset) 856 *dma_handle = sg_dma_address( 857 &chunk->mem[i]) + dma_offset; 858 dma_offset -= length; 859 } 860 861 if (length > (u32)offset) { 862 addr = chunk->buf[i] + offset; 863 goto out; 864 } 865 offset -= length; 866 } 867 } 868 869 out: 870 mutex_unlock(&table->mutex); 871 return addr; 872 } 873 874 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, 875 struct hns_roce_hem_table *table, 876 unsigned long start, unsigned long end) 877 { 878 struct hns_roce_hem_mhop mhop; 879 unsigned long inc = table->table_chunk_size / table->obj_size; 880 unsigned long i = 0; 881 int ret; 882 883 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 884 ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 885 if (ret) 886 goto fail; 887 inc = mhop.bt_chunk_size / table->obj_size; 888 } 889 890 /* Allocate MTT entry memory according to chunk(128K) */ 891 for (i = start; i <= end; i += inc) { 892 ret = hns_roce_table_get(hr_dev, table, i); 893 if (ret) 894 goto fail; 895 } 896 897 return 0; 898 899 fail: 900 while (i > start) { 901 i -= inc; 902 hns_roce_table_put(hr_dev, table, i); 903 } 904 return ret; 905 } 906 907 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, 908 struct hns_roce_hem_table *table, 909 unsigned long start, unsigned long end) 910 { 911 struct hns_roce_hem_mhop mhop; 912 unsigned long inc = table->table_chunk_size / table->obj_size; 913 unsigned long i; 914 915 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 916 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 917 return; 918 inc = mhop.bt_chunk_size / table->obj_size; 919 } 920 921 for (i = start; i <= end; i += inc) 922 hns_roce_table_put(hr_dev, table, i); 923 } 924 925 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 926 struct hns_roce_hem_table *table, u32 type, 927 unsigned long obj_size, unsigned long nobj, 928 int use_lowmem) 929 { 930 unsigned long obj_per_chunk; 931 unsigned long num_hem; 932 933 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 934 table->table_chunk_size = hr_dev->caps.chunk_sz; 935 obj_per_chunk = table->table_chunk_size / obj_size; 936 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 937 938 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 939 if (!table->hem) 940 return -ENOMEM; 941 } else { 942 struct hns_roce_hem_mhop mhop = {}; 943 unsigned long buf_chunk_size; 944 unsigned long bt_chunk_size; 945 unsigned long bt_chunk_num; 946 unsigned long num_bt_l0 = 0; 947 u32 hop_num; 948 949 if (get_hem_table_config(hr_dev, &mhop, type)) 950 return -EINVAL; 951 952 buf_chunk_size = mhop.buf_chunk_size; 953 bt_chunk_size = mhop.bt_chunk_size; 954 num_bt_l0 = mhop.ba_l0_num; 955 hop_num = mhop.hop_num; 956 957 obj_per_chunk = buf_chunk_size / obj_size; 958 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 959 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; 960 if (type >= HEM_TYPE_MTT) 961 num_bt_l0 = bt_chunk_num; 962 963 table->hem = kcalloc(num_hem, sizeof(*table->hem), 964 GFP_KERNEL); 965 if (!table->hem) 966 goto err_kcalloc_hem_buf; 967 968 if (check_whether_bt_num_3(type, hop_num)) { 969 unsigned long num_bt_l1; 970 971 num_bt_l1 = (num_hem + bt_chunk_num - 1) / 972 bt_chunk_num; 973 table->bt_l1 = kcalloc(num_bt_l1, 974 sizeof(*table->bt_l1), 975 GFP_KERNEL); 976 if (!table->bt_l1) 977 goto err_kcalloc_bt_l1; 978 979 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 980 sizeof(*table->bt_l1_dma_addr), 981 GFP_KERNEL); 982 983 if (!table->bt_l1_dma_addr) 984 goto err_kcalloc_l1_dma; 985 } 986 987 if (check_whether_bt_num_2(type, hop_num) || 988 check_whether_bt_num_3(type, hop_num)) { 989 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 990 GFP_KERNEL); 991 if (!table->bt_l0) 992 goto err_kcalloc_bt_l0; 993 994 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 995 sizeof(*table->bt_l0_dma_addr), 996 GFP_KERNEL); 997 if (!table->bt_l0_dma_addr) 998 goto err_kcalloc_l0_dma; 999 } 1000 } 1001 1002 table->type = type; 1003 table->num_hem = num_hem; 1004 table->num_obj = nobj; 1005 table->obj_size = obj_size; 1006 table->lowmem = use_lowmem; 1007 mutex_init(&table->mutex); 1008 1009 return 0; 1010 1011 err_kcalloc_l0_dma: 1012 kfree(table->bt_l0); 1013 table->bt_l0 = NULL; 1014 1015 err_kcalloc_bt_l0: 1016 kfree(table->bt_l1_dma_addr); 1017 table->bt_l1_dma_addr = NULL; 1018 1019 err_kcalloc_l1_dma: 1020 kfree(table->bt_l1); 1021 table->bt_l1 = NULL; 1022 1023 err_kcalloc_bt_l1: 1024 kfree(table->hem); 1025 table->hem = NULL; 1026 1027 err_kcalloc_hem_buf: 1028 return -ENOMEM; 1029 } 1030 1031 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 1032 struct hns_roce_hem_table *table) 1033 { 1034 struct hns_roce_hem_mhop mhop; 1035 u32 buf_chunk_size; 1036 int i; 1037 u64 obj; 1038 1039 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 1040 return; 1041 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 1042 mhop.bt_chunk_size; 1043 1044 for (i = 0; i < table->num_hem; ++i) { 1045 obj = i * buf_chunk_size / table->obj_size; 1046 if (table->hem[i]) 1047 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 1048 } 1049 1050 kfree(table->hem); 1051 table->hem = NULL; 1052 kfree(table->bt_l1); 1053 table->bt_l1 = NULL; 1054 kfree(table->bt_l1_dma_addr); 1055 table->bt_l1_dma_addr = NULL; 1056 kfree(table->bt_l0); 1057 table->bt_l0 = NULL; 1058 kfree(table->bt_l0_dma_addr); 1059 table->bt_l0_dma_addr = NULL; 1060 } 1061 1062 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 1063 struct hns_roce_hem_table *table) 1064 { 1065 struct device *dev = hr_dev->dev; 1066 unsigned long i; 1067 1068 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 1069 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 1070 return; 1071 } 1072 1073 for (i = 0; i < table->num_hem; ++i) 1074 if (table->hem[i]) { 1075 if (hr_dev->hw->clear_hem(hr_dev, table, 1076 i * table->table_chunk_size / table->obj_size, 0)) 1077 dev_err(dev, "Clear HEM base address failed.\n"); 1078 1079 hns_roce_free_hem(hr_dev, table->hem[i]); 1080 } 1081 1082 kfree(table->hem); 1083 } 1084 1085 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 1086 { 1087 if ((hr_dev->caps.num_idx_segs)) 1088 hns_roce_cleanup_hem_table(hr_dev, 1089 &hr_dev->mr_table.mtt_idx_table); 1090 if (hr_dev->caps.num_srqwqe_segs) 1091 hns_roce_cleanup_hem_table(hr_dev, 1092 &hr_dev->mr_table.mtt_srqwqe_table); 1093 if (hr_dev->caps.srqc_entry_sz) 1094 hns_roce_cleanup_hem_table(hr_dev, 1095 &hr_dev->srq_table.table); 1096 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 1097 if (hr_dev->caps.qpc_timer_entry_sz) 1098 hns_roce_cleanup_hem_table(hr_dev, 1099 &hr_dev->qpc_timer_table); 1100 if (hr_dev->caps.cqc_timer_entry_sz) 1101 hns_roce_cleanup_hem_table(hr_dev, 1102 &hr_dev->cqc_timer_table); 1103 if (hr_dev->caps.sccc_entry_sz) 1104 hns_roce_cleanup_hem_table(hr_dev, 1105 &hr_dev->qp_table.sccc_table); 1106 if (hr_dev->caps.trrl_entry_sz) 1107 hns_roce_cleanup_hem_table(hr_dev, 1108 &hr_dev->qp_table.trrl_table); 1109 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 1110 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 1111 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1112 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 1113 hns_roce_cleanup_hem_table(hr_dev, 1114 &hr_dev->mr_table.mtt_cqe_table); 1115 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); 1116 } 1117 1118 struct roce_hem_item { 1119 struct list_head list; /* link all hems in the same bt level */ 1120 struct list_head sibling; /* link all hems in last hop for mtt */ 1121 void *addr; 1122 dma_addr_t dma_addr; 1123 size_t count; /* max ba numbers */ 1124 int start; /* start buf offset in this hem */ 1125 int end; /* end buf offset in this hem */ 1126 }; 1127 1128 static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, 1129 int start, int end, 1130 int count, bool exist_bt, 1131 int bt_level) 1132 { 1133 struct roce_hem_item *hem; 1134 1135 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 1136 if (!hem) 1137 return NULL; 1138 1139 if (exist_bt) { 1140 hem->addr = dma_alloc_coherent(hr_dev->dev, 1141 count * BA_BYTE_LEN, 1142 &hem->dma_addr, GFP_KERNEL); 1143 if (!hem->addr) { 1144 kfree(hem); 1145 return NULL; 1146 } 1147 } 1148 1149 hem->count = count; 1150 hem->start = start; 1151 hem->end = end; 1152 INIT_LIST_HEAD(&hem->list); 1153 INIT_LIST_HEAD(&hem->sibling); 1154 1155 return hem; 1156 } 1157 1158 static void hem_list_free_item(struct hns_roce_dev *hr_dev, 1159 struct roce_hem_item *hem, bool exist_bt) 1160 { 1161 if (exist_bt) 1162 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, 1163 hem->addr, hem->dma_addr); 1164 kfree(hem); 1165 } 1166 1167 static void hem_list_free_all(struct hns_roce_dev *hr_dev, 1168 struct list_head *head, bool exist_bt) 1169 { 1170 struct roce_hem_item *hem, *temp_hem; 1171 1172 list_for_each_entry_safe(hem, temp_hem, head, list) { 1173 list_del(&hem->list); 1174 hem_list_free_item(hr_dev, hem, exist_bt); 1175 } 1176 } 1177 1178 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, 1179 u64 table_addr) 1180 { 1181 *(u64 *)(base_addr) = table_addr; 1182 } 1183 1184 /* assign L0 table address to hem from root bt */ 1185 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 1186 struct roce_hem_item *hem, void *cpu_addr, 1187 u64 phy_addr) 1188 { 1189 hem->addr = cpu_addr; 1190 hem->dma_addr = (dma_addr_t)phy_addr; 1191 } 1192 1193 static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, 1194 int offset) 1195 { 1196 return (hem->start <= offset && offset <= hem->end); 1197 } 1198 1199 static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1200 int page_offset) 1201 { 1202 struct roce_hem_item *hem, *temp_hem; 1203 struct roce_hem_item *found = NULL; 1204 1205 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1206 if (hem_list_page_is_in_range(hem, page_offset)) { 1207 found = hem; 1208 break; 1209 } 1210 } 1211 1212 return found; 1213 } 1214 1215 static bool hem_list_is_bottom_bt(int hopnum, int bt_level) 1216 { 1217 /* 1218 * hopnum base address table levels 1219 * 0 L0(buf) 1220 * 1 L0 -> buf 1221 * 2 L0 -> L1 -> buf 1222 * 3 L0 -> L1 -> L2 -> buf 1223 */ 1224 return bt_level >= (hopnum ? hopnum - 1 : hopnum); 1225 } 1226 1227 /** 1228 * calc base address entries num 1229 * @hopnum: num of mutihop addressing 1230 * @bt_level: base address table level 1231 * @unit: ba entries per bt page 1232 */ 1233 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) 1234 { 1235 u32 step; 1236 int max; 1237 int i; 1238 1239 if (hopnum <= bt_level) 1240 return 0; 1241 /* 1242 * hopnum bt_level range 1243 * 1 0 unit 1244 * ------------ 1245 * 2 0 unit * unit 1246 * 2 1 unit 1247 * ------------ 1248 * 3 0 unit * unit * unit 1249 * 3 1 unit * unit 1250 * 3 2 unit 1251 */ 1252 step = 1; 1253 max = hopnum - bt_level; 1254 for (i = 0; i < max; i++) 1255 step = step * unit; 1256 1257 return step; 1258 } 1259 1260 /** 1261 * calc the root ba entries which could cover all regions 1262 * @regions: buf region array 1263 * @region_cnt: array size of @regions 1264 * @unit: ba entries per bt page 1265 */ 1266 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, 1267 int region_cnt, int unit) 1268 { 1269 struct hns_roce_buf_region *r; 1270 int total = 0; 1271 int step; 1272 int i; 1273 1274 for (i = 0; i < region_cnt; i++) { 1275 r = (struct hns_roce_buf_region *)®ions[i]; 1276 if (r->hopnum > 1) { 1277 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1278 if (step > 0) 1279 total += (r->count + step - 1) / step; 1280 } else { 1281 total += r->count; 1282 } 1283 } 1284 1285 return total; 1286 } 1287 1288 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, 1289 const struct hns_roce_buf_region *r, int unit, 1290 int offset, struct list_head *mid_bt, 1291 struct list_head *btm_bt) 1292 { 1293 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1294 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1295 struct roce_hem_item *cur, *pre; 1296 const int hopnum = r->hopnum; 1297 int start_aligned; 1298 int distance; 1299 int ret = 0; 1300 int max_ofs; 1301 int level; 1302 u32 step; 1303 int end; 1304 1305 if (hopnum <= 1) 1306 return 0; 1307 1308 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { 1309 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); 1310 return -EINVAL; 1311 } 1312 1313 if (offset < r->offset) { 1314 dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", 1315 offset, r->offset); 1316 return -EINVAL; 1317 } 1318 1319 distance = offset - r->offset; 1320 max_ofs = r->offset + r->count - 1; 1321 for (level = 0; level < hopnum; level++) 1322 INIT_LIST_HEAD(&temp_list[level]); 1323 1324 /* config L1 bt to last bt and link them to corresponding parent */ 1325 for (level = 1; level < hopnum; level++) { 1326 cur = hem_list_search_item(&mid_bt[level], offset); 1327 if (cur) { 1328 hem_ptrs[level] = cur; 1329 continue; 1330 } 1331 1332 step = hem_list_calc_ba_range(hopnum, level, unit); 1333 if (step < 1) { 1334 ret = -EINVAL; 1335 goto err_exit; 1336 } 1337 1338 start_aligned = (distance / step) * step + r->offset; 1339 end = min_t(int, start_aligned + step - 1, max_ofs); 1340 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, 1341 true, level); 1342 if (!cur) { 1343 ret = -ENOMEM; 1344 goto err_exit; 1345 } 1346 hem_ptrs[level] = cur; 1347 list_add(&cur->list, &temp_list[level]); 1348 if (hem_list_is_bottom_bt(hopnum, level)) 1349 list_add(&cur->sibling, &temp_list[0]); 1350 1351 /* link bt to parent bt */ 1352 if (level > 1) { 1353 pre = hem_ptrs[level - 1]; 1354 step = (cur->start - pre->start) / step * BA_BYTE_LEN; 1355 hem_list_link_bt(hr_dev, pre->addr + step, 1356 cur->dma_addr); 1357 } 1358 } 1359 1360 list_splice(&temp_list[0], btm_bt); 1361 for (level = 1; level < hopnum; level++) 1362 list_splice(&temp_list[level], &mid_bt[level]); 1363 1364 return 0; 1365 1366 err_exit: 1367 for (level = 1; level < hopnum; level++) 1368 hem_list_free_all(hr_dev, &temp_list[level], true); 1369 1370 return ret; 1371 } 1372 1373 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1374 struct hns_roce_hem_list *hem_list, int unit, 1375 const struct hns_roce_buf_region *regions, 1376 int region_cnt) 1377 { 1378 struct roce_hem_item *hem, *temp_hem, *root_hem; 1379 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; 1380 const struct hns_roce_buf_region *r; 1381 struct list_head temp_root; 1382 struct list_head temp_btm; 1383 void *cpu_base; 1384 u64 phy_base; 1385 int ret = 0; 1386 int offset; 1387 int total; 1388 int step; 1389 int i; 1390 1391 r = ®ions[0]; 1392 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); 1393 if (root_hem) 1394 return 0; 1395 1396 INIT_LIST_HEAD(&temp_root); 1397 total = r->offset; 1398 /* indicate to last region */ 1399 r = ®ions[region_cnt - 1]; 1400 root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1, 1401 unit, true, 0); 1402 if (!root_hem) 1403 return -ENOMEM; 1404 list_add(&root_hem->list, &temp_root); 1405 1406 hem_list->root_ba = root_hem->dma_addr; 1407 1408 INIT_LIST_HEAD(&temp_btm); 1409 for (i = 0; i < region_cnt; i++) 1410 INIT_LIST_HEAD(&temp_list[i]); 1411 1412 total = 0; 1413 for (i = 0; i < region_cnt && total < unit; i++) { 1414 r = ®ions[i]; 1415 if (!r->count) 1416 continue; 1417 1418 /* all regions's mid[x][0] shared the root_bt's trunk */ 1419 cpu_base = root_hem->addr + total * BA_BYTE_LEN; 1420 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; 1421 1422 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1423 * which's address share to all regions. 1424 */ 1425 if (hem_list_is_bottom_bt(r->hopnum, 0)) { 1426 hem = hem_list_alloc_item(hr_dev, r->offset, 1427 r->offset + r->count - 1, 1428 r->count, false, 0); 1429 if (!hem) { 1430 ret = -ENOMEM; 1431 goto err_exit; 1432 } 1433 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1434 list_add(&hem->list, &temp_list[i]); 1435 list_add(&hem->sibling, &temp_btm); 1436 total += r->count; 1437 } else { 1438 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1439 if (step < 1) { 1440 ret = -EINVAL; 1441 goto err_exit; 1442 } 1443 /* if exist mid bt, link L1 to L0 */ 1444 list_for_each_entry_safe(hem, temp_hem, 1445 &hem_list->mid_bt[i][1], list) { 1446 offset = hem->start / step * BA_BYTE_LEN; 1447 hem_list_link_bt(hr_dev, cpu_base + offset, 1448 hem->dma_addr); 1449 total++; 1450 } 1451 } 1452 } 1453 1454 list_splice(&temp_btm, &hem_list->btm_bt); 1455 list_splice(&temp_root, &hem_list->root_bt); 1456 for (i = 0; i < region_cnt; i++) 1457 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); 1458 1459 return 0; 1460 1461 err_exit: 1462 for (i = 0; i < region_cnt; i++) 1463 hem_list_free_all(hr_dev, &temp_list[i], false); 1464 1465 hem_list_free_all(hr_dev, &temp_root, true); 1466 1467 return ret; 1468 } 1469 1470 /* construct the base address table and link them by address hop config */ 1471 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, 1472 struct hns_roce_hem_list *hem_list, 1473 const struct hns_roce_buf_region *regions, 1474 int region_cnt) 1475 { 1476 const struct hns_roce_buf_region *r; 1477 int ofs, end; 1478 int ret = 0; 1479 int unit; 1480 int i; 1481 1482 if (region_cnt > HNS_ROCE_MAX_BT_REGION) { 1483 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", 1484 region_cnt); 1485 return -EINVAL; 1486 } 1487 1488 unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN; 1489 for (i = 0; i < region_cnt; i++) { 1490 r = ®ions[i]; 1491 if (!r->count) 1492 continue; 1493 1494 end = r->offset + r->count; 1495 for (ofs = r->offset; ofs < end; ofs += unit) { 1496 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, 1497 hem_list->mid_bt[i], 1498 &hem_list->btm_bt); 1499 if (ret) { 1500 dev_err(hr_dev->dev, 1501 "alloc hem trunk fail ret=%d!\n", ret); 1502 goto err_alloc; 1503 } 1504 } 1505 } 1506 1507 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, 1508 region_cnt); 1509 if (ret) 1510 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); 1511 else 1512 return 0; 1513 1514 err_alloc: 1515 hns_roce_hem_list_release(hr_dev, hem_list); 1516 1517 return ret; 1518 } 1519 1520 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, 1521 struct hns_roce_hem_list *hem_list) 1522 { 1523 int i, j; 1524 1525 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1526 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1527 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], 1528 j != 0); 1529 1530 hem_list_free_all(hr_dev, &hem_list->root_bt, true); 1531 INIT_LIST_HEAD(&hem_list->btm_bt); 1532 hem_list->root_ba = 0; 1533 } 1534 1535 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, 1536 int bt_page_order) 1537 { 1538 int i, j; 1539 1540 INIT_LIST_HEAD(&hem_list->root_bt); 1541 INIT_LIST_HEAD(&hem_list->btm_bt); 1542 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1543 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1544 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); 1545 1546 hem_list->bt_pg_shift = bt_page_order; 1547 } 1548 1549 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, 1550 struct hns_roce_hem_list *hem_list, 1551 int offset, int *mtt_cnt, u64 *phy_addr) 1552 { 1553 struct list_head *head = &hem_list->btm_bt; 1554 struct roce_hem_item *hem, *temp_hem; 1555 void *cpu_base = NULL; 1556 u64 phy_base = 0; 1557 int nr = 0; 1558 1559 list_for_each_entry_safe(hem, temp_hem, head, sibling) { 1560 if (hem_list_page_is_in_range(hem, offset)) { 1561 nr = offset - hem->start; 1562 cpu_base = hem->addr + nr * BA_BYTE_LEN; 1563 phy_base = hem->dma_addr + nr * BA_BYTE_LEN; 1564 nr = hem->end + 1 - offset; 1565 break; 1566 } 1567 } 1568 1569 if (mtt_cnt) 1570 *mtt_cnt = nr; 1571 1572 if (phy_addr) 1573 *phy_addr = phy_base; 1574 1575 return cpu_base; 1576 } 1577