1 /* 2 * linux/drivers/net/ehea/ehea_qmr.c 3 * 4 * eHEA ethernet device driver for IBM eServer System p 5 * 6 * (C) Copyright IBM Corp. 2006 7 * 8 * Authors: 9 * Christoph Raisch <raisch@de.ibm.com> 10 * Jan-Bernd Themann <themann@de.ibm.com> 11 * Thomas Klein <tklein@de.ibm.com> 12 * 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License as published by 16 * the Free Software Foundation; either version 2, or (at your option) 17 * any later version. 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 * 24 * You should have received a copy of the GNU General Public License 25 * along with this program; if not, write to the Free Software 26 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 27 */ 28 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 30 31 #include <linux/mm.h> 32 #include <linux/slab.h> 33 #include "ehea.h" 34 #include "ehea_phyp.h" 35 #include "ehea_qmr.h" 36 37 struct ehea_bmap *ehea_bmap = NULL; 38 39 40 41 static void *hw_qpageit_get_inc(struct hw_queue *queue) 42 { 43 void *retvalue = hw_qeit_get(queue); 44 45 queue->current_q_offset += queue->pagesize; 46 if (queue->current_q_offset > queue->queue_length) { 47 queue->current_q_offset -= queue->pagesize; 48 retvalue = NULL; 49 } else if (((u64) retvalue) & (EHEA_PAGESIZE-1)) { 50 pr_err("not on pageboundary\n"); 51 retvalue = NULL; 52 } 53 return retvalue; 54 } 55 56 static int hw_queue_ctor(struct hw_queue *queue, const u32 nr_of_pages, 57 const u32 pagesize, const u32 qe_size) 58 { 59 int pages_per_kpage = PAGE_SIZE / pagesize; 60 int i, k; 61 62 if ((pagesize > PAGE_SIZE) || (!pages_per_kpage)) { 63 pr_err("pagesize conflict! kernel pagesize=%d, ehea pagesize=%d\n", 64 (int)PAGE_SIZE, (int)pagesize); 65 return -EINVAL; 66 } 67 68 queue->queue_length = nr_of_pages * pagesize; 69 queue->queue_pages = kmalloc(nr_of_pages * sizeof(void *), GFP_KERNEL); 70 if (!queue->queue_pages) { 71 pr_err("no mem for queue_pages\n"); 72 return -ENOMEM; 73 } 74 75 /* 76 * allocate pages for queue: 77 * outer loop allocates whole kernel pages (page aligned) and 78 * inner loop divides a kernel page into smaller hea queue pages 79 */ 80 i = 0; 81 while (i < nr_of_pages) { 82 u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL); 83 if (!kpage) 84 goto out_nomem; 85 for (k = 0; k < pages_per_kpage && i < nr_of_pages; k++) { 86 (queue->queue_pages)[i] = (struct ehea_page *)kpage; 87 kpage += pagesize; 88 i++; 89 } 90 } 91 92 queue->current_q_offset = 0; 93 queue->qe_size = qe_size; 94 queue->pagesize = pagesize; 95 queue->toggle_state = 1; 96 97 return 0; 98 out_nomem: 99 for (i = 0; i < nr_of_pages; i += pages_per_kpage) { 100 if (!(queue->queue_pages)[i]) 101 break; 102 free_page((unsigned long)(queue->queue_pages)[i]); 103 } 104 return -ENOMEM; 105 } 106 107 static void hw_queue_dtor(struct hw_queue *queue) 108 { 109 int pages_per_kpage = PAGE_SIZE / queue->pagesize; 110 int i, nr_pages; 111 112 if (!queue || !queue->queue_pages) 113 return; 114 115 nr_pages = queue->queue_length / queue->pagesize; 116 117 for (i = 0; i < nr_pages; i += pages_per_kpage) 118 free_page((unsigned long)(queue->queue_pages)[i]); 119 120 kfree(queue->queue_pages); 121 } 122 123 struct ehea_cq *ehea_create_cq(struct ehea_adapter *adapter, 124 int nr_of_cqe, u64 eq_handle, u32 cq_token) 125 { 126 struct ehea_cq *cq; 127 struct h_epa epa; 128 u64 *cq_handle_ref, hret, rpage; 129 u32 act_nr_of_entries, act_pages, counter; 130 int ret; 131 void *vpage; 132 133 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 134 if (!cq) { 135 pr_err("no mem for cq\n"); 136 goto out_nomem; 137 } 138 139 cq->attr.max_nr_of_cqes = nr_of_cqe; 140 cq->attr.cq_token = cq_token; 141 cq->attr.eq_handle = eq_handle; 142 143 cq->adapter = adapter; 144 145 cq_handle_ref = &cq->fw_handle; 146 act_nr_of_entries = 0; 147 act_pages = 0; 148 149 hret = ehea_h_alloc_resource_cq(adapter->handle, &cq->attr, 150 &cq->fw_handle, &cq->epas); 151 if (hret != H_SUCCESS) { 152 pr_err("alloc_resource_cq failed\n"); 153 goto out_freemem; 154 } 155 156 ret = hw_queue_ctor(&cq->hw_queue, cq->attr.nr_pages, 157 EHEA_PAGESIZE, sizeof(struct ehea_cqe)); 158 if (ret) 159 goto out_freeres; 160 161 for (counter = 0; counter < cq->attr.nr_pages; counter++) { 162 vpage = hw_qpageit_get_inc(&cq->hw_queue); 163 if (!vpage) { 164 pr_err("hw_qpageit_get_inc failed\n"); 165 goto out_kill_hwq; 166 } 167 168 rpage = virt_to_abs(vpage); 169 hret = ehea_h_register_rpage(adapter->handle, 170 0, EHEA_CQ_REGISTER_ORIG, 171 cq->fw_handle, rpage, 1); 172 if (hret < H_SUCCESS) { 173 pr_err("register_rpage_cq failed ehea_cq=%p hret=%llx counter=%i act_pages=%i\n", 174 cq, hret, counter, cq->attr.nr_pages); 175 goto out_kill_hwq; 176 } 177 178 if (counter == (cq->attr.nr_pages - 1)) { 179 vpage = hw_qpageit_get_inc(&cq->hw_queue); 180 181 if ((hret != H_SUCCESS) || (vpage)) { 182 pr_err("registration of pages not complete hret=%llx\n", 183 hret); 184 goto out_kill_hwq; 185 } 186 } else { 187 if (hret != H_PAGE_REGISTERED) { 188 pr_err("CQ: registration of page failed hret=%llx\n", 189 hret); 190 goto out_kill_hwq; 191 } 192 } 193 } 194 195 hw_qeit_reset(&cq->hw_queue); 196 epa = cq->epas.kernel; 197 ehea_reset_cq_ep(cq); 198 ehea_reset_cq_n1(cq); 199 200 return cq; 201 202 out_kill_hwq: 203 hw_queue_dtor(&cq->hw_queue); 204 205 out_freeres: 206 ehea_h_free_resource(adapter->handle, cq->fw_handle, FORCE_FREE); 207 208 out_freemem: 209 kfree(cq); 210 211 out_nomem: 212 return NULL; 213 } 214 215 u64 ehea_destroy_cq_res(struct ehea_cq *cq, u64 force) 216 { 217 u64 hret; 218 u64 adapter_handle = cq->adapter->handle; 219 220 /* deregister all previous registered pages */ 221 hret = ehea_h_free_resource(adapter_handle, cq->fw_handle, force); 222 if (hret != H_SUCCESS) 223 return hret; 224 225 hw_queue_dtor(&cq->hw_queue); 226 kfree(cq); 227 228 return hret; 229 } 230 231 int ehea_destroy_cq(struct ehea_cq *cq) 232 { 233 u64 hret, aer, aerr; 234 if (!cq) 235 return 0; 236 237 hcp_epas_dtor(&cq->epas); 238 hret = ehea_destroy_cq_res(cq, NORMAL_FREE); 239 if (hret == H_R_STATE) { 240 ehea_error_data(cq->adapter, cq->fw_handle, &aer, &aerr); 241 hret = ehea_destroy_cq_res(cq, FORCE_FREE); 242 } 243 244 if (hret != H_SUCCESS) { 245 pr_err("destroy CQ failed\n"); 246 return -EIO; 247 } 248 249 return 0; 250 } 251 252 struct ehea_eq *ehea_create_eq(struct ehea_adapter *adapter, 253 const enum ehea_eq_type type, 254 const u32 max_nr_of_eqes, const u8 eqe_gen) 255 { 256 int ret, i; 257 u64 hret, rpage; 258 void *vpage; 259 struct ehea_eq *eq; 260 261 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 262 if (!eq) { 263 pr_err("no mem for eq\n"); 264 return NULL; 265 } 266 267 eq->adapter = adapter; 268 eq->attr.type = type; 269 eq->attr.max_nr_of_eqes = max_nr_of_eqes; 270 eq->attr.eqe_gen = eqe_gen; 271 spin_lock_init(&eq->spinlock); 272 273 hret = ehea_h_alloc_resource_eq(adapter->handle, 274 &eq->attr, &eq->fw_handle); 275 if (hret != H_SUCCESS) { 276 pr_err("alloc_resource_eq failed\n"); 277 goto out_freemem; 278 } 279 280 ret = hw_queue_ctor(&eq->hw_queue, eq->attr.nr_pages, 281 EHEA_PAGESIZE, sizeof(struct ehea_eqe)); 282 if (ret) { 283 pr_err("can't allocate eq pages\n"); 284 goto out_freeres; 285 } 286 287 for (i = 0; i < eq->attr.nr_pages; i++) { 288 vpage = hw_qpageit_get_inc(&eq->hw_queue); 289 if (!vpage) { 290 pr_err("hw_qpageit_get_inc failed\n"); 291 hret = H_RESOURCE; 292 goto out_kill_hwq; 293 } 294 295 rpage = virt_to_abs(vpage); 296 297 hret = ehea_h_register_rpage(adapter->handle, 0, 298 EHEA_EQ_REGISTER_ORIG, 299 eq->fw_handle, rpage, 1); 300 301 if (i == (eq->attr.nr_pages - 1)) { 302 /* last page */ 303 vpage = hw_qpageit_get_inc(&eq->hw_queue); 304 if ((hret != H_SUCCESS) || (vpage)) 305 goto out_kill_hwq; 306 307 } else { 308 if (hret != H_PAGE_REGISTERED) 309 goto out_kill_hwq; 310 311 } 312 } 313 314 hw_qeit_reset(&eq->hw_queue); 315 return eq; 316 317 out_kill_hwq: 318 hw_queue_dtor(&eq->hw_queue); 319 320 out_freeres: 321 ehea_h_free_resource(adapter->handle, eq->fw_handle, FORCE_FREE); 322 323 out_freemem: 324 kfree(eq); 325 return NULL; 326 } 327 328 struct ehea_eqe *ehea_poll_eq(struct ehea_eq *eq) 329 { 330 struct ehea_eqe *eqe; 331 unsigned long flags; 332 333 spin_lock_irqsave(&eq->spinlock, flags); 334 eqe = hw_eqit_eq_get_inc_valid(&eq->hw_queue); 335 spin_unlock_irqrestore(&eq->spinlock, flags); 336 337 return eqe; 338 } 339 340 u64 ehea_destroy_eq_res(struct ehea_eq *eq, u64 force) 341 { 342 u64 hret; 343 unsigned long flags; 344 345 spin_lock_irqsave(&eq->spinlock, flags); 346 347 hret = ehea_h_free_resource(eq->adapter->handle, eq->fw_handle, force); 348 spin_unlock_irqrestore(&eq->spinlock, flags); 349 350 if (hret != H_SUCCESS) 351 return hret; 352 353 hw_queue_dtor(&eq->hw_queue); 354 kfree(eq); 355 356 return hret; 357 } 358 359 int ehea_destroy_eq(struct ehea_eq *eq) 360 { 361 u64 hret, aer, aerr; 362 if (!eq) 363 return 0; 364 365 hcp_epas_dtor(&eq->epas); 366 367 hret = ehea_destroy_eq_res(eq, NORMAL_FREE); 368 if (hret == H_R_STATE) { 369 ehea_error_data(eq->adapter, eq->fw_handle, &aer, &aerr); 370 hret = ehea_destroy_eq_res(eq, FORCE_FREE); 371 } 372 373 if (hret != H_SUCCESS) { 374 pr_err("destroy EQ failed\n"); 375 return -EIO; 376 } 377 378 return 0; 379 } 380 381 /** 382 * allocates memory for a queue and registers pages in phyp 383 */ 384 int ehea_qp_alloc_register(struct ehea_qp *qp, struct hw_queue *hw_queue, 385 int nr_pages, int wqe_size, int act_nr_sges, 386 struct ehea_adapter *adapter, int h_call_q_selector) 387 { 388 u64 hret, rpage; 389 int ret, cnt; 390 void *vpage; 391 392 ret = hw_queue_ctor(hw_queue, nr_pages, EHEA_PAGESIZE, wqe_size); 393 if (ret) 394 return ret; 395 396 for (cnt = 0; cnt < nr_pages; cnt++) { 397 vpage = hw_qpageit_get_inc(hw_queue); 398 if (!vpage) { 399 pr_err("hw_qpageit_get_inc failed\n"); 400 goto out_kill_hwq; 401 } 402 rpage = virt_to_abs(vpage); 403 hret = ehea_h_register_rpage(adapter->handle, 404 0, h_call_q_selector, 405 qp->fw_handle, rpage, 1); 406 if (hret < H_SUCCESS) { 407 pr_err("register_rpage_qp failed\n"); 408 goto out_kill_hwq; 409 } 410 } 411 hw_qeit_reset(hw_queue); 412 return 0; 413 414 out_kill_hwq: 415 hw_queue_dtor(hw_queue); 416 return -EIO; 417 } 418 419 static inline u32 map_wqe_size(u8 wqe_enc_size) 420 { 421 return 128 << wqe_enc_size; 422 } 423 424 struct ehea_qp *ehea_create_qp(struct ehea_adapter *adapter, 425 u32 pd, struct ehea_qp_init_attr *init_attr) 426 { 427 int ret; 428 u64 hret; 429 struct ehea_qp *qp; 430 u32 wqe_size_in_bytes_sq, wqe_size_in_bytes_rq1; 431 u32 wqe_size_in_bytes_rq2, wqe_size_in_bytes_rq3; 432 433 434 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 435 if (!qp) { 436 pr_err("no mem for qp\n"); 437 return NULL; 438 } 439 440 qp->adapter = adapter; 441 442 hret = ehea_h_alloc_resource_qp(adapter->handle, init_attr, pd, 443 &qp->fw_handle, &qp->epas); 444 if (hret != H_SUCCESS) { 445 pr_err("ehea_h_alloc_resource_qp failed\n"); 446 goto out_freemem; 447 } 448 449 wqe_size_in_bytes_sq = map_wqe_size(init_attr->act_wqe_size_enc_sq); 450 wqe_size_in_bytes_rq1 = map_wqe_size(init_attr->act_wqe_size_enc_rq1); 451 wqe_size_in_bytes_rq2 = map_wqe_size(init_attr->act_wqe_size_enc_rq2); 452 wqe_size_in_bytes_rq3 = map_wqe_size(init_attr->act_wqe_size_enc_rq3); 453 454 ret = ehea_qp_alloc_register(qp, &qp->hw_squeue, init_attr->nr_sq_pages, 455 wqe_size_in_bytes_sq, 456 init_attr->act_wqe_size_enc_sq, adapter, 457 0); 458 if (ret) { 459 pr_err("can't register for sq ret=%x\n", ret); 460 goto out_freeres; 461 } 462 463 ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue1, 464 init_attr->nr_rq1_pages, 465 wqe_size_in_bytes_rq1, 466 init_attr->act_wqe_size_enc_rq1, 467 adapter, 1); 468 if (ret) { 469 pr_err("can't register for rq1 ret=%x\n", ret); 470 goto out_kill_hwsq; 471 } 472 473 if (init_attr->rq_count > 1) { 474 ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue2, 475 init_attr->nr_rq2_pages, 476 wqe_size_in_bytes_rq2, 477 init_attr->act_wqe_size_enc_rq2, 478 adapter, 2); 479 if (ret) { 480 pr_err("can't register for rq2 ret=%x\n", ret); 481 goto out_kill_hwr1q; 482 } 483 } 484 485 if (init_attr->rq_count > 2) { 486 ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue3, 487 init_attr->nr_rq3_pages, 488 wqe_size_in_bytes_rq3, 489 init_attr->act_wqe_size_enc_rq3, 490 adapter, 3); 491 if (ret) { 492 pr_err("can't register for rq3 ret=%x\n", ret); 493 goto out_kill_hwr2q; 494 } 495 } 496 497 qp->init_attr = *init_attr; 498 499 return qp; 500 501 out_kill_hwr2q: 502 hw_queue_dtor(&qp->hw_rqueue2); 503 504 out_kill_hwr1q: 505 hw_queue_dtor(&qp->hw_rqueue1); 506 507 out_kill_hwsq: 508 hw_queue_dtor(&qp->hw_squeue); 509 510 out_freeres: 511 ehea_h_disable_and_get_hea(adapter->handle, qp->fw_handle); 512 ehea_h_free_resource(adapter->handle, qp->fw_handle, FORCE_FREE); 513 514 out_freemem: 515 kfree(qp); 516 return NULL; 517 } 518 519 u64 ehea_destroy_qp_res(struct ehea_qp *qp, u64 force) 520 { 521 u64 hret; 522 struct ehea_qp_init_attr *qp_attr = &qp->init_attr; 523 524 525 ehea_h_disable_and_get_hea(qp->adapter->handle, qp->fw_handle); 526 hret = ehea_h_free_resource(qp->adapter->handle, qp->fw_handle, force); 527 if (hret != H_SUCCESS) 528 return hret; 529 530 hw_queue_dtor(&qp->hw_squeue); 531 hw_queue_dtor(&qp->hw_rqueue1); 532 533 if (qp_attr->rq_count > 1) 534 hw_queue_dtor(&qp->hw_rqueue2); 535 if (qp_attr->rq_count > 2) 536 hw_queue_dtor(&qp->hw_rqueue3); 537 kfree(qp); 538 539 return hret; 540 } 541 542 int ehea_destroy_qp(struct ehea_qp *qp) 543 { 544 u64 hret, aer, aerr; 545 if (!qp) 546 return 0; 547 548 hcp_epas_dtor(&qp->epas); 549 550 hret = ehea_destroy_qp_res(qp, NORMAL_FREE); 551 if (hret == H_R_STATE) { 552 ehea_error_data(qp->adapter, qp->fw_handle, &aer, &aerr); 553 hret = ehea_destroy_qp_res(qp, FORCE_FREE); 554 } 555 556 if (hret != H_SUCCESS) { 557 pr_err("destroy QP failed\n"); 558 return -EIO; 559 } 560 561 return 0; 562 } 563 564 static inline int ehea_calc_index(unsigned long i, unsigned long s) 565 { 566 return (i >> s) & EHEA_INDEX_MASK; 567 } 568 569 static inline int ehea_init_top_bmap(struct ehea_top_bmap *ehea_top_bmap, 570 int dir) 571 { 572 if (!ehea_top_bmap->dir[dir]) { 573 ehea_top_bmap->dir[dir] = 574 kzalloc(sizeof(struct ehea_dir_bmap), GFP_KERNEL); 575 if (!ehea_top_bmap->dir[dir]) 576 return -ENOMEM; 577 } 578 return 0; 579 } 580 581 static inline int ehea_init_bmap(struct ehea_bmap *ehea_bmap, int top, int dir) 582 { 583 if (!ehea_bmap->top[top]) { 584 ehea_bmap->top[top] = 585 kzalloc(sizeof(struct ehea_top_bmap), GFP_KERNEL); 586 if (!ehea_bmap->top[top]) 587 return -ENOMEM; 588 } 589 return ehea_init_top_bmap(ehea_bmap->top[top], dir); 590 } 591 592 static DEFINE_MUTEX(ehea_busmap_mutex); 593 static unsigned long ehea_mr_len; 594 595 #define EHEA_BUSMAP_ADD_SECT 1 596 #define EHEA_BUSMAP_REM_SECT 0 597 598 static void ehea_rebuild_busmap(void) 599 { 600 u64 vaddr = EHEA_BUSMAP_START; 601 int top, dir, idx; 602 603 for (top = 0; top < EHEA_MAP_ENTRIES; top++) { 604 struct ehea_top_bmap *ehea_top; 605 int valid_dir_entries = 0; 606 607 if (!ehea_bmap->top[top]) 608 continue; 609 ehea_top = ehea_bmap->top[top]; 610 for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { 611 struct ehea_dir_bmap *ehea_dir; 612 int valid_entries = 0; 613 614 if (!ehea_top->dir[dir]) 615 continue; 616 valid_dir_entries++; 617 ehea_dir = ehea_top->dir[dir]; 618 for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { 619 if (!ehea_dir->ent[idx]) 620 continue; 621 valid_entries++; 622 ehea_dir->ent[idx] = vaddr; 623 vaddr += EHEA_SECTSIZE; 624 } 625 if (!valid_entries) { 626 ehea_top->dir[dir] = NULL; 627 kfree(ehea_dir); 628 } 629 } 630 if (!valid_dir_entries) { 631 ehea_bmap->top[top] = NULL; 632 kfree(ehea_top); 633 } 634 } 635 } 636 637 static int ehea_update_busmap(unsigned long pfn, unsigned long nr_pages, int add) 638 { 639 unsigned long i, start_section, end_section; 640 641 if (!nr_pages) 642 return 0; 643 644 if (!ehea_bmap) { 645 ehea_bmap = kzalloc(sizeof(struct ehea_bmap), GFP_KERNEL); 646 if (!ehea_bmap) 647 return -ENOMEM; 648 } 649 650 start_section = (pfn * PAGE_SIZE) / EHEA_SECTSIZE; 651 end_section = start_section + ((nr_pages * PAGE_SIZE) / EHEA_SECTSIZE); 652 /* Mark entries as valid or invalid only; address is assigned later */ 653 for (i = start_section; i < end_section; i++) { 654 u64 flag; 655 int top = ehea_calc_index(i, EHEA_TOP_INDEX_SHIFT); 656 int dir = ehea_calc_index(i, EHEA_DIR_INDEX_SHIFT); 657 int idx = i & EHEA_INDEX_MASK; 658 659 if (add) { 660 int ret = ehea_init_bmap(ehea_bmap, top, dir); 661 if (ret) 662 return ret; 663 flag = 1; /* valid */ 664 ehea_mr_len += EHEA_SECTSIZE; 665 } else { 666 if (!ehea_bmap->top[top]) 667 continue; 668 if (!ehea_bmap->top[top]->dir[dir]) 669 continue; 670 flag = 0; /* invalid */ 671 ehea_mr_len -= EHEA_SECTSIZE; 672 } 673 674 ehea_bmap->top[top]->dir[dir]->ent[idx] = flag; 675 } 676 ehea_rebuild_busmap(); /* Assign contiguous addresses for mr */ 677 return 0; 678 } 679 680 int ehea_add_sect_bmap(unsigned long pfn, unsigned long nr_pages) 681 { 682 int ret; 683 684 mutex_lock(&ehea_busmap_mutex); 685 ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); 686 mutex_unlock(&ehea_busmap_mutex); 687 return ret; 688 } 689 690 int ehea_rem_sect_bmap(unsigned long pfn, unsigned long nr_pages) 691 { 692 int ret; 693 694 mutex_lock(&ehea_busmap_mutex); 695 ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_REM_SECT); 696 mutex_unlock(&ehea_busmap_mutex); 697 return ret; 698 } 699 700 static int ehea_is_hugepage(unsigned long pfn) 701 { 702 int page_order; 703 704 if (pfn & EHEA_HUGEPAGE_PFN_MASK) 705 return 0; 706 707 page_order = compound_order(pfn_to_page(pfn)); 708 if (page_order + PAGE_SHIFT != EHEA_HUGEPAGESHIFT) 709 return 0; 710 711 return 1; 712 } 713 714 static int ehea_create_busmap_callback(unsigned long initial_pfn, 715 unsigned long total_nr_pages, void *arg) 716 { 717 int ret; 718 unsigned long pfn, start_pfn, end_pfn, nr_pages; 719 720 if ((total_nr_pages * PAGE_SIZE) < EHEA_HUGEPAGE_SIZE) 721 return ehea_update_busmap(initial_pfn, total_nr_pages, 722 EHEA_BUSMAP_ADD_SECT); 723 724 /* Given chunk is >= 16GB -> check for hugepages */ 725 start_pfn = initial_pfn; 726 end_pfn = initial_pfn + total_nr_pages; 727 pfn = start_pfn; 728 729 while (pfn < end_pfn) { 730 if (ehea_is_hugepage(pfn)) { 731 /* Add mem found in front of the hugepage */ 732 nr_pages = pfn - start_pfn; 733 ret = ehea_update_busmap(start_pfn, nr_pages, 734 EHEA_BUSMAP_ADD_SECT); 735 if (ret) 736 return ret; 737 738 /* Skip the hugepage */ 739 pfn += (EHEA_HUGEPAGE_SIZE / PAGE_SIZE); 740 start_pfn = pfn; 741 } else 742 pfn += (EHEA_SECTSIZE / PAGE_SIZE); 743 } 744 745 /* Add mem found behind the hugepage(s) */ 746 nr_pages = pfn - start_pfn; 747 return ehea_update_busmap(start_pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); 748 } 749 750 int ehea_create_busmap(void) 751 { 752 int ret; 753 754 mutex_lock(&ehea_busmap_mutex); 755 ehea_mr_len = 0; 756 ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL, 757 ehea_create_busmap_callback); 758 mutex_unlock(&ehea_busmap_mutex); 759 return ret; 760 } 761 762 void ehea_destroy_busmap(void) 763 { 764 int top, dir; 765 mutex_lock(&ehea_busmap_mutex); 766 if (!ehea_bmap) 767 goto out_destroy; 768 769 for (top = 0; top < EHEA_MAP_ENTRIES; top++) { 770 if (!ehea_bmap->top[top]) 771 continue; 772 773 for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { 774 if (!ehea_bmap->top[top]->dir[dir]) 775 continue; 776 777 kfree(ehea_bmap->top[top]->dir[dir]); 778 } 779 780 kfree(ehea_bmap->top[top]); 781 } 782 783 kfree(ehea_bmap); 784 ehea_bmap = NULL; 785 out_destroy: 786 mutex_unlock(&ehea_busmap_mutex); 787 } 788 789 u64 ehea_map_vaddr(void *caddr) 790 { 791 int top, dir, idx; 792 unsigned long index, offset; 793 794 if (!ehea_bmap) 795 return EHEA_INVAL_ADDR; 796 797 index = virt_to_abs(caddr) >> SECTION_SIZE_BITS; 798 top = (index >> EHEA_TOP_INDEX_SHIFT) & EHEA_INDEX_MASK; 799 if (!ehea_bmap->top[top]) 800 return EHEA_INVAL_ADDR; 801 802 dir = (index >> EHEA_DIR_INDEX_SHIFT) & EHEA_INDEX_MASK; 803 if (!ehea_bmap->top[top]->dir[dir]) 804 return EHEA_INVAL_ADDR; 805 806 idx = index & EHEA_INDEX_MASK; 807 if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) 808 return EHEA_INVAL_ADDR; 809 810 offset = (unsigned long)caddr & (EHEA_SECTSIZE - 1); 811 return ehea_bmap->top[top]->dir[dir]->ent[idx] | offset; 812 } 813 814 static inline void *ehea_calc_sectbase(int top, int dir, int idx) 815 { 816 unsigned long ret = idx; 817 ret |= dir << EHEA_DIR_INDEX_SHIFT; 818 ret |= top << EHEA_TOP_INDEX_SHIFT; 819 return abs_to_virt(ret << SECTION_SIZE_BITS); 820 } 821 822 static u64 ehea_reg_mr_section(int top, int dir, int idx, u64 *pt, 823 struct ehea_adapter *adapter, 824 struct ehea_mr *mr) 825 { 826 void *pg; 827 u64 j, m, hret; 828 unsigned long k = 0; 829 u64 pt_abs = virt_to_abs(pt); 830 831 void *sectbase = ehea_calc_sectbase(top, dir, idx); 832 833 for (j = 0; j < (EHEA_PAGES_PER_SECTION / EHEA_MAX_RPAGE); j++) { 834 835 for (m = 0; m < EHEA_MAX_RPAGE; m++) { 836 pg = sectbase + ((k++) * EHEA_PAGESIZE); 837 pt[m] = virt_to_abs(pg); 838 } 839 hret = ehea_h_register_rpage_mr(adapter->handle, mr->handle, 0, 840 0, pt_abs, EHEA_MAX_RPAGE); 841 842 if ((hret != H_SUCCESS) && 843 (hret != H_PAGE_REGISTERED)) { 844 ehea_h_free_resource(adapter->handle, mr->handle, 845 FORCE_FREE); 846 pr_err("register_rpage_mr failed\n"); 847 return hret; 848 } 849 } 850 return hret; 851 } 852 853 static u64 ehea_reg_mr_sections(int top, int dir, u64 *pt, 854 struct ehea_adapter *adapter, 855 struct ehea_mr *mr) 856 { 857 u64 hret = H_SUCCESS; 858 int idx; 859 860 for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { 861 if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) 862 continue; 863 864 hret = ehea_reg_mr_section(top, dir, idx, pt, adapter, mr); 865 if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) 866 return hret; 867 } 868 return hret; 869 } 870 871 static u64 ehea_reg_mr_dir_sections(int top, u64 *pt, 872 struct ehea_adapter *adapter, 873 struct ehea_mr *mr) 874 { 875 u64 hret = H_SUCCESS; 876 int dir; 877 878 for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { 879 if (!ehea_bmap->top[top]->dir[dir]) 880 continue; 881 882 hret = ehea_reg_mr_sections(top, dir, pt, adapter, mr); 883 if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) 884 return hret; 885 } 886 return hret; 887 } 888 889 int ehea_reg_kernel_mr(struct ehea_adapter *adapter, struct ehea_mr *mr) 890 { 891 int ret; 892 u64 *pt; 893 u64 hret; 894 u32 acc_ctrl = EHEA_MR_ACC_CTRL; 895 896 unsigned long top; 897 898 pt = (void *)get_zeroed_page(GFP_KERNEL); 899 if (!pt) { 900 pr_err("no mem\n"); 901 ret = -ENOMEM; 902 goto out; 903 } 904 905 hret = ehea_h_alloc_resource_mr(adapter->handle, EHEA_BUSMAP_START, 906 ehea_mr_len, acc_ctrl, adapter->pd, 907 &mr->handle, &mr->lkey); 908 909 if (hret != H_SUCCESS) { 910 pr_err("alloc_resource_mr failed\n"); 911 ret = -EIO; 912 goto out; 913 } 914 915 if (!ehea_bmap) { 916 ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); 917 pr_err("no busmap available\n"); 918 ret = -EIO; 919 goto out; 920 } 921 922 for (top = 0; top < EHEA_MAP_ENTRIES; top++) { 923 if (!ehea_bmap->top[top]) 924 continue; 925 926 hret = ehea_reg_mr_dir_sections(top, pt, adapter, mr); 927 if((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS)) 928 break; 929 } 930 931 if (hret != H_SUCCESS) { 932 ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); 933 pr_err("registering mr failed\n"); 934 ret = -EIO; 935 goto out; 936 } 937 938 mr->vaddr = EHEA_BUSMAP_START; 939 mr->adapter = adapter; 940 ret = 0; 941 out: 942 free_page((unsigned long)pt); 943 return ret; 944 } 945 946 int ehea_rem_mr(struct ehea_mr *mr) 947 { 948 u64 hret; 949 950 if (!mr || !mr->adapter) 951 return -EINVAL; 952 953 hret = ehea_h_free_resource(mr->adapter->handle, mr->handle, 954 FORCE_FREE); 955 if (hret != H_SUCCESS) { 956 pr_err("destroy MR failed\n"); 957 return -EIO; 958 } 959 960 return 0; 961 } 962 963 int ehea_gen_smr(struct ehea_adapter *adapter, struct ehea_mr *old_mr, 964 struct ehea_mr *shared_mr) 965 { 966 u64 hret; 967 968 hret = ehea_h_register_smr(adapter->handle, old_mr->handle, 969 old_mr->vaddr, EHEA_MR_ACC_CTRL, 970 adapter->pd, shared_mr); 971 if (hret != H_SUCCESS) 972 return -EIO; 973 974 shared_mr->adapter = adapter; 975 976 return 0; 977 } 978 979 void print_error_data(u64 *data) 980 { 981 int length; 982 u64 type = EHEA_BMASK_GET(ERROR_DATA_TYPE, data[2]); 983 u64 resource = data[1]; 984 985 length = EHEA_BMASK_GET(ERROR_DATA_LENGTH, data[0]); 986 987 if (length > EHEA_PAGESIZE) 988 length = EHEA_PAGESIZE; 989 990 if (type == EHEA_AER_RESTYPE_QP) 991 pr_err("QP (resource=%llX) state: AER=0x%llX, AERR=0x%llX, port=%llX\n", 992 resource, data[6], data[12], data[22]); 993 else if (type == EHEA_AER_RESTYPE_CQ) 994 pr_err("CQ (resource=%llX) state: AER=0x%llX\n", 995 resource, data[6]); 996 else if (type == EHEA_AER_RESTYPE_EQ) 997 pr_err("EQ (resource=%llX) state: AER=0x%llX\n", 998 resource, data[6]); 999 1000 ehea_dump(data, length, "error data"); 1001 } 1002 1003 u64 ehea_error_data(struct ehea_adapter *adapter, u64 res_handle, 1004 u64 *aer, u64 *aerr) 1005 { 1006 unsigned long ret; 1007 u64 *rblock; 1008 u64 type = 0; 1009 1010 rblock = (void *)get_zeroed_page(GFP_KERNEL); 1011 if (!rblock) { 1012 pr_err("Cannot allocate rblock memory\n"); 1013 goto out; 1014 } 1015 1016 ret = ehea_h_error_data(adapter->handle, res_handle, rblock); 1017 1018 if (ret == H_SUCCESS) { 1019 type = EHEA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]); 1020 *aer = rblock[6]; 1021 *aerr = rblock[12]; 1022 print_error_data(rblock); 1023 } else if (ret == H_R_STATE) { 1024 pr_err("No error data available: %llX\n", res_handle); 1025 } else 1026 pr_err("Error data could not be fetched: %llX\n", res_handle); 1027 1028 free_page((unsigned long)rblock); 1029 out: 1030 return type; 1031 } 1032