1 /* cnic.c: Broadcom CNIC core network driver. 2 * 3 * Copyright (c) 2006-2012 Broadcom Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 * 9 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com) 10 * Modified and maintained by: Michael Chan <mchan@broadcom.com> 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/module.h> 16 17 #include <linux/kernel.h> 18 #include <linux/errno.h> 19 #include <linux/list.h> 20 #include <linux/slab.h> 21 #include <linux/pci.h> 22 #include <linux/init.h> 23 #include <linux/netdevice.h> 24 #include <linux/uio_driver.h> 25 #include <linux/in.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/delay.h> 28 #include <linux/ethtool.h> 29 #include <linux/if_vlan.h> 30 #include <linux/prefetch.h> 31 #include <linux/random.h> 32 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE) 33 #define BCM_VLAN 1 34 #endif 35 #include <net/ip.h> 36 #include <net/tcp.h> 37 #include <net/route.h> 38 #include <net/ipv6.h> 39 #include <net/ip6_route.h> 40 #include <net/ip6_checksum.h> 41 #include <scsi/iscsi_if.h> 42 43 #define BCM_CNIC 1 44 #include "cnic_if.h" 45 #include "bnx2.h" 46 #include "bnx2x/bnx2x.h" 47 #include "bnx2x/bnx2x_reg.h" 48 #include "bnx2x/bnx2x_fw_defs.h" 49 #include "bnx2x/bnx2x_hsi.h" 50 #include "../../../scsi/bnx2i/57xx_iscsi_constants.h" 51 #include "../../../scsi/bnx2i/57xx_iscsi_hsi.h" 52 #include "../../../scsi/bnx2fc/bnx2fc_constants.h" 53 #include "cnic.h" 54 #include "cnic_defs.h" 55 56 #define CNIC_MODULE_NAME "cnic" 57 58 static char version[] = 59 "Broadcom NetXtreme II CNIC Driver " CNIC_MODULE_NAME " v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n"; 60 61 MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) " 62 "Chen (zongxi@broadcom.com"); 63 MODULE_DESCRIPTION("Broadcom NetXtreme II CNIC Driver"); 64 MODULE_LICENSE("GPL"); 65 MODULE_VERSION(CNIC_MODULE_VERSION); 66 67 /* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */ 68 static LIST_HEAD(cnic_dev_list); 69 static LIST_HEAD(cnic_udev_list); 70 static DEFINE_RWLOCK(cnic_dev_lock); 71 static DEFINE_MUTEX(cnic_lock); 72 73 static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE]; 74 75 /* helper function, assuming cnic_lock is held */ 76 static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type) 77 { 78 return rcu_dereference_protected(cnic_ulp_tbl[type], 79 lockdep_is_held(&cnic_lock)); 80 } 81 82 static int cnic_service_bnx2(void *, void *); 83 static int cnic_service_bnx2x(void *, void *); 84 static int cnic_ctl(void *, struct cnic_ctl_info *); 85 86 static struct cnic_ops cnic_bnx2_ops = { 87 .cnic_owner = THIS_MODULE, 88 .cnic_handler = cnic_service_bnx2, 89 .cnic_ctl = cnic_ctl, 90 }; 91 92 static struct cnic_ops cnic_bnx2x_ops = { 93 .cnic_owner = THIS_MODULE, 94 .cnic_handler = cnic_service_bnx2x, 95 .cnic_ctl = cnic_ctl, 96 }; 97 98 static struct workqueue_struct *cnic_wq; 99 100 static void cnic_shutdown_rings(struct cnic_dev *); 101 static void cnic_init_rings(struct cnic_dev *); 102 static int cnic_cm_set_pg(struct cnic_sock *); 103 104 static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode) 105 { 106 struct cnic_uio_dev *udev = uinfo->priv; 107 struct cnic_dev *dev; 108 109 if (!capable(CAP_NET_ADMIN)) 110 return -EPERM; 111 112 if (udev->uio_dev != -1) 113 return -EBUSY; 114 115 rtnl_lock(); 116 dev = udev->dev; 117 118 if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 119 rtnl_unlock(); 120 return -ENODEV; 121 } 122 123 udev->uio_dev = iminor(inode); 124 125 cnic_shutdown_rings(dev); 126 cnic_init_rings(dev); 127 rtnl_unlock(); 128 129 return 0; 130 } 131 132 static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode) 133 { 134 struct cnic_uio_dev *udev = uinfo->priv; 135 136 udev->uio_dev = -1; 137 return 0; 138 } 139 140 static inline void cnic_hold(struct cnic_dev *dev) 141 { 142 atomic_inc(&dev->ref_count); 143 } 144 145 static inline void cnic_put(struct cnic_dev *dev) 146 { 147 atomic_dec(&dev->ref_count); 148 } 149 150 static inline void csk_hold(struct cnic_sock *csk) 151 { 152 atomic_inc(&csk->ref_count); 153 } 154 155 static inline void csk_put(struct cnic_sock *csk) 156 { 157 atomic_dec(&csk->ref_count); 158 } 159 160 static struct cnic_dev *cnic_from_netdev(struct net_device *netdev) 161 { 162 struct cnic_dev *cdev; 163 164 read_lock(&cnic_dev_lock); 165 list_for_each_entry(cdev, &cnic_dev_list, list) { 166 if (netdev == cdev->netdev) { 167 cnic_hold(cdev); 168 read_unlock(&cnic_dev_lock); 169 return cdev; 170 } 171 } 172 read_unlock(&cnic_dev_lock); 173 return NULL; 174 } 175 176 static inline void ulp_get(struct cnic_ulp_ops *ulp_ops) 177 { 178 atomic_inc(&ulp_ops->ref_count); 179 } 180 181 static inline void ulp_put(struct cnic_ulp_ops *ulp_ops) 182 { 183 atomic_dec(&ulp_ops->ref_count); 184 } 185 186 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val) 187 { 188 struct cnic_local *cp = dev->cnic_priv; 189 struct cnic_eth_dev *ethdev = cp->ethdev; 190 struct drv_ctl_info info; 191 struct drv_ctl_io *io = &info.data.io; 192 193 info.cmd = DRV_CTL_CTX_WR_CMD; 194 io->cid_addr = cid_addr; 195 io->offset = off; 196 io->data = val; 197 ethdev->drv_ctl(dev->netdev, &info); 198 } 199 200 static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr) 201 { 202 struct cnic_local *cp = dev->cnic_priv; 203 struct cnic_eth_dev *ethdev = cp->ethdev; 204 struct drv_ctl_info info; 205 struct drv_ctl_io *io = &info.data.io; 206 207 info.cmd = DRV_CTL_CTXTBL_WR_CMD; 208 io->offset = off; 209 io->dma_addr = addr; 210 ethdev->drv_ctl(dev->netdev, &info); 211 } 212 213 static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start) 214 { 215 struct cnic_local *cp = dev->cnic_priv; 216 struct cnic_eth_dev *ethdev = cp->ethdev; 217 struct drv_ctl_info info; 218 struct drv_ctl_l2_ring *ring = &info.data.ring; 219 220 if (start) 221 info.cmd = DRV_CTL_START_L2_CMD; 222 else 223 info.cmd = DRV_CTL_STOP_L2_CMD; 224 225 ring->cid = cid; 226 ring->client_id = cl_id; 227 ethdev->drv_ctl(dev->netdev, &info); 228 } 229 230 static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val) 231 { 232 struct cnic_local *cp = dev->cnic_priv; 233 struct cnic_eth_dev *ethdev = cp->ethdev; 234 struct drv_ctl_info info; 235 struct drv_ctl_io *io = &info.data.io; 236 237 info.cmd = DRV_CTL_IO_WR_CMD; 238 io->offset = off; 239 io->data = val; 240 ethdev->drv_ctl(dev->netdev, &info); 241 } 242 243 static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off) 244 { 245 struct cnic_local *cp = dev->cnic_priv; 246 struct cnic_eth_dev *ethdev = cp->ethdev; 247 struct drv_ctl_info info; 248 struct drv_ctl_io *io = &info.data.io; 249 250 info.cmd = DRV_CTL_IO_RD_CMD; 251 io->offset = off; 252 ethdev->drv_ctl(dev->netdev, &info); 253 return io->data; 254 } 255 256 static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg) 257 { 258 struct cnic_local *cp = dev->cnic_priv; 259 struct cnic_eth_dev *ethdev = cp->ethdev; 260 struct drv_ctl_info info; 261 struct fcoe_capabilities *fcoe_cap = 262 &info.data.register_data.fcoe_features; 263 264 if (reg) { 265 info.cmd = DRV_CTL_ULP_REGISTER_CMD; 266 if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap) 267 memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap)); 268 } else { 269 info.cmd = DRV_CTL_ULP_UNREGISTER_CMD; 270 } 271 272 info.data.ulp_type = ulp_type; 273 ethdev->drv_ctl(dev->netdev, &info); 274 } 275 276 static int cnic_in_use(struct cnic_sock *csk) 277 { 278 return test_bit(SK_F_INUSE, &csk->flags); 279 } 280 281 static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count) 282 { 283 struct cnic_local *cp = dev->cnic_priv; 284 struct cnic_eth_dev *ethdev = cp->ethdev; 285 struct drv_ctl_info info; 286 287 info.cmd = cmd; 288 info.data.credit.credit_count = count; 289 ethdev->drv_ctl(dev->netdev, &info); 290 } 291 292 static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid) 293 { 294 u32 i; 295 296 if (!cp->ctx_tbl) 297 return -EINVAL; 298 299 for (i = 0; i < cp->max_cid_space; i++) { 300 if (cp->ctx_tbl[i].cid == cid) { 301 *l5_cid = i; 302 return 0; 303 } 304 } 305 return -EINVAL; 306 } 307 308 static int cnic_send_nlmsg(struct cnic_local *cp, u32 type, 309 struct cnic_sock *csk) 310 { 311 struct iscsi_path path_req; 312 char *buf = NULL; 313 u16 len = 0; 314 u32 msg_type = ISCSI_KEVENT_IF_DOWN; 315 struct cnic_ulp_ops *ulp_ops; 316 struct cnic_uio_dev *udev = cp->udev; 317 int rc = 0, retry = 0; 318 319 if (!udev || udev->uio_dev == -1) 320 return -ENODEV; 321 322 if (csk) { 323 len = sizeof(path_req); 324 buf = (char *) &path_req; 325 memset(&path_req, 0, len); 326 327 msg_type = ISCSI_KEVENT_PATH_REQ; 328 path_req.handle = (u64) csk->l5_cid; 329 if (test_bit(SK_F_IPV6, &csk->flags)) { 330 memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0], 331 sizeof(struct in6_addr)); 332 path_req.ip_addr_len = 16; 333 } else { 334 memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0], 335 sizeof(struct in_addr)); 336 path_req.ip_addr_len = 4; 337 } 338 path_req.vlan_id = csk->vlan_id; 339 path_req.pmtu = csk->mtu; 340 } 341 342 while (retry < 3) { 343 rc = 0; 344 rcu_read_lock(); 345 ulp_ops = rcu_dereference(cnic_ulp_tbl[CNIC_ULP_ISCSI]); 346 if (ulp_ops) 347 rc = ulp_ops->iscsi_nl_send_msg( 348 cp->ulp_handle[CNIC_ULP_ISCSI], 349 msg_type, buf, len); 350 rcu_read_unlock(); 351 if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ) 352 break; 353 354 msleep(100); 355 retry++; 356 } 357 return rc; 358 } 359 360 static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8); 361 362 static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type, 363 char *buf, u16 len) 364 { 365 int rc = -EINVAL; 366 367 switch (msg_type) { 368 case ISCSI_UEVENT_PATH_UPDATE: { 369 struct cnic_local *cp; 370 u32 l5_cid; 371 struct cnic_sock *csk; 372 struct iscsi_path *path_resp; 373 374 if (len < sizeof(*path_resp)) 375 break; 376 377 path_resp = (struct iscsi_path *) buf; 378 cp = dev->cnic_priv; 379 l5_cid = (u32) path_resp->handle; 380 if (l5_cid >= MAX_CM_SK_TBL_SZ) 381 break; 382 383 rcu_read_lock(); 384 if (!rcu_dereference(cp->ulp_ops[CNIC_ULP_L4])) { 385 rc = -ENODEV; 386 rcu_read_unlock(); 387 break; 388 } 389 csk = &cp->csk_tbl[l5_cid]; 390 csk_hold(csk); 391 if (cnic_in_use(csk) && 392 test_bit(SK_F_CONNECT_START, &csk->flags)) { 393 394 csk->vlan_id = path_resp->vlan_id; 395 396 memcpy(csk->ha, path_resp->mac_addr, 6); 397 if (test_bit(SK_F_IPV6, &csk->flags)) 398 memcpy(&csk->src_ip[0], &path_resp->src.v6_addr, 399 sizeof(struct in6_addr)); 400 else 401 memcpy(&csk->src_ip[0], &path_resp->src.v4_addr, 402 sizeof(struct in_addr)); 403 404 if (is_valid_ether_addr(csk->ha)) { 405 cnic_cm_set_pg(csk); 406 } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) && 407 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 408 409 cnic_cm_upcall(cp, csk, 410 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 411 clear_bit(SK_F_CONNECT_START, &csk->flags); 412 } 413 } 414 csk_put(csk); 415 rcu_read_unlock(); 416 rc = 0; 417 } 418 } 419 420 return rc; 421 } 422 423 static int cnic_offld_prep(struct cnic_sock *csk) 424 { 425 if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 426 return 0; 427 428 if (!test_bit(SK_F_CONNECT_START, &csk->flags)) { 429 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 430 return 0; 431 } 432 433 return 1; 434 } 435 436 static int cnic_close_prep(struct cnic_sock *csk) 437 { 438 clear_bit(SK_F_CONNECT_START, &csk->flags); 439 smp_mb__after_clear_bit(); 440 441 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 442 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 443 msleep(1); 444 445 return 1; 446 } 447 return 0; 448 } 449 450 static int cnic_abort_prep(struct cnic_sock *csk) 451 { 452 clear_bit(SK_F_CONNECT_START, &csk->flags); 453 smp_mb__after_clear_bit(); 454 455 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 456 msleep(1); 457 458 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 459 csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP; 460 return 1; 461 } 462 463 return 0; 464 } 465 466 int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops) 467 { 468 struct cnic_dev *dev; 469 470 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 471 pr_err("%s: Bad type %d\n", __func__, ulp_type); 472 return -EINVAL; 473 } 474 mutex_lock(&cnic_lock); 475 if (cnic_ulp_tbl_prot(ulp_type)) { 476 pr_err("%s: Type %d has already been registered\n", 477 __func__, ulp_type); 478 mutex_unlock(&cnic_lock); 479 return -EBUSY; 480 } 481 482 read_lock(&cnic_dev_lock); 483 list_for_each_entry(dev, &cnic_dev_list, list) { 484 struct cnic_local *cp = dev->cnic_priv; 485 486 clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]); 487 } 488 read_unlock(&cnic_dev_lock); 489 490 atomic_set(&ulp_ops->ref_count, 0); 491 rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops); 492 mutex_unlock(&cnic_lock); 493 494 /* Prevent race conditions with netdev_event */ 495 rtnl_lock(); 496 list_for_each_entry(dev, &cnic_dev_list, list) { 497 struct cnic_local *cp = dev->cnic_priv; 498 499 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 500 ulp_ops->cnic_init(dev); 501 } 502 rtnl_unlock(); 503 504 return 0; 505 } 506 507 int cnic_unregister_driver(int ulp_type) 508 { 509 struct cnic_dev *dev; 510 struct cnic_ulp_ops *ulp_ops; 511 int i = 0; 512 513 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 514 pr_err("%s: Bad type %d\n", __func__, ulp_type); 515 return -EINVAL; 516 } 517 mutex_lock(&cnic_lock); 518 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 519 if (!ulp_ops) { 520 pr_err("%s: Type %d has not been registered\n", 521 __func__, ulp_type); 522 goto out_unlock; 523 } 524 read_lock(&cnic_dev_lock); 525 list_for_each_entry(dev, &cnic_dev_list, list) { 526 struct cnic_local *cp = dev->cnic_priv; 527 528 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 529 pr_err("%s: Type %d still has devices registered\n", 530 __func__, ulp_type); 531 read_unlock(&cnic_dev_lock); 532 goto out_unlock; 533 } 534 } 535 read_unlock(&cnic_dev_lock); 536 537 RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL); 538 539 mutex_unlock(&cnic_lock); 540 synchronize_rcu(); 541 while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) { 542 msleep(100); 543 i++; 544 } 545 546 if (atomic_read(&ulp_ops->ref_count) != 0) 547 pr_warn("%s: Failed waiting for ref count to go to zero\n", 548 __func__); 549 return 0; 550 551 out_unlock: 552 mutex_unlock(&cnic_lock); 553 return -EINVAL; 554 } 555 556 static int cnic_start_hw(struct cnic_dev *); 557 static void cnic_stop_hw(struct cnic_dev *); 558 559 static int cnic_register_device(struct cnic_dev *dev, int ulp_type, 560 void *ulp_ctx) 561 { 562 struct cnic_local *cp = dev->cnic_priv; 563 struct cnic_ulp_ops *ulp_ops; 564 565 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 566 pr_err("%s: Bad type %d\n", __func__, ulp_type); 567 return -EINVAL; 568 } 569 mutex_lock(&cnic_lock); 570 if (cnic_ulp_tbl_prot(ulp_type) == NULL) { 571 pr_err("%s: Driver with type %d has not been registered\n", 572 __func__, ulp_type); 573 mutex_unlock(&cnic_lock); 574 return -EAGAIN; 575 } 576 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 577 pr_err("%s: Type %d has already been registered to this device\n", 578 __func__, ulp_type); 579 mutex_unlock(&cnic_lock); 580 return -EBUSY; 581 } 582 583 clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]); 584 cp->ulp_handle[ulp_type] = ulp_ctx; 585 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 586 rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops); 587 cnic_hold(dev); 588 589 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 590 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type])) 591 ulp_ops->cnic_start(cp->ulp_handle[ulp_type]); 592 593 mutex_unlock(&cnic_lock); 594 595 cnic_ulp_ctl(dev, ulp_type, true); 596 597 return 0; 598 599 } 600 EXPORT_SYMBOL(cnic_register_driver); 601 602 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type) 603 { 604 struct cnic_local *cp = dev->cnic_priv; 605 int i = 0; 606 607 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 608 pr_err("%s: Bad type %d\n", __func__, ulp_type); 609 return -EINVAL; 610 } 611 mutex_lock(&cnic_lock); 612 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 613 RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL); 614 cnic_put(dev); 615 } else { 616 pr_err("%s: device not registered to this ulp type %d\n", 617 __func__, ulp_type); 618 mutex_unlock(&cnic_lock); 619 return -EINVAL; 620 } 621 mutex_unlock(&cnic_lock); 622 623 if (ulp_type == CNIC_ULP_ISCSI) 624 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 625 else if (ulp_type == CNIC_ULP_FCOE) 626 dev->fcoe_cap = NULL; 627 628 synchronize_rcu(); 629 630 while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) && 631 i < 20) { 632 msleep(100); 633 i++; 634 } 635 if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type])) 636 netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n"); 637 638 cnic_ulp_ctl(dev, ulp_type, false); 639 640 return 0; 641 } 642 EXPORT_SYMBOL(cnic_unregister_driver); 643 644 static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id, 645 u32 next) 646 { 647 id_tbl->start = start_id; 648 id_tbl->max = size; 649 id_tbl->next = next; 650 spin_lock_init(&id_tbl->lock); 651 id_tbl->table = kzalloc(DIV_ROUND_UP(size, 32) * 4, GFP_KERNEL); 652 if (!id_tbl->table) 653 return -ENOMEM; 654 655 return 0; 656 } 657 658 static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl) 659 { 660 kfree(id_tbl->table); 661 id_tbl->table = NULL; 662 } 663 664 static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id) 665 { 666 int ret = -1; 667 668 id -= id_tbl->start; 669 if (id >= id_tbl->max) 670 return ret; 671 672 spin_lock(&id_tbl->lock); 673 if (!test_bit(id, id_tbl->table)) { 674 set_bit(id, id_tbl->table); 675 ret = 0; 676 } 677 spin_unlock(&id_tbl->lock); 678 return ret; 679 } 680 681 /* Returns -1 if not successful */ 682 static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl) 683 { 684 u32 id; 685 686 spin_lock(&id_tbl->lock); 687 id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next); 688 if (id >= id_tbl->max) { 689 id = -1; 690 if (id_tbl->next != 0) { 691 id = find_first_zero_bit(id_tbl->table, id_tbl->next); 692 if (id >= id_tbl->next) 693 id = -1; 694 } 695 } 696 697 if (id < id_tbl->max) { 698 set_bit(id, id_tbl->table); 699 id_tbl->next = (id + 1) & (id_tbl->max - 1); 700 id += id_tbl->start; 701 } 702 703 spin_unlock(&id_tbl->lock); 704 705 return id; 706 } 707 708 static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id) 709 { 710 if (id == -1) 711 return; 712 713 id -= id_tbl->start; 714 if (id >= id_tbl->max) 715 return; 716 717 clear_bit(id, id_tbl->table); 718 } 719 720 static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma) 721 { 722 int i; 723 724 if (!dma->pg_arr) 725 return; 726 727 for (i = 0; i < dma->num_pages; i++) { 728 if (dma->pg_arr[i]) { 729 dma_free_coherent(&dev->pcidev->dev, BNX2_PAGE_SIZE, 730 dma->pg_arr[i], dma->pg_map_arr[i]); 731 dma->pg_arr[i] = NULL; 732 } 733 } 734 if (dma->pgtbl) { 735 dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size, 736 dma->pgtbl, dma->pgtbl_map); 737 dma->pgtbl = NULL; 738 } 739 kfree(dma->pg_arr); 740 dma->pg_arr = NULL; 741 dma->num_pages = 0; 742 } 743 744 static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma) 745 { 746 int i; 747 __le32 *page_table = (__le32 *) dma->pgtbl; 748 749 for (i = 0; i < dma->num_pages; i++) { 750 /* Each entry needs to be in big endian format. */ 751 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 752 page_table++; 753 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 754 page_table++; 755 } 756 } 757 758 static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma) 759 { 760 int i; 761 __le32 *page_table = (__le32 *) dma->pgtbl; 762 763 for (i = 0; i < dma->num_pages; i++) { 764 /* Each entry needs to be in little endian format. */ 765 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 766 page_table++; 767 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 768 page_table++; 769 } 770 } 771 772 static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma, 773 int pages, int use_pg_tbl) 774 { 775 int i, size; 776 struct cnic_local *cp = dev->cnic_priv; 777 778 size = pages * (sizeof(void *) + sizeof(dma_addr_t)); 779 dma->pg_arr = kzalloc(size, GFP_ATOMIC); 780 if (dma->pg_arr == NULL) 781 return -ENOMEM; 782 783 dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages); 784 dma->num_pages = pages; 785 786 for (i = 0; i < pages; i++) { 787 dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev, 788 BNX2_PAGE_SIZE, 789 &dma->pg_map_arr[i], 790 GFP_ATOMIC); 791 if (dma->pg_arr[i] == NULL) 792 goto error; 793 } 794 if (!use_pg_tbl) 795 return 0; 796 797 dma->pgtbl_size = ((pages * 8) + BNX2_PAGE_SIZE - 1) & 798 ~(BNX2_PAGE_SIZE - 1); 799 dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size, 800 &dma->pgtbl_map, GFP_ATOMIC); 801 if (dma->pgtbl == NULL) 802 goto error; 803 804 cp->setup_pgtbl(dev, dma); 805 806 return 0; 807 808 error: 809 cnic_free_dma(dev, dma); 810 return -ENOMEM; 811 } 812 813 static void cnic_free_context(struct cnic_dev *dev) 814 { 815 struct cnic_local *cp = dev->cnic_priv; 816 int i; 817 818 for (i = 0; i < cp->ctx_blks; i++) { 819 if (cp->ctx_arr[i].ctx) { 820 dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 821 cp->ctx_arr[i].ctx, 822 cp->ctx_arr[i].mapping); 823 cp->ctx_arr[i].ctx = NULL; 824 } 825 } 826 } 827 828 static void __cnic_free_uio_rings(struct cnic_uio_dev *udev) 829 { 830 if (udev->l2_buf) { 831 dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size, 832 udev->l2_buf, udev->l2_buf_map); 833 udev->l2_buf = NULL; 834 } 835 836 if (udev->l2_ring) { 837 dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size, 838 udev->l2_ring, udev->l2_ring_map); 839 udev->l2_ring = NULL; 840 } 841 842 } 843 844 static void __cnic_free_uio(struct cnic_uio_dev *udev) 845 { 846 uio_unregister_device(&udev->cnic_uinfo); 847 848 __cnic_free_uio_rings(udev); 849 850 pci_dev_put(udev->pdev); 851 kfree(udev); 852 } 853 854 static void cnic_free_uio(struct cnic_uio_dev *udev) 855 { 856 if (!udev) 857 return; 858 859 write_lock(&cnic_dev_lock); 860 list_del_init(&udev->list); 861 write_unlock(&cnic_dev_lock); 862 __cnic_free_uio(udev); 863 } 864 865 static void cnic_free_resc(struct cnic_dev *dev) 866 { 867 struct cnic_local *cp = dev->cnic_priv; 868 struct cnic_uio_dev *udev = cp->udev; 869 870 if (udev) { 871 udev->dev = NULL; 872 cp->udev = NULL; 873 if (udev->uio_dev == -1) 874 __cnic_free_uio_rings(udev); 875 } 876 877 cnic_free_context(dev); 878 kfree(cp->ctx_arr); 879 cp->ctx_arr = NULL; 880 cp->ctx_blks = 0; 881 882 cnic_free_dma(dev, &cp->gbl_buf_info); 883 cnic_free_dma(dev, &cp->kwq_info); 884 cnic_free_dma(dev, &cp->kwq_16_data_info); 885 cnic_free_dma(dev, &cp->kcq2.dma); 886 cnic_free_dma(dev, &cp->kcq1.dma); 887 kfree(cp->iscsi_tbl); 888 cp->iscsi_tbl = NULL; 889 kfree(cp->ctx_tbl); 890 cp->ctx_tbl = NULL; 891 892 cnic_free_id_tbl(&cp->fcoe_cid_tbl); 893 cnic_free_id_tbl(&cp->cid_tbl); 894 } 895 896 static int cnic_alloc_context(struct cnic_dev *dev) 897 { 898 struct cnic_local *cp = dev->cnic_priv; 899 900 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 901 int i, k, arr_size; 902 903 cp->ctx_blk_size = BNX2_PAGE_SIZE; 904 cp->cids_per_blk = BNX2_PAGE_SIZE / 128; 905 arr_size = BNX2_MAX_CID / cp->cids_per_blk * 906 sizeof(struct cnic_ctx); 907 cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL); 908 if (cp->ctx_arr == NULL) 909 return -ENOMEM; 910 911 k = 0; 912 for (i = 0; i < 2; i++) { 913 u32 j, reg, off, lo, hi; 914 915 if (i == 0) 916 off = BNX2_PG_CTX_MAP; 917 else 918 off = BNX2_ISCSI_CTX_MAP; 919 920 reg = cnic_reg_rd_ind(dev, off); 921 lo = reg >> 16; 922 hi = reg & 0xffff; 923 for (j = lo; j < hi; j += cp->cids_per_blk, k++) 924 cp->ctx_arr[k].cid = j; 925 } 926 927 cp->ctx_blks = k; 928 if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) { 929 cp->ctx_blks = 0; 930 return -ENOMEM; 931 } 932 933 for (i = 0; i < cp->ctx_blks; i++) { 934 cp->ctx_arr[i].ctx = 935 dma_alloc_coherent(&dev->pcidev->dev, 936 BNX2_PAGE_SIZE, 937 &cp->ctx_arr[i].mapping, 938 GFP_KERNEL); 939 if (cp->ctx_arr[i].ctx == NULL) 940 return -ENOMEM; 941 } 942 } 943 return 0; 944 } 945 946 static u16 cnic_bnx2_next_idx(u16 idx) 947 { 948 return idx + 1; 949 } 950 951 static u16 cnic_bnx2_hw_idx(u16 idx) 952 { 953 return idx; 954 } 955 956 static u16 cnic_bnx2x_next_idx(u16 idx) 957 { 958 idx++; 959 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 960 idx++; 961 962 return idx; 963 } 964 965 static u16 cnic_bnx2x_hw_idx(u16 idx) 966 { 967 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 968 idx++; 969 return idx; 970 } 971 972 static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info, 973 bool use_pg_tbl) 974 { 975 int err, i, use_page_tbl = 0; 976 struct kcqe **kcq; 977 978 if (use_pg_tbl) 979 use_page_tbl = 1; 980 981 err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl); 982 if (err) 983 return err; 984 985 kcq = (struct kcqe **) info->dma.pg_arr; 986 info->kcq = kcq; 987 988 info->next_idx = cnic_bnx2_next_idx; 989 info->hw_idx = cnic_bnx2_hw_idx; 990 if (use_pg_tbl) 991 return 0; 992 993 info->next_idx = cnic_bnx2x_next_idx; 994 info->hw_idx = cnic_bnx2x_hw_idx; 995 996 for (i = 0; i < KCQ_PAGE_CNT; i++) { 997 struct bnx2x_bd_chain_next *next = 998 (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT]; 999 int j = i + 1; 1000 1001 if (j >= KCQ_PAGE_CNT) 1002 j = 0; 1003 next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32; 1004 next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff; 1005 } 1006 return 0; 1007 } 1008 1009 static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages) 1010 { 1011 struct cnic_local *cp = udev->dev->cnic_priv; 1012 1013 if (udev->l2_ring) 1014 return 0; 1015 1016 udev->l2_ring_size = pages * BNX2_PAGE_SIZE; 1017 udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size, 1018 &udev->l2_ring_map, 1019 GFP_KERNEL | __GFP_COMP); 1020 if (!udev->l2_ring) 1021 return -ENOMEM; 1022 1023 udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size; 1024 udev->l2_buf_size = PAGE_ALIGN(udev->l2_buf_size); 1025 udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size, 1026 &udev->l2_buf_map, 1027 GFP_KERNEL | __GFP_COMP); 1028 if (!udev->l2_buf) { 1029 __cnic_free_uio_rings(udev); 1030 return -ENOMEM; 1031 } 1032 1033 return 0; 1034 1035 } 1036 1037 static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages) 1038 { 1039 struct cnic_local *cp = dev->cnic_priv; 1040 struct cnic_uio_dev *udev; 1041 1042 read_lock(&cnic_dev_lock); 1043 list_for_each_entry(udev, &cnic_udev_list, list) { 1044 if (udev->pdev == dev->pcidev) { 1045 udev->dev = dev; 1046 if (__cnic_alloc_uio_rings(udev, pages)) { 1047 udev->dev = NULL; 1048 read_unlock(&cnic_dev_lock); 1049 return -ENOMEM; 1050 } 1051 cp->udev = udev; 1052 read_unlock(&cnic_dev_lock); 1053 return 0; 1054 } 1055 } 1056 read_unlock(&cnic_dev_lock); 1057 1058 udev = kzalloc(sizeof(struct cnic_uio_dev), GFP_ATOMIC); 1059 if (!udev) 1060 return -ENOMEM; 1061 1062 udev->uio_dev = -1; 1063 1064 udev->dev = dev; 1065 udev->pdev = dev->pcidev; 1066 1067 if (__cnic_alloc_uio_rings(udev, pages)) 1068 goto err_udev; 1069 1070 write_lock(&cnic_dev_lock); 1071 list_add(&udev->list, &cnic_udev_list); 1072 write_unlock(&cnic_dev_lock); 1073 1074 pci_dev_get(udev->pdev); 1075 1076 cp->udev = udev; 1077 1078 return 0; 1079 1080 err_udev: 1081 kfree(udev); 1082 return -ENOMEM; 1083 } 1084 1085 static int cnic_init_uio(struct cnic_dev *dev) 1086 { 1087 struct cnic_local *cp = dev->cnic_priv; 1088 struct cnic_uio_dev *udev = cp->udev; 1089 struct uio_info *uinfo; 1090 int ret = 0; 1091 1092 if (!udev) 1093 return -ENOMEM; 1094 1095 uinfo = &udev->cnic_uinfo; 1096 1097 uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0); 1098 uinfo->mem[0].internal_addr = dev->regview; 1099 uinfo->mem[0].memtype = UIO_MEM_PHYS; 1100 1101 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 1102 uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID + 1103 TX_MAX_TSS_RINGS + 1); 1104 uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen & 1105 PAGE_MASK; 1106 if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 1107 uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE * 9; 1108 else 1109 uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE; 1110 1111 uinfo->name = "bnx2_cnic"; 1112 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 1113 uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0); 1114 1115 uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk & 1116 PAGE_MASK; 1117 uinfo->mem[1].size = sizeof(*cp->bnx2x_def_status_blk); 1118 1119 uinfo->name = "bnx2x_cnic"; 1120 } 1121 1122 uinfo->mem[1].memtype = UIO_MEM_LOGICAL; 1123 1124 uinfo->mem[2].addr = (unsigned long) udev->l2_ring; 1125 uinfo->mem[2].size = udev->l2_ring_size; 1126 uinfo->mem[2].memtype = UIO_MEM_LOGICAL; 1127 1128 uinfo->mem[3].addr = (unsigned long) udev->l2_buf; 1129 uinfo->mem[3].size = udev->l2_buf_size; 1130 uinfo->mem[3].memtype = UIO_MEM_LOGICAL; 1131 1132 uinfo->version = CNIC_MODULE_VERSION; 1133 uinfo->irq = UIO_IRQ_CUSTOM; 1134 1135 uinfo->open = cnic_uio_open; 1136 uinfo->release = cnic_uio_close; 1137 1138 if (udev->uio_dev == -1) { 1139 if (!uinfo->priv) { 1140 uinfo->priv = udev; 1141 1142 ret = uio_register_device(&udev->pdev->dev, uinfo); 1143 } 1144 } else { 1145 cnic_init_rings(dev); 1146 } 1147 1148 return ret; 1149 } 1150 1151 static int cnic_alloc_bnx2_resc(struct cnic_dev *dev) 1152 { 1153 struct cnic_local *cp = dev->cnic_priv; 1154 int ret; 1155 1156 ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1); 1157 if (ret) 1158 goto error; 1159 cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr; 1160 1161 ret = cnic_alloc_kcq(dev, &cp->kcq1, true); 1162 if (ret) 1163 goto error; 1164 1165 ret = cnic_alloc_context(dev); 1166 if (ret) 1167 goto error; 1168 1169 ret = cnic_alloc_uio_rings(dev, 2); 1170 if (ret) 1171 goto error; 1172 1173 ret = cnic_init_uio(dev); 1174 if (ret) 1175 goto error; 1176 1177 return 0; 1178 1179 error: 1180 cnic_free_resc(dev); 1181 return ret; 1182 } 1183 1184 static int cnic_alloc_bnx2x_context(struct cnic_dev *dev) 1185 { 1186 struct cnic_local *cp = dev->cnic_priv; 1187 int ctx_blk_size = cp->ethdev->ctx_blk_size; 1188 int total_mem, blks, i; 1189 1190 total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space; 1191 blks = total_mem / ctx_blk_size; 1192 if (total_mem % ctx_blk_size) 1193 blks++; 1194 1195 if (blks > cp->ethdev->ctx_tbl_len) 1196 return -ENOMEM; 1197 1198 cp->ctx_arr = kcalloc(blks, sizeof(struct cnic_ctx), GFP_KERNEL); 1199 if (cp->ctx_arr == NULL) 1200 return -ENOMEM; 1201 1202 cp->ctx_blks = blks; 1203 cp->ctx_blk_size = ctx_blk_size; 1204 if (!BNX2X_CHIP_IS_57710(cp->chip_id)) 1205 cp->ctx_align = 0; 1206 else 1207 cp->ctx_align = ctx_blk_size; 1208 1209 cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE; 1210 1211 for (i = 0; i < blks; i++) { 1212 cp->ctx_arr[i].ctx = 1213 dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 1214 &cp->ctx_arr[i].mapping, 1215 GFP_KERNEL); 1216 if (cp->ctx_arr[i].ctx == NULL) 1217 return -ENOMEM; 1218 1219 if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) { 1220 if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) { 1221 cnic_free_context(dev); 1222 cp->ctx_blk_size += cp->ctx_align; 1223 i = -1; 1224 continue; 1225 } 1226 } 1227 } 1228 return 0; 1229 } 1230 1231 static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev) 1232 { 1233 struct cnic_local *cp = dev->cnic_priv; 1234 struct cnic_eth_dev *ethdev = cp->ethdev; 1235 u32 start_cid = ethdev->starting_cid; 1236 int i, j, n, ret, pages; 1237 struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info; 1238 1239 cp->max_cid_space = MAX_ISCSI_TBL_SZ; 1240 cp->iscsi_start_cid = start_cid; 1241 cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ; 1242 1243 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 1244 cp->max_cid_space += dev->max_fcoe_conn; 1245 cp->fcoe_init_cid = ethdev->fcoe_init_cid; 1246 if (!cp->fcoe_init_cid) 1247 cp->fcoe_init_cid = 0x10; 1248 } 1249 1250 cp->iscsi_tbl = kzalloc(sizeof(struct cnic_iscsi) * MAX_ISCSI_TBL_SZ, 1251 GFP_KERNEL); 1252 if (!cp->iscsi_tbl) 1253 goto error; 1254 1255 cp->ctx_tbl = kzalloc(sizeof(struct cnic_context) * 1256 cp->max_cid_space, GFP_KERNEL); 1257 if (!cp->ctx_tbl) 1258 goto error; 1259 1260 for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) { 1261 cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i]; 1262 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI; 1263 } 1264 1265 for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++) 1266 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE; 1267 1268 pages = PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) / 1269 PAGE_SIZE; 1270 1271 ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0); 1272 if (ret) 1273 return -ENOMEM; 1274 1275 n = PAGE_SIZE / CNIC_KWQ16_DATA_SIZE; 1276 for (i = 0, j = 0; i < cp->max_cid_space; i++) { 1277 long off = CNIC_KWQ16_DATA_SIZE * (i % n); 1278 1279 cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off; 1280 cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] + 1281 off; 1282 1283 if ((i % n) == (n - 1)) 1284 j++; 1285 } 1286 1287 ret = cnic_alloc_kcq(dev, &cp->kcq1, false); 1288 if (ret) 1289 goto error; 1290 1291 if (CNIC_SUPPORTS_FCOE(cp)) { 1292 ret = cnic_alloc_kcq(dev, &cp->kcq2, true); 1293 if (ret) 1294 goto error; 1295 } 1296 1297 pages = PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / PAGE_SIZE; 1298 ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0); 1299 if (ret) 1300 goto error; 1301 1302 ret = cnic_alloc_bnx2x_context(dev); 1303 if (ret) 1304 goto error; 1305 1306 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 1307 return 0; 1308 1309 cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk; 1310 1311 cp->l2_rx_ring_size = 15; 1312 1313 ret = cnic_alloc_uio_rings(dev, 4); 1314 if (ret) 1315 goto error; 1316 1317 ret = cnic_init_uio(dev); 1318 if (ret) 1319 goto error; 1320 1321 return 0; 1322 1323 error: 1324 cnic_free_resc(dev); 1325 return -ENOMEM; 1326 } 1327 1328 static inline u32 cnic_kwq_avail(struct cnic_local *cp) 1329 { 1330 return cp->max_kwq_idx - 1331 ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx); 1332 } 1333 1334 static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 1335 u32 num_wqes) 1336 { 1337 struct cnic_local *cp = dev->cnic_priv; 1338 struct kwqe *prod_qe; 1339 u16 prod, sw_prod, i; 1340 1341 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 1342 return -EAGAIN; /* bnx2 is down */ 1343 1344 spin_lock_bh(&cp->cnic_ulp_lock); 1345 if (num_wqes > cnic_kwq_avail(cp) && 1346 !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) { 1347 spin_unlock_bh(&cp->cnic_ulp_lock); 1348 return -EAGAIN; 1349 } 1350 1351 clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 1352 1353 prod = cp->kwq_prod_idx; 1354 sw_prod = prod & MAX_KWQ_IDX; 1355 for (i = 0; i < num_wqes; i++) { 1356 prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)]; 1357 memcpy(prod_qe, wqes[i], sizeof(struct kwqe)); 1358 prod++; 1359 sw_prod = prod & MAX_KWQ_IDX; 1360 } 1361 cp->kwq_prod_idx = prod; 1362 1363 CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx); 1364 1365 spin_unlock_bh(&cp->cnic_ulp_lock); 1366 return 0; 1367 } 1368 1369 static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid, 1370 union l5cm_specific_data *l5_data) 1371 { 1372 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1373 dma_addr_t map; 1374 1375 map = ctx->kwqe_data_mapping; 1376 l5_data->phy_address.lo = (u64) map & 0xffffffff; 1377 l5_data->phy_address.hi = (u64) map >> 32; 1378 return ctx->kwqe_data; 1379 } 1380 1381 static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid, 1382 u32 type, union l5cm_specific_data *l5_data) 1383 { 1384 struct cnic_local *cp = dev->cnic_priv; 1385 struct l5cm_spe kwqe; 1386 struct kwqe_16 *kwq[1]; 1387 u16 type_16; 1388 int ret; 1389 1390 kwqe.hdr.conn_and_cmd_data = 1391 cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) | 1392 BNX2X_HW_CID(cp, cid))); 1393 1394 type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE; 1395 type_16 |= (cp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) & 1396 SPE_HDR_FUNCTION_ID; 1397 1398 kwqe.hdr.type = cpu_to_le16(type_16); 1399 kwqe.hdr.reserved1 = 0; 1400 kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo); 1401 kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi); 1402 1403 kwq[0] = (struct kwqe_16 *) &kwqe; 1404 1405 spin_lock_bh(&cp->cnic_ulp_lock); 1406 ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1); 1407 spin_unlock_bh(&cp->cnic_ulp_lock); 1408 1409 if (ret == 1) 1410 return 0; 1411 1412 return ret; 1413 } 1414 1415 static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type, 1416 struct kcqe *cqes[], u32 num_cqes) 1417 { 1418 struct cnic_local *cp = dev->cnic_priv; 1419 struct cnic_ulp_ops *ulp_ops; 1420 1421 rcu_read_lock(); 1422 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 1423 if (likely(ulp_ops)) { 1424 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 1425 cqes, num_cqes); 1426 } 1427 rcu_read_unlock(); 1428 } 1429 1430 static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe) 1431 { 1432 struct cnic_local *cp = dev->cnic_priv; 1433 struct bnx2x *bp = netdev_priv(dev->netdev); 1434 struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe; 1435 int hq_bds, pages; 1436 u32 pfid = cp->pfid; 1437 1438 cp->num_iscsi_tasks = req1->num_tasks_per_conn; 1439 cp->num_ccells = req1->num_ccells_per_conn; 1440 cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE * 1441 cp->num_iscsi_tasks; 1442 cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS * 1443 BNX2X_ISCSI_R2TQE_SIZE; 1444 cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE; 1445 pages = PAGE_ALIGN(cp->hq_size) / PAGE_SIZE; 1446 hq_bds = pages * (PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE); 1447 cp->num_cqs = req1->num_cqs; 1448 1449 if (!dev->max_iscsi_conn) 1450 return 0; 1451 1452 /* init Tstorm RAM */ 1453 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1454 req1->rq_num_wqes); 1455 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1456 PAGE_SIZE); 1457 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 1458 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1459 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1460 TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1461 req1->num_tasks_per_conn); 1462 1463 /* init Ustorm RAM */ 1464 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1465 USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid), 1466 req1->rq_buffer_size); 1467 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1468 PAGE_SIZE); 1469 CNIC_WR8(dev, BAR_USTRORM_INTMEM + 1470 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1471 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1472 USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1473 req1->num_tasks_per_conn); 1474 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1475 req1->rq_num_wqes); 1476 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1477 req1->cq_num_wqes); 1478 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1479 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1480 1481 /* init Xstorm RAM */ 1482 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1483 PAGE_SIZE); 1484 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1485 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1486 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 1487 XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1488 req1->num_tasks_per_conn); 1489 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1490 hq_bds); 1491 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid), 1492 req1->num_tasks_per_conn); 1493 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1494 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1495 1496 /* init Cstorm RAM */ 1497 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1498 PAGE_SIZE); 1499 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 1500 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1501 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1502 CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1503 req1->num_tasks_per_conn); 1504 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1505 req1->cq_num_wqes); 1506 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1507 hq_bds); 1508 1509 return 0; 1510 } 1511 1512 static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe) 1513 { 1514 struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe; 1515 struct cnic_local *cp = dev->cnic_priv; 1516 struct bnx2x *bp = netdev_priv(dev->netdev); 1517 u32 pfid = cp->pfid; 1518 struct iscsi_kcqe kcqe; 1519 struct kcqe *cqes[1]; 1520 1521 memset(&kcqe, 0, sizeof(kcqe)); 1522 if (!dev->max_iscsi_conn) { 1523 kcqe.completion_status = 1524 ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED; 1525 goto done; 1526 } 1527 1528 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1529 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1530 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1531 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1532 req2->error_bit_map[1]); 1533 1534 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1535 USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1536 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1537 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1538 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1539 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1540 req2->error_bit_map[1]); 1541 1542 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1543 CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1544 1545 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1546 1547 done: 1548 kcqe.op_code = ISCSI_KCQE_OPCODE_INIT; 1549 cqes[0] = (struct kcqe *) &kcqe; 1550 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1551 1552 return 0; 1553 } 1554 1555 static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1556 { 1557 struct cnic_local *cp = dev->cnic_priv; 1558 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1559 1560 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) { 1561 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1562 1563 cnic_free_dma(dev, &iscsi->hq_info); 1564 cnic_free_dma(dev, &iscsi->r2tq_info); 1565 cnic_free_dma(dev, &iscsi->task_array_info); 1566 cnic_free_id(&cp->cid_tbl, ctx->cid); 1567 } else { 1568 cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid); 1569 } 1570 1571 ctx->cid = 0; 1572 } 1573 1574 static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1575 { 1576 u32 cid; 1577 int ret, pages; 1578 struct cnic_local *cp = dev->cnic_priv; 1579 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1580 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1581 1582 if (ctx->ulp_proto_id == CNIC_ULP_FCOE) { 1583 cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl); 1584 if (cid == -1) { 1585 ret = -ENOMEM; 1586 goto error; 1587 } 1588 ctx->cid = cid; 1589 return 0; 1590 } 1591 1592 cid = cnic_alloc_new_id(&cp->cid_tbl); 1593 if (cid == -1) { 1594 ret = -ENOMEM; 1595 goto error; 1596 } 1597 1598 ctx->cid = cid; 1599 pages = PAGE_ALIGN(cp->task_array_size) / PAGE_SIZE; 1600 1601 ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1); 1602 if (ret) 1603 goto error; 1604 1605 pages = PAGE_ALIGN(cp->r2tq_size) / PAGE_SIZE; 1606 ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1); 1607 if (ret) 1608 goto error; 1609 1610 pages = PAGE_ALIGN(cp->hq_size) / PAGE_SIZE; 1611 ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1); 1612 if (ret) 1613 goto error; 1614 1615 return 0; 1616 1617 error: 1618 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1619 return ret; 1620 } 1621 1622 static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init, 1623 struct regpair *ctx_addr) 1624 { 1625 struct cnic_local *cp = dev->cnic_priv; 1626 struct cnic_eth_dev *ethdev = cp->ethdev; 1627 int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk; 1628 int off = (cid - ethdev->starting_cid) % cp->cids_per_blk; 1629 unsigned long align_off = 0; 1630 dma_addr_t ctx_map; 1631 void *ctx; 1632 1633 if (cp->ctx_align) { 1634 unsigned long mask = cp->ctx_align - 1; 1635 1636 if (cp->ctx_arr[blk].mapping & mask) 1637 align_off = cp->ctx_align - 1638 (cp->ctx_arr[blk].mapping & mask); 1639 } 1640 ctx_map = cp->ctx_arr[blk].mapping + align_off + 1641 (off * BNX2X_CONTEXT_MEM_SIZE); 1642 ctx = cp->ctx_arr[blk].ctx + align_off + 1643 (off * BNX2X_CONTEXT_MEM_SIZE); 1644 if (init) 1645 memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE); 1646 1647 ctx_addr->lo = ctx_map & 0xffffffff; 1648 ctx_addr->hi = (u64) ctx_map >> 32; 1649 return ctx; 1650 } 1651 1652 static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[], 1653 u32 num) 1654 { 1655 struct cnic_local *cp = dev->cnic_priv; 1656 struct iscsi_kwqe_conn_offload1 *req1 = 1657 (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1658 struct iscsi_kwqe_conn_offload2 *req2 = 1659 (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1660 struct iscsi_kwqe_conn_offload3 *req3; 1661 struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id]; 1662 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1663 u32 cid = ctx->cid; 1664 u32 hw_cid = BNX2X_HW_CID(cp, cid); 1665 struct iscsi_context *ictx; 1666 struct regpair context_addr; 1667 int i, j, n = 2, n_max; 1668 u8 port = CNIC_PORT(cp); 1669 1670 ctx->ctx_flags = 0; 1671 if (!req2->num_additional_wqes) 1672 return -EINVAL; 1673 1674 n_max = req2->num_additional_wqes + 2; 1675 1676 ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr); 1677 if (ictx == NULL) 1678 return -ENOMEM; 1679 1680 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1681 1682 ictx->xstorm_ag_context.hq_prod = 1; 1683 1684 ictx->xstorm_st_context.iscsi.first_burst_length = 1685 ISCSI_DEF_FIRST_BURST_LEN; 1686 ictx->xstorm_st_context.iscsi.max_send_pdu_length = 1687 ISCSI_DEF_MAX_RECV_SEG_LEN; 1688 ictx->xstorm_st_context.iscsi.sq_pbl_base.lo = 1689 req1->sq_page_table_addr_lo; 1690 ictx->xstorm_st_context.iscsi.sq_pbl_base.hi = 1691 req1->sq_page_table_addr_hi; 1692 ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi; 1693 ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo; 1694 ictx->xstorm_st_context.iscsi.hq_pbl_base.lo = 1695 iscsi->hq_info.pgtbl_map & 0xffffffff; 1696 ictx->xstorm_st_context.iscsi.hq_pbl_base.hi = 1697 (u64) iscsi->hq_info.pgtbl_map >> 32; 1698 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo = 1699 iscsi->hq_info.pgtbl[0]; 1700 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi = 1701 iscsi->hq_info.pgtbl[1]; 1702 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo = 1703 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1704 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi = 1705 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1706 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo = 1707 iscsi->r2tq_info.pgtbl[0]; 1708 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi = 1709 iscsi->r2tq_info.pgtbl[1]; 1710 ictx->xstorm_st_context.iscsi.task_pbl_base.lo = 1711 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1712 ictx->xstorm_st_context.iscsi.task_pbl_base.hi = 1713 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1714 ictx->xstorm_st_context.iscsi.task_pbl_cache_idx = 1715 BNX2X_ISCSI_PBL_NOT_CACHED; 1716 ictx->xstorm_st_context.iscsi.flags.flags |= 1717 XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA; 1718 ictx->xstorm_st_context.iscsi.flags.flags |= 1719 XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T; 1720 ictx->xstorm_st_context.common.ethernet.reserved_vlan_type = 1721 ETH_P_8021Q; 1722 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id) && 1723 cp->port_mode == CHIP_2_PORT_MODE) { 1724 1725 port = 0; 1726 } 1727 ictx->xstorm_st_context.common.flags = 1728 1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT; 1729 ictx->xstorm_st_context.common.flags = 1730 port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT; 1731 1732 ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE; 1733 /* TSTORM requires the base address of RQ DB & not PTE */ 1734 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo = 1735 req2->rq_page_table_addr_lo & PAGE_MASK; 1736 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi = 1737 req2->rq_page_table_addr_hi; 1738 ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id; 1739 ictx->tstorm_st_context.tcp.cwnd = 0x5A8; 1740 ictx->tstorm_st_context.tcp.flags2 |= 1741 TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN; 1742 ictx->tstorm_st_context.tcp.ooo_support_mode = 1743 TCP_TSTORM_OOO_DROP_AND_PROC_ACK; 1744 1745 ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG; 1746 1747 ictx->ustorm_st_context.ring.rq.pbl_base.lo = 1748 req2->rq_page_table_addr_lo; 1749 ictx->ustorm_st_context.ring.rq.pbl_base.hi = 1750 req2->rq_page_table_addr_hi; 1751 ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi; 1752 ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo; 1753 ictx->ustorm_st_context.ring.r2tq.pbl_base.lo = 1754 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1755 ictx->ustorm_st_context.ring.r2tq.pbl_base.hi = 1756 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1757 ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo = 1758 iscsi->r2tq_info.pgtbl[0]; 1759 ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi = 1760 iscsi->r2tq_info.pgtbl[1]; 1761 ictx->ustorm_st_context.ring.cq_pbl_base.lo = 1762 req1->cq_page_table_addr_lo; 1763 ictx->ustorm_st_context.ring.cq_pbl_base.hi = 1764 req1->cq_page_table_addr_hi; 1765 ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN; 1766 ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi; 1767 ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo; 1768 ictx->ustorm_st_context.task_pbe_cache_index = 1769 BNX2X_ISCSI_PBL_NOT_CACHED; 1770 ictx->ustorm_st_context.task_pdu_cache_index = 1771 BNX2X_ISCSI_PDU_HEADER_NOT_CACHED; 1772 1773 for (i = 1, j = 1; i < cp->num_cqs; i++, j++) { 1774 if (j == 3) { 1775 if (n >= n_max) 1776 break; 1777 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1778 j = 0; 1779 } 1780 ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN; 1781 ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo = 1782 req3->qp_first_pte[j].hi; 1783 ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi = 1784 req3->qp_first_pte[j].lo; 1785 } 1786 1787 ictx->ustorm_st_context.task_pbl_base.lo = 1788 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1789 ictx->ustorm_st_context.task_pbl_base.hi = 1790 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1791 ictx->ustorm_st_context.tce_phy_addr.lo = 1792 iscsi->task_array_info.pgtbl[0]; 1793 ictx->ustorm_st_context.tce_phy_addr.hi = 1794 iscsi->task_array_info.pgtbl[1]; 1795 ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1796 ictx->ustorm_st_context.num_cqs = cp->num_cqs; 1797 ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN; 1798 ictx->ustorm_st_context.negotiated_rx_and_flags |= 1799 ISCSI_DEF_MAX_BURST_LEN; 1800 ictx->ustorm_st_context.negotiated_rx |= 1801 ISCSI_DEFAULT_MAX_OUTSTANDING_R2T << 1802 USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT; 1803 1804 ictx->cstorm_st_context.hq_pbl_base.lo = 1805 iscsi->hq_info.pgtbl_map & 0xffffffff; 1806 ictx->cstorm_st_context.hq_pbl_base.hi = 1807 (u64) iscsi->hq_info.pgtbl_map >> 32; 1808 ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0]; 1809 ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1]; 1810 ictx->cstorm_st_context.task_pbl_base.lo = 1811 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1812 ictx->cstorm_st_context.task_pbl_base.hi = 1813 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1814 /* CSTORM and USTORM initialization is different, CSTORM requires 1815 * CQ DB base & not PTE addr */ 1816 ictx->cstorm_st_context.cq_db_base.lo = 1817 req1->cq_page_table_addr_lo & PAGE_MASK; 1818 ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi; 1819 ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1820 ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1; 1821 for (i = 0; i < cp->num_cqs; i++) { 1822 ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] = 1823 ISCSI_INITIAL_SN; 1824 ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] = 1825 ISCSI_INITIAL_SN; 1826 } 1827 1828 ictx->xstorm_ag_context.cdu_reserved = 1829 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 1830 ISCSI_CONNECTION_TYPE); 1831 ictx->ustorm_ag_context.cdu_usage = 1832 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 1833 ISCSI_CONNECTION_TYPE); 1834 return 0; 1835 1836 } 1837 1838 static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 1839 u32 num, int *work) 1840 { 1841 struct iscsi_kwqe_conn_offload1 *req1; 1842 struct iscsi_kwqe_conn_offload2 *req2; 1843 struct cnic_local *cp = dev->cnic_priv; 1844 struct cnic_context *ctx; 1845 struct iscsi_kcqe kcqe; 1846 struct kcqe *cqes[1]; 1847 u32 l5_cid; 1848 int ret = 0; 1849 1850 if (num < 2) { 1851 *work = num; 1852 return -EINVAL; 1853 } 1854 1855 req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1856 req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1857 if ((num - 2) < req2->num_additional_wqes) { 1858 *work = num; 1859 return -EINVAL; 1860 } 1861 *work = 2 + req2->num_additional_wqes; 1862 1863 l5_cid = req1->iscsi_conn_id; 1864 if (l5_cid >= MAX_ISCSI_TBL_SZ) 1865 return -EINVAL; 1866 1867 memset(&kcqe, 0, sizeof(kcqe)); 1868 kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN; 1869 kcqe.iscsi_conn_id = l5_cid; 1870 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 1871 1872 ctx = &cp->ctx_tbl[l5_cid]; 1873 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) { 1874 kcqe.completion_status = 1875 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY; 1876 goto done; 1877 } 1878 1879 if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) { 1880 atomic_dec(&cp->iscsi_conn); 1881 goto done; 1882 } 1883 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 1884 if (ret) { 1885 atomic_dec(&cp->iscsi_conn); 1886 ret = 0; 1887 goto done; 1888 } 1889 ret = cnic_setup_bnx2x_ctx(dev, wqes, num); 1890 if (ret < 0) { 1891 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1892 atomic_dec(&cp->iscsi_conn); 1893 goto done; 1894 } 1895 1896 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1897 kcqe.iscsi_conn_context_id = BNX2X_HW_CID(cp, cp->ctx_tbl[l5_cid].cid); 1898 1899 done: 1900 cqes[0] = (struct kcqe *) &kcqe; 1901 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1902 return 0; 1903 } 1904 1905 1906 static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe) 1907 { 1908 struct cnic_local *cp = dev->cnic_priv; 1909 struct iscsi_kwqe_conn_update *req = 1910 (struct iscsi_kwqe_conn_update *) kwqe; 1911 void *data; 1912 union l5cm_specific_data l5_data; 1913 u32 l5_cid, cid = BNX2X_SW_CID(req->context_id); 1914 int ret; 1915 1916 if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0) 1917 return -EINVAL; 1918 1919 data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 1920 if (!data) 1921 return -ENOMEM; 1922 1923 memcpy(data, kwqe, sizeof(struct kwqe)); 1924 1925 ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, 1926 req->context_id, ISCSI_CONNECTION_TYPE, &l5_data); 1927 return ret; 1928 } 1929 1930 static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid) 1931 { 1932 struct cnic_local *cp = dev->cnic_priv; 1933 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1934 union l5cm_specific_data l5_data; 1935 int ret; 1936 u32 hw_cid; 1937 1938 init_waitqueue_head(&ctx->waitq); 1939 ctx->wait_cond = 0; 1940 memset(&l5_data, 0, sizeof(l5_data)); 1941 hw_cid = BNX2X_HW_CID(cp, ctx->cid); 1942 1943 ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 1944 hw_cid, NONE_CONNECTION_TYPE, &l5_data); 1945 1946 if (ret == 0) { 1947 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 1948 if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags))) 1949 return -EBUSY; 1950 } 1951 1952 return 0; 1953 } 1954 1955 static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 1956 { 1957 struct cnic_local *cp = dev->cnic_priv; 1958 struct iscsi_kwqe_conn_destroy *req = 1959 (struct iscsi_kwqe_conn_destroy *) kwqe; 1960 u32 l5_cid = req->reserved0; 1961 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1962 int ret = 0; 1963 struct iscsi_kcqe kcqe; 1964 struct kcqe *cqes[1]; 1965 1966 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 1967 goto skip_cfc_delete; 1968 1969 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 1970 unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies; 1971 1972 if (delta > (2 * HZ)) 1973 delta = 0; 1974 1975 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 1976 queue_delayed_work(cnic_wq, &cp->delete_task, delta); 1977 goto destroy_reply; 1978 } 1979 1980 ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid); 1981 1982 skip_cfc_delete: 1983 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1984 1985 if (!ret) { 1986 atomic_dec(&cp->iscsi_conn); 1987 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 1988 } 1989 1990 destroy_reply: 1991 memset(&kcqe, 0, sizeof(kcqe)); 1992 kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN; 1993 kcqe.iscsi_conn_id = l5_cid; 1994 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1995 kcqe.iscsi_conn_context_id = req->context_id; 1996 1997 cqes[0] = (struct kcqe *) &kcqe; 1998 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1999 2000 return 0; 2001 } 2002 2003 static void cnic_init_storm_conn_bufs(struct cnic_dev *dev, 2004 struct l4_kwq_connect_req1 *kwqe1, 2005 struct l4_kwq_connect_req3 *kwqe3, 2006 struct l5cm_active_conn_buffer *conn_buf) 2007 { 2008 struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf; 2009 struct l5cm_xstorm_conn_buffer *xstorm_buf = 2010 &conn_buf->xstorm_conn_buffer; 2011 struct l5cm_tstorm_conn_buffer *tstorm_buf = 2012 &conn_buf->tstorm_conn_buffer; 2013 struct regpair context_addr; 2014 u32 cid = BNX2X_SW_CID(kwqe1->cid); 2015 struct in6_addr src_ip, dst_ip; 2016 int i; 2017 u32 *addrp; 2018 2019 addrp = (u32 *) &conn_addr->local_ip_addr; 2020 for (i = 0; i < 4; i++, addrp++) 2021 src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2022 2023 addrp = (u32 *) &conn_addr->remote_ip_addr; 2024 for (i = 0; i < 4; i++, addrp++) 2025 dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2026 2027 cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr); 2028 2029 xstorm_buf->context_addr.hi = context_addr.hi; 2030 xstorm_buf->context_addr.lo = context_addr.lo; 2031 xstorm_buf->mss = 0xffff; 2032 xstorm_buf->rcv_buf = kwqe3->rcv_buf; 2033 if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE) 2034 xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE; 2035 xstorm_buf->pseudo_header_checksum = 2036 swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0)); 2037 2038 if (!(kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK)) 2039 tstorm_buf->params |= 2040 L5CM_TSTORM_CONN_BUFFER_DELAYED_ACK_ENABLE; 2041 if (kwqe3->ka_timeout) { 2042 tstorm_buf->ka_enable = 1; 2043 tstorm_buf->ka_timeout = kwqe3->ka_timeout; 2044 tstorm_buf->ka_interval = kwqe3->ka_interval; 2045 tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count; 2046 } 2047 tstorm_buf->max_rt_time = 0xffffffff; 2048 } 2049 2050 static void cnic_init_bnx2x_mac(struct cnic_dev *dev) 2051 { 2052 struct cnic_local *cp = dev->cnic_priv; 2053 struct bnx2x *bp = netdev_priv(dev->netdev); 2054 u32 pfid = cp->pfid; 2055 u8 *mac = dev->mac_addr; 2056 2057 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2058 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]); 2059 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2060 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]); 2061 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2062 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]); 2063 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2064 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]); 2065 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2066 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]); 2067 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2068 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]); 2069 2070 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2071 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]); 2072 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2073 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2074 mac[4]); 2075 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2076 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]); 2077 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2078 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2079 mac[2]); 2080 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2081 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]); 2082 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2083 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2084 mac[0]); 2085 } 2086 2087 static void cnic_bnx2x_set_tcp_timestamp(struct cnic_dev *dev, int tcp_ts) 2088 { 2089 struct cnic_local *cp = dev->cnic_priv; 2090 struct bnx2x *bp = netdev_priv(dev->netdev); 2091 u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN; 2092 u16 tstorm_flags = 0; 2093 2094 if (tcp_ts) { 2095 xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 2096 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 2097 } 2098 2099 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2100 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(cp->pfid), xstorm_flags); 2101 2102 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 2103 TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(cp->pfid), tstorm_flags); 2104 } 2105 2106 static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[], 2107 u32 num, int *work) 2108 { 2109 struct cnic_local *cp = dev->cnic_priv; 2110 struct bnx2x *bp = netdev_priv(dev->netdev); 2111 struct l4_kwq_connect_req1 *kwqe1 = 2112 (struct l4_kwq_connect_req1 *) wqes[0]; 2113 struct l4_kwq_connect_req3 *kwqe3; 2114 struct l5cm_active_conn_buffer *conn_buf; 2115 struct l5cm_conn_addr_params *conn_addr; 2116 union l5cm_specific_data l5_data; 2117 u32 l5_cid = kwqe1->pg_cid; 2118 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 2119 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2120 int ret; 2121 2122 if (num < 2) { 2123 *work = num; 2124 return -EINVAL; 2125 } 2126 2127 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) 2128 *work = 3; 2129 else 2130 *work = 2; 2131 2132 if (num < *work) { 2133 *work = num; 2134 return -EINVAL; 2135 } 2136 2137 if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) { 2138 netdev_err(dev->netdev, "conn_buf size too big\n"); 2139 return -ENOMEM; 2140 } 2141 conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2142 if (!conn_buf) 2143 return -ENOMEM; 2144 2145 memset(conn_buf, 0, sizeof(*conn_buf)); 2146 2147 conn_addr = &conn_buf->conn_addr_buf; 2148 conn_addr->remote_addr_0 = csk->ha[0]; 2149 conn_addr->remote_addr_1 = csk->ha[1]; 2150 conn_addr->remote_addr_2 = csk->ha[2]; 2151 conn_addr->remote_addr_3 = csk->ha[3]; 2152 conn_addr->remote_addr_4 = csk->ha[4]; 2153 conn_addr->remote_addr_5 = csk->ha[5]; 2154 2155 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) { 2156 struct l4_kwq_connect_req2 *kwqe2 = 2157 (struct l4_kwq_connect_req2 *) wqes[1]; 2158 2159 conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4; 2160 conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3; 2161 conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2; 2162 2163 conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4; 2164 conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3; 2165 conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2; 2166 conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION; 2167 } 2168 kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1]; 2169 2170 conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip; 2171 conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip; 2172 conn_addr->local_tcp_port = kwqe1->src_port; 2173 conn_addr->remote_tcp_port = kwqe1->dst_port; 2174 2175 conn_addr->pmtu = kwqe3->pmtu; 2176 cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf); 2177 2178 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 2179 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(cp->pfid), csk->vlan_id); 2180 2181 cnic_bnx2x_set_tcp_timestamp(dev, 2182 kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_TIME_STAMP); 2183 2184 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT, 2185 kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2186 if (!ret) 2187 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2188 2189 return ret; 2190 } 2191 2192 static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe) 2193 { 2194 struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe; 2195 union l5cm_specific_data l5_data; 2196 int ret; 2197 2198 memset(&l5_data, 0, sizeof(l5_data)); 2199 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE, 2200 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2201 return ret; 2202 } 2203 2204 static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe) 2205 { 2206 struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe; 2207 union l5cm_specific_data l5_data; 2208 int ret; 2209 2210 memset(&l5_data, 0, sizeof(l5_data)); 2211 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT, 2212 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2213 return ret; 2214 } 2215 static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2216 { 2217 struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe; 2218 struct l4_kcq kcqe; 2219 struct kcqe *cqes[1]; 2220 2221 memset(&kcqe, 0, sizeof(kcqe)); 2222 kcqe.pg_host_opaque = req->host_opaque; 2223 kcqe.pg_cid = req->host_opaque; 2224 kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG; 2225 cqes[0] = (struct kcqe *) &kcqe; 2226 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2227 return 0; 2228 } 2229 2230 static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2231 { 2232 struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe; 2233 struct l4_kcq kcqe; 2234 struct kcqe *cqes[1]; 2235 2236 memset(&kcqe, 0, sizeof(kcqe)); 2237 kcqe.pg_host_opaque = req->pg_host_opaque; 2238 kcqe.pg_cid = req->pg_cid; 2239 kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG; 2240 cqes[0] = (struct kcqe *) &kcqe; 2241 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2242 return 0; 2243 } 2244 2245 static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe) 2246 { 2247 struct fcoe_kwqe_stat *req; 2248 struct fcoe_stat_ramrod_params *fcoe_stat; 2249 union l5cm_specific_data l5_data; 2250 struct cnic_local *cp = dev->cnic_priv; 2251 int ret; 2252 u32 cid; 2253 2254 req = (struct fcoe_kwqe_stat *) kwqe; 2255 cid = BNX2X_HW_CID(cp, cp->fcoe_init_cid); 2256 2257 fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2258 if (!fcoe_stat) 2259 return -ENOMEM; 2260 2261 memset(fcoe_stat, 0, sizeof(*fcoe_stat)); 2262 memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req)); 2263 2264 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid, 2265 FCOE_CONNECTION_TYPE, &l5_data); 2266 return ret; 2267 } 2268 2269 static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[], 2270 u32 num, int *work) 2271 { 2272 int ret; 2273 struct cnic_local *cp = dev->cnic_priv; 2274 u32 cid; 2275 struct fcoe_init_ramrod_params *fcoe_init; 2276 struct fcoe_kwqe_init1 *req1; 2277 struct fcoe_kwqe_init2 *req2; 2278 struct fcoe_kwqe_init3 *req3; 2279 union l5cm_specific_data l5_data; 2280 2281 if (num < 3) { 2282 *work = num; 2283 return -EINVAL; 2284 } 2285 req1 = (struct fcoe_kwqe_init1 *) wqes[0]; 2286 req2 = (struct fcoe_kwqe_init2 *) wqes[1]; 2287 req3 = (struct fcoe_kwqe_init3 *) wqes[2]; 2288 if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) { 2289 *work = 1; 2290 return -EINVAL; 2291 } 2292 if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) { 2293 *work = 2; 2294 return -EINVAL; 2295 } 2296 2297 if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) { 2298 netdev_err(dev->netdev, "fcoe_init size too big\n"); 2299 return -ENOMEM; 2300 } 2301 fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2302 if (!fcoe_init) 2303 return -ENOMEM; 2304 2305 memset(fcoe_init, 0, sizeof(*fcoe_init)); 2306 memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1)); 2307 memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2)); 2308 memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3)); 2309 fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff; 2310 fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32; 2311 fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages; 2312 2313 fcoe_init->sb_num = cp->status_blk_num; 2314 fcoe_init->eq_prod = MAX_KCQ_IDX; 2315 fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS; 2316 cp->kcq2.sw_prod_idx = 0; 2317 2318 cid = BNX2X_HW_CID(cp, cp->fcoe_init_cid); 2319 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid, 2320 FCOE_CONNECTION_TYPE, &l5_data); 2321 *work = 3; 2322 return ret; 2323 } 2324 2325 static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 2326 u32 num, int *work) 2327 { 2328 int ret = 0; 2329 u32 cid = -1, l5_cid; 2330 struct cnic_local *cp = dev->cnic_priv; 2331 struct fcoe_kwqe_conn_offload1 *req1; 2332 struct fcoe_kwqe_conn_offload2 *req2; 2333 struct fcoe_kwqe_conn_offload3 *req3; 2334 struct fcoe_kwqe_conn_offload4 *req4; 2335 struct fcoe_conn_offload_ramrod_params *fcoe_offload; 2336 struct cnic_context *ctx; 2337 struct fcoe_context *fctx; 2338 struct regpair ctx_addr; 2339 union l5cm_specific_data l5_data; 2340 struct fcoe_kcqe kcqe; 2341 struct kcqe *cqes[1]; 2342 2343 if (num < 4) { 2344 *work = num; 2345 return -EINVAL; 2346 } 2347 req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0]; 2348 req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1]; 2349 req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2]; 2350 req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3]; 2351 2352 *work = 4; 2353 2354 l5_cid = req1->fcoe_conn_id; 2355 if (l5_cid >= dev->max_fcoe_conn) 2356 goto err_reply; 2357 2358 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2359 2360 ctx = &cp->ctx_tbl[l5_cid]; 2361 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2362 goto err_reply; 2363 2364 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 2365 if (ret) { 2366 ret = 0; 2367 goto err_reply; 2368 } 2369 cid = ctx->cid; 2370 2371 fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr); 2372 if (fctx) { 2373 u32 hw_cid = BNX2X_HW_CID(cp, cid); 2374 u32 val; 2375 2376 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 2377 FCOE_CONNECTION_TYPE); 2378 fctx->xstorm_ag_context.cdu_reserved = val; 2379 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 2380 FCOE_CONNECTION_TYPE); 2381 fctx->ustorm_ag_context.cdu_usage = val; 2382 } 2383 if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) { 2384 netdev_err(dev->netdev, "fcoe_offload size too big\n"); 2385 goto err_reply; 2386 } 2387 fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2388 if (!fcoe_offload) 2389 goto err_reply; 2390 2391 memset(fcoe_offload, 0, sizeof(*fcoe_offload)); 2392 memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1)); 2393 memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2)); 2394 memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3)); 2395 memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4)); 2396 2397 cid = BNX2X_HW_CID(cp, cid); 2398 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid, 2399 FCOE_CONNECTION_TYPE, &l5_data); 2400 if (!ret) 2401 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2402 2403 return ret; 2404 2405 err_reply: 2406 if (cid != -1) 2407 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2408 2409 memset(&kcqe, 0, sizeof(kcqe)); 2410 kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN; 2411 kcqe.fcoe_conn_id = req1->fcoe_conn_id; 2412 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 2413 2414 cqes[0] = (struct kcqe *) &kcqe; 2415 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2416 return ret; 2417 } 2418 2419 static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe) 2420 { 2421 struct fcoe_kwqe_conn_enable_disable *req; 2422 struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable; 2423 union l5cm_specific_data l5_data; 2424 int ret; 2425 u32 cid, l5_cid; 2426 struct cnic_local *cp = dev->cnic_priv; 2427 2428 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2429 cid = req->context_id; 2430 l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE; 2431 2432 if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) { 2433 netdev_err(dev->netdev, "fcoe_enable size too big\n"); 2434 return -ENOMEM; 2435 } 2436 fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2437 if (!fcoe_enable) 2438 return -ENOMEM; 2439 2440 memset(fcoe_enable, 0, sizeof(*fcoe_enable)); 2441 memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req)); 2442 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid, 2443 FCOE_CONNECTION_TYPE, &l5_data); 2444 return ret; 2445 } 2446 2447 static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe) 2448 { 2449 struct fcoe_kwqe_conn_enable_disable *req; 2450 struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable; 2451 union l5cm_specific_data l5_data; 2452 int ret; 2453 u32 cid, l5_cid; 2454 struct cnic_local *cp = dev->cnic_priv; 2455 2456 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2457 cid = req->context_id; 2458 l5_cid = req->conn_id; 2459 if (l5_cid >= dev->max_fcoe_conn) 2460 return -EINVAL; 2461 2462 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2463 2464 if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) { 2465 netdev_err(dev->netdev, "fcoe_disable size too big\n"); 2466 return -ENOMEM; 2467 } 2468 fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2469 if (!fcoe_disable) 2470 return -ENOMEM; 2471 2472 memset(fcoe_disable, 0, sizeof(*fcoe_disable)); 2473 memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req)); 2474 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid, 2475 FCOE_CONNECTION_TYPE, &l5_data); 2476 return ret; 2477 } 2478 2479 static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2480 { 2481 struct fcoe_kwqe_conn_destroy *req; 2482 union l5cm_specific_data l5_data; 2483 int ret; 2484 u32 cid, l5_cid; 2485 struct cnic_local *cp = dev->cnic_priv; 2486 struct cnic_context *ctx; 2487 struct fcoe_kcqe kcqe; 2488 struct kcqe *cqes[1]; 2489 2490 req = (struct fcoe_kwqe_conn_destroy *) kwqe; 2491 cid = req->context_id; 2492 l5_cid = req->conn_id; 2493 if (l5_cid >= dev->max_fcoe_conn) 2494 return -EINVAL; 2495 2496 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2497 2498 ctx = &cp->ctx_tbl[l5_cid]; 2499 2500 init_waitqueue_head(&ctx->waitq); 2501 ctx->wait_cond = 0; 2502 2503 memset(&kcqe, 0, sizeof(kcqe)); 2504 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR; 2505 memset(&l5_data, 0, sizeof(l5_data)); 2506 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid, 2507 FCOE_CONNECTION_TYPE, &l5_data); 2508 if (ret == 0) { 2509 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 2510 if (ctx->wait_cond) 2511 kcqe.completion_status = 0; 2512 } 2513 2514 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2515 queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000)); 2516 2517 kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN; 2518 kcqe.fcoe_conn_id = req->conn_id; 2519 kcqe.fcoe_conn_context_id = cid; 2520 2521 cqes[0] = (struct kcqe *) &kcqe; 2522 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2523 return ret; 2524 } 2525 2526 static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid) 2527 { 2528 struct cnic_local *cp = dev->cnic_priv; 2529 u32 i; 2530 2531 for (i = start_cid; i < cp->max_cid_space; i++) { 2532 struct cnic_context *ctx = &cp->ctx_tbl[i]; 2533 int j; 2534 2535 while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 2536 msleep(10); 2537 2538 for (j = 0; j < 5; j++) { 2539 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2540 break; 2541 msleep(20); 2542 } 2543 2544 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2545 netdev_warn(dev->netdev, "CID %x not deleted\n", 2546 ctx->cid); 2547 } 2548 } 2549 2550 static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2551 { 2552 struct fcoe_kwqe_destroy *req; 2553 union l5cm_specific_data l5_data; 2554 struct cnic_local *cp = dev->cnic_priv; 2555 int ret; 2556 u32 cid; 2557 2558 cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ); 2559 2560 req = (struct fcoe_kwqe_destroy *) kwqe; 2561 cid = BNX2X_HW_CID(cp, cp->fcoe_init_cid); 2562 2563 memset(&l5_data, 0, sizeof(l5_data)); 2564 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid, 2565 FCOE_CONNECTION_TYPE, &l5_data); 2566 return ret; 2567 } 2568 2569 static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe) 2570 { 2571 struct cnic_local *cp = dev->cnic_priv; 2572 struct kcqe kcqe; 2573 struct kcqe *cqes[1]; 2574 u32 cid; 2575 u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2576 u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK; 2577 u32 kcqe_op; 2578 int ulp_type; 2579 2580 cid = kwqe->kwqe_info0; 2581 memset(&kcqe, 0, sizeof(kcqe)); 2582 2583 if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) { 2584 u32 l5_cid = 0; 2585 2586 ulp_type = CNIC_ULP_FCOE; 2587 if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) { 2588 struct fcoe_kwqe_conn_enable_disable *req; 2589 2590 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2591 kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN; 2592 cid = req->context_id; 2593 l5_cid = req->conn_id; 2594 } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) { 2595 kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC; 2596 } else { 2597 return; 2598 } 2599 kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT; 2600 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE; 2601 kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2602 kcqe.kcqe_info2 = cid; 2603 kcqe.kcqe_info0 = l5_cid; 2604 2605 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) { 2606 ulp_type = CNIC_ULP_ISCSI; 2607 if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN) 2608 cid = kwqe->kwqe_info1; 2609 2610 kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT; 2611 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI; 2612 kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR; 2613 kcqe.kcqe_info2 = cid; 2614 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0); 2615 2616 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) { 2617 struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe; 2618 2619 ulp_type = CNIC_ULP_L4; 2620 if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1) 2621 kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE; 2622 else if (opcode == L4_KWQE_OPCODE_VALUE_RESET) 2623 kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP; 2624 else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE) 2625 kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 2626 else 2627 return; 2628 2629 kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) | 2630 KCQE_FLAGS_LAYER_MASK_L4; 2631 l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2632 l4kcqe->cid = cid; 2633 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id); 2634 } else { 2635 return; 2636 } 2637 2638 cqes[0] = &kcqe; 2639 cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1); 2640 } 2641 2642 static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev, 2643 struct kwqe *wqes[], u32 num_wqes) 2644 { 2645 int i, work, ret; 2646 u32 opcode; 2647 struct kwqe *kwqe; 2648 2649 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2650 return -EAGAIN; /* bnx2 is down */ 2651 2652 for (i = 0; i < num_wqes; ) { 2653 kwqe = wqes[i]; 2654 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2655 work = 1; 2656 2657 switch (opcode) { 2658 case ISCSI_KWQE_OPCODE_INIT1: 2659 ret = cnic_bnx2x_iscsi_init1(dev, kwqe); 2660 break; 2661 case ISCSI_KWQE_OPCODE_INIT2: 2662 ret = cnic_bnx2x_iscsi_init2(dev, kwqe); 2663 break; 2664 case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1: 2665 ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i], 2666 num_wqes - i, &work); 2667 break; 2668 case ISCSI_KWQE_OPCODE_UPDATE_CONN: 2669 ret = cnic_bnx2x_iscsi_update(dev, kwqe); 2670 break; 2671 case ISCSI_KWQE_OPCODE_DESTROY_CONN: 2672 ret = cnic_bnx2x_iscsi_destroy(dev, kwqe); 2673 break; 2674 case L4_KWQE_OPCODE_VALUE_CONNECT1: 2675 ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i, 2676 &work); 2677 break; 2678 case L4_KWQE_OPCODE_VALUE_CLOSE: 2679 ret = cnic_bnx2x_close(dev, kwqe); 2680 break; 2681 case L4_KWQE_OPCODE_VALUE_RESET: 2682 ret = cnic_bnx2x_reset(dev, kwqe); 2683 break; 2684 case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG: 2685 ret = cnic_bnx2x_offload_pg(dev, kwqe); 2686 break; 2687 case L4_KWQE_OPCODE_VALUE_UPDATE_PG: 2688 ret = cnic_bnx2x_update_pg(dev, kwqe); 2689 break; 2690 case L4_KWQE_OPCODE_VALUE_UPLOAD_PG: 2691 ret = 0; 2692 break; 2693 default: 2694 ret = 0; 2695 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2696 opcode); 2697 break; 2698 } 2699 if (ret < 0) { 2700 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2701 opcode); 2702 2703 /* Possibly bnx2x parity error, send completion 2704 * to ulp drivers with error code to speed up 2705 * cleanup and reset recovery. 2706 */ 2707 if (ret == -EIO || ret == -EAGAIN) 2708 cnic_bnx2x_kwqe_err(dev, kwqe); 2709 } 2710 i += work; 2711 } 2712 return 0; 2713 } 2714 2715 static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev, 2716 struct kwqe *wqes[], u32 num_wqes) 2717 { 2718 struct cnic_local *cp = dev->cnic_priv; 2719 int i, work, ret; 2720 u32 opcode; 2721 struct kwqe *kwqe; 2722 2723 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2724 return -EAGAIN; /* bnx2 is down */ 2725 2726 if (!BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) 2727 return -EINVAL; 2728 2729 for (i = 0; i < num_wqes; ) { 2730 kwqe = wqes[i]; 2731 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2732 work = 1; 2733 2734 switch (opcode) { 2735 case FCOE_KWQE_OPCODE_INIT1: 2736 ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i], 2737 num_wqes - i, &work); 2738 break; 2739 case FCOE_KWQE_OPCODE_OFFLOAD_CONN1: 2740 ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i], 2741 num_wqes - i, &work); 2742 break; 2743 case FCOE_KWQE_OPCODE_ENABLE_CONN: 2744 ret = cnic_bnx2x_fcoe_enable(dev, kwqe); 2745 break; 2746 case FCOE_KWQE_OPCODE_DISABLE_CONN: 2747 ret = cnic_bnx2x_fcoe_disable(dev, kwqe); 2748 break; 2749 case FCOE_KWQE_OPCODE_DESTROY_CONN: 2750 ret = cnic_bnx2x_fcoe_destroy(dev, kwqe); 2751 break; 2752 case FCOE_KWQE_OPCODE_DESTROY: 2753 ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe); 2754 break; 2755 case FCOE_KWQE_OPCODE_STAT: 2756 ret = cnic_bnx2x_fcoe_stat(dev, kwqe); 2757 break; 2758 default: 2759 ret = 0; 2760 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2761 opcode); 2762 break; 2763 } 2764 if (ret < 0) { 2765 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2766 opcode); 2767 2768 /* Possibly bnx2x parity error, send completion 2769 * to ulp drivers with error code to speed up 2770 * cleanup and reset recovery. 2771 */ 2772 if (ret == -EIO || ret == -EAGAIN) 2773 cnic_bnx2x_kwqe_err(dev, kwqe); 2774 } 2775 i += work; 2776 } 2777 return 0; 2778 } 2779 2780 static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 2781 u32 num_wqes) 2782 { 2783 int ret = -EINVAL; 2784 u32 layer_code; 2785 2786 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2787 return -EAGAIN; /* bnx2x is down */ 2788 2789 if (!num_wqes) 2790 return 0; 2791 2792 layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK; 2793 switch (layer_code) { 2794 case KWQE_FLAGS_LAYER_MASK_L5_ISCSI: 2795 case KWQE_FLAGS_LAYER_MASK_L4: 2796 case KWQE_FLAGS_LAYER_MASK_L2: 2797 ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes); 2798 break; 2799 2800 case KWQE_FLAGS_LAYER_MASK_L5_FCOE: 2801 ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes); 2802 break; 2803 } 2804 return ret; 2805 } 2806 2807 static inline u32 cnic_get_kcqe_layer_mask(u32 opflag) 2808 { 2809 if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN)) 2810 return KCQE_FLAGS_LAYER_MASK_L4; 2811 2812 return opflag & KCQE_FLAGS_LAYER_MASK; 2813 } 2814 2815 static void service_kcqes(struct cnic_dev *dev, int num_cqes) 2816 { 2817 struct cnic_local *cp = dev->cnic_priv; 2818 int i, j, comp = 0; 2819 2820 i = 0; 2821 j = 1; 2822 while (num_cqes) { 2823 struct cnic_ulp_ops *ulp_ops; 2824 int ulp_type; 2825 u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag; 2826 u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag); 2827 2828 if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION)) 2829 comp++; 2830 2831 while (j < num_cqes) { 2832 u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag; 2833 2834 if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer) 2835 break; 2836 2837 if (unlikely(next_op & KCQE_RAMROD_COMPLETION)) 2838 comp++; 2839 j++; 2840 } 2841 2842 if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA) 2843 ulp_type = CNIC_ULP_RDMA; 2844 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI) 2845 ulp_type = CNIC_ULP_ISCSI; 2846 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE) 2847 ulp_type = CNIC_ULP_FCOE; 2848 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4) 2849 ulp_type = CNIC_ULP_L4; 2850 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2) 2851 goto end; 2852 else { 2853 netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n", 2854 kcqe_op_flag); 2855 goto end; 2856 } 2857 2858 rcu_read_lock(); 2859 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 2860 if (likely(ulp_ops)) { 2861 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 2862 cp->completed_kcq + i, j); 2863 } 2864 rcu_read_unlock(); 2865 end: 2866 num_cqes -= j; 2867 i += j; 2868 j = 1; 2869 } 2870 if (unlikely(comp)) 2871 cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp); 2872 } 2873 2874 static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info) 2875 { 2876 struct cnic_local *cp = dev->cnic_priv; 2877 u16 i, ri, hw_prod, last; 2878 struct kcqe *kcqe; 2879 int kcqe_cnt = 0, last_cnt = 0; 2880 2881 i = ri = last = info->sw_prod_idx; 2882 ri &= MAX_KCQ_IDX; 2883 hw_prod = *info->hw_prod_idx_ptr; 2884 hw_prod = info->hw_idx(hw_prod); 2885 2886 while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) { 2887 kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)]; 2888 cp->completed_kcq[kcqe_cnt++] = kcqe; 2889 i = info->next_idx(i); 2890 ri = i & MAX_KCQ_IDX; 2891 if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) { 2892 last_cnt = kcqe_cnt; 2893 last = i; 2894 } 2895 } 2896 2897 info->sw_prod_idx = last; 2898 return last_cnt; 2899 } 2900 2901 static int cnic_l2_completion(struct cnic_local *cp) 2902 { 2903 u16 hw_cons, sw_cons; 2904 struct cnic_uio_dev *udev = cp->udev; 2905 union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *) 2906 (udev->l2_ring + (2 * BNX2_PAGE_SIZE)); 2907 u32 cmd; 2908 int comp = 0; 2909 2910 if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags)) 2911 return 0; 2912 2913 hw_cons = *cp->rx_cons_ptr; 2914 if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT) 2915 hw_cons++; 2916 2917 sw_cons = cp->rx_cons; 2918 while (sw_cons != hw_cons) { 2919 u8 cqe_fp_flags; 2920 2921 cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT]; 2922 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 2923 if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) { 2924 cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data); 2925 cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT; 2926 if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP || 2927 cmd == RAMROD_CMD_ID_ETH_HALT) 2928 comp++; 2929 } 2930 sw_cons = BNX2X_NEXT_RCQE(sw_cons); 2931 } 2932 return comp; 2933 } 2934 2935 static void cnic_chk_pkt_rings(struct cnic_local *cp) 2936 { 2937 u16 rx_cons, tx_cons; 2938 int comp = 0; 2939 2940 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 2941 return; 2942 2943 rx_cons = *cp->rx_cons_ptr; 2944 tx_cons = *cp->tx_cons_ptr; 2945 if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) { 2946 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 2947 comp = cnic_l2_completion(cp); 2948 2949 cp->tx_cons = tx_cons; 2950 cp->rx_cons = rx_cons; 2951 2952 if (cp->udev) 2953 uio_event_notify(&cp->udev->cnic_uinfo); 2954 } 2955 if (comp) 2956 clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 2957 } 2958 2959 static u32 cnic_service_bnx2_queues(struct cnic_dev *dev) 2960 { 2961 struct cnic_local *cp = dev->cnic_priv; 2962 u32 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2963 int kcqe_cnt; 2964 2965 /* status block index must be read before reading other fields */ 2966 rmb(); 2967 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2968 2969 while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) { 2970 2971 service_kcqes(dev, kcqe_cnt); 2972 2973 /* Tell compiler that status_blk fields can change. */ 2974 barrier(); 2975 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2976 /* status block index must be read first */ 2977 rmb(); 2978 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2979 } 2980 2981 CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx); 2982 2983 cnic_chk_pkt_rings(cp); 2984 2985 return status_idx; 2986 } 2987 2988 static int cnic_service_bnx2(void *data, void *status_blk) 2989 { 2990 struct cnic_dev *dev = data; 2991 2992 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 2993 struct status_block *sblk = status_blk; 2994 2995 return sblk->status_idx; 2996 } 2997 2998 return cnic_service_bnx2_queues(dev); 2999 } 3000 3001 static void cnic_service_bnx2_msix(unsigned long data) 3002 { 3003 struct cnic_dev *dev = (struct cnic_dev *) data; 3004 struct cnic_local *cp = dev->cnic_priv; 3005 3006 cp->last_status_idx = cnic_service_bnx2_queues(dev); 3007 3008 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 3009 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 3010 } 3011 3012 static void cnic_doirq(struct cnic_dev *dev) 3013 { 3014 struct cnic_local *cp = dev->cnic_priv; 3015 3016 if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3017 u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX; 3018 3019 prefetch(cp->status_blk.gen); 3020 prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]); 3021 3022 tasklet_schedule(&cp->cnic_irq_task); 3023 } 3024 } 3025 3026 static irqreturn_t cnic_irq(int irq, void *dev_instance) 3027 { 3028 struct cnic_dev *dev = dev_instance; 3029 struct cnic_local *cp = dev->cnic_priv; 3030 3031 if (cp->ack_int) 3032 cp->ack_int(dev); 3033 3034 cnic_doirq(dev); 3035 3036 return IRQ_HANDLED; 3037 } 3038 3039 static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm, 3040 u16 index, u8 op, u8 update) 3041 { 3042 struct cnic_local *cp = dev->cnic_priv; 3043 u32 hc_addr = (HC_REG_COMMAND_REG + CNIC_PORT(cp) * 32 + 3044 COMMAND_REG_INT_ACK); 3045 struct igu_ack_register igu_ack; 3046 3047 igu_ack.status_block_index = index; 3048 igu_ack.sb_id_and_flags = 3049 ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) | 3050 (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) | 3051 (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) | 3052 (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT)); 3053 3054 CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack)); 3055 } 3056 3057 static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment, 3058 u16 index, u8 op, u8 update) 3059 { 3060 struct igu_regular cmd_data; 3061 u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8; 3062 3063 cmd_data.sb_id_and_flags = 3064 (index << IGU_REGULAR_SB_INDEX_SHIFT) | 3065 (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) | 3066 (update << IGU_REGULAR_BUPDATE_SHIFT) | 3067 (op << IGU_REGULAR_ENABLE_INT_SHIFT); 3068 3069 3070 CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags); 3071 } 3072 3073 static void cnic_ack_bnx2x_msix(struct cnic_dev *dev) 3074 { 3075 struct cnic_local *cp = dev->cnic_priv; 3076 3077 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0, 3078 IGU_INT_DISABLE, 0); 3079 } 3080 3081 static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev) 3082 { 3083 struct cnic_local *cp = dev->cnic_priv; 3084 3085 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0, 3086 IGU_INT_DISABLE, 0); 3087 } 3088 3089 static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx) 3090 { 3091 struct cnic_local *cp = dev->cnic_priv; 3092 3093 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx, 3094 IGU_INT_ENABLE, 1); 3095 } 3096 3097 static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx) 3098 { 3099 struct cnic_local *cp = dev->cnic_priv; 3100 3101 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx, 3102 IGU_INT_ENABLE, 1); 3103 } 3104 3105 static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info) 3106 { 3107 u32 last_status = *info->status_idx_ptr; 3108 int kcqe_cnt; 3109 3110 /* status block index must be read before reading the KCQ */ 3111 rmb(); 3112 while ((kcqe_cnt = cnic_get_kcqes(dev, info))) { 3113 3114 service_kcqes(dev, kcqe_cnt); 3115 3116 /* Tell compiler that sblk fields can change. */ 3117 barrier(); 3118 3119 last_status = *info->status_idx_ptr; 3120 /* status block index must be read before reading the KCQ */ 3121 rmb(); 3122 } 3123 return last_status; 3124 } 3125 3126 static void cnic_service_bnx2x_bh(unsigned long data) 3127 { 3128 struct cnic_dev *dev = (struct cnic_dev *) data; 3129 struct cnic_local *cp = dev->cnic_priv; 3130 u32 status_idx, new_status_idx; 3131 3132 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) 3133 return; 3134 3135 while (1) { 3136 status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1); 3137 3138 CNIC_WR16(dev, cp->kcq1.io_addr, 3139 cp->kcq1.sw_prod_idx + MAX_KCQ_IDX); 3140 3141 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_FCOE) { 3142 cp->arm_int(dev, status_idx); 3143 break; 3144 } 3145 3146 new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2); 3147 3148 if (new_status_idx != status_idx) 3149 continue; 3150 3151 CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx + 3152 MAX_KCQ_IDX); 3153 3154 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 3155 status_idx, IGU_INT_ENABLE, 1); 3156 3157 break; 3158 } 3159 } 3160 3161 static int cnic_service_bnx2x(void *data, void *status_blk) 3162 { 3163 struct cnic_dev *dev = data; 3164 struct cnic_local *cp = dev->cnic_priv; 3165 3166 if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 3167 cnic_doirq(dev); 3168 3169 cnic_chk_pkt_rings(cp); 3170 3171 return 0; 3172 } 3173 3174 static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type) 3175 { 3176 struct cnic_ulp_ops *ulp_ops; 3177 3178 if (if_type == CNIC_ULP_ISCSI) 3179 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 3180 3181 mutex_lock(&cnic_lock); 3182 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3183 lockdep_is_held(&cnic_lock)); 3184 if (!ulp_ops) { 3185 mutex_unlock(&cnic_lock); 3186 return; 3187 } 3188 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3189 mutex_unlock(&cnic_lock); 3190 3191 if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3192 ulp_ops->cnic_stop(cp->ulp_handle[if_type]); 3193 3194 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3195 } 3196 3197 static void cnic_ulp_stop(struct cnic_dev *dev) 3198 { 3199 struct cnic_local *cp = dev->cnic_priv; 3200 int if_type; 3201 3202 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) 3203 cnic_ulp_stop_one(cp, if_type); 3204 } 3205 3206 static void cnic_ulp_start(struct cnic_dev *dev) 3207 { 3208 struct cnic_local *cp = dev->cnic_priv; 3209 int if_type; 3210 3211 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 3212 struct cnic_ulp_ops *ulp_ops; 3213 3214 mutex_lock(&cnic_lock); 3215 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3216 lockdep_is_held(&cnic_lock)); 3217 if (!ulp_ops || !ulp_ops->cnic_start) { 3218 mutex_unlock(&cnic_lock); 3219 continue; 3220 } 3221 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3222 mutex_unlock(&cnic_lock); 3223 3224 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3225 ulp_ops->cnic_start(cp->ulp_handle[if_type]); 3226 3227 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3228 } 3229 } 3230 3231 static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type) 3232 { 3233 struct cnic_local *cp = dev->cnic_priv; 3234 struct cnic_ulp_ops *ulp_ops; 3235 int rc; 3236 3237 mutex_lock(&cnic_lock); 3238 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 3239 if (ulp_ops && ulp_ops->cnic_get_stats) 3240 rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]); 3241 else 3242 rc = -ENODEV; 3243 mutex_unlock(&cnic_lock); 3244 return rc; 3245 } 3246 3247 static int cnic_ctl(void *data, struct cnic_ctl_info *info) 3248 { 3249 struct cnic_dev *dev = data; 3250 int ulp_type = CNIC_ULP_ISCSI; 3251 3252 switch (info->cmd) { 3253 case CNIC_CTL_STOP_CMD: 3254 cnic_hold(dev); 3255 3256 cnic_ulp_stop(dev); 3257 cnic_stop_hw(dev); 3258 3259 cnic_put(dev); 3260 break; 3261 case CNIC_CTL_START_CMD: 3262 cnic_hold(dev); 3263 3264 if (!cnic_start_hw(dev)) 3265 cnic_ulp_start(dev); 3266 3267 cnic_put(dev); 3268 break; 3269 case CNIC_CTL_STOP_ISCSI_CMD: { 3270 struct cnic_local *cp = dev->cnic_priv; 3271 set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags); 3272 queue_delayed_work(cnic_wq, &cp->delete_task, 0); 3273 break; 3274 } 3275 case CNIC_CTL_COMPLETION_CMD: { 3276 struct cnic_ctl_completion *comp = &info->data.comp; 3277 u32 cid = BNX2X_SW_CID(comp->cid); 3278 u32 l5_cid; 3279 struct cnic_local *cp = dev->cnic_priv; 3280 3281 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 3282 break; 3283 3284 if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) { 3285 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3286 3287 if (unlikely(comp->error)) { 3288 set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags); 3289 netdev_err(dev->netdev, 3290 "CID %x CFC delete comp error %x\n", 3291 cid, comp->error); 3292 } 3293 3294 ctx->wait_cond = 1; 3295 wake_up(&ctx->waitq); 3296 } 3297 break; 3298 } 3299 case CNIC_CTL_FCOE_STATS_GET_CMD: 3300 ulp_type = CNIC_ULP_FCOE; 3301 /* fall through */ 3302 case CNIC_CTL_ISCSI_STATS_GET_CMD: 3303 cnic_hold(dev); 3304 cnic_copy_ulp_stats(dev, ulp_type); 3305 cnic_put(dev); 3306 break; 3307 3308 default: 3309 return -EINVAL; 3310 } 3311 return 0; 3312 } 3313 3314 static void cnic_ulp_init(struct cnic_dev *dev) 3315 { 3316 int i; 3317 struct cnic_local *cp = dev->cnic_priv; 3318 3319 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3320 struct cnic_ulp_ops *ulp_ops; 3321 3322 mutex_lock(&cnic_lock); 3323 ulp_ops = cnic_ulp_tbl_prot(i); 3324 if (!ulp_ops || !ulp_ops->cnic_init) { 3325 mutex_unlock(&cnic_lock); 3326 continue; 3327 } 3328 ulp_get(ulp_ops); 3329 mutex_unlock(&cnic_lock); 3330 3331 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3332 ulp_ops->cnic_init(dev); 3333 3334 ulp_put(ulp_ops); 3335 } 3336 } 3337 3338 static void cnic_ulp_exit(struct cnic_dev *dev) 3339 { 3340 int i; 3341 struct cnic_local *cp = dev->cnic_priv; 3342 3343 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3344 struct cnic_ulp_ops *ulp_ops; 3345 3346 mutex_lock(&cnic_lock); 3347 ulp_ops = cnic_ulp_tbl_prot(i); 3348 if (!ulp_ops || !ulp_ops->cnic_exit) { 3349 mutex_unlock(&cnic_lock); 3350 continue; 3351 } 3352 ulp_get(ulp_ops); 3353 mutex_unlock(&cnic_lock); 3354 3355 if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3356 ulp_ops->cnic_exit(dev); 3357 3358 ulp_put(ulp_ops); 3359 } 3360 } 3361 3362 static int cnic_cm_offload_pg(struct cnic_sock *csk) 3363 { 3364 struct cnic_dev *dev = csk->dev; 3365 struct l4_kwq_offload_pg *l4kwqe; 3366 struct kwqe *wqes[1]; 3367 3368 l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1; 3369 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3370 wqes[0] = (struct kwqe *) l4kwqe; 3371 3372 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG; 3373 l4kwqe->flags = 3374 L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT; 3375 l4kwqe->l2hdr_nbytes = ETH_HLEN; 3376 3377 l4kwqe->da0 = csk->ha[0]; 3378 l4kwqe->da1 = csk->ha[1]; 3379 l4kwqe->da2 = csk->ha[2]; 3380 l4kwqe->da3 = csk->ha[3]; 3381 l4kwqe->da4 = csk->ha[4]; 3382 l4kwqe->da5 = csk->ha[5]; 3383 3384 l4kwqe->sa0 = dev->mac_addr[0]; 3385 l4kwqe->sa1 = dev->mac_addr[1]; 3386 l4kwqe->sa2 = dev->mac_addr[2]; 3387 l4kwqe->sa3 = dev->mac_addr[3]; 3388 l4kwqe->sa4 = dev->mac_addr[4]; 3389 l4kwqe->sa5 = dev->mac_addr[5]; 3390 3391 l4kwqe->etype = ETH_P_IP; 3392 l4kwqe->ipid_start = DEF_IPID_START; 3393 l4kwqe->host_opaque = csk->l5_cid; 3394 3395 if (csk->vlan_id) { 3396 l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING; 3397 l4kwqe->vlan_tag = csk->vlan_id; 3398 l4kwqe->l2hdr_nbytes += 4; 3399 } 3400 3401 return dev->submit_kwqes(dev, wqes, 1); 3402 } 3403 3404 static int cnic_cm_update_pg(struct cnic_sock *csk) 3405 { 3406 struct cnic_dev *dev = csk->dev; 3407 struct l4_kwq_update_pg *l4kwqe; 3408 struct kwqe *wqes[1]; 3409 3410 l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1; 3411 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3412 wqes[0] = (struct kwqe *) l4kwqe; 3413 3414 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG; 3415 l4kwqe->flags = 3416 L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT; 3417 l4kwqe->pg_cid = csk->pg_cid; 3418 3419 l4kwqe->da0 = csk->ha[0]; 3420 l4kwqe->da1 = csk->ha[1]; 3421 l4kwqe->da2 = csk->ha[2]; 3422 l4kwqe->da3 = csk->ha[3]; 3423 l4kwqe->da4 = csk->ha[4]; 3424 l4kwqe->da5 = csk->ha[5]; 3425 3426 l4kwqe->pg_host_opaque = csk->l5_cid; 3427 l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA; 3428 3429 return dev->submit_kwqes(dev, wqes, 1); 3430 } 3431 3432 static int cnic_cm_upload_pg(struct cnic_sock *csk) 3433 { 3434 struct cnic_dev *dev = csk->dev; 3435 struct l4_kwq_upload *l4kwqe; 3436 struct kwqe *wqes[1]; 3437 3438 l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1; 3439 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3440 wqes[0] = (struct kwqe *) l4kwqe; 3441 3442 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG; 3443 l4kwqe->flags = 3444 L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT; 3445 l4kwqe->cid = csk->pg_cid; 3446 3447 return dev->submit_kwqes(dev, wqes, 1); 3448 } 3449 3450 static int cnic_cm_conn_req(struct cnic_sock *csk) 3451 { 3452 struct cnic_dev *dev = csk->dev; 3453 struct l4_kwq_connect_req1 *l4kwqe1; 3454 struct l4_kwq_connect_req2 *l4kwqe2; 3455 struct l4_kwq_connect_req3 *l4kwqe3; 3456 struct kwqe *wqes[3]; 3457 u8 tcp_flags = 0; 3458 int num_wqes = 2; 3459 3460 l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1; 3461 l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2; 3462 l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3; 3463 memset(l4kwqe1, 0, sizeof(*l4kwqe1)); 3464 memset(l4kwqe2, 0, sizeof(*l4kwqe2)); 3465 memset(l4kwqe3, 0, sizeof(*l4kwqe3)); 3466 3467 l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3; 3468 l4kwqe3->flags = 3469 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT; 3470 l4kwqe3->ka_timeout = csk->ka_timeout; 3471 l4kwqe3->ka_interval = csk->ka_interval; 3472 l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count; 3473 l4kwqe3->tos = csk->tos; 3474 l4kwqe3->ttl = csk->ttl; 3475 l4kwqe3->snd_seq_scale = csk->snd_seq_scale; 3476 l4kwqe3->pmtu = csk->mtu; 3477 l4kwqe3->rcv_buf = csk->rcv_buf; 3478 l4kwqe3->snd_buf = csk->snd_buf; 3479 l4kwqe3->seed = csk->seed; 3480 3481 wqes[0] = (struct kwqe *) l4kwqe1; 3482 if (test_bit(SK_F_IPV6, &csk->flags)) { 3483 wqes[1] = (struct kwqe *) l4kwqe2; 3484 wqes[2] = (struct kwqe *) l4kwqe3; 3485 num_wqes = 3; 3486 3487 l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6; 3488 l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2; 3489 l4kwqe2->flags = 3490 L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT | 3491 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT; 3492 l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]); 3493 l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]); 3494 l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]); 3495 l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]); 3496 l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]); 3497 l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]); 3498 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) - 3499 sizeof(struct tcphdr); 3500 } else { 3501 wqes[1] = (struct kwqe *) l4kwqe3; 3502 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) - 3503 sizeof(struct tcphdr); 3504 } 3505 3506 l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1; 3507 l4kwqe1->flags = 3508 (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) | 3509 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT; 3510 l4kwqe1->cid = csk->cid; 3511 l4kwqe1->pg_cid = csk->pg_cid; 3512 l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]); 3513 l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]); 3514 l4kwqe1->src_port = be16_to_cpu(csk->src_port); 3515 l4kwqe1->dst_port = be16_to_cpu(csk->dst_port); 3516 if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK) 3517 tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK; 3518 if (csk->tcp_flags & SK_TCP_KEEP_ALIVE) 3519 tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE; 3520 if (csk->tcp_flags & SK_TCP_NAGLE) 3521 tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE; 3522 if (csk->tcp_flags & SK_TCP_TIMESTAMP) 3523 tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP; 3524 if (csk->tcp_flags & SK_TCP_SACK) 3525 tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK; 3526 if (csk->tcp_flags & SK_TCP_SEG_SCALING) 3527 tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING; 3528 3529 l4kwqe1->tcp_flags = tcp_flags; 3530 3531 return dev->submit_kwqes(dev, wqes, num_wqes); 3532 } 3533 3534 static int cnic_cm_close_req(struct cnic_sock *csk) 3535 { 3536 struct cnic_dev *dev = csk->dev; 3537 struct l4_kwq_close_req *l4kwqe; 3538 struct kwqe *wqes[1]; 3539 3540 l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2; 3541 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3542 wqes[0] = (struct kwqe *) l4kwqe; 3543 3544 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE; 3545 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT; 3546 l4kwqe->cid = csk->cid; 3547 3548 return dev->submit_kwqes(dev, wqes, 1); 3549 } 3550 3551 static int cnic_cm_abort_req(struct cnic_sock *csk) 3552 { 3553 struct cnic_dev *dev = csk->dev; 3554 struct l4_kwq_reset_req *l4kwqe; 3555 struct kwqe *wqes[1]; 3556 3557 l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2; 3558 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3559 wqes[0] = (struct kwqe *) l4kwqe; 3560 3561 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET; 3562 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT; 3563 l4kwqe->cid = csk->cid; 3564 3565 return dev->submit_kwqes(dev, wqes, 1); 3566 } 3567 3568 static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid, 3569 u32 l5_cid, struct cnic_sock **csk, void *context) 3570 { 3571 struct cnic_local *cp = dev->cnic_priv; 3572 struct cnic_sock *csk1; 3573 3574 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3575 return -EINVAL; 3576 3577 if (cp->ctx_tbl) { 3578 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3579 3580 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 3581 return -EAGAIN; 3582 } 3583 3584 csk1 = &cp->csk_tbl[l5_cid]; 3585 if (atomic_read(&csk1->ref_count)) 3586 return -EAGAIN; 3587 3588 if (test_and_set_bit(SK_F_INUSE, &csk1->flags)) 3589 return -EBUSY; 3590 3591 csk1->dev = dev; 3592 csk1->cid = cid; 3593 csk1->l5_cid = l5_cid; 3594 csk1->ulp_type = ulp_type; 3595 csk1->context = context; 3596 3597 csk1->ka_timeout = DEF_KA_TIMEOUT; 3598 csk1->ka_interval = DEF_KA_INTERVAL; 3599 csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT; 3600 csk1->tos = DEF_TOS; 3601 csk1->ttl = DEF_TTL; 3602 csk1->snd_seq_scale = DEF_SND_SEQ_SCALE; 3603 csk1->rcv_buf = DEF_RCV_BUF; 3604 csk1->snd_buf = DEF_SND_BUF; 3605 csk1->seed = DEF_SEED; 3606 3607 *csk = csk1; 3608 return 0; 3609 } 3610 3611 static void cnic_cm_cleanup(struct cnic_sock *csk) 3612 { 3613 if (csk->src_port) { 3614 struct cnic_dev *dev = csk->dev; 3615 struct cnic_local *cp = dev->cnic_priv; 3616 3617 cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port)); 3618 csk->src_port = 0; 3619 } 3620 } 3621 3622 static void cnic_close_conn(struct cnic_sock *csk) 3623 { 3624 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) { 3625 cnic_cm_upload_pg(csk); 3626 clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3627 } 3628 cnic_cm_cleanup(csk); 3629 } 3630 3631 static int cnic_cm_destroy(struct cnic_sock *csk) 3632 { 3633 if (!cnic_in_use(csk)) 3634 return -EINVAL; 3635 3636 csk_hold(csk); 3637 clear_bit(SK_F_INUSE, &csk->flags); 3638 smp_mb__after_clear_bit(); 3639 while (atomic_read(&csk->ref_count) != 1) 3640 msleep(1); 3641 cnic_cm_cleanup(csk); 3642 3643 csk->flags = 0; 3644 csk_put(csk); 3645 return 0; 3646 } 3647 3648 static inline u16 cnic_get_vlan(struct net_device *dev, 3649 struct net_device **vlan_dev) 3650 { 3651 if (dev->priv_flags & IFF_802_1Q_VLAN) { 3652 *vlan_dev = vlan_dev_real_dev(dev); 3653 return vlan_dev_vlan_id(dev); 3654 } 3655 *vlan_dev = dev; 3656 return 0; 3657 } 3658 3659 static int cnic_get_v4_route(struct sockaddr_in *dst_addr, 3660 struct dst_entry **dst) 3661 { 3662 #if defined(CONFIG_INET) 3663 struct rtable *rt; 3664 3665 rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0); 3666 if (!IS_ERR(rt)) { 3667 *dst = &rt->dst; 3668 return 0; 3669 } 3670 return PTR_ERR(rt); 3671 #else 3672 return -ENETUNREACH; 3673 #endif 3674 } 3675 3676 static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr, 3677 struct dst_entry **dst) 3678 { 3679 #if defined(CONFIG_IPV6) || (defined(CONFIG_IPV6_MODULE) && defined(MODULE)) 3680 struct flowi6 fl6; 3681 3682 memset(&fl6, 0, sizeof(fl6)); 3683 fl6.daddr = dst_addr->sin6_addr; 3684 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 3685 fl6.flowi6_oif = dst_addr->sin6_scope_id; 3686 3687 *dst = ip6_route_output(&init_net, NULL, &fl6); 3688 if ((*dst)->error) { 3689 dst_release(*dst); 3690 *dst = NULL; 3691 return -ENETUNREACH; 3692 } else 3693 return 0; 3694 #endif 3695 3696 return -ENETUNREACH; 3697 } 3698 3699 static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr, 3700 int ulp_type) 3701 { 3702 struct cnic_dev *dev = NULL; 3703 struct dst_entry *dst; 3704 struct net_device *netdev = NULL; 3705 int err = -ENETUNREACH; 3706 3707 if (dst_addr->sin_family == AF_INET) 3708 err = cnic_get_v4_route(dst_addr, &dst); 3709 else if (dst_addr->sin_family == AF_INET6) { 3710 struct sockaddr_in6 *dst_addr6 = 3711 (struct sockaddr_in6 *) dst_addr; 3712 3713 err = cnic_get_v6_route(dst_addr6, &dst); 3714 } else 3715 return NULL; 3716 3717 if (err) 3718 return NULL; 3719 3720 if (!dst->dev) 3721 goto done; 3722 3723 cnic_get_vlan(dst->dev, &netdev); 3724 3725 dev = cnic_from_netdev(netdev); 3726 3727 done: 3728 dst_release(dst); 3729 if (dev) 3730 cnic_put(dev); 3731 return dev; 3732 } 3733 3734 static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3735 { 3736 struct cnic_dev *dev = csk->dev; 3737 struct cnic_local *cp = dev->cnic_priv; 3738 3739 return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk); 3740 } 3741 3742 static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3743 { 3744 struct cnic_dev *dev = csk->dev; 3745 struct cnic_local *cp = dev->cnic_priv; 3746 int is_v6, rc = 0; 3747 struct dst_entry *dst = NULL; 3748 struct net_device *realdev; 3749 __be16 local_port; 3750 u32 port_id; 3751 3752 if (saddr->local.v6.sin6_family == AF_INET6 && 3753 saddr->remote.v6.sin6_family == AF_INET6) 3754 is_v6 = 1; 3755 else if (saddr->local.v4.sin_family == AF_INET && 3756 saddr->remote.v4.sin_family == AF_INET) 3757 is_v6 = 0; 3758 else 3759 return -EINVAL; 3760 3761 clear_bit(SK_F_IPV6, &csk->flags); 3762 3763 if (is_v6) { 3764 set_bit(SK_F_IPV6, &csk->flags); 3765 cnic_get_v6_route(&saddr->remote.v6, &dst); 3766 3767 memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr, 3768 sizeof(struct in6_addr)); 3769 csk->dst_port = saddr->remote.v6.sin6_port; 3770 local_port = saddr->local.v6.sin6_port; 3771 3772 } else { 3773 cnic_get_v4_route(&saddr->remote.v4, &dst); 3774 3775 csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr; 3776 csk->dst_port = saddr->remote.v4.sin_port; 3777 local_port = saddr->local.v4.sin_port; 3778 } 3779 3780 csk->vlan_id = 0; 3781 csk->mtu = dev->netdev->mtu; 3782 if (dst && dst->dev) { 3783 u16 vlan = cnic_get_vlan(dst->dev, &realdev); 3784 if (realdev == dev->netdev) { 3785 csk->vlan_id = vlan; 3786 csk->mtu = dst_mtu(dst); 3787 } 3788 } 3789 3790 port_id = be16_to_cpu(local_port); 3791 if (port_id >= CNIC_LOCAL_PORT_MIN && 3792 port_id < CNIC_LOCAL_PORT_MAX) { 3793 if (cnic_alloc_id(&cp->csk_port_tbl, port_id)) 3794 port_id = 0; 3795 } else 3796 port_id = 0; 3797 3798 if (!port_id) { 3799 port_id = cnic_alloc_new_id(&cp->csk_port_tbl); 3800 if (port_id == -1) { 3801 rc = -ENOMEM; 3802 goto err_out; 3803 } 3804 local_port = cpu_to_be16(port_id); 3805 } 3806 csk->src_port = local_port; 3807 3808 err_out: 3809 dst_release(dst); 3810 return rc; 3811 } 3812 3813 static void cnic_init_csk_state(struct cnic_sock *csk) 3814 { 3815 csk->state = 0; 3816 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3817 clear_bit(SK_F_CLOSING, &csk->flags); 3818 } 3819 3820 static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3821 { 3822 struct cnic_local *cp = csk->dev->cnic_priv; 3823 int err = 0; 3824 3825 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 3826 return -EOPNOTSUPP; 3827 3828 if (!cnic_in_use(csk)) 3829 return -EINVAL; 3830 3831 if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags)) 3832 return -EINVAL; 3833 3834 cnic_init_csk_state(csk); 3835 3836 err = cnic_get_route(csk, saddr); 3837 if (err) 3838 goto err_out; 3839 3840 err = cnic_resolve_addr(csk, saddr); 3841 if (!err) 3842 return 0; 3843 3844 err_out: 3845 clear_bit(SK_F_CONNECT_START, &csk->flags); 3846 return err; 3847 } 3848 3849 static int cnic_cm_abort(struct cnic_sock *csk) 3850 { 3851 struct cnic_local *cp = csk->dev->cnic_priv; 3852 u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP; 3853 3854 if (!cnic_in_use(csk)) 3855 return -EINVAL; 3856 3857 if (cnic_abort_prep(csk)) 3858 return cnic_cm_abort_req(csk); 3859 3860 /* Getting here means that we haven't started connect, or 3861 * connect was not successful, or it has been reset by the target. 3862 */ 3863 3864 cp->close_conn(csk, opcode); 3865 if (csk->state != opcode) { 3866 /* Wait for remote reset sequence to complete */ 3867 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3868 msleep(1); 3869 3870 return -EALREADY; 3871 } 3872 3873 return 0; 3874 } 3875 3876 static int cnic_cm_close(struct cnic_sock *csk) 3877 { 3878 if (!cnic_in_use(csk)) 3879 return -EINVAL; 3880 3881 if (cnic_close_prep(csk)) { 3882 csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 3883 return cnic_cm_close_req(csk); 3884 } else { 3885 /* Wait for remote reset sequence to complete */ 3886 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3887 msleep(1); 3888 3889 return -EALREADY; 3890 } 3891 return 0; 3892 } 3893 3894 static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk, 3895 u8 opcode) 3896 { 3897 struct cnic_ulp_ops *ulp_ops; 3898 int ulp_type = csk->ulp_type; 3899 3900 rcu_read_lock(); 3901 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 3902 if (ulp_ops) { 3903 if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE) 3904 ulp_ops->cm_connect_complete(csk); 3905 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 3906 ulp_ops->cm_close_complete(csk); 3907 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) 3908 ulp_ops->cm_remote_abort(csk); 3909 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP) 3910 ulp_ops->cm_abort_complete(csk); 3911 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED) 3912 ulp_ops->cm_remote_close(csk); 3913 } 3914 rcu_read_unlock(); 3915 } 3916 3917 static int cnic_cm_set_pg(struct cnic_sock *csk) 3918 { 3919 if (cnic_offld_prep(csk)) { 3920 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3921 cnic_cm_update_pg(csk); 3922 else 3923 cnic_cm_offload_pg(csk); 3924 } 3925 return 0; 3926 } 3927 3928 static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe) 3929 { 3930 struct cnic_local *cp = dev->cnic_priv; 3931 u32 l5_cid = kcqe->pg_host_opaque; 3932 u8 opcode = kcqe->op_code; 3933 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 3934 3935 csk_hold(csk); 3936 if (!cnic_in_use(csk)) 3937 goto done; 3938 3939 if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3940 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3941 goto done; 3942 } 3943 /* Possible PG kcqe status: SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */ 3944 if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) { 3945 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3946 cnic_cm_upcall(cp, csk, 3947 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 3948 goto done; 3949 } 3950 3951 csk->pg_cid = kcqe->pg_cid; 3952 set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3953 cnic_cm_conn_req(csk); 3954 3955 done: 3956 csk_put(csk); 3957 } 3958 3959 static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe) 3960 { 3961 struct cnic_local *cp = dev->cnic_priv; 3962 struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe; 3963 u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE; 3964 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3965 3966 ctx->timestamp = jiffies; 3967 ctx->wait_cond = 1; 3968 wake_up(&ctx->waitq); 3969 } 3970 3971 static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe) 3972 { 3973 struct cnic_local *cp = dev->cnic_priv; 3974 struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe; 3975 u8 opcode = l4kcqe->op_code; 3976 u32 l5_cid; 3977 struct cnic_sock *csk; 3978 3979 if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) { 3980 cnic_process_fcoe_term_conn(dev, kcqe); 3981 return; 3982 } 3983 if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG || 3984 opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3985 cnic_cm_process_offld_pg(dev, l4kcqe); 3986 return; 3987 } 3988 3989 l5_cid = l4kcqe->conn_id; 3990 if (opcode & 0x80) 3991 l5_cid = l4kcqe->cid; 3992 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3993 return; 3994 3995 csk = &cp->csk_tbl[l5_cid]; 3996 csk_hold(csk); 3997 3998 if (!cnic_in_use(csk)) { 3999 csk_put(csk); 4000 return; 4001 } 4002 4003 switch (opcode) { 4004 case L5CM_RAMROD_CMD_ID_TCP_CONNECT: 4005 if (l4kcqe->status != 0) { 4006 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4007 cnic_cm_upcall(cp, csk, 4008 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 4009 } 4010 break; 4011 case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE: 4012 if (l4kcqe->status == 0) 4013 set_bit(SK_F_OFFLD_COMPLETE, &csk->flags); 4014 else if (l4kcqe->status == 4015 L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4016 set_bit(SK_F_HW_ERR, &csk->flags); 4017 4018 smp_mb__before_clear_bit(); 4019 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4020 cnic_cm_upcall(cp, csk, opcode); 4021 break; 4022 4023 case L5CM_RAMROD_CMD_ID_CLOSE: 4024 if (l4kcqe->status != 0) { 4025 netdev_warn(dev->netdev, "RAMROD CLOSE compl with " 4026 "status 0x%x\n", l4kcqe->status); 4027 opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 4028 /* Fall through */ 4029 } else { 4030 break; 4031 } 4032 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4033 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4034 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4035 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4036 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4037 if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4038 set_bit(SK_F_HW_ERR, &csk->flags); 4039 4040 cp->close_conn(csk, opcode); 4041 break; 4042 4043 case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED: 4044 /* after we already sent CLOSE_REQ */ 4045 if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) && 4046 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) && 4047 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 4048 cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP); 4049 else 4050 cnic_cm_upcall(cp, csk, opcode); 4051 break; 4052 } 4053 csk_put(csk); 4054 } 4055 4056 static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num) 4057 { 4058 struct cnic_dev *dev = data; 4059 int i; 4060 4061 for (i = 0; i < num; i++) 4062 cnic_cm_process_kcqe(dev, kcqe[i]); 4063 } 4064 4065 static struct cnic_ulp_ops cm_ulp_ops = { 4066 .indicate_kcqes = cnic_cm_indicate_kcqe, 4067 }; 4068 4069 static void cnic_cm_free_mem(struct cnic_dev *dev) 4070 { 4071 struct cnic_local *cp = dev->cnic_priv; 4072 4073 kfree(cp->csk_tbl); 4074 cp->csk_tbl = NULL; 4075 cnic_free_id_tbl(&cp->csk_port_tbl); 4076 } 4077 4078 static int cnic_cm_alloc_mem(struct cnic_dev *dev) 4079 { 4080 struct cnic_local *cp = dev->cnic_priv; 4081 u32 port_id; 4082 4083 cp->csk_tbl = kzalloc(sizeof(struct cnic_sock) * MAX_CM_SK_TBL_SZ, 4084 GFP_KERNEL); 4085 if (!cp->csk_tbl) 4086 return -ENOMEM; 4087 4088 port_id = random32(); 4089 port_id %= CNIC_LOCAL_PORT_RANGE; 4090 if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, 4091 CNIC_LOCAL_PORT_MIN, port_id)) { 4092 cnic_cm_free_mem(dev); 4093 return -ENOMEM; 4094 } 4095 return 0; 4096 } 4097 4098 static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode) 4099 { 4100 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 4101 /* Unsolicited RESET_COMP or RESET_RECEIVED */ 4102 opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED; 4103 csk->state = opcode; 4104 } 4105 4106 /* 1. If event opcode matches the expected event in csk->state 4107 * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any 4108 * event 4109 * 3. If the expected event is 0, meaning the connection was never 4110 * never established, we accept the opcode from cm_abort. 4111 */ 4112 if (opcode == csk->state || csk->state == 0 || 4113 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP || 4114 csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) { 4115 if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) { 4116 if (csk->state == 0) 4117 csk->state = opcode; 4118 return 1; 4119 } 4120 } 4121 return 0; 4122 } 4123 4124 static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode) 4125 { 4126 struct cnic_dev *dev = csk->dev; 4127 struct cnic_local *cp = dev->cnic_priv; 4128 4129 if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) { 4130 cnic_cm_upcall(cp, csk, opcode); 4131 return; 4132 } 4133 4134 clear_bit(SK_F_CONNECT_START, &csk->flags); 4135 cnic_close_conn(csk); 4136 csk->state = opcode; 4137 cnic_cm_upcall(cp, csk, opcode); 4138 } 4139 4140 static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev) 4141 { 4142 } 4143 4144 static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) 4145 { 4146 u32 seed; 4147 4148 seed = random32(); 4149 cnic_ctx_wr(dev, 45, 0, seed); 4150 return 0; 4151 } 4152 4153 static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode) 4154 { 4155 struct cnic_dev *dev = csk->dev; 4156 struct cnic_local *cp = dev->cnic_priv; 4157 struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid]; 4158 union l5cm_specific_data l5_data; 4159 u32 cmd = 0; 4160 int close_complete = 0; 4161 4162 switch (opcode) { 4163 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4164 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4165 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4166 if (cnic_ready_to_close(csk, opcode)) { 4167 if (test_bit(SK_F_HW_ERR, &csk->flags)) 4168 close_complete = 1; 4169 else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 4170 cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE; 4171 else 4172 close_complete = 1; 4173 } 4174 break; 4175 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4176 cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD; 4177 break; 4178 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4179 close_complete = 1; 4180 break; 4181 } 4182 if (cmd) { 4183 memset(&l5_data, 0, sizeof(l5_data)); 4184 4185 cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE, 4186 &l5_data); 4187 } else if (close_complete) { 4188 ctx->timestamp = jiffies; 4189 cnic_close_conn(csk); 4190 cnic_cm_upcall(cp, csk, csk->state); 4191 } 4192 } 4193 4194 static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) 4195 { 4196 struct cnic_local *cp = dev->cnic_priv; 4197 4198 if (!cp->ctx_tbl) 4199 return; 4200 4201 if (!netif_running(dev->netdev)) 4202 return; 4203 4204 cnic_bnx2x_delete_wait(dev, 0); 4205 4206 cancel_delayed_work(&cp->delete_task); 4207 flush_workqueue(cnic_wq); 4208 4209 if (atomic_read(&cp->iscsi_conn) != 0) 4210 netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", 4211 atomic_read(&cp->iscsi_conn)); 4212 } 4213 4214 static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev) 4215 { 4216 struct cnic_local *cp = dev->cnic_priv; 4217 struct bnx2x *bp = netdev_priv(dev->netdev); 4218 u32 pfid = cp->pfid; 4219 u32 port = CNIC_PORT(cp); 4220 4221 cnic_init_bnx2x_mac(dev); 4222 cnic_bnx2x_set_tcp_timestamp(dev, 1); 4223 4224 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 4225 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0); 4226 4227 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4228 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1); 4229 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4230 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port), 4231 DEF_MAX_DA_COUNT); 4232 4233 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4234 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL); 4235 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4236 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS); 4237 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4238 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2); 4239 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4240 XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER); 4241 4242 CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid), 4243 DEF_MAX_CWND); 4244 return 0; 4245 } 4246 4247 static void cnic_delete_task(struct work_struct *work) 4248 { 4249 struct cnic_local *cp; 4250 struct cnic_dev *dev; 4251 u32 i; 4252 int need_resched = 0; 4253 4254 cp = container_of(work, struct cnic_local, delete_task.work); 4255 dev = cp->dev; 4256 4257 if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) { 4258 struct drv_ctl_info info; 4259 4260 cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI); 4261 4262 info.cmd = DRV_CTL_ISCSI_STOPPED_CMD; 4263 cp->ethdev->drv_ctl(dev->netdev, &info); 4264 } 4265 4266 for (i = 0; i < cp->max_cid_space; i++) { 4267 struct cnic_context *ctx = &cp->ctx_tbl[i]; 4268 int err; 4269 4270 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) || 4271 !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4272 continue; 4273 4274 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 4275 need_resched = 1; 4276 continue; 4277 } 4278 4279 if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4280 continue; 4281 4282 err = cnic_bnx2x_destroy_ramrod(dev, i); 4283 4284 cnic_free_bnx2x_conn_resc(dev, i); 4285 if (!err) { 4286 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) 4287 atomic_dec(&cp->iscsi_conn); 4288 4289 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 4290 } 4291 } 4292 4293 if (need_resched) 4294 queue_delayed_work(cnic_wq, &cp->delete_task, 4295 msecs_to_jiffies(10)); 4296 4297 } 4298 4299 static int cnic_cm_open(struct cnic_dev *dev) 4300 { 4301 struct cnic_local *cp = dev->cnic_priv; 4302 int err; 4303 4304 err = cnic_cm_alloc_mem(dev); 4305 if (err) 4306 return err; 4307 4308 err = cp->start_cm(dev); 4309 4310 if (err) 4311 goto err_out; 4312 4313 INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task); 4314 4315 dev->cm_create = cnic_cm_create; 4316 dev->cm_destroy = cnic_cm_destroy; 4317 dev->cm_connect = cnic_cm_connect; 4318 dev->cm_abort = cnic_cm_abort; 4319 dev->cm_close = cnic_cm_close; 4320 dev->cm_select_dev = cnic_cm_select_dev; 4321 4322 cp->ulp_handle[CNIC_ULP_L4] = dev; 4323 rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops); 4324 return 0; 4325 4326 err_out: 4327 cnic_cm_free_mem(dev); 4328 return err; 4329 } 4330 4331 static int cnic_cm_shutdown(struct cnic_dev *dev) 4332 { 4333 struct cnic_local *cp = dev->cnic_priv; 4334 int i; 4335 4336 if (!cp->csk_tbl) 4337 return 0; 4338 4339 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) { 4340 struct cnic_sock *csk = &cp->csk_tbl[i]; 4341 4342 clear_bit(SK_F_INUSE, &csk->flags); 4343 cnic_cm_cleanup(csk); 4344 } 4345 cnic_cm_free_mem(dev); 4346 4347 return 0; 4348 } 4349 4350 static void cnic_init_context(struct cnic_dev *dev, u32 cid) 4351 { 4352 u32 cid_addr; 4353 int i; 4354 4355 cid_addr = GET_CID_ADDR(cid); 4356 4357 for (i = 0; i < CTX_SIZE; i += 4) 4358 cnic_ctx_wr(dev, cid_addr, i, 0); 4359 } 4360 4361 static int cnic_setup_5709_context(struct cnic_dev *dev, int valid) 4362 { 4363 struct cnic_local *cp = dev->cnic_priv; 4364 int ret = 0, i; 4365 u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0; 4366 4367 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4368 return 0; 4369 4370 for (i = 0; i < cp->ctx_blks; i++) { 4371 int j; 4372 u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk; 4373 u32 val; 4374 4375 memset(cp->ctx_arr[i].ctx, 0, BNX2_PAGE_SIZE); 4376 4377 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0, 4378 (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit); 4379 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1, 4380 (u64) cp->ctx_arr[i].mapping >> 32); 4381 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx | 4382 BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ); 4383 for (j = 0; j < 10; j++) { 4384 4385 val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL); 4386 if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ)) 4387 break; 4388 udelay(5); 4389 } 4390 if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) { 4391 ret = -EBUSY; 4392 break; 4393 } 4394 } 4395 return ret; 4396 } 4397 4398 static void cnic_free_irq(struct cnic_dev *dev) 4399 { 4400 struct cnic_local *cp = dev->cnic_priv; 4401 struct cnic_eth_dev *ethdev = cp->ethdev; 4402 4403 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4404 cp->disable_int_sync(dev); 4405 tasklet_kill(&cp->cnic_irq_task); 4406 free_irq(ethdev->irq_arr[0].vector, dev); 4407 } 4408 } 4409 4410 static int cnic_request_irq(struct cnic_dev *dev) 4411 { 4412 struct cnic_local *cp = dev->cnic_priv; 4413 struct cnic_eth_dev *ethdev = cp->ethdev; 4414 int err; 4415 4416 err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev); 4417 if (err) 4418 tasklet_disable(&cp->cnic_irq_task); 4419 4420 return err; 4421 } 4422 4423 static int cnic_init_bnx2_irq(struct cnic_dev *dev) 4424 { 4425 struct cnic_local *cp = dev->cnic_priv; 4426 struct cnic_eth_dev *ethdev = cp->ethdev; 4427 4428 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4429 int err, i = 0; 4430 int sblk_num = cp->status_blk_num; 4431 u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) + 4432 BNX2_HC_SB_CONFIG_1; 4433 4434 CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT); 4435 4436 CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8); 4437 CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220); 4438 CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220); 4439 4440 cp->last_status_idx = cp->status_blk.bnx2->status_idx; 4441 tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2_msix, 4442 (unsigned long) dev); 4443 err = cnic_request_irq(dev); 4444 if (err) 4445 return err; 4446 4447 while (cp->status_blk.bnx2->status_completion_producer_index && 4448 i < 10) { 4449 CNIC_WR(dev, BNX2_HC_COALESCE_NOW, 4450 1 << (11 + sblk_num)); 4451 udelay(10); 4452 i++; 4453 barrier(); 4454 } 4455 if (cp->status_blk.bnx2->status_completion_producer_index) { 4456 cnic_free_irq(dev); 4457 goto failed; 4458 } 4459 4460 } else { 4461 struct status_block *sblk = cp->status_blk.gen; 4462 u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND); 4463 int i = 0; 4464 4465 while (sblk->status_completion_producer_index && i < 10) { 4466 CNIC_WR(dev, BNX2_HC_COMMAND, 4467 hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT); 4468 udelay(10); 4469 i++; 4470 barrier(); 4471 } 4472 if (sblk->status_completion_producer_index) 4473 goto failed; 4474 4475 } 4476 return 0; 4477 4478 failed: 4479 netdev_err(dev->netdev, "KCQ index not resetting to 0\n"); 4480 return -EBUSY; 4481 } 4482 4483 static void cnic_enable_bnx2_int(struct cnic_dev *dev) 4484 { 4485 struct cnic_local *cp = dev->cnic_priv; 4486 struct cnic_eth_dev *ethdev = cp->ethdev; 4487 4488 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4489 return; 4490 4491 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4492 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 4493 } 4494 4495 static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev) 4496 { 4497 struct cnic_local *cp = dev->cnic_priv; 4498 struct cnic_eth_dev *ethdev = cp->ethdev; 4499 4500 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4501 return; 4502 4503 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4504 BNX2_PCICFG_INT_ACK_CMD_MASK_INT); 4505 CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD); 4506 synchronize_irq(ethdev->irq_arr[0].vector); 4507 } 4508 4509 static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev) 4510 { 4511 struct cnic_local *cp = dev->cnic_priv; 4512 struct cnic_eth_dev *ethdev = cp->ethdev; 4513 struct cnic_uio_dev *udev = cp->udev; 4514 u32 cid_addr, tx_cid, sb_id; 4515 u32 val, offset0, offset1, offset2, offset3; 4516 int i; 4517 struct bnx2_tx_bd *txbd; 4518 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4519 struct status_block *s_blk = cp->status_blk.gen; 4520 4521 sb_id = cp->status_blk_num; 4522 tx_cid = 20; 4523 cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2; 4524 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4525 struct status_block_msix *sblk = cp->status_blk.bnx2; 4526 4527 tx_cid = TX_TSS_CID + sb_id - 1; 4528 CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) | 4529 (TX_TSS_CID << 7)); 4530 cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index; 4531 } 4532 cp->tx_cons = *cp->tx_cons_ptr; 4533 4534 cid_addr = GET_CID_ADDR(tx_cid); 4535 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 4536 u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40; 4537 4538 for (i = 0; i < PHY_CTX_SIZE; i += 4) 4539 cnic_ctx_wr(dev, cid_addr2, i, 0); 4540 4541 offset0 = BNX2_L2CTX_TYPE_XI; 4542 offset1 = BNX2_L2CTX_CMD_TYPE_XI; 4543 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI; 4544 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI; 4545 } else { 4546 cnic_init_context(dev, tx_cid); 4547 cnic_init_context(dev, tx_cid + 1); 4548 4549 offset0 = BNX2_L2CTX_TYPE; 4550 offset1 = BNX2_L2CTX_CMD_TYPE; 4551 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI; 4552 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO; 4553 } 4554 val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2; 4555 cnic_ctx_wr(dev, cid_addr, offset0, val); 4556 4557 val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16); 4558 cnic_ctx_wr(dev, cid_addr, offset1, val); 4559 4560 txbd = udev->l2_ring; 4561 4562 buf_map = udev->l2_buf_map; 4563 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) { 4564 txbd->tx_bd_haddr_hi = (u64) buf_map >> 32; 4565 txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4566 } 4567 val = (u64) ring_map >> 32; 4568 cnic_ctx_wr(dev, cid_addr, offset2, val); 4569 txbd->tx_bd_haddr_hi = val; 4570 4571 val = (u64) ring_map & 0xffffffff; 4572 cnic_ctx_wr(dev, cid_addr, offset3, val); 4573 txbd->tx_bd_haddr_lo = val; 4574 } 4575 4576 static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev) 4577 { 4578 struct cnic_local *cp = dev->cnic_priv; 4579 struct cnic_eth_dev *ethdev = cp->ethdev; 4580 struct cnic_uio_dev *udev = cp->udev; 4581 u32 cid_addr, sb_id, val, coal_reg, coal_val; 4582 int i; 4583 struct bnx2_rx_bd *rxbd; 4584 struct status_block *s_blk = cp->status_blk.gen; 4585 dma_addr_t ring_map = udev->l2_ring_map; 4586 4587 sb_id = cp->status_blk_num; 4588 cnic_init_context(dev, 2); 4589 cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2; 4590 coal_reg = BNX2_HC_COMMAND; 4591 coal_val = CNIC_RD(dev, coal_reg); 4592 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4593 struct status_block_msix *sblk = cp->status_blk.bnx2; 4594 4595 cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index; 4596 coal_reg = BNX2_HC_COALESCE_NOW; 4597 coal_val = 1 << (11 + sb_id); 4598 } 4599 i = 0; 4600 while (!(*cp->rx_cons_ptr != 0) && i < 10) { 4601 CNIC_WR(dev, coal_reg, coal_val); 4602 udelay(10); 4603 i++; 4604 barrier(); 4605 } 4606 cp->rx_cons = *cp->rx_cons_ptr; 4607 4608 cid_addr = GET_CID_ADDR(2); 4609 val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE | 4610 BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8); 4611 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val); 4612 4613 if (sb_id == 0) 4614 val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT; 4615 else 4616 val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id); 4617 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val); 4618 4619 rxbd = udev->l2_ring + BNX2_PAGE_SIZE; 4620 for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) { 4621 dma_addr_t buf_map; 4622 int n = (i % cp->l2_rx_ring_size) + 1; 4623 4624 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4625 rxbd->rx_bd_len = cp->l2_single_buf_size; 4626 rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END; 4627 rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32; 4628 rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4629 } 4630 val = (u64) (ring_map + BNX2_PAGE_SIZE) >> 32; 4631 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val); 4632 rxbd->rx_bd_haddr_hi = val; 4633 4634 val = (u64) (ring_map + BNX2_PAGE_SIZE) & 0xffffffff; 4635 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val); 4636 rxbd->rx_bd_haddr_lo = val; 4637 4638 val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD); 4639 cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2)); 4640 } 4641 4642 static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev) 4643 { 4644 struct kwqe *wqes[1], l2kwqe; 4645 4646 memset(&l2kwqe, 0, sizeof(l2kwqe)); 4647 wqes[0] = &l2kwqe; 4648 l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) | 4649 (L2_KWQE_OPCODE_VALUE_FLUSH << 4650 KWQE_OPCODE_SHIFT) | 2; 4651 dev->submit_kwqes(dev, wqes, 1); 4652 } 4653 4654 static void cnic_set_bnx2_mac(struct cnic_dev *dev) 4655 { 4656 struct cnic_local *cp = dev->cnic_priv; 4657 u32 val; 4658 4659 val = cp->func << 2; 4660 4661 cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val); 4662 4663 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4664 BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER); 4665 dev->mac_addr[0] = (u8) (val >> 8); 4666 dev->mac_addr[1] = (u8) val; 4667 4668 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val); 4669 4670 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4671 BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER); 4672 dev->mac_addr[2] = (u8) (val >> 24); 4673 dev->mac_addr[3] = (u8) (val >> 16); 4674 dev->mac_addr[4] = (u8) (val >> 8); 4675 dev->mac_addr[5] = (u8) val; 4676 4677 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val); 4678 4679 val = 4 | BNX2_RPM_SORT_USER2_BC_EN; 4680 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4681 val |= BNX2_RPM_SORT_USER2_PROM_VLAN; 4682 4683 CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0); 4684 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val); 4685 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA); 4686 } 4687 4688 static int cnic_start_bnx2_hw(struct cnic_dev *dev) 4689 { 4690 struct cnic_local *cp = dev->cnic_priv; 4691 struct cnic_eth_dev *ethdev = cp->ethdev; 4692 struct status_block *sblk = cp->status_blk.gen; 4693 u32 val, kcq_cid_addr, kwq_cid_addr; 4694 int err; 4695 4696 cnic_set_bnx2_mac(dev); 4697 4698 val = CNIC_RD(dev, BNX2_MQ_CONFIG); 4699 val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE; 4700 if (BNX2_PAGE_BITS > 12) 4701 val |= (12 - 8) << 4; 4702 else 4703 val |= (BNX2_PAGE_BITS - 8) << 4; 4704 4705 CNIC_WR(dev, BNX2_MQ_CONFIG, val); 4706 4707 CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8); 4708 CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220); 4709 CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220); 4710 4711 err = cnic_setup_5709_context(dev, 1); 4712 if (err) 4713 return err; 4714 4715 cnic_init_context(dev, KWQ_CID); 4716 cnic_init_context(dev, KCQ_CID); 4717 4718 kwq_cid_addr = GET_CID_ADDR(KWQ_CID); 4719 cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX; 4720 4721 cp->max_kwq_idx = MAX_KWQ_IDX; 4722 cp->kwq_prod_idx = 0; 4723 cp->kwq_con_idx = 0; 4724 set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 4725 4726 if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708) 4727 cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15; 4728 else 4729 cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index; 4730 4731 /* Initialize the kernel work queue context. */ 4732 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4733 (BNX2_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4734 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val); 4735 4736 val = (BNX2_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16; 4737 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4738 4739 val = ((BNX2_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT; 4740 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4741 4742 val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32); 4743 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4744 4745 val = (u32) cp->kwq_info.pgtbl_map; 4746 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4747 4748 kcq_cid_addr = GET_CID_ADDR(KCQ_CID); 4749 cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX; 4750 4751 cp->kcq1.sw_prod_idx = 0; 4752 cp->kcq1.hw_prod_idx_ptr = 4753 &sblk->status_completion_producer_index; 4754 4755 cp->kcq1.status_idx_ptr = &sblk->status_idx; 4756 4757 /* Initialize the kernel complete queue context. */ 4758 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4759 (BNX2_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4760 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val); 4761 4762 val = (BNX2_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16; 4763 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4764 4765 val = ((BNX2_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT; 4766 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4767 4768 val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32); 4769 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4770 4771 val = (u32) cp->kcq1.dma.pgtbl_map; 4772 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4773 4774 cp->int_num = 0; 4775 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4776 struct status_block_msix *msblk = cp->status_blk.bnx2; 4777 u32 sb_id = cp->status_blk_num; 4778 u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id); 4779 4780 cp->kcq1.hw_prod_idx_ptr = 4781 &msblk->status_completion_producer_index; 4782 cp->kcq1.status_idx_ptr = &msblk->status_idx; 4783 cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index; 4784 cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT; 4785 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4786 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4787 } 4788 4789 /* Enable Commnad Scheduler notification when we write to the 4790 * host producer index of the kernel contexts. */ 4791 CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2); 4792 4793 /* Enable Command Scheduler notification when we write to either 4794 * the Send Queue or Receive Queue producer indexes of the kernel 4795 * bypass contexts. */ 4796 CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7); 4797 CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7); 4798 4799 /* Notify COM when the driver post an application buffer. */ 4800 CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000); 4801 4802 /* Set the CP and COM doorbells. These two processors polls the 4803 * doorbell for a non zero value before running. This must be done 4804 * after setting up the kernel queue contexts. */ 4805 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1); 4806 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1); 4807 4808 cnic_init_bnx2_tx_ring(dev); 4809 cnic_init_bnx2_rx_ring(dev); 4810 4811 err = cnic_init_bnx2_irq(dev); 4812 if (err) { 4813 netdev_err(dev->netdev, "cnic_init_irq failed\n"); 4814 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 4815 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 4816 return err; 4817 } 4818 4819 return 0; 4820 } 4821 4822 static void cnic_setup_bnx2x_context(struct cnic_dev *dev) 4823 { 4824 struct cnic_local *cp = dev->cnic_priv; 4825 struct cnic_eth_dev *ethdev = cp->ethdev; 4826 u32 start_offset = ethdev->ctx_tbl_offset; 4827 int i; 4828 4829 for (i = 0; i < cp->ctx_blks; i++) { 4830 struct cnic_ctx *ctx = &cp->ctx_arr[i]; 4831 dma_addr_t map = ctx->mapping; 4832 4833 if (cp->ctx_align) { 4834 unsigned long mask = cp->ctx_align - 1; 4835 4836 map = (map + mask) & ~mask; 4837 } 4838 4839 cnic_ctx_tbl_wr(dev, start_offset + i, map); 4840 } 4841 } 4842 4843 static int cnic_init_bnx2x_irq(struct cnic_dev *dev) 4844 { 4845 struct cnic_local *cp = dev->cnic_priv; 4846 struct cnic_eth_dev *ethdev = cp->ethdev; 4847 int err = 0; 4848 4849 tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2x_bh, 4850 (unsigned long) dev); 4851 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 4852 err = cnic_request_irq(dev); 4853 4854 return err; 4855 } 4856 4857 static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev, 4858 u16 sb_id, u8 sb_index, 4859 u8 disable) 4860 { 4861 struct bnx2x *bp = netdev_priv(dev->netdev); 4862 4863 u32 addr = BAR_CSTRORM_INTMEM + 4864 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4865 offsetof(struct hc_status_block_data_e1x, index_data) + 4866 sizeof(struct hc_index_data)*sb_index + 4867 offsetof(struct hc_index_data, flags); 4868 u16 flags = CNIC_RD16(dev, addr); 4869 /* clear and set */ 4870 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4871 flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) & 4872 HC_INDEX_DATA_HC_ENABLED); 4873 CNIC_WR16(dev, addr, flags); 4874 } 4875 4876 static void cnic_enable_bnx2x_int(struct cnic_dev *dev) 4877 { 4878 struct cnic_local *cp = dev->cnic_priv; 4879 struct bnx2x *bp = netdev_priv(dev->netdev); 4880 u8 sb_id = cp->status_blk_num; 4881 4882 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 4883 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4884 offsetof(struct hc_status_block_data_e1x, index_data) + 4885 sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS + 4886 offsetof(struct hc_index_data, timeout), 64 / 4); 4887 cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0); 4888 } 4889 4890 static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev) 4891 { 4892 } 4893 4894 static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev, 4895 struct client_init_ramrod_data *data) 4896 { 4897 struct cnic_local *cp = dev->cnic_priv; 4898 struct cnic_uio_dev *udev = cp->udev; 4899 union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring; 4900 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4901 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4902 int i; 4903 u32 cli = cp->ethdev->iscsi_l2_client_id; 4904 u32 val; 4905 4906 memset(txbd, 0, BNX2_PAGE_SIZE); 4907 4908 buf_map = udev->l2_buf_map; 4909 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) { 4910 struct eth_tx_start_bd *start_bd = &txbd->start_bd; 4911 struct eth_tx_parse_bd_e1x *pbd_e1x = 4912 &((txbd + 1)->parse_bd_e1x); 4913 struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2); 4914 struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd); 4915 4916 start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4917 start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4918 reg_bd->addr_hi = start_bd->addr_hi; 4919 reg_bd->addr_lo = start_bd->addr_lo + 0x10; 4920 start_bd->nbytes = cpu_to_le16(0x10); 4921 start_bd->nbd = cpu_to_le16(3); 4922 start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 4923 start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS; 4924 start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 4925 4926 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) 4927 pbd_e2->parsing_data = (UNICAST_ADDRESS << 4928 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT); 4929 else 4930 pbd_e1x->global_data = (UNICAST_ADDRESS << 4931 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT); 4932 } 4933 4934 val = (u64) ring_map >> 32; 4935 txbd->next_bd.addr_hi = cpu_to_le32(val); 4936 4937 data->tx.tx_bd_page_base.hi = cpu_to_le32(val); 4938 4939 val = (u64) ring_map & 0xffffffff; 4940 txbd->next_bd.addr_lo = cpu_to_le32(val); 4941 4942 data->tx.tx_bd_page_base.lo = cpu_to_le32(val); 4943 4944 /* Other ramrod params */ 4945 data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS; 4946 data->tx.tx_status_block_id = BNX2X_DEF_SB_ID; 4947 4948 /* reset xstorm per client statistics */ 4949 if (cli < MAX_STAT_COUNTER_ID) { 4950 data->general.statistics_zero_flg = 1; 4951 data->general.statistics_en_flg = 1; 4952 data->general.statistics_counter_id = cli; 4953 } 4954 4955 cp->tx_cons_ptr = 4956 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS]; 4957 } 4958 4959 static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev, 4960 struct client_init_ramrod_data *data) 4961 { 4962 struct cnic_local *cp = dev->cnic_priv; 4963 struct cnic_uio_dev *udev = cp->udev; 4964 struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring + 4965 BNX2_PAGE_SIZE); 4966 struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *) 4967 (udev->l2_ring + (2 * BNX2_PAGE_SIZE)); 4968 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4969 int i; 4970 u32 cli = cp->ethdev->iscsi_l2_client_id; 4971 int cl_qzone_id = BNX2X_CL_QZONE_ID(cp, cli); 4972 u32 val; 4973 dma_addr_t ring_map = udev->l2_ring_map; 4974 4975 /* General data */ 4976 data->general.client_id = cli; 4977 data->general.activate_flg = 1; 4978 data->general.sp_client_id = cli; 4979 data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14); 4980 data->general.func_id = cp->pfid; 4981 4982 for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) { 4983 dma_addr_t buf_map; 4984 int n = (i % cp->l2_rx_ring_size) + 1; 4985 4986 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4987 rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4988 rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4989 } 4990 4991 val = (u64) (ring_map + BNX2_PAGE_SIZE) >> 32; 4992 rxbd->addr_hi = cpu_to_le32(val); 4993 data->rx.bd_page_base.hi = cpu_to_le32(val); 4994 4995 val = (u64) (ring_map + BNX2_PAGE_SIZE) & 0xffffffff; 4996 rxbd->addr_lo = cpu_to_le32(val); 4997 data->rx.bd_page_base.lo = cpu_to_le32(val); 4998 4999 rxcqe += BNX2X_MAX_RCQ_DESC_CNT; 5000 val = (u64) (ring_map + (2 * BNX2_PAGE_SIZE)) >> 32; 5001 rxcqe->addr_hi = cpu_to_le32(val); 5002 data->rx.cqe_page_base.hi = cpu_to_le32(val); 5003 5004 val = (u64) (ring_map + (2 * BNX2_PAGE_SIZE)) & 0xffffffff; 5005 rxcqe->addr_lo = cpu_to_le32(val); 5006 data->rx.cqe_page_base.lo = cpu_to_le32(val); 5007 5008 /* Other ramrod params */ 5009 data->rx.client_qzone_id = cl_qzone_id; 5010 data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS; 5011 data->rx.status_block_id = BNX2X_DEF_SB_ID; 5012 5013 data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT; 5014 5015 data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size); 5016 data->rx.outer_vlan_removal_enable_flg = 1; 5017 data->rx.silent_vlan_removal_flg = 1; 5018 data->rx.silent_vlan_value = 0; 5019 data->rx.silent_vlan_mask = 0xffff; 5020 5021 cp->rx_cons_ptr = 5022 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS]; 5023 cp->rx_cons = *cp->rx_cons_ptr; 5024 } 5025 5026 static void cnic_init_bnx2x_kcq(struct cnic_dev *dev) 5027 { 5028 struct cnic_local *cp = dev->cnic_priv; 5029 struct bnx2x *bp = netdev_priv(dev->netdev); 5030 u32 pfid = cp->pfid; 5031 5032 cp->kcq1.io_addr = BAR_CSTRORM_INTMEM + 5033 CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0); 5034 cp->kcq1.sw_prod_idx = 0; 5035 5036 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 5037 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5038 5039 cp->kcq1.hw_prod_idx_ptr = 5040 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5041 cp->kcq1.status_idx_ptr = 5042 &sb->sb.running_index[SM_RX_ID]; 5043 } else { 5044 struct host_hc_status_block_e1x *sb = cp->status_blk.gen; 5045 5046 cp->kcq1.hw_prod_idx_ptr = 5047 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5048 cp->kcq1.status_idx_ptr = 5049 &sb->sb.running_index[SM_RX_ID]; 5050 } 5051 5052 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 5053 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5054 5055 cp->kcq2.io_addr = BAR_USTRORM_INTMEM + 5056 USTORM_FCOE_EQ_PROD_OFFSET(pfid); 5057 cp->kcq2.sw_prod_idx = 0; 5058 cp->kcq2.hw_prod_idx_ptr = 5059 &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS]; 5060 cp->kcq2.status_idx_ptr = 5061 &sb->sb.running_index[SM_RX_ID]; 5062 } 5063 } 5064 5065 static int cnic_start_bnx2x_hw(struct cnic_dev *dev) 5066 { 5067 struct cnic_local *cp = dev->cnic_priv; 5068 struct bnx2x *bp = netdev_priv(dev->netdev); 5069 struct cnic_eth_dev *ethdev = cp->ethdev; 5070 int func, ret; 5071 u32 pfid; 5072 5073 dev->stats_addr = ethdev->addr_drv_info_to_mcp; 5074 cp->port_mode = bp->common.chip_port_mode; 5075 cp->pfid = bp->pfid; 5076 cp->func = bp->pf_num; 5077 5078 func = CNIC_FUNC(cp); 5079 pfid = cp->pfid; 5080 5081 ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ, 5082 cp->iscsi_start_cid, 0); 5083 5084 if (ret) 5085 return -ENOMEM; 5086 5087 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 5088 ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn, 5089 cp->fcoe_start_cid, 0); 5090 5091 if (ret) 5092 return -ENOMEM; 5093 } 5094 5095 cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2; 5096 5097 cnic_init_bnx2x_kcq(dev); 5098 5099 /* Only 1 EQ */ 5100 CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX); 5101 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5102 CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0); 5103 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5104 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0), 5105 cp->kcq1.dma.pg_map_arr[1] & 0xffffffff); 5106 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5107 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4, 5108 (u64) cp->kcq1.dma.pg_map_arr[1] >> 32); 5109 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5110 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0), 5111 cp->kcq1.dma.pg_map_arr[0] & 0xffffffff); 5112 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5113 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4, 5114 (u64) cp->kcq1.dma.pg_map_arr[0] >> 32); 5115 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5116 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1); 5117 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 5118 CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num); 5119 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5120 CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0), 5121 HC_INDEX_ISCSI_EQ_CONS); 5122 5123 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5124 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid), 5125 cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff); 5126 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5127 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4, 5128 (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32); 5129 5130 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 5131 TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF); 5132 5133 cnic_setup_bnx2x_context(dev); 5134 5135 ret = cnic_init_bnx2x_irq(dev); 5136 if (ret) 5137 return ret; 5138 5139 return 0; 5140 } 5141 5142 static void cnic_init_rings(struct cnic_dev *dev) 5143 { 5144 struct cnic_local *cp = dev->cnic_priv; 5145 struct bnx2x *bp = netdev_priv(dev->netdev); 5146 struct cnic_uio_dev *udev = cp->udev; 5147 5148 if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5149 return; 5150 5151 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5152 cnic_init_bnx2_tx_ring(dev); 5153 cnic_init_bnx2_rx_ring(dev); 5154 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5155 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5156 u32 cli = cp->ethdev->iscsi_l2_client_id; 5157 u32 cid = cp->ethdev->iscsi_l2_cid; 5158 u32 cl_qzone_id; 5159 struct client_init_ramrod_data *data; 5160 union l5cm_specific_data l5_data; 5161 struct ustorm_eth_rx_producers rx_prods = {0}; 5162 u32 off, i, *cid_ptr; 5163 5164 rx_prods.bd_prod = 0; 5165 rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT; 5166 barrier(); 5167 5168 cl_qzone_id = BNX2X_CL_QZONE_ID(cp, cli); 5169 5170 off = BAR_USTRORM_INTMEM + 5171 (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id) ? 5172 USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) : 5173 USTORM_RX_PRODS_E1X_OFFSET(CNIC_PORT(cp), cli)); 5174 5175 for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++) 5176 CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]); 5177 5178 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5179 5180 data = udev->l2_buf; 5181 cid_ptr = udev->l2_buf + 12; 5182 5183 memset(data, 0, sizeof(*data)); 5184 5185 cnic_init_bnx2x_tx_ring(dev, data); 5186 cnic_init_bnx2x_rx_ring(dev, data); 5187 5188 l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff; 5189 l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32; 5190 5191 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5192 5193 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP, 5194 cid, ETH_CONNECTION_TYPE, &l5_data); 5195 5196 i = 0; 5197 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5198 ++i < 10) 5199 msleep(1); 5200 5201 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5202 netdev_err(dev->netdev, 5203 "iSCSI CLIENT_SETUP did not complete\n"); 5204 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5205 cnic_ring_ctl(dev, cid, cli, 1); 5206 *cid_ptr = cid; 5207 } 5208 } 5209 5210 static void cnic_shutdown_rings(struct cnic_dev *dev) 5211 { 5212 struct cnic_local *cp = dev->cnic_priv; 5213 struct cnic_uio_dev *udev = cp->udev; 5214 void *rx_ring; 5215 5216 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5217 return; 5218 5219 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5220 cnic_shutdown_bnx2_rx_ring(dev); 5221 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5222 u32 cli = cp->ethdev->iscsi_l2_client_id; 5223 u32 cid = cp->ethdev->iscsi_l2_cid; 5224 union l5cm_specific_data l5_data; 5225 int i; 5226 5227 cnic_ring_ctl(dev, cid, cli, 0); 5228 5229 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5230 5231 l5_data.phy_address.lo = cli; 5232 l5_data.phy_address.hi = 0; 5233 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT, 5234 cid, ETH_CONNECTION_TYPE, &l5_data); 5235 i = 0; 5236 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5237 ++i < 10) 5238 msleep(1); 5239 5240 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5241 netdev_err(dev->netdev, 5242 "iSCSI CLIENT_HALT did not complete\n"); 5243 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5244 5245 memset(&l5_data, 0, sizeof(l5_data)); 5246 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 5247 cid, NONE_CONNECTION_TYPE, &l5_data); 5248 msleep(10); 5249 } 5250 clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5251 rx_ring = udev->l2_ring + BNX2_PAGE_SIZE; 5252 memset(rx_ring, 0, BNX2_PAGE_SIZE); 5253 } 5254 5255 static int cnic_register_netdev(struct cnic_dev *dev) 5256 { 5257 struct cnic_local *cp = dev->cnic_priv; 5258 struct cnic_eth_dev *ethdev = cp->ethdev; 5259 int err; 5260 5261 if (!ethdev) 5262 return -ENODEV; 5263 5264 if (ethdev->drv_state & CNIC_DRV_STATE_REGD) 5265 return 0; 5266 5267 err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev); 5268 if (err) 5269 netdev_err(dev->netdev, "register_cnic failed\n"); 5270 5271 return err; 5272 } 5273 5274 static void cnic_unregister_netdev(struct cnic_dev *dev) 5275 { 5276 struct cnic_local *cp = dev->cnic_priv; 5277 struct cnic_eth_dev *ethdev = cp->ethdev; 5278 5279 if (!ethdev) 5280 return; 5281 5282 ethdev->drv_unregister_cnic(dev->netdev); 5283 } 5284 5285 static int cnic_start_hw(struct cnic_dev *dev) 5286 { 5287 struct cnic_local *cp = dev->cnic_priv; 5288 struct cnic_eth_dev *ethdev = cp->ethdev; 5289 int err; 5290 5291 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5292 return -EALREADY; 5293 5294 dev->regview = ethdev->io_base; 5295 pci_dev_get(dev->pcidev); 5296 cp->func = PCI_FUNC(dev->pcidev->devfn); 5297 cp->status_blk.gen = ethdev->irq_arr[0].status_blk; 5298 cp->status_blk_num = ethdev->irq_arr[0].status_blk_num; 5299 5300 err = cp->alloc_resc(dev); 5301 if (err) { 5302 netdev_err(dev->netdev, "allocate resource failure\n"); 5303 goto err1; 5304 } 5305 5306 err = cp->start_hw(dev); 5307 if (err) 5308 goto err1; 5309 5310 err = cnic_cm_open(dev); 5311 if (err) 5312 goto err1; 5313 5314 set_bit(CNIC_F_CNIC_UP, &dev->flags); 5315 5316 cp->enable_int(dev); 5317 5318 return 0; 5319 5320 err1: 5321 cp->free_resc(dev); 5322 pci_dev_put(dev->pcidev); 5323 return err; 5324 } 5325 5326 static void cnic_stop_bnx2_hw(struct cnic_dev *dev) 5327 { 5328 cnic_disable_bnx2_int_sync(dev); 5329 5330 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 5331 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 5332 5333 cnic_init_context(dev, KWQ_CID); 5334 cnic_init_context(dev, KCQ_CID); 5335 5336 cnic_setup_5709_context(dev, 0); 5337 cnic_free_irq(dev); 5338 5339 cnic_free_resc(dev); 5340 } 5341 5342 5343 static void cnic_stop_bnx2x_hw(struct cnic_dev *dev) 5344 { 5345 struct cnic_local *cp = dev->cnic_priv; 5346 struct bnx2x *bp = netdev_priv(dev->netdev); 5347 u32 hc_index = HC_INDEX_ISCSI_EQ_CONS; 5348 u32 sb_id = cp->status_blk_num; 5349 u32 idx_off, syn_off; 5350 5351 cnic_free_irq(dev); 5352 5353 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 5354 idx_off = offsetof(struct hc_status_block_e2, index_values) + 5355 (hc_index * sizeof(u16)); 5356 5357 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id); 5358 } else { 5359 idx_off = offsetof(struct hc_status_block_e1x, index_values) + 5360 (hc_index * sizeof(u16)); 5361 5362 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id); 5363 } 5364 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0); 5365 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) + 5366 idx_off, 0); 5367 5368 *cp->kcq1.hw_prod_idx_ptr = 0; 5369 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5370 CSTORM_ISCSI_EQ_CONS_OFFSET(cp->pfid, 0), 0); 5371 CNIC_WR16(dev, cp->kcq1.io_addr, 0); 5372 cnic_free_resc(dev); 5373 } 5374 5375 static void cnic_stop_hw(struct cnic_dev *dev) 5376 { 5377 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5378 struct cnic_local *cp = dev->cnic_priv; 5379 int i = 0; 5380 5381 /* Need to wait for the ring shutdown event to complete 5382 * before clearing the CNIC_UP flag. 5383 */ 5384 while (cp->udev && cp->udev->uio_dev != -1 && i < 15) { 5385 msleep(100); 5386 i++; 5387 } 5388 cnic_shutdown_rings(dev); 5389 cp->stop_cm(dev); 5390 clear_bit(CNIC_F_CNIC_UP, &dev->flags); 5391 RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL); 5392 synchronize_rcu(); 5393 cnic_cm_shutdown(dev); 5394 cp->stop_hw(dev); 5395 pci_dev_put(dev->pcidev); 5396 } 5397 } 5398 5399 static void cnic_free_dev(struct cnic_dev *dev) 5400 { 5401 int i = 0; 5402 5403 while ((atomic_read(&dev->ref_count) != 0) && i < 10) { 5404 msleep(100); 5405 i++; 5406 } 5407 if (atomic_read(&dev->ref_count) != 0) 5408 netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n"); 5409 5410 netdev_info(dev->netdev, "Removed CNIC device\n"); 5411 dev_put(dev->netdev); 5412 kfree(dev); 5413 } 5414 5415 static struct cnic_dev *cnic_alloc_dev(struct net_device *dev, 5416 struct pci_dev *pdev) 5417 { 5418 struct cnic_dev *cdev; 5419 struct cnic_local *cp; 5420 int alloc_size; 5421 5422 alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local); 5423 5424 cdev = kzalloc(alloc_size , GFP_KERNEL); 5425 if (cdev == NULL) { 5426 netdev_err(dev, "allocate dev struct failure\n"); 5427 return NULL; 5428 } 5429 5430 cdev->netdev = dev; 5431 cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev); 5432 cdev->register_device = cnic_register_device; 5433 cdev->unregister_device = cnic_unregister_device; 5434 cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv; 5435 5436 cp = cdev->cnic_priv; 5437 cp->dev = cdev; 5438 cp->l2_single_buf_size = 0x400; 5439 cp->l2_rx_ring_size = 3; 5440 5441 spin_lock_init(&cp->cnic_ulp_lock); 5442 5443 netdev_info(dev, "Added CNIC device\n"); 5444 5445 return cdev; 5446 } 5447 5448 static struct cnic_dev *init_bnx2_cnic(struct net_device *dev) 5449 { 5450 struct pci_dev *pdev; 5451 struct cnic_dev *cdev; 5452 struct cnic_local *cp; 5453 struct bnx2 *bp = netdev_priv(dev); 5454 struct cnic_eth_dev *ethdev = NULL; 5455 5456 if (bp->cnic_probe) 5457 ethdev = (bp->cnic_probe)(dev); 5458 5459 if (!ethdev) 5460 return NULL; 5461 5462 pdev = ethdev->pdev; 5463 if (!pdev) 5464 return NULL; 5465 5466 dev_hold(dev); 5467 pci_dev_get(pdev); 5468 if ((pdev->device == PCI_DEVICE_ID_NX2_5709 || 5469 pdev->device == PCI_DEVICE_ID_NX2_5709S) && 5470 (pdev->revision < 0x10)) { 5471 pci_dev_put(pdev); 5472 goto cnic_err; 5473 } 5474 pci_dev_put(pdev); 5475 5476 cdev = cnic_alloc_dev(dev, pdev); 5477 if (cdev == NULL) 5478 goto cnic_err; 5479 5480 set_bit(CNIC_F_BNX2_CLASS, &cdev->flags); 5481 cdev->submit_kwqes = cnic_submit_bnx2_kwqes; 5482 5483 cp = cdev->cnic_priv; 5484 cp->ethdev = ethdev; 5485 cdev->pcidev = pdev; 5486 cp->chip_id = ethdev->chip_id; 5487 5488 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5489 5490 cp->cnic_ops = &cnic_bnx2_ops; 5491 cp->start_hw = cnic_start_bnx2_hw; 5492 cp->stop_hw = cnic_stop_bnx2_hw; 5493 cp->setup_pgtbl = cnic_setup_page_tbl; 5494 cp->alloc_resc = cnic_alloc_bnx2_resc; 5495 cp->free_resc = cnic_free_resc; 5496 cp->start_cm = cnic_cm_init_bnx2_hw; 5497 cp->stop_cm = cnic_cm_stop_bnx2_hw; 5498 cp->enable_int = cnic_enable_bnx2_int; 5499 cp->disable_int_sync = cnic_disable_bnx2_int_sync; 5500 cp->close_conn = cnic_close_bnx2_conn; 5501 return cdev; 5502 5503 cnic_err: 5504 dev_put(dev); 5505 return NULL; 5506 } 5507 5508 static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev) 5509 { 5510 struct pci_dev *pdev; 5511 struct cnic_dev *cdev; 5512 struct cnic_local *cp; 5513 struct bnx2x *bp = netdev_priv(dev); 5514 struct cnic_eth_dev *ethdev = NULL; 5515 5516 if (bp->cnic_probe) 5517 ethdev = bp->cnic_probe(dev); 5518 5519 if (!ethdev) 5520 return NULL; 5521 5522 pdev = ethdev->pdev; 5523 if (!pdev) 5524 return NULL; 5525 5526 dev_hold(dev); 5527 cdev = cnic_alloc_dev(dev, pdev); 5528 if (cdev == NULL) { 5529 dev_put(dev); 5530 return NULL; 5531 } 5532 5533 set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags); 5534 cdev->submit_kwqes = cnic_submit_bnx2x_kwqes; 5535 5536 cp = cdev->cnic_priv; 5537 cp->ethdev = ethdev; 5538 cdev->pcidev = pdev; 5539 cp->chip_id = ethdev->chip_id; 5540 5541 cdev->stats_addr = ethdev->addr_drv_info_to_mcp; 5542 5543 if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)) 5544 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5545 if (CNIC_SUPPORTS_FCOE(cp)) 5546 cdev->max_fcoe_conn = ethdev->max_fcoe_conn; 5547 5548 if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS) 5549 cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS; 5550 5551 memcpy(cdev->mac_addr, ethdev->iscsi_mac, 6); 5552 5553 cp->cnic_ops = &cnic_bnx2x_ops; 5554 cp->start_hw = cnic_start_bnx2x_hw; 5555 cp->stop_hw = cnic_stop_bnx2x_hw; 5556 cp->setup_pgtbl = cnic_setup_page_tbl_le; 5557 cp->alloc_resc = cnic_alloc_bnx2x_resc; 5558 cp->free_resc = cnic_free_resc; 5559 cp->start_cm = cnic_cm_init_bnx2x_hw; 5560 cp->stop_cm = cnic_cm_stop_bnx2x_hw; 5561 cp->enable_int = cnic_enable_bnx2x_int; 5562 cp->disable_int_sync = cnic_disable_bnx2x_int_sync; 5563 if (BNX2X_CHIP_IS_E2_PLUS(cp->chip_id)) { 5564 cp->ack_int = cnic_ack_bnx2x_e2_msix; 5565 cp->arm_int = cnic_arm_bnx2x_e2_msix; 5566 } else { 5567 cp->ack_int = cnic_ack_bnx2x_msix; 5568 cp->arm_int = cnic_arm_bnx2x_msix; 5569 } 5570 cp->close_conn = cnic_close_bnx2x_conn; 5571 return cdev; 5572 } 5573 5574 static struct cnic_dev *is_cnic_dev(struct net_device *dev) 5575 { 5576 struct ethtool_drvinfo drvinfo; 5577 struct cnic_dev *cdev = NULL; 5578 5579 if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) { 5580 memset(&drvinfo, 0, sizeof(drvinfo)); 5581 dev->ethtool_ops->get_drvinfo(dev, &drvinfo); 5582 5583 if (!strcmp(drvinfo.driver, "bnx2")) 5584 cdev = init_bnx2_cnic(dev); 5585 if (!strcmp(drvinfo.driver, "bnx2x")) 5586 cdev = init_bnx2x_cnic(dev); 5587 if (cdev) { 5588 write_lock(&cnic_dev_lock); 5589 list_add(&cdev->list, &cnic_dev_list); 5590 write_unlock(&cnic_dev_lock); 5591 } 5592 } 5593 return cdev; 5594 } 5595 5596 static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event, 5597 u16 vlan_id) 5598 { 5599 int if_type; 5600 5601 rcu_read_lock(); 5602 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 5603 struct cnic_ulp_ops *ulp_ops; 5604 void *ctx; 5605 5606 ulp_ops = rcu_dereference(cp->ulp_ops[if_type]); 5607 if (!ulp_ops || !ulp_ops->indicate_netevent) 5608 continue; 5609 5610 ctx = cp->ulp_handle[if_type]; 5611 5612 ulp_ops->indicate_netevent(ctx, event, vlan_id); 5613 } 5614 rcu_read_unlock(); 5615 } 5616 5617 /* netdev event handler */ 5618 static int cnic_netdev_event(struct notifier_block *this, unsigned long event, 5619 void *ptr) 5620 { 5621 struct net_device *netdev = ptr; 5622 struct cnic_dev *dev; 5623 int new_dev = 0; 5624 5625 dev = cnic_from_netdev(netdev); 5626 5627 if (!dev && (event == NETDEV_REGISTER || netif_running(netdev))) { 5628 /* Check for the hot-plug device */ 5629 dev = is_cnic_dev(netdev); 5630 if (dev) { 5631 new_dev = 1; 5632 cnic_hold(dev); 5633 } 5634 } 5635 if (dev) { 5636 struct cnic_local *cp = dev->cnic_priv; 5637 5638 if (new_dev) 5639 cnic_ulp_init(dev); 5640 else if (event == NETDEV_UNREGISTER) 5641 cnic_ulp_exit(dev); 5642 5643 if (event == NETDEV_UP || (new_dev && netif_running(netdev))) { 5644 if (cnic_register_netdev(dev) != 0) { 5645 cnic_put(dev); 5646 goto done; 5647 } 5648 if (!cnic_start_hw(dev)) 5649 cnic_ulp_start(dev); 5650 } 5651 5652 cnic_rcv_netevent(cp, event, 0); 5653 5654 if (event == NETDEV_GOING_DOWN) { 5655 cnic_ulp_stop(dev); 5656 cnic_stop_hw(dev); 5657 cnic_unregister_netdev(dev); 5658 } else if (event == NETDEV_UNREGISTER) { 5659 write_lock(&cnic_dev_lock); 5660 list_del_init(&dev->list); 5661 write_unlock(&cnic_dev_lock); 5662 5663 cnic_put(dev); 5664 cnic_free_dev(dev); 5665 goto done; 5666 } 5667 cnic_put(dev); 5668 } else { 5669 struct net_device *realdev; 5670 u16 vid; 5671 5672 vid = cnic_get_vlan(netdev, &realdev); 5673 if (realdev) { 5674 dev = cnic_from_netdev(realdev); 5675 if (dev) { 5676 vid |= VLAN_TAG_PRESENT; 5677 cnic_rcv_netevent(dev->cnic_priv, event, vid); 5678 cnic_put(dev); 5679 } 5680 } 5681 } 5682 done: 5683 return NOTIFY_DONE; 5684 } 5685 5686 static struct notifier_block cnic_netdev_notifier = { 5687 .notifier_call = cnic_netdev_event 5688 }; 5689 5690 static void cnic_release(void) 5691 { 5692 struct cnic_dev *dev; 5693 struct cnic_uio_dev *udev; 5694 5695 while (!list_empty(&cnic_dev_list)) { 5696 dev = list_entry(cnic_dev_list.next, struct cnic_dev, list); 5697 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5698 cnic_ulp_stop(dev); 5699 cnic_stop_hw(dev); 5700 } 5701 5702 cnic_ulp_exit(dev); 5703 cnic_unregister_netdev(dev); 5704 list_del_init(&dev->list); 5705 cnic_free_dev(dev); 5706 } 5707 while (!list_empty(&cnic_udev_list)) { 5708 udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev, 5709 list); 5710 cnic_free_uio(udev); 5711 } 5712 } 5713 5714 static int __init cnic_init(void) 5715 { 5716 int rc = 0; 5717 5718 pr_info("%s", version); 5719 5720 rc = register_netdevice_notifier(&cnic_netdev_notifier); 5721 if (rc) { 5722 cnic_release(); 5723 return rc; 5724 } 5725 5726 cnic_wq = create_singlethread_workqueue("cnic_wq"); 5727 if (!cnic_wq) { 5728 cnic_release(); 5729 unregister_netdevice_notifier(&cnic_netdev_notifier); 5730 return -ENOMEM; 5731 } 5732 5733 return 0; 5734 } 5735 5736 static void __exit cnic_exit(void) 5737 { 5738 unregister_netdevice_notifier(&cnic_netdev_notifier); 5739 cnic_release(); 5740 destroy_workqueue(cnic_wq); 5741 } 5742 5743 module_init(cnic_init); 5744 module_exit(cnic_exit); 5745