1 /* cnic.c: QLogic CNIC core network driver. 2 * 3 * Copyright (c) 2006-2014 Broadcom Corporation 4 * Copyright (c) 2014-2015 QLogic Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com) 11 * Previously modified and maintained by: Michael Chan <mchan@broadcom.com> 12 * Maintained By: Dept-HSGLinuxNICDev@qlogic.com 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/module.h> 18 19 #include <linux/kernel.h> 20 #include <linux/errno.h> 21 #include <linux/list.h> 22 #include <linux/slab.h> 23 #include <linux/pci.h> 24 #include <linux/init.h> 25 #include <linux/netdevice.h> 26 #include <linux/uio_driver.h> 27 #include <linux/in.h> 28 #include <linux/dma-mapping.h> 29 #include <linux/delay.h> 30 #include <linux/ethtool.h> 31 #include <linux/if_vlan.h> 32 #include <linux/prefetch.h> 33 #include <linux/random.h> 34 #include <linux/workqueue.h> 35 #if IS_ENABLED(CONFIG_VLAN_8021Q) 36 #define BCM_VLAN 1 37 #endif 38 #include <net/ip.h> 39 #include <net/tcp.h> 40 #include <net/route.h> 41 #include <net/ipv6.h> 42 #include <net/ip6_route.h> 43 #include <net/ip6_checksum.h> 44 #include <scsi/iscsi_if.h> 45 46 #define BCM_CNIC 1 47 #include "cnic_if.h" 48 #include "bnx2.h" 49 #include "bnx2x/bnx2x.h" 50 #include "bnx2x/bnx2x_reg.h" 51 #include "bnx2x/bnx2x_fw_defs.h" 52 #include "bnx2x/bnx2x_hsi.h" 53 #include "../../../scsi/bnx2i/57xx_iscsi_constants.h" 54 #include "../../../scsi/bnx2i/57xx_iscsi_hsi.h" 55 #include "../../../scsi/bnx2fc/bnx2fc_constants.h" 56 #include "cnic.h" 57 #include "cnic_defs.h" 58 59 #define CNIC_MODULE_NAME "cnic" 60 61 static char version[] = 62 "QLogic " CNIC_MODULE_NAME "Driver v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n"; 63 64 MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) " 65 "Chen (zongxi@broadcom.com"); 66 MODULE_DESCRIPTION("QLogic cnic Driver"); 67 MODULE_LICENSE("GPL"); 68 MODULE_VERSION(CNIC_MODULE_VERSION); 69 70 /* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */ 71 static LIST_HEAD(cnic_dev_list); 72 static LIST_HEAD(cnic_udev_list); 73 static DEFINE_RWLOCK(cnic_dev_lock); 74 static DEFINE_MUTEX(cnic_lock); 75 76 static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE]; 77 78 /* helper function, assuming cnic_lock is held */ 79 static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type) 80 { 81 return rcu_dereference_protected(cnic_ulp_tbl[type], 82 lockdep_is_held(&cnic_lock)); 83 } 84 85 static int cnic_service_bnx2(void *, void *); 86 static int cnic_service_bnx2x(void *, void *); 87 static int cnic_ctl(void *, struct cnic_ctl_info *); 88 89 static struct cnic_ops cnic_bnx2_ops = { 90 .cnic_owner = THIS_MODULE, 91 .cnic_handler = cnic_service_bnx2, 92 .cnic_ctl = cnic_ctl, 93 }; 94 95 static struct cnic_ops cnic_bnx2x_ops = { 96 .cnic_owner = THIS_MODULE, 97 .cnic_handler = cnic_service_bnx2x, 98 .cnic_ctl = cnic_ctl, 99 }; 100 101 static struct workqueue_struct *cnic_wq; 102 103 static void cnic_shutdown_rings(struct cnic_dev *); 104 static void cnic_init_rings(struct cnic_dev *); 105 static int cnic_cm_set_pg(struct cnic_sock *); 106 107 static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode) 108 { 109 struct cnic_uio_dev *udev = uinfo->priv; 110 struct cnic_dev *dev; 111 112 if (!capable(CAP_NET_ADMIN)) 113 return -EPERM; 114 115 if (udev->uio_dev != -1) 116 return -EBUSY; 117 118 rtnl_lock(); 119 dev = udev->dev; 120 121 if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 122 rtnl_unlock(); 123 return -ENODEV; 124 } 125 126 udev->uio_dev = iminor(inode); 127 128 cnic_shutdown_rings(dev); 129 cnic_init_rings(dev); 130 rtnl_unlock(); 131 132 return 0; 133 } 134 135 static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode) 136 { 137 struct cnic_uio_dev *udev = uinfo->priv; 138 139 udev->uio_dev = -1; 140 return 0; 141 } 142 143 static inline void cnic_hold(struct cnic_dev *dev) 144 { 145 atomic_inc(&dev->ref_count); 146 } 147 148 static inline void cnic_put(struct cnic_dev *dev) 149 { 150 atomic_dec(&dev->ref_count); 151 } 152 153 static inline void csk_hold(struct cnic_sock *csk) 154 { 155 atomic_inc(&csk->ref_count); 156 } 157 158 static inline void csk_put(struct cnic_sock *csk) 159 { 160 atomic_dec(&csk->ref_count); 161 } 162 163 static struct cnic_dev *cnic_from_netdev(struct net_device *netdev) 164 { 165 struct cnic_dev *cdev; 166 167 read_lock(&cnic_dev_lock); 168 list_for_each_entry(cdev, &cnic_dev_list, list) { 169 if (netdev == cdev->netdev) { 170 cnic_hold(cdev); 171 read_unlock(&cnic_dev_lock); 172 return cdev; 173 } 174 } 175 read_unlock(&cnic_dev_lock); 176 return NULL; 177 } 178 179 static inline void ulp_get(struct cnic_ulp_ops *ulp_ops) 180 { 181 atomic_inc(&ulp_ops->ref_count); 182 } 183 184 static inline void ulp_put(struct cnic_ulp_ops *ulp_ops) 185 { 186 atomic_dec(&ulp_ops->ref_count); 187 } 188 189 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val) 190 { 191 struct cnic_local *cp = dev->cnic_priv; 192 struct cnic_eth_dev *ethdev = cp->ethdev; 193 struct drv_ctl_info info; 194 struct drv_ctl_io *io = &info.data.io; 195 196 memset(&info, 0, sizeof(struct drv_ctl_info)); 197 info.cmd = DRV_CTL_CTX_WR_CMD; 198 io->cid_addr = cid_addr; 199 io->offset = off; 200 io->data = val; 201 ethdev->drv_ctl(dev->netdev, &info); 202 } 203 204 static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr) 205 { 206 struct cnic_local *cp = dev->cnic_priv; 207 struct cnic_eth_dev *ethdev = cp->ethdev; 208 struct drv_ctl_info info; 209 struct drv_ctl_io *io = &info.data.io; 210 211 memset(&info, 0, sizeof(struct drv_ctl_info)); 212 info.cmd = DRV_CTL_CTXTBL_WR_CMD; 213 io->offset = off; 214 io->dma_addr = addr; 215 ethdev->drv_ctl(dev->netdev, &info); 216 } 217 218 static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start) 219 { 220 struct cnic_local *cp = dev->cnic_priv; 221 struct cnic_eth_dev *ethdev = cp->ethdev; 222 struct drv_ctl_info info; 223 struct drv_ctl_l2_ring *ring = &info.data.ring; 224 225 memset(&info, 0, sizeof(struct drv_ctl_info)); 226 if (start) 227 info.cmd = DRV_CTL_START_L2_CMD; 228 else 229 info.cmd = DRV_CTL_STOP_L2_CMD; 230 231 ring->cid = cid; 232 ring->client_id = cl_id; 233 ethdev->drv_ctl(dev->netdev, &info); 234 } 235 236 static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val) 237 { 238 struct cnic_local *cp = dev->cnic_priv; 239 struct cnic_eth_dev *ethdev = cp->ethdev; 240 struct drv_ctl_info info; 241 struct drv_ctl_io *io = &info.data.io; 242 243 memset(&info, 0, sizeof(struct drv_ctl_info)); 244 info.cmd = DRV_CTL_IO_WR_CMD; 245 io->offset = off; 246 io->data = val; 247 ethdev->drv_ctl(dev->netdev, &info); 248 } 249 250 static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off) 251 { 252 struct cnic_local *cp = dev->cnic_priv; 253 struct cnic_eth_dev *ethdev = cp->ethdev; 254 struct drv_ctl_info info; 255 struct drv_ctl_io *io = &info.data.io; 256 257 memset(&info, 0, sizeof(struct drv_ctl_info)); 258 info.cmd = DRV_CTL_IO_RD_CMD; 259 io->offset = off; 260 ethdev->drv_ctl(dev->netdev, &info); 261 return io->data; 262 } 263 264 static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg, int state) 265 { 266 struct cnic_local *cp = dev->cnic_priv; 267 struct cnic_eth_dev *ethdev = cp->ethdev; 268 struct drv_ctl_info info; 269 struct fcoe_capabilities *fcoe_cap = 270 &info.data.register_data.fcoe_features; 271 272 memset(&info, 0, sizeof(struct drv_ctl_info)); 273 if (reg) { 274 info.cmd = DRV_CTL_ULP_REGISTER_CMD; 275 if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap) 276 memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap)); 277 } else { 278 info.cmd = DRV_CTL_ULP_UNREGISTER_CMD; 279 } 280 281 info.data.ulp_type = ulp_type; 282 info.drv_state = state; 283 ethdev->drv_ctl(dev->netdev, &info); 284 } 285 286 static int cnic_in_use(struct cnic_sock *csk) 287 { 288 return test_bit(SK_F_INUSE, &csk->flags); 289 } 290 291 static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count) 292 { 293 struct cnic_local *cp = dev->cnic_priv; 294 struct cnic_eth_dev *ethdev = cp->ethdev; 295 struct drv_ctl_info info; 296 297 memset(&info, 0, sizeof(struct drv_ctl_info)); 298 info.cmd = cmd; 299 info.data.credit.credit_count = count; 300 ethdev->drv_ctl(dev->netdev, &info); 301 } 302 303 static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid) 304 { 305 u32 i; 306 307 if (!cp->ctx_tbl) 308 return -EINVAL; 309 310 for (i = 0; i < cp->max_cid_space; i++) { 311 if (cp->ctx_tbl[i].cid == cid) { 312 *l5_cid = i; 313 return 0; 314 } 315 } 316 return -EINVAL; 317 } 318 319 static int cnic_send_nlmsg(struct cnic_local *cp, u32 type, 320 struct cnic_sock *csk) 321 { 322 struct iscsi_path path_req; 323 char *buf = NULL; 324 u16 len = 0; 325 u32 msg_type = ISCSI_KEVENT_IF_DOWN; 326 struct cnic_ulp_ops *ulp_ops; 327 struct cnic_uio_dev *udev = cp->udev; 328 int rc = 0, retry = 0; 329 330 if (!udev || udev->uio_dev == -1) 331 return -ENODEV; 332 333 if (csk) { 334 len = sizeof(path_req); 335 buf = (char *) &path_req; 336 memset(&path_req, 0, len); 337 338 msg_type = ISCSI_KEVENT_PATH_REQ; 339 path_req.handle = (u64) csk->l5_cid; 340 if (test_bit(SK_F_IPV6, &csk->flags)) { 341 memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0], 342 sizeof(struct in6_addr)); 343 path_req.ip_addr_len = 16; 344 } else { 345 memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0], 346 sizeof(struct in_addr)); 347 path_req.ip_addr_len = 4; 348 } 349 path_req.vlan_id = csk->vlan_id; 350 path_req.pmtu = csk->mtu; 351 } 352 353 while (retry < 3) { 354 rc = 0; 355 rcu_read_lock(); 356 ulp_ops = rcu_dereference(cp->ulp_ops[CNIC_ULP_ISCSI]); 357 if (ulp_ops) 358 rc = ulp_ops->iscsi_nl_send_msg( 359 cp->ulp_handle[CNIC_ULP_ISCSI], 360 msg_type, buf, len); 361 rcu_read_unlock(); 362 if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ) 363 break; 364 365 msleep(100); 366 retry++; 367 } 368 return rc; 369 } 370 371 static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8); 372 373 static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type, 374 char *buf, u16 len) 375 { 376 int rc = -EINVAL; 377 378 switch (msg_type) { 379 case ISCSI_UEVENT_PATH_UPDATE: { 380 struct cnic_local *cp; 381 u32 l5_cid; 382 struct cnic_sock *csk; 383 struct iscsi_path *path_resp; 384 385 if (len < sizeof(*path_resp)) 386 break; 387 388 path_resp = (struct iscsi_path *) buf; 389 cp = dev->cnic_priv; 390 l5_cid = (u32) path_resp->handle; 391 if (l5_cid >= MAX_CM_SK_TBL_SZ) 392 break; 393 394 if (!rcu_access_pointer(cp->ulp_ops[CNIC_ULP_L4])) { 395 rc = -ENODEV; 396 break; 397 } 398 csk = &cp->csk_tbl[l5_cid]; 399 csk_hold(csk); 400 if (cnic_in_use(csk) && 401 test_bit(SK_F_CONNECT_START, &csk->flags)) { 402 403 csk->vlan_id = path_resp->vlan_id; 404 405 memcpy(csk->ha, path_resp->mac_addr, ETH_ALEN); 406 if (test_bit(SK_F_IPV6, &csk->flags)) 407 memcpy(&csk->src_ip[0], &path_resp->src.v6_addr, 408 sizeof(struct in6_addr)); 409 else 410 memcpy(&csk->src_ip[0], &path_resp->src.v4_addr, 411 sizeof(struct in_addr)); 412 413 if (is_valid_ether_addr(csk->ha)) { 414 cnic_cm_set_pg(csk); 415 } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) && 416 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 417 418 cnic_cm_upcall(cp, csk, 419 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 420 clear_bit(SK_F_CONNECT_START, &csk->flags); 421 } 422 } 423 csk_put(csk); 424 rc = 0; 425 } 426 } 427 428 return rc; 429 } 430 431 static int cnic_offld_prep(struct cnic_sock *csk) 432 { 433 if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 434 return 0; 435 436 if (!test_bit(SK_F_CONNECT_START, &csk->flags)) { 437 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 438 return 0; 439 } 440 441 return 1; 442 } 443 444 static int cnic_close_prep(struct cnic_sock *csk) 445 { 446 clear_bit(SK_F_CONNECT_START, &csk->flags); 447 smp_mb__after_atomic(); 448 449 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 450 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 451 msleep(1); 452 453 return 1; 454 } 455 return 0; 456 } 457 458 static int cnic_abort_prep(struct cnic_sock *csk) 459 { 460 clear_bit(SK_F_CONNECT_START, &csk->flags); 461 smp_mb__after_atomic(); 462 463 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 464 msleep(1); 465 466 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 467 csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP; 468 return 1; 469 } 470 471 return 0; 472 } 473 474 int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops) 475 { 476 struct cnic_dev *dev; 477 478 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 479 pr_err("%s: Bad type %d\n", __func__, ulp_type); 480 return -EINVAL; 481 } 482 mutex_lock(&cnic_lock); 483 if (cnic_ulp_tbl_prot(ulp_type)) { 484 pr_err("%s: Type %d has already been registered\n", 485 __func__, ulp_type); 486 mutex_unlock(&cnic_lock); 487 return -EBUSY; 488 } 489 490 read_lock(&cnic_dev_lock); 491 list_for_each_entry(dev, &cnic_dev_list, list) { 492 struct cnic_local *cp = dev->cnic_priv; 493 494 clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]); 495 } 496 read_unlock(&cnic_dev_lock); 497 498 atomic_set(&ulp_ops->ref_count, 0); 499 rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops); 500 mutex_unlock(&cnic_lock); 501 502 /* Prevent race conditions with netdev_event */ 503 rtnl_lock(); 504 list_for_each_entry(dev, &cnic_dev_list, list) { 505 struct cnic_local *cp = dev->cnic_priv; 506 507 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 508 ulp_ops->cnic_init(dev); 509 } 510 rtnl_unlock(); 511 512 return 0; 513 } 514 515 int cnic_unregister_driver(int ulp_type) 516 { 517 struct cnic_dev *dev; 518 struct cnic_ulp_ops *ulp_ops; 519 int i = 0; 520 521 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 522 pr_err("%s: Bad type %d\n", __func__, ulp_type); 523 return -EINVAL; 524 } 525 mutex_lock(&cnic_lock); 526 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 527 if (!ulp_ops) { 528 pr_err("%s: Type %d has not been registered\n", 529 __func__, ulp_type); 530 goto out_unlock; 531 } 532 read_lock(&cnic_dev_lock); 533 list_for_each_entry(dev, &cnic_dev_list, list) { 534 struct cnic_local *cp = dev->cnic_priv; 535 536 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 537 pr_err("%s: Type %d still has devices registered\n", 538 __func__, ulp_type); 539 read_unlock(&cnic_dev_lock); 540 goto out_unlock; 541 } 542 } 543 read_unlock(&cnic_dev_lock); 544 545 RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL); 546 547 mutex_unlock(&cnic_lock); 548 synchronize_rcu(); 549 while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) { 550 msleep(100); 551 i++; 552 } 553 554 if (atomic_read(&ulp_ops->ref_count) != 0) 555 pr_warn("%s: Failed waiting for ref count to go to zero\n", 556 __func__); 557 return 0; 558 559 out_unlock: 560 mutex_unlock(&cnic_lock); 561 return -EINVAL; 562 } 563 564 static int cnic_start_hw(struct cnic_dev *); 565 static void cnic_stop_hw(struct cnic_dev *); 566 567 static int cnic_register_device(struct cnic_dev *dev, int ulp_type, 568 void *ulp_ctx) 569 { 570 struct cnic_local *cp = dev->cnic_priv; 571 struct cnic_ulp_ops *ulp_ops; 572 573 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 574 pr_err("%s: Bad type %d\n", __func__, ulp_type); 575 return -EINVAL; 576 } 577 mutex_lock(&cnic_lock); 578 if (cnic_ulp_tbl_prot(ulp_type) == NULL) { 579 pr_err("%s: Driver with type %d has not been registered\n", 580 __func__, ulp_type); 581 mutex_unlock(&cnic_lock); 582 return -EAGAIN; 583 } 584 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 585 pr_err("%s: Type %d has already been registered to this device\n", 586 __func__, ulp_type); 587 mutex_unlock(&cnic_lock); 588 return -EBUSY; 589 } 590 591 clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]); 592 cp->ulp_handle[ulp_type] = ulp_ctx; 593 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 594 rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops); 595 cnic_hold(dev); 596 597 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 598 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type])) 599 ulp_ops->cnic_start(cp->ulp_handle[ulp_type]); 600 601 mutex_unlock(&cnic_lock); 602 603 cnic_ulp_ctl(dev, ulp_type, true, DRV_ACTIVE); 604 605 return 0; 606 607 } 608 EXPORT_SYMBOL(cnic_register_driver); 609 610 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type) 611 { 612 struct cnic_local *cp = dev->cnic_priv; 613 int i = 0; 614 615 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 616 pr_err("%s: Bad type %d\n", __func__, ulp_type); 617 return -EINVAL; 618 } 619 620 if (ulp_type == CNIC_ULP_ISCSI) 621 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 622 623 mutex_lock(&cnic_lock); 624 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 625 RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL); 626 cnic_put(dev); 627 } else { 628 pr_err("%s: device not registered to this ulp type %d\n", 629 __func__, ulp_type); 630 mutex_unlock(&cnic_lock); 631 return -EINVAL; 632 } 633 mutex_unlock(&cnic_lock); 634 635 if (ulp_type == CNIC_ULP_FCOE) 636 dev->fcoe_cap = NULL; 637 638 synchronize_rcu(); 639 640 while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) && 641 i < 20) { 642 msleep(100); 643 i++; 644 } 645 if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type])) 646 netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n"); 647 648 if (test_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 649 cnic_ulp_ctl(dev, ulp_type, false, DRV_UNLOADED); 650 else 651 cnic_ulp_ctl(dev, ulp_type, false, DRV_INACTIVE); 652 653 return 0; 654 } 655 EXPORT_SYMBOL(cnic_unregister_driver); 656 657 static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id, 658 u32 next) 659 { 660 id_tbl->start = start_id; 661 id_tbl->max = size; 662 id_tbl->next = next; 663 spin_lock_init(&id_tbl->lock); 664 id_tbl->table = bitmap_zalloc(size, GFP_KERNEL); 665 if (!id_tbl->table) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl) 672 { 673 bitmap_free(id_tbl->table); 674 id_tbl->table = NULL; 675 } 676 677 static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id) 678 { 679 int ret = -1; 680 681 id -= id_tbl->start; 682 if (id >= id_tbl->max) 683 return ret; 684 685 spin_lock(&id_tbl->lock); 686 if (!test_bit(id, id_tbl->table)) { 687 set_bit(id, id_tbl->table); 688 ret = 0; 689 } 690 spin_unlock(&id_tbl->lock); 691 return ret; 692 } 693 694 /* Returns -1 if not successful */ 695 static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl) 696 { 697 u32 id; 698 699 spin_lock(&id_tbl->lock); 700 id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next); 701 if (id >= id_tbl->max) { 702 id = -1; 703 if (id_tbl->next != 0) { 704 id = find_first_zero_bit(id_tbl->table, id_tbl->next); 705 if (id >= id_tbl->next) 706 id = -1; 707 } 708 } 709 710 if (id < id_tbl->max) { 711 set_bit(id, id_tbl->table); 712 id_tbl->next = (id + 1) & (id_tbl->max - 1); 713 id += id_tbl->start; 714 } 715 716 spin_unlock(&id_tbl->lock); 717 718 return id; 719 } 720 721 static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id) 722 { 723 if (id == -1) 724 return; 725 726 id -= id_tbl->start; 727 if (id >= id_tbl->max) 728 return; 729 730 clear_bit(id, id_tbl->table); 731 } 732 733 static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma) 734 { 735 int i; 736 737 if (!dma->pg_arr) 738 return; 739 740 for (i = 0; i < dma->num_pages; i++) { 741 if (dma->pg_arr[i]) { 742 dma_free_coherent(&dev->pcidev->dev, CNIC_PAGE_SIZE, 743 dma->pg_arr[i], dma->pg_map_arr[i]); 744 dma->pg_arr[i] = NULL; 745 } 746 } 747 if (dma->pgtbl) { 748 dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size, 749 dma->pgtbl, dma->pgtbl_map); 750 dma->pgtbl = NULL; 751 } 752 kfree(dma->pg_arr); 753 dma->pg_arr = NULL; 754 dma->num_pages = 0; 755 } 756 757 static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma) 758 { 759 int i; 760 __le32 *page_table = (__le32 *) dma->pgtbl; 761 762 for (i = 0; i < dma->num_pages; i++) { 763 /* Each entry needs to be in big endian format. */ 764 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 765 page_table++; 766 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 767 page_table++; 768 } 769 } 770 771 static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma) 772 { 773 int i; 774 __le32 *page_table = (__le32 *) dma->pgtbl; 775 776 for (i = 0; i < dma->num_pages; i++) { 777 /* Each entry needs to be in little endian format. */ 778 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 779 page_table++; 780 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 781 page_table++; 782 } 783 } 784 785 static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma, 786 int pages, int use_pg_tbl) 787 { 788 int i, size; 789 struct cnic_local *cp = dev->cnic_priv; 790 791 size = pages * (sizeof(void *) + sizeof(dma_addr_t)); 792 dma->pg_arr = kzalloc(size, GFP_ATOMIC); 793 if (dma->pg_arr == NULL) 794 return -ENOMEM; 795 796 dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages); 797 dma->num_pages = pages; 798 799 for (i = 0; i < pages; i++) { 800 dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev, 801 CNIC_PAGE_SIZE, 802 &dma->pg_map_arr[i], 803 GFP_ATOMIC); 804 if (dma->pg_arr[i] == NULL) 805 goto error; 806 } 807 if (!use_pg_tbl) 808 return 0; 809 810 dma->pgtbl_size = ((pages * 8) + CNIC_PAGE_SIZE - 1) & 811 ~(CNIC_PAGE_SIZE - 1); 812 dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size, 813 &dma->pgtbl_map, GFP_ATOMIC); 814 if (dma->pgtbl == NULL) 815 goto error; 816 817 cp->setup_pgtbl(dev, dma); 818 819 return 0; 820 821 error: 822 cnic_free_dma(dev, dma); 823 return -ENOMEM; 824 } 825 826 static void cnic_free_context(struct cnic_dev *dev) 827 { 828 struct cnic_local *cp = dev->cnic_priv; 829 int i; 830 831 for (i = 0; i < cp->ctx_blks; i++) { 832 if (cp->ctx_arr[i].ctx) { 833 dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 834 cp->ctx_arr[i].ctx, 835 cp->ctx_arr[i].mapping); 836 cp->ctx_arr[i].ctx = NULL; 837 } 838 } 839 } 840 841 static void __cnic_free_uio_rings(struct cnic_uio_dev *udev) 842 { 843 if (udev->l2_buf) { 844 dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size, 845 udev->l2_buf, udev->l2_buf_map); 846 udev->l2_buf = NULL; 847 } 848 849 if (udev->l2_ring) { 850 dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size, 851 udev->l2_ring, udev->l2_ring_map); 852 udev->l2_ring = NULL; 853 } 854 855 } 856 857 static void __cnic_free_uio(struct cnic_uio_dev *udev) 858 { 859 uio_unregister_device(&udev->cnic_uinfo); 860 861 __cnic_free_uio_rings(udev); 862 863 pci_dev_put(udev->pdev); 864 kfree(udev); 865 } 866 867 static void cnic_free_uio(struct cnic_uio_dev *udev) 868 { 869 if (!udev) 870 return; 871 872 write_lock(&cnic_dev_lock); 873 list_del_init(&udev->list); 874 write_unlock(&cnic_dev_lock); 875 __cnic_free_uio(udev); 876 } 877 878 static void cnic_free_resc(struct cnic_dev *dev) 879 { 880 struct cnic_local *cp = dev->cnic_priv; 881 struct cnic_uio_dev *udev = cp->udev; 882 883 if (udev) { 884 udev->dev = NULL; 885 cp->udev = NULL; 886 if (udev->uio_dev == -1) 887 __cnic_free_uio_rings(udev); 888 } 889 890 cnic_free_context(dev); 891 kfree(cp->ctx_arr); 892 cp->ctx_arr = NULL; 893 cp->ctx_blks = 0; 894 895 cnic_free_dma(dev, &cp->gbl_buf_info); 896 cnic_free_dma(dev, &cp->kwq_info); 897 cnic_free_dma(dev, &cp->kwq_16_data_info); 898 cnic_free_dma(dev, &cp->kcq2.dma); 899 cnic_free_dma(dev, &cp->kcq1.dma); 900 kfree(cp->iscsi_tbl); 901 cp->iscsi_tbl = NULL; 902 kfree(cp->ctx_tbl); 903 cp->ctx_tbl = NULL; 904 905 cnic_free_id_tbl(&cp->fcoe_cid_tbl); 906 cnic_free_id_tbl(&cp->cid_tbl); 907 } 908 909 static int cnic_alloc_context(struct cnic_dev *dev) 910 { 911 struct cnic_local *cp = dev->cnic_priv; 912 913 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 914 int i, k, arr_size; 915 916 cp->ctx_blk_size = CNIC_PAGE_SIZE; 917 cp->cids_per_blk = CNIC_PAGE_SIZE / 128; 918 arr_size = BNX2_MAX_CID / cp->cids_per_blk * 919 sizeof(struct cnic_ctx); 920 cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL); 921 if (cp->ctx_arr == NULL) 922 return -ENOMEM; 923 924 k = 0; 925 for (i = 0; i < 2; i++) { 926 u32 j, reg, off, lo, hi; 927 928 if (i == 0) 929 off = BNX2_PG_CTX_MAP; 930 else 931 off = BNX2_ISCSI_CTX_MAP; 932 933 reg = cnic_reg_rd_ind(dev, off); 934 lo = reg >> 16; 935 hi = reg & 0xffff; 936 for (j = lo; j < hi; j += cp->cids_per_blk, k++) 937 cp->ctx_arr[k].cid = j; 938 } 939 940 cp->ctx_blks = k; 941 if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) { 942 cp->ctx_blks = 0; 943 return -ENOMEM; 944 } 945 946 for (i = 0; i < cp->ctx_blks; i++) { 947 cp->ctx_arr[i].ctx = 948 dma_alloc_coherent(&dev->pcidev->dev, 949 CNIC_PAGE_SIZE, 950 &cp->ctx_arr[i].mapping, 951 GFP_KERNEL); 952 if (cp->ctx_arr[i].ctx == NULL) 953 return -ENOMEM; 954 } 955 } 956 return 0; 957 } 958 959 static u16 cnic_bnx2_next_idx(u16 idx) 960 { 961 return idx + 1; 962 } 963 964 static u16 cnic_bnx2_hw_idx(u16 idx) 965 { 966 return idx; 967 } 968 969 static u16 cnic_bnx2x_next_idx(u16 idx) 970 { 971 idx++; 972 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 973 idx++; 974 975 return idx; 976 } 977 978 static u16 cnic_bnx2x_hw_idx(u16 idx) 979 { 980 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 981 idx++; 982 return idx; 983 } 984 985 static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info, 986 bool use_pg_tbl) 987 { 988 int err, i, use_page_tbl = 0; 989 struct kcqe **kcq; 990 991 if (use_pg_tbl) 992 use_page_tbl = 1; 993 994 err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl); 995 if (err) 996 return err; 997 998 kcq = (struct kcqe **) info->dma.pg_arr; 999 info->kcq = kcq; 1000 1001 info->next_idx = cnic_bnx2_next_idx; 1002 info->hw_idx = cnic_bnx2_hw_idx; 1003 if (use_pg_tbl) 1004 return 0; 1005 1006 info->next_idx = cnic_bnx2x_next_idx; 1007 info->hw_idx = cnic_bnx2x_hw_idx; 1008 1009 for (i = 0; i < KCQ_PAGE_CNT; i++) { 1010 struct bnx2x_bd_chain_next *next = 1011 (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT]; 1012 int j = i + 1; 1013 1014 if (j >= KCQ_PAGE_CNT) 1015 j = 0; 1016 next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32; 1017 next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff; 1018 } 1019 return 0; 1020 } 1021 1022 static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages) 1023 { 1024 struct cnic_local *cp = udev->dev->cnic_priv; 1025 1026 if (udev->l2_ring) 1027 return 0; 1028 1029 udev->l2_ring_size = pages * CNIC_PAGE_SIZE; 1030 udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size, 1031 &udev->l2_ring_map, GFP_KERNEL); 1032 if (!udev->l2_ring) 1033 return -ENOMEM; 1034 1035 udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size; 1036 udev->l2_buf_size = CNIC_PAGE_ALIGN(udev->l2_buf_size); 1037 udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size, 1038 &udev->l2_buf_map, GFP_KERNEL); 1039 if (!udev->l2_buf) { 1040 __cnic_free_uio_rings(udev); 1041 return -ENOMEM; 1042 } 1043 1044 return 0; 1045 1046 } 1047 1048 static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages) 1049 { 1050 struct cnic_local *cp = dev->cnic_priv; 1051 struct cnic_uio_dev *udev; 1052 1053 list_for_each_entry(udev, &cnic_udev_list, list) { 1054 if (udev->pdev == dev->pcidev) { 1055 udev->dev = dev; 1056 if (__cnic_alloc_uio_rings(udev, pages)) { 1057 udev->dev = NULL; 1058 return -ENOMEM; 1059 } 1060 cp->udev = udev; 1061 return 0; 1062 } 1063 } 1064 1065 udev = kzalloc(sizeof(struct cnic_uio_dev), GFP_ATOMIC); 1066 if (!udev) 1067 return -ENOMEM; 1068 1069 udev->uio_dev = -1; 1070 1071 udev->dev = dev; 1072 udev->pdev = dev->pcidev; 1073 1074 if (__cnic_alloc_uio_rings(udev, pages)) 1075 goto err_udev; 1076 1077 list_add(&udev->list, &cnic_udev_list); 1078 1079 pci_dev_get(udev->pdev); 1080 1081 cp->udev = udev; 1082 1083 return 0; 1084 1085 err_udev: 1086 kfree(udev); 1087 return -ENOMEM; 1088 } 1089 1090 static int cnic_init_uio(struct cnic_dev *dev) 1091 { 1092 struct cnic_local *cp = dev->cnic_priv; 1093 struct cnic_uio_dev *udev = cp->udev; 1094 struct uio_info *uinfo; 1095 int ret = 0; 1096 1097 if (!udev) 1098 return -ENOMEM; 1099 1100 uinfo = &udev->cnic_uinfo; 1101 1102 uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0); 1103 uinfo->mem[0].internal_addr = dev->regview; 1104 uinfo->mem[0].memtype = UIO_MEM_PHYS; 1105 1106 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 1107 uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID + 1108 TX_MAX_TSS_RINGS + 1); 1109 uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen & 1110 CNIC_PAGE_MASK; 1111 uinfo->mem[1].dma_addr = cp->status_blk_map; 1112 if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 1113 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE * 9); 1114 else 1115 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE); 1116 1117 uinfo->name = "bnx2_cnic"; 1118 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 1119 uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0); 1120 1121 uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk & 1122 CNIC_PAGE_MASK; 1123 uinfo->mem[1].dma_addr = cp->status_blk_map; 1124 uinfo->mem[1].size = PAGE_ALIGN(sizeof(*cp->bnx2x_def_status_blk)); 1125 1126 uinfo->name = "bnx2x_cnic"; 1127 } 1128 1129 uinfo->mem[1].dma_device = &dev->pcidev->dev; 1130 uinfo->mem[1].memtype = UIO_MEM_DMA_COHERENT; 1131 1132 uinfo->mem[2].addr = (unsigned long) udev->l2_ring; 1133 uinfo->mem[2].dma_addr = udev->l2_ring_map; 1134 uinfo->mem[2].size = PAGE_ALIGN(udev->l2_ring_size); 1135 uinfo->mem[2].dma_device = &dev->pcidev->dev; 1136 uinfo->mem[2].memtype = UIO_MEM_DMA_COHERENT; 1137 1138 uinfo->mem[3].addr = (unsigned long) udev->l2_buf; 1139 uinfo->mem[3].dma_addr = udev->l2_buf_map; 1140 uinfo->mem[3].size = PAGE_ALIGN(udev->l2_buf_size); 1141 uinfo->mem[3].dma_device = &dev->pcidev->dev; 1142 uinfo->mem[3].memtype = UIO_MEM_DMA_COHERENT; 1143 1144 uinfo->version = CNIC_MODULE_VERSION; 1145 uinfo->irq = UIO_IRQ_CUSTOM; 1146 1147 uinfo->open = cnic_uio_open; 1148 uinfo->release = cnic_uio_close; 1149 1150 if (udev->uio_dev == -1) { 1151 if (!uinfo->priv) { 1152 uinfo->priv = udev; 1153 1154 ret = uio_register_device(&udev->pdev->dev, uinfo); 1155 } 1156 } else { 1157 cnic_init_rings(dev); 1158 } 1159 1160 return ret; 1161 } 1162 1163 static int cnic_alloc_bnx2_resc(struct cnic_dev *dev) 1164 { 1165 struct cnic_local *cp = dev->cnic_priv; 1166 int ret; 1167 1168 ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1); 1169 if (ret) 1170 goto error; 1171 cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr; 1172 1173 ret = cnic_alloc_kcq(dev, &cp->kcq1, true); 1174 if (ret) 1175 goto error; 1176 1177 ret = cnic_alloc_context(dev); 1178 if (ret) 1179 goto error; 1180 1181 ret = cnic_alloc_uio_rings(dev, 2); 1182 if (ret) 1183 goto error; 1184 1185 ret = cnic_init_uio(dev); 1186 if (ret) 1187 goto error; 1188 1189 return 0; 1190 1191 error: 1192 cnic_free_resc(dev); 1193 return ret; 1194 } 1195 1196 static int cnic_alloc_bnx2x_context(struct cnic_dev *dev) 1197 { 1198 struct cnic_local *cp = dev->cnic_priv; 1199 struct bnx2x *bp = netdev_priv(dev->netdev); 1200 int ctx_blk_size = cp->ethdev->ctx_blk_size; 1201 int total_mem, blks, i; 1202 1203 total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space; 1204 blks = total_mem / ctx_blk_size; 1205 if (total_mem % ctx_blk_size) 1206 blks++; 1207 1208 if (blks > cp->ethdev->ctx_tbl_len) 1209 return -ENOMEM; 1210 1211 cp->ctx_arr = kcalloc(blks, sizeof(struct cnic_ctx), GFP_KERNEL); 1212 if (cp->ctx_arr == NULL) 1213 return -ENOMEM; 1214 1215 cp->ctx_blks = blks; 1216 cp->ctx_blk_size = ctx_blk_size; 1217 if (!CHIP_IS_E1(bp)) 1218 cp->ctx_align = 0; 1219 else 1220 cp->ctx_align = ctx_blk_size; 1221 1222 cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE; 1223 1224 for (i = 0; i < blks; i++) { 1225 cp->ctx_arr[i].ctx = 1226 dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 1227 &cp->ctx_arr[i].mapping, 1228 GFP_KERNEL); 1229 if (cp->ctx_arr[i].ctx == NULL) 1230 return -ENOMEM; 1231 1232 if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) { 1233 if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) { 1234 cnic_free_context(dev); 1235 cp->ctx_blk_size += cp->ctx_align; 1236 i = -1; 1237 continue; 1238 } 1239 } 1240 } 1241 return 0; 1242 } 1243 1244 static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev) 1245 { 1246 struct cnic_local *cp = dev->cnic_priv; 1247 struct bnx2x *bp = netdev_priv(dev->netdev); 1248 struct cnic_eth_dev *ethdev = cp->ethdev; 1249 u32 start_cid = ethdev->starting_cid; 1250 int i, j, n, ret, pages; 1251 struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info; 1252 1253 cp->max_cid_space = MAX_ISCSI_TBL_SZ; 1254 cp->iscsi_start_cid = start_cid; 1255 cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ; 1256 1257 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 1258 cp->max_cid_space += dev->max_fcoe_conn; 1259 cp->fcoe_init_cid = ethdev->fcoe_init_cid; 1260 if (!cp->fcoe_init_cid) 1261 cp->fcoe_init_cid = 0x10; 1262 } 1263 1264 cp->iscsi_tbl = kcalloc(MAX_ISCSI_TBL_SZ, sizeof(struct cnic_iscsi), 1265 GFP_KERNEL); 1266 if (!cp->iscsi_tbl) 1267 goto error; 1268 1269 cp->ctx_tbl = kcalloc(cp->max_cid_space, sizeof(struct cnic_context), 1270 GFP_KERNEL); 1271 if (!cp->ctx_tbl) 1272 goto error; 1273 1274 for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) { 1275 cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i]; 1276 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI; 1277 } 1278 1279 for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++) 1280 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE; 1281 1282 pages = CNIC_PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) / 1283 CNIC_PAGE_SIZE; 1284 1285 ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0); 1286 if (ret) 1287 goto error; 1288 1289 n = CNIC_PAGE_SIZE / CNIC_KWQ16_DATA_SIZE; 1290 for (i = 0, j = 0; i < cp->max_cid_space; i++) { 1291 long off = CNIC_KWQ16_DATA_SIZE * (i % n); 1292 1293 cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off; 1294 cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] + 1295 off; 1296 1297 if ((i % n) == (n - 1)) 1298 j++; 1299 } 1300 1301 ret = cnic_alloc_kcq(dev, &cp->kcq1, false); 1302 if (ret) 1303 goto error; 1304 1305 if (CNIC_SUPPORTS_FCOE(bp)) { 1306 ret = cnic_alloc_kcq(dev, &cp->kcq2, true); 1307 if (ret) 1308 goto error; 1309 } 1310 1311 pages = CNIC_PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / CNIC_PAGE_SIZE; 1312 ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0); 1313 if (ret) 1314 goto error; 1315 1316 ret = cnic_alloc_bnx2x_context(dev); 1317 if (ret) 1318 goto error; 1319 1320 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 1321 return 0; 1322 1323 cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk; 1324 cp->status_blk_map = cp->ethdev->irq_arr[1].status_blk_map; 1325 1326 cp->l2_rx_ring_size = 15; 1327 1328 ret = cnic_alloc_uio_rings(dev, 4); 1329 if (ret) 1330 goto error; 1331 1332 ret = cnic_init_uio(dev); 1333 if (ret) 1334 goto error; 1335 1336 return 0; 1337 1338 error: 1339 cnic_free_resc(dev); 1340 return -ENOMEM; 1341 } 1342 1343 static inline u32 cnic_kwq_avail(struct cnic_local *cp) 1344 { 1345 return cp->max_kwq_idx - 1346 ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx); 1347 } 1348 1349 static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 1350 u32 num_wqes) 1351 { 1352 struct cnic_local *cp = dev->cnic_priv; 1353 struct kwqe *prod_qe; 1354 u16 prod, sw_prod, i; 1355 1356 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 1357 return -EAGAIN; /* bnx2 is down */ 1358 1359 spin_lock_bh(&cp->cnic_ulp_lock); 1360 if (num_wqes > cnic_kwq_avail(cp) && 1361 !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) { 1362 spin_unlock_bh(&cp->cnic_ulp_lock); 1363 return -EAGAIN; 1364 } 1365 1366 clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 1367 1368 prod = cp->kwq_prod_idx; 1369 sw_prod = prod & MAX_KWQ_IDX; 1370 for (i = 0; i < num_wqes; i++) { 1371 prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)]; 1372 memcpy(prod_qe, wqes[i], sizeof(struct kwqe)); 1373 prod++; 1374 sw_prod = prod & MAX_KWQ_IDX; 1375 } 1376 cp->kwq_prod_idx = prod; 1377 1378 CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx); 1379 1380 spin_unlock_bh(&cp->cnic_ulp_lock); 1381 return 0; 1382 } 1383 1384 static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid, 1385 union l5cm_specific_data *l5_data) 1386 { 1387 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1388 dma_addr_t map; 1389 1390 map = ctx->kwqe_data_mapping; 1391 l5_data->phy_address.lo = (u64) map & 0xffffffff; 1392 l5_data->phy_address.hi = (u64) map >> 32; 1393 return ctx->kwqe_data; 1394 } 1395 1396 static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid, 1397 u32 type, union l5cm_specific_data *l5_data) 1398 { 1399 struct cnic_local *cp = dev->cnic_priv; 1400 struct bnx2x *bp = netdev_priv(dev->netdev); 1401 struct l5cm_spe kwqe; 1402 struct kwqe_16 *kwq[1]; 1403 u16 type_16; 1404 int ret; 1405 1406 kwqe.hdr.conn_and_cmd_data = 1407 cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) | 1408 BNX2X_HW_CID(bp, cid))); 1409 1410 type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE; 1411 type_16 |= (bp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) & 1412 SPE_HDR_FUNCTION_ID; 1413 1414 kwqe.hdr.type = cpu_to_le16(type_16); 1415 kwqe.hdr.reserved1 = 0; 1416 kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo); 1417 kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi); 1418 1419 kwq[0] = (struct kwqe_16 *) &kwqe; 1420 1421 spin_lock_bh(&cp->cnic_ulp_lock); 1422 ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1); 1423 spin_unlock_bh(&cp->cnic_ulp_lock); 1424 1425 if (ret == 1) 1426 return 0; 1427 1428 return ret; 1429 } 1430 1431 static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type, 1432 struct kcqe *cqes[], u32 num_cqes) 1433 { 1434 struct cnic_local *cp = dev->cnic_priv; 1435 struct cnic_ulp_ops *ulp_ops; 1436 1437 rcu_read_lock(); 1438 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 1439 if (likely(ulp_ops)) { 1440 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 1441 cqes, num_cqes); 1442 } 1443 rcu_read_unlock(); 1444 } 1445 1446 static void cnic_bnx2x_set_tcp_options(struct cnic_dev *dev, int time_stamps, 1447 int en_tcp_dack) 1448 { 1449 struct bnx2x *bp = netdev_priv(dev->netdev); 1450 u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN; 1451 u16 tstorm_flags = 0; 1452 1453 if (time_stamps) { 1454 xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1455 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1456 } 1457 if (en_tcp_dack) 1458 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_DELAYED_ACK_EN; 1459 1460 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1461 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), xstorm_flags); 1462 1463 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1464 TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), tstorm_flags); 1465 } 1466 1467 static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe) 1468 { 1469 struct cnic_local *cp = dev->cnic_priv; 1470 struct bnx2x *bp = netdev_priv(dev->netdev); 1471 struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe; 1472 int hq_bds, pages; 1473 u32 pfid = bp->pfid; 1474 1475 cp->num_iscsi_tasks = req1->num_tasks_per_conn; 1476 cp->num_ccells = req1->num_ccells_per_conn; 1477 cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE * 1478 cp->num_iscsi_tasks; 1479 cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS * 1480 BNX2X_ISCSI_R2TQE_SIZE; 1481 cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE; 1482 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1483 hq_bds = pages * (CNIC_PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE); 1484 cp->num_cqs = req1->num_cqs; 1485 1486 if (!dev->max_iscsi_conn) 1487 return 0; 1488 1489 /* init Tstorm RAM */ 1490 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1491 req1->rq_num_wqes); 1492 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1493 CNIC_PAGE_SIZE); 1494 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 1495 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1496 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1497 TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1498 req1->num_tasks_per_conn); 1499 1500 /* init Ustorm RAM */ 1501 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1502 USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid), 1503 req1->rq_buffer_size); 1504 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1505 CNIC_PAGE_SIZE); 1506 CNIC_WR8(dev, BAR_USTRORM_INTMEM + 1507 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1508 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1509 USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1510 req1->num_tasks_per_conn); 1511 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1512 req1->rq_num_wqes); 1513 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1514 req1->cq_num_wqes); 1515 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1516 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1517 1518 /* init Xstorm RAM */ 1519 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1520 CNIC_PAGE_SIZE); 1521 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1522 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1523 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 1524 XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1525 req1->num_tasks_per_conn); 1526 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1527 hq_bds); 1528 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid), 1529 req1->num_tasks_per_conn); 1530 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1531 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1532 1533 /* init Cstorm RAM */ 1534 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1535 CNIC_PAGE_SIZE); 1536 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 1537 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1538 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1539 CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1540 req1->num_tasks_per_conn); 1541 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1542 req1->cq_num_wqes); 1543 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1544 hq_bds); 1545 1546 cnic_bnx2x_set_tcp_options(dev, 1547 req1->flags & ISCSI_KWQE_INIT1_TIME_STAMPS_ENABLE, 1548 req1->flags & ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE); 1549 1550 return 0; 1551 } 1552 1553 static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe) 1554 { 1555 struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe; 1556 struct bnx2x *bp = netdev_priv(dev->netdev); 1557 u32 pfid = bp->pfid; 1558 struct iscsi_kcqe kcqe; 1559 struct kcqe *cqes[1]; 1560 1561 memset(&kcqe, 0, sizeof(kcqe)); 1562 if (!dev->max_iscsi_conn) { 1563 kcqe.completion_status = 1564 ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED; 1565 goto done; 1566 } 1567 1568 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1569 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1570 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1571 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1572 req2->error_bit_map[1]); 1573 1574 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1575 USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1576 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1577 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1578 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1579 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1580 req2->error_bit_map[1]); 1581 1582 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1583 CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1584 1585 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1586 1587 done: 1588 kcqe.op_code = ISCSI_KCQE_OPCODE_INIT; 1589 cqes[0] = (struct kcqe *) &kcqe; 1590 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1591 1592 return 0; 1593 } 1594 1595 static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1596 { 1597 struct cnic_local *cp = dev->cnic_priv; 1598 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1599 1600 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) { 1601 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1602 1603 cnic_free_dma(dev, &iscsi->hq_info); 1604 cnic_free_dma(dev, &iscsi->r2tq_info); 1605 cnic_free_dma(dev, &iscsi->task_array_info); 1606 cnic_free_id(&cp->cid_tbl, ctx->cid); 1607 } else { 1608 cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid); 1609 } 1610 1611 ctx->cid = 0; 1612 } 1613 1614 static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1615 { 1616 u32 cid; 1617 int ret, pages; 1618 struct cnic_local *cp = dev->cnic_priv; 1619 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1620 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1621 1622 if (ctx->ulp_proto_id == CNIC_ULP_FCOE) { 1623 cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl); 1624 if (cid == -1) { 1625 ret = -ENOMEM; 1626 goto error; 1627 } 1628 ctx->cid = cid; 1629 return 0; 1630 } 1631 1632 cid = cnic_alloc_new_id(&cp->cid_tbl); 1633 if (cid == -1) { 1634 ret = -ENOMEM; 1635 goto error; 1636 } 1637 1638 ctx->cid = cid; 1639 pages = CNIC_PAGE_ALIGN(cp->task_array_size) / CNIC_PAGE_SIZE; 1640 1641 ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1); 1642 if (ret) 1643 goto error; 1644 1645 pages = CNIC_PAGE_ALIGN(cp->r2tq_size) / CNIC_PAGE_SIZE; 1646 ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1); 1647 if (ret) 1648 goto error; 1649 1650 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1651 ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1); 1652 if (ret) 1653 goto error; 1654 1655 return 0; 1656 1657 error: 1658 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1659 return ret; 1660 } 1661 1662 static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init, 1663 struct regpair *ctx_addr) 1664 { 1665 struct cnic_local *cp = dev->cnic_priv; 1666 struct cnic_eth_dev *ethdev = cp->ethdev; 1667 int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk; 1668 int off = (cid - ethdev->starting_cid) % cp->cids_per_blk; 1669 unsigned long align_off = 0; 1670 dma_addr_t ctx_map; 1671 void *ctx; 1672 1673 if (cp->ctx_align) { 1674 unsigned long mask = cp->ctx_align - 1; 1675 1676 if (cp->ctx_arr[blk].mapping & mask) 1677 align_off = cp->ctx_align - 1678 (cp->ctx_arr[blk].mapping & mask); 1679 } 1680 ctx_map = cp->ctx_arr[blk].mapping + align_off + 1681 (off * BNX2X_CONTEXT_MEM_SIZE); 1682 ctx = cp->ctx_arr[blk].ctx + align_off + 1683 (off * BNX2X_CONTEXT_MEM_SIZE); 1684 if (init) 1685 memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE); 1686 1687 ctx_addr->lo = ctx_map & 0xffffffff; 1688 ctx_addr->hi = (u64) ctx_map >> 32; 1689 return ctx; 1690 } 1691 1692 static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[], 1693 u32 num) 1694 { 1695 struct cnic_local *cp = dev->cnic_priv; 1696 struct bnx2x *bp = netdev_priv(dev->netdev); 1697 struct iscsi_kwqe_conn_offload1 *req1 = 1698 (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1699 struct iscsi_kwqe_conn_offload2 *req2 = 1700 (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1701 struct iscsi_kwqe_conn_offload3 *req3; 1702 struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id]; 1703 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1704 u32 cid = ctx->cid; 1705 u32 hw_cid = BNX2X_HW_CID(bp, cid); 1706 struct iscsi_context *ictx; 1707 struct regpair context_addr; 1708 int i, j, n = 2, n_max; 1709 u8 port = BP_PORT(bp); 1710 1711 ctx->ctx_flags = 0; 1712 if (!req2->num_additional_wqes) 1713 return -EINVAL; 1714 1715 n_max = req2->num_additional_wqes + 2; 1716 1717 ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr); 1718 if (ictx == NULL) 1719 return -ENOMEM; 1720 1721 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1722 1723 ictx->xstorm_ag_context.hq_prod = 1; 1724 1725 ictx->xstorm_st_context.iscsi.first_burst_length = 1726 ISCSI_DEF_FIRST_BURST_LEN; 1727 ictx->xstorm_st_context.iscsi.max_send_pdu_length = 1728 ISCSI_DEF_MAX_RECV_SEG_LEN; 1729 ictx->xstorm_st_context.iscsi.sq_pbl_base.lo = 1730 req1->sq_page_table_addr_lo; 1731 ictx->xstorm_st_context.iscsi.sq_pbl_base.hi = 1732 req1->sq_page_table_addr_hi; 1733 ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi; 1734 ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo; 1735 ictx->xstorm_st_context.iscsi.hq_pbl_base.lo = 1736 iscsi->hq_info.pgtbl_map & 0xffffffff; 1737 ictx->xstorm_st_context.iscsi.hq_pbl_base.hi = 1738 (u64) iscsi->hq_info.pgtbl_map >> 32; 1739 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo = 1740 iscsi->hq_info.pgtbl[0]; 1741 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi = 1742 iscsi->hq_info.pgtbl[1]; 1743 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo = 1744 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1745 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi = 1746 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1747 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo = 1748 iscsi->r2tq_info.pgtbl[0]; 1749 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi = 1750 iscsi->r2tq_info.pgtbl[1]; 1751 ictx->xstorm_st_context.iscsi.task_pbl_base.lo = 1752 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1753 ictx->xstorm_st_context.iscsi.task_pbl_base.hi = 1754 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1755 ictx->xstorm_st_context.iscsi.task_pbl_cache_idx = 1756 BNX2X_ISCSI_PBL_NOT_CACHED; 1757 ictx->xstorm_st_context.iscsi.flags.flags |= 1758 XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA; 1759 ictx->xstorm_st_context.iscsi.flags.flags |= 1760 XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T; 1761 ictx->xstorm_st_context.common.ethernet.reserved_vlan_type = 1762 ETH_P_8021Q; 1763 if (BNX2X_CHIP_IS_E2_PLUS(bp) && 1764 bp->common.chip_port_mode == CHIP_2_PORT_MODE) { 1765 1766 port = 0; 1767 } 1768 ictx->xstorm_st_context.common.flags = 1769 1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT; 1770 ictx->xstorm_st_context.common.flags = 1771 port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT; 1772 1773 ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE; 1774 /* TSTORM requires the base address of RQ DB & not PTE */ 1775 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo = 1776 req2->rq_page_table_addr_lo & CNIC_PAGE_MASK; 1777 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi = 1778 req2->rq_page_table_addr_hi; 1779 ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id; 1780 ictx->tstorm_st_context.tcp.cwnd = 0x5A8; 1781 ictx->tstorm_st_context.tcp.flags2 |= 1782 TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN; 1783 ictx->tstorm_st_context.tcp.ooo_support_mode = 1784 TCP_TSTORM_OOO_DROP_AND_PROC_ACK; 1785 1786 ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG; 1787 1788 ictx->ustorm_st_context.ring.rq.pbl_base.lo = 1789 req2->rq_page_table_addr_lo; 1790 ictx->ustorm_st_context.ring.rq.pbl_base.hi = 1791 req2->rq_page_table_addr_hi; 1792 ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi; 1793 ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo; 1794 ictx->ustorm_st_context.ring.r2tq.pbl_base.lo = 1795 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1796 ictx->ustorm_st_context.ring.r2tq.pbl_base.hi = 1797 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1798 ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo = 1799 iscsi->r2tq_info.pgtbl[0]; 1800 ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi = 1801 iscsi->r2tq_info.pgtbl[1]; 1802 ictx->ustorm_st_context.ring.cq_pbl_base.lo = 1803 req1->cq_page_table_addr_lo; 1804 ictx->ustorm_st_context.ring.cq_pbl_base.hi = 1805 req1->cq_page_table_addr_hi; 1806 ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN; 1807 ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi; 1808 ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo; 1809 ictx->ustorm_st_context.task_pbe_cache_index = 1810 BNX2X_ISCSI_PBL_NOT_CACHED; 1811 ictx->ustorm_st_context.task_pdu_cache_index = 1812 BNX2X_ISCSI_PDU_HEADER_NOT_CACHED; 1813 1814 for (i = 1, j = 1; i < cp->num_cqs; i++, j++) { 1815 if (j == 3) { 1816 if (n >= n_max) 1817 break; 1818 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1819 j = 0; 1820 } 1821 ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN; 1822 ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo = 1823 req3->qp_first_pte[j].hi; 1824 ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi = 1825 req3->qp_first_pte[j].lo; 1826 } 1827 1828 ictx->ustorm_st_context.task_pbl_base.lo = 1829 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1830 ictx->ustorm_st_context.task_pbl_base.hi = 1831 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1832 ictx->ustorm_st_context.tce_phy_addr.lo = 1833 iscsi->task_array_info.pgtbl[0]; 1834 ictx->ustorm_st_context.tce_phy_addr.hi = 1835 iscsi->task_array_info.pgtbl[1]; 1836 ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1837 ictx->ustorm_st_context.num_cqs = cp->num_cqs; 1838 ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN; 1839 ictx->ustorm_st_context.negotiated_rx_and_flags |= 1840 ISCSI_DEF_MAX_BURST_LEN; 1841 ictx->ustorm_st_context.negotiated_rx |= 1842 ISCSI_DEFAULT_MAX_OUTSTANDING_R2T << 1843 USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT; 1844 1845 ictx->cstorm_st_context.hq_pbl_base.lo = 1846 iscsi->hq_info.pgtbl_map & 0xffffffff; 1847 ictx->cstorm_st_context.hq_pbl_base.hi = 1848 (u64) iscsi->hq_info.pgtbl_map >> 32; 1849 ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0]; 1850 ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1]; 1851 ictx->cstorm_st_context.task_pbl_base.lo = 1852 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1853 ictx->cstorm_st_context.task_pbl_base.hi = 1854 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1855 /* CSTORM and USTORM initialization is different, CSTORM requires 1856 * CQ DB base & not PTE addr */ 1857 ictx->cstorm_st_context.cq_db_base.lo = 1858 req1->cq_page_table_addr_lo & CNIC_PAGE_MASK; 1859 ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi; 1860 ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1861 ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1; 1862 for (i = 0; i < cp->num_cqs; i++) { 1863 ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] = 1864 ISCSI_INITIAL_SN; 1865 ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] = 1866 ISCSI_INITIAL_SN; 1867 } 1868 1869 ictx->xstorm_ag_context.cdu_reserved = 1870 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 1871 ISCSI_CONNECTION_TYPE); 1872 ictx->ustorm_ag_context.cdu_usage = 1873 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 1874 ISCSI_CONNECTION_TYPE); 1875 return 0; 1876 1877 } 1878 1879 static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 1880 u32 num, int *work) 1881 { 1882 struct iscsi_kwqe_conn_offload1 *req1; 1883 struct iscsi_kwqe_conn_offload2 *req2; 1884 struct cnic_local *cp = dev->cnic_priv; 1885 struct bnx2x *bp = netdev_priv(dev->netdev); 1886 struct cnic_context *ctx; 1887 struct iscsi_kcqe kcqe; 1888 struct kcqe *cqes[1]; 1889 u32 l5_cid; 1890 int ret = 0; 1891 1892 if (num < 2) { 1893 *work = num; 1894 return -EINVAL; 1895 } 1896 1897 req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1898 req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1899 if ((num - 2) < req2->num_additional_wqes) { 1900 *work = num; 1901 return -EINVAL; 1902 } 1903 *work = 2 + req2->num_additional_wqes; 1904 1905 l5_cid = req1->iscsi_conn_id; 1906 if (l5_cid >= MAX_ISCSI_TBL_SZ) 1907 return -EINVAL; 1908 1909 memset(&kcqe, 0, sizeof(kcqe)); 1910 kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN; 1911 kcqe.iscsi_conn_id = l5_cid; 1912 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 1913 1914 ctx = &cp->ctx_tbl[l5_cid]; 1915 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) { 1916 kcqe.completion_status = 1917 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY; 1918 goto done; 1919 } 1920 1921 if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) { 1922 atomic_dec(&cp->iscsi_conn); 1923 goto done; 1924 } 1925 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 1926 if (ret) { 1927 atomic_dec(&cp->iscsi_conn); 1928 goto done; 1929 } 1930 ret = cnic_setup_bnx2x_ctx(dev, wqes, num); 1931 if (ret < 0) { 1932 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1933 atomic_dec(&cp->iscsi_conn); 1934 goto done; 1935 } 1936 1937 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1938 kcqe.iscsi_conn_context_id = BNX2X_HW_CID(bp, cp->ctx_tbl[l5_cid].cid); 1939 1940 done: 1941 cqes[0] = (struct kcqe *) &kcqe; 1942 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1943 return 0; 1944 } 1945 1946 1947 static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe) 1948 { 1949 struct cnic_local *cp = dev->cnic_priv; 1950 struct iscsi_kwqe_conn_update *req = 1951 (struct iscsi_kwqe_conn_update *) kwqe; 1952 void *data; 1953 union l5cm_specific_data l5_data; 1954 u32 l5_cid, cid = BNX2X_SW_CID(req->context_id); 1955 int ret; 1956 1957 if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0) 1958 return -EINVAL; 1959 1960 data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 1961 if (!data) 1962 return -ENOMEM; 1963 1964 memcpy(data, kwqe, sizeof(struct kwqe)); 1965 1966 ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, 1967 req->context_id, ISCSI_CONNECTION_TYPE, &l5_data); 1968 return ret; 1969 } 1970 1971 static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid) 1972 { 1973 struct cnic_local *cp = dev->cnic_priv; 1974 struct bnx2x *bp = netdev_priv(dev->netdev); 1975 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1976 union l5cm_specific_data l5_data; 1977 int ret; 1978 u32 hw_cid; 1979 1980 init_waitqueue_head(&ctx->waitq); 1981 ctx->wait_cond = 0; 1982 memset(&l5_data, 0, sizeof(l5_data)); 1983 hw_cid = BNX2X_HW_CID(bp, ctx->cid); 1984 1985 ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 1986 hw_cid, NONE_CONNECTION_TYPE, &l5_data); 1987 1988 if (ret == 0) { 1989 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 1990 if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags))) 1991 return -EBUSY; 1992 } 1993 1994 return 0; 1995 } 1996 1997 static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 1998 { 1999 struct cnic_local *cp = dev->cnic_priv; 2000 struct iscsi_kwqe_conn_destroy *req = 2001 (struct iscsi_kwqe_conn_destroy *) kwqe; 2002 u32 l5_cid = req->reserved0; 2003 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2004 int ret = 0; 2005 struct iscsi_kcqe kcqe; 2006 struct kcqe *cqes[1]; 2007 2008 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2009 goto skip_cfc_delete; 2010 2011 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 2012 unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies; 2013 2014 if (delta > (2 * HZ)) 2015 delta = 0; 2016 2017 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2018 queue_delayed_work(cnic_wq, &cp->delete_task, delta); 2019 goto destroy_reply; 2020 } 2021 2022 ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid); 2023 2024 skip_cfc_delete: 2025 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2026 2027 if (!ret) { 2028 atomic_dec(&cp->iscsi_conn); 2029 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2030 } 2031 2032 destroy_reply: 2033 memset(&kcqe, 0, sizeof(kcqe)); 2034 kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN; 2035 kcqe.iscsi_conn_id = l5_cid; 2036 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 2037 kcqe.iscsi_conn_context_id = req->context_id; 2038 2039 cqes[0] = (struct kcqe *) &kcqe; 2040 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 2041 2042 return 0; 2043 } 2044 2045 static void cnic_init_storm_conn_bufs(struct cnic_dev *dev, 2046 struct l4_kwq_connect_req1 *kwqe1, 2047 struct l4_kwq_connect_req3 *kwqe3, 2048 struct l5cm_active_conn_buffer *conn_buf) 2049 { 2050 struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf; 2051 struct l5cm_xstorm_conn_buffer *xstorm_buf = 2052 &conn_buf->xstorm_conn_buffer; 2053 struct l5cm_tstorm_conn_buffer *tstorm_buf = 2054 &conn_buf->tstorm_conn_buffer; 2055 struct regpair context_addr; 2056 u32 cid = BNX2X_SW_CID(kwqe1->cid); 2057 struct in6_addr src_ip, dst_ip; 2058 int i; 2059 u32 *addrp; 2060 2061 addrp = (u32 *) &conn_addr->local_ip_addr; 2062 for (i = 0; i < 4; i++, addrp++) 2063 src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2064 2065 addrp = (u32 *) &conn_addr->remote_ip_addr; 2066 for (i = 0; i < 4; i++, addrp++) 2067 dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2068 2069 cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr); 2070 2071 xstorm_buf->context_addr.hi = context_addr.hi; 2072 xstorm_buf->context_addr.lo = context_addr.lo; 2073 xstorm_buf->mss = 0xffff; 2074 xstorm_buf->rcv_buf = kwqe3->rcv_buf; 2075 if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE) 2076 xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE; 2077 xstorm_buf->pseudo_header_checksum = 2078 swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0)); 2079 2080 if (kwqe3->ka_timeout) { 2081 tstorm_buf->ka_enable = 1; 2082 tstorm_buf->ka_timeout = kwqe3->ka_timeout; 2083 tstorm_buf->ka_interval = kwqe3->ka_interval; 2084 tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count; 2085 } 2086 tstorm_buf->max_rt_time = 0xffffffff; 2087 } 2088 2089 static void cnic_init_bnx2x_mac(struct cnic_dev *dev) 2090 { 2091 struct bnx2x *bp = netdev_priv(dev->netdev); 2092 u32 pfid = bp->pfid; 2093 u8 *mac = dev->mac_addr; 2094 2095 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2096 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]); 2097 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2098 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]); 2099 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2100 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]); 2101 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2102 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]); 2103 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2104 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]); 2105 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2106 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]); 2107 2108 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2109 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]); 2110 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2111 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2112 mac[4]); 2113 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2114 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]); 2115 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2116 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2117 mac[2]); 2118 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2119 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]); 2120 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2121 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2122 mac[0]); 2123 } 2124 2125 static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[], 2126 u32 num, int *work) 2127 { 2128 struct cnic_local *cp = dev->cnic_priv; 2129 struct bnx2x *bp = netdev_priv(dev->netdev); 2130 struct l4_kwq_connect_req1 *kwqe1 = 2131 (struct l4_kwq_connect_req1 *) wqes[0]; 2132 struct l4_kwq_connect_req3 *kwqe3; 2133 struct l5cm_active_conn_buffer *conn_buf; 2134 struct l5cm_conn_addr_params *conn_addr; 2135 union l5cm_specific_data l5_data; 2136 u32 l5_cid = kwqe1->pg_cid; 2137 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 2138 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2139 int ret; 2140 2141 if (num < 2) { 2142 *work = num; 2143 return -EINVAL; 2144 } 2145 2146 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) 2147 *work = 3; 2148 else 2149 *work = 2; 2150 2151 if (num < *work) { 2152 *work = num; 2153 return -EINVAL; 2154 } 2155 2156 if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) { 2157 netdev_err(dev->netdev, "conn_buf size too big\n"); 2158 return -ENOMEM; 2159 } 2160 conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2161 if (!conn_buf) 2162 return -ENOMEM; 2163 2164 memset(conn_buf, 0, sizeof(*conn_buf)); 2165 2166 conn_addr = &conn_buf->conn_addr_buf; 2167 conn_addr->remote_addr_0 = csk->ha[0]; 2168 conn_addr->remote_addr_1 = csk->ha[1]; 2169 conn_addr->remote_addr_2 = csk->ha[2]; 2170 conn_addr->remote_addr_3 = csk->ha[3]; 2171 conn_addr->remote_addr_4 = csk->ha[4]; 2172 conn_addr->remote_addr_5 = csk->ha[5]; 2173 2174 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) { 2175 struct l4_kwq_connect_req2 *kwqe2 = 2176 (struct l4_kwq_connect_req2 *) wqes[1]; 2177 2178 conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4; 2179 conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3; 2180 conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2; 2181 2182 conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4; 2183 conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3; 2184 conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2; 2185 conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION; 2186 } 2187 kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1]; 2188 2189 conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip; 2190 conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip; 2191 conn_addr->local_tcp_port = kwqe1->src_port; 2192 conn_addr->remote_tcp_port = kwqe1->dst_port; 2193 2194 conn_addr->pmtu = kwqe3->pmtu; 2195 cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf); 2196 2197 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 2198 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(bp->pfid), csk->vlan_id); 2199 2200 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT, 2201 kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2202 if (!ret) 2203 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2204 2205 return ret; 2206 } 2207 2208 static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe) 2209 { 2210 struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe; 2211 union l5cm_specific_data l5_data; 2212 int ret; 2213 2214 memset(&l5_data, 0, sizeof(l5_data)); 2215 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE, 2216 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2217 return ret; 2218 } 2219 2220 static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe) 2221 { 2222 struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe; 2223 union l5cm_specific_data l5_data; 2224 int ret; 2225 2226 memset(&l5_data, 0, sizeof(l5_data)); 2227 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT, 2228 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2229 return ret; 2230 } 2231 static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2232 { 2233 struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe; 2234 struct l4_kcq kcqe; 2235 struct kcqe *cqes[1]; 2236 2237 memset(&kcqe, 0, sizeof(kcqe)); 2238 kcqe.pg_host_opaque = req->host_opaque; 2239 kcqe.pg_cid = req->host_opaque; 2240 kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG; 2241 cqes[0] = (struct kcqe *) &kcqe; 2242 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2243 return 0; 2244 } 2245 2246 static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2247 { 2248 struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe; 2249 struct l4_kcq kcqe; 2250 struct kcqe *cqes[1]; 2251 2252 memset(&kcqe, 0, sizeof(kcqe)); 2253 kcqe.pg_host_opaque = req->pg_host_opaque; 2254 kcqe.pg_cid = req->pg_cid; 2255 kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG; 2256 cqes[0] = (struct kcqe *) &kcqe; 2257 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2258 return 0; 2259 } 2260 2261 static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe) 2262 { 2263 struct fcoe_kwqe_stat *req; 2264 struct fcoe_stat_ramrod_params *fcoe_stat; 2265 union l5cm_specific_data l5_data; 2266 struct cnic_local *cp = dev->cnic_priv; 2267 struct bnx2x *bp = netdev_priv(dev->netdev); 2268 int ret; 2269 u32 cid; 2270 2271 req = (struct fcoe_kwqe_stat *) kwqe; 2272 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2273 2274 fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2275 if (!fcoe_stat) 2276 return -ENOMEM; 2277 2278 memset(fcoe_stat, 0, sizeof(*fcoe_stat)); 2279 memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req)); 2280 2281 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid, 2282 FCOE_CONNECTION_TYPE, &l5_data); 2283 return ret; 2284 } 2285 2286 static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[], 2287 u32 num, int *work) 2288 { 2289 int ret; 2290 struct cnic_local *cp = dev->cnic_priv; 2291 struct bnx2x *bp = netdev_priv(dev->netdev); 2292 u32 cid; 2293 struct fcoe_init_ramrod_params *fcoe_init; 2294 struct fcoe_kwqe_init1 *req1; 2295 struct fcoe_kwqe_init2 *req2; 2296 struct fcoe_kwqe_init3 *req3; 2297 union l5cm_specific_data l5_data; 2298 2299 if (num < 3) { 2300 *work = num; 2301 return -EINVAL; 2302 } 2303 req1 = (struct fcoe_kwqe_init1 *) wqes[0]; 2304 req2 = (struct fcoe_kwqe_init2 *) wqes[1]; 2305 req3 = (struct fcoe_kwqe_init3 *) wqes[2]; 2306 if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) { 2307 *work = 1; 2308 return -EINVAL; 2309 } 2310 if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) { 2311 *work = 2; 2312 return -EINVAL; 2313 } 2314 2315 if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) { 2316 netdev_err(dev->netdev, "fcoe_init size too big\n"); 2317 return -ENOMEM; 2318 } 2319 fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2320 if (!fcoe_init) 2321 return -ENOMEM; 2322 2323 memset(fcoe_init, 0, sizeof(*fcoe_init)); 2324 memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1)); 2325 memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2)); 2326 memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3)); 2327 fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff; 2328 fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32; 2329 fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages; 2330 2331 fcoe_init->sb_num = cp->status_blk_num; 2332 fcoe_init->eq_prod = MAX_KCQ_IDX; 2333 fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS; 2334 cp->kcq2.sw_prod_idx = 0; 2335 2336 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2337 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid, 2338 FCOE_CONNECTION_TYPE, &l5_data); 2339 *work = 3; 2340 return ret; 2341 } 2342 2343 static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 2344 u32 num, int *work) 2345 { 2346 int ret = 0; 2347 u32 cid = -1, l5_cid; 2348 struct cnic_local *cp = dev->cnic_priv; 2349 struct bnx2x *bp = netdev_priv(dev->netdev); 2350 struct fcoe_kwqe_conn_offload1 *req1; 2351 struct fcoe_kwqe_conn_offload2 *req2; 2352 struct fcoe_kwqe_conn_offload3 *req3; 2353 struct fcoe_kwqe_conn_offload4 *req4; 2354 struct fcoe_conn_offload_ramrod_params *fcoe_offload; 2355 struct cnic_context *ctx; 2356 struct fcoe_context *fctx; 2357 struct regpair ctx_addr; 2358 union l5cm_specific_data l5_data; 2359 struct fcoe_kcqe kcqe; 2360 struct kcqe *cqes[1]; 2361 2362 if (num < 4) { 2363 *work = num; 2364 return -EINVAL; 2365 } 2366 req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0]; 2367 req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1]; 2368 req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2]; 2369 req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3]; 2370 2371 *work = 4; 2372 2373 l5_cid = req1->fcoe_conn_id; 2374 if (l5_cid >= dev->max_fcoe_conn) 2375 goto err_reply; 2376 2377 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2378 2379 ctx = &cp->ctx_tbl[l5_cid]; 2380 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2381 goto err_reply; 2382 2383 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 2384 if (ret) { 2385 ret = 0; 2386 goto err_reply; 2387 } 2388 cid = ctx->cid; 2389 2390 fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr); 2391 if (fctx) { 2392 u32 hw_cid = BNX2X_HW_CID(bp, cid); 2393 u32 val; 2394 2395 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 2396 FCOE_CONNECTION_TYPE); 2397 fctx->xstorm_ag_context.cdu_reserved = val; 2398 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 2399 FCOE_CONNECTION_TYPE); 2400 fctx->ustorm_ag_context.cdu_usage = val; 2401 } 2402 if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) { 2403 netdev_err(dev->netdev, "fcoe_offload size too big\n"); 2404 goto err_reply; 2405 } 2406 fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2407 if (!fcoe_offload) 2408 goto err_reply; 2409 2410 memset(fcoe_offload, 0, sizeof(*fcoe_offload)); 2411 memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1)); 2412 memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2)); 2413 memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3)); 2414 memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4)); 2415 2416 cid = BNX2X_HW_CID(bp, cid); 2417 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid, 2418 FCOE_CONNECTION_TYPE, &l5_data); 2419 if (!ret) 2420 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2421 2422 return ret; 2423 2424 err_reply: 2425 if (cid != -1) 2426 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2427 2428 memset(&kcqe, 0, sizeof(kcqe)); 2429 kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN; 2430 kcqe.fcoe_conn_id = req1->fcoe_conn_id; 2431 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 2432 2433 cqes[0] = (struct kcqe *) &kcqe; 2434 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2435 return ret; 2436 } 2437 2438 static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe) 2439 { 2440 struct fcoe_kwqe_conn_enable_disable *req; 2441 struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable; 2442 union l5cm_specific_data l5_data; 2443 int ret; 2444 u32 cid, l5_cid; 2445 struct cnic_local *cp = dev->cnic_priv; 2446 2447 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2448 cid = req->context_id; 2449 l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE; 2450 2451 if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) { 2452 netdev_err(dev->netdev, "fcoe_enable size too big\n"); 2453 return -ENOMEM; 2454 } 2455 fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2456 if (!fcoe_enable) 2457 return -ENOMEM; 2458 2459 memset(fcoe_enable, 0, sizeof(*fcoe_enable)); 2460 memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req)); 2461 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid, 2462 FCOE_CONNECTION_TYPE, &l5_data); 2463 return ret; 2464 } 2465 2466 static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe) 2467 { 2468 struct fcoe_kwqe_conn_enable_disable *req; 2469 struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable; 2470 union l5cm_specific_data l5_data; 2471 int ret; 2472 u32 cid, l5_cid; 2473 struct cnic_local *cp = dev->cnic_priv; 2474 2475 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2476 cid = req->context_id; 2477 l5_cid = req->conn_id; 2478 if (l5_cid >= dev->max_fcoe_conn) 2479 return -EINVAL; 2480 2481 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2482 2483 if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) { 2484 netdev_err(dev->netdev, "fcoe_disable size too big\n"); 2485 return -ENOMEM; 2486 } 2487 fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2488 if (!fcoe_disable) 2489 return -ENOMEM; 2490 2491 memset(fcoe_disable, 0, sizeof(*fcoe_disable)); 2492 memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req)); 2493 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid, 2494 FCOE_CONNECTION_TYPE, &l5_data); 2495 return ret; 2496 } 2497 2498 static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2499 { 2500 struct fcoe_kwqe_conn_destroy *req; 2501 union l5cm_specific_data l5_data; 2502 int ret; 2503 u32 cid, l5_cid; 2504 struct cnic_local *cp = dev->cnic_priv; 2505 struct cnic_context *ctx; 2506 struct fcoe_kcqe kcqe; 2507 struct kcqe *cqes[1]; 2508 2509 req = (struct fcoe_kwqe_conn_destroy *) kwqe; 2510 cid = req->context_id; 2511 l5_cid = req->conn_id; 2512 if (l5_cid >= dev->max_fcoe_conn) 2513 return -EINVAL; 2514 2515 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2516 2517 ctx = &cp->ctx_tbl[l5_cid]; 2518 2519 init_waitqueue_head(&ctx->waitq); 2520 ctx->wait_cond = 0; 2521 2522 memset(&kcqe, 0, sizeof(kcqe)); 2523 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR; 2524 memset(&l5_data, 0, sizeof(l5_data)); 2525 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid, 2526 FCOE_CONNECTION_TYPE, &l5_data); 2527 if (ret == 0) { 2528 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 2529 if (ctx->wait_cond) 2530 kcqe.completion_status = 0; 2531 } 2532 2533 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2534 queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000)); 2535 2536 kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN; 2537 kcqe.fcoe_conn_id = req->conn_id; 2538 kcqe.fcoe_conn_context_id = cid; 2539 2540 cqes[0] = (struct kcqe *) &kcqe; 2541 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2542 return ret; 2543 } 2544 2545 static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid) 2546 { 2547 struct cnic_local *cp = dev->cnic_priv; 2548 u32 i; 2549 2550 for (i = start_cid; i < cp->max_cid_space; i++) { 2551 struct cnic_context *ctx = &cp->ctx_tbl[i]; 2552 int j; 2553 2554 while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 2555 msleep(10); 2556 2557 for (j = 0; j < 5; j++) { 2558 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2559 break; 2560 msleep(20); 2561 } 2562 2563 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2564 netdev_warn(dev->netdev, "CID %x not deleted\n", 2565 ctx->cid); 2566 } 2567 } 2568 2569 static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2570 { 2571 union l5cm_specific_data l5_data; 2572 struct cnic_local *cp = dev->cnic_priv; 2573 struct bnx2x *bp = netdev_priv(dev->netdev); 2574 int ret; 2575 u32 cid; 2576 2577 cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ); 2578 2579 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2580 2581 memset(&l5_data, 0, sizeof(l5_data)); 2582 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid, 2583 FCOE_CONNECTION_TYPE, &l5_data); 2584 return ret; 2585 } 2586 2587 static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe) 2588 { 2589 struct cnic_local *cp = dev->cnic_priv; 2590 struct kcqe kcqe; 2591 struct kcqe *cqes[1]; 2592 u32 cid; 2593 u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2594 u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK; 2595 u32 kcqe_op; 2596 int ulp_type; 2597 2598 cid = kwqe->kwqe_info0; 2599 memset(&kcqe, 0, sizeof(kcqe)); 2600 2601 if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) { 2602 u32 l5_cid = 0; 2603 2604 ulp_type = CNIC_ULP_FCOE; 2605 if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) { 2606 struct fcoe_kwqe_conn_enable_disable *req; 2607 2608 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2609 kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN; 2610 cid = req->context_id; 2611 l5_cid = req->conn_id; 2612 } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) { 2613 kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC; 2614 } else { 2615 return; 2616 } 2617 kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT; 2618 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE; 2619 kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2620 kcqe.kcqe_info2 = cid; 2621 kcqe.kcqe_info0 = l5_cid; 2622 2623 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) { 2624 ulp_type = CNIC_ULP_ISCSI; 2625 if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN) 2626 cid = kwqe->kwqe_info1; 2627 2628 kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT; 2629 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI; 2630 kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR; 2631 kcqe.kcqe_info2 = cid; 2632 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0); 2633 2634 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) { 2635 struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe; 2636 2637 ulp_type = CNIC_ULP_L4; 2638 if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1) 2639 kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE; 2640 else if (opcode == L4_KWQE_OPCODE_VALUE_RESET) 2641 kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP; 2642 else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE) 2643 kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 2644 else 2645 return; 2646 2647 kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) | 2648 KCQE_FLAGS_LAYER_MASK_L4; 2649 l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2650 l4kcqe->cid = cid; 2651 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id); 2652 } else { 2653 return; 2654 } 2655 2656 cqes[0] = &kcqe; 2657 cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1); 2658 } 2659 2660 static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev, 2661 struct kwqe *wqes[], u32 num_wqes) 2662 { 2663 int i, work, ret; 2664 u32 opcode; 2665 struct kwqe *kwqe; 2666 2667 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2668 return -EAGAIN; /* bnx2 is down */ 2669 2670 for (i = 0; i < num_wqes; ) { 2671 kwqe = wqes[i]; 2672 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2673 work = 1; 2674 2675 switch (opcode) { 2676 case ISCSI_KWQE_OPCODE_INIT1: 2677 ret = cnic_bnx2x_iscsi_init1(dev, kwqe); 2678 break; 2679 case ISCSI_KWQE_OPCODE_INIT2: 2680 ret = cnic_bnx2x_iscsi_init2(dev, kwqe); 2681 break; 2682 case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1: 2683 ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i], 2684 num_wqes - i, &work); 2685 break; 2686 case ISCSI_KWQE_OPCODE_UPDATE_CONN: 2687 ret = cnic_bnx2x_iscsi_update(dev, kwqe); 2688 break; 2689 case ISCSI_KWQE_OPCODE_DESTROY_CONN: 2690 ret = cnic_bnx2x_iscsi_destroy(dev, kwqe); 2691 break; 2692 case L4_KWQE_OPCODE_VALUE_CONNECT1: 2693 ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i, 2694 &work); 2695 break; 2696 case L4_KWQE_OPCODE_VALUE_CLOSE: 2697 ret = cnic_bnx2x_close(dev, kwqe); 2698 break; 2699 case L4_KWQE_OPCODE_VALUE_RESET: 2700 ret = cnic_bnx2x_reset(dev, kwqe); 2701 break; 2702 case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG: 2703 ret = cnic_bnx2x_offload_pg(dev, kwqe); 2704 break; 2705 case L4_KWQE_OPCODE_VALUE_UPDATE_PG: 2706 ret = cnic_bnx2x_update_pg(dev, kwqe); 2707 break; 2708 case L4_KWQE_OPCODE_VALUE_UPLOAD_PG: 2709 ret = 0; 2710 break; 2711 default: 2712 ret = 0; 2713 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2714 opcode); 2715 break; 2716 } 2717 if (ret < 0) { 2718 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2719 opcode); 2720 2721 /* Possibly bnx2x parity error, send completion 2722 * to ulp drivers with error code to speed up 2723 * cleanup and reset recovery. 2724 */ 2725 if (ret == -EIO || ret == -EAGAIN) 2726 cnic_bnx2x_kwqe_err(dev, kwqe); 2727 } 2728 i += work; 2729 } 2730 return 0; 2731 } 2732 2733 static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev, 2734 struct kwqe *wqes[], u32 num_wqes) 2735 { 2736 struct bnx2x *bp = netdev_priv(dev->netdev); 2737 int i, work, ret; 2738 u32 opcode; 2739 struct kwqe *kwqe; 2740 2741 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2742 return -EAGAIN; /* bnx2 is down */ 2743 2744 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 2745 return -EINVAL; 2746 2747 for (i = 0; i < num_wqes; ) { 2748 kwqe = wqes[i]; 2749 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2750 work = 1; 2751 2752 switch (opcode) { 2753 case FCOE_KWQE_OPCODE_INIT1: 2754 ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i], 2755 num_wqes - i, &work); 2756 break; 2757 case FCOE_KWQE_OPCODE_OFFLOAD_CONN1: 2758 ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i], 2759 num_wqes - i, &work); 2760 break; 2761 case FCOE_KWQE_OPCODE_ENABLE_CONN: 2762 ret = cnic_bnx2x_fcoe_enable(dev, kwqe); 2763 break; 2764 case FCOE_KWQE_OPCODE_DISABLE_CONN: 2765 ret = cnic_bnx2x_fcoe_disable(dev, kwqe); 2766 break; 2767 case FCOE_KWQE_OPCODE_DESTROY_CONN: 2768 ret = cnic_bnx2x_fcoe_destroy(dev, kwqe); 2769 break; 2770 case FCOE_KWQE_OPCODE_DESTROY: 2771 ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe); 2772 break; 2773 case FCOE_KWQE_OPCODE_STAT: 2774 ret = cnic_bnx2x_fcoe_stat(dev, kwqe); 2775 break; 2776 default: 2777 ret = 0; 2778 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2779 opcode); 2780 break; 2781 } 2782 if (ret < 0) { 2783 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2784 opcode); 2785 2786 /* Possibly bnx2x parity error, send completion 2787 * to ulp drivers with error code to speed up 2788 * cleanup and reset recovery. 2789 */ 2790 if (ret == -EIO || ret == -EAGAIN) 2791 cnic_bnx2x_kwqe_err(dev, kwqe); 2792 } 2793 i += work; 2794 } 2795 return 0; 2796 } 2797 2798 static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 2799 u32 num_wqes) 2800 { 2801 int ret = -EINVAL; 2802 u32 layer_code; 2803 2804 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2805 return -EAGAIN; /* bnx2x is down */ 2806 2807 if (!num_wqes) 2808 return 0; 2809 2810 layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK; 2811 switch (layer_code) { 2812 case KWQE_FLAGS_LAYER_MASK_L5_ISCSI: 2813 case KWQE_FLAGS_LAYER_MASK_L4: 2814 case KWQE_FLAGS_LAYER_MASK_L2: 2815 ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes); 2816 break; 2817 2818 case KWQE_FLAGS_LAYER_MASK_L5_FCOE: 2819 ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes); 2820 break; 2821 } 2822 return ret; 2823 } 2824 2825 static inline u32 cnic_get_kcqe_layer_mask(u32 opflag) 2826 { 2827 if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN)) 2828 return KCQE_FLAGS_LAYER_MASK_L4; 2829 2830 return opflag & KCQE_FLAGS_LAYER_MASK; 2831 } 2832 2833 static void service_kcqes(struct cnic_dev *dev, int num_cqes) 2834 { 2835 struct cnic_local *cp = dev->cnic_priv; 2836 int i, j, comp = 0; 2837 2838 i = 0; 2839 j = 1; 2840 while (num_cqes) { 2841 struct cnic_ulp_ops *ulp_ops; 2842 int ulp_type; 2843 u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag; 2844 u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag); 2845 2846 if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION)) 2847 comp++; 2848 2849 while (j < num_cqes) { 2850 u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag; 2851 2852 if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer) 2853 break; 2854 2855 if (unlikely(next_op & KCQE_RAMROD_COMPLETION)) 2856 comp++; 2857 j++; 2858 } 2859 2860 if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA) 2861 ulp_type = CNIC_ULP_RDMA; 2862 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI) 2863 ulp_type = CNIC_ULP_ISCSI; 2864 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE) 2865 ulp_type = CNIC_ULP_FCOE; 2866 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4) 2867 ulp_type = CNIC_ULP_L4; 2868 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2) 2869 goto end; 2870 else { 2871 netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n", 2872 kcqe_op_flag); 2873 goto end; 2874 } 2875 2876 rcu_read_lock(); 2877 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 2878 if (likely(ulp_ops)) { 2879 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 2880 cp->completed_kcq + i, j); 2881 } 2882 rcu_read_unlock(); 2883 end: 2884 num_cqes -= j; 2885 i += j; 2886 j = 1; 2887 } 2888 if (unlikely(comp)) 2889 cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp); 2890 } 2891 2892 static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info) 2893 { 2894 struct cnic_local *cp = dev->cnic_priv; 2895 u16 i, ri, hw_prod, last; 2896 struct kcqe *kcqe; 2897 int kcqe_cnt = 0, last_cnt = 0; 2898 2899 i = ri = last = info->sw_prod_idx; 2900 ri &= MAX_KCQ_IDX; 2901 hw_prod = *info->hw_prod_idx_ptr; 2902 hw_prod = info->hw_idx(hw_prod); 2903 2904 while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) { 2905 kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)]; 2906 cp->completed_kcq[kcqe_cnt++] = kcqe; 2907 i = info->next_idx(i); 2908 ri = i & MAX_KCQ_IDX; 2909 if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) { 2910 last_cnt = kcqe_cnt; 2911 last = i; 2912 } 2913 } 2914 2915 info->sw_prod_idx = last; 2916 return last_cnt; 2917 } 2918 2919 static int cnic_l2_completion(struct cnic_local *cp) 2920 { 2921 u16 hw_cons, sw_cons; 2922 struct cnic_uio_dev *udev = cp->udev; 2923 union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *) 2924 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 2925 u32 cmd; 2926 int comp = 0; 2927 2928 if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags)) 2929 return 0; 2930 2931 hw_cons = *cp->rx_cons_ptr; 2932 if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT) 2933 hw_cons++; 2934 2935 sw_cons = cp->rx_cons; 2936 while (sw_cons != hw_cons) { 2937 u8 cqe_fp_flags; 2938 2939 cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT]; 2940 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 2941 if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) { 2942 cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data); 2943 cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT; 2944 if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP || 2945 cmd == RAMROD_CMD_ID_ETH_HALT) 2946 comp++; 2947 } 2948 sw_cons = BNX2X_NEXT_RCQE(sw_cons); 2949 } 2950 return comp; 2951 } 2952 2953 static void cnic_chk_pkt_rings(struct cnic_local *cp) 2954 { 2955 u16 rx_cons, tx_cons; 2956 int comp = 0; 2957 2958 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 2959 return; 2960 2961 rx_cons = *cp->rx_cons_ptr; 2962 tx_cons = *cp->tx_cons_ptr; 2963 if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) { 2964 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 2965 comp = cnic_l2_completion(cp); 2966 2967 cp->tx_cons = tx_cons; 2968 cp->rx_cons = rx_cons; 2969 2970 if (cp->udev) 2971 uio_event_notify(&cp->udev->cnic_uinfo); 2972 } 2973 if (comp) 2974 clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 2975 } 2976 2977 static u32 cnic_service_bnx2_queues(struct cnic_dev *dev) 2978 { 2979 struct cnic_local *cp = dev->cnic_priv; 2980 u32 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2981 int kcqe_cnt; 2982 2983 /* status block index must be read before reading other fields */ 2984 rmb(); 2985 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2986 2987 while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) { 2988 2989 service_kcqes(dev, kcqe_cnt); 2990 2991 /* Tell compiler that status_blk fields can change. */ 2992 barrier(); 2993 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2994 /* status block index must be read first */ 2995 rmb(); 2996 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2997 } 2998 2999 CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx); 3000 3001 cnic_chk_pkt_rings(cp); 3002 3003 return status_idx; 3004 } 3005 3006 static int cnic_service_bnx2(void *data, void *status_blk) 3007 { 3008 struct cnic_dev *dev = data; 3009 3010 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3011 struct status_block *sblk = status_blk; 3012 3013 return sblk->status_idx; 3014 } 3015 3016 return cnic_service_bnx2_queues(dev); 3017 } 3018 3019 static void cnic_service_bnx2_msix(struct work_struct *work) 3020 { 3021 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3022 struct cnic_dev *dev = cp->dev; 3023 3024 cp->last_status_idx = cnic_service_bnx2_queues(dev); 3025 3026 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 3027 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 3028 } 3029 3030 static void cnic_doirq(struct cnic_dev *dev) 3031 { 3032 struct cnic_local *cp = dev->cnic_priv; 3033 3034 if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3035 u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX; 3036 3037 prefetch(cp->status_blk.gen); 3038 prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]); 3039 3040 queue_work(system_bh_wq, &cp->cnic_irq_bh_work); 3041 } 3042 } 3043 3044 static irqreturn_t cnic_irq(int irq, void *dev_instance) 3045 { 3046 struct cnic_dev *dev = dev_instance; 3047 struct cnic_local *cp = dev->cnic_priv; 3048 3049 if (cp->ack_int) 3050 cp->ack_int(dev); 3051 3052 cnic_doirq(dev); 3053 3054 return IRQ_HANDLED; 3055 } 3056 3057 static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm, 3058 u16 index, u8 op, u8 update) 3059 { 3060 struct bnx2x *bp = netdev_priv(dev->netdev); 3061 u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp) * 32 + 3062 COMMAND_REG_INT_ACK); 3063 struct igu_ack_register igu_ack; 3064 3065 igu_ack.status_block_index = index; 3066 igu_ack.sb_id_and_flags = 3067 ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) | 3068 (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) | 3069 (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) | 3070 (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT)); 3071 3072 CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack)); 3073 } 3074 3075 static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment, 3076 u16 index, u8 op, u8 update) 3077 { 3078 struct igu_regular cmd_data; 3079 u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8; 3080 3081 cmd_data.sb_id_and_flags = 3082 (index << IGU_REGULAR_SB_INDEX_SHIFT) | 3083 (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) | 3084 (update << IGU_REGULAR_BUPDATE_SHIFT) | 3085 (op << IGU_REGULAR_ENABLE_INT_SHIFT); 3086 3087 3088 CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags); 3089 } 3090 3091 static void cnic_ack_bnx2x_msix(struct cnic_dev *dev) 3092 { 3093 struct cnic_local *cp = dev->cnic_priv; 3094 3095 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0, 3096 IGU_INT_DISABLE, 0); 3097 } 3098 3099 static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev) 3100 { 3101 struct cnic_local *cp = dev->cnic_priv; 3102 3103 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0, 3104 IGU_INT_DISABLE, 0); 3105 } 3106 3107 static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx) 3108 { 3109 struct cnic_local *cp = dev->cnic_priv; 3110 3111 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx, 3112 IGU_INT_ENABLE, 1); 3113 } 3114 3115 static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx) 3116 { 3117 struct cnic_local *cp = dev->cnic_priv; 3118 3119 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx, 3120 IGU_INT_ENABLE, 1); 3121 } 3122 3123 static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info) 3124 { 3125 u32 last_status = *info->status_idx_ptr; 3126 int kcqe_cnt; 3127 3128 /* status block index must be read before reading the KCQ */ 3129 rmb(); 3130 while ((kcqe_cnt = cnic_get_kcqes(dev, info))) { 3131 3132 service_kcqes(dev, kcqe_cnt); 3133 3134 /* Tell compiler that sblk fields can change. */ 3135 barrier(); 3136 3137 last_status = *info->status_idx_ptr; 3138 /* status block index must be read before reading the KCQ */ 3139 rmb(); 3140 } 3141 return last_status; 3142 } 3143 3144 static void cnic_service_bnx2x_bh_work(struct work_struct *work) 3145 { 3146 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3147 struct cnic_dev *dev = cp->dev; 3148 struct bnx2x *bp = netdev_priv(dev->netdev); 3149 u32 status_idx, new_status_idx; 3150 3151 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) 3152 return; 3153 3154 while (1) { 3155 status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1); 3156 3157 CNIC_WR16(dev, cp->kcq1.io_addr, 3158 cp->kcq1.sw_prod_idx + MAX_KCQ_IDX); 3159 3160 if (!CNIC_SUPPORTS_FCOE(bp)) { 3161 cp->arm_int(dev, status_idx); 3162 break; 3163 } 3164 3165 new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2); 3166 3167 if (new_status_idx != status_idx) 3168 continue; 3169 3170 CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx + 3171 MAX_KCQ_IDX); 3172 3173 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 3174 status_idx, IGU_INT_ENABLE, 1); 3175 3176 break; 3177 } 3178 } 3179 3180 static int cnic_service_bnx2x(void *data, void *status_blk) 3181 { 3182 struct cnic_dev *dev = data; 3183 struct cnic_local *cp = dev->cnic_priv; 3184 3185 if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 3186 cnic_doirq(dev); 3187 3188 cnic_chk_pkt_rings(cp); 3189 3190 return 0; 3191 } 3192 3193 static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type) 3194 { 3195 struct cnic_ulp_ops *ulp_ops; 3196 3197 if (if_type == CNIC_ULP_ISCSI) 3198 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 3199 3200 mutex_lock(&cnic_lock); 3201 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3202 lockdep_is_held(&cnic_lock)); 3203 if (!ulp_ops) { 3204 mutex_unlock(&cnic_lock); 3205 return; 3206 } 3207 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3208 mutex_unlock(&cnic_lock); 3209 3210 if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3211 ulp_ops->cnic_stop(cp->ulp_handle[if_type]); 3212 3213 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3214 } 3215 3216 static void cnic_ulp_stop(struct cnic_dev *dev) 3217 { 3218 struct cnic_local *cp = dev->cnic_priv; 3219 int if_type; 3220 3221 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) 3222 cnic_ulp_stop_one(cp, if_type); 3223 } 3224 3225 static void cnic_ulp_start(struct cnic_dev *dev) 3226 { 3227 struct cnic_local *cp = dev->cnic_priv; 3228 int if_type; 3229 3230 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 3231 struct cnic_ulp_ops *ulp_ops; 3232 3233 mutex_lock(&cnic_lock); 3234 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3235 lockdep_is_held(&cnic_lock)); 3236 if (!ulp_ops || !ulp_ops->cnic_start) { 3237 mutex_unlock(&cnic_lock); 3238 continue; 3239 } 3240 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3241 mutex_unlock(&cnic_lock); 3242 3243 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3244 ulp_ops->cnic_start(cp->ulp_handle[if_type]); 3245 3246 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3247 } 3248 } 3249 3250 static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type) 3251 { 3252 struct cnic_local *cp = dev->cnic_priv; 3253 struct cnic_ulp_ops *ulp_ops; 3254 int rc; 3255 3256 mutex_lock(&cnic_lock); 3257 ulp_ops = rcu_dereference_protected(cp->ulp_ops[ulp_type], 3258 lockdep_is_held(&cnic_lock)); 3259 if (ulp_ops && ulp_ops->cnic_get_stats) 3260 rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]); 3261 else 3262 rc = -ENODEV; 3263 mutex_unlock(&cnic_lock); 3264 return rc; 3265 } 3266 3267 static int cnic_ctl(void *data, struct cnic_ctl_info *info) 3268 { 3269 struct cnic_dev *dev = data; 3270 int ulp_type = CNIC_ULP_ISCSI; 3271 3272 switch (info->cmd) { 3273 case CNIC_CTL_STOP_CMD: 3274 cnic_hold(dev); 3275 3276 cnic_ulp_stop(dev); 3277 cnic_stop_hw(dev); 3278 3279 cnic_put(dev); 3280 break; 3281 case CNIC_CTL_START_CMD: 3282 cnic_hold(dev); 3283 3284 if (!cnic_start_hw(dev)) 3285 cnic_ulp_start(dev); 3286 3287 cnic_put(dev); 3288 break; 3289 case CNIC_CTL_STOP_ISCSI_CMD: { 3290 struct cnic_local *cp = dev->cnic_priv; 3291 set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags); 3292 queue_delayed_work(cnic_wq, &cp->delete_task, 0); 3293 break; 3294 } 3295 case CNIC_CTL_COMPLETION_CMD: { 3296 struct cnic_ctl_completion *comp = &info->data.comp; 3297 u32 cid = BNX2X_SW_CID(comp->cid); 3298 u32 l5_cid; 3299 struct cnic_local *cp = dev->cnic_priv; 3300 3301 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 3302 break; 3303 3304 if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) { 3305 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3306 3307 if (unlikely(comp->error)) { 3308 set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags); 3309 netdev_err(dev->netdev, 3310 "CID %x CFC delete comp error %x\n", 3311 cid, comp->error); 3312 } 3313 3314 ctx->wait_cond = 1; 3315 wake_up(&ctx->waitq); 3316 } 3317 break; 3318 } 3319 case CNIC_CTL_FCOE_STATS_GET_CMD: 3320 ulp_type = CNIC_ULP_FCOE; 3321 fallthrough; 3322 case CNIC_CTL_ISCSI_STATS_GET_CMD: 3323 cnic_hold(dev); 3324 cnic_copy_ulp_stats(dev, ulp_type); 3325 cnic_put(dev); 3326 break; 3327 3328 default: 3329 return -EINVAL; 3330 } 3331 return 0; 3332 } 3333 3334 static void cnic_ulp_init(struct cnic_dev *dev) 3335 { 3336 int i; 3337 struct cnic_local *cp = dev->cnic_priv; 3338 3339 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3340 struct cnic_ulp_ops *ulp_ops; 3341 3342 mutex_lock(&cnic_lock); 3343 ulp_ops = cnic_ulp_tbl_prot(i); 3344 if (!ulp_ops || !ulp_ops->cnic_init) { 3345 mutex_unlock(&cnic_lock); 3346 continue; 3347 } 3348 ulp_get(ulp_ops); 3349 mutex_unlock(&cnic_lock); 3350 3351 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3352 ulp_ops->cnic_init(dev); 3353 3354 ulp_put(ulp_ops); 3355 } 3356 } 3357 3358 static void cnic_ulp_exit(struct cnic_dev *dev) 3359 { 3360 int i; 3361 struct cnic_local *cp = dev->cnic_priv; 3362 3363 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3364 struct cnic_ulp_ops *ulp_ops; 3365 3366 mutex_lock(&cnic_lock); 3367 ulp_ops = cnic_ulp_tbl_prot(i); 3368 if (!ulp_ops || !ulp_ops->cnic_exit) { 3369 mutex_unlock(&cnic_lock); 3370 continue; 3371 } 3372 ulp_get(ulp_ops); 3373 mutex_unlock(&cnic_lock); 3374 3375 if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3376 ulp_ops->cnic_exit(dev); 3377 3378 ulp_put(ulp_ops); 3379 } 3380 } 3381 3382 static int cnic_cm_offload_pg(struct cnic_sock *csk) 3383 { 3384 struct cnic_dev *dev = csk->dev; 3385 struct l4_kwq_offload_pg *l4kwqe; 3386 struct kwqe *wqes[1]; 3387 3388 l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1; 3389 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3390 wqes[0] = (struct kwqe *) l4kwqe; 3391 3392 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG; 3393 l4kwqe->flags = 3394 L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT; 3395 l4kwqe->l2hdr_nbytes = ETH_HLEN; 3396 3397 l4kwqe->da0 = csk->ha[0]; 3398 l4kwqe->da1 = csk->ha[1]; 3399 l4kwqe->da2 = csk->ha[2]; 3400 l4kwqe->da3 = csk->ha[3]; 3401 l4kwqe->da4 = csk->ha[4]; 3402 l4kwqe->da5 = csk->ha[5]; 3403 3404 l4kwqe->sa0 = dev->mac_addr[0]; 3405 l4kwqe->sa1 = dev->mac_addr[1]; 3406 l4kwqe->sa2 = dev->mac_addr[2]; 3407 l4kwqe->sa3 = dev->mac_addr[3]; 3408 l4kwqe->sa4 = dev->mac_addr[4]; 3409 l4kwqe->sa5 = dev->mac_addr[5]; 3410 3411 l4kwqe->etype = ETH_P_IP; 3412 l4kwqe->ipid_start = DEF_IPID_START; 3413 l4kwqe->host_opaque = csk->l5_cid; 3414 3415 if (csk->vlan_id) { 3416 l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING; 3417 l4kwqe->vlan_tag = csk->vlan_id; 3418 l4kwqe->l2hdr_nbytes += 4; 3419 } 3420 3421 return dev->submit_kwqes(dev, wqes, 1); 3422 } 3423 3424 static int cnic_cm_update_pg(struct cnic_sock *csk) 3425 { 3426 struct cnic_dev *dev = csk->dev; 3427 struct l4_kwq_update_pg *l4kwqe; 3428 struct kwqe *wqes[1]; 3429 3430 l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1; 3431 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3432 wqes[0] = (struct kwqe *) l4kwqe; 3433 3434 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG; 3435 l4kwqe->flags = 3436 L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT; 3437 l4kwqe->pg_cid = csk->pg_cid; 3438 3439 l4kwqe->da0 = csk->ha[0]; 3440 l4kwqe->da1 = csk->ha[1]; 3441 l4kwqe->da2 = csk->ha[2]; 3442 l4kwqe->da3 = csk->ha[3]; 3443 l4kwqe->da4 = csk->ha[4]; 3444 l4kwqe->da5 = csk->ha[5]; 3445 3446 l4kwqe->pg_host_opaque = csk->l5_cid; 3447 l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA; 3448 3449 return dev->submit_kwqes(dev, wqes, 1); 3450 } 3451 3452 static int cnic_cm_upload_pg(struct cnic_sock *csk) 3453 { 3454 struct cnic_dev *dev = csk->dev; 3455 struct l4_kwq_upload *l4kwqe; 3456 struct kwqe *wqes[1]; 3457 3458 l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1; 3459 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3460 wqes[0] = (struct kwqe *) l4kwqe; 3461 3462 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG; 3463 l4kwqe->flags = 3464 L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT; 3465 l4kwqe->cid = csk->pg_cid; 3466 3467 return dev->submit_kwqes(dev, wqes, 1); 3468 } 3469 3470 static int cnic_cm_conn_req(struct cnic_sock *csk) 3471 { 3472 struct cnic_dev *dev = csk->dev; 3473 struct l4_kwq_connect_req1 *l4kwqe1; 3474 struct l4_kwq_connect_req2 *l4kwqe2; 3475 struct l4_kwq_connect_req3 *l4kwqe3; 3476 struct kwqe *wqes[3]; 3477 u8 tcp_flags = 0; 3478 int num_wqes = 2; 3479 3480 l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1; 3481 l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2; 3482 l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3; 3483 memset(l4kwqe1, 0, sizeof(*l4kwqe1)); 3484 memset(l4kwqe2, 0, sizeof(*l4kwqe2)); 3485 memset(l4kwqe3, 0, sizeof(*l4kwqe3)); 3486 3487 l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3; 3488 l4kwqe3->flags = 3489 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT; 3490 l4kwqe3->ka_timeout = csk->ka_timeout; 3491 l4kwqe3->ka_interval = csk->ka_interval; 3492 l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count; 3493 l4kwqe3->tos = csk->tos; 3494 l4kwqe3->ttl = csk->ttl; 3495 l4kwqe3->snd_seq_scale = csk->snd_seq_scale; 3496 l4kwqe3->pmtu = csk->mtu; 3497 l4kwqe3->rcv_buf = csk->rcv_buf; 3498 l4kwqe3->snd_buf = csk->snd_buf; 3499 l4kwqe3->seed = csk->seed; 3500 3501 wqes[0] = (struct kwqe *) l4kwqe1; 3502 if (test_bit(SK_F_IPV6, &csk->flags)) { 3503 wqes[1] = (struct kwqe *) l4kwqe2; 3504 wqes[2] = (struct kwqe *) l4kwqe3; 3505 num_wqes = 3; 3506 3507 l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6; 3508 l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2; 3509 l4kwqe2->flags = 3510 L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT | 3511 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT; 3512 l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]); 3513 l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]); 3514 l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]); 3515 l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]); 3516 l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]); 3517 l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]); 3518 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) - 3519 sizeof(struct tcphdr); 3520 } else { 3521 wqes[1] = (struct kwqe *) l4kwqe3; 3522 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) - 3523 sizeof(struct tcphdr); 3524 } 3525 3526 l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1; 3527 l4kwqe1->flags = 3528 (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) | 3529 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT; 3530 l4kwqe1->cid = csk->cid; 3531 l4kwqe1->pg_cid = csk->pg_cid; 3532 l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]); 3533 l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]); 3534 l4kwqe1->src_port = be16_to_cpu(csk->src_port); 3535 l4kwqe1->dst_port = be16_to_cpu(csk->dst_port); 3536 if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK) 3537 tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK; 3538 if (csk->tcp_flags & SK_TCP_KEEP_ALIVE) 3539 tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE; 3540 if (csk->tcp_flags & SK_TCP_NAGLE) 3541 tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE; 3542 if (csk->tcp_flags & SK_TCP_TIMESTAMP) 3543 tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP; 3544 if (csk->tcp_flags & SK_TCP_SACK) 3545 tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK; 3546 if (csk->tcp_flags & SK_TCP_SEG_SCALING) 3547 tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING; 3548 3549 l4kwqe1->tcp_flags = tcp_flags; 3550 3551 return dev->submit_kwqes(dev, wqes, num_wqes); 3552 } 3553 3554 static int cnic_cm_close_req(struct cnic_sock *csk) 3555 { 3556 struct cnic_dev *dev = csk->dev; 3557 struct l4_kwq_close_req *l4kwqe; 3558 struct kwqe *wqes[1]; 3559 3560 l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2; 3561 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3562 wqes[0] = (struct kwqe *) l4kwqe; 3563 3564 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE; 3565 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT; 3566 l4kwqe->cid = csk->cid; 3567 3568 return dev->submit_kwqes(dev, wqes, 1); 3569 } 3570 3571 static int cnic_cm_abort_req(struct cnic_sock *csk) 3572 { 3573 struct cnic_dev *dev = csk->dev; 3574 struct l4_kwq_reset_req *l4kwqe; 3575 struct kwqe *wqes[1]; 3576 3577 l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2; 3578 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3579 wqes[0] = (struct kwqe *) l4kwqe; 3580 3581 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET; 3582 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT; 3583 l4kwqe->cid = csk->cid; 3584 3585 return dev->submit_kwqes(dev, wqes, 1); 3586 } 3587 3588 static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid, 3589 u32 l5_cid, struct cnic_sock **csk, void *context) 3590 { 3591 struct cnic_local *cp = dev->cnic_priv; 3592 struct cnic_sock *csk1; 3593 3594 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3595 return -EINVAL; 3596 3597 if (cp->ctx_tbl) { 3598 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3599 3600 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 3601 return -EAGAIN; 3602 } 3603 3604 csk1 = &cp->csk_tbl[l5_cid]; 3605 if (atomic_read(&csk1->ref_count)) 3606 return -EAGAIN; 3607 3608 if (test_and_set_bit(SK_F_INUSE, &csk1->flags)) 3609 return -EBUSY; 3610 3611 csk1->dev = dev; 3612 csk1->cid = cid; 3613 csk1->l5_cid = l5_cid; 3614 csk1->ulp_type = ulp_type; 3615 csk1->context = context; 3616 3617 csk1->ka_timeout = DEF_KA_TIMEOUT; 3618 csk1->ka_interval = DEF_KA_INTERVAL; 3619 csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT; 3620 csk1->tos = DEF_TOS; 3621 csk1->ttl = DEF_TTL; 3622 csk1->snd_seq_scale = DEF_SND_SEQ_SCALE; 3623 csk1->rcv_buf = DEF_RCV_BUF; 3624 csk1->snd_buf = DEF_SND_BUF; 3625 csk1->seed = DEF_SEED; 3626 csk1->tcp_flags = 0; 3627 3628 *csk = csk1; 3629 return 0; 3630 } 3631 3632 static void cnic_cm_cleanup(struct cnic_sock *csk) 3633 { 3634 if (csk->src_port) { 3635 struct cnic_dev *dev = csk->dev; 3636 struct cnic_local *cp = dev->cnic_priv; 3637 3638 cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port)); 3639 csk->src_port = 0; 3640 } 3641 } 3642 3643 static void cnic_close_conn(struct cnic_sock *csk) 3644 { 3645 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) { 3646 cnic_cm_upload_pg(csk); 3647 clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3648 } 3649 cnic_cm_cleanup(csk); 3650 } 3651 3652 static int cnic_cm_destroy(struct cnic_sock *csk) 3653 { 3654 if (!cnic_in_use(csk)) 3655 return -EINVAL; 3656 3657 csk_hold(csk); 3658 clear_bit(SK_F_INUSE, &csk->flags); 3659 smp_mb__after_atomic(); 3660 while (atomic_read(&csk->ref_count) != 1) 3661 msleep(1); 3662 cnic_cm_cleanup(csk); 3663 3664 csk->flags = 0; 3665 csk_put(csk); 3666 return 0; 3667 } 3668 3669 static inline u16 cnic_get_vlan(struct net_device *dev, 3670 struct net_device **vlan_dev) 3671 { 3672 if (is_vlan_dev(dev)) { 3673 *vlan_dev = vlan_dev_real_dev(dev); 3674 return vlan_dev_vlan_id(dev); 3675 } 3676 *vlan_dev = dev; 3677 return 0; 3678 } 3679 3680 static int cnic_get_v4_route(struct sockaddr_in *dst_addr, 3681 struct dst_entry **dst) 3682 { 3683 #if defined(CONFIG_INET) 3684 struct rtable *rt; 3685 3686 rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0, 3687 RT_SCOPE_UNIVERSE); 3688 if (!IS_ERR(rt)) { 3689 *dst = &rt->dst; 3690 return 0; 3691 } 3692 return PTR_ERR(rt); 3693 #else 3694 return -ENETUNREACH; 3695 #endif 3696 } 3697 3698 static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr, 3699 struct dst_entry **dst) 3700 { 3701 #if IS_ENABLED(CONFIG_IPV6) 3702 struct flowi6 fl6; 3703 3704 memset(&fl6, 0, sizeof(fl6)); 3705 fl6.daddr = dst_addr->sin6_addr; 3706 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 3707 fl6.flowi6_oif = dst_addr->sin6_scope_id; 3708 3709 *dst = ip6_route_output(&init_net, NULL, &fl6); 3710 if ((*dst)->error) { 3711 dst_release(*dst); 3712 *dst = NULL; 3713 return -ENETUNREACH; 3714 } else 3715 return 0; 3716 #endif 3717 3718 return -ENETUNREACH; 3719 } 3720 3721 static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr, 3722 int ulp_type) 3723 { 3724 struct cnic_dev *dev = NULL; 3725 struct dst_entry *dst; 3726 struct net_device *netdev = NULL; 3727 int err = -ENETUNREACH; 3728 3729 if (dst_addr->sin_family == AF_INET) 3730 err = cnic_get_v4_route(dst_addr, &dst); 3731 else if (dst_addr->sin_family == AF_INET6) { 3732 struct sockaddr_in6 *dst_addr6 = 3733 (struct sockaddr_in6 *) dst_addr; 3734 3735 err = cnic_get_v6_route(dst_addr6, &dst); 3736 } else 3737 return NULL; 3738 3739 if (err) 3740 return NULL; 3741 3742 if (!dst->dev) 3743 goto done; 3744 3745 cnic_get_vlan(dst->dev, &netdev); 3746 3747 dev = cnic_from_netdev(netdev); 3748 3749 done: 3750 dst_release(dst); 3751 if (dev) 3752 cnic_put(dev); 3753 return dev; 3754 } 3755 3756 static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3757 { 3758 struct cnic_dev *dev = csk->dev; 3759 struct cnic_local *cp = dev->cnic_priv; 3760 3761 return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk); 3762 } 3763 3764 static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3765 { 3766 struct cnic_dev *dev = csk->dev; 3767 struct cnic_local *cp = dev->cnic_priv; 3768 int is_v6, rc = 0; 3769 struct dst_entry *dst = NULL; 3770 struct net_device *realdev; 3771 __be16 local_port; 3772 u32 port_id; 3773 3774 if (saddr->local.v6.sin6_family == AF_INET6 && 3775 saddr->remote.v6.sin6_family == AF_INET6) 3776 is_v6 = 1; 3777 else if (saddr->local.v4.sin_family == AF_INET && 3778 saddr->remote.v4.sin_family == AF_INET) 3779 is_v6 = 0; 3780 else 3781 return -EINVAL; 3782 3783 clear_bit(SK_F_IPV6, &csk->flags); 3784 3785 if (is_v6) { 3786 set_bit(SK_F_IPV6, &csk->flags); 3787 cnic_get_v6_route(&saddr->remote.v6, &dst); 3788 3789 memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr, 3790 sizeof(struct in6_addr)); 3791 csk->dst_port = saddr->remote.v6.sin6_port; 3792 local_port = saddr->local.v6.sin6_port; 3793 3794 } else { 3795 cnic_get_v4_route(&saddr->remote.v4, &dst); 3796 3797 csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr; 3798 csk->dst_port = saddr->remote.v4.sin_port; 3799 local_port = saddr->local.v4.sin_port; 3800 } 3801 3802 csk->vlan_id = 0; 3803 csk->mtu = dev->netdev->mtu; 3804 if (dst && dst->dev) { 3805 u16 vlan = cnic_get_vlan(dst->dev, &realdev); 3806 if (realdev == dev->netdev) { 3807 csk->vlan_id = vlan; 3808 csk->mtu = dst_mtu(dst); 3809 } 3810 } 3811 3812 port_id = be16_to_cpu(local_port); 3813 if (port_id >= CNIC_LOCAL_PORT_MIN && 3814 port_id < CNIC_LOCAL_PORT_MAX) { 3815 if (cnic_alloc_id(&cp->csk_port_tbl, port_id)) 3816 port_id = 0; 3817 } else 3818 port_id = 0; 3819 3820 if (!port_id) { 3821 port_id = cnic_alloc_new_id(&cp->csk_port_tbl); 3822 if (port_id == -1) { 3823 rc = -ENOMEM; 3824 goto err_out; 3825 } 3826 local_port = cpu_to_be16(port_id); 3827 } 3828 csk->src_port = local_port; 3829 3830 err_out: 3831 dst_release(dst); 3832 return rc; 3833 } 3834 3835 static void cnic_init_csk_state(struct cnic_sock *csk) 3836 { 3837 csk->state = 0; 3838 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3839 clear_bit(SK_F_CLOSING, &csk->flags); 3840 } 3841 3842 static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3843 { 3844 struct cnic_local *cp = csk->dev->cnic_priv; 3845 int err = 0; 3846 3847 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 3848 return -EOPNOTSUPP; 3849 3850 if (!cnic_in_use(csk)) 3851 return -EINVAL; 3852 3853 if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags)) 3854 return -EINVAL; 3855 3856 cnic_init_csk_state(csk); 3857 3858 err = cnic_get_route(csk, saddr); 3859 if (err) 3860 goto err_out; 3861 3862 err = cnic_resolve_addr(csk, saddr); 3863 if (!err) 3864 return 0; 3865 3866 err_out: 3867 clear_bit(SK_F_CONNECT_START, &csk->flags); 3868 return err; 3869 } 3870 3871 static int cnic_cm_abort(struct cnic_sock *csk) 3872 { 3873 struct cnic_local *cp = csk->dev->cnic_priv; 3874 u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP; 3875 3876 if (!cnic_in_use(csk)) 3877 return -EINVAL; 3878 3879 if (cnic_abort_prep(csk)) 3880 return cnic_cm_abort_req(csk); 3881 3882 /* Getting here means that we haven't started connect, or 3883 * connect was not successful, or it has been reset by the target. 3884 */ 3885 3886 cp->close_conn(csk, opcode); 3887 if (csk->state != opcode) { 3888 /* Wait for remote reset sequence to complete */ 3889 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3890 msleep(1); 3891 3892 return -EALREADY; 3893 } 3894 3895 return 0; 3896 } 3897 3898 static int cnic_cm_close(struct cnic_sock *csk) 3899 { 3900 if (!cnic_in_use(csk)) 3901 return -EINVAL; 3902 3903 if (cnic_close_prep(csk)) { 3904 csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 3905 return cnic_cm_close_req(csk); 3906 } else { 3907 /* Wait for remote reset sequence to complete */ 3908 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3909 msleep(1); 3910 3911 return -EALREADY; 3912 } 3913 return 0; 3914 } 3915 3916 static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk, 3917 u8 opcode) 3918 { 3919 struct cnic_ulp_ops *ulp_ops; 3920 int ulp_type = csk->ulp_type; 3921 3922 rcu_read_lock(); 3923 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 3924 if (ulp_ops) { 3925 if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE) 3926 ulp_ops->cm_connect_complete(csk); 3927 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 3928 ulp_ops->cm_close_complete(csk); 3929 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) 3930 ulp_ops->cm_remote_abort(csk); 3931 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP) 3932 ulp_ops->cm_abort_complete(csk); 3933 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED) 3934 ulp_ops->cm_remote_close(csk); 3935 } 3936 rcu_read_unlock(); 3937 } 3938 3939 static int cnic_cm_set_pg(struct cnic_sock *csk) 3940 { 3941 if (cnic_offld_prep(csk)) { 3942 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3943 cnic_cm_update_pg(csk); 3944 else 3945 cnic_cm_offload_pg(csk); 3946 } 3947 return 0; 3948 } 3949 3950 static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe) 3951 { 3952 struct cnic_local *cp = dev->cnic_priv; 3953 u32 l5_cid = kcqe->pg_host_opaque; 3954 u8 opcode = kcqe->op_code; 3955 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 3956 3957 csk_hold(csk); 3958 if (!cnic_in_use(csk)) 3959 goto done; 3960 3961 if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3962 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3963 goto done; 3964 } 3965 /* Possible PG kcqe status: SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */ 3966 if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) { 3967 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3968 cnic_cm_upcall(cp, csk, 3969 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 3970 goto done; 3971 } 3972 3973 csk->pg_cid = kcqe->pg_cid; 3974 set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3975 cnic_cm_conn_req(csk); 3976 3977 done: 3978 csk_put(csk); 3979 } 3980 3981 static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe) 3982 { 3983 struct cnic_local *cp = dev->cnic_priv; 3984 struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe; 3985 u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE; 3986 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3987 3988 ctx->timestamp = jiffies; 3989 ctx->wait_cond = 1; 3990 wake_up(&ctx->waitq); 3991 } 3992 3993 static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe) 3994 { 3995 struct cnic_local *cp = dev->cnic_priv; 3996 struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe; 3997 u8 opcode = l4kcqe->op_code; 3998 u32 l5_cid; 3999 struct cnic_sock *csk; 4000 4001 if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) { 4002 cnic_process_fcoe_term_conn(dev, kcqe); 4003 return; 4004 } 4005 if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG || 4006 opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 4007 cnic_cm_process_offld_pg(dev, l4kcqe); 4008 return; 4009 } 4010 4011 l5_cid = l4kcqe->conn_id; 4012 if (opcode & 0x80) 4013 l5_cid = l4kcqe->cid; 4014 if (l5_cid >= MAX_CM_SK_TBL_SZ) 4015 return; 4016 4017 csk = &cp->csk_tbl[l5_cid]; 4018 csk_hold(csk); 4019 4020 if (!cnic_in_use(csk)) { 4021 csk_put(csk); 4022 return; 4023 } 4024 4025 switch (opcode) { 4026 case L5CM_RAMROD_CMD_ID_TCP_CONNECT: 4027 if (l4kcqe->status != 0) { 4028 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4029 cnic_cm_upcall(cp, csk, 4030 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 4031 } 4032 break; 4033 case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE: 4034 if (l4kcqe->status == 0) 4035 set_bit(SK_F_OFFLD_COMPLETE, &csk->flags); 4036 else if (l4kcqe->status == 4037 L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4038 set_bit(SK_F_HW_ERR, &csk->flags); 4039 4040 smp_mb__before_atomic(); 4041 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4042 cnic_cm_upcall(cp, csk, opcode); 4043 break; 4044 4045 case L5CM_RAMROD_CMD_ID_CLOSE: { 4046 struct iscsi_kcqe *l5kcqe = (struct iscsi_kcqe *) kcqe; 4047 4048 if (l4kcqe->status == 0 && l5kcqe->completion_status == 0) 4049 break; 4050 4051 netdev_warn(dev->netdev, "RAMROD CLOSE compl with status 0x%x completion status 0x%x\n", 4052 l4kcqe->status, l5kcqe->completion_status); 4053 opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 4054 } 4055 fallthrough; 4056 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4057 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4058 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4059 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4060 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4061 if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4062 set_bit(SK_F_HW_ERR, &csk->flags); 4063 4064 cp->close_conn(csk, opcode); 4065 break; 4066 4067 case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED: 4068 /* after we already sent CLOSE_REQ */ 4069 if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) && 4070 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) && 4071 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 4072 cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP); 4073 else 4074 cnic_cm_upcall(cp, csk, opcode); 4075 break; 4076 } 4077 csk_put(csk); 4078 } 4079 4080 static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num) 4081 { 4082 struct cnic_dev *dev = data; 4083 int i; 4084 4085 for (i = 0; i < num; i++) 4086 cnic_cm_process_kcqe(dev, kcqe[i]); 4087 } 4088 4089 static struct cnic_ulp_ops cm_ulp_ops = { 4090 .indicate_kcqes = cnic_cm_indicate_kcqe, 4091 }; 4092 4093 static void cnic_cm_free_mem(struct cnic_dev *dev) 4094 { 4095 struct cnic_local *cp = dev->cnic_priv; 4096 4097 kvfree(cp->csk_tbl); 4098 cp->csk_tbl = NULL; 4099 cnic_free_id_tbl(&cp->csk_port_tbl); 4100 } 4101 4102 static int cnic_cm_alloc_mem(struct cnic_dev *dev) 4103 { 4104 struct cnic_local *cp = dev->cnic_priv; 4105 u32 port_id; 4106 int i; 4107 4108 cp->csk_tbl = kvcalloc(MAX_CM_SK_TBL_SZ, sizeof(struct cnic_sock), 4109 GFP_KERNEL); 4110 if (!cp->csk_tbl) 4111 return -ENOMEM; 4112 4113 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) 4114 atomic_set(&cp->csk_tbl[i].ref_count, 0); 4115 4116 port_id = get_random_u32_below(CNIC_LOCAL_PORT_RANGE); 4117 if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, 4118 CNIC_LOCAL_PORT_MIN, port_id)) { 4119 cnic_cm_free_mem(dev); 4120 return -ENOMEM; 4121 } 4122 return 0; 4123 } 4124 4125 static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode) 4126 { 4127 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 4128 /* Unsolicited RESET_COMP or RESET_RECEIVED */ 4129 opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED; 4130 csk->state = opcode; 4131 } 4132 4133 /* 1. If event opcode matches the expected event in csk->state 4134 * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any 4135 * event 4136 * 3. If the expected event is 0, meaning the connection was never 4137 * never established, we accept the opcode from cm_abort. 4138 */ 4139 if (opcode == csk->state || csk->state == 0 || 4140 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP || 4141 csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) { 4142 if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) { 4143 if (csk->state == 0) 4144 csk->state = opcode; 4145 return 1; 4146 } 4147 } 4148 return 0; 4149 } 4150 4151 static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode) 4152 { 4153 struct cnic_dev *dev = csk->dev; 4154 struct cnic_local *cp = dev->cnic_priv; 4155 4156 if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) { 4157 cnic_cm_upcall(cp, csk, opcode); 4158 return; 4159 } 4160 4161 clear_bit(SK_F_CONNECT_START, &csk->flags); 4162 cnic_close_conn(csk); 4163 csk->state = opcode; 4164 cnic_cm_upcall(cp, csk, opcode); 4165 } 4166 4167 static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev) 4168 { 4169 } 4170 4171 static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) 4172 { 4173 u32 seed; 4174 4175 seed = get_random_u32(); 4176 cnic_ctx_wr(dev, 45, 0, seed); 4177 return 0; 4178 } 4179 4180 static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode) 4181 { 4182 struct cnic_dev *dev = csk->dev; 4183 struct cnic_local *cp = dev->cnic_priv; 4184 struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid]; 4185 union l5cm_specific_data l5_data; 4186 u32 cmd = 0; 4187 int close_complete = 0; 4188 4189 switch (opcode) { 4190 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4191 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4192 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4193 if (cnic_ready_to_close(csk, opcode)) { 4194 if (test_bit(SK_F_HW_ERR, &csk->flags)) 4195 close_complete = 1; 4196 else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 4197 cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE; 4198 else 4199 close_complete = 1; 4200 } 4201 break; 4202 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4203 cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD; 4204 break; 4205 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4206 close_complete = 1; 4207 break; 4208 } 4209 if (cmd) { 4210 memset(&l5_data, 0, sizeof(l5_data)); 4211 4212 cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE, 4213 &l5_data); 4214 } else if (close_complete) { 4215 ctx->timestamp = jiffies; 4216 cnic_close_conn(csk); 4217 cnic_cm_upcall(cp, csk, csk->state); 4218 } 4219 } 4220 4221 static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) 4222 { 4223 struct cnic_local *cp = dev->cnic_priv; 4224 4225 if (!cp->ctx_tbl) 4226 return; 4227 4228 if (!netif_running(dev->netdev)) 4229 return; 4230 4231 cnic_bnx2x_delete_wait(dev, 0); 4232 4233 cancel_delayed_work(&cp->delete_task); 4234 flush_workqueue(cnic_wq); 4235 4236 if (atomic_read(&cp->iscsi_conn) != 0) 4237 netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", 4238 atomic_read(&cp->iscsi_conn)); 4239 } 4240 4241 static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev) 4242 { 4243 struct bnx2x *bp = netdev_priv(dev->netdev); 4244 u32 pfid = bp->pfid; 4245 u32 port = BP_PORT(bp); 4246 4247 cnic_init_bnx2x_mac(dev); 4248 cnic_bnx2x_set_tcp_options(dev, 0, 1); 4249 4250 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 4251 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0); 4252 4253 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4254 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1); 4255 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4256 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port), 4257 DEF_MAX_DA_COUNT); 4258 4259 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4260 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL); 4261 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4262 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS); 4263 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4264 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2); 4265 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4266 XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER); 4267 4268 CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid), 4269 DEF_MAX_CWND); 4270 return 0; 4271 } 4272 4273 static void cnic_delete_task(struct work_struct *work) 4274 { 4275 struct cnic_local *cp; 4276 struct cnic_dev *dev; 4277 u32 i; 4278 int need_resched = 0; 4279 4280 cp = container_of(work, struct cnic_local, delete_task.work); 4281 dev = cp->dev; 4282 4283 if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) { 4284 struct drv_ctl_info info; 4285 4286 cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI); 4287 4288 memset(&info, 0, sizeof(struct drv_ctl_info)); 4289 info.cmd = DRV_CTL_ISCSI_STOPPED_CMD; 4290 cp->ethdev->drv_ctl(dev->netdev, &info); 4291 } 4292 4293 for (i = 0; i < cp->max_cid_space; i++) { 4294 struct cnic_context *ctx = &cp->ctx_tbl[i]; 4295 int err; 4296 4297 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) || 4298 !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4299 continue; 4300 4301 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 4302 need_resched = 1; 4303 continue; 4304 } 4305 4306 if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4307 continue; 4308 4309 err = cnic_bnx2x_destroy_ramrod(dev, i); 4310 4311 cnic_free_bnx2x_conn_resc(dev, i); 4312 if (!err) { 4313 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) 4314 atomic_dec(&cp->iscsi_conn); 4315 4316 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 4317 } 4318 } 4319 4320 if (need_resched) 4321 queue_delayed_work(cnic_wq, &cp->delete_task, 4322 msecs_to_jiffies(10)); 4323 4324 } 4325 4326 static int cnic_cm_open(struct cnic_dev *dev) 4327 { 4328 struct cnic_local *cp = dev->cnic_priv; 4329 int err; 4330 4331 err = cnic_cm_alloc_mem(dev); 4332 if (err) 4333 return err; 4334 4335 err = cp->start_cm(dev); 4336 4337 if (err) 4338 goto err_out; 4339 4340 INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task); 4341 4342 dev->cm_create = cnic_cm_create; 4343 dev->cm_destroy = cnic_cm_destroy; 4344 dev->cm_connect = cnic_cm_connect; 4345 dev->cm_abort = cnic_cm_abort; 4346 dev->cm_close = cnic_cm_close; 4347 dev->cm_select_dev = cnic_cm_select_dev; 4348 4349 cp->ulp_handle[CNIC_ULP_L4] = dev; 4350 rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops); 4351 return 0; 4352 4353 err_out: 4354 cnic_cm_free_mem(dev); 4355 return err; 4356 } 4357 4358 static int cnic_cm_shutdown(struct cnic_dev *dev) 4359 { 4360 struct cnic_local *cp = dev->cnic_priv; 4361 int i; 4362 4363 if (!cp->csk_tbl) 4364 return 0; 4365 4366 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) { 4367 struct cnic_sock *csk = &cp->csk_tbl[i]; 4368 4369 clear_bit(SK_F_INUSE, &csk->flags); 4370 cnic_cm_cleanup(csk); 4371 } 4372 cnic_cm_free_mem(dev); 4373 4374 return 0; 4375 } 4376 4377 static void cnic_init_context(struct cnic_dev *dev, u32 cid) 4378 { 4379 u32 cid_addr; 4380 int i; 4381 4382 cid_addr = GET_CID_ADDR(cid); 4383 4384 for (i = 0; i < CTX_SIZE; i += 4) 4385 cnic_ctx_wr(dev, cid_addr, i, 0); 4386 } 4387 4388 static int cnic_setup_5709_context(struct cnic_dev *dev, int valid) 4389 { 4390 struct cnic_local *cp = dev->cnic_priv; 4391 int ret = 0, i; 4392 u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0; 4393 4394 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4395 return 0; 4396 4397 for (i = 0; i < cp->ctx_blks; i++) { 4398 int j; 4399 u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk; 4400 u32 val; 4401 4402 memset(cp->ctx_arr[i].ctx, 0, CNIC_PAGE_SIZE); 4403 4404 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0, 4405 (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit); 4406 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1, 4407 (u64) cp->ctx_arr[i].mapping >> 32); 4408 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx | 4409 BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ); 4410 for (j = 0; j < 10; j++) { 4411 4412 val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL); 4413 if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ)) 4414 break; 4415 udelay(5); 4416 } 4417 if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) { 4418 ret = -EBUSY; 4419 break; 4420 } 4421 } 4422 return ret; 4423 } 4424 4425 static void cnic_free_irq(struct cnic_dev *dev) 4426 { 4427 struct cnic_local *cp = dev->cnic_priv; 4428 struct cnic_eth_dev *ethdev = cp->ethdev; 4429 4430 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4431 cp->disable_int_sync(dev); 4432 cancel_work_sync(&cp->cnic_irq_bh_work); 4433 free_irq(ethdev->irq_arr[0].vector, dev); 4434 } 4435 } 4436 4437 static int cnic_request_irq(struct cnic_dev *dev) 4438 { 4439 struct cnic_local *cp = dev->cnic_priv; 4440 struct cnic_eth_dev *ethdev = cp->ethdev; 4441 int err; 4442 4443 err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev); 4444 if (err) 4445 disable_work_sync(&cp->cnic_irq_bh_work); 4446 4447 return err; 4448 } 4449 4450 static int cnic_init_bnx2_irq(struct cnic_dev *dev) 4451 { 4452 struct cnic_local *cp = dev->cnic_priv; 4453 struct cnic_eth_dev *ethdev = cp->ethdev; 4454 4455 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4456 int err, i = 0; 4457 int sblk_num = cp->status_blk_num; 4458 u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) + 4459 BNX2_HC_SB_CONFIG_1; 4460 4461 CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT); 4462 4463 CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8); 4464 CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220); 4465 CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220); 4466 4467 cp->last_status_idx = cp->status_blk.bnx2->status_idx; 4468 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2_msix); 4469 err = cnic_request_irq(dev); 4470 if (err) 4471 return err; 4472 4473 while (cp->status_blk.bnx2->status_completion_producer_index && 4474 i < 10) { 4475 CNIC_WR(dev, BNX2_HC_COALESCE_NOW, 4476 1 << (11 + sblk_num)); 4477 udelay(10); 4478 i++; 4479 barrier(); 4480 } 4481 if (cp->status_blk.bnx2->status_completion_producer_index) { 4482 cnic_free_irq(dev); 4483 goto failed; 4484 } 4485 4486 } else { 4487 struct status_block *sblk = cp->status_blk.gen; 4488 u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND); 4489 int i = 0; 4490 4491 while (sblk->status_completion_producer_index && i < 10) { 4492 CNIC_WR(dev, BNX2_HC_COMMAND, 4493 hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT); 4494 udelay(10); 4495 i++; 4496 barrier(); 4497 } 4498 if (sblk->status_completion_producer_index) 4499 goto failed; 4500 4501 } 4502 return 0; 4503 4504 failed: 4505 netdev_err(dev->netdev, "KCQ index not resetting to 0\n"); 4506 return -EBUSY; 4507 } 4508 4509 static void cnic_enable_bnx2_int(struct cnic_dev *dev) 4510 { 4511 struct cnic_local *cp = dev->cnic_priv; 4512 struct cnic_eth_dev *ethdev = cp->ethdev; 4513 4514 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4515 return; 4516 4517 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4518 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 4519 } 4520 4521 static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev) 4522 { 4523 struct cnic_local *cp = dev->cnic_priv; 4524 struct cnic_eth_dev *ethdev = cp->ethdev; 4525 4526 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4527 return; 4528 4529 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4530 BNX2_PCICFG_INT_ACK_CMD_MASK_INT); 4531 CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD); 4532 synchronize_irq(ethdev->irq_arr[0].vector); 4533 } 4534 4535 static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev) 4536 { 4537 struct cnic_local *cp = dev->cnic_priv; 4538 struct cnic_eth_dev *ethdev = cp->ethdev; 4539 struct cnic_uio_dev *udev = cp->udev; 4540 u32 cid_addr, tx_cid, sb_id; 4541 u32 val, offset0, offset1, offset2, offset3; 4542 int i; 4543 struct bnx2_tx_bd *txbd; 4544 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4545 struct status_block *s_blk = cp->status_blk.gen; 4546 4547 sb_id = cp->status_blk_num; 4548 tx_cid = 20; 4549 cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2; 4550 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4551 struct status_block_msix *sblk = cp->status_blk.bnx2; 4552 4553 tx_cid = TX_TSS_CID + sb_id - 1; 4554 CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) | 4555 (TX_TSS_CID << 7)); 4556 cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index; 4557 } 4558 cp->tx_cons = *cp->tx_cons_ptr; 4559 4560 cid_addr = GET_CID_ADDR(tx_cid); 4561 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 4562 u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40; 4563 4564 for (i = 0; i < PHY_CTX_SIZE; i += 4) 4565 cnic_ctx_wr(dev, cid_addr2, i, 0); 4566 4567 offset0 = BNX2_L2CTX_TYPE_XI; 4568 offset1 = BNX2_L2CTX_CMD_TYPE_XI; 4569 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI; 4570 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI; 4571 } else { 4572 cnic_init_context(dev, tx_cid); 4573 cnic_init_context(dev, tx_cid + 1); 4574 4575 offset0 = BNX2_L2CTX_TYPE; 4576 offset1 = BNX2_L2CTX_CMD_TYPE; 4577 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI; 4578 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO; 4579 } 4580 val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2; 4581 cnic_ctx_wr(dev, cid_addr, offset0, val); 4582 4583 val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16); 4584 cnic_ctx_wr(dev, cid_addr, offset1, val); 4585 4586 txbd = udev->l2_ring; 4587 4588 buf_map = udev->l2_buf_map; 4589 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) { 4590 txbd->tx_bd_haddr_hi = (u64) buf_map >> 32; 4591 txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4592 } 4593 val = (u64) ring_map >> 32; 4594 cnic_ctx_wr(dev, cid_addr, offset2, val); 4595 txbd->tx_bd_haddr_hi = val; 4596 4597 val = (u64) ring_map & 0xffffffff; 4598 cnic_ctx_wr(dev, cid_addr, offset3, val); 4599 txbd->tx_bd_haddr_lo = val; 4600 } 4601 4602 static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev) 4603 { 4604 struct cnic_local *cp = dev->cnic_priv; 4605 struct cnic_eth_dev *ethdev = cp->ethdev; 4606 struct cnic_uio_dev *udev = cp->udev; 4607 u32 cid_addr, sb_id, val, coal_reg, coal_val; 4608 int i; 4609 struct bnx2_rx_bd *rxbd; 4610 struct status_block *s_blk = cp->status_blk.gen; 4611 dma_addr_t ring_map = udev->l2_ring_map; 4612 4613 sb_id = cp->status_blk_num; 4614 cnic_init_context(dev, 2); 4615 cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2; 4616 coal_reg = BNX2_HC_COMMAND; 4617 coal_val = CNIC_RD(dev, coal_reg); 4618 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4619 struct status_block_msix *sblk = cp->status_blk.bnx2; 4620 4621 cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index; 4622 coal_reg = BNX2_HC_COALESCE_NOW; 4623 coal_val = 1 << (11 + sb_id); 4624 } 4625 i = 0; 4626 while (!(*cp->rx_cons_ptr != 0) && i < 10) { 4627 CNIC_WR(dev, coal_reg, coal_val); 4628 udelay(10); 4629 i++; 4630 barrier(); 4631 } 4632 cp->rx_cons = *cp->rx_cons_ptr; 4633 4634 cid_addr = GET_CID_ADDR(2); 4635 val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE | 4636 BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8); 4637 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val); 4638 4639 if (sb_id == 0) 4640 val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT; 4641 else 4642 val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id); 4643 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val); 4644 4645 rxbd = udev->l2_ring + CNIC_PAGE_SIZE; 4646 for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) { 4647 dma_addr_t buf_map; 4648 int n = (i % cp->l2_rx_ring_size) + 1; 4649 4650 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4651 rxbd->rx_bd_len = cp->l2_single_buf_size; 4652 rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END; 4653 rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32; 4654 rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4655 } 4656 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 4657 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val); 4658 rxbd->rx_bd_haddr_hi = val; 4659 4660 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 4661 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val); 4662 rxbd->rx_bd_haddr_lo = val; 4663 4664 val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD); 4665 cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2)); 4666 } 4667 4668 static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev) 4669 { 4670 struct kwqe *wqes[1], l2kwqe; 4671 4672 memset(&l2kwqe, 0, sizeof(l2kwqe)); 4673 wqes[0] = &l2kwqe; 4674 l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) | 4675 (L2_KWQE_OPCODE_VALUE_FLUSH << 4676 KWQE_OPCODE_SHIFT) | 2; 4677 dev->submit_kwqes(dev, wqes, 1); 4678 } 4679 4680 static void cnic_set_bnx2_mac(struct cnic_dev *dev) 4681 { 4682 struct cnic_local *cp = dev->cnic_priv; 4683 u32 val; 4684 4685 val = cp->func << 2; 4686 4687 cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val); 4688 4689 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4690 BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER); 4691 dev->mac_addr[0] = (u8) (val >> 8); 4692 dev->mac_addr[1] = (u8) val; 4693 4694 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val); 4695 4696 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4697 BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER); 4698 dev->mac_addr[2] = (u8) (val >> 24); 4699 dev->mac_addr[3] = (u8) (val >> 16); 4700 dev->mac_addr[4] = (u8) (val >> 8); 4701 dev->mac_addr[5] = (u8) val; 4702 4703 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val); 4704 4705 val = 4 | BNX2_RPM_SORT_USER2_BC_EN; 4706 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4707 val |= BNX2_RPM_SORT_USER2_PROM_VLAN; 4708 4709 CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0); 4710 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val); 4711 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA); 4712 } 4713 4714 static int cnic_start_bnx2_hw(struct cnic_dev *dev) 4715 { 4716 struct cnic_local *cp = dev->cnic_priv; 4717 struct cnic_eth_dev *ethdev = cp->ethdev; 4718 struct status_block *sblk = cp->status_blk.gen; 4719 u32 val, kcq_cid_addr, kwq_cid_addr; 4720 int err; 4721 4722 cnic_set_bnx2_mac(dev); 4723 4724 val = CNIC_RD(dev, BNX2_MQ_CONFIG); 4725 val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE; 4726 if (CNIC_PAGE_BITS > 12) 4727 val |= (12 - 8) << 4; 4728 else 4729 val |= (CNIC_PAGE_BITS - 8) << 4; 4730 4731 CNIC_WR(dev, BNX2_MQ_CONFIG, val); 4732 4733 CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8); 4734 CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220); 4735 CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220); 4736 4737 err = cnic_setup_5709_context(dev, 1); 4738 if (err) 4739 return err; 4740 4741 cnic_init_context(dev, KWQ_CID); 4742 cnic_init_context(dev, KCQ_CID); 4743 4744 kwq_cid_addr = GET_CID_ADDR(KWQ_CID); 4745 cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX; 4746 4747 cp->max_kwq_idx = MAX_KWQ_IDX; 4748 cp->kwq_prod_idx = 0; 4749 cp->kwq_con_idx = 0; 4750 set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 4751 4752 if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708) 4753 cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15; 4754 else 4755 cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index; 4756 4757 /* Initialize the kernel work queue context. */ 4758 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4759 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4760 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val); 4761 4762 val = (CNIC_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16; 4763 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4764 4765 val = ((CNIC_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT; 4766 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4767 4768 val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32); 4769 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4770 4771 val = (u32) cp->kwq_info.pgtbl_map; 4772 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4773 4774 kcq_cid_addr = GET_CID_ADDR(KCQ_CID); 4775 cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX; 4776 4777 cp->kcq1.sw_prod_idx = 0; 4778 cp->kcq1.hw_prod_idx_ptr = 4779 &sblk->status_completion_producer_index; 4780 4781 cp->kcq1.status_idx_ptr = &sblk->status_idx; 4782 4783 /* Initialize the kernel complete queue context. */ 4784 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4785 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4786 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val); 4787 4788 val = (CNIC_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16; 4789 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4790 4791 val = ((CNIC_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT; 4792 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4793 4794 val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32); 4795 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4796 4797 val = (u32) cp->kcq1.dma.pgtbl_map; 4798 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4799 4800 cp->int_num = 0; 4801 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4802 struct status_block_msix *msblk = cp->status_blk.bnx2; 4803 u32 sb_id = cp->status_blk_num; 4804 u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id); 4805 4806 cp->kcq1.hw_prod_idx_ptr = 4807 &msblk->status_completion_producer_index; 4808 cp->kcq1.status_idx_ptr = &msblk->status_idx; 4809 cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index; 4810 cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT; 4811 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4812 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4813 } 4814 4815 /* Enable Commnad Scheduler notification when we write to the 4816 * host producer index of the kernel contexts. */ 4817 CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2); 4818 4819 /* Enable Command Scheduler notification when we write to either 4820 * the Send Queue or Receive Queue producer indexes of the kernel 4821 * bypass contexts. */ 4822 CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7); 4823 CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7); 4824 4825 /* Notify COM when the driver post an application buffer. */ 4826 CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000); 4827 4828 /* Set the CP and COM doorbells. These two processors polls the 4829 * doorbell for a non zero value before running. This must be done 4830 * after setting up the kernel queue contexts. */ 4831 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1); 4832 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1); 4833 4834 cnic_init_bnx2_tx_ring(dev); 4835 cnic_init_bnx2_rx_ring(dev); 4836 4837 err = cnic_init_bnx2_irq(dev); 4838 if (err) { 4839 netdev_err(dev->netdev, "cnic_init_irq failed\n"); 4840 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 4841 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 4842 return err; 4843 } 4844 4845 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 4846 4847 return 0; 4848 } 4849 4850 static void cnic_setup_bnx2x_context(struct cnic_dev *dev) 4851 { 4852 struct cnic_local *cp = dev->cnic_priv; 4853 struct cnic_eth_dev *ethdev = cp->ethdev; 4854 u32 start_offset = ethdev->ctx_tbl_offset; 4855 int i; 4856 4857 for (i = 0; i < cp->ctx_blks; i++) { 4858 struct cnic_ctx *ctx = &cp->ctx_arr[i]; 4859 dma_addr_t map = ctx->mapping; 4860 4861 if (cp->ctx_align) { 4862 unsigned long mask = cp->ctx_align - 1; 4863 4864 map = (map + mask) & ~mask; 4865 } 4866 4867 cnic_ctx_tbl_wr(dev, start_offset + i, map); 4868 } 4869 } 4870 4871 static int cnic_init_bnx2x_irq(struct cnic_dev *dev) 4872 { 4873 struct cnic_local *cp = dev->cnic_priv; 4874 struct cnic_eth_dev *ethdev = cp->ethdev; 4875 int err = 0; 4876 4877 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2x_bh_work); 4878 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 4879 err = cnic_request_irq(dev); 4880 4881 return err; 4882 } 4883 4884 static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev, 4885 u16 sb_id, u8 sb_index, 4886 u8 disable) 4887 { 4888 struct bnx2x *bp = netdev_priv(dev->netdev); 4889 4890 u32 addr = BAR_CSTRORM_INTMEM + 4891 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4892 offsetof(struct hc_status_block_data_e1x, index_data) + 4893 sizeof(struct hc_index_data)*sb_index + 4894 offsetof(struct hc_index_data, flags); 4895 u16 flags = CNIC_RD16(dev, addr); 4896 /* clear and set */ 4897 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4898 flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) & 4899 HC_INDEX_DATA_HC_ENABLED); 4900 CNIC_WR16(dev, addr, flags); 4901 } 4902 4903 static void cnic_enable_bnx2x_int(struct cnic_dev *dev) 4904 { 4905 struct cnic_local *cp = dev->cnic_priv; 4906 struct bnx2x *bp = netdev_priv(dev->netdev); 4907 u8 sb_id = cp->status_blk_num; 4908 4909 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 4910 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4911 offsetof(struct hc_status_block_data_e1x, index_data) + 4912 sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS + 4913 offsetof(struct hc_index_data, timeout), 64 / 4); 4914 cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0); 4915 } 4916 4917 static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev) 4918 { 4919 } 4920 4921 static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev, 4922 struct client_init_ramrod_data *data) 4923 { 4924 struct cnic_local *cp = dev->cnic_priv; 4925 struct bnx2x *bp = netdev_priv(dev->netdev); 4926 struct cnic_uio_dev *udev = cp->udev; 4927 union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring; 4928 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4929 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4930 int i; 4931 u32 cli = cp->ethdev->iscsi_l2_client_id; 4932 u32 val; 4933 4934 memset(txbd, 0, CNIC_PAGE_SIZE); 4935 4936 buf_map = udev->l2_buf_map; 4937 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) { 4938 struct eth_tx_start_bd *start_bd = &txbd->start_bd; 4939 struct eth_tx_parse_bd_e1x *pbd_e1x = 4940 &((txbd + 1)->parse_bd_e1x); 4941 struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2); 4942 struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd); 4943 4944 start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4945 start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4946 reg_bd->addr_hi = start_bd->addr_hi; 4947 reg_bd->addr_lo = start_bd->addr_lo + 0x10; 4948 start_bd->nbytes = cpu_to_le16(0x10); 4949 start_bd->nbd = cpu_to_le16(3); 4950 start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 4951 start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS; 4952 start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 4953 4954 if (BNX2X_CHIP_IS_E2_PLUS(bp)) 4955 pbd_e2->parsing_data = (UNICAST_ADDRESS << 4956 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT); 4957 else 4958 pbd_e1x->global_data = (UNICAST_ADDRESS << 4959 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT); 4960 } 4961 4962 val = (u64) ring_map >> 32; 4963 txbd->next_bd.addr_hi = cpu_to_le32(val); 4964 4965 data->tx.tx_bd_page_base.hi = cpu_to_le32(val); 4966 4967 val = (u64) ring_map & 0xffffffff; 4968 txbd->next_bd.addr_lo = cpu_to_le32(val); 4969 4970 data->tx.tx_bd_page_base.lo = cpu_to_le32(val); 4971 4972 /* Other ramrod params */ 4973 data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS; 4974 data->tx.tx_status_block_id = BNX2X_DEF_SB_ID; 4975 4976 /* reset xstorm per client statistics */ 4977 if (cli < MAX_STAT_COUNTER_ID) { 4978 data->general.statistics_zero_flg = 1; 4979 data->general.statistics_en_flg = 1; 4980 data->general.statistics_counter_id = cli; 4981 } 4982 4983 cp->tx_cons_ptr = 4984 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS]; 4985 } 4986 4987 static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev, 4988 struct client_init_ramrod_data *data) 4989 { 4990 struct cnic_local *cp = dev->cnic_priv; 4991 struct bnx2x *bp = netdev_priv(dev->netdev); 4992 struct cnic_uio_dev *udev = cp->udev; 4993 struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring + 4994 CNIC_PAGE_SIZE); 4995 struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *) 4996 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 4997 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4998 int i; 4999 u32 cli = cp->ethdev->iscsi_l2_client_id; 5000 int cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5001 u32 val; 5002 dma_addr_t ring_map = udev->l2_ring_map; 5003 5004 /* General data */ 5005 data->general.client_id = cli; 5006 data->general.activate_flg = 1; 5007 data->general.sp_client_id = cli; 5008 data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14); 5009 data->general.func_id = bp->pfid; 5010 5011 for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) { 5012 dma_addr_t buf_map; 5013 int n = (i % cp->l2_rx_ring_size) + 1; 5014 5015 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 5016 rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 5017 rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 5018 } 5019 5020 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 5021 rxbd->addr_hi = cpu_to_le32(val); 5022 data->rx.bd_page_base.hi = cpu_to_le32(val); 5023 5024 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 5025 rxbd->addr_lo = cpu_to_le32(val); 5026 data->rx.bd_page_base.lo = cpu_to_le32(val); 5027 5028 rxcqe += BNX2X_MAX_RCQ_DESC_CNT; 5029 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) >> 32; 5030 rxcqe->addr_hi = cpu_to_le32(val); 5031 data->rx.cqe_page_base.hi = cpu_to_le32(val); 5032 5033 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) & 0xffffffff; 5034 rxcqe->addr_lo = cpu_to_le32(val); 5035 data->rx.cqe_page_base.lo = cpu_to_le32(val); 5036 5037 /* Other ramrod params */ 5038 data->rx.client_qzone_id = cl_qzone_id; 5039 data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS; 5040 data->rx.status_block_id = BNX2X_DEF_SB_ID; 5041 5042 data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT; 5043 5044 data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size); 5045 data->rx.outer_vlan_removal_enable_flg = 1; 5046 data->rx.silent_vlan_removal_flg = 1; 5047 data->rx.silent_vlan_value = 0; 5048 data->rx.silent_vlan_mask = 0xffff; 5049 5050 cp->rx_cons_ptr = 5051 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS]; 5052 cp->rx_cons = *cp->rx_cons_ptr; 5053 } 5054 5055 static void cnic_init_bnx2x_kcq(struct cnic_dev *dev) 5056 { 5057 struct cnic_local *cp = dev->cnic_priv; 5058 struct bnx2x *bp = netdev_priv(dev->netdev); 5059 u32 pfid = bp->pfid; 5060 5061 cp->kcq1.io_addr = BAR_CSTRORM_INTMEM + 5062 CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0); 5063 cp->kcq1.sw_prod_idx = 0; 5064 5065 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5066 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5067 5068 cp->kcq1.hw_prod_idx_ptr = 5069 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5070 cp->kcq1.status_idx_ptr = 5071 &sb->sb.running_index[SM_RX_ID]; 5072 } else { 5073 struct host_hc_status_block_e1x *sb = cp->status_blk.gen; 5074 5075 cp->kcq1.hw_prod_idx_ptr = 5076 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5077 cp->kcq1.status_idx_ptr = 5078 &sb->sb.running_index[SM_RX_ID]; 5079 } 5080 5081 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5082 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5083 5084 cp->kcq2.io_addr = BAR_USTRORM_INTMEM + 5085 USTORM_FCOE_EQ_PROD_OFFSET(pfid); 5086 cp->kcq2.sw_prod_idx = 0; 5087 cp->kcq2.hw_prod_idx_ptr = 5088 &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS]; 5089 cp->kcq2.status_idx_ptr = 5090 &sb->sb.running_index[SM_RX_ID]; 5091 } 5092 } 5093 5094 static int cnic_start_bnx2x_hw(struct cnic_dev *dev) 5095 { 5096 struct cnic_local *cp = dev->cnic_priv; 5097 struct bnx2x *bp = netdev_priv(dev->netdev); 5098 struct cnic_eth_dev *ethdev = cp->ethdev; 5099 int ret; 5100 u32 pfid; 5101 5102 dev->stats_addr = ethdev->addr_drv_info_to_mcp; 5103 cp->func = bp->pf_num; 5104 5105 pfid = bp->pfid; 5106 5107 ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ, 5108 cp->iscsi_start_cid, 0); 5109 5110 if (ret) 5111 return -ENOMEM; 5112 5113 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5114 ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn, 5115 cp->fcoe_start_cid, 0); 5116 5117 if (ret) 5118 return -ENOMEM; 5119 } 5120 5121 cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2; 5122 5123 cnic_init_bnx2x_kcq(dev); 5124 5125 /* Only 1 EQ */ 5126 CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX); 5127 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5128 CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0); 5129 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5130 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0), 5131 cp->kcq1.dma.pg_map_arr[1] & 0xffffffff); 5132 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5133 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4, 5134 (u64) cp->kcq1.dma.pg_map_arr[1] >> 32); 5135 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5136 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0), 5137 cp->kcq1.dma.pg_map_arr[0] & 0xffffffff); 5138 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5139 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4, 5140 (u64) cp->kcq1.dma.pg_map_arr[0] >> 32); 5141 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5142 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1); 5143 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 5144 CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num); 5145 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5146 CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0), 5147 HC_INDEX_ISCSI_EQ_CONS); 5148 5149 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5150 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid), 5151 cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff); 5152 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5153 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4, 5154 (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32); 5155 5156 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 5157 TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF); 5158 5159 cnic_setup_bnx2x_context(dev); 5160 5161 ret = cnic_init_bnx2x_irq(dev); 5162 if (ret) 5163 return ret; 5164 5165 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 5166 return 0; 5167 } 5168 5169 static void cnic_init_rings(struct cnic_dev *dev) 5170 { 5171 struct cnic_local *cp = dev->cnic_priv; 5172 struct bnx2x *bp = netdev_priv(dev->netdev); 5173 struct cnic_uio_dev *udev = cp->udev; 5174 5175 if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5176 return; 5177 5178 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5179 cnic_init_bnx2_tx_ring(dev); 5180 cnic_init_bnx2_rx_ring(dev); 5181 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5182 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5183 u32 cli = cp->ethdev->iscsi_l2_client_id; 5184 u32 cid = cp->ethdev->iscsi_l2_cid; 5185 u32 cl_qzone_id; 5186 struct client_init_ramrod_data *data; 5187 union l5cm_specific_data l5_data; 5188 struct ustorm_eth_rx_producers rx_prods = {0}; 5189 u32 off, i, *cid_ptr; 5190 5191 rx_prods.bd_prod = 0; 5192 rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT; 5193 barrier(); 5194 5195 cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5196 5197 off = BAR_USTRORM_INTMEM + 5198 (BNX2X_CHIP_IS_E2_PLUS(bp) ? 5199 USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) : 5200 USTORM_RX_PRODS_E1X_OFFSET(BP_PORT(bp), cli)); 5201 5202 for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++) 5203 CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]); 5204 5205 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5206 5207 data = udev->l2_buf; 5208 cid_ptr = udev->l2_buf + 12; 5209 5210 memset(data, 0, sizeof(*data)); 5211 5212 cnic_init_bnx2x_tx_ring(dev, data); 5213 cnic_init_bnx2x_rx_ring(dev, data); 5214 5215 data->general.fp_hsi_ver = ETH_FP_HSI_VERSION; 5216 5217 l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff; 5218 l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32; 5219 5220 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5221 5222 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP, 5223 cid, ETH_CONNECTION_TYPE, &l5_data); 5224 5225 i = 0; 5226 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5227 ++i < 10) 5228 msleep(1); 5229 5230 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5231 netdev_err(dev->netdev, 5232 "iSCSI CLIENT_SETUP did not complete\n"); 5233 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5234 cnic_ring_ctl(dev, cid, cli, 1); 5235 *cid_ptr = cid >> 4; 5236 *(cid_ptr + 1) = cid * bp->db_size; 5237 *(cid_ptr + 2) = UIO_USE_TX_DOORBELL; 5238 } 5239 } 5240 5241 static void cnic_shutdown_rings(struct cnic_dev *dev) 5242 { 5243 struct cnic_local *cp = dev->cnic_priv; 5244 struct cnic_uio_dev *udev = cp->udev; 5245 void *rx_ring; 5246 5247 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5248 return; 5249 5250 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5251 cnic_shutdown_bnx2_rx_ring(dev); 5252 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5253 u32 cli = cp->ethdev->iscsi_l2_client_id; 5254 u32 cid = cp->ethdev->iscsi_l2_cid; 5255 union l5cm_specific_data l5_data; 5256 int i; 5257 5258 cnic_ring_ctl(dev, cid, cli, 0); 5259 5260 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5261 5262 l5_data.phy_address.lo = cli; 5263 l5_data.phy_address.hi = 0; 5264 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT, 5265 cid, ETH_CONNECTION_TYPE, &l5_data); 5266 i = 0; 5267 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5268 ++i < 10) 5269 msleep(1); 5270 5271 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5272 netdev_err(dev->netdev, 5273 "iSCSI CLIENT_HALT did not complete\n"); 5274 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5275 5276 memset(&l5_data, 0, sizeof(l5_data)); 5277 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 5278 cid, NONE_CONNECTION_TYPE, &l5_data); 5279 msleep(10); 5280 } 5281 clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5282 rx_ring = udev->l2_ring + CNIC_PAGE_SIZE; 5283 memset(rx_ring, 0, CNIC_PAGE_SIZE); 5284 } 5285 5286 static int cnic_register_netdev(struct cnic_dev *dev) 5287 { 5288 struct cnic_local *cp = dev->cnic_priv; 5289 struct cnic_eth_dev *ethdev = cp->ethdev; 5290 int err; 5291 5292 if (!ethdev) 5293 return -ENODEV; 5294 5295 if (ethdev->drv_state & CNIC_DRV_STATE_REGD) 5296 return 0; 5297 5298 err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev); 5299 if (err) 5300 netdev_err(dev->netdev, "register_cnic failed\n"); 5301 5302 /* Read iSCSI config again. On some bnx2x device, iSCSI config 5303 * can change after firmware is downloaded. 5304 */ 5305 dev->max_iscsi_conn = ethdev->max_iscsi_conn; 5306 if (ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 5307 dev->max_iscsi_conn = 0; 5308 5309 return err; 5310 } 5311 5312 static void cnic_unregister_netdev(struct cnic_dev *dev) 5313 { 5314 struct cnic_local *cp = dev->cnic_priv; 5315 struct cnic_eth_dev *ethdev = cp->ethdev; 5316 5317 if (!ethdev) 5318 return; 5319 5320 ethdev->drv_unregister_cnic(dev->netdev); 5321 } 5322 5323 static int cnic_start_hw(struct cnic_dev *dev) 5324 { 5325 struct cnic_local *cp = dev->cnic_priv; 5326 struct cnic_eth_dev *ethdev = cp->ethdev; 5327 int err; 5328 5329 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5330 return -EALREADY; 5331 5332 dev->regview = ethdev->io_base; 5333 pci_dev_get(dev->pcidev); 5334 cp->func = PCI_FUNC(dev->pcidev->devfn); 5335 cp->status_blk.gen = ethdev->irq_arr[0].status_blk; 5336 cp->status_blk_map = ethdev->irq_arr[0].status_blk_map; 5337 cp->status_blk_num = ethdev->irq_arr[0].status_blk_num; 5338 5339 err = cp->alloc_resc(dev); 5340 if (err) { 5341 netdev_err(dev->netdev, "allocate resource failure\n"); 5342 goto err1; 5343 } 5344 5345 err = cp->start_hw(dev); 5346 if (err) 5347 goto err1; 5348 5349 err = cnic_cm_open(dev); 5350 if (err) 5351 goto err1; 5352 5353 set_bit(CNIC_F_CNIC_UP, &dev->flags); 5354 5355 cp->enable_int(dev); 5356 5357 return 0; 5358 5359 err1: 5360 if (ethdev->drv_state & CNIC_DRV_STATE_HANDLES_IRQ) 5361 cp->stop_hw(dev); 5362 else 5363 cp->free_resc(dev); 5364 pci_dev_put(dev->pcidev); 5365 return err; 5366 } 5367 5368 static void cnic_stop_bnx2_hw(struct cnic_dev *dev) 5369 { 5370 cnic_disable_bnx2_int_sync(dev); 5371 5372 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 5373 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 5374 5375 cnic_init_context(dev, KWQ_CID); 5376 cnic_init_context(dev, KCQ_CID); 5377 5378 cnic_setup_5709_context(dev, 0); 5379 cnic_free_irq(dev); 5380 5381 cnic_free_resc(dev); 5382 } 5383 5384 5385 static void cnic_stop_bnx2x_hw(struct cnic_dev *dev) 5386 { 5387 struct cnic_local *cp = dev->cnic_priv; 5388 struct bnx2x *bp = netdev_priv(dev->netdev); 5389 u32 hc_index = HC_INDEX_ISCSI_EQ_CONS; 5390 u32 sb_id = cp->status_blk_num; 5391 u32 idx_off, syn_off; 5392 5393 cnic_free_irq(dev); 5394 5395 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5396 idx_off = offsetof(struct hc_status_block_e2, index_values) + 5397 (hc_index * sizeof(u16)); 5398 5399 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id); 5400 } else { 5401 idx_off = offsetof(struct hc_status_block_e1x, index_values) + 5402 (hc_index * sizeof(u16)); 5403 5404 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id); 5405 } 5406 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0); 5407 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) + 5408 idx_off, 0); 5409 5410 *cp->kcq1.hw_prod_idx_ptr = 0; 5411 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5412 CSTORM_ISCSI_EQ_CONS_OFFSET(bp->pfid, 0), 0); 5413 CNIC_WR16(dev, cp->kcq1.io_addr, 0); 5414 cnic_free_resc(dev); 5415 } 5416 5417 static void cnic_stop_hw(struct cnic_dev *dev) 5418 { 5419 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5420 struct cnic_local *cp = dev->cnic_priv; 5421 int i = 0; 5422 5423 /* Need to wait for the ring shutdown event to complete 5424 * before clearing the CNIC_UP flag. 5425 */ 5426 while (cp->udev && cp->udev->uio_dev != -1 && i < 15) { 5427 msleep(100); 5428 i++; 5429 } 5430 cnic_shutdown_rings(dev); 5431 cp->stop_cm(dev); 5432 cp->ethdev->drv_state &= ~CNIC_DRV_STATE_HANDLES_IRQ; 5433 clear_bit(CNIC_F_CNIC_UP, &dev->flags); 5434 RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL); 5435 synchronize_rcu(); 5436 cnic_cm_shutdown(dev); 5437 cp->stop_hw(dev); 5438 pci_dev_put(dev->pcidev); 5439 } 5440 } 5441 5442 static void cnic_free_dev(struct cnic_dev *dev) 5443 { 5444 int i = 0; 5445 5446 while ((atomic_read(&dev->ref_count) != 0) && i < 10) { 5447 msleep(100); 5448 i++; 5449 } 5450 if (atomic_read(&dev->ref_count) != 0) 5451 netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n"); 5452 5453 netdev_info(dev->netdev, "Removed CNIC device\n"); 5454 dev_put(dev->netdev); 5455 kfree(dev); 5456 } 5457 5458 static int cnic_get_fc_npiv_tbl(struct cnic_dev *dev, 5459 struct cnic_fc_npiv_tbl *npiv_tbl) 5460 { 5461 struct cnic_local *cp = dev->cnic_priv; 5462 struct bnx2x *bp = netdev_priv(dev->netdev); 5463 int ret; 5464 5465 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5466 return -EAGAIN; /* bnx2x is down */ 5467 5468 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 5469 return -EINVAL; 5470 5471 ret = cp->ethdev->drv_get_fc_npiv_tbl(dev->netdev, npiv_tbl); 5472 return ret; 5473 } 5474 5475 static struct cnic_dev *cnic_alloc_dev(struct net_device *dev, 5476 struct pci_dev *pdev) 5477 { 5478 struct cnic_dev *cdev; 5479 struct cnic_local *cp; 5480 int alloc_size; 5481 5482 alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local); 5483 5484 cdev = kzalloc(alloc_size, GFP_KERNEL); 5485 if (cdev == NULL) 5486 return NULL; 5487 5488 cdev->netdev = dev; 5489 cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev); 5490 cdev->register_device = cnic_register_device; 5491 cdev->unregister_device = cnic_unregister_device; 5492 cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv; 5493 cdev->get_fc_npiv_tbl = cnic_get_fc_npiv_tbl; 5494 atomic_set(&cdev->ref_count, 0); 5495 5496 cp = cdev->cnic_priv; 5497 cp->dev = cdev; 5498 cp->l2_single_buf_size = 0x400; 5499 cp->l2_rx_ring_size = 3; 5500 5501 spin_lock_init(&cp->cnic_ulp_lock); 5502 5503 netdev_info(dev, "Added CNIC device\n"); 5504 5505 return cdev; 5506 } 5507 5508 static struct cnic_dev *init_bnx2_cnic(struct net_device *dev) 5509 { 5510 struct pci_dev *pdev; 5511 struct cnic_dev *cdev; 5512 struct cnic_local *cp; 5513 struct bnx2 *bp = netdev_priv(dev); 5514 struct cnic_eth_dev *ethdev = NULL; 5515 5516 if (bp->cnic_probe) 5517 ethdev = (bp->cnic_probe)(dev); 5518 5519 if (!ethdev) 5520 return NULL; 5521 5522 pdev = ethdev->pdev; 5523 if (!pdev) 5524 return NULL; 5525 5526 dev_hold(dev); 5527 pci_dev_get(pdev); 5528 if ((pdev->device == PCI_DEVICE_ID_NX2_5709 || 5529 pdev->device == PCI_DEVICE_ID_NX2_5709S) && 5530 (pdev->revision < 0x10)) { 5531 pci_dev_put(pdev); 5532 goto cnic_err; 5533 } 5534 pci_dev_put(pdev); 5535 5536 cdev = cnic_alloc_dev(dev, pdev); 5537 if (cdev == NULL) 5538 goto cnic_err; 5539 5540 set_bit(CNIC_F_BNX2_CLASS, &cdev->flags); 5541 cdev->submit_kwqes = cnic_submit_bnx2_kwqes; 5542 5543 cp = cdev->cnic_priv; 5544 cp->ethdev = ethdev; 5545 cdev->pcidev = pdev; 5546 cp->chip_id = ethdev->chip_id; 5547 5548 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5549 5550 cp->cnic_ops = &cnic_bnx2_ops; 5551 cp->start_hw = cnic_start_bnx2_hw; 5552 cp->stop_hw = cnic_stop_bnx2_hw; 5553 cp->setup_pgtbl = cnic_setup_page_tbl; 5554 cp->alloc_resc = cnic_alloc_bnx2_resc; 5555 cp->free_resc = cnic_free_resc; 5556 cp->start_cm = cnic_cm_init_bnx2_hw; 5557 cp->stop_cm = cnic_cm_stop_bnx2_hw; 5558 cp->enable_int = cnic_enable_bnx2_int; 5559 cp->disable_int_sync = cnic_disable_bnx2_int_sync; 5560 cp->close_conn = cnic_close_bnx2_conn; 5561 return cdev; 5562 5563 cnic_err: 5564 dev_put(dev); 5565 return NULL; 5566 } 5567 5568 static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev) 5569 { 5570 struct pci_dev *pdev; 5571 struct cnic_dev *cdev; 5572 struct cnic_local *cp; 5573 struct bnx2x *bp = netdev_priv(dev); 5574 struct cnic_eth_dev *ethdev = NULL; 5575 5576 if (bp->cnic_probe) 5577 ethdev = bp->cnic_probe(dev); 5578 5579 if (!ethdev) 5580 return NULL; 5581 5582 pdev = ethdev->pdev; 5583 if (!pdev) 5584 return NULL; 5585 5586 dev_hold(dev); 5587 cdev = cnic_alloc_dev(dev, pdev); 5588 if (cdev == NULL) { 5589 dev_put(dev); 5590 return NULL; 5591 } 5592 5593 set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags); 5594 cdev->submit_kwqes = cnic_submit_bnx2x_kwqes; 5595 5596 cp = cdev->cnic_priv; 5597 cp->ethdev = ethdev; 5598 cdev->pcidev = pdev; 5599 cp->chip_id = ethdev->chip_id; 5600 5601 cdev->stats_addr = ethdev->addr_drv_info_to_mcp; 5602 5603 if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)) 5604 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5605 if (CNIC_SUPPORTS_FCOE(bp)) { 5606 cdev->max_fcoe_conn = ethdev->max_fcoe_conn; 5607 cdev->max_fcoe_exchanges = ethdev->max_fcoe_exchanges; 5608 } 5609 5610 if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS) 5611 cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS; 5612 5613 memcpy(cdev->mac_addr, ethdev->iscsi_mac, ETH_ALEN); 5614 5615 cp->cnic_ops = &cnic_bnx2x_ops; 5616 cp->start_hw = cnic_start_bnx2x_hw; 5617 cp->stop_hw = cnic_stop_bnx2x_hw; 5618 cp->setup_pgtbl = cnic_setup_page_tbl_le; 5619 cp->alloc_resc = cnic_alloc_bnx2x_resc; 5620 cp->free_resc = cnic_free_resc; 5621 cp->start_cm = cnic_cm_init_bnx2x_hw; 5622 cp->stop_cm = cnic_cm_stop_bnx2x_hw; 5623 cp->enable_int = cnic_enable_bnx2x_int; 5624 cp->disable_int_sync = cnic_disable_bnx2x_int_sync; 5625 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5626 cp->ack_int = cnic_ack_bnx2x_e2_msix; 5627 cp->arm_int = cnic_arm_bnx2x_e2_msix; 5628 } else { 5629 cp->ack_int = cnic_ack_bnx2x_msix; 5630 cp->arm_int = cnic_arm_bnx2x_msix; 5631 } 5632 cp->close_conn = cnic_close_bnx2x_conn; 5633 return cdev; 5634 } 5635 5636 static struct cnic_dev *is_cnic_dev(struct net_device *dev) 5637 { 5638 struct ethtool_drvinfo drvinfo; 5639 struct cnic_dev *cdev = NULL; 5640 5641 if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) { 5642 memset(&drvinfo, 0, sizeof(drvinfo)); 5643 dev->ethtool_ops->get_drvinfo(dev, &drvinfo); 5644 5645 if (!strcmp(drvinfo.driver, "bnx2")) 5646 cdev = init_bnx2_cnic(dev); 5647 if (!strcmp(drvinfo.driver, "bnx2x")) 5648 cdev = init_bnx2x_cnic(dev); 5649 if (cdev) { 5650 write_lock(&cnic_dev_lock); 5651 list_add(&cdev->list, &cnic_dev_list); 5652 write_unlock(&cnic_dev_lock); 5653 } 5654 } 5655 return cdev; 5656 } 5657 5658 static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event, 5659 u16 vlan_id) 5660 { 5661 int if_type; 5662 5663 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 5664 struct cnic_ulp_ops *ulp_ops; 5665 void *ctx; 5666 5667 mutex_lock(&cnic_lock); 5668 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 5669 lockdep_is_held(&cnic_lock)); 5670 if (!ulp_ops || !ulp_ops->indicate_netevent) { 5671 mutex_unlock(&cnic_lock); 5672 continue; 5673 } 5674 5675 ctx = cp->ulp_handle[if_type]; 5676 5677 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5678 mutex_unlock(&cnic_lock); 5679 5680 ulp_ops->indicate_netevent(ctx, event, vlan_id); 5681 5682 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5683 } 5684 } 5685 5686 /* netdev event handler */ 5687 static int cnic_netdev_event(struct notifier_block *this, unsigned long event, 5688 void *ptr) 5689 { 5690 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 5691 struct cnic_dev *dev; 5692 int new_dev = 0; 5693 5694 dev = cnic_from_netdev(netdev); 5695 5696 if (!dev && event == NETDEV_REGISTER) { 5697 /* Check for the hot-plug device */ 5698 dev = is_cnic_dev(netdev); 5699 if (dev) { 5700 new_dev = 1; 5701 cnic_hold(dev); 5702 } 5703 } 5704 if (dev) { 5705 struct cnic_local *cp = dev->cnic_priv; 5706 5707 if (new_dev) 5708 cnic_ulp_init(dev); 5709 else if (event == NETDEV_UNREGISTER) 5710 cnic_ulp_exit(dev); 5711 5712 if (event == NETDEV_UP) { 5713 if (cnic_register_netdev(dev) != 0) { 5714 cnic_put(dev); 5715 goto done; 5716 } 5717 if (!cnic_start_hw(dev)) 5718 cnic_ulp_start(dev); 5719 } 5720 5721 cnic_rcv_netevent(cp, event, 0); 5722 5723 if (event == NETDEV_GOING_DOWN) { 5724 cnic_ulp_stop(dev); 5725 cnic_stop_hw(dev); 5726 cnic_unregister_netdev(dev); 5727 } else if (event == NETDEV_UNREGISTER) { 5728 write_lock(&cnic_dev_lock); 5729 list_del_init(&dev->list); 5730 write_unlock(&cnic_dev_lock); 5731 5732 cnic_put(dev); 5733 cnic_free_dev(dev); 5734 goto done; 5735 } 5736 cnic_put(dev); 5737 } else { 5738 struct net_device *realdev; 5739 u16 vid; 5740 5741 vid = cnic_get_vlan(netdev, &realdev); 5742 if (realdev) { 5743 dev = cnic_from_netdev(realdev); 5744 if (dev) { 5745 vid |= VLAN_CFI_MASK; /* make non-zero */ 5746 cnic_rcv_netevent(dev->cnic_priv, event, vid); 5747 cnic_put(dev); 5748 } 5749 } 5750 } 5751 done: 5752 return NOTIFY_DONE; 5753 } 5754 5755 static struct notifier_block cnic_netdev_notifier = { 5756 .notifier_call = cnic_netdev_event 5757 }; 5758 5759 static void cnic_release(void) 5760 { 5761 struct cnic_uio_dev *udev; 5762 5763 while (!list_empty(&cnic_udev_list)) { 5764 udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev, 5765 list); 5766 cnic_free_uio(udev); 5767 } 5768 } 5769 5770 static int __init cnic_init(void) 5771 { 5772 int rc = 0; 5773 5774 pr_info("%s", version); 5775 5776 rc = register_netdevice_notifier(&cnic_netdev_notifier); 5777 if (rc) { 5778 cnic_release(); 5779 return rc; 5780 } 5781 5782 cnic_wq = create_singlethread_workqueue("cnic_wq"); 5783 if (!cnic_wq) { 5784 cnic_release(); 5785 unregister_netdevice_notifier(&cnic_netdev_notifier); 5786 return -ENOMEM; 5787 } 5788 5789 return 0; 5790 } 5791 5792 static void __exit cnic_exit(void) 5793 { 5794 unregister_netdevice_notifier(&cnic_netdev_notifier); 5795 cnic_release(); 5796 destroy_workqueue(cnic_wq); 5797 } 5798 5799 module_init(cnic_init); 5800 module_exit(cnic_exit); 5801