1 /* cnic.c: QLogic CNIC core network driver. 2 * 3 * Copyright (c) 2006-2014 Broadcom Corporation 4 * Copyright (c) 2014-2015 QLogic Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com) 11 * Previously modified and maintained by: Michael Chan <mchan@broadcom.com> 12 * Maintained By: Dept-HSGLinuxNICDev@qlogic.com 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/module.h> 18 19 #include <linux/kernel.h> 20 #include <linux/errno.h> 21 #include <linux/list.h> 22 #include <linux/slab.h> 23 #include <linux/pci.h> 24 #include <linux/init.h> 25 #include <linux/netdevice.h> 26 #include <linux/uio_driver.h> 27 #include <linux/in.h> 28 #include <linux/dma-mapping.h> 29 #include <linux/delay.h> 30 #include <linux/ethtool.h> 31 #include <linux/if_vlan.h> 32 #include <linux/prefetch.h> 33 #include <linux/random.h> 34 #include <linux/workqueue.h> 35 #if IS_ENABLED(CONFIG_VLAN_8021Q) 36 #define BCM_VLAN 1 37 #endif 38 #include <net/ip.h> 39 #include <net/tcp.h> 40 #include <net/route.h> 41 #include <net/ipv6.h> 42 #include <net/ip6_route.h> 43 #include <net/ip6_checksum.h> 44 #include <scsi/iscsi_if.h> 45 46 #define BCM_CNIC 1 47 #include "cnic_if.h" 48 #include "bnx2.h" 49 #include "bnx2x/bnx2x.h" 50 #include "bnx2x/bnx2x_reg.h" 51 #include "bnx2x/bnx2x_fw_defs.h" 52 #include "bnx2x/bnx2x_hsi.h" 53 #include "../../../scsi/bnx2i/57xx_iscsi_constants.h" 54 #include "../../../scsi/bnx2i/57xx_iscsi_hsi.h" 55 #include "../../../scsi/bnx2fc/bnx2fc_constants.h" 56 #include "cnic.h" 57 #include "cnic_defs.h" 58 59 #define CNIC_MODULE_NAME "cnic" 60 61 static char version[] = 62 "QLogic " CNIC_MODULE_NAME "Driver v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n"; 63 64 MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) " 65 "Chen (zongxi@broadcom.com"); 66 MODULE_DESCRIPTION("QLogic cnic Driver"); 67 MODULE_LICENSE("GPL"); 68 MODULE_VERSION(CNIC_MODULE_VERSION); 69 70 /* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */ 71 static LIST_HEAD(cnic_dev_list); 72 static LIST_HEAD(cnic_udev_list); 73 static DEFINE_RWLOCK(cnic_dev_lock); 74 static DEFINE_MUTEX(cnic_lock); 75 76 static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE]; 77 78 /* helper function, assuming cnic_lock is held */ 79 static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type) 80 { 81 return rcu_dereference_protected(cnic_ulp_tbl[type], 82 lockdep_is_held(&cnic_lock)); 83 } 84 85 static int cnic_service_bnx2(void *, void *); 86 static int cnic_service_bnx2x(void *, void *); 87 static int cnic_ctl(void *, struct cnic_ctl_info *); 88 89 static struct cnic_ops cnic_bnx2_ops = { 90 .cnic_owner = THIS_MODULE, 91 .cnic_handler = cnic_service_bnx2, 92 .cnic_ctl = cnic_ctl, 93 }; 94 95 static struct cnic_ops cnic_bnx2x_ops = { 96 .cnic_owner = THIS_MODULE, 97 .cnic_handler = cnic_service_bnx2x, 98 .cnic_ctl = cnic_ctl, 99 }; 100 101 static struct workqueue_struct *cnic_wq; 102 103 static void cnic_shutdown_rings(struct cnic_dev *); 104 static void cnic_init_rings(struct cnic_dev *); 105 static int cnic_cm_set_pg(struct cnic_sock *); 106 107 static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode) 108 { 109 struct cnic_uio_dev *udev = uinfo->priv; 110 struct cnic_dev *dev; 111 112 if (!capable(CAP_NET_ADMIN)) 113 return -EPERM; 114 115 if (udev->uio_dev != -1) 116 return -EBUSY; 117 118 rtnl_lock(); 119 dev = udev->dev; 120 121 if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 122 rtnl_unlock(); 123 return -ENODEV; 124 } 125 126 udev->uio_dev = iminor(inode); 127 128 cnic_shutdown_rings(dev); 129 cnic_init_rings(dev); 130 rtnl_unlock(); 131 132 return 0; 133 } 134 135 static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode) 136 { 137 struct cnic_uio_dev *udev = uinfo->priv; 138 139 udev->uio_dev = -1; 140 return 0; 141 } 142 143 static inline void cnic_hold(struct cnic_dev *dev) 144 { 145 atomic_inc(&dev->ref_count); 146 } 147 148 static inline void cnic_put(struct cnic_dev *dev) 149 { 150 atomic_dec(&dev->ref_count); 151 } 152 153 static inline void csk_hold(struct cnic_sock *csk) 154 { 155 atomic_inc(&csk->ref_count); 156 } 157 158 static inline void csk_put(struct cnic_sock *csk) 159 { 160 atomic_dec(&csk->ref_count); 161 } 162 163 static struct cnic_dev *cnic_from_netdev(struct net_device *netdev) 164 { 165 struct cnic_dev *cdev; 166 167 read_lock(&cnic_dev_lock); 168 list_for_each_entry(cdev, &cnic_dev_list, list) { 169 if (netdev == cdev->netdev) { 170 cnic_hold(cdev); 171 read_unlock(&cnic_dev_lock); 172 return cdev; 173 } 174 } 175 read_unlock(&cnic_dev_lock); 176 return NULL; 177 } 178 179 static inline void ulp_get(struct cnic_ulp_ops *ulp_ops) 180 { 181 atomic_inc(&ulp_ops->ref_count); 182 } 183 184 static inline void ulp_put(struct cnic_ulp_ops *ulp_ops) 185 { 186 atomic_dec(&ulp_ops->ref_count); 187 } 188 189 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val) 190 { 191 struct cnic_local *cp = dev->cnic_priv; 192 struct cnic_eth_dev *ethdev = cp->ethdev; 193 struct drv_ctl_info info; 194 struct drv_ctl_io *io = &info.data.io; 195 196 memset(&info, 0, sizeof(struct drv_ctl_info)); 197 info.cmd = DRV_CTL_CTX_WR_CMD; 198 io->cid_addr = cid_addr; 199 io->offset = off; 200 io->data = val; 201 ethdev->drv_ctl(dev->netdev, &info); 202 } 203 204 static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr) 205 { 206 struct cnic_local *cp = dev->cnic_priv; 207 struct cnic_eth_dev *ethdev = cp->ethdev; 208 struct drv_ctl_info info; 209 struct drv_ctl_io *io = &info.data.io; 210 211 memset(&info, 0, sizeof(struct drv_ctl_info)); 212 info.cmd = DRV_CTL_CTXTBL_WR_CMD; 213 io->offset = off; 214 io->dma_addr = addr; 215 ethdev->drv_ctl(dev->netdev, &info); 216 } 217 218 static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start) 219 { 220 struct cnic_local *cp = dev->cnic_priv; 221 struct cnic_eth_dev *ethdev = cp->ethdev; 222 struct drv_ctl_info info; 223 struct drv_ctl_l2_ring *ring = &info.data.ring; 224 225 memset(&info, 0, sizeof(struct drv_ctl_info)); 226 if (start) 227 info.cmd = DRV_CTL_START_L2_CMD; 228 else 229 info.cmd = DRV_CTL_STOP_L2_CMD; 230 231 ring->cid = cid; 232 ring->client_id = cl_id; 233 ethdev->drv_ctl(dev->netdev, &info); 234 } 235 236 static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val) 237 { 238 struct cnic_local *cp = dev->cnic_priv; 239 struct cnic_eth_dev *ethdev = cp->ethdev; 240 struct drv_ctl_info info; 241 struct drv_ctl_io *io = &info.data.io; 242 243 memset(&info, 0, sizeof(struct drv_ctl_info)); 244 info.cmd = DRV_CTL_IO_WR_CMD; 245 io->offset = off; 246 io->data = val; 247 ethdev->drv_ctl(dev->netdev, &info); 248 } 249 250 static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off) 251 { 252 struct cnic_local *cp = dev->cnic_priv; 253 struct cnic_eth_dev *ethdev = cp->ethdev; 254 struct drv_ctl_info info; 255 struct drv_ctl_io *io = &info.data.io; 256 257 memset(&info, 0, sizeof(struct drv_ctl_info)); 258 info.cmd = DRV_CTL_IO_RD_CMD; 259 io->offset = off; 260 ethdev->drv_ctl(dev->netdev, &info); 261 return io->data; 262 } 263 264 static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg, int state) 265 { 266 struct cnic_local *cp = dev->cnic_priv; 267 struct cnic_eth_dev *ethdev = cp->ethdev; 268 struct drv_ctl_info info; 269 struct fcoe_capabilities *fcoe_cap = 270 &info.data.register_data.fcoe_features; 271 272 memset(&info, 0, sizeof(struct drv_ctl_info)); 273 if (reg) { 274 info.cmd = DRV_CTL_ULP_REGISTER_CMD; 275 if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap) 276 memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap)); 277 } else { 278 info.cmd = DRV_CTL_ULP_UNREGISTER_CMD; 279 } 280 281 info.data.ulp_type = ulp_type; 282 info.drv_state = state; 283 ethdev->drv_ctl(dev->netdev, &info); 284 } 285 286 static int cnic_in_use(struct cnic_sock *csk) 287 { 288 return test_bit(SK_F_INUSE, &csk->flags); 289 } 290 291 static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count) 292 { 293 struct cnic_local *cp = dev->cnic_priv; 294 struct cnic_eth_dev *ethdev = cp->ethdev; 295 struct drv_ctl_info info; 296 297 memset(&info, 0, sizeof(struct drv_ctl_info)); 298 info.cmd = cmd; 299 info.data.credit.credit_count = count; 300 ethdev->drv_ctl(dev->netdev, &info); 301 } 302 303 static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid) 304 { 305 u32 i; 306 307 if (!cp->ctx_tbl) 308 return -EINVAL; 309 310 for (i = 0; i < cp->max_cid_space; i++) { 311 if (cp->ctx_tbl[i].cid == cid) { 312 *l5_cid = i; 313 return 0; 314 } 315 } 316 return -EINVAL; 317 } 318 319 static int cnic_send_nlmsg(struct cnic_local *cp, u32 type, 320 struct cnic_sock *csk) 321 { 322 struct iscsi_path path_req; 323 char *buf = NULL; 324 u16 len = 0; 325 u32 msg_type = ISCSI_KEVENT_IF_DOWN; 326 struct cnic_ulp_ops *ulp_ops; 327 struct cnic_uio_dev *udev = cp->udev; 328 int rc = 0, retry = 0; 329 330 if (!udev || udev->uio_dev == -1) 331 return -ENODEV; 332 333 if (csk) { 334 len = sizeof(path_req); 335 buf = (char *) &path_req; 336 memset(&path_req, 0, len); 337 338 msg_type = ISCSI_KEVENT_PATH_REQ; 339 path_req.handle = (u64) csk->l5_cid; 340 if (test_bit(SK_F_IPV6, &csk->flags)) { 341 memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0], 342 sizeof(struct in6_addr)); 343 path_req.ip_addr_len = 16; 344 } else { 345 memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0], 346 sizeof(struct in_addr)); 347 path_req.ip_addr_len = 4; 348 } 349 path_req.vlan_id = csk->vlan_id; 350 path_req.pmtu = csk->mtu; 351 } 352 353 while (retry < 3) { 354 rc = 0; 355 rcu_read_lock(); 356 ulp_ops = rcu_dereference(cp->ulp_ops[CNIC_ULP_ISCSI]); 357 if (ulp_ops) 358 rc = ulp_ops->iscsi_nl_send_msg( 359 cp->ulp_handle[CNIC_ULP_ISCSI], 360 msg_type, buf, len); 361 rcu_read_unlock(); 362 if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ) 363 break; 364 365 msleep(100); 366 retry++; 367 } 368 return rc; 369 } 370 371 static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8); 372 373 static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type, 374 char *buf, u16 len) 375 { 376 int rc = -EINVAL; 377 378 switch (msg_type) { 379 case ISCSI_UEVENT_PATH_UPDATE: { 380 struct cnic_local *cp; 381 u32 l5_cid; 382 struct cnic_sock *csk; 383 struct iscsi_path *path_resp; 384 385 if (len < sizeof(*path_resp)) 386 break; 387 388 path_resp = (struct iscsi_path *) buf; 389 cp = dev->cnic_priv; 390 l5_cid = (u32) path_resp->handle; 391 if (l5_cid >= MAX_CM_SK_TBL_SZ) 392 break; 393 394 if (!rcu_access_pointer(cp->ulp_ops[CNIC_ULP_L4])) { 395 rc = -ENODEV; 396 break; 397 } 398 csk = &cp->csk_tbl[l5_cid]; 399 csk_hold(csk); 400 if (cnic_in_use(csk) && 401 test_bit(SK_F_CONNECT_START, &csk->flags)) { 402 403 csk->vlan_id = path_resp->vlan_id; 404 405 memcpy(csk->ha, path_resp->mac_addr, ETH_ALEN); 406 if (test_bit(SK_F_IPV6, &csk->flags)) 407 memcpy(&csk->src_ip[0], &path_resp->src.v6_addr, 408 sizeof(struct in6_addr)); 409 else 410 memcpy(&csk->src_ip[0], &path_resp->src.v4_addr, 411 sizeof(struct in_addr)); 412 413 if (is_valid_ether_addr(csk->ha)) { 414 cnic_cm_set_pg(csk); 415 } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) && 416 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 417 418 cnic_cm_upcall(cp, csk, 419 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 420 clear_bit(SK_F_CONNECT_START, &csk->flags); 421 } 422 } 423 csk_put(csk); 424 rc = 0; 425 } 426 } 427 428 return rc; 429 } 430 431 static int cnic_offld_prep(struct cnic_sock *csk) 432 { 433 if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 434 return 0; 435 436 if (!test_bit(SK_F_CONNECT_START, &csk->flags)) { 437 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 438 return 0; 439 } 440 441 return 1; 442 } 443 444 static int cnic_close_prep(struct cnic_sock *csk) 445 { 446 clear_bit(SK_F_CONNECT_START, &csk->flags); 447 smp_mb__after_atomic(); 448 449 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 450 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 451 msleep(1); 452 453 return 1; 454 } 455 return 0; 456 } 457 458 static int cnic_abort_prep(struct cnic_sock *csk) 459 { 460 clear_bit(SK_F_CONNECT_START, &csk->flags); 461 smp_mb__after_atomic(); 462 463 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 464 msleep(1); 465 466 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 467 csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP; 468 return 1; 469 } 470 471 return 0; 472 } 473 474 int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops) 475 { 476 struct cnic_dev *dev; 477 478 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 479 pr_err("%s: Bad type %d\n", __func__, ulp_type); 480 return -EINVAL; 481 } 482 mutex_lock(&cnic_lock); 483 if (cnic_ulp_tbl_prot(ulp_type)) { 484 pr_err("%s: Type %d has already been registered\n", 485 __func__, ulp_type); 486 mutex_unlock(&cnic_lock); 487 return -EBUSY; 488 } 489 490 read_lock(&cnic_dev_lock); 491 list_for_each_entry(dev, &cnic_dev_list, list) { 492 struct cnic_local *cp = dev->cnic_priv; 493 494 clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]); 495 } 496 read_unlock(&cnic_dev_lock); 497 498 atomic_set(&ulp_ops->ref_count, 0); 499 rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops); 500 mutex_unlock(&cnic_lock); 501 502 /* Prevent race conditions with netdev_event */ 503 rtnl_lock(); 504 list_for_each_entry(dev, &cnic_dev_list, list) { 505 struct cnic_local *cp = dev->cnic_priv; 506 507 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 508 ulp_ops->cnic_init(dev); 509 } 510 rtnl_unlock(); 511 512 return 0; 513 } 514 515 int cnic_unregister_driver(int ulp_type) 516 { 517 struct cnic_dev *dev; 518 struct cnic_ulp_ops *ulp_ops; 519 int i = 0; 520 521 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 522 pr_err("%s: Bad type %d\n", __func__, ulp_type); 523 return -EINVAL; 524 } 525 mutex_lock(&cnic_lock); 526 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 527 if (!ulp_ops) { 528 pr_err("%s: Type %d has not been registered\n", 529 __func__, ulp_type); 530 goto out_unlock; 531 } 532 read_lock(&cnic_dev_lock); 533 list_for_each_entry(dev, &cnic_dev_list, list) { 534 struct cnic_local *cp = dev->cnic_priv; 535 536 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 537 pr_err("%s: Type %d still has devices registered\n", 538 __func__, ulp_type); 539 read_unlock(&cnic_dev_lock); 540 goto out_unlock; 541 } 542 } 543 read_unlock(&cnic_dev_lock); 544 545 RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL); 546 547 mutex_unlock(&cnic_lock); 548 synchronize_rcu(); 549 while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) { 550 msleep(100); 551 i++; 552 } 553 554 if (atomic_read(&ulp_ops->ref_count) != 0) 555 pr_warn("%s: Failed waiting for ref count to go to zero\n", 556 __func__); 557 return 0; 558 559 out_unlock: 560 mutex_unlock(&cnic_lock); 561 return -EINVAL; 562 } 563 564 static int cnic_start_hw(struct cnic_dev *); 565 static void cnic_stop_hw(struct cnic_dev *); 566 567 static int cnic_register_device(struct cnic_dev *dev, int ulp_type, 568 void *ulp_ctx) 569 { 570 struct cnic_local *cp = dev->cnic_priv; 571 struct cnic_ulp_ops *ulp_ops; 572 573 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 574 pr_err("%s: Bad type %d\n", __func__, ulp_type); 575 return -EINVAL; 576 } 577 mutex_lock(&cnic_lock); 578 if (cnic_ulp_tbl_prot(ulp_type) == NULL) { 579 pr_err("%s: Driver with type %d has not been registered\n", 580 __func__, ulp_type); 581 mutex_unlock(&cnic_lock); 582 return -EAGAIN; 583 } 584 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 585 pr_err("%s: Type %d has already been registered to this device\n", 586 __func__, ulp_type); 587 mutex_unlock(&cnic_lock); 588 return -EBUSY; 589 } 590 591 clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]); 592 cp->ulp_handle[ulp_type] = ulp_ctx; 593 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 594 rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops); 595 cnic_hold(dev); 596 597 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 598 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type])) 599 ulp_ops->cnic_start(cp->ulp_handle[ulp_type]); 600 601 mutex_unlock(&cnic_lock); 602 603 cnic_ulp_ctl(dev, ulp_type, true, DRV_ACTIVE); 604 605 return 0; 606 607 } 608 EXPORT_SYMBOL(cnic_register_driver); 609 610 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type) 611 { 612 struct cnic_local *cp = dev->cnic_priv; 613 int i = 0; 614 615 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 616 pr_err("%s: Bad type %d\n", __func__, ulp_type); 617 return -EINVAL; 618 } 619 620 if (ulp_type == CNIC_ULP_ISCSI) 621 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 622 623 mutex_lock(&cnic_lock); 624 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 625 RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL); 626 cnic_put(dev); 627 } else { 628 pr_err("%s: device not registered to this ulp type %d\n", 629 __func__, ulp_type); 630 mutex_unlock(&cnic_lock); 631 return -EINVAL; 632 } 633 mutex_unlock(&cnic_lock); 634 635 if (ulp_type == CNIC_ULP_FCOE) 636 dev->fcoe_cap = NULL; 637 638 synchronize_rcu(); 639 640 while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) && 641 i < 20) { 642 msleep(100); 643 i++; 644 } 645 if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type])) 646 netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n"); 647 648 if (test_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 649 cnic_ulp_ctl(dev, ulp_type, false, DRV_UNLOADED); 650 else 651 cnic_ulp_ctl(dev, ulp_type, false, DRV_INACTIVE); 652 653 return 0; 654 } 655 EXPORT_SYMBOL(cnic_unregister_driver); 656 657 static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id, 658 u32 next) 659 { 660 id_tbl->start = start_id; 661 id_tbl->max = size; 662 id_tbl->next = next; 663 spin_lock_init(&id_tbl->lock); 664 id_tbl->table = bitmap_zalloc(size, GFP_KERNEL); 665 if (!id_tbl->table) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl) 672 { 673 bitmap_free(id_tbl->table); 674 id_tbl->table = NULL; 675 } 676 677 static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id) 678 { 679 int ret = -1; 680 681 id -= id_tbl->start; 682 if (id >= id_tbl->max) 683 return ret; 684 685 spin_lock(&id_tbl->lock); 686 if (!test_bit(id, id_tbl->table)) { 687 set_bit(id, id_tbl->table); 688 ret = 0; 689 } 690 spin_unlock(&id_tbl->lock); 691 return ret; 692 } 693 694 /* Returns -1 if not successful */ 695 static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl) 696 { 697 u32 id; 698 699 spin_lock(&id_tbl->lock); 700 id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next); 701 if (id >= id_tbl->max) { 702 id = -1; 703 if (id_tbl->next != 0) { 704 id = find_first_zero_bit(id_tbl->table, id_tbl->next); 705 if (id >= id_tbl->next) 706 id = -1; 707 } 708 } 709 710 if (id < id_tbl->max) { 711 set_bit(id, id_tbl->table); 712 id_tbl->next = (id + 1) & (id_tbl->max - 1); 713 id += id_tbl->start; 714 } 715 716 spin_unlock(&id_tbl->lock); 717 718 return id; 719 } 720 721 static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id) 722 { 723 if (id == -1) 724 return; 725 726 id -= id_tbl->start; 727 if (id >= id_tbl->max) 728 return; 729 730 clear_bit(id, id_tbl->table); 731 } 732 733 static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma) 734 { 735 int i; 736 737 if (!dma->pg_arr) 738 return; 739 740 for (i = 0; i < dma->num_pages; i++) { 741 if (dma->pg_arr[i]) { 742 dma_free_coherent(&dev->pcidev->dev, CNIC_PAGE_SIZE, 743 dma->pg_arr[i], dma->pg_map_arr[i]); 744 dma->pg_arr[i] = NULL; 745 } 746 } 747 if (dma->pgtbl) { 748 dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size, 749 dma->pgtbl, dma->pgtbl_map); 750 dma->pgtbl = NULL; 751 } 752 kfree(dma->pg_arr); 753 dma->pg_arr = NULL; 754 dma->num_pages = 0; 755 } 756 757 static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma) 758 { 759 int i; 760 __le32 *page_table = (__le32 *) dma->pgtbl; 761 762 for (i = 0; i < dma->num_pages; i++) { 763 /* Each entry needs to be in big endian format. */ 764 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 765 page_table++; 766 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 767 page_table++; 768 } 769 } 770 771 static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma) 772 { 773 int i; 774 __le32 *page_table = (__le32 *) dma->pgtbl; 775 776 for (i = 0; i < dma->num_pages; i++) { 777 /* Each entry needs to be in little endian format. */ 778 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 779 page_table++; 780 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 781 page_table++; 782 } 783 } 784 785 static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma, 786 int pages, int use_pg_tbl) 787 { 788 int i, size; 789 struct cnic_local *cp = dev->cnic_priv; 790 791 size = pages * (sizeof(void *) + sizeof(dma_addr_t)); 792 dma->pg_arr = kzalloc(size, GFP_ATOMIC); 793 if (dma->pg_arr == NULL) 794 return -ENOMEM; 795 796 dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages); 797 dma->num_pages = pages; 798 799 for (i = 0; i < pages; i++) { 800 dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev, 801 CNIC_PAGE_SIZE, 802 &dma->pg_map_arr[i], 803 GFP_ATOMIC); 804 if (dma->pg_arr[i] == NULL) 805 goto error; 806 } 807 if (!use_pg_tbl) 808 return 0; 809 810 dma->pgtbl_size = ((pages * 8) + CNIC_PAGE_SIZE - 1) & 811 ~(CNIC_PAGE_SIZE - 1); 812 dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size, 813 &dma->pgtbl_map, GFP_ATOMIC); 814 if (dma->pgtbl == NULL) 815 goto error; 816 817 cp->setup_pgtbl(dev, dma); 818 819 return 0; 820 821 error: 822 cnic_free_dma(dev, dma); 823 return -ENOMEM; 824 } 825 826 static void cnic_free_context(struct cnic_dev *dev) 827 { 828 struct cnic_local *cp = dev->cnic_priv; 829 int i; 830 831 for (i = 0; i < cp->ctx_blks; i++) { 832 if (cp->ctx_arr[i].ctx) { 833 dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 834 cp->ctx_arr[i].ctx, 835 cp->ctx_arr[i].mapping); 836 cp->ctx_arr[i].ctx = NULL; 837 } 838 } 839 } 840 841 static void __cnic_free_uio_rings(struct cnic_uio_dev *udev) 842 { 843 if (udev->l2_buf) { 844 dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size, 845 udev->l2_buf, udev->l2_buf_map); 846 udev->l2_buf = NULL; 847 } 848 849 if (udev->l2_ring) { 850 dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size, 851 udev->l2_ring, udev->l2_ring_map); 852 udev->l2_ring = NULL; 853 } 854 855 } 856 857 static void __cnic_free_uio(struct cnic_uio_dev *udev) 858 { 859 uio_unregister_device(&udev->cnic_uinfo); 860 861 __cnic_free_uio_rings(udev); 862 863 pci_dev_put(udev->pdev); 864 kfree(udev); 865 } 866 867 static void cnic_free_uio(struct cnic_uio_dev *udev) 868 { 869 if (!udev) 870 return; 871 872 write_lock(&cnic_dev_lock); 873 list_del_init(&udev->list); 874 write_unlock(&cnic_dev_lock); 875 __cnic_free_uio(udev); 876 } 877 878 static void cnic_free_resc(struct cnic_dev *dev) 879 { 880 struct cnic_local *cp = dev->cnic_priv; 881 struct cnic_uio_dev *udev = cp->udev; 882 883 if (udev) { 884 udev->dev = NULL; 885 cp->udev = NULL; 886 if (udev->uio_dev == -1) 887 __cnic_free_uio_rings(udev); 888 } 889 890 cnic_free_context(dev); 891 kfree(cp->ctx_arr); 892 cp->ctx_arr = NULL; 893 cp->ctx_blks = 0; 894 895 cnic_free_dma(dev, &cp->gbl_buf_info); 896 cnic_free_dma(dev, &cp->kwq_info); 897 cnic_free_dma(dev, &cp->kwq_16_data_info); 898 cnic_free_dma(dev, &cp->kcq2.dma); 899 cnic_free_dma(dev, &cp->kcq1.dma); 900 kfree(cp->iscsi_tbl); 901 cp->iscsi_tbl = NULL; 902 kfree(cp->ctx_tbl); 903 cp->ctx_tbl = NULL; 904 905 cnic_free_id_tbl(&cp->fcoe_cid_tbl); 906 cnic_free_id_tbl(&cp->cid_tbl); 907 } 908 909 static int cnic_alloc_context(struct cnic_dev *dev) 910 { 911 struct cnic_local *cp = dev->cnic_priv; 912 913 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 914 int i, k, arr_size; 915 916 cp->ctx_blk_size = CNIC_PAGE_SIZE; 917 cp->cids_per_blk = CNIC_PAGE_SIZE / 128; 918 arr_size = BNX2_MAX_CID / cp->cids_per_blk * 919 sizeof(struct cnic_ctx); 920 cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL); 921 if (cp->ctx_arr == NULL) 922 return -ENOMEM; 923 924 k = 0; 925 for (i = 0; i < 2; i++) { 926 u32 j, reg, off, lo, hi; 927 928 if (i == 0) 929 off = BNX2_PG_CTX_MAP; 930 else 931 off = BNX2_ISCSI_CTX_MAP; 932 933 reg = cnic_reg_rd_ind(dev, off); 934 lo = reg >> 16; 935 hi = reg & 0xffff; 936 for (j = lo; j < hi; j += cp->cids_per_blk, k++) 937 cp->ctx_arr[k].cid = j; 938 } 939 940 cp->ctx_blks = k; 941 if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) { 942 cp->ctx_blks = 0; 943 return -ENOMEM; 944 } 945 946 for (i = 0; i < cp->ctx_blks; i++) { 947 cp->ctx_arr[i].ctx = 948 dma_alloc_coherent(&dev->pcidev->dev, 949 CNIC_PAGE_SIZE, 950 &cp->ctx_arr[i].mapping, 951 GFP_KERNEL); 952 if (cp->ctx_arr[i].ctx == NULL) 953 return -ENOMEM; 954 } 955 } 956 return 0; 957 } 958 959 static u16 cnic_bnx2_next_idx(u16 idx) 960 { 961 return idx + 1; 962 } 963 964 static u16 cnic_bnx2_hw_idx(u16 idx) 965 { 966 return idx; 967 } 968 969 static u16 cnic_bnx2x_next_idx(u16 idx) 970 { 971 idx++; 972 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 973 idx++; 974 975 return idx; 976 } 977 978 static u16 cnic_bnx2x_hw_idx(u16 idx) 979 { 980 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 981 idx++; 982 return idx; 983 } 984 985 static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info, 986 bool use_pg_tbl) 987 { 988 int err, i, use_page_tbl = 0; 989 struct kcqe **kcq; 990 991 if (use_pg_tbl) 992 use_page_tbl = 1; 993 994 err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl); 995 if (err) 996 return err; 997 998 kcq = (struct kcqe **) info->dma.pg_arr; 999 info->kcq = kcq; 1000 1001 info->next_idx = cnic_bnx2_next_idx; 1002 info->hw_idx = cnic_bnx2_hw_idx; 1003 if (use_pg_tbl) 1004 return 0; 1005 1006 info->next_idx = cnic_bnx2x_next_idx; 1007 info->hw_idx = cnic_bnx2x_hw_idx; 1008 1009 for (i = 0; i < KCQ_PAGE_CNT; i++) { 1010 struct bnx2x_bd_chain_next *next = 1011 (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT]; 1012 int j = i + 1; 1013 1014 if (j >= KCQ_PAGE_CNT) 1015 j = 0; 1016 next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32; 1017 next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff; 1018 } 1019 return 0; 1020 } 1021 1022 static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages) 1023 { 1024 struct cnic_local *cp = udev->dev->cnic_priv; 1025 1026 if (udev->l2_ring) 1027 return 0; 1028 1029 udev->l2_ring_size = pages * CNIC_PAGE_SIZE; 1030 udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size, 1031 &udev->l2_ring_map, GFP_KERNEL); 1032 if (!udev->l2_ring) 1033 return -ENOMEM; 1034 1035 udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size; 1036 udev->l2_buf_size = CNIC_PAGE_ALIGN(udev->l2_buf_size); 1037 udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size, 1038 &udev->l2_buf_map, GFP_KERNEL); 1039 if (!udev->l2_buf) { 1040 __cnic_free_uio_rings(udev); 1041 return -ENOMEM; 1042 } 1043 1044 return 0; 1045 1046 } 1047 1048 static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages) 1049 { 1050 struct cnic_local *cp = dev->cnic_priv; 1051 struct cnic_uio_dev *udev; 1052 1053 list_for_each_entry(udev, &cnic_udev_list, list) { 1054 if (udev->pdev == dev->pcidev) { 1055 udev->dev = dev; 1056 if (__cnic_alloc_uio_rings(udev, pages)) { 1057 udev->dev = NULL; 1058 return -ENOMEM; 1059 } 1060 cp->udev = udev; 1061 return 0; 1062 } 1063 } 1064 1065 udev = kzalloc_obj(struct cnic_uio_dev, GFP_ATOMIC); 1066 if (!udev) 1067 return -ENOMEM; 1068 1069 udev->uio_dev = -1; 1070 1071 udev->dev = dev; 1072 udev->pdev = dev->pcidev; 1073 1074 if (__cnic_alloc_uio_rings(udev, pages)) 1075 goto err_udev; 1076 1077 list_add(&udev->list, &cnic_udev_list); 1078 1079 pci_dev_get(udev->pdev); 1080 1081 cp->udev = udev; 1082 1083 return 0; 1084 1085 err_udev: 1086 kfree(udev); 1087 return -ENOMEM; 1088 } 1089 1090 static int cnic_init_uio(struct cnic_dev *dev) 1091 { 1092 struct cnic_local *cp = dev->cnic_priv; 1093 struct cnic_uio_dev *udev = cp->udev; 1094 struct uio_info *uinfo; 1095 int ret = 0; 1096 1097 if (!udev) 1098 return -ENOMEM; 1099 1100 uinfo = &udev->cnic_uinfo; 1101 1102 uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0); 1103 uinfo->mem[0].internal_addr = dev->regview; 1104 uinfo->mem[0].memtype = UIO_MEM_PHYS; 1105 1106 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 1107 uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID + 1108 TX_MAX_TSS_RINGS + 1); 1109 uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen & 1110 CNIC_PAGE_MASK; 1111 uinfo->mem[1].dma_addr = cp->status_blk_map; 1112 if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 1113 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE * 9); 1114 else 1115 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE); 1116 1117 uinfo->name = "bnx2_cnic"; 1118 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 1119 uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0); 1120 1121 uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk & 1122 CNIC_PAGE_MASK; 1123 uinfo->mem[1].dma_addr = cp->status_blk_map; 1124 uinfo->mem[1].size = PAGE_ALIGN(sizeof(*cp->bnx2x_def_status_blk)); 1125 1126 uinfo->name = "bnx2x_cnic"; 1127 } 1128 1129 uinfo->mem[1].dma_device = &dev->pcidev->dev; 1130 uinfo->mem[1].memtype = UIO_MEM_DMA_COHERENT; 1131 1132 uinfo->mem[2].addr = (unsigned long) udev->l2_ring; 1133 uinfo->mem[2].dma_addr = udev->l2_ring_map; 1134 uinfo->mem[2].size = PAGE_ALIGN(udev->l2_ring_size); 1135 uinfo->mem[2].dma_device = &dev->pcidev->dev; 1136 uinfo->mem[2].memtype = UIO_MEM_DMA_COHERENT; 1137 1138 uinfo->mem[3].addr = (unsigned long) udev->l2_buf; 1139 uinfo->mem[3].dma_addr = udev->l2_buf_map; 1140 uinfo->mem[3].size = PAGE_ALIGN(udev->l2_buf_size); 1141 uinfo->mem[3].dma_device = &dev->pcidev->dev; 1142 uinfo->mem[3].memtype = UIO_MEM_DMA_COHERENT; 1143 1144 uinfo->version = CNIC_MODULE_VERSION; 1145 uinfo->irq = UIO_IRQ_CUSTOM; 1146 1147 uinfo->open = cnic_uio_open; 1148 uinfo->release = cnic_uio_close; 1149 1150 if (udev->uio_dev == -1) { 1151 if (!uinfo->priv) { 1152 uinfo->priv = udev; 1153 1154 ret = uio_register_device(&udev->pdev->dev, uinfo); 1155 } 1156 } else { 1157 cnic_init_rings(dev); 1158 } 1159 1160 return ret; 1161 } 1162 1163 static int cnic_alloc_bnx2_resc(struct cnic_dev *dev) 1164 { 1165 struct cnic_local *cp = dev->cnic_priv; 1166 int ret; 1167 1168 ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1); 1169 if (ret) 1170 goto error; 1171 cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr; 1172 1173 ret = cnic_alloc_kcq(dev, &cp->kcq1, true); 1174 if (ret) 1175 goto error; 1176 1177 ret = cnic_alloc_context(dev); 1178 if (ret) 1179 goto error; 1180 1181 ret = cnic_alloc_uio_rings(dev, 2); 1182 if (ret) 1183 goto error; 1184 1185 ret = cnic_init_uio(dev); 1186 if (ret) 1187 goto error; 1188 1189 return 0; 1190 1191 error: 1192 cnic_free_resc(dev); 1193 return ret; 1194 } 1195 1196 static int cnic_alloc_bnx2x_context(struct cnic_dev *dev) 1197 { 1198 struct cnic_local *cp = dev->cnic_priv; 1199 struct bnx2x *bp = netdev_priv(dev->netdev); 1200 int ctx_blk_size = cp->ethdev->ctx_blk_size; 1201 int total_mem, blks, i; 1202 1203 total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space; 1204 blks = total_mem / ctx_blk_size; 1205 if (total_mem % ctx_blk_size) 1206 blks++; 1207 1208 if (blks > cp->ethdev->ctx_tbl_len) 1209 return -ENOMEM; 1210 1211 cp->ctx_arr = kzalloc_objs(struct cnic_ctx, blks); 1212 if (cp->ctx_arr == NULL) 1213 return -ENOMEM; 1214 1215 cp->ctx_blks = blks; 1216 cp->ctx_blk_size = ctx_blk_size; 1217 if (!CHIP_IS_E1(bp)) 1218 cp->ctx_align = 0; 1219 else 1220 cp->ctx_align = ctx_blk_size; 1221 1222 cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE; 1223 1224 for (i = 0; i < blks; i++) { 1225 cp->ctx_arr[i].ctx = 1226 dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 1227 &cp->ctx_arr[i].mapping, 1228 GFP_KERNEL); 1229 if (cp->ctx_arr[i].ctx == NULL) 1230 return -ENOMEM; 1231 1232 if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) { 1233 if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) { 1234 cnic_free_context(dev); 1235 cp->ctx_blk_size += cp->ctx_align; 1236 i = -1; 1237 continue; 1238 } 1239 } 1240 } 1241 return 0; 1242 } 1243 1244 static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev) 1245 { 1246 struct cnic_local *cp = dev->cnic_priv; 1247 struct bnx2x *bp = netdev_priv(dev->netdev); 1248 struct cnic_eth_dev *ethdev = cp->ethdev; 1249 u32 start_cid = ethdev->starting_cid; 1250 int i, j, n, ret, pages; 1251 struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info; 1252 1253 cp->max_cid_space = MAX_ISCSI_TBL_SZ; 1254 cp->iscsi_start_cid = start_cid; 1255 cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ; 1256 1257 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 1258 cp->max_cid_space += dev->max_fcoe_conn; 1259 cp->fcoe_init_cid = ethdev->fcoe_init_cid; 1260 if (!cp->fcoe_init_cid) 1261 cp->fcoe_init_cid = 0x10; 1262 } 1263 1264 cp->iscsi_tbl = kzalloc_objs(struct cnic_iscsi, MAX_ISCSI_TBL_SZ); 1265 if (!cp->iscsi_tbl) 1266 goto error; 1267 1268 cp->ctx_tbl = kzalloc_objs(struct cnic_context, cp->max_cid_space); 1269 if (!cp->ctx_tbl) 1270 goto error; 1271 1272 for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) { 1273 cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i]; 1274 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI; 1275 } 1276 1277 for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++) 1278 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE; 1279 1280 pages = CNIC_PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) / 1281 CNIC_PAGE_SIZE; 1282 1283 ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0); 1284 if (ret) 1285 goto error; 1286 1287 n = CNIC_PAGE_SIZE / CNIC_KWQ16_DATA_SIZE; 1288 for (i = 0, j = 0; i < cp->max_cid_space; i++) { 1289 long off = CNIC_KWQ16_DATA_SIZE * (i % n); 1290 1291 cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off; 1292 cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] + 1293 off; 1294 1295 if ((i % n) == (n - 1)) 1296 j++; 1297 } 1298 1299 ret = cnic_alloc_kcq(dev, &cp->kcq1, false); 1300 if (ret) 1301 goto error; 1302 1303 if (CNIC_SUPPORTS_FCOE(bp)) { 1304 ret = cnic_alloc_kcq(dev, &cp->kcq2, true); 1305 if (ret) 1306 goto error; 1307 } 1308 1309 pages = CNIC_PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / CNIC_PAGE_SIZE; 1310 ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0); 1311 if (ret) 1312 goto error; 1313 1314 ret = cnic_alloc_bnx2x_context(dev); 1315 if (ret) 1316 goto error; 1317 1318 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 1319 return 0; 1320 1321 cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk; 1322 cp->status_blk_map = cp->ethdev->irq_arr[1].status_blk_map; 1323 1324 cp->l2_rx_ring_size = 15; 1325 1326 ret = cnic_alloc_uio_rings(dev, 4); 1327 if (ret) 1328 goto error; 1329 1330 ret = cnic_init_uio(dev); 1331 if (ret) 1332 goto error; 1333 1334 return 0; 1335 1336 error: 1337 cnic_free_resc(dev); 1338 return -ENOMEM; 1339 } 1340 1341 static inline u32 cnic_kwq_avail(struct cnic_local *cp) 1342 { 1343 return cp->max_kwq_idx - 1344 ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx); 1345 } 1346 1347 static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 1348 u32 num_wqes) 1349 { 1350 struct cnic_local *cp = dev->cnic_priv; 1351 struct kwqe *prod_qe; 1352 u16 prod, sw_prod, i; 1353 1354 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 1355 return -EAGAIN; /* bnx2 is down */ 1356 1357 spin_lock_bh(&cp->cnic_ulp_lock); 1358 if (num_wqes > cnic_kwq_avail(cp) && 1359 !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) { 1360 spin_unlock_bh(&cp->cnic_ulp_lock); 1361 return -EAGAIN; 1362 } 1363 1364 clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 1365 1366 prod = cp->kwq_prod_idx; 1367 sw_prod = prod & MAX_KWQ_IDX; 1368 for (i = 0; i < num_wqes; i++) { 1369 prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)]; 1370 memcpy(prod_qe, wqes[i], sizeof(struct kwqe)); 1371 prod++; 1372 sw_prod = prod & MAX_KWQ_IDX; 1373 } 1374 cp->kwq_prod_idx = prod; 1375 1376 CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx); 1377 1378 spin_unlock_bh(&cp->cnic_ulp_lock); 1379 return 0; 1380 } 1381 1382 static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid, 1383 union l5cm_specific_data *l5_data) 1384 { 1385 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1386 dma_addr_t map; 1387 1388 map = ctx->kwqe_data_mapping; 1389 l5_data->phy_address.lo = (u64) map & 0xffffffff; 1390 l5_data->phy_address.hi = (u64) map >> 32; 1391 return ctx->kwqe_data; 1392 } 1393 1394 static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid, 1395 u32 type, union l5cm_specific_data *l5_data) 1396 { 1397 struct cnic_local *cp = dev->cnic_priv; 1398 struct bnx2x *bp = netdev_priv(dev->netdev); 1399 struct l5cm_spe kwqe; 1400 struct kwqe_16 *kwq[1]; 1401 u16 type_16; 1402 int ret; 1403 1404 kwqe.hdr.conn_and_cmd_data = 1405 cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) | 1406 BNX2X_HW_CID(bp, cid))); 1407 1408 type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE; 1409 type_16 |= (bp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) & 1410 SPE_HDR_FUNCTION_ID; 1411 1412 kwqe.hdr.type = cpu_to_le16(type_16); 1413 kwqe.hdr.reserved1 = 0; 1414 kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo); 1415 kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi); 1416 1417 kwq[0] = (struct kwqe_16 *) &kwqe; 1418 1419 spin_lock_bh(&cp->cnic_ulp_lock); 1420 ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1); 1421 spin_unlock_bh(&cp->cnic_ulp_lock); 1422 1423 if (ret == 1) 1424 return 0; 1425 1426 return ret; 1427 } 1428 1429 static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type, 1430 struct kcqe *cqes[], u32 num_cqes) 1431 { 1432 struct cnic_local *cp = dev->cnic_priv; 1433 struct cnic_ulp_ops *ulp_ops; 1434 1435 rcu_read_lock(); 1436 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 1437 if (likely(ulp_ops)) { 1438 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 1439 cqes, num_cqes); 1440 } 1441 rcu_read_unlock(); 1442 } 1443 1444 static void cnic_bnx2x_set_tcp_options(struct cnic_dev *dev, int time_stamps, 1445 int en_tcp_dack) 1446 { 1447 struct bnx2x *bp = netdev_priv(dev->netdev); 1448 u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN; 1449 u16 tstorm_flags = 0; 1450 1451 if (time_stamps) { 1452 xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1453 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1454 } 1455 if (en_tcp_dack) 1456 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_DELAYED_ACK_EN; 1457 1458 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1459 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), xstorm_flags); 1460 1461 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1462 TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), tstorm_flags); 1463 } 1464 1465 static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe) 1466 { 1467 struct cnic_local *cp = dev->cnic_priv; 1468 struct bnx2x *bp = netdev_priv(dev->netdev); 1469 struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe; 1470 int hq_bds, pages; 1471 u32 pfid = bp->pfid; 1472 1473 cp->num_iscsi_tasks = req1->num_tasks_per_conn; 1474 cp->num_ccells = req1->num_ccells_per_conn; 1475 cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE * 1476 cp->num_iscsi_tasks; 1477 cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS * 1478 BNX2X_ISCSI_R2TQE_SIZE; 1479 cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE; 1480 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1481 hq_bds = pages * (CNIC_PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE); 1482 cp->num_cqs = req1->num_cqs; 1483 1484 if (!dev->max_iscsi_conn) 1485 return 0; 1486 1487 /* init Tstorm RAM */ 1488 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1489 req1->rq_num_wqes); 1490 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1491 CNIC_PAGE_SIZE); 1492 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 1493 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1494 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1495 TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1496 req1->num_tasks_per_conn); 1497 1498 /* init Ustorm RAM */ 1499 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1500 USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid), 1501 req1->rq_buffer_size); 1502 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1503 CNIC_PAGE_SIZE); 1504 CNIC_WR8(dev, BAR_USTRORM_INTMEM + 1505 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1506 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1507 USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1508 req1->num_tasks_per_conn); 1509 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1510 req1->rq_num_wqes); 1511 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1512 req1->cq_num_wqes); 1513 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1514 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1515 1516 /* init Xstorm RAM */ 1517 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1518 CNIC_PAGE_SIZE); 1519 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1520 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1521 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 1522 XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1523 req1->num_tasks_per_conn); 1524 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1525 hq_bds); 1526 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid), 1527 req1->num_tasks_per_conn); 1528 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1529 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1530 1531 /* init Cstorm RAM */ 1532 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1533 CNIC_PAGE_SIZE); 1534 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 1535 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1536 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1537 CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1538 req1->num_tasks_per_conn); 1539 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1540 req1->cq_num_wqes); 1541 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1542 hq_bds); 1543 1544 cnic_bnx2x_set_tcp_options(dev, 1545 req1->flags & ISCSI_KWQE_INIT1_TIME_STAMPS_ENABLE, 1546 req1->flags & ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE); 1547 1548 return 0; 1549 } 1550 1551 static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe) 1552 { 1553 struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe; 1554 struct bnx2x *bp = netdev_priv(dev->netdev); 1555 u32 pfid = bp->pfid; 1556 struct iscsi_kcqe kcqe; 1557 struct kcqe *cqes[1]; 1558 1559 memset(&kcqe, 0, sizeof(kcqe)); 1560 if (!dev->max_iscsi_conn) { 1561 kcqe.completion_status = 1562 ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED; 1563 goto done; 1564 } 1565 1566 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1567 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1568 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1569 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1570 req2->error_bit_map[1]); 1571 1572 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1573 USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1574 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1575 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1576 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1577 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1578 req2->error_bit_map[1]); 1579 1580 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1581 CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1582 1583 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1584 1585 done: 1586 kcqe.op_code = ISCSI_KCQE_OPCODE_INIT; 1587 cqes[0] = (struct kcqe *) &kcqe; 1588 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1589 1590 return 0; 1591 } 1592 1593 static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1594 { 1595 struct cnic_local *cp = dev->cnic_priv; 1596 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1597 1598 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) { 1599 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1600 1601 cnic_free_dma(dev, &iscsi->hq_info); 1602 cnic_free_dma(dev, &iscsi->r2tq_info); 1603 cnic_free_dma(dev, &iscsi->task_array_info); 1604 cnic_free_id(&cp->cid_tbl, ctx->cid); 1605 } else { 1606 cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid); 1607 } 1608 1609 ctx->cid = 0; 1610 } 1611 1612 static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1613 { 1614 u32 cid; 1615 int ret, pages; 1616 struct cnic_local *cp = dev->cnic_priv; 1617 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1618 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1619 1620 if (ctx->ulp_proto_id == CNIC_ULP_FCOE) { 1621 cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl); 1622 if (cid == -1) { 1623 ret = -ENOMEM; 1624 goto error; 1625 } 1626 ctx->cid = cid; 1627 return 0; 1628 } 1629 1630 cid = cnic_alloc_new_id(&cp->cid_tbl); 1631 if (cid == -1) { 1632 ret = -ENOMEM; 1633 goto error; 1634 } 1635 1636 ctx->cid = cid; 1637 pages = CNIC_PAGE_ALIGN(cp->task_array_size) / CNIC_PAGE_SIZE; 1638 1639 ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1); 1640 if (ret) 1641 goto error; 1642 1643 pages = CNIC_PAGE_ALIGN(cp->r2tq_size) / CNIC_PAGE_SIZE; 1644 ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1); 1645 if (ret) 1646 goto error; 1647 1648 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1649 ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1); 1650 if (ret) 1651 goto error; 1652 1653 return 0; 1654 1655 error: 1656 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1657 return ret; 1658 } 1659 1660 static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init, 1661 struct regpair *ctx_addr) 1662 { 1663 struct cnic_local *cp = dev->cnic_priv; 1664 struct cnic_eth_dev *ethdev = cp->ethdev; 1665 int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk; 1666 int off = (cid - ethdev->starting_cid) % cp->cids_per_blk; 1667 unsigned long align_off = 0; 1668 dma_addr_t ctx_map; 1669 void *ctx; 1670 1671 if (cp->ctx_align) { 1672 unsigned long mask = cp->ctx_align - 1; 1673 1674 if (cp->ctx_arr[blk].mapping & mask) 1675 align_off = cp->ctx_align - 1676 (cp->ctx_arr[blk].mapping & mask); 1677 } 1678 ctx_map = cp->ctx_arr[blk].mapping + align_off + 1679 (off * BNX2X_CONTEXT_MEM_SIZE); 1680 ctx = cp->ctx_arr[blk].ctx + align_off + 1681 (off * BNX2X_CONTEXT_MEM_SIZE); 1682 if (init) 1683 memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE); 1684 1685 ctx_addr->lo = ctx_map & 0xffffffff; 1686 ctx_addr->hi = (u64) ctx_map >> 32; 1687 return ctx; 1688 } 1689 1690 static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[], 1691 u32 num) 1692 { 1693 struct cnic_local *cp = dev->cnic_priv; 1694 struct bnx2x *bp = netdev_priv(dev->netdev); 1695 struct iscsi_kwqe_conn_offload1 *req1 = 1696 (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1697 struct iscsi_kwqe_conn_offload2 *req2 = 1698 (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1699 struct iscsi_kwqe_conn_offload3 *req3; 1700 struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id]; 1701 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1702 u32 cid = ctx->cid; 1703 u32 hw_cid = BNX2X_HW_CID(bp, cid); 1704 struct iscsi_context *ictx; 1705 struct regpair context_addr; 1706 int i, j, n = 2, n_max; 1707 u8 port = BP_PORT(bp); 1708 1709 ctx->ctx_flags = 0; 1710 if (!req2->num_additional_wqes) 1711 return -EINVAL; 1712 1713 n_max = req2->num_additional_wqes + 2; 1714 1715 ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr); 1716 if (ictx == NULL) 1717 return -ENOMEM; 1718 1719 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1720 1721 ictx->xstorm_ag_context.hq_prod = 1; 1722 1723 ictx->xstorm_st_context.iscsi.first_burst_length = 1724 ISCSI_DEF_FIRST_BURST_LEN; 1725 ictx->xstorm_st_context.iscsi.max_send_pdu_length = 1726 ISCSI_DEF_MAX_RECV_SEG_LEN; 1727 ictx->xstorm_st_context.iscsi.sq_pbl_base.lo = 1728 req1->sq_page_table_addr_lo; 1729 ictx->xstorm_st_context.iscsi.sq_pbl_base.hi = 1730 req1->sq_page_table_addr_hi; 1731 ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi; 1732 ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo; 1733 ictx->xstorm_st_context.iscsi.hq_pbl_base.lo = 1734 iscsi->hq_info.pgtbl_map & 0xffffffff; 1735 ictx->xstorm_st_context.iscsi.hq_pbl_base.hi = 1736 (u64) iscsi->hq_info.pgtbl_map >> 32; 1737 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo = 1738 iscsi->hq_info.pgtbl[0]; 1739 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi = 1740 iscsi->hq_info.pgtbl[1]; 1741 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo = 1742 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1743 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi = 1744 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1745 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo = 1746 iscsi->r2tq_info.pgtbl[0]; 1747 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi = 1748 iscsi->r2tq_info.pgtbl[1]; 1749 ictx->xstorm_st_context.iscsi.task_pbl_base.lo = 1750 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1751 ictx->xstorm_st_context.iscsi.task_pbl_base.hi = 1752 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1753 ictx->xstorm_st_context.iscsi.task_pbl_cache_idx = 1754 BNX2X_ISCSI_PBL_NOT_CACHED; 1755 ictx->xstorm_st_context.iscsi.flags.flags |= 1756 XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA; 1757 ictx->xstorm_st_context.iscsi.flags.flags |= 1758 XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T; 1759 ictx->xstorm_st_context.common.ethernet.reserved_vlan_type = 1760 ETH_P_8021Q; 1761 if (BNX2X_CHIP_IS_E2_PLUS(bp) && 1762 bp->common.chip_port_mode == CHIP_2_PORT_MODE) { 1763 1764 port = 0; 1765 } 1766 ictx->xstorm_st_context.common.flags = 1767 1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT; 1768 ictx->xstorm_st_context.common.flags = 1769 port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT; 1770 1771 ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE; 1772 /* TSTORM requires the base address of RQ DB & not PTE */ 1773 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo = 1774 req2->rq_page_table_addr_lo & CNIC_PAGE_MASK; 1775 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi = 1776 req2->rq_page_table_addr_hi; 1777 ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id; 1778 ictx->tstorm_st_context.tcp.cwnd = 0x5A8; 1779 ictx->tstorm_st_context.tcp.flags2 |= 1780 TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN; 1781 ictx->tstorm_st_context.tcp.ooo_support_mode = 1782 TCP_TSTORM_OOO_DROP_AND_PROC_ACK; 1783 1784 ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG; 1785 1786 ictx->ustorm_st_context.ring.rq.pbl_base.lo = 1787 req2->rq_page_table_addr_lo; 1788 ictx->ustorm_st_context.ring.rq.pbl_base.hi = 1789 req2->rq_page_table_addr_hi; 1790 ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi; 1791 ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo; 1792 ictx->ustorm_st_context.ring.r2tq.pbl_base.lo = 1793 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1794 ictx->ustorm_st_context.ring.r2tq.pbl_base.hi = 1795 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1796 ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo = 1797 iscsi->r2tq_info.pgtbl[0]; 1798 ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi = 1799 iscsi->r2tq_info.pgtbl[1]; 1800 ictx->ustorm_st_context.ring.cq_pbl_base.lo = 1801 req1->cq_page_table_addr_lo; 1802 ictx->ustorm_st_context.ring.cq_pbl_base.hi = 1803 req1->cq_page_table_addr_hi; 1804 ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN; 1805 ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi; 1806 ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo; 1807 ictx->ustorm_st_context.task_pbe_cache_index = 1808 BNX2X_ISCSI_PBL_NOT_CACHED; 1809 ictx->ustorm_st_context.task_pdu_cache_index = 1810 BNX2X_ISCSI_PDU_HEADER_NOT_CACHED; 1811 1812 for (i = 1, j = 1; i < cp->num_cqs; i++, j++) { 1813 if (j == 3) { 1814 if (n >= n_max) 1815 break; 1816 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1817 j = 0; 1818 } 1819 ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN; 1820 ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo = 1821 req3->qp_first_pte[j].hi; 1822 ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi = 1823 req3->qp_first_pte[j].lo; 1824 } 1825 1826 ictx->ustorm_st_context.task_pbl_base.lo = 1827 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1828 ictx->ustorm_st_context.task_pbl_base.hi = 1829 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1830 ictx->ustorm_st_context.tce_phy_addr.lo = 1831 iscsi->task_array_info.pgtbl[0]; 1832 ictx->ustorm_st_context.tce_phy_addr.hi = 1833 iscsi->task_array_info.pgtbl[1]; 1834 ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1835 ictx->ustorm_st_context.num_cqs = cp->num_cqs; 1836 ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN; 1837 ictx->ustorm_st_context.negotiated_rx_and_flags |= 1838 ISCSI_DEF_MAX_BURST_LEN; 1839 ictx->ustorm_st_context.negotiated_rx |= 1840 ISCSI_DEFAULT_MAX_OUTSTANDING_R2T << 1841 USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT; 1842 1843 ictx->cstorm_st_context.hq_pbl_base.lo = 1844 iscsi->hq_info.pgtbl_map & 0xffffffff; 1845 ictx->cstorm_st_context.hq_pbl_base.hi = 1846 (u64) iscsi->hq_info.pgtbl_map >> 32; 1847 ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0]; 1848 ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1]; 1849 ictx->cstorm_st_context.task_pbl_base.lo = 1850 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1851 ictx->cstorm_st_context.task_pbl_base.hi = 1852 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1853 /* CSTORM and USTORM initialization is different, CSTORM requires 1854 * CQ DB base & not PTE addr */ 1855 ictx->cstorm_st_context.cq_db_base.lo = 1856 req1->cq_page_table_addr_lo & CNIC_PAGE_MASK; 1857 ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi; 1858 ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1859 ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1; 1860 for (i = 0; i < cp->num_cqs; i++) { 1861 ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] = 1862 ISCSI_INITIAL_SN; 1863 ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] = 1864 ISCSI_INITIAL_SN; 1865 } 1866 1867 ictx->xstorm_ag_context.cdu_reserved = 1868 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 1869 ISCSI_CONNECTION_TYPE); 1870 ictx->ustorm_ag_context.cdu_usage = 1871 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 1872 ISCSI_CONNECTION_TYPE); 1873 return 0; 1874 1875 } 1876 1877 static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 1878 u32 num, int *work) 1879 { 1880 struct iscsi_kwqe_conn_offload1 *req1; 1881 struct iscsi_kwqe_conn_offload2 *req2; 1882 struct cnic_local *cp = dev->cnic_priv; 1883 struct bnx2x *bp = netdev_priv(dev->netdev); 1884 struct cnic_context *ctx; 1885 struct iscsi_kcqe kcqe; 1886 struct kcqe *cqes[1]; 1887 u32 l5_cid; 1888 int ret = 0; 1889 1890 if (num < 2) { 1891 *work = num; 1892 return -EINVAL; 1893 } 1894 1895 req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1896 req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1897 if ((num - 2) < req2->num_additional_wqes) { 1898 *work = num; 1899 return -EINVAL; 1900 } 1901 *work = 2 + req2->num_additional_wqes; 1902 1903 l5_cid = req1->iscsi_conn_id; 1904 if (l5_cid >= MAX_ISCSI_TBL_SZ) 1905 return -EINVAL; 1906 1907 memset(&kcqe, 0, sizeof(kcqe)); 1908 kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN; 1909 kcqe.iscsi_conn_id = l5_cid; 1910 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 1911 1912 ctx = &cp->ctx_tbl[l5_cid]; 1913 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) { 1914 kcqe.completion_status = 1915 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY; 1916 goto done; 1917 } 1918 1919 if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) { 1920 atomic_dec(&cp->iscsi_conn); 1921 goto done; 1922 } 1923 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 1924 if (ret) { 1925 atomic_dec(&cp->iscsi_conn); 1926 goto done; 1927 } 1928 ret = cnic_setup_bnx2x_ctx(dev, wqes, num); 1929 if (ret < 0) { 1930 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1931 atomic_dec(&cp->iscsi_conn); 1932 goto done; 1933 } 1934 1935 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1936 kcqe.iscsi_conn_context_id = BNX2X_HW_CID(bp, cp->ctx_tbl[l5_cid].cid); 1937 1938 done: 1939 cqes[0] = (struct kcqe *) &kcqe; 1940 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1941 return 0; 1942 } 1943 1944 1945 static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe) 1946 { 1947 struct cnic_local *cp = dev->cnic_priv; 1948 struct iscsi_kwqe_conn_update *req = 1949 (struct iscsi_kwqe_conn_update *) kwqe; 1950 void *data; 1951 union l5cm_specific_data l5_data; 1952 u32 l5_cid, cid = BNX2X_SW_CID(req->context_id); 1953 int ret; 1954 1955 if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0) 1956 return -EINVAL; 1957 1958 data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 1959 if (!data) 1960 return -ENOMEM; 1961 1962 memcpy(data, kwqe, sizeof(struct kwqe)); 1963 1964 ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, 1965 req->context_id, ISCSI_CONNECTION_TYPE, &l5_data); 1966 return ret; 1967 } 1968 1969 static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid) 1970 { 1971 struct cnic_local *cp = dev->cnic_priv; 1972 struct bnx2x *bp = netdev_priv(dev->netdev); 1973 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1974 union l5cm_specific_data l5_data; 1975 int ret; 1976 u32 hw_cid; 1977 1978 init_waitqueue_head(&ctx->waitq); 1979 ctx->wait_cond = 0; 1980 memset(&l5_data, 0, sizeof(l5_data)); 1981 hw_cid = BNX2X_HW_CID(bp, ctx->cid); 1982 1983 ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 1984 hw_cid, NONE_CONNECTION_TYPE, &l5_data); 1985 1986 if (ret == 0) { 1987 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 1988 if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags))) 1989 return -EBUSY; 1990 } 1991 1992 return 0; 1993 } 1994 1995 static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 1996 { 1997 struct cnic_local *cp = dev->cnic_priv; 1998 struct iscsi_kwqe_conn_destroy *req = 1999 (struct iscsi_kwqe_conn_destroy *) kwqe; 2000 u32 l5_cid = req->reserved0; 2001 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2002 int ret = 0; 2003 struct iscsi_kcqe kcqe; 2004 struct kcqe *cqes[1]; 2005 2006 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2007 goto skip_cfc_delete; 2008 2009 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 2010 unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies; 2011 2012 if (delta > (2 * HZ)) 2013 delta = 0; 2014 2015 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2016 queue_delayed_work(cnic_wq, &cp->delete_task, delta); 2017 goto destroy_reply; 2018 } 2019 2020 ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid); 2021 2022 skip_cfc_delete: 2023 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2024 2025 if (!ret) { 2026 atomic_dec(&cp->iscsi_conn); 2027 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2028 } 2029 2030 destroy_reply: 2031 memset(&kcqe, 0, sizeof(kcqe)); 2032 kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN; 2033 kcqe.iscsi_conn_id = l5_cid; 2034 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 2035 kcqe.iscsi_conn_context_id = req->context_id; 2036 2037 cqes[0] = (struct kcqe *) &kcqe; 2038 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 2039 2040 return 0; 2041 } 2042 2043 static void cnic_init_storm_conn_bufs(struct cnic_dev *dev, 2044 struct l4_kwq_connect_req1 *kwqe1, 2045 struct l4_kwq_connect_req3 *kwqe3, 2046 struct l5cm_active_conn_buffer *conn_buf) 2047 { 2048 struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf; 2049 struct l5cm_xstorm_conn_buffer *xstorm_buf = 2050 &conn_buf->xstorm_conn_buffer; 2051 struct l5cm_tstorm_conn_buffer *tstorm_buf = 2052 &conn_buf->tstorm_conn_buffer; 2053 struct regpair context_addr; 2054 u32 cid = BNX2X_SW_CID(kwqe1->cid); 2055 struct in6_addr src_ip, dst_ip; 2056 int i; 2057 u32 *addrp; 2058 2059 addrp = (u32 *) &conn_addr->local_ip_addr; 2060 for (i = 0; i < 4; i++, addrp++) 2061 src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2062 2063 addrp = (u32 *) &conn_addr->remote_ip_addr; 2064 for (i = 0; i < 4; i++, addrp++) 2065 dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2066 2067 cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr); 2068 2069 xstorm_buf->context_addr.hi = context_addr.hi; 2070 xstorm_buf->context_addr.lo = context_addr.lo; 2071 xstorm_buf->mss = 0xffff; 2072 xstorm_buf->rcv_buf = kwqe3->rcv_buf; 2073 if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE) 2074 xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE; 2075 xstorm_buf->pseudo_header_checksum = 2076 swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0)); 2077 2078 if (kwqe3->ka_timeout) { 2079 tstorm_buf->ka_enable = 1; 2080 tstorm_buf->ka_timeout = kwqe3->ka_timeout; 2081 tstorm_buf->ka_interval = kwqe3->ka_interval; 2082 tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count; 2083 } 2084 tstorm_buf->max_rt_time = 0xffffffff; 2085 } 2086 2087 static void cnic_init_bnx2x_mac(struct cnic_dev *dev) 2088 { 2089 struct bnx2x *bp = netdev_priv(dev->netdev); 2090 u32 pfid = bp->pfid; 2091 u8 *mac = dev->mac_addr; 2092 2093 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2094 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]); 2095 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2096 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]); 2097 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2098 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]); 2099 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2100 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]); 2101 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2102 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]); 2103 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2104 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]); 2105 2106 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2107 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]); 2108 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2109 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2110 mac[4]); 2111 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2112 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]); 2113 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2114 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2115 mac[2]); 2116 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2117 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]); 2118 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2119 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2120 mac[0]); 2121 } 2122 2123 static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[], 2124 u32 num, int *work) 2125 { 2126 struct cnic_local *cp = dev->cnic_priv; 2127 struct bnx2x *bp = netdev_priv(dev->netdev); 2128 struct l4_kwq_connect_req1 *kwqe1 = 2129 (struct l4_kwq_connect_req1 *) wqes[0]; 2130 struct l4_kwq_connect_req3 *kwqe3; 2131 struct l5cm_active_conn_buffer *conn_buf; 2132 struct l5cm_conn_addr_params *conn_addr; 2133 union l5cm_specific_data l5_data; 2134 u32 l5_cid = kwqe1->pg_cid; 2135 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 2136 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2137 int ret; 2138 2139 if (num < 2) { 2140 *work = num; 2141 return -EINVAL; 2142 } 2143 2144 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) 2145 *work = 3; 2146 else 2147 *work = 2; 2148 2149 if (num < *work) { 2150 *work = num; 2151 return -EINVAL; 2152 } 2153 2154 if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) { 2155 netdev_err(dev->netdev, "conn_buf size too big\n"); 2156 return -ENOMEM; 2157 } 2158 conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2159 if (!conn_buf) 2160 return -ENOMEM; 2161 2162 memset(conn_buf, 0, sizeof(*conn_buf)); 2163 2164 conn_addr = &conn_buf->conn_addr_buf; 2165 conn_addr->remote_addr_0 = csk->ha[0]; 2166 conn_addr->remote_addr_1 = csk->ha[1]; 2167 conn_addr->remote_addr_2 = csk->ha[2]; 2168 conn_addr->remote_addr_3 = csk->ha[3]; 2169 conn_addr->remote_addr_4 = csk->ha[4]; 2170 conn_addr->remote_addr_5 = csk->ha[5]; 2171 2172 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) { 2173 struct l4_kwq_connect_req2 *kwqe2 = 2174 (struct l4_kwq_connect_req2 *) wqes[1]; 2175 2176 conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4; 2177 conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3; 2178 conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2; 2179 2180 conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4; 2181 conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3; 2182 conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2; 2183 conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION; 2184 } 2185 kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1]; 2186 2187 conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip; 2188 conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip; 2189 conn_addr->local_tcp_port = kwqe1->src_port; 2190 conn_addr->remote_tcp_port = kwqe1->dst_port; 2191 2192 conn_addr->pmtu = kwqe3->pmtu; 2193 cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf); 2194 2195 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 2196 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(bp->pfid), csk->vlan_id); 2197 2198 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT, 2199 kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2200 if (!ret) 2201 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2202 2203 return ret; 2204 } 2205 2206 static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe) 2207 { 2208 struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe; 2209 union l5cm_specific_data l5_data; 2210 int ret; 2211 2212 memset(&l5_data, 0, sizeof(l5_data)); 2213 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE, 2214 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2215 return ret; 2216 } 2217 2218 static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe) 2219 { 2220 struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe; 2221 union l5cm_specific_data l5_data; 2222 int ret; 2223 2224 memset(&l5_data, 0, sizeof(l5_data)); 2225 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT, 2226 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2227 return ret; 2228 } 2229 static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2230 { 2231 struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe; 2232 struct l4_kcq kcqe; 2233 struct kcqe *cqes[1]; 2234 2235 memset(&kcqe, 0, sizeof(kcqe)); 2236 kcqe.pg_host_opaque = req->host_opaque; 2237 kcqe.pg_cid = req->host_opaque; 2238 kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG; 2239 cqes[0] = (struct kcqe *) &kcqe; 2240 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2241 return 0; 2242 } 2243 2244 static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2245 { 2246 struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe; 2247 struct l4_kcq kcqe; 2248 struct kcqe *cqes[1]; 2249 2250 memset(&kcqe, 0, sizeof(kcqe)); 2251 kcqe.pg_host_opaque = req->pg_host_opaque; 2252 kcqe.pg_cid = req->pg_cid; 2253 kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG; 2254 cqes[0] = (struct kcqe *) &kcqe; 2255 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2256 return 0; 2257 } 2258 2259 static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe) 2260 { 2261 struct fcoe_kwqe_stat *req; 2262 struct fcoe_stat_ramrod_params *fcoe_stat; 2263 union l5cm_specific_data l5_data; 2264 struct cnic_local *cp = dev->cnic_priv; 2265 struct bnx2x *bp = netdev_priv(dev->netdev); 2266 int ret; 2267 u32 cid; 2268 2269 req = (struct fcoe_kwqe_stat *) kwqe; 2270 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2271 2272 fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2273 if (!fcoe_stat) 2274 return -ENOMEM; 2275 2276 memset(fcoe_stat, 0, sizeof(*fcoe_stat)); 2277 memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req)); 2278 2279 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid, 2280 FCOE_CONNECTION_TYPE, &l5_data); 2281 return ret; 2282 } 2283 2284 static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[], 2285 u32 num, int *work) 2286 { 2287 int ret; 2288 struct cnic_local *cp = dev->cnic_priv; 2289 struct bnx2x *bp = netdev_priv(dev->netdev); 2290 u32 cid; 2291 struct fcoe_init_ramrod_params *fcoe_init; 2292 struct fcoe_kwqe_init1 *req1; 2293 struct fcoe_kwqe_init2 *req2; 2294 struct fcoe_kwqe_init3 *req3; 2295 union l5cm_specific_data l5_data; 2296 2297 if (num < 3) { 2298 *work = num; 2299 return -EINVAL; 2300 } 2301 req1 = (struct fcoe_kwqe_init1 *) wqes[0]; 2302 req2 = (struct fcoe_kwqe_init2 *) wqes[1]; 2303 req3 = (struct fcoe_kwqe_init3 *) wqes[2]; 2304 if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) { 2305 *work = 1; 2306 return -EINVAL; 2307 } 2308 if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) { 2309 *work = 2; 2310 return -EINVAL; 2311 } 2312 2313 if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) { 2314 netdev_err(dev->netdev, "fcoe_init size too big\n"); 2315 return -ENOMEM; 2316 } 2317 fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2318 if (!fcoe_init) 2319 return -ENOMEM; 2320 2321 memset(fcoe_init, 0, sizeof(*fcoe_init)); 2322 memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1)); 2323 memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2)); 2324 memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3)); 2325 fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff; 2326 fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32; 2327 fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages; 2328 2329 fcoe_init->sb_num = cp->status_blk_num; 2330 fcoe_init->eq_prod = MAX_KCQ_IDX; 2331 fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS; 2332 cp->kcq2.sw_prod_idx = 0; 2333 2334 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2335 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid, 2336 FCOE_CONNECTION_TYPE, &l5_data); 2337 *work = 3; 2338 return ret; 2339 } 2340 2341 static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 2342 u32 num, int *work) 2343 { 2344 int ret = 0; 2345 u32 cid = -1, l5_cid; 2346 struct cnic_local *cp = dev->cnic_priv; 2347 struct bnx2x *bp = netdev_priv(dev->netdev); 2348 struct fcoe_kwqe_conn_offload1 *req1; 2349 struct fcoe_kwqe_conn_offload2 *req2; 2350 struct fcoe_kwqe_conn_offload3 *req3; 2351 struct fcoe_kwqe_conn_offload4 *req4; 2352 struct fcoe_conn_offload_ramrod_params *fcoe_offload; 2353 struct cnic_context *ctx; 2354 struct fcoe_context *fctx; 2355 struct regpair ctx_addr; 2356 union l5cm_specific_data l5_data; 2357 struct fcoe_kcqe kcqe; 2358 struct kcqe *cqes[1]; 2359 2360 if (num < 4) { 2361 *work = num; 2362 return -EINVAL; 2363 } 2364 req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0]; 2365 req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1]; 2366 req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2]; 2367 req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3]; 2368 2369 *work = 4; 2370 2371 l5_cid = req1->fcoe_conn_id; 2372 if (l5_cid >= dev->max_fcoe_conn) 2373 goto err_reply; 2374 2375 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2376 2377 ctx = &cp->ctx_tbl[l5_cid]; 2378 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2379 goto err_reply; 2380 2381 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 2382 if (ret) { 2383 ret = 0; 2384 goto err_reply; 2385 } 2386 cid = ctx->cid; 2387 2388 fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr); 2389 if (fctx) { 2390 u32 hw_cid = BNX2X_HW_CID(bp, cid); 2391 u32 val; 2392 2393 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 2394 FCOE_CONNECTION_TYPE); 2395 fctx->xstorm_ag_context.cdu_reserved = val; 2396 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 2397 FCOE_CONNECTION_TYPE); 2398 fctx->ustorm_ag_context.cdu_usage = val; 2399 } 2400 if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) { 2401 netdev_err(dev->netdev, "fcoe_offload size too big\n"); 2402 goto err_reply; 2403 } 2404 fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2405 if (!fcoe_offload) 2406 goto err_reply; 2407 2408 memset(fcoe_offload, 0, sizeof(*fcoe_offload)); 2409 memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1)); 2410 memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2)); 2411 memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3)); 2412 memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4)); 2413 2414 cid = BNX2X_HW_CID(bp, cid); 2415 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid, 2416 FCOE_CONNECTION_TYPE, &l5_data); 2417 if (!ret) 2418 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2419 2420 return ret; 2421 2422 err_reply: 2423 if (cid != -1) 2424 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2425 2426 memset(&kcqe, 0, sizeof(kcqe)); 2427 kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN; 2428 kcqe.fcoe_conn_id = req1->fcoe_conn_id; 2429 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 2430 2431 cqes[0] = (struct kcqe *) &kcqe; 2432 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2433 return ret; 2434 } 2435 2436 static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe) 2437 { 2438 struct fcoe_kwqe_conn_enable_disable *req; 2439 struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable; 2440 union l5cm_specific_data l5_data; 2441 int ret; 2442 u32 cid, l5_cid; 2443 struct cnic_local *cp = dev->cnic_priv; 2444 2445 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2446 cid = req->context_id; 2447 l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE; 2448 2449 if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) { 2450 netdev_err(dev->netdev, "fcoe_enable size too big\n"); 2451 return -ENOMEM; 2452 } 2453 fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2454 if (!fcoe_enable) 2455 return -ENOMEM; 2456 2457 memset(fcoe_enable, 0, sizeof(*fcoe_enable)); 2458 memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req)); 2459 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid, 2460 FCOE_CONNECTION_TYPE, &l5_data); 2461 return ret; 2462 } 2463 2464 static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe) 2465 { 2466 struct fcoe_kwqe_conn_enable_disable *req; 2467 struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable; 2468 union l5cm_specific_data l5_data; 2469 int ret; 2470 u32 cid, l5_cid; 2471 struct cnic_local *cp = dev->cnic_priv; 2472 2473 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2474 cid = req->context_id; 2475 l5_cid = req->conn_id; 2476 if (l5_cid >= dev->max_fcoe_conn) 2477 return -EINVAL; 2478 2479 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2480 2481 if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) { 2482 netdev_err(dev->netdev, "fcoe_disable size too big\n"); 2483 return -ENOMEM; 2484 } 2485 fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2486 if (!fcoe_disable) 2487 return -ENOMEM; 2488 2489 memset(fcoe_disable, 0, sizeof(*fcoe_disable)); 2490 memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req)); 2491 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid, 2492 FCOE_CONNECTION_TYPE, &l5_data); 2493 return ret; 2494 } 2495 2496 static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2497 { 2498 struct fcoe_kwqe_conn_destroy *req; 2499 union l5cm_specific_data l5_data; 2500 int ret; 2501 u32 cid, l5_cid; 2502 struct cnic_local *cp = dev->cnic_priv; 2503 struct cnic_context *ctx; 2504 struct fcoe_kcqe kcqe; 2505 struct kcqe *cqes[1]; 2506 2507 req = (struct fcoe_kwqe_conn_destroy *) kwqe; 2508 cid = req->context_id; 2509 l5_cid = req->conn_id; 2510 if (l5_cid >= dev->max_fcoe_conn) 2511 return -EINVAL; 2512 2513 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2514 2515 ctx = &cp->ctx_tbl[l5_cid]; 2516 2517 init_waitqueue_head(&ctx->waitq); 2518 ctx->wait_cond = 0; 2519 2520 memset(&kcqe, 0, sizeof(kcqe)); 2521 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR; 2522 memset(&l5_data, 0, sizeof(l5_data)); 2523 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid, 2524 FCOE_CONNECTION_TYPE, &l5_data); 2525 if (ret == 0) { 2526 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 2527 if (ctx->wait_cond) 2528 kcqe.completion_status = 0; 2529 } 2530 2531 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2532 queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000)); 2533 2534 kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN; 2535 kcqe.fcoe_conn_id = req->conn_id; 2536 kcqe.fcoe_conn_context_id = cid; 2537 2538 cqes[0] = (struct kcqe *) &kcqe; 2539 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2540 return ret; 2541 } 2542 2543 static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid) 2544 { 2545 struct cnic_local *cp = dev->cnic_priv; 2546 u32 i; 2547 2548 for (i = start_cid; i < cp->max_cid_space; i++) { 2549 struct cnic_context *ctx = &cp->ctx_tbl[i]; 2550 int j; 2551 2552 while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 2553 msleep(10); 2554 2555 for (j = 0; j < 5; j++) { 2556 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2557 break; 2558 msleep(20); 2559 } 2560 2561 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2562 netdev_warn(dev->netdev, "CID %x not deleted\n", 2563 ctx->cid); 2564 } 2565 } 2566 2567 static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2568 { 2569 union l5cm_specific_data l5_data; 2570 struct cnic_local *cp = dev->cnic_priv; 2571 struct bnx2x *bp = netdev_priv(dev->netdev); 2572 int ret; 2573 u32 cid; 2574 2575 cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ); 2576 2577 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2578 2579 memset(&l5_data, 0, sizeof(l5_data)); 2580 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid, 2581 FCOE_CONNECTION_TYPE, &l5_data); 2582 return ret; 2583 } 2584 2585 static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe) 2586 { 2587 struct cnic_local *cp = dev->cnic_priv; 2588 struct kcqe kcqe; 2589 struct kcqe *cqes[1]; 2590 u32 cid; 2591 u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2592 u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK; 2593 u32 kcqe_op; 2594 int ulp_type; 2595 2596 cid = kwqe->kwqe_info0; 2597 memset(&kcqe, 0, sizeof(kcqe)); 2598 2599 if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) { 2600 u32 l5_cid = 0; 2601 2602 ulp_type = CNIC_ULP_FCOE; 2603 if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) { 2604 struct fcoe_kwqe_conn_enable_disable *req; 2605 2606 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2607 kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN; 2608 cid = req->context_id; 2609 l5_cid = req->conn_id; 2610 } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) { 2611 kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC; 2612 } else { 2613 return; 2614 } 2615 kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT; 2616 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE; 2617 kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2618 kcqe.kcqe_info2 = cid; 2619 kcqe.kcqe_info0 = l5_cid; 2620 2621 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) { 2622 ulp_type = CNIC_ULP_ISCSI; 2623 if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN) 2624 cid = kwqe->kwqe_info1; 2625 2626 kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT; 2627 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI; 2628 kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR; 2629 kcqe.kcqe_info2 = cid; 2630 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0); 2631 2632 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) { 2633 struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe; 2634 2635 ulp_type = CNIC_ULP_L4; 2636 if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1) 2637 kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE; 2638 else if (opcode == L4_KWQE_OPCODE_VALUE_RESET) 2639 kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP; 2640 else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE) 2641 kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 2642 else 2643 return; 2644 2645 kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) | 2646 KCQE_FLAGS_LAYER_MASK_L4; 2647 l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2648 l4kcqe->cid = cid; 2649 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id); 2650 } else { 2651 return; 2652 } 2653 2654 cqes[0] = &kcqe; 2655 cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1); 2656 } 2657 2658 static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev, 2659 struct kwqe *wqes[], u32 num_wqes) 2660 { 2661 int i, work, ret; 2662 u32 opcode; 2663 struct kwqe *kwqe; 2664 2665 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2666 return -EAGAIN; /* bnx2 is down */ 2667 2668 for (i = 0; i < num_wqes; ) { 2669 kwqe = wqes[i]; 2670 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2671 work = 1; 2672 2673 switch (opcode) { 2674 case ISCSI_KWQE_OPCODE_INIT1: 2675 ret = cnic_bnx2x_iscsi_init1(dev, kwqe); 2676 break; 2677 case ISCSI_KWQE_OPCODE_INIT2: 2678 ret = cnic_bnx2x_iscsi_init2(dev, kwqe); 2679 break; 2680 case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1: 2681 ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i], 2682 num_wqes - i, &work); 2683 break; 2684 case ISCSI_KWQE_OPCODE_UPDATE_CONN: 2685 ret = cnic_bnx2x_iscsi_update(dev, kwqe); 2686 break; 2687 case ISCSI_KWQE_OPCODE_DESTROY_CONN: 2688 ret = cnic_bnx2x_iscsi_destroy(dev, kwqe); 2689 break; 2690 case L4_KWQE_OPCODE_VALUE_CONNECT1: 2691 ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i, 2692 &work); 2693 break; 2694 case L4_KWQE_OPCODE_VALUE_CLOSE: 2695 ret = cnic_bnx2x_close(dev, kwqe); 2696 break; 2697 case L4_KWQE_OPCODE_VALUE_RESET: 2698 ret = cnic_bnx2x_reset(dev, kwqe); 2699 break; 2700 case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG: 2701 ret = cnic_bnx2x_offload_pg(dev, kwqe); 2702 break; 2703 case L4_KWQE_OPCODE_VALUE_UPDATE_PG: 2704 ret = cnic_bnx2x_update_pg(dev, kwqe); 2705 break; 2706 case L4_KWQE_OPCODE_VALUE_UPLOAD_PG: 2707 ret = 0; 2708 break; 2709 default: 2710 ret = 0; 2711 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2712 opcode); 2713 break; 2714 } 2715 if (ret < 0) { 2716 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2717 opcode); 2718 2719 /* Possibly bnx2x parity error, send completion 2720 * to ulp drivers with error code to speed up 2721 * cleanup and reset recovery. 2722 */ 2723 if (ret == -EIO || ret == -EAGAIN) 2724 cnic_bnx2x_kwqe_err(dev, kwqe); 2725 } 2726 i += work; 2727 } 2728 return 0; 2729 } 2730 2731 static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev, 2732 struct kwqe *wqes[], u32 num_wqes) 2733 { 2734 struct bnx2x *bp = netdev_priv(dev->netdev); 2735 int i, work, ret; 2736 u32 opcode; 2737 struct kwqe *kwqe; 2738 2739 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2740 return -EAGAIN; /* bnx2 is down */ 2741 2742 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 2743 return -EINVAL; 2744 2745 for (i = 0; i < num_wqes; ) { 2746 kwqe = wqes[i]; 2747 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2748 work = 1; 2749 2750 switch (opcode) { 2751 case FCOE_KWQE_OPCODE_INIT1: 2752 ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i], 2753 num_wqes - i, &work); 2754 break; 2755 case FCOE_KWQE_OPCODE_OFFLOAD_CONN1: 2756 ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i], 2757 num_wqes - i, &work); 2758 break; 2759 case FCOE_KWQE_OPCODE_ENABLE_CONN: 2760 ret = cnic_bnx2x_fcoe_enable(dev, kwqe); 2761 break; 2762 case FCOE_KWQE_OPCODE_DISABLE_CONN: 2763 ret = cnic_bnx2x_fcoe_disable(dev, kwqe); 2764 break; 2765 case FCOE_KWQE_OPCODE_DESTROY_CONN: 2766 ret = cnic_bnx2x_fcoe_destroy(dev, kwqe); 2767 break; 2768 case FCOE_KWQE_OPCODE_DESTROY: 2769 ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe); 2770 break; 2771 case FCOE_KWQE_OPCODE_STAT: 2772 ret = cnic_bnx2x_fcoe_stat(dev, kwqe); 2773 break; 2774 default: 2775 ret = 0; 2776 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2777 opcode); 2778 break; 2779 } 2780 if (ret < 0) { 2781 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2782 opcode); 2783 2784 /* Possibly bnx2x parity error, send completion 2785 * to ulp drivers with error code to speed up 2786 * cleanup and reset recovery. 2787 */ 2788 if (ret == -EIO || ret == -EAGAIN) 2789 cnic_bnx2x_kwqe_err(dev, kwqe); 2790 } 2791 i += work; 2792 } 2793 return 0; 2794 } 2795 2796 static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 2797 u32 num_wqes) 2798 { 2799 int ret = -EINVAL; 2800 u32 layer_code; 2801 2802 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2803 return -EAGAIN; /* bnx2x is down */ 2804 2805 if (!num_wqes) 2806 return 0; 2807 2808 layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK; 2809 switch (layer_code) { 2810 case KWQE_FLAGS_LAYER_MASK_L5_ISCSI: 2811 case KWQE_FLAGS_LAYER_MASK_L4: 2812 case KWQE_FLAGS_LAYER_MASK_L2: 2813 ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes); 2814 break; 2815 2816 case KWQE_FLAGS_LAYER_MASK_L5_FCOE: 2817 ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes); 2818 break; 2819 } 2820 return ret; 2821 } 2822 2823 static inline u32 cnic_get_kcqe_layer_mask(u32 opflag) 2824 { 2825 if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN)) 2826 return KCQE_FLAGS_LAYER_MASK_L4; 2827 2828 return opflag & KCQE_FLAGS_LAYER_MASK; 2829 } 2830 2831 static void service_kcqes(struct cnic_dev *dev, int num_cqes) 2832 { 2833 struct cnic_local *cp = dev->cnic_priv; 2834 int i, j, comp = 0; 2835 2836 i = 0; 2837 j = 1; 2838 while (num_cqes) { 2839 struct cnic_ulp_ops *ulp_ops; 2840 int ulp_type; 2841 u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag; 2842 u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag); 2843 2844 if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION)) 2845 comp++; 2846 2847 while (j < num_cqes) { 2848 u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag; 2849 2850 if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer) 2851 break; 2852 2853 if (unlikely(next_op & KCQE_RAMROD_COMPLETION)) 2854 comp++; 2855 j++; 2856 } 2857 2858 if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA) 2859 ulp_type = CNIC_ULP_RDMA; 2860 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI) 2861 ulp_type = CNIC_ULP_ISCSI; 2862 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE) 2863 ulp_type = CNIC_ULP_FCOE; 2864 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4) 2865 ulp_type = CNIC_ULP_L4; 2866 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2) 2867 goto end; 2868 else { 2869 netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n", 2870 kcqe_op_flag); 2871 goto end; 2872 } 2873 2874 rcu_read_lock(); 2875 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 2876 if (likely(ulp_ops)) { 2877 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 2878 cp->completed_kcq + i, j); 2879 } 2880 rcu_read_unlock(); 2881 end: 2882 num_cqes -= j; 2883 i += j; 2884 j = 1; 2885 } 2886 if (unlikely(comp)) 2887 cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp); 2888 } 2889 2890 static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info) 2891 { 2892 struct cnic_local *cp = dev->cnic_priv; 2893 u16 i, ri, hw_prod, last; 2894 struct kcqe *kcqe; 2895 int kcqe_cnt = 0, last_cnt = 0; 2896 2897 i = ri = last = info->sw_prod_idx; 2898 ri &= MAX_KCQ_IDX; 2899 hw_prod = *info->hw_prod_idx_ptr; 2900 hw_prod = info->hw_idx(hw_prod); 2901 2902 while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) { 2903 kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)]; 2904 cp->completed_kcq[kcqe_cnt++] = kcqe; 2905 i = info->next_idx(i); 2906 ri = i & MAX_KCQ_IDX; 2907 if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) { 2908 last_cnt = kcqe_cnt; 2909 last = i; 2910 } 2911 } 2912 2913 info->sw_prod_idx = last; 2914 return last_cnt; 2915 } 2916 2917 static int cnic_l2_completion(struct cnic_local *cp) 2918 { 2919 u16 hw_cons, sw_cons; 2920 struct cnic_uio_dev *udev = cp->udev; 2921 union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *) 2922 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 2923 u32 cmd; 2924 int comp = 0; 2925 2926 if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags)) 2927 return 0; 2928 2929 hw_cons = *cp->rx_cons_ptr; 2930 if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT) 2931 hw_cons++; 2932 2933 sw_cons = cp->rx_cons; 2934 while (sw_cons != hw_cons) { 2935 u8 cqe_fp_flags; 2936 2937 cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT]; 2938 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 2939 if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) { 2940 cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data); 2941 cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT; 2942 if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP || 2943 cmd == RAMROD_CMD_ID_ETH_HALT) 2944 comp++; 2945 } 2946 sw_cons = BNX2X_NEXT_RCQE(sw_cons); 2947 } 2948 return comp; 2949 } 2950 2951 static void cnic_chk_pkt_rings(struct cnic_local *cp) 2952 { 2953 u16 rx_cons, tx_cons; 2954 int comp = 0; 2955 2956 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 2957 return; 2958 2959 rx_cons = *cp->rx_cons_ptr; 2960 tx_cons = *cp->tx_cons_ptr; 2961 if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) { 2962 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 2963 comp = cnic_l2_completion(cp); 2964 2965 cp->tx_cons = tx_cons; 2966 cp->rx_cons = rx_cons; 2967 2968 if (cp->udev) 2969 uio_event_notify(&cp->udev->cnic_uinfo); 2970 } 2971 if (comp) 2972 clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 2973 } 2974 2975 static u32 cnic_service_bnx2_queues(struct cnic_dev *dev) 2976 { 2977 struct cnic_local *cp = dev->cnic_priv; 2978 u32 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2979 int kcqe_cnt; 2980 2981 /* status block index must be read before reading other fields */ 2982 rmb(); 2983 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2984 2985 while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) { 2986 2987 service_kcqes(dev, kcqe_cnt); 2988 2989 /* Tell compiler that status_blk fields can change. */ 2990 barrier(); 2991 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2992 /* status block index must be read first */ 2993 rmb(); 2994 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2995 } 2996 2997 CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx); 2998 2999 cnic_chk_pkt_rings(cp); 3000 3001 return status_idx; 3002 } 3003 3004 static int cnic_service_bnx2(void *data, void *status_blk) 3005 { 3006 struct cnic_dev *dev = data; 3007 3008 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3009 struct status_block *sblk = status_blk; 3010 3011 return sblk->status_idx; 3012 } 3013 3014 return cnic_service_bnx2_queues(dev); 3015 } 3016 3017 static void cnic_service_bnx2_msix(struct work_struct *work) 3018 { 3019 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3020 struct cnic_dev *dev = cp->dev; 3021 3022 cp->last_status_idx = cnic_service_bnx2_queues(dev); 3023 3024 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 3025 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 3026 } 3027 3028 static void cnic_doirq(struct cnic_dev *dev) 3029 { 3030 struct cnic_local *cp = dev->cnic_priv; 3031 3032 if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3033 u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX; 3034 3035 prefetch(cp->status_blk.gen); 3036 prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]); 3037 3038 queue_work(system_bh_wq, &cp->cnic_irq_bh_work); 3039 } 3040 } 3041 3042 static irqreturn_t cnic_irq(int irq, void *dev_instance) 3043 { 3044 struct cnic_dev *dev = dev_instance; 3045 struct cnic_local *cp = dev->cnic_priv; 3046 3047 if (cp->ack_int) 3048 cp->ack_int(dev); 3049 3050 cnic_doirq(dev); 3051 3052 return IRQ_HANDLED; 3053 } 3054 3055 static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm, 3056 u16 index, u8 op, u8 update) 3057 { 3058 struct bnx2x *bp = netdev_priv(dev->netdev); 3059 u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp) * 32 + 3060 COMMAND_REG_INT_ACK); 3061 struct igu_ack_register igu_ack; 3062 3063 igu_ack.status_block_index = index; 3064 igu_ack.sb_id_and_flags = 3065 ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) | 3066 (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) | 3067 (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) | 3068 (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT)); 3069 3070 CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack)); 3071 } 3072 3073 static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment, 3074 u16 index, u8 op, u8 update) 3075 { 3076 struct igu_regular cmd_data; 3077 u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8; 3078 3079 cmd_data.sb_id_and_flags = 3080 (index << IGU_REGULAR_SB_INDEX_SHIFT) | 3081 (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) | 3082 (update << IGU_REGULAR_BUPDATE_SHIFT) | 3083 (op << IGU_REGULAR_ENABLE_INT_SHIFT); 3084 3085 3086 CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags); 3087 } 3088 3089 static void cnic_ack_bnx2x_msix(struct cnic_dev *dev) 3090 { 3091 struct cnic_local *cp = dev->cnic_priv; 3092 3093 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0, 3094 IGU_INT_DISABLE, 0); 3095 } 3096 3097 static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev) 3098 { 3099 struct cnic_local *cp = dev->cnic_priv; 3100 3101 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0, 3102 IGU_INT_DISABLE, 0); 3103 } 3104 3105 static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx) 3106 { 3107 struct cnic_local *cp = dev->cnic_priv; 3108 3109 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx, 3110 IGU_INT_ENABLE, 1); 3111 } 3112 3113 static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx) 3114 { 3115 struct cnic_local *cp = dev->cnic_priv; 3116 3117 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx, 3118 IGU_INT_ENABLE, 1); 3119 } 3120 3121 static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info) 3122 { 3123 u32 last_status = *info->status_idx_ptr; 3124 int kcqe_cnt; 3125 3126 /* status block index must be read before reading the KCQ */ 3127 rmb(); 3128 while ((kcqe_cnt = cnic_get_kcqes(dev, info))) { 3129 3130 service_kcqes(dev, kcqe_cnt); 3131 3132 /* Tell compiler that sblk fields can change. */ 3133 barrier(); 3134 3135 last_status = *info->status_idx_ptr; 3136 /* status block index must be read before reading the KCQ */ 3137 rmb(); 3138 } 3139 return last_status; 3140 } 3141 3142 static void cnic_service_bnx2x_bh_work(struct work_struct *work) 3143 { 3144 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3145 struct cnic_dev *dev = cp->dev; 3146 struct bnx2x *bp = netdev_priv(dev->netdev); 3147 u32 status_idx, new_status_idx; 3148 3149 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) 3150 return; 3151 3152 while (1) { 3153 status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1); 3154 3155 CNIC_WR16(dev, cp->kcq1.io_addr, 3156 cp->kcq1.sw_prod_idx + MAX_KCQ_IDX); 3157 3158 if (!CNIC_SUPPORTS_FCOE(bp)) { 3159 cp->arm_int(dev, status_idx); 3160 break; 3161 } 3162 3163 new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2); 3164 3165 if (new_status_idx != status_idx) 3166 continue; 3167 3168 CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx + 3169 MAX_KCQ_IDX); 3170 3171 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 3172 status_idx, IGU_INT_ENABLE, 1); 3173 3174 break; 3175 } 3176 } 3177 3178 static int cnic_service_bnx2x(void *data, void *status_blk) 3179 { 3180 struct cnic_dev *dev = data; 3181 struct cnic_local *cp = dev->cnic_priv; 3182 3183 if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 3184 cnic_doirq(dev); 3185 3186 cnic_chk_pkt_rings(cp); 3187 3188 return 0; 3189 } 3190 3191 static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type) 3192 { 3193 struct cnic_ulp_ops *ulp_ops; 3194 3195 if (if_type == CNIC_ULP_ISCSI) 3196 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 3197 3198 mutex_lock(&cnic_lock); 3199 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3200 lockdep_is_held(&cnic_lock)); 3201 if (!ulp_ops) { 3202 mutex_unlock(&cnic_lock); 3203 return; 3204 } 3205 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3206 mutex_unlock(&cnic_lock); 3207 3208 if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3209 ulp_ops->cnic_stop(cp->ulp_handle[if_type]); 3210 3211 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3212 } 3213 3214 static void cnic_ulp_stop(struct cnic_dev *dev) 3215 { 3216 struct cnic_local *cp = dev->cnic_priv; 3217 int if_type; 3218 3219 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) 3220 cnic_ulp_stop_one(cp, if_type); 3221 } 3222 3223 static void cnic_ulp_start(struct cnic_dev *dev) 3224 { 3225 struct cnic_local *cp = dev->cnic_priv; 3226 int if_type; 3227 3228 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 3229 struct cnic_ulp_ops *ulp_ops; 3230 3231 mutex_lock(&cnic_lock); 3232 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3233 lockdep_is_held(&cnic_lock)); 3234 if (!ulp_ops || !ulp_ops->cnic_start) { 3235 mutex_unlock(&cnic_lock); 3236 continue; 3237 } 3238 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3239 mutex_unlock(&cnic_lock); 3240 3241 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3242 ulp_ops->cnic_start(cp->ulp_handle[if_type]); 3243 3244 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3245 } 3246 } 3247 3248 static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type) 3249 { 3250 struct cnic_local *cp = dev->cnic_priv; 3251 struct cnic_ulp_ops *ulp_ops; 3252 int rc; 3253 3254 mutex_lock(&cnic_lock); 3255 ulp_ops = rcu_dereference_protected(cp->ulp_ops[ulp_type], 3256 lockdep_is_held(&cnic_lock)); 3257 if (ulp_ops && ulp_ops->cnic_get_stats) 3258 rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]); 3259 else 3260 rc = -ENODEV; 3261 mutex_unlock(&cnic_lock); 3262 return rc; 3263 } 3264 3265 static int cnic_ctl(void *data, struct cnic_ctl_info *info) 3266 { 3267 struct cnic_dev *dev = data; 3268 int ulp_type = CNIC_ULP_ISCSI; 3269 3270 switch (info->cmd) { 3271 case CNIC_CTL_STOP_CMD: 3272 cnic_hold(dev); 3273 3274 cnic_ulp_stop(dev); 3275 cnic_stop_hw(dev); 3276 3277 cnic_put(dev); 3278 break; 3279 case CNIC_CTL_START_CMD: 3280 cnic_hold(dev); 3281 3282 if (!cnic_start_hw(dev)) 3283 cnic_ulp_start(dev); 3284 3285 cnic_put(dev); 3286 break; 3287 case CNIC_CTL_STOP_ISCSI_CMD: { 3288 struct cnic_local *cp = dev->cnic_priv; 3289 set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags); 3290 queue_delayed_work(cnic_wq, &cp->delete_task, 0); 3291 break; 3292 } 3293 case CNIC_CTL_COMPLETION_CMD: { 3294 struct cnic_ctl_completion *comp = &info->data.comp; 3295 u32 cid = BNX2X_SW_CID(comp->cid); 3296 u32 l5_cid; 3297 struct cnic_local *cp = dev->cnic_priv; 3298 3299 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 3300 break; 3301 3302 if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) { 3303 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3304 3305 if (unlikely(comp->error)) { 3306 set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags); 3307 netdev_err(dev->netdev, 3308 "CID %x CFC delete comp error %x\n", 3309 cid, comp->error); 3310 } 3311 3312 ctx->wait_cond = 1; 3313 wake_up(&ctx->waitq); 3314 } 3315 break; 3316 } 3317 case CNIC_CTL_FCOE_STATS_GET_CMD: 3318 ulp_type = CNIC_ULP_FCOE; 3319 fallthrough; 3320 case CNIC_CTL_ISCSI_STATS_GET_CMD: 3321 cnic_hold(dev); 3322 cnic_copy_ulp_stats(dev, ulp_type); 3323 cnic_put(dev); 3324 break; 3325 3326 default: 3327 return -EINVAL; 3328 } 3329 return 0; 3330 } 3331 3332 static void cnic_ulp_init(struct cnic_dev *dev) 3333 { 3334 int i; 3335 struct cnic_local *cp = dev->cnic_priv; 3336 3337 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3338 struct cnic_ulp_ops *ulp_ops; 3339 3340 mutex_lock(&cnic_lock); 3341 ulp_ops = cnic_ulp_tbl_prot(i); 3342 if (!ulp_ops || !ulp_ops->cnic_init) { 3343 mutex_unlock(&cnic_lock); 3344 continue; 3345 } 3346 ulp_get(ulp_ops); 3347 mutex_unlock(&cnic_lock); 3348 3349 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3350 ulp_ops->cnic_init(dev); 3351 3352 ulp_put(ulp_ops); 3353 } 3354 } 3355 3356 static void cnic_ulp_exit(struct cnic_dev *dev) 3357 { 3358 int i; 3359 struct cnic_local *cp = dev->cnic_priv; 3360 3361 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3362 struct cnic_ulp_ops *ulp_ops; 3363 3364 mutex_lock(&cnic_lock); 3365 ulp_ops = cnic_ulp_tbl_prot(i); 3366 if (!ulp_ops || !ulp_ops->cnic_exit) { 3367 mutex_unlock(&cnic_lock); 3368 continue; 3369 } 3370 ulp_get(ulp_ops); 3371 mutex_unlock(&cnic_lock); 3372 3373 if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3374 ulp_ops->cnic_exit(dev); 3375 3376 ulp_put(ulp_ops); 3377 } 3378 } 3379 3380 static int cnic_cm_offload_pg(struct cnic_sock *csk) 3381 { 3382 struct cnic_dev *dev = csk->dev; 3383 struct l4_kwq_offload_pg *l4kwqe; 3384 struct kwqe *wqes[1]; 3385 3386 l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1; 3387 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3388 wqes[0] = (struct kwqe *) l4kwqe; 3389 3390 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG; 3391 l4kwqe->flags = 3392 L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT; 3393 l4kwqe->l2hdr_nbytes = ETH_HLEN; 3394 3395 l4kwqe->da0 = csk->ha[0]; 3396 l4kwqe->da1 = csk->ha[1]; 3397 l4kwqe->da2 = csk->ha[2]; 3398 l4kwqe->da3 = csk->ha[3]; 3399 l4kwqe->da4 = csk->ha[4]; 3400 l4kwqe->da5 = csk->ha[5]; 3401 3402 l4kwqe->sa0 = dev->mac_addr[0]; 3403 l4kwqe->sa1 = dev->mac_addr[1]; 3404 l4kwqe->sa2 = dev->mac_addr[2]; 3405 l4kwqe->sa3 = dev->mac_addr[3]; 3406 l4kwqe->sa4 = dev->mac_addr[4]; 3407 l4kwqe->sa5 = dev->mac_addr[5]; 3408 3409 l4kwqe->etype = ETH_P_IP; 3410 l4kwqe->ipid_start = DEF_IPID_START; 3411 l4kwqe->host_opaque = csk->l5_cid; 3412 3413 if (csk->vlan_id) { 3414 l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING; 3415 l4kwqe->vlan_tag = csk->vlan_id; 3416 l4kwqe->l2hdr_nbytes += 4; 3417 } 3418 3419 return dev->submit_kwqes(dev, wqes, 1); 3420 } 3421 3422 static int cnic_cm_update_pg(struct cnic_sock *csk) 3423 { 3424 struct cnic_dev *dev = csk->dev; 3425 struct l4_kwq_update_pg *l4kwqe; 3426 struct kwqe *wqes[1]; 3427 3428 l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1; 3429 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3430 wqes[0] = (struct kwqe *) l4kwqe; 3431 3432 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG; 3433 l4kwqe->flags = 3434 L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT; 3435 l4kwqe->pg_cid = csk->pg_cid; 3436 3437 l4kwqe->da0 = csk->ha[0]; 3438 l4kwqe->da1 = csk->ha[1]; 3439 l4kwqe->da2 = csk->ha[2]; 3440 l4kwqe->da3 = csk->ha[3]; 3441 l4kwqe->da4 = csk->ha[4]; 3442 l4kwqe->da5 = csk->ha[5]; 3443 3444 l4kwqe->pg_host_opaque = csk->l5_cid; 3445 l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA; 3446 3447 return dev->submit_kwqes(dev, wqes, 1); 3448 } 3449 3450 static int cnic_cm_upload_pg(struct cnic_sock *csk) 3451 { 3452 struct cnic_dev *dev = csk->dev; 3453 struct l4_kwq_upload *l4kwqe; 3454 struct kwqe *wqes[1]; 3455 3456 l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1; 3457 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3458 wqes[0] = (struct kwqe *) l4kwqe; 3459 3460 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG; 3461 l4kwqe->flags = 3462 L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT; 3463 l4kwqe->cid = csk->pg_cid; 3464 3465 return dev->submit_kwqes(dev, wqes, 1); 3466 } 3467 3468 static int cnic_cm_conn_req(struct cnic_sock *csk) 3469 { 3470 struct cnic_dev *dev = csk->dev; 3471 struct l4_kwq_connect_req1 *l4kwqe1; 3472 struct l4_kwq_connect_req2 *l4kwqe2; 3473 struct l4_kwq_connect_req3 *l4kwqe3; 3474 struct kwqe *wqes[3]; 3475 u8 tcp_flags = 0; 3476 int num_wqes = 2; 3477 3478 l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1; 3479 l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2; 3480 l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3; 3481 memset(l4kwqe1, 0, sizeof(*l4kwqe1)); 3482 memset(l4kwqe2, 0, sizeof(*l4kwqe2)); 3483 memset(l4kwqe3, 0, sizeof(*l4kwqe3)); 3484 3485 l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3; 3486 l4kwqe3->flags = 3487 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT; 3488 l4kwqe3->ka_timeout = csk->ka_timeout; 3489 l4kwqe3->ka_interval = csk->ka_interval; 3490 l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count; 3491 l4kwqe3->tos = csk->tos; 3492 l4kwqe3->ttl = csk->ttl; 3493 l4kwqe3->snd_seq_scale = csk->snd_seq_scale; 3494 l4kwqe3->pmtu = csk->mtu; 3495 l4kwqe3->rcv_buf = csk->rcv_buf; 3496 l4kwqe3->snd_buf = csk->snd_buf; 3497 l4kwqe3->seed = csk->seed; 3498 3499 wqes[0] = (struct kwqe *) l4kwqe1; 3500 if (test_bit(SK_F_IPV6, &csk->flags)) { 3501 wqes[1] = (struct kwqe *) l4kwqe2; 3502 wqes[2] = (struct kwqe *) l4kwqe3; 3503 num_wqes = 3; 3504 3505 l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6; 3506 l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2; 3507 l4kwqe2->flags = 3508 L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT | 3509 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT; 3510 l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]); 3511 l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]); 3512 l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]); 3513 l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]); 3514 l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]); 3515 l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]); 3516 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) - 3517 sizeof(struct tcphdr); 3518 } else { 3519 wqes[1] = (struct kwqe *) l4kwqe3; 3520 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) - 3521 sizeof(struct tcphdr); 3522 } 3523 3524 l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1; 3525 l4kwqe1->flags = 3526 (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) | 3527 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT; 3528 l4kwqe1->cid = csk->cid; 3529 l4kwqe1->pg_cid = csk->pg_cid; 3530 l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]); 3531 l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]); 3532 l4kwqe1->src_port = be16_to_cpu(csk->src_port); 3533 l4kwqe1->dst_port = be16_to_cpu(csk->dst_port); 3534 if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK) 3535 tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK; 3536 if (csk->tcp_flags & SK_TCP_KEEP_ALIVE) 3537 tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE; 3538 if (csk->tcp_flags & SK_TCP_NAGLE) 3539 tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE; 3540 if (csk->tcp_flags & SK_TCP_TIMESTAMP) 3541 tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP; 3542 if (csk->tcp_flags & SK_TCP_SACK) 3543 tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK; 3544 if (csk->tcp_flags & SK_TCP_SEG_SCALING) 3545 tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING; 3546 3547 l4kwqe1->tcp_flags = tcp_flags; 3548 3549 return dev->submit_kwqes(dev, wqes, num_wqes); 3550 } 3551 3552 static int cnic_cm_close_req(struct cnic_sock *csk) 3553 { 3554 struct cnic_dev *dev = csk->dev; 3555 struct l4_kwq_close_req *l4kwqe; 3556 struct kwqe *wqes[1]; 3557 3558 l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2; 3559 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3560 wqes[0] = (struct kwqe *) l4kwqe; 3561 3562 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE; 3563 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT; 3564 l4kwqe->cid = csk->cid; 3565 3566 return dev->submit_kwqes(dev, wqes, 1); 3567 } 3568 3569 static int cnic_cm_abort_req(struct cnic_sock *csk) 3570 { 3571 struct cnic_dev *dev = csk->dev; 3572 struct l4_kwq_reset_req *l4kwqe; 3573 struct kwqe *wqes[1]; 3574 3575 l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2; 3576 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3577 wqes[0] = (struct kwqe *) l4kwqe; 3578 3579 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET; 3580 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT; 3581 l4kwqe->cid = csk->cid; 3582 3583 return dev->submit_kwqes(dev, wqes, 1); 3584 } 3585 3586 static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid, 3587 u32 l5_cid, struct cnic_sock **csk, void *context) 3588 { 3589 struct cnic_local *cp = dev->cnic_priv; 3590 struct cnic_sock *csk1; 3591 3592 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3593 return -EINVAL; 3594 3595 if (cp->ctx_tbl) { 3596 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3597 3598 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 3599 return -EAGAIN; 3600 } 3601 3602 csk1 = &cp->csk_tbl[l5_cid]; 3603 if (atomic_read(&csk1->ref_count)) 3604 return -EAGAIN; 3605 3606 if (test_and_set_bit(SK_F_INUSE, &csk1->flags)) 3607 return -EBUSY; 3608 3609 csk1->dev = dev; 3610 csk1->cid = cid; 3611 csk1->l5_cid = l5_cid; 3612 csk1->ulp_type = ulp_type; 3613 csk1->context = context; 3614 3615 csk1->ka_timeout = DEF_KA_TIMEOUT; 3616 csk1->ka_interval = DEF_KA_INTERVAL; 3617 csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT; 3618 csk1->tos = DEF_TOS; 3619 csk1->ttl = DEF_TTL; 3620 csk1->snd_seq_scale = DEF_SND_SEQ_SCALE; 3621 csk1->rcv_buf = DEF_RCV_BUF; 3622 csk1->snd_buf = DEF_SND_BUF; 3623 csk1->seed = DEF_SEED; 3624 csk1->tcp_flags = 0; 3625 3626 *csk = csk1; 3627 return 0; 3628 } 3629 3630 static void cnic_cm_cleanup(struct cnic_sock *csk) 3631 { 3632 if (csk->src_port) { 3633 struct cnic_dev *dev = csk->dev; 3634 struct cnic_local *cp = dev->cnic_priv; 3635 3636 cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port)); 3637 csk->src_port = 0; 3638 } 3639 } 3640 3641 static void cnic_close_conn(struct cnic_sock *csk) 3642 { 3643 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) { 3644 cnic_cm_upload_pg(csk); 3645 clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3646 } 3647 cnic_cm_cleanup(csk); 3648 } 3649 3650 static int cnic_cm_destroy(struct cnic_sock *csk) 3651 { 3652 if (!cnic_in_use(csk)) 3653 return -EINVAL; 3654 3655 csk_hold(csk); 3656 clear_bit(SK_F_INUSE, &csk->flags); 3657 smp_mb__after_atomic(); 3658 while (atomic_read(&csk->ref_count) != 1) 3659 msleep(1); 3660 cnic_cm_cleanup(csk); 3661 3662 csk->flags = 0; 3663 csk_put(csk); 3664 return 0; 3665 } 3666 3667 static inline u16 cnic_get_vlan(struct net_device *dev, 3668 struct net_device **vlan_dev) 3669 { 3670 if (is_vlan_dev(dev)) { 3671 *vlan_dev = vlan_dev_real_dev(dev); 3672 return vlan_dev_vlan_id(dev); 3673 } 3674 *vlan_dev = dev; 3675 return 0; 3676 } 3677 3678 static int cnic_get_v4_route(struct sockaddr_in *dst_addr, 3679 struct dst_entry **dst) 3680 { 3681 #if defined(CONFIG_INET) 3682 struct rtable *rt; 3683 3684 rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0, 3685 RT_SCOPE_UNIVERSE); 3686 if (!IS_ERR(rt)) { 3687 *dst = &rt->dst; 3688 return 0; 3689 } 3690 return PTR_ERR(rt); 3691 #else 3692 return -ENETUNREACH; 3693 #endif 3694 } 3695 3696 static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr, 3697 struct dst_entry **dst) 3698 { 3699 #if IS_ENABLED(CONFIG_IPV6) 3700 struct flowi6 fl6; 3701 3702 memset(&fl6, 0, sizeof(fl6)); 3703 fl6.daddr = dst_addr->sin6_addr; 3704 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 3705 fl6.flowi6_oif = dst_addr->sin6_scope_id; 3706 3707 *dst = ip6_route_output(&init_net, NULL, &fl6); 3708 if ((*dst)->error) { 3709 dst_release(*dst); 3710 *dst = NULL; 3711 return -ENETUNREACH; 3712 } else 3713 return 0; 3714 #endif 3715 3716 return -ENETUNREACH; 3717 } 3718 3719 static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr, 3720 int ulp_type) 3721 { 3722 struct cnic_dev *dev = NULL; 3723 struct dst_entry *dst; 3724 struct net_device *netdev = NULL; 3725 int err = -ENETUNREACH; 3726 3727 if (dst_addr->sin_family == AF_INET) 3728 err = cnic_get_v4_route(dst_addr, &dst); 3729 else if (dst_addr->sin_family == AF_INET6) { 3730 struct sockaddr_in6 *dst_addr6 = 3731 (struct sockaddr_in6 *) dst_addr; 3732 3733 err = cnic_get_v6_route(dst_addr6, &dst); 3734 } else 3735 return NULL; 3736 3737 if (err) 3738 return NULL; 3739 3740 if (!dst->dev) 3741 goto done; 3742 3743 cnic_get_vlan(dst->dev, &netdev); 3744 3745 dev = cnic_from_netdev(netdev); 3746 3747 done: 3748 dst_release(dst); 3749 if (dev) 3750 cnic_put(dev); 3751 return dev; 3752 } 3753 3754 static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3755 { 3756 struct cnic_dev *dev = csk->dev; 3757 struct cnic_local *cp = dev->cnic_priv; 3758 3759 return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk); 3760 } 3761 3762 static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3763 { 3764 struct cnic_dev *dev = csk->dev; 3765 struct cnic_local *cp = dev->cnic_priv; 3766 int is_v6, rc = 0; 3767 struct dst_entry *dst = NULL; 3768 struct net_device *realdev; 3769 __be16 local_port; 3770 u32 port_id; 3771 3772 if (saddr->local.v6.sin6_family == AF_INET6 && 3773 saddr->remote.v6.sin6_family == AF_INET6) 3774 is_v6 = 1; 3775 else if (saddr->local.v4.sin_family == AF_INET && 3776 saddr->remote.v4.sin_family == AF_INET) 3777 is_v6 = 0; 3778 else 3779 return -EINVAL; 3780 3781 clear_bit(SK_F_IPV6, &csk->flags); 3782 3783 if (is_v6) { 3784 set_bit(SK_F_IPV6, &csk->flags); 3785 cnic_get_v6_route(&saddr->remote.v6, &dst); 3786 3787 memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr, 3788 sizeof(struct in6_addr)); 3789 csk->dst_port = saddr->remote.v6.sin6_port; 3790 local_port = saddr->local.v6.sin6_port; 3791 3792 } else { 3793 cnic_get_v4_route(&saddr->remote.v4, &dst); 3794 3795 csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr; 3796 csk->dst_port = saddr->remote.v4.sin_port; 3797 local_port = saddr->local.v4.sin_port; 3798 } 3799 3800 csk->vlan_id = 0; 3801 csk->mtu = dev->netdev->mtu; 3802 if (dst && dst->dev) { 3803 u16 vlan = cnic_get_vlan(dst->dev, &realdev); 3804 if (realdev == dev->netdev) { 3805 csk->vlan_id = vlan; 3806 csk->mtu = dst_mtu(dst); 3807 } 3808 } 3809 3810 port_id = be16_to_cpu(local_port); 3811 if (port_id >= CNIC_LOCAL_PORT_MIN && 3812 port_id < CNIC_LOCAL_PORT_MAX) { 3813 if (cnic_alloc_id(&cp->csk_port_tbl, port_id)) 3814 port_id = 0; 3815 } else 3816 port_id = 0; 3817 3818 if (!port_id) { 3819 port_id = cnic_alloc_new_id(&cp->csk_port_tbl); 3820 if (port_id == -1) { 3821 rc = -ENOMEM; 3822 goto err_out; 3823 } 3824 local_port = cpu_to_be16(port_id); 3825 } 3826 csk->src_port = local_port; 3827 3828 err_out: 3829 dst_release(dst); 3830 return rc; 3831 } 3832 3833 static void cnic_init_csk_state(struct cnic_sock *csk) 3834 { 3835 csk->state = 0; 3836 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3837 clear_bit(SK_F_CLOSING, &csk->flags); 3838 } 3839 3840 static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3841 { 3842 struct cnic_local *cp = csk->dev->cnic_priv; 3843 int err = 0; 3844 3845 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 3846 return -EOPNOTSUPP; 3847 3848 if (!cnic_in_use(csk)) 3849 return -EINVAL; 3850 3851 if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags)) 3852 return -EINVAL; 3853 3854 cnic_init_csk_state(csk); 3855 3856 err = cnic_get_route(csk, saddr); 3857 if (err) 3858 goto err_out; 3859 3860 err = cnic_resolve_addr(csk, saddr); 3861 if (!err) 3862 return 0; 3863 3864 err_out: 3865 clear_bit(SK_F_CONNECT_START, &csk->flags); 3866 return err; 3867 } 3868 3869 static int cnic_cm_abort(struct cnic_sock *csk) 3870 { 3871 struct cnic_local *cp = csk->dev->cnic_priv; 3872 u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP; 3873 3874 if (!cnic_in_use(csk)) 3875 return -EINVAL; 3876 3877 if (cnic_abort_prep(csk)) 3878 return cnic_cm_abort_req(csk); 3879 3880 /* Getting here means that we haven't started connect, or 3881 * connect was not successful, or it has been reset by the target. 3882 */ 3883 3884 cp->close_conn(csk, opcode); 3885 if (csk->state != opcode) { 3886 /* Wait for remote reset sequence to complete */ 3887 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3888 msleep(1); 3889 3890 return -EALREADY; 3891 } 3892 3893 return 0; 3894 } 3895 3896 static int cnic_cm_close(struct cnic_sock *csk) 3897 { 3898 if (!cnic_in_use(csk)) 3899 return -EINVAL; 3900 3901 if (cnic_close_prep(csk)) { 3902 csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 3903 return cnic_cm_close_req(csk); 3904 } else { 3905 /* Wait for remote reset sequence to complete */ 3906 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3907 msleep(1); 3908 3909 return -EALREADY; 3910 } 3911 return 0; 3912 } 3913 3914 static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk, 3915 u8 opcode) 3916 { 3917 struct cnic_ulp_ops *ulp_ops; 3918 int ulp_type = csk->ulp_type; 3919 3920 rcu_read_lock(); 3921 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 3922 if (ulp_ops) { 3923 if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE) 3924 ulp_ops->cm_connect_complete(csk); 3925 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 3926 ulp_ops->cm_close_complete(csk); 3927 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) 3928 ulp_ops->cm_remote_abort(csk); 3929 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP) 3930 ulp_ops->cm_abort_complete(csk); 3931 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED) 3932 ulp_ops->cm_remote_close(csk); 3933 } 3934 rcu_read_unlock(); 3935 } 3936 3937 static int cnic_cm_set_pg(struct cnic_sock *csk) 3938 { 3939 if (cnic_offld_prep(csk)) { 3940 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3941 cnic_cm_update_pg(csk); 3942 else 3943 cnic_cm_offload_pg(csk); 3944 } 3945 return 0; 3946 } 3947 3948 static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe) 3949 { 3950 struct cnic_local *cp = dev->cnic_priv; 3951 u32 l5_cid = kcqe->pg_host_opaque; 3952 u8 opcode = kcqe->op_code; 3953 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 3954 3955 csk_hold(csk); 3956 if (!cnic_in_use(csk)) 3957 goto done; 3958 3959 if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3960 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3961 goto done; 3962 } 3963 /* Possible PG kcqe status: SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */ 3964 if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) { 3965 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3966 cnic_cm_upcall(cp, csk, 3967 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 3968 goto done; 3969 } 3970 3971 csk->pg_cid = kcqe->pg_cid; 3972 set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3973 cnic_cm_conn_req(csk); 3974 3975 done: 3976 csk_put(csk); 3977 } 3978 3979 static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe) 3980 { 3981 struct cnic_local *cp = dev->cnic_priv; 3982 struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe; 3983 u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE; 3984 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3985 3986 ctx->timestamp = jiffies; 3987 ctx->wait_cond = 1; 3988 wake_up(&ctx->waitq); 3989 } 3990 3991 static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe) 3992 { 3993 struct cnic_local *cp = dev->cnic_priv; 3994 struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe; 3995 u8 opcode = l4kcqe->op_code; 3996 u32 l5_cid; 3997 struct cnic_sock *csk; 3998 3999 if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) { 4000 cnic_process_fcoe_term_conn(dev, kcqe); 4001 return; 4002 } 4003 if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG || 4004 opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 4005 cnic_cm_process_offld_pg(dev, l4kcqe); 4006 return; 4007 } 4008 4009 l5_cid = l4kcqe->conn_id; 4010 if (opcode & 0x80) 4011 l5_cid = l4kcqe->cid; 4012 if (l5_cid >= MAX_CM_SK_TBL_SZ) 4013 return; 4014 4015 csk = &cp->csk_tbl[l5_cid]; 4016 csk_hold(csk); 4017 4018 if (!cnic_in_use(csk)) { 4019 csk_put(csk); 4020 return; 4021 } 4022 4023 switch (opcode) { 4024 case L5CM_RAMROD_CMD_ID_TCP_CONNECT: 4025 if (l4kcqe->status != 0) { 4026 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4027 cnic_cm_upcall(cp, csk, 4028 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 4029 } 4030 break; 4031 case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE: 4032 if (l4kcqe->status == 0) 4033 set_bit(SK_F_OFFLD_COMPLETE, &csk->flags); 4034 else if (l4kcqe->status == 4035 L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4036 set_bit(SK_F_HW_ERR, &csk->flags); 4037 4038 smp_mb__before_atomic(); 4039 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4040 cnic_cm_upcall(cp, csk, opcode); 4041 break; 4042 4043 case L5CM_RAMROD_CMD_ID_CLOSE: { 4044 struct iscsi_kcqe *l5kcqe = (struct iscsi_kcqe *) kcqe; 4045 4046 if (l4kcqe->status == 0 && l5kcqe->completion_status == 0) 4047 break; 4048 4049 netdev_warn(dev->netdev, "RAMROD CLOSE compl with status 0x%x completion status 0x%x\n", 4050 l4kcqe->status, l5kcqe->completion_status); 4051 opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 4052 } 4053 fallthrough; 4054 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4055 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4056 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4057 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4058 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4059 if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4060 set_bit(SK_F_HW_ERR, &csk->flags); 4061 4062 cp->close_conn(csk, opcode); 4063 break; 4064 4065 case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED: 4066 /* after we already sent CLOSE_REQ */ 4067 if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) && 4068 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) && 4069 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 4070 cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP); 4071 else 4072 cnic_cm_upcall(cp, csk, opcode); 4073 break; 4074 } 4075 csk_put(csk); 4076 } 4077 4078 static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num) 4079 { 4080 struct cnic_dev *dev = data; 4081 int i; 4082 4083 for (i = 0; i < num; i++) 4084 cnic_cm_process_kcqe(dev, kcqe[i]); 4085 } 4086 4087 static struct cnic_ulp_ops cm_ulp_ops = { 4088 .indicate_kcqes = cnic_cm_indicate_kcqe, 4089 }; 4090 4091 static void cnic_cm_free_mem(struct cnic_dev *dev) 4092 { 4093 struct cnic_local *cp = dev->cnic_priv; 4094 4095 kvfree(cp->csk_tbl); 4096 cp->csk_tbl = NULL; 4097 cnic_free_id_tbl(&cp->csk_port_tbl); 4098 } 4099 4100 static int cnic_cm_alloc_mem(struct cnic_dev *dev) 4101 { 4102 struct cnic_local *cp = dev->cnic_priv; 4103 u32 port_id; 4104 int i; 4105 4106 cp->csk_tbl = kvzalloc_objs(struct cnic_sock, MAX_CM_SK_TBL_SZ); 4107 if (!cp->csk_tbl) 4108 return -ENOMEM; 4109 4110 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) 4111 atomic_set(&cp->csk_tbl[i].ref_count, 0); 4112 4113 port_id = get_random_u32_below(CNIC_LOCAL_PORT_RANGE); 4114 if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, 4115 CNIC_LOCAL_PORT_MIN, port_id)) { 4116 cnic_cm_free_mem(dev); 4117 return -ENOMEM; 4118 } 4119 return 0; 4120 } 4121 4122 static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode) 4123 { 4124 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 4125 /* Unsolicited RESET_COMP or RESET_RECEIVED */ 4126 opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED; 4127 csk->state = opcode; 4128 } 4129 4130 /* 1. If event opcode matches the expected event in csk->state 4131 * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any 4132 * event 4133 * 3. If the expected event is 0, meaning the connection was never 4134 * never established, we accept the opcode from cm_abort. 4135 */ 4136 if (opcode == csk->state || csk->state == 0 || 4137 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP || 4138 csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) { 4139 if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) { 4140 if (csk->state == 0) 4141 csk->state = opcode; 4142 return 1; 4143 } 4144 } 4145 return 0; 4146 } 4147 4148 static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode) 4149 { 4150 struct cnic_dev *dev = csk->dev; 4151 struct cnic_local *cp = dev->cnic_priv; 4152 4153 if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) { 4154 cnic_cm_upcall(cp, csk, opcode); 4155 return; 4156 } 4157 4158 clear_bit(SK_F_CONNECT_START, &csk->flags); 4159 cnic_close_conn(csk); 4160 csk->state = opcode; 4161 cnic_cm_upcall(cp, csk, opcode); 4162 } 4163 4164 static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev) 4165 { 4166 } 4167 4168 static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) 4169 { 4170 u32 seed; 4171 4172 seed = get_random_u32(); 4173 cnic_ctx_wr(dev, 45, 0, seed); 4174 return 0; 4175 } 4176 4177 static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode) 4178 { 4179 struct cnic_dev *dev = csk->dev; 4180 struct cnic_local *cp = dev->cnic_priv; 4181 struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid]; 4182 union l5cm_specific_data l5_data; 4183 u32 cmd = 0; 4184 int close_complete = 0; 4185 4186 switch (opcode) { 4187 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4188 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4189 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4190 if (cnic_ready_to_close(csk, opcode)) { 4191 if (test_bit(SK_F_HW_ERR, &csk->flags)) 4192 close_complete = 1; 4193 else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 4194 cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE; 4195 else 4196 close_complete = 1; 4197 } 4198 break; 4199 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4200 cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD; 4201 break; 4202 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4203 close_complete = 1; 4204 break; 4205 } 4206 if (cmd) { 4207 memset(&l5_data, 0, sizeof(l5_data)); 4208 4209 cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE, 4210 &l5_data); 4211 } else if (close_complete) { 4212 ctx->timestamp = jiffies; 4213 cnic_close_conn(csk); 4214 cnic_cm_upcall(cp, csk, csk->state); 4215 } 4216 } 4217 4218 static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) 4219 { 4220 struct cnic_local *cp = dev->cnic_priv; 4221 4222 if (!cp->ctx_tbl) 4223 return; 4224 4225 if (!netif_running(dev->netdev)) 4226 return; 4227 4228 cnic_bnx2x_delete_wait(dev, 0); 4229 4230 cancel_delayed_work_sync(&cp->delete_task); 4231 4232 if (atomic_read(&cp->iscsi_conn) != 0) 4233 netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", 4234 atomic_read(&cp->iscsi_conn)); 4235 } 4236 4237 static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev) 4238 { 4239 struct bnx2x *bp = netdev_priv(dev->netdev); 4240 u32 pfid = bp->pfid; 4241 u32 port = BP_PORT(bp); 4242 4243 cnic_init_bnx2x_mac(dev); 4244 cnic_bnx2x_set_tcp_options(dev, 0, 1); 4245 4246 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 4247 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0); 4248 4249 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4250 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1); 4251 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4252 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port), 4253 DEF_MAX_DA_COUNT); 4254 4255 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4256 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL); 4257 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4258 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS); 4259 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4260 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2); 4261 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4262 XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER); 4263 4264 CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid), 4265 DEF_MAX_CWND); 4266 return 0; 4267 } 4268 4269 static void cnic_delete_task(struct work_struct *work) 4270 { 4271 struct cnic_local *cp; 4272 struct cnic_dev *dev; 4273 u32 i; 4274 int need_resched = 0; 4275 4276 cp = container_of(work, struct cnic_local, delete_task.work); 4277 dev = cp->dev; 4278 4279 if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) { 4280 struct drv_ctl_info info; 4281 4282 cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI); 4283 4284 memset(&info, 0, sizeof(struct drv_ctl_info)); 4285 info.cmd = DRV_CTL_ISCSI_STOPPED_CMD; 4286 cp->ethdev->drv_ctl(dev->netdev, &info); 4287 } 4288 4289 for (i = 0; i < cp->max_cid_space; i++) { 4290 struct cnic_context *ctx = &cp->ctx_tbl[i]; 4291 int err; 4292 4293 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) || 4294 !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4295 continue; 4296 4297 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 4298 need_resched = 1; 4299 continue; 4300 } 4301 4302 if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4303 continue; 4304 4305 err = cnic_bnx2x_destroy_ramrod(dev, i); 4306 4307 cnic_free_bnx2x_conn_resc(dev, i); 4308 if (!err) { 4309 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) 4310 atomic_dec(&cp->iscsi_conn); 4311 4312 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 4313 } 4314 } 4315 4316 if (need_resched) 4317 queue_delayed_work(cnic_wq, &cp->delete_task, 4318 msecs_to_jiffies(10)); 4319 4320 } 4321 4322 static int cnic_cm_open(struct cnic_dev *dev) 4323 { 4324 struct cnic_local *cp = dev->cnic_priv; 4325 int err; 4326 4327 err = cnic_cm_alloc_mem(dev); 4328 if (err) 4329 return err; 4330 4331 err = cp->start_cm(dev); 4332 4333 if (err) 4334 goto err_out; 4335 4336 INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task); 4337 4338 dev->cm_create = cnic_cm_create; 4339 dev->cm_destroy = cnic_cm_destroy; 4340 dev->cm_connect = cnic_cm_connect; 4341 dev->cm_abort = cnic_cm_abort; 4342 dev->cm_close = cnic_cm_close; 4343 dev->cm_select_dev = cnic_cm_select_dev; 4344 4345 cp->ulp_handle[CNIC_ULP_L4] = dev; 4346 rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops); 4347 return 0; 4348 4349 err_out: 4350 cnic_cm_free_mem(dev); 4351 return err; 4352 } 4353 4354 static int cnic_cm_shutdown(struct cnic_dev *dev) 4355 { 4356 struct cnic_local *cp = dev->cnic_priv; 4357 int i; 4358 4359 if (!cp->csk_tbl) 4360 return 0; 4361 4362 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) { 4363 struct cnic_sock *csk = &cp->csk_tbl[i]; 4364 4365 clear_bit(SK_F_INUSE, &csk->flags); 4366 cnic_cm_cleanup(csk); 4367 } 4368 cnic_cm_free_mem(dev); 4369 4370 return 0; 4371 } 4372 4373 static void cnic_init_context(struct cnic_dev *dev, u32 cid) 4374 { 4375 u32 cid_addr; 4376 int i; 4377 4378 cid_addr = GET_CID_ADDR(cid); 4379 4380 for (i = 0; i < CTX_SIZE; i += 4) 4381 cnic_ctx_wr(dev, cid_addr, i, 0); 4382 } 4383 4384 static int cnic_setup_5709_context(struct cnic_dev *dev, int valid) 4385 { 4386 struct cnic_local *cp = dev->cnic_priv; 4387 int ret = 0, i; 4388 u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0; 4389 4390 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4391 return 0; 4392 4393 for (i = 0; i < cp->ctx_blks; i++) { 4394 int j; 4395 u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk; 4396 u32 val; 4397 4398 memset(cp->ctx_arr[i].ctx, 0, CNIC_PAGE_SIZE); 4399 4400 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0, 4401 (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit); 4402 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1, 4403 (u64) cp->ctx_arr[i].mapping >> 32); 4404 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx | 4405 BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ); 4406 for (j = 0; j < 10; j++) { 4407 4408 val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL); 4409 if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ)) 4410 break; 4411 udelay(5); 4412 } 4413 if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) { 4414 ret = -EBUSY; 4415 break; 4416 } 4417 } 4418 return ret; 4419 } 4420 4421 static void cnic_free_irq(struct cnic_dev *dev) 4422 { 4423 struct cnic_local *cp = dev->cnic_priv; 4424 struct cnic_eth_dev *ethdev = cp->ethdev; 4425 4426 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4427 cp->disable_int_sync(dev); 4428 cancel_work_sync(&cp->cnic_irq_bh_work); 4429 free_irq(ethdev->irq_arr[0].vector, dev); 4430 } 4431 } 4432 4433 static int cnic_request_irq(struct cnic_dev *dev) 4434 { 4435 struct cnic_local *cp = dev->cnic_priv; 4436 struct cnic_eth_dev *ethdev = cp->ethdev; 4437 int err; 4438 4439 err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev); 4440 if (err) 4441 disable_work_sync(&cp->cnic_irq_bh_work); 4442 4443 return err; 4444 } 4445 4446 static int cnic_init_bnx2_irq(struct cnic_dev *dev) 4447 { 4448 struct cnic_local *cp = dev->cnic_priv; 4449 struct cnic_eth_dev *ethdev = cp->ethdev; 4450 4451 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4452 int err, i = 0; 4453 int sblk_num = cp->status_blk_num; 4454 u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) + 4455 BNX2_HC_SB_CONFIG_1; 4456 4457 CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT); 4458 4459 CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8); 4460 CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220); 4461 CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220); 4462 4463 cp->last_status_idx = cp->status_blk.bnx2->status_idx; 4464 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2_msix); 4465 err = cnic_request_irq(dev); 4466 if (err) 4467 return err; 4468 4469 while (cp->status_blk.bnx2->status_completion_producer_index && 4470 i < 10) { 4471 CNIC_WR(dev, BNX2_HC_COALESCE_NOW, 4472 1 << (11 + sblk_num)); 4473 udelay(10); 4474 i++; 4475 barrier(); 4476 } 4477 if (cp->status_blk.bnx2->status_completion_producer_index) { 4478 cnic_free_irq(dev); 4479 goto failed; 4480 } 4481 4482 } else { 4483 struct status_block *sblk = cp->status_blk.gen; 4484 u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND); 4485 int i = 0; 4486 4487 while (sblk->status_completion_producer_index && i < 10) { 4488 CNIC_WR(dev, BNX2_HC_COMMAND, 4489 hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT); 4490 udelay(10); 4491 i++; 4492 barrier(); 4493 } 4494 if (sblk->status_completion_producer_index) 4495 goto failed; 4496 4497 } 4498 return 0; 4499 4500 failed: 4501 netdev_err(dev->netdev, "KCQ index not resetting to 0\n"); 4502 return -EBUSY; 4503 } 4504 4505 static void cnic_enable_bnx2_int(struct cnic_dev *dev) 4506 { 4507 struct cnic_local *cp = dev->cnic_priv; 4508 struct cnic_eth_dev *ethdev = cp->ethdev; 4509 4510 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4511 return; 4512 4513 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4514 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 4515 } 4516 4517 static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev) 4518 { 4519 struct cnic_local *cp = dev->cnic_priv; 4520 struct cnic_eth_dev *ethdev = cp->ethdev; 4521 4522 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4523 return; 4524 4525 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4526 BNX2_PCICFG_INT_ACK_CMD_MASK_INT); 4527 CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD); 4528 synchronize_irq(ethdev->irq_arr[0].vector); 4529 } 4530 4531 static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev) 4532 { 4533 struct cnic_local *cp = dev->cnic_priv; 4534 struct cnic_eth_dev *ethdev = cp->ethdev; 4535 struct cnic_uio_dev *udev = cp->udev; 4536 u32 cid_addr, tx_cid, sb_id; 4537 u32 val, offset0, offset1, offset2, offset3; 4538 int i; 4539 struct bnx2_tx_bd *txbd; 4540 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4541 struct status_block *s_blk = cp->status_blk.gen; 4542 4543 sb_id = cp->status_blk_num; 4544 tx_cid = 20; 4545 cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2; 4546 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4547 struct status_block_msix *sblk = cp->status_blk.bnx2; 4548 4549 tx_cid = TX_TSS_CID + sb_id - 1; 4550 CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) | 4551 (TX_TSS_CID << 7)); 4552 cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index; 4553 } 4554 cp->tx_cons = *cp->tx_cons_ptr; 4555 4556 cid_addr = GET_CID_ADDR(tx_cid); 4557 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 4558 u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40; 4559 4560 for (i = 0; i < PHY_CTX_SIZE; i += 4) 4561 cnic_ctx_wr(dev, cid_addr2, i, 0); 4562 4563 offset0 = BNX2_L2CTX_TYPE_XI; 4564 offset1 = BNX2_L2CTX_CMD_TYPE_XI; 4565 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI; 4566 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI; 4567 } else { 4568 cnic_init_context(dev, tx_cid); 4569 cnic_init_context(dev, tx_cid + 1); 4570 4571 offset0 = BNX2_L2CTX_TYPE; 4572 offset1 = BNX2_L2CTX_CMD_TYPE; 4573 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI; 4574 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO; 4575 } 4576 val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2; 4577 cnic_ctx_wr(dev, cid_addr, offset0, val); 4578 4579 val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16); 4580 cnic_ctx_wr(dev, cid_addr, offset1, val); 4581 4582 txbd = udev->l2_ring; 4583 4584 buf_map = udev->l2_buf_map; 4585 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) { 4586 txbd->tx_bd_haddr_hi = (u64) buf_map >> 32; 4587 txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4588 } 4589 val = (u64) ring_map >> 32; 4590 cnic_ctx_wr(dev, cid_addr, offset2, val); 4591 txbd->tx_bd_haddr_hi = val; 4592 4593 val = (u64) ring_map & 0xffffffff; 4594 cnic_ctx_wr(dev, cid_addr, offset3, val); 4595 txbd->tx_bd_haddr_lo = val; 4596 } 4597 4598 static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev) 4599 { 4600 struct cnic_local *cp = dev->cnic_priv; 4601 struct cnic_eth_dev *ethdev = cp->ethdev; 4602 struct cnic_uio_dev *udev = cp->udev; 4603 u32 cid_addr, sb_id, val, coal_reg, coal_val; 4604 int i; 4605 struct bnx2_rx_bd *rxbd; 4606 struct status_block *s_blk = cp->status_blk.gen; 4607 dma_addr_t ring_map = udev->l2_ring_map; 4608 4609 sb_id = cp->status_blk_num; 4610 cnic_init_context(dev, 2); 4611 cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2; 4612 coal_reg = BNX2_HC_COMMAND; 4613 coal_val = CNIC_RD(dev, coal_reg); 4614 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4615 struct status_block_msix *sblk = cp->status_blk.bnx2; 4616 4617 cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index; 4618 coal_reg = BNX2_HC_COALESCE_NOW; 4619 coal_val = 1 << (11 + sb_id); 4620 } 4621 i = 0; 4622 while (!(*cp->rx_cons_ptr != 0) && i < 10) { 4623 CNIC_WR(dev, coal_reg, coal_val); 4624 udelay(10); 4625 i++; 4626 barrier(); 4627 } 4628 cp->rx_cons = *cp->rx_cons_ptr; 4629 4630 cid_addr = GET_CID_ADDR(2); 4631 val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE | 4632 BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8); 4633 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val); 4634 4635 if (sb_id == 0) 4636 val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT; 4637 else 4638 val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id); 4639 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val); 4640 4641 rxbd = udev->l2_ring + CNIC_PAGE_SIZE; 4642 for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) { 4643 dma_addr_t buf_map; 4644 int n = (i % cp->l2_rx_ring_size) + 1; 4645 4646 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4647 rxbd->rx_bd_len = cp->l2_single_buf_size; 4648 rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END; 4649 rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32; 4650 rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4651 } 4652 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 4653 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val); 4654 rxbd->rx_bd_haddr_hi = val; 4655 4656 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 4657 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val); 4658 rxbd->rx_bd_haddr_lo = val; 4659 4660 val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD); 4661 cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2)); 4662 } 4663 4664 static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev) 4665 { 4666 struct kwqe *wqes[1], l2kwqe; 4667 4668 memset(&l2kwqe, 0, sizeof(l2kwqe)); 4669 wqes[0] = &l2kwqe; 4670 l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) | 4671 (L2_KWQE_OPCODE_VALUE_FLUSH << 4672 KWQE_OPCODE_SHIFT) | 2; 4673 dev->submit_kwqes(dev, wqes, 1); 4674 } 4675 4676 static void cnic_set_bnx2_mac(struct cnic_dev *dev) 4677 { 4678 struct cnic_local *cp = dev->cnic_priv; 4679 u32 val; 4680 4681 val = cp->func << 2; 4682 4683 cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val); 4684 4685 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4686 BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER); 4687 dev->mac_addr[0] = (u8) (val >> 8); 4688 dev->mac_addr[1] = (u8) val; 4689 4690 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val); 4691 4692 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4693 BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER); 4694 dev->mac_addr[2] = (u8) (val >> 24); 4695 dev->mac_addr[3] = (u8) (val >> 16); 4696 dev->mac_addr[4] = (u8) (val >> 8); 4697 dev->mac_addr[5] = (u8) val; 4698 4699 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val); 4700 4701 val = 4 | BNX2_RPM_SORT_USER2_BC_EN; 4702 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4703 val |= BNX2_RPM_SORT_USER2_PROM_VLAN; 4704 4705 CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0); 4706 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val); 4707 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA); 4708 } 4709 4710 static int cnic_start_bnx2_hw(struct cnic_dev *dev) 4711 { 4712 struct cnic_local *cp = dev->cnic_priv; 4713 struct cnic_eth_dev *ethdev = cp->ethdev; 4714 struct status_block *sblk = cp->status_blk.gen; 4715 u32 val, kcq_cid_addr, kwq_cid_addr; 4716 int err; 4717 4718 cnic_set_bnx2_mac(dev); 4719 4720 val = CNIC_RD(dev, BNX2_MQ_CONFIG); 4721 val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE; 4722 if (CNIC_PAGE_BITS > 12) 4723 val |= (12 - 8) << 4; 4724 else 4725 val |= (CNIC_PAGE_BITS - 8) << 4; 4726 4727 CNIC_WR(dev, BNX2_MQ_CONFIG, val); 4728 4729 CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8); 4730 CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220); 4731 CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220); 4732 4733 err = cnic_setup_5709_context(dev, 1); 4734 if (err) 4735 return err; 4736 4737 cnic_init_context(dev, KWQ_CID); 4738 cnic_init_context(dev, KCQ_CID); 4739 4740 kwq_cid_addr = GET_CID_ADDR(KWQ_CID); 4741 cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX; 4742 4743 cp->max_kwq_idx = MAX_KWQ_IDX; 4744 cp->kwq_prod_idx = 0; 4745 cp->kwq_con_idx = 0; 4746 set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 4747 4748 if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708) 4749 cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15; 4750 else 4751 cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index; 4752 4753 /* Initialize the kernel work queue context. */ 4754 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4755 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4756 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val); 4757 4758 val = (CNIC_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16; 4759 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4760 4761 val = ((CNIC_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT; 4762 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4763 4764 val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32); 4765 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4766 4767 val = (u32) cp->kwq_info.pgtbl_map; 4768 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4769 4770 kcq_cid_addr = GET_CID_ADDR(KCQ_CID); 4771 cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX; 4772 4773 cp->kcq1.sw_prod_idx = 0; 4774 cp->kcq1.hw_prod_idx_ptr = 4775 &sblk->status_completion_producer_index; 4776 4777 cp->kcq1.status_idx_ptr = &sblk->status_idx; 4778 4779 /* Initialize the kernel complete queue context. */ 4780 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4781 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4782 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val); 4783 4784 val = (CNIC_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16; 4785 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4786 4787 val = ((CNIC_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT; 4788 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4789 4790 val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32); 4791 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4792 4793 val = (u32) cp->kcq1.dma.pgtbl_map; 4794 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4795 4796 cp->int_num = 0; 4797 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4798 struct status_block_msix *msblk = cp->status_blk.bnx2; 4799 u32 sb_id = cp->status_blk_num; 4800 u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id); 4801 4802 cp->kcq1.hw_prod_idx_ptr = 4803 &msblk->status_completion_producer_index; 4804 cp->kcq1.status_idx_ptr = &msblk->status_idx; 4805 cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index; 4806 cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT; 4807 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4808 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4809 } 4810 4811 /* Enable Commnad Scheduler notification when we write to the 4812 * host producer index of the kernel contexts. */ 4813 CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2); 4814 4815 /* Enable Command Scheduler notification when we write to either 4816 * the Send Queue or Receive Queue producer indexes of the kernel 4817 * bypass contexts. */ 4818 CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7); 4819 CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7); 4820 4821 /* Notify COM when the driver post an application buffer. */ 4822 CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000); 4823 4824 /* Set the CP and COM doorbells. These two processors polls the 4825 * doorbell for a non zero value before running. This must be done 4826 * after setting up the kernel queue contexts. */ 4827 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1); 4828 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1); 4829 4830 cnic_init_bnx2_tx_ring(dev); 4831 cnic_init_bnx2_rx_ring(dev); 4832 4833 err = cnic_init_bnx2_irq(dev); 4834 if (err) { 4835 netdev_err(dev->netdev, "cnic_init_irq failed\n"); 4836 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 4837 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 4838 return err; 4839 } 4840 4841 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 4842 4843 return 0; 4844 } 4845 4846 static void cnic_setup_bnx2x_context(struct cnic_dev *dev) 4847 { 4848 struct cnic_local *cp = dev->cnic_priv; 4849 struct cnic_eth_dev *ethdev = cp->ethdev; 4850 u32 start_offset = ethdev->ctx_tbl_offset; 4851 int i; 4852 4853 for (i = 0; i < cp->ctx_blks; i++) { 4854 struct cnic_ctx *ctx = &cp->ctx_arr[i]; 4855 dma_addr_t map = ctx->mapping; 4856 4857 if (cp->ctx_align) { 4858 unsigned long mask = cp->ctx_align - 1; 4859 4860 map = (map + mask) & ~mask; 4861 } 4862 4863 cnic_ctx_tbl_wr(dev, start_offset + i, map); 4864 } 4865 } 4866 4867 static int cnic_init_bnx2x_irq(struct cnic_dev *dev) 4868 { 4869 struct cnic_local *cp = dev->cnic_priv; 4870 struct cnic_eth_dev *ethdev = cp->ethdev; 4871 int err = 0; 4872 4873 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2x_bh_work); 4874 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 4875 err = cnic_request_irq(dev); 4876 4877 return err; 4878 } 4879 4880 static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev, 4881 u16 sb_id, u8 sb_index, 4882 u8 disable) 4883 { 4884 struct bnx2x *bp = netdev_priv(dev->netdev); 4885 4886 u32 addr = BAR_CSTRORM_INTMEM + 4887 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4888 offsetof(struct hc_status_block_data_e1x, index_data) + 4889 sizeof(struct hc_index_data)*sb_index + 4890 offsetof(struct hc_index_data, flags); 4891 u16 flags = CNIC_RD16(dev, addr); 4892 /* clear and set */ 4893 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4894 flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) & 4895 HC_INDEX_DATA_HC_ENABLED); 4896 CNIC_WR16(dev, addr, flags); 4897 } 4898 4899 static void cnic_enable_bnx2x_int(struct cnic_dev *dev) 4900 { 4901 struct cnic_local *cp = dev->cnic_priv; 4902 struct bnx2x *bp = netdev_priv(dev->netdev); 4903 u8 sb_id = cp->status_blk_num; 4904 4905 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 4906 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4907 offsetof(struct hc_status_block_data_e1x, index_data) + 4908 sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS + 4909 offsetof(struct hc_index_data, timeout), 64 / 4); 4910 cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0); 4911 } 4912 4913 static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev) 4914 { 4915 } 4916 4917 static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev, 4918 struct client_init_ramrod_data *data) 4919 { 4920 struct cnic_local *cp = dev->cnic_priv; 4921 struct bnx2x *bp = netdev_priv(dev->netdev); 4922 struct cnic_uio_dev *udev = cp->udev; 4923 union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring; 4924 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4925 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4926 int i; 4927 u32 cli = cp->ethdev->iscsi_l2_client_id; 4928 u32 val; 4929 4930 memset(txbd, 0, CNIC_PAGE_SIZE); 4931 4932 buf_map = udev->l2_buf_map; 4933 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) { 4934 struct eth_tx_start_bd *start_bd = &txbd->start_bd; 4935 struct eth_tx_parse_bd_e1x *pbd_e1x = 4936 &((txbd + 1)->parse_bd_e1x); 4937 struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2); 4938 struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd); 4939 4940 start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4941 start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4942 reg_bd->addr_hi = start_bd->addr_hi; 4943 reg_bd->addr_lo = start_bd->addr_lo + 0x10; 4944 start_bd->nbytes = cpu_to_le16(0x10); 4945 start_bd->nbd = cpu_to_le16(3); 4946 start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 4947 start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS; 4948 start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 4949 4950 if (BNX2X_CHIP_IS_E2_PLUS(bp)) 4951 pbd_e2->parsing_data = (UNICAST_ADDRESS << 4952 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT); 4953 else 4954 pbd_e1x->global_data = (UNICAST_ADDRESS << 4955 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT); 4956 } 4957 4958 val = (u64) ring_map >> 32; 4959 txbd->next_bd.addr_hi = cpu_to_le32(val); 4960 4961 data->tx.tx_bd_page_base.hi = cpu_to_le32(val); 4962 4963 val = (u64) ring_map & 0xffffffff; 4964 txbd->next_bd.addr_lo = cpu_to_le32(val); 4965 4966 data->tx.tx_bd_page_base.lo = cpu_to_le32(val); 4967 4968 /* Other ramrod params */ 4969 data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS; 4970 data->tx.tx_status_block_id = BNX2X_DEF_SB_ID; 4971 4972 /* reset xstorm per client statistics */ 4973 if (cli < MAX_STAT_COUNTER_ID) { 4974 data->general.statistics_zero_flg = 1; 4975 data->general.statistics_en_flg = 1; 4976 data->general.statistics_counter_id = cli; 4977 } 4978 4979 cp->tx_cons_ptr = 4980 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS]; 4981 } 4982 4983 static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev, 4984 struct client_init_ramrod_data *data) 4985 { 4986 struct cnic_local *cp = dev->cnic_priv; 4987 struct bnx2x *bp = netdev_priv(dev->netdev); 4988 struct cnic_uio_dev *udev = cp->udev; 4989 struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring + 4990 CNIC_PAGE_SIZE); 4991 struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *) 4992 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 4993 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4994 int i; 4995 u32 cli = cp->ethdev->iscsi_l2_client_id; 4996 int cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 4997 u32 val; 4998 dma_addr_t ring_map = udev->l2_ring_map; 4999 5000 /* General data */ 5001 data->general.client_id = cli; 5002 data->general.activate_flg = 1; 5003 data->general.sp_client_id = cli; 5004 data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14); 5005 data->general.func_id = bp->pfid; 5006 5007 for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) { 5008 dma_addr_t buf_map; 5009 int n = (i % cp->l2_rx_ring_size) + 1; 5010 5011 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 5012 rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 5013 rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 5014 } 5015 5016 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 5017 rxbd->addr_hi = cpu_to_le32(val); 5018 data->rx.bd_page_base.hi = cpu_to_le32(val); 5019 5020 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 5021 rxbd->addr_lo = cpu_to_le32(val); 5022 data->rx.bd_page_base.lo = cpu_to_le32(val); 5023 5024 rxcqe += BNX2X_MAX_RCQ_DESC_CNT; 5025 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) >> 32; 5026 rxcqe->addr_hi = cpu_to_le32(val); 5027 data->rx.cqe_page_base.hi = cpu_to_le32(val); 5028 5029 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) & 0xffffffff; 5030 rxcqe->addr_lo = cpu_to_le32(val); 5031 data->rx.cqe_page_base.lo = cpu_to_le32(val); 5032 5033 /* Other ramrod params */ 5034 data->rx.client_qzone_id = cl_qzone_id; 5035 data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS; 5036 data->rx.status_block_id = BNX2X_DEF_SB_ID; 5037 5038 data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT; 5039 5040 data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size); 5041 data->rx.outer_vlan_removal_enable_flg = 1; 5042 data->rx.silent_vlan_removal_flg = 1; 5043 data->rx.silent_vlan_value = 0; 5044 data->rx.silent_vlan_mask = 0xffff; 5045 5046 cp->rx_cons_ptr = 5047 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS]; 5048 cp->rx_cons = *cp->rx_cons_ptr; 5049 } 5050 5051 static void cnic_init_bnx2x_kcq(struct cnic_dev *dev) 5052 { 5053 struct cnic_local *cp = dev->cnic_priv; 5054 struct bnx2x *bp = netdev_priv(dev->netdev); 5055 u32 pfid = bp->pfid; 5056 5057 cp->kcq1.io_addr = BAR_CSTRORM_INTMEM + 5058 CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0); 5059 cp->kcq1.sw_prod_idx = 0; 5060 5061 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5062 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5063 5064 cp->kcq1.hw_prod_idx_ptr = 5065 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5066 cp->kcq1.status_idx_ptr = 5067 &sb->sb.running_index[SM_RX_ID]; 5068 } else { 5069 struct host_hc_status_block_e1x *sb = cp->status_blk.gen; 5070 5071 cp->kcq1.hw_prod_idx_ptr = 5072 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5073 cp->kcq1.status_idx_ptr = 5074 &sb->sb.running_index[SM_RX_ID]; 5075 } 5076 5077 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5078 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5079 5080 cp->kcq2.io_addr = BAR_USTRORM_INTMEM + 5081 USTORM_FCOE_EQ_PROD_OFFSET(pfid); 5082 cp->kcq2.sw_prod_idx = 0; 5083 cp->kcq2.hw_prod_idx_ptr = 5084 &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS]; 5085 cp->kcq2.status_idx_ptr = 5086 &sb->sb.running_index[SM_RX_ID]; 5087 } 5088 } 5089 5090 static int cnic_start_bnx2x_hw(struct cnic_dev *dev) 5091 { 5092 struct cnic_local *cp = dev->cnic_priv; 5093 struct bnx2x *bp = netdev_priv(dev->netdev); 5094 struct cnic_eth_dev *ethdev = cp->ethdev; 5095 int ret; 5096 u32 pfid; 5097 5098 dev->stats_addr = ethdev->addr_drv_info_to_mcp; 5099 cp->func = bp->pf_num; 5100 5101 pfid = bp->pfid; 5102 5103 ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ, 5104 cp->iscsi_start_cid, 0); 5105 5106 if (ret) 5107 return -ENOMEM; 5108 5109 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5110 ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn, 5111 cp->fcoe_start_cid, 0); 5112 5113 if (ret) 5114 return -ENOMEM; 5115 } 5116 5117 cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2; 5118 5119 cnic_init_bnx2x_kcq(dev); 5120 5121 /* Only 1 EQ */ 5122 CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX); 5123 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5124 CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0); 5125 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5126 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0), 5127 cp->kcq1.dma.pg_map_arr[1] & 0xffffffff); 5128 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5129 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4, 5130 (u64) cp->kcq1.dma.pg_map_arr[1] >> 32); 5131 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5132 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0), 5133 cp->kcq1.dma.pg_map_arr[0] & 0xffffffff); 5134 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5135 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4, 5136 (u64) cp->kcq1.dma.pg_map_arr[0] >> 32); 5137 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5138 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1); 5139 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 5140 CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num); 5141 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5142 CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0), 5143 HC_INDEX_ISCSI_EQ_CONS); 5144 5145 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5146 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid), 5147 cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff); 5148 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5149 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4, 5150 (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32); 5151 5152 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 5153 TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF); 5154 5155 cnic_setup_bnx2x_context(dev); 5156 5157 ret = cnic_init_bnx2x_irq(dev); 5158 if (ret) 5159 return ret; 5160 5161 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 5162 return 0; 5163 } 5164 5165 static void cnic_init_rings(struct cnic_dev *dev) 5166 { 5167 struct cnic_local *cp = dev->cnic_priv; 5168 struct bnx2x *bp = netdev_priv(dev->netdev); 5169 struct cnic_uio_dev *udev = cp->udev; 5170 5171 if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5172 return; 5173 5174 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5175 cnic_init_bnx2_tx_ring(dev); 5176 cnic_init_bnx2_rx_ring(dev); 5177 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5178 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5179 u32 cli = cp->ethdev->iscsi_l2_client_id; 5180 u32 cid = cp->ethdev->iscsi_l2_cid; 5181 u32 cl_qzone_id; 5182 struct client_init_ramrod_data *data; 5183 union l5cm_specific_data l5_data; 5184 struct ustorm_eth_rx_producers rx_prods = {0}; 5185 u32 off, i, *cid_ptr; 5186 5187 rx_prods.bd_prod = 0; 5188 rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT; 5189 barrier(); 5190 5191 cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5192 5193 off = BAR_USTRORM_INTMEM + 5194 (BNX2X_CHIP_IS_E2_PLUS(bp) ? 5195 USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) : 5196 USTORM_RX_PRODS_E1X_OFFSET(BP_PORT(bp), cli)); 5197 5198 for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++) 5199 CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]); 5200 5201 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5202 5203 data = udev->l2_buf; 5204 cid_ptr = udev->l2_buf + 12; 5205 5206 memset(data, 0, sizeof(*data)); 5207 5208 cnic_init_bnx2x_tx_ring(dev, data); 5209 cnic_init_bnx2x_rx_ring(dev, data); 5210 5211 data->general.fp_hsi_ver = ETH_FP_HSI_VERSION; 5212 5213 l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff; 5214 l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32; 5215 5216 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5217 5218 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP, 5219 cid, ETH_CONNECTION_TYPE, &l5_data); 5220 5221 i = 0; 5222 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5223 ++i < 10) 5224 msleep(1); 5225 5226 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5227 netdev_err(dev->netdev, 5228 "iSCSI CLIENT_SETUP did not complete\n"); 5229 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5230 cnic_ring_ctl(dev, cid, cli, 1); 5231 *cid_ptr = cid >> 4; 5232 *(cid_ptr + 1) = cid * bp->db_size; 5233 *(cid_ptr + 2) = UIO_USE_TX_DOORBELL; 5234 } 5235 } 5236 5237 static void cnic_shutdown_rings(struct cnic_dev *dev) 5238 { 5239 struct cnic_local *cp = dev->cnic_priv; 5240 struct cnic_uio_dev *udev = cp->udev; 5241 void *rx_ring; 5242 5243 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5244 return; 5245 5246 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5247 cnic_shutdown_bnx2_rx_ring(dev); 5248 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5249 u32 cli = cp->ethdev->iscsi_l2_client_id; 5250 u32 cid = cp->ethdev->iscsi_l2_cid; 5251 union l5cm_specific_data l5_data; 5252 int i; 5253 5254 cnic_ring_ctl(dev, cid, cli, 0); 5255 5256 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5257 5258 l5_data.phy_address.lo = cli; 5259 l5_data.phy_address.hi = 0; 5260 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT, 5261 cid, ETH_CONNECTION_TYPE, &l5_data); 5262 i = 0; 5263 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5264 ++i < 10) 5265 msleep(1); 5266 5267 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5268 netdev_err(dev->netdev, 5269 "iSCSI CLIENT_HALT did not complete\n"); 5270 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5271 5272 memset(&l5_data, 0, sizeof(l5_data)); 5273 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 5274 cid, NONE_CONNECTION_TYPE, &l5_data); 5275 msleep(10); 5276 } 5277 clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5278 rx_ring = udev->l2_ring + CNIC_PAGE_SIZE; 5279 memset(rx_ring, 0, CNIC_PAGE_SIZE); 5280 } 5281 5282 static int cnic_register_netdev(struct cnic_dev *dev) 5283 { 5284 struct cnic_local *cp = dev->cnic_priv; 5285 struct cnic_eth_dev *ethdev = cp->ethdev; 5286 int err; 5287 5288 if (!ethdev) 5289 return -ENODEV; 5290 5291 if (ethdev->drv_state & CNIC_DRV_STATE_REGD) 5292 return 0; 5293 5294 err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev); 5295 if (err) 5296 netdev_err(dev->netdev, "register_cnic failed\n"); 5297 5298 /* Read iSCSI config again. On some bnx2x device, iSCSI config 5299 * can change after firmware is downloaded. 5300 */ 5301 dev->max_iscsi_conn = ethdev->max_iscsi_conn; 5302 if (ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 5303 dev->max_iscsi_conn = 0; 5304 5305 return err; 5306 } 5307 5308 static void cnic_unregister_netdev(struct cnic_dev *dev) 5309 { 5310 struct cnic_local *cp = dev->cnic_priv; 5311 struct cnic_eth_dev *ethdev = cp->ethdev; 5312 5313 if (!ethdev) 5314 return; 5315 5316 ethdev->drv_unregister_cnic(dev->netdev); 5317 } 5318 5319 static int cnic_start_hw(struct cnic_dev *dev) 5320 { 5321 struct cnic_local *cp = dev->cnic_priv; 5322 struct cnic_eth_dev *ethdev = cp->ethdev; 5323 int err; 5324 5325 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5326 return -EALREADY; 5327 5328 dev->regview = ethdev->io_base; 5329 pci_dev_get(dev->pcidev); 5330 cp->func = PCI_FUNC(dev->pcidev->devfn); 5331 cp->status_blk.gen = ethdev->irq_arr[0].status_blk; 5332 cp->status_blk_map = ethdev->irq_arr[0].status_blk_map; 5333 cp->status_blk_num = ethdev->irq_arr[0].status_blk_num; 5334 5335 err = cp->alloc_resc(dev); 5336 if (err) { 5337 netdev_err(dev->netdev, "allocate resource failure\n"); 5338 goto err1; 5339 } 5340 5341 err = cp->start_hw(dev); 5342 if (err) 5343 goto err1; 5344 5345 err = cnic_cm_open(dev); 5346 if (err) 5347 goto err1; 5348 5349 set_bit(CNIC_F_CNIC_UP, &dev->flags); 5350 5351 cp->enable_int(dev); 5352 5353 return 0; 5354 5355 err1: 5356 if (ethdev->drv_state & CNIC_DRV_STATE_HANDLES_IRQ) 5357 cp->stop_hw(dev); 5358 else 5359 cp->free_resc(dev); 5360 pci_dev_put(dev->pcidev); 5361 return err; 5362 } 5363 5364 static void cnic_stop_bnx2_hw(struct cnic_dev *dev) 5365 { 5366 cnic_disable_bnx2_int_sync(dev); 5367 5368 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 5369 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 5370 5371 cnic_init_context(dev, KWQ_CID); 5372 cnic_init_context(dev, KCQ_CID); 5373 5374 cnic_setup_5709_context(dev, 0); 5375 cnic_free_irq(dev); 5376 5377 cnic_free_resc(dev); 5378 } 5379 5380 5381 static void cnic_stop_bnx2x_hw(struct cnic_dev *dev) 5382 { 5383 struct cnic_local *cp = dev->cnic_priv; 5384 struct bnx2x *bp = netdev_priv(dev->netdev); 5385 u32 hc_index = HC_INDEX_ISCSI_EQ_CONS; 5386 u32 sb_id = cp->status_blk_num; 5387 u32 idx_off, syn_off; 5388 5389 cnic_free_irq(dev); 5390 5391 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5392 idx_off = offsetof(struct hc_status_block_e2, index_values) + 5393 (hc_index * sizeof(u16)); 5394 5395 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id); 5396 } else { 5397 idx_off = offsetof(struct hc_status_block_e1x, index_values) + 5398 (hc_index * sizeof(u16)); 5399 5400 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id); 5401 } 5402 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0); 5403 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) + 5404 idx_off, 0); 5405 5406 *cp->kcq1.hw_prod_idx_ptr = 0; 5407 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5408 CSTORM_ISCSI_EQ_CONS_OFFSET(bp->pfid, 0), 0); 5409 CNIC_WR16(dev, cp->kcq1.io_addr, 0); 5410 cnic_free_resc(dev); 5411 } 5412 5413 static void cnic_stop_hw(struct cnic_dev *dev) 5414 { 5415 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5416 struct cnic_local *cp = dev->cnic_priv; 5417 int i = 0; 5418 5419 /* Need to wait for the ring shutdown event to complete 5420 * before clearing the CNIC_UP flag. 5421 */ 5422 while (cp->udev && cp->udev->uio_dev != -1 && i < 15) { 5423 msleep(100); 5424 i++; 5425 } 5426 cnic_shutdown_rings(dev); 5427 cp->stop_cm(dev); 5428 cp->ethdev->drv_state &= ~CNIC_DRV_STATE_HANDLES_IRQ; 5429 clear_bit(CNIC_F_CNIC_UP, &dev->flags); 5430 RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL); 5431 synchronize_rcu(); 5432 cnic_cm_shutdown(dev); 5433 cp->stop_hw(dev); 5434 pci_dev_put(dev->pcidev); 5435 } 5436 } 5437 5438 static void cnic_free_dev(struct cnic_dev *dev) 5439 { 5440 int i = 0; 5441 5442 while ((atomic_read(&dev->ref_count) != 0) && i < 10) { 5443 msleep(100); 5444 i++; 5445 } 5446 if (atomic_read(&dev->ref_count) != 0) 5447 netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n"); 5448 5449 netdev_info(dev->netdev, "Removed CNIC device\n"); 5450 dev_put(dev->netdev); 5451 kfree(dev); 5452 } 5453 5454 static int cnic_get_fc_npiv_tbl(struct cnic_dev *dev, 5455 struct cnic_fc_npiv_tbl *npiv_tbl) 5456 { 5457 struct cnic_local *cp = dev->cnic_priv; 5458 struct bnx2x *bp = netdev_priv(dev->netdev); 5459 int ret; 5460 5461 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5462 return -EAGAIN; /* bnx2x is down */ 5463 5464 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 5465 return -EINVAL; 5466 5467 ret = cp->ethdev->drv_get_fc_npiv_tbl(dev->netdev, npiv_tbl); 5468 return ret; 5469 } 5470 5471 static struct cnic_dev *cnic_alloc_dev(struct net_device *dev, 5472 struct pci_dev *pdev) 5473 { 5474 struct cnic_dev *cdev; 5475 struct cnic_local *cp; 5476 int alloc_size; 5477 5478 alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local); 5479 5480 cdev = kzalloc(alloc_size, GFP_KERNEL); 5481 if (cdev == NULL) 5482 return NULL; 5483 5484 cdev->netdev = dev; 5485 cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev); 5486 cdev->register_device = cnic_register_device; 5487 cdev->unregister_device = cnic_unregister_device; 5488 cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv; 5489 cdev->get_fc_npiv_tbl = cnic_get_fc_npiv_tbl; 5490 atomic_set(&cdev->ref_count, 0); 5491 5492 cp = cdev->cnic_priv; 5493 cp->dev = cdev; 5494 cp->l2_single_buf_size = 0x400; 5495 cp->l2_rx_ring_size = 3; 5496 5497 spin_lock_init(&cp->cnic_ulp_lock); 5498 5499 netdev_info(dev, "Added CNIC device\n"); 5500 5501 return cdev; 5502 } 5503 5504 static struct cnic_dev *init_bnx2_cnic(struct net_device *dev) 5505 { 5506 struct pci_dev *pdev; 5507 struct cnic_dev *cdev; 5508 struct cnic_local *cp; 5509 struct bnx2 *bp = netdev_priv(dev); 5510 struct cnic_eth_dev *ethdev = NULL; 5511 5512 if (bp->cnic_probe) 5513 ethdev = (bp->cnic_probe)(dev); 5514 5515 if (!ethdev) 5516 return NULL; 5517 5518 pdev = ethdev->pdev; 5519 if (!pdev) 5520 return NULL; 5521 5522 dev_hold(dev); 5523 pci_dev_get(pdev); 5524 if ((pdev->device == PCI_DEVICE_ID_NX2_5709 || 5525 pdev->device == PCI_DEVICE_ID_NX2_5709S) && 5526 (pdev->revision < 0x10)) { 5527 pci_dev_put(pdev); 5528 goto cnic_err; 5529 } 5530 pci_dev_put(pdev); 5531 5532 cdev = cnic_alloc_dev(dev, pdev); 5533 if (cdev == NULL) 5534 goto cnic_err; 5535 5536 set_bit(CNIC_F_BNX2_CLASS, &cdev->flags); 5537 cdev->submit_kwqes = cnic_submit_bnx2_kwqes; 5538 5539 cp = cdev->cnic_priv; 5540 cp->ethdev = ethdev; 5541 cdev->pcidev = pdev; 5542 cp->chip_id = ethdev->chip_id; 5543 5544 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5545 5546 cp->cnic_ops = &cnic_bnx2_ops; 5547 cp->start_hw = cnic_start_bnx2_hw; 5548 cp->stop_hw = cnic_stop_bnx2_hw; 5549 cp->setup_pgtbl = cnic_setup_page_tbl; 5550 cp->alloc_resc = cnic_alloc_bnx2_resc; 5551 cp->free_resc = cnic_free_resc; 5552 cp->start_cm = cnic_cm_init_bnx2_hw; 5553 cp->stop_cm = cnic_cm_stop_bnx2_hw; 5554 cp->enable_int = cnic_enable_bnx2_int; 5555 cp->disable_int_sync = cnic_disable_bnx2_int_sync; 5556 cp->close_conn = cnic_close_bnx2_conn; 5557 return cdev; 5558 5559 cnic_err: 5560 dev_put(dev); 5561 return NULL; 5562 } 5563 5564 static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev) 5565 { 5566 struct pci_dev *pdev; 5567 struct cnic_dev *cdev; 5568 struct cnic_local *cp; 5569 struct bnx2x *bp = netdev_priv(dev); 5570 struct cnic_eth_dev *ethdev = NULL; 5571 5572 if (bp->cnic_probe) 5573 ethdev = bp->cnic_probe(dev); 5574 5575 if (!ethdev) 5576 return NULL; 5577 5578 pdev = ethdev->pdev; 5579 if (!pdev) 5580 return NULL; 5581 5582 dev_hold(dev); 5583 cdev = cnic_alloc_dev(dev, pdev); 5584 if (cdev == NULL) { 5585 dev_put(dev); 5586 return NULL; 5587 } 5588 5589 set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags); 5590 cdev->submit_kwqes = cnic_submit_bnx2x_kwqes; 5591 5592 cp = cdev->cnic_priv; 5593 cp->ethdev = ethdev; 5594 cdev->pcidev = pdev; 5595 cp->chip_id = ethdev->chip_id; 5596 5597 cdev->stats_addr = ethdev->addr_drv_info_to_mcp; 5598 5599 if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)) 5600 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5601 if (CNIC_SUPPORTS_FCOE(bp)) { 5602 cdev->max_fcoe_conn = ethdev->max_fcoe_conn; 5603 cdev->max_fcoe_exchanges = ethdev->max_fcoe_exchanges; 5604 } 5605 5606 if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS) 5607 cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS; 5608 5609 memcpy(cdev->mac_addr, ethdev->iscsi_mac, ETH_ALEN); 5610 5611 cp->cnic_ops = &cnic_bnx2x_ops; 5612 cp->start_hw = cnic_start_bnx2x_hw; 5613 cp->stop_hw = cnic_stop_bnx2x_hw; 5614 cp->setup_pgtbl = cnic_setup_page_tbl_le; 5615 cp->alloc_resc = cnic_alloc_bnx2x_resc; 5616 cp->free_resc = cnic_free_resc; 5617 cp->start_cm = cnic_cm_init_bnx2x_hw; 5618 cp->stop_cm = cnic_cm_stop_bnx2x_hw; 5619 cp->enable_int = cnic_enable_bnx2x_int; 5620 cp->disable_int_sync = cnic_disable_bnx2x_int_sync; 5621 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5622 cp->ack_int = cnic_ack_bnx2x_e2_msix; 5623 cp->arm_int = cnic_arm_bnx2x_e2_msix; 5624 } else { 5625 cp->ack_int = cnic_ack_bnx2x_msix; 5626 cp->arm_int = cnic_arm_bnx2x_msix; 5627 } 5628 cp->close_conn = cnic_close_bnx2x_conn; 5629 return cdev; 5630 } 5631 5632 static struct cnic_dev *is_cnic_dev(struct net_device *dev) 5633 { 5634 struct ethtool_drvinfo drvinfo; 5635 struct cnic_dev *cdev = NULL; 5636 5637 if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) { 5638 memset(&drvinfo, 0, sizeof(drvinfo)); 5639 dev->ethtool_ops->get_drvinfo(dev, &drvinfo); 5640 5641 if (!strcmp(drvinfo.driver, "bnx2")) 5642 cdev = init_bnx2_cnic(dev); 5643 if (!strcmp(drvinfo.driver, "bnx2x")) 5644 cdev = init_bnx2x_cnic(dev); 5645 if (cdev) { 5646 write_lock(&cnic_dev_lock); 5647 list_add(&cdev->list, &cnic_dev_list); 5648 write_unlock(&cnic_dev_lock); 5649 } 5650 } 5651 return cdev; 5652 } 5653 5654 static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event, 5655 u16 vlan_id) 5656 { 5657 int if_type; 5658 5659 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 5660 struct cnic_ulp_ops *ulp_ops; 5661 void *ctx; 5662 5663 mutex_lock(&cnic_lock); 5664 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 5665 lockdep_is_held(&cnic_lock)); 5666 if (!ulp_ops || !ulp_ops->indicate_netevent) { 5667 mutex_unlock(&cnic_lock); 5668 continue; 5669 } 5670 5671 ctx = cp->ulp_handle[if_type]; 5672 5673 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5674 mutex_unlock(&cnic_lock); 5675 5676 ulp_ops->indicate_netevent(ctx, event, vlan_id); 5677 5678 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5679 } 5680 } 5681 5682 /* netdev event handler */ 5683 static int cnic_netdev_event(struct notifier_block *this, unsigned long event, 5684 void *ptr) 5685 { 5686 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 5687 struct cnic_dev *dev; 5688 int new_dev = 0; 5689 5690 dev = cnic_from_netdev(netdev); 5691 5692 if (!dev && event == NETDEV_REGISTER) { 5693 /* Check for the hot-plug device */ 5694 dev = is_cnic_dev(netdev); 5695 if (dev) { 5696 new_dev = 1; 5697 cnic_hold(dev); 5698 } 5699 } 5700 if (dev) { 5701 struct cnic_local *cp = dev->cnic_priv; 5702 5703 if (new_dev) 5704 cnic_ulp_init(dev); 5705 else if (event == NETDEV_UNREGISTER) 5706 cnic_ulp_exit(dev); 5707 5708 if (event == NETDEV_UP) { 5709 if (cnic_register_netdev(dev) != 0) { 5710 cnic_put(dev); 5711 goto done; 5712 } 5713 if (!cnic_start_hw(dev)) 5714 cnic_ulp_start(dev); 5715 } 5716 5717 cnic_rcv_netevent(cp, event, 0); 5718 5719 if (event == NETDEV_GOING_DOWN) { 5720 cnic_ulp_stop(dev); 5721 cnic_stop_hw(dev); 5722 cnic_unregister_netdev(dev); 5723 } else if (event == NETDEV_UNREGISTER) { 5724 write_lock(&cnic_dev_lock); 5725 list_del_init(&dev->list); 5726 write_unlock(&cnic_dev_lock); 5727 5728 cnic_put(dev); 5729 cnic_free_dev(dev); 5730 goto done; 5731 } 5732 cnic_put(dev); 5733 } else { 5734 struct net_device *realdev; 5735 u16 vid; 5736 5737 vid = cnic_get_vlan(netdev, &realdev); 5738 if (realdev) { 5739 dev = cnic_from_netdev(realdev); 5740 if (dev) { 5741 vid |= VLAN_CFI_MASK; /* make non-zero */ 5742 cnic_rcv_netevent(dev->cnic_priv, event, vid); 5743 cnic_put(dev); 5744 } 5745 } 5746 } 5747 done: 5748 return NOTIFY_DONE; 5749 } 5750 5751 static struct notifier_block cnic_netdev_notifier = { 5752 .notifier_call = cnic_netdev_event 5753 }; 5754 5755 static void cnic_release(void) 5756 { 5757 struct cnic_uio_dev *udev; 5758 5759 while (!list_empty(&cnic_udev_list)) { 5760 udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev, 5761 list); 5762 cnic_free_uio(udev); 5763 } 5764 } 5765 5766 static int __init cnic_init(void) 5767 { 5768 int rc = 0; 5769 5770 pr_info("%s", version); 5771 5772 rc = register_netdevice_notifier(&cnic_netdev_notifier); 5773 if (rc) { 5774 cnic_release(); 5775 return rc; 5776 } 5777 5778 cnic_wq = create_singlethread_workqueue("cnic_wq"); 5779 if (!cnic_wq) { 5780 cnic_release(); 5781 unregister_netdevice_notifier(&cnic_netdev_notifier); 5782 return -ENOMEM; 5783 } 5784 5785 return 0; 5786 } 5787 5788 static void __exit cnic_exit(void) 5789 { 5790 unregister_netdevice_notifier(&cnic_netdev_notifier); 5791 cnic_release(); 5792 destroy_workqueue(cnic_wq); 5793 } 5794 5795 module_init(cnic_init); 5796 module_exit(cnic_exit); 5797