1 /* cnic.c: QLogic CNIC core network driver. 2 * 3 * Copyright (c) 2006-2014 Broadcom Corporation 4 * Copyright (c) 2014-2015 QLogic Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com) 11 * Previously modified and maintained by: Michael Chan <mchan@broadcom.com> 12 * Maintained By: Dept-HSGLinuxNICDev@qlogic.com 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/module.h> 18 19 #include <linux/kernel.h> 20 #include <linux/errno.h> 21 #include <linux/list.h> 22 #include <linux/slab.h> 23 #include <linux/pci.h> 24 #include <linux/init.h> 25 #include <linux/netdevice.h> 26 #include <linux/uio_driver.h> 27 #include <linux/in.h> 28 #include <linux/dma-mapping.h> 29 #include <linux/delay.h> 30 #include <linux/ethtool.h> 31 #include <linux/if_vlan.h> 32 #include <linux/prefetch.h> 33 #include <linux/random.h> 34 #include <linux/workqueue.h> 35 #if IS_ENABLED(CONFIG_VLAN_8021Q) 36 #define BCM_VLAN 1 37 #endif 38 #include <net/ip.h> 39 #include <net/tcp.h> 40 #include <net/route.h> 41 #include <net/ipv6.h> 42 #include <net/ip6_route.h> 43 #include <net/ip6_checksum.h> 44 #include <scsi/iscsi_if.h> 45 46 #define BCM_CNIC 1 47 #include "cnic_if.h" 48 #include "bnx2.h" 49 #include "bnx2x/bnx2x.h" 50 #include "bnx2x/bnx2x_reg.h" 51 #include "bnx2x/bnx2x_fw_defs.h" 52 #include "bnx2x/bnx2x_hsi.h" 53 #include "../../../scsi/bnx2i/57xx_iscsi_constants.h" 54 #include "../../../scsi/bnx2i/57xx_iscsi_hsi.h" 55 #include "../../../scsi/bnx2fc/bnx2fc_constants.h" 56 #include "cnic.h" 57 #include "cnic_defs.h" 58 59 #define CNIC_MODULE_NAME "cnic" 60 61 static char version[] = 62 "QLogic " CNIC_MODULE_NAME "Driver v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n"; 63 64 MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) " 65 "Chen (zongxi@broadcom.com"); 66 MODULE_DESCRIPTION("QLogic cnic Driver"); 67 MODULE_LICENSE("GPL"); 68 MODULE_VERSION(CNIC_MODULE_VERSION); 69 70 /* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */ 71 static LIST_HEAD(cnic_dev_list); 72 static LIST_HEAD(cnic_udev_list); 73 static DEFINE_RWLOCK(cnic_dev_lock); 74 static DEFINE_MUTEX(cnic_lock); 75 76 static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE]; 77 78 /* helper function, assuming cnic_lock is held */ 79 static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type) 80 { 81 return rcu_dereference_protected(cnic_ulp_tbl[type], 82 lockdep_is_held(&cnic_lock)); 83 } 84 85 static int cnic_service_bnx2(void *, void *); 86 static int cnic_service_bnx2x(void *, void *); 87 static int cnic_ctl(void *, struct cnic_ctl_info *); 88 89 static struct cnic_ops cnic_bnx2_ops = { 90 .cnic_owner = THIS_MODULE, 91 .cnic_handler = cnic_service_bnx2, 92 .cnic_ctl = cnic_ctl, 93 }; 94 95 static struct cnic_ops cnic_bnx2x_ops = { 96 .cnic_owner = THIS_MODULE, 97 .cnic_handler = cnic_service_bnx2x, 98 .cnic_ctl = cnic_ctl, 99 }; 100 101 static struct workqueue_struct *cnic_wq; 102 103 static void cnic_shutdown_rings(struct cnic_dev *); 104 static void cnic_init_rings(struct cnic_dev *); 105 static int cnic_cm_set_pg(struct cnic_sock *); 106 107 static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode) 108 { 109 struct cnic_uio_dev *udev = uinfo->priv; 110 struct cnic_dev *dev; 111 112 if (!capable(CAP_NET_ADMIN)) 113 return -EPERM; 114 115 if (udev->uio_dev != -1) 116 return -EBUSY; 117 118 rtnl_lock(); 119 dev = udev->dev; 120 121 if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 122 rtnl_unlock(); 123 return -ENODEV; 124 } 125 126 udev->uio_dev = iminor(inode); 127 128 cnic_shutdown_rings(dev); 129 cnic_init_rings(dev); 130 rtnl_unlock(); 131 132 return 0; 133 } 134 135 static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode) 136 { 137 struct cnic_uio_dev *udev = uinfo->priv; 138 139 udev->uio_dev = -1; 140 return 0; 141 } 142 143 static inline void cnic_hold(struct cnic_dev *dev) 144 { 145 atomic_inc(&dev->ref_count); 146 } 147 148 static inline void cnic_put(struct cnic_dev *dev) 149 { 150 atomic_dec(&dev->ref_count); 151 } 152 153 static inline void csk_hold(struct cnic_sock *csk) 154 { 155 atomic_inc(&csk->ref_count); 156 } 157 158 static inline void csk_put(struct cnic_sock *csk) 159 { 160 atomic_dec(&csk->ref_count); 161 } 162 163 static struct cnic_dev *cnic_from_netdev(struct net_device *netdev) 164 { 165 struct cnic_dev *cdev; 166 167 read_lock(&cnic_dev_lock); 168 list_for_each_entry(cdev, &cnic_dev_list, list) { 169 if (netdev == cdev->netdev) { 170 cnic_hold(cdev); 171 read_unlock(&cnic_dev_lock); 172 return cdev; 173 } 174 } 175 read_unlock(&cnic_dev_lock); 176 return NULL; 177 } 178 179 static inline void ulp_get(struct cnic_ulp_ops *ulp_ops) 180 { 181 atomic_inc(&ulp_ops->ref_count); 182 } 183 184 static inline void ulp_put(struct cnic_ulp_ops *ulp_ops) 185 { 186 atomic_dec(&ulp_ops->ref_count); 187 } 188 189 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val) 190 { 191 struct cnic_local *cp = dev->cnic_priv; 192 struct cnic_eth_dev *ethdev = cp->ethdev; 193 struct drv_ctl_info info; 194 struct drv_ctl_io *io = &info.data.io; 195 196 memset(&info, 0, sizeof(struct drv_ctl_info)); 197 info.cmd = DRV_CTL_CTX_WR_CMD; 198 io->cid_addr = cid_addr; 199 io->offset = off; 200 io->data = val; 201 ethdev->drv_ctl(dev->netdev, &info); 202 } 203 204 static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr) 205 { 206 struct cnic_local *cp = dev->cnic_priv; 207 struct cnic_eth_dev *ethdev = cp->ethdev; 208 struct drv_ctl_info info; 209 struct drv_ctl_io *io = &info.data.io; 210 211 memset(&info, 0, sizeof(struct drv_ctl_info)); 212 info.cmd = DRV_CTL_CTXTBL_WR_CMD; 213 io->offset = off; 214 io->dma_addr = addr; 215 ethdev->drv_ctl(dev->netdev, &info); 216 } 217 218 static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start) 219 { 220 struct cnic_local *cp = dev->cnic_priv; 221 struct cnic_eth_dev *ethdev = cp->ethdev; 222 struct drv_ctl_info info; 223 struct drv_ctl_l2_ring *ring = &info.data.ring; 224 225 memset(&info, 0, sizeof(struct drv_ctl_info)); 226 if (start) 227 info.cmd = DRV_CTL_START_L2_CMD; 228 else 229 info.cmd = DRV_CTL_STOP_L2_CMD; 230 231 ring->cid = cid; 232 ring->client_id = cl_id; 233 ethdev->drv_ctl(dev->netdev, &info); 234 } 235 236 static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val) 237 { 238 struct cnic_local *cp = dev->cnic_priv; 239 struct cnic_eth_dev *ethdev = cp->ethdev; 240 struct drv_ctl_info info; 241 struct drv_ctl_io *io = &info.data.io; 242 243 memset(&info, 0, sizeof(struct drv_ctl_info)); 244 info.cmd = DRV_CTL_IO_WR_CMD; 245 io->offset = off; 246 io->data = val; 247 ethdev->drv_ctl(dev->netdev, &info); 248 } 249 250 static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off) 251 { 252 struct cnic_local *cp = dev->cnic_priv; 253 struct cnic_eth_dev *ethdev = cp->ethdev; 254 struct drv_ctl_info info; 255 struct drv_ctl_io *io = &info.data.io; 256 257 memset(&info, 0, sizeof(struct drv_ctl_info)); 258 info.cmd = DRV_CTL_IO_RD_CMD; 259 io->offset = off; 260 ethdev->drv_ctl(dev->netdev, &info); 261 return io->data; 262 } 263 264 static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg, int state) 265 { 266 struct cnic_local *cp = dev->cnic_priv; 267 struct cnic_eth_dev *ethdev = cp->ethdev; 268 struct drv_ctl_info info; 269 struct fcoe_capabilities *fcoe_cap = 270 &info.data.register_data.fcoe_features; 271 272 memset(&info, 0, sizeof(struct drv_ctl_info)); 273 if (reg) { 274 info.cmd = DRV_CTL_ULP_REGISTER_CMD; 275 if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap) 276 memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap)); 277 } else { 278 info.cmd = DRV_CTL_ULP_UNREGISTER_CMD; 279 } 280 281 info.data.ulp_type = ulp_type; 282 info.drv_state = state; 283 ethdev->drv_ctl(dev->netdev, &info); 284 } 285 286 static int cnic_in_use(struct cnic_sock *csk) 287 { 288 return test_bit(SK_F_INUSE, &csk->flags); 289 } 290 291 static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count) 292 { 293 struct cnic_local *cp = dev->cnic_priv; 294 struct cnic_eth_dev *ethdev = cp->ethdev; 295 struct drv_ctl_info info; 296 297 memset(&info, 0, sizeof(struct drv_ctl_info)); 298 info.cmd = cmd; 299 info.data.credit.credit_count = count; 300 ethdev->drv_ctl(dev->netdev, &info); 301 } 302 303 static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid) 304 { 305 u32 i; 306 307 if (!cp->ctx_tbl) 308 return -EINVAL; 309 310 for (i = 0; i < cp->max_cid_space; i++) { 311 if (cp->ctx_tbl[i].cid == cid) { 312 *l5_cid = i; 313 return 0; 314 } 315 } 316 return -EINVAL; 317 } 318 319 static int cnic_send_nlmsg(struct cnic_local *cp, u32 type, 320 struct cnic_sock *csk) 321 { 322 struct iscsi_path path_req; 323 char *buf = NULL; 324 u16 len = 0; 325 u32 msg_type = ISCSI_KEVENT_IF_DOWN; 326 struct cnic_ulp_ops *ulp_ops; 327 struct cnic_uio_dev *udev = cp->udev; 328 int rc = 0, retry = 0; 329 330 if (!udev || udev->uio_dev == -1) 331 return -ENODEV; 332 333 if (csk) { 334 len = sizeof(path_req); 335 buf = (char *) &path_req; 336 memset(&path_req, 0, len); 337 338 msg_type = ISCSI_KEVENT_PATH_REQ; 339 path_req.handle = (u64) csk->l5_cid; 340 if (test_bit(SK_F_IPV6, &csk->flags)) { 341 memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0], 342 sizeof(struct in6_addr)); 343 path_req.ip_addr_len = 16; 344 } else { 345 memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0], 346 sizeof(struct in_addr)); 347 path_req.ip_addr_len = 4; 348 } 349 path_req.vlan_id = csk->vlan_id; 350 path_req.pmtu = csk->mtu; 351 } 352 353 while (retry < 3) { 354 rc = 0; 355 rcu_read_lock(); 356 ulp_ops = rcu_dereference(cp->ulp_ops[CNIC_ULP_ISCSI]); 357 if (ulp_ops) 358 rc = ulp_ops->iscsi_nl_send_msg( 359 cp->ulp_handle[CNIC_ULP_ISCSI], 360 msg_type, buf, len); 361 rcu_read_unlock(); 362 if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ) 363 break; 364 365 msleep(100); 366 retry++; 367 } 368 return rc; 369 } 370 371 static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8); 372 373 static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type, 374 char *buf, u16 len) 375 { 376 int rc = -EINVAL; 377 378 switch (msg_type) { 379 case ISCSI_UEVENT_PATH_UPDATE: { 380 struct cnic_local *cp; 381 u32 l5_cid; 382 struct cnic_sock *csk; 383 struct iscsi_path *path_resp; 384 385 if (len < sizeof(*path_resp)) 386 break; 387 388 path_resp = (struct iscsi_path *) buf; 389 cp = dev->cnic_priv; 390 l5_cid = (u32) path_resp->handle; 391 if (l5_cid >= MAX_CM_SK_TBL_SZ) 392 break; 393 394 if (!rcu_access_pointer(cp->ulp_ops[CNIC_ULP_L4])) { 395 rc = -ENODEV; 396 break; 397 } 398 csk = &cp->csk_tbl[l5_cid]; 399 csk_hold(csk); 400 if (cnic_in_use(csk) && 401 test_bit(SK_F_CONNECT_START, &csk->flags)) { 402 403 csk->vlan_id = path_resp->vlan_id; 404 405 memcpy(csk->ha, path_resp->mac_addr, ETH_ALEN); 406 if (test_bit(SK_F_IPV6, &csk->flags)) 407 memcpy(&csk->src_ip[0], &path_resp->src.v6_addr, 408 sizeof(struct in6_addr)); 409 else 410 memcpy(&csk->src_ip[0], &path_resp->src.v4_addr, 411 sizeof(struct in_addr)); 412 413 if (is_valid_ether_addr(csk->ha)) { 414 cnic_cm_set_pg(csk); 415 } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) && 416 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 417 418 cnic_cm_upcall(cp, csk, 419 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 420 clear_bit(SK_F_CONNECT_START, &csk->flags); 421 } 422 } 423 csk_put(csk); 424 rc = 0; 425 } 426 } 427 428 return rc; 429 } 430 431 static int cnic_offld_prep(struct cnic_sock *csk) 432 { 433 if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 434 return 0; 435 436 if (!test_bit(SK_F_CONNECT_START, &csk->flags)) { 437 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 438 return 0; 439 } 440 441 return 1; 442 } 443 444 static int cnic_close_prep(struct cnic_sock *csk) 445 { 446 clear_bit(SK_F_CONNECT_START, &csk->flags); 447 smp_mb__after_atomic(); 448 449 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 450 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 451 msleep(1); 452 453 return 1; 454 } 455 return 0; 456 } 457 458 static int cnic_abort_prep(struct cnic_sock *csk) 459 { 460 clear_bit(SK_F_CONNECT_START, &csk->flags); 461 smp_mb__after_atomic(); 462 463 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 464 msleep(1); 465 466 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 467 csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP; 468 return 1; 469 } 470 471 return 0; 472 } 473 474 int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops) 475 { 476 struct cnic_dev *dev; 477 478 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 479 pr_err("%s: Bad type %d\n", __func__, ulp_type); 480 return -EINVAL; 481 } 482 mutex_lock(&cnic_lock); 483 if (cnic_ulp_tbl_prot(ulp_type)) { 484 pr_err("%s: Type %d has already been registered\n", 485 __func__, ulp_type); 486 mutex_unlock(&cnic_lock); 487 return -EBUSY; 488 } 489 490 read_lock(&cnic_dev_lock); 491 list_for_each_entry(dev, &cnic_dev_list, list) { 492 struct cnic_local *cp = dev->cnic_priv; 493 494 clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]); 495 } 496 read_unlock(&cnic_dev_lock); 497 498 atomic_set(&ulp_ops->ref_count, 0); 499 rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops); 500 mutex_unlock(&cnic_lock); 501 502 /* Prevent race conditions with netdev_event */ 503 rtnl_lock(); 504 list_for_each_entry(dev, &cnic_dev_list, list) { 505 struct cnic_local *cp = dev->cnic_priv; 506 507 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 508 ulp_ops->cnic_init(dev); 509 } 510 rtnl_unlock(); 511 512 return 0; 513 } 514 515 int cnic_unregister_driver(int ulp_type) 516 { 517 struct cnic_dev *dev; 518 struct cnic_ulp_ops *ulp_ops; 519 int i = 0; 520 521 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 522 pr_err("%s: Bad type %d\n", __func__, ulp_type); 523 return -EINVAL; 524 } 525 mutex_lock(&cnic_lock); 526 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 527 if (!ulp_ops) { 528 pr_err("%s: Type %d has not been registered\n", 529 __func__, ulp_type); 530 goto out_unlock; 531 } 532 read_lock(&cnic_dev_lock); 533 list_for_each_entry(dev, &cnic_dev_list, list) { 534 struct cnic_local *cp = dev->cnic_priv; 535 536 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 537 pr_err("%s: Type %d still has devices registered\n", 538 __func__, ulp_type); 539 read_unlock(&cnic_dev_lock); 540 goto out_unlock; 541 } 542 } 543 read_unlock(&cnic_dev_lock); 544 545 RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL); 546 547 mutex_unlock(&cnic_lock); 548 synchronize_rcu(); 549 while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) { 550 msleep(100); 551 i++; 552 } 553 554 if (atomic_read(&ulp_ops->ref_count) != 0) 555 pr_warn("%s: Failed waiting for ref count to go to zero\n", 556 __func__); 557 return 0; 558 559 out_unlock: 560 mutex_unlock(&cnic_lock); 561 return -EINVAL; 562 } 563 564 static int cnic_start_hw(struct cnic_dev *); 565 static void cnic_stop_hw(struct cnic_dev *); 566 567 static int cnic_register_device(struct cnic_dev *dev, int ulp_type, 568 void *ulp_ctx) 569 { 570 struct cnic_local *cp = dev->cnic_priv; 571 struct cnic_ulp_ops *ulp_ops; 572 573 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 574 pr_err("%s: Bad type %d\n", __func__, ulp_type); 575 return -EINVAL; 576 } 577 mutex_lock(&cnic_lock); 578 if (cnic_ulp_tbl_prot(ulp_type) == NULL) { 579 pr_err("%s: Driver with type %d has not been registered\n", 580 __func__, ulp_type); 581 mutex_unlock(&cnic_lock); 582 return -EAGAIN; 583 } 584 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 585 pr_err("%s: Type %d has already been registered to this device\n", 586 __func__, ulp_type); 587 mutex_unlock(&cnic_lock); 588 return -EBUSY; 589 } 590 591 clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]); 592 cp->ulp_handle[ulp_type] = ulp_ctx; 593 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 594 rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops); 595 cnic_hold(dev); 596 597 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 598 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type])) 599 ulp_ops->cnic_start(cp->ulp_handle[ulp_type]); 600 601 mutex_unlock(&cnic_lock); 602 603 cnic_ulp_ctl(dev, ulp_type, true, DRV_ACTIVE); 604 605 return 0; 606 607 } 608 EXPORT_SYMBOL(cnic_register_driver); 609 610 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type) 611 { 612 struct cnic_local *cp = dev->cnic_priv; 613 int i = 0; 614 615 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 616 pr_err("%s: Bad type %d\n", __func__, ulp_type); 617 return -EINVAL; 618 } 619 620 if (ulp_type == CNIC_ULP_ISCSI) 621 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 622 623 mutex_lock(&cnic_lock); 624 if (rcu_access_pointer(cp->ulp_ops[ulp_type])) { 625 RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL); 626 cnic_put(dev); 627 } else { 628 pr_err("%s: device not registered to this ulp type %d\n", 629 __func__, ulp_type); 630 mutex_unlock(&cnic_lock); 631 return -EINVAL; 632 } 633 mutex_unlock(&cnic_lock); 634 635 if (ulp_type == CNIC_ULP_FCOE) 636 dev->fcoe_cap = NULL; 637 638 synchronize_rcu(); 639 640 while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) && 641 i < 20) { 642 msleep(100); 643 i++; 644 } 645 if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type])) 646 netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n"); 647 648 if (test_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 649 cnic_ulp_ctl(dev, ulp_type, false, DRV_UNLOADED); 650 else 651 cnic_ulp_ctl(dev, ulp_type, false, DRV_INACTIVE); 652 653 return 0; 654 } 655 EXPORT_SYMBOL(cnic_unregister_driver); 656 657 static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id, 658 u32 next) 659 { 660 id_tbl->start = start_id; 661 id_tbl->max = size; 662 id_tbl->next = next; 663 spin_lock_init(&id_tbl->lock); 664 id_tbl->table = bitmap_zalloc(size, GFP_KERNEL); 665 if (!id_tbl->table) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl) 672 { 673 bitmap_free(id_tbl->table); 674 id_tbl->table = NULL; 675 } 676 677 static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id) 678 { 679 int ret = -1; 680 681 id -= id_tbl->start; 682 if (id >= id_tbl->max) 683 return ret; 684 685 spin_lock(&id_tbl->lock); 686 if (!test_bit(id, id_tbl->table)) { 687 set_bit(id, id_tbl->table); 688 ret = 0; 689 } 690 spin_unlock(&id_tbl->lock); 691 return ret; 692 } 693 694 /* Returns -1 if not successful */ 695 static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl) 696 { 697 u32 id; 698 699 spin_lock(&id_tbl->lock); 700 id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next); 701 if (id >= id_tbl->max) { 702 id = -1; 703 if (id_tbl->next != 0) { 704 id = find_first_zero_bit(id_tbl->table, id_tbl->next); 705 if (id >= id_tbl->next) 706 id = -1; 707 } 708 } 709 710 if (id < id_tbl->max) { 711 set_bit(id, id_tbl->table); 712 id_tbl->next = (id + 1) & (id_tbl->max - 1); 713 id += id_tbl->start; 714 } 715 716 spin_unlock(&id_tbl->lock); 717 718 return id; 719 } 720 721 static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id) 722 { 723 if (id == -1) 724 return; 725 726 id -= id_tbl->start; 727 if (id >= id_tbl->max) 728 return; 729 730 clear_bit(id, id_tbl->table); 731 } 732 733 static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma) 734 { 735 int i; 736 737 if (!dma->pg_arr) 738 return; 739 740 for (i = 0; i < dma->num_pages; i++) { 741 if (dma->pg_arr[i]) { 742 dma_free_coherent(&dev->pcidev->dev, CNIC_PAGE_SIZE, 743 dma->pg_arr[i], dma->pg_map_arr[i]); 744 dma->pg_arr[i] = NULL; 745 } 746 } 747 if (dma->pgtbl) { 748 dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size, 749 dma->pgtbl, dma->pgtbl_map); 750 dma->pgtbl = NULL; 751 } 752 kfree(dma->pg_arr); 753 dma->pg_arr = NULL; 754 dma->num_pages = 0; 755 } 756 757 static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma) 758 { 759 int i; 760 __le32 *page_table = (__le32 *) dma->pgtbl; 761 762 for (i = 0; i < dma->num_pages; i++) { 763 /* Each entry needs to be in big endian format. */ 764 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 765 page_table++; 766 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 767 page_table++; 768 } 769 } 770 771 static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma) 772 { 773 int i; 774 __le32 *page_table = (__le32 *) dma->pgtbl; 775 776 for (i = 0; i < dma->num_pages; i++) { 777 /* Each entry needs to be in little endian format. */ 778 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 779 page_table++; 780 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 781 page_table++; 782 } 783 } 784 785 static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma, 786 int pages, int use_pg_tbl) 787 { 788 int i, size; 789 struct cnic_local *cp = dev->cnic_priv; 790 791 size = pages * (sizeof(void *) + sizeof(dma_addr_t)); 792 dma->pg_arr = kzalloc(size, GFP_ATOMIC); 793 if (dma->pg_arr == NULL) 794 return -ENOMEM; 795 796 dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages); 797 dma->num_pages = pages; 798 799 for (i = 0; i < pages; i++) { 800 dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev, 801 CNIC_PAGE_SIZE, 802 &dma->pg_map_arr[i], 803 GFP_ATOMIC); 804 if (dma->pg_arr[i] == NULL) 805 goto error; 806 } 807 if (!use_pg_tbl) 808 return 0; 809 810 dma->pgtbl_size = ((pages * 8) + CNIC_PAGE_SIZE - 1) & 811 ~(CNIC_PAGE_SIZE - 1); 812 dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size, 813 &dma->pgtbl_map, GFP_ATOMIC); 814 if (dma->pgtbl == NULL) 815 goto error; 816 817 cp->setup_pgtbl(dev, dma); 818 819 return 0; 820 821 error: 822 cnic_free_dma(dev, dma); 823 return -ENOMEM; 824 } 825 826 static void cnic_free_context(struct cnic_dev *dev) 827 { 828 struct cnic_local *cp = dev->cnic_priv; 829 int i; 830 831 for (i = 0; i < cp->ctx_blks; i++) { 832 if (cp->ctx_arr[i].ctx) { 833 dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 834 cp->ctx_arr[i].ctx, 835 cp->ctx_arr[i].mapping); 836 cp->ctx_arr[i].ctx = NULL; 837 } 838 } 839 } 840 841 static void __cnic_free_uio_rings(struct cnic_uio_dev *udev) 842 { 843 if (udev->l2_buf) { 844 dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size, 845 udev->l2_buf, udev->l2_buf_map); 846 udev->l2_buf = NULL; 847 } 848 849 if (udev->l2_ring) { 850 dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size, 851 udev->l2_ring, udev->l2_ring_map); 852 udev->l2_ring = NULL; 853 } 854 855 } 856 857 static void __cnic_free_uio(struct cnic_uio_dev *udev) 858 { 859 uio_unregister_device(&udev->cnic_uinfo); 860 861 __cnic_free_uio_rings(udev); 862 863 pci_dev_put(udev->pdev); 864 kfree(udev); 865 } 866 867 static void cnic_free_uio(struct cnic_uio_dev *udev) 868 { 869 if (!udev) 870 return; 871 872 write_lock(&cnic_dev_lock); 873 list_del_init(&udev->list); 874 write_unlock(&cnic_dev_lock); 875 __cnic_free_uio(udev); 876 } 877 878 static void cnic_free_resc(struct cnic_dev *dev) 879 { 880 struct cnic_local *cp = dev->cnic_priv; 881 struct cnic_uio_dev *udev = cp->udev; 882 883 if (udev) { 884 udev->dev = NULL; 885 cp->udev = NULL; 886 if (udev->uio_dev == -1) 887 __cnic_free_uio_rings(udev); 888 } 889 890 cnic_free_context(dev); 891 kfree(cp->ctx_arr); 892 cp->ctx_arr = NULL; 893 cp->ctx_blks = 0; 894 895 cnic_free_dma(dev, &cp->gbl_buf_info); 896 cnic_free_dma(dev, &cp->kwq_info); 897 cnic_free_dma(dev, &cp->kwq_16_data_info); 898 cnic_free_dma(dev, &cp->kcq2.dma); 899 cnic_free_dma(dev, &cp->kcq1.dma); 900 kfree(cp->iscsi_tbl); 901 cp->iscsi_tbl = NULL; 902 kfree(cp->ctx_tbl); 903 cp->ctx_tbl = NULL; 904 905 cnic_free_id_tbl(&cp->fcoe_cid_tbl); 906 cnic_free_id_tbl(&cp->cid_tbl); 907 } 908 909 static int cnic_alloc_context(struct cnic_dev *dev) 910 { 911 struct cnic_local *cp = dev->cnic_priv; 912 913 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 914 int i, k, arr_size; 915 916 cp->ctx_blk_size = CNIC_PAGE_SIZE; 917 cp->cids_per_blk = CNIC_PAGE_SIZE / 128; 918 arr_size = BNX2_MAX_CID / cp->cids_per_blk * 919 sizeof(struct cnic_ctx); 920 cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL); 921 if (cp->ctx_arr == NULL) 922 return -ENOMEM; 923 924 k = 0; 925 for (i = 0; i < 2; i++) { 926 u32 j, reg, off, lo, hi; 927 928 if (i == 0) 929 off = BNX2_PG_CTX_MAP; 930 else 931 off = BNX2_ISCSI_CTX_MAP; 932 933 reg = cnic_reg_rd_ind(dev, off); 934 lo = reg >> 16; 935 hi = reg & 0xffff; 936 for (j = lo; j < hi; j += cp->cids_per_blk, k++) 937 cp->ctx_arr[k].cid = j; 938 } 939 940 cp->ctx_blks = k; 941 if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) { 942 cp->ctx_blks = 0; 943 return -ENOMEM; 944 } 945 946 for (i = 0; i < cp->ctx_blks; i++) { 947 cp->ctx_arr[i].ctx = 948 dma_alloc_coherent(&dev->pcidev->dev, 949 CNIC_PAGE_SIZE, 950 &cp->ctx_arr[i].mapping, 951 GFP_KERNEL); 952 if (cp->ctx_arr[i].ctx == NULL) 953 return -ENOMEM; 954 } 955 } 956 return 0; 957 } 958 959 static u16 cnic_bnx2_next_idx(u16 idx) 960 { 961 return idx + 1; 962 } 963 964 static u16 cnic_bnx2_hw_idx(u16 idx) 965 { 966 return idx; 967 } 968 969 static u16 cnic_bnx2x_next_idx(u16 idx) 970 { 971 idx++; 972 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 973 idx++; 974 975 return idx; 976 } 977 978 static u16 cnic_bnx2x_hw_idx(u16 idx) 979 { 980 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 981 idx++; 982 return idx; 983 } 984 985 static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info, 986 bool use_pg_tbl) 987 { 988 int err, i, use_page_tbl = 0; 989 struct kcqe **kcq; 990 991 if (use_pg_tbl) 992 use_page_tbl = 1; 993 994 err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl); 995 if (err) 996 return err; 997 998 kcq = (struct kcqe **) info->dma.pg_arr; 999 info->kcq = kcq; 1000 1001 info->next_idx = cnic_bnx2_next_idx; 1002 info->hw_idx = cnic_bnx2_hw_idx; 1003 if (use_pg_tbl) 1004 return 0; 1005 1006 info->next_idx = cnic_bnx2x_next_idx; 1007 info->hw_idx = cnic_bnx2x_hw_idx; 1008 1009 for (i = 0; i < KCQ_PAGE_CNT; i++) { 1010 struct bnx2x_bd_chain_next *next = 1011 (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT]; 1012 int j = i + 1; 1013 1014 if (j >= KCQ_PAGE_CNT) 1015 j = 0; 1016 next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32; 1017 next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff; 1018 } 1019 return 0; 1020 } 1021 1022 static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages) 1023 { 1024 struct cnic_local *cp = udev->dev->cnic_priv; 1025 1026 if (udev->l2_ring) 1027 return 0; 1028 1029 udev->l2_ring_size = pages * CNIC_PAGE_SIZE; 1030 udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size, 1031 &udev->l2_ring_map, GFP_KERNEL); 1032 if (!udev->l2_ring) 1033 return -ENOMEM; 1034 1035 udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size; 1036 udev->l2_buf_size = CNIC_PAGE_ALIGN(udev->l2_buf_size); 1037 udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size, 1038 &udev->l2_buf_map, GFP_KERNEL); 1039 if (!udev->l2_buf) { 1040 __cnic_free_uio_rings(udev); 1041 return -ENOMEM; 1042 } 1043 1044 return 0; 1045 1046 } 1047 1048 static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages) 1049 { 1050 struct cnic_local *cp = dev->cnic_priv; 1051 struct cnic_uio_dev *udev; 1052 1053 list_for_each_entry(udev, &cnic_udev_list, list) { 1054 if (udev->pdev == dev->pcidev) { 1055 udev->dev = dev; 1056 if (__cnic_alloc_uio_rings(udev, pages)) { 1057 udev->dev = NULL; 1058 return -ENOMEM; 1059 } 1060 cp->udev = udev; 1061 return 0; 1062 } 1063 } 1064 1065 udev = kzalloc(sizeof(struct cnic_uio_dev), GFP_ATOMIC); 1066 if (!udev) 1067 return -ENOMEM; 1068 1069 udev->uio_dev = -1; 1070 1071 udev->dev = dev; 1072 udev->pdev = dev->pcidev; 1073 1074 if (__cnic_alloc_uio_rings(udev, pages)) 1075 goto err_udev; 1076 1077 list_add(&udev->list, &cnic_udev_list); 1078 1079 pci_dev_get(udev->pdev); 1080 1081 cp->udev = udev; 1082 1083 return 0; 1084 1085 err_udev: 1086 kfree(udev); 1087 return -ENOMEM; 1088 } 1089 1090 static int cnic_init_uio(struct cnic_dev *dev) 1091 { 1092 struct cnic_local *cp = dev->cnic_priv; 1093 struct cnic_uio_dev *udev = cp->udev; 1094 struct uio_info *uinfo; 1095 int ret = 0; 1096 1097 if (!udev) 1098 return -ENOMEM; 1099 1100 uinfo = &udev->cnic_uinfo; 1101 1102 uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0); 1103 uinfo->mem[0].internal_addr = dev->regview; 1104 uinfo->mem[0].memtype = UIO_MEM_PHYS; 1105 1106 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 1107 uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID + 1108 TX_MAX_TSS_RINGS + 1); 1109 uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen & 1110 CNIC_PAGE_MASK; 1111 uinfo->mem[1].dma_addr = cp->status_blk_map; 1112 if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 1113 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE * 9); 1114 else 1115 uinfo->mem[1].size = PAGE_ALIGN(BNX2_SBLK_MSIX_ALIGN_SIZE); 1116 1117 uinfo->name = "bnx2_cnic"; 1118 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 1119 uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0); 1120 1121 uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk & 1122 CNIC_PAGE_MASK; 1123 uinfo->mem[1].dma_addr = cp->status_blk_map; 1124 uinfo->mem[1].size = PAGE_ALIGN(sizeof(*cp->bnx2x_def_status_blk)); 1125 1126 uinfo->name = "bnx2x_cnic"; 1127 } 1128 1129 uinfo->mem[1].dma_device = &dev->pcidev->dev; 1130 uinfo->mem[1].memtype = UIO_MEM_DMA_COHERENT; 1131 1132 uinfo->mem[2].addr = (unsigned long) udev->l2_ring; 1133 uinfo->mem[2].dma_addr = udev->l2_ring_map; 1134 uinfo->mem[2].size = PAGE_ALIGN(udev->l2_ring_size); 1135 uinfo->mem[2].dma_device = &dev->pcidev->dev; 1136 uinfo->mem[2].memtype = UIO_MEM_DMA_COHERENT; 1137 1138 uinfo->mem[3].addr = (unsigned long) udev->l2_buf; 1139 uinfo->mem[3].dma_addr = udev->l2_buf_map; 1140 uinfo->mem[3].size = PAGE_ALIGN(udev->l2_buf_size); 1141 uinfo->mem[3].dma_device = &dev->pcidev->dev; 1142 uinfo->mem[3].memtype = UIO_MEM_DMA_COHERENT; 1143 1144 uinfo->version = CNIC_MODULE_VERSION; 1145 uinfo->irq = UIO_IRQ_CUSTOM; 1146 1147 uinfo->open = cnic_uio_open; 1148 uinfo->release = cnic_uio_close; 1149 1150 if (udev->uio_dev == -1) { 1151 if (!uinfo->priv) { 1152 uinfo->priv = udev; 1153 1154 ret = uio_register_device(&udev->pdev->dev, uinfo); 1155 } 1156 } else { 1157 cnic_init_rings(dev); 1158 } 1159 1160 return ret; 1161 } 1162 1163 static int cnic_alloc_bnx2_resc(struct cnic_dev *dev) 1164 { 1165 struct cnic_local *cp = dev->cnic_priv; 1166 int ret; 1167 1168 ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1); 1169 if (ret) 1170 goto error; 1171 cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr; 1172 1173 ret = cnic_alloc_kcq(dev, &cp->kcq1, true); 1174 if (ret) 1175 goto error; 1176 1177 ret = cnic_alloc_context(dev); 1178 if (ret) 1179 goto error; 1180 1181 ret = cnic_alloc_uio_rings(dev, 2); 1182 if (ret) 1183 goto error; 1184 1185 ret = cnic_init_uio(dev); 1186 if (ret) 1187 goto error; 1188 1189 return 0; 1190 1191 error: 1192 cnic_free_resc(dev); 1193 return ret; 1194 } 1195 1196 static int cnic_alloc_bnx2x_context(struct cnic_dev *dev) 1197 { 1198 struct cnic_local *cp = dev->cnic_priv; 1199 struct bnx2x *bp = netdev_priv(dev->netdev); 1200 int ctx_blk_size = cp->ethdev->ctx_blk_size; 1201 int total_mem, blks, i; 1202 1203 total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space; 1204 blks = total_mem / ctx_blk_size; 1205 if (total_mem % ctx_blk_size) 1206 blks++; 1207 1208 if (blks > cp->ethdev->ctx_tbl_len) 1209 return -ENOMEM; 1210 1211 cp->ctx_arr = kcalloc(blks, sizeof(struct cnic_ctx), GFP_KERNEL); 1212 if (cp->ctx_arr == NULL) 1213 return -ENOMEM; 1214 1215 cp->ctx_blks = blks; 1216 cp->ctx_blk_size = ctx_blk_size; 1217 if (!CHIP_IS_E1(bp)) 1218 cp->ctx_align = 0; 1219 else 1220 cp->ctx_align = ctx_blk_size; 1221 1222 cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE; 1223 1224 for (i = 0; i < blks; i++) { 1225 cp->ctx_arr[i].ctx = 1226 dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 1227 &cp->ctx_arr[i].mapping, 1228 GFP_KERNEL); 1229 if (cp->ctx_arr[i].ctx == NULL) 1230 return -ENOMEM; 1231 1232 if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) { 1233 if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) { 1234 cnic_free_context(dev); 1235 cp->ctx_blk_size += cp->ctx_align; 1236 i = -1; 1237 continue; 1238 } 1239 } 1240 } 1241 return 0; 1242 } 1243 1244 static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev) 1245 { 1246 struct cnic_local *cp = dev->cnic_priv; 1247 struct bnx2x *bp = netdev_priv(dev->netdev); 1248 struct cnic_eth_dev *ethdev = cp->ethdev; 1249 u32 start_cid = ethdev->starting_cid; 1250 int i, j, n, ret, pages; 1251 struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info; 1252 1253 cp->max_cid_space = MAX_ISCSI_TBL_SZ; 1254 cp->iscsi_start_cid = start_cid; 1255 cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ; 1256 1257 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 1258 cp->max_cid_space += dev->max_fcoe_conn; 1259 cp->fcoe_init_cid = ethdev->fcoe_init_cid; 1260 if (!cp->fcoe_init_cid) 1261 cp->fcoe_init_cid = 0x10; 1262 } 1263 1264 cp->iscsi_tbl = kcalloc(MAX_ISCSI_TBL_SZ, sizeof(struct cnic_iscsi), 1265 GFP_KERNEL); 1266 if (!cp->iscsi_tbl) 1267 goto error; 1268 1269 cp->ctx_tbl = kcalloc(cp->max_cid_space, sizeof(struct cnic_context), 1270 GFP_KERNEL); 1271 if (!cp->ctx_tbl) 1272 goto error; 1273 1274 for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) { 1275 cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i]; 1276 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI; 1277 } 1278 1279 for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++) 1280 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE; 1281 1282 pages = CNIC_PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) / 1283 CNIC_PAGE_SIZE; 1284 1285 ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0); 1286 if (ret) 1287 goto error; 1288 1289 n = CNIC_PAGE_SIZE / CNIC_KWQ16_DATA_SIZE; 1290 for (i = 0, j = 0; i < cp->max_cid_space; i++) { 1291 long off = CNIC_KWQ16_DATA_SIZE * (i % n); 1292 1293 cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off; 1294 cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] + 1295 off; 1296 1297 if ((i % n) == (n - 1)) 1298 j++; 1299 } 1300 1301 ret = cnic_alloc_kcq(dev, &cp->kcq1, false); 1302 if (ret) 1303 goto error; 1304 1305 if (CNIC_SUPPORTS_FCOE(bp)) { 1306 ret = cnic_alloc_kcq(dev, &cp->kcq2, true); 1307 if (ret) 1308 goto error; 1309 } 1310 1311 pages = CNIC_PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / CNIC_PAGE_SIZE; 1312 ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0); 1313 if (ret) 1314 goto error; 1315 1316 ret = cnic_alloc_bnx2x_context(dev); 1317 if (ret) 1318 goto error; 1319 1320 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 1321 return 0; 1322 1323 cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk; 1324 cp->status_blk_map = cp->ethdev->irq_arr[1].status_blk_map; 1325 1326 cp->l2_rx_ring_size = 15; 1327 1328 ret = cnic_alloc_uio_rings(dev, 4); 1329 if (ret) 1330 goto error; 1331 1332 ret = cnic_init_uio(dev); 1333 if (ret) 1334 goto error; 1335 1336 return 0; 1337 1338 error: 1339 cnic_free_resc(dev); 1340 return -ENOMEM; 1341 } 1342 1343 static inline u32 cnic_kwq_avail(struct cnic_local *cp) 1344 { 1345 return cp->max_kwq_idx - 1346 ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx); 1347 } 1348 1349 static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 1350 u32 num_wqes) 1351 { 1352 struct cnic_local *cp = dev->cnic_priv; 1353 struct kwqe *prod_qe; 1354 u16 prod, sw_prod, i; 1355 1356 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 1357 return -EAGAIN; /* bnx2 is down */ 1358 1359 spin_lock_bh(&cp->cnic_ulp_lock); 1360 if (num_wqes > cnic_kwq_avail(cp) && 1361 !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) { 1362 spin_unlock_bh(&cp->cnic_ulp_lock); 1363 return -EAGAIN; 1364 } 1365 1366 clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 1367 1368 prod = cp->kwq_prod_idx; 1369 sw_prod = prod & MAX_KWQ_IDX; 1370 for (i = 0; i < num_wqes; i++) { 1371 prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)]; 1372 memcpy(prod_qe, wqes[i], sizeof(struct kwqe)); 1373 prod++; 1374 sw_prod = prod & MAX_KWQ_IDX; 1375 } 1376 cp->kwq_prod_idx = prod; 1377 1378 CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx); 1379 1380 spin_unlock_bh(&cp->cnic_ulp_lock); 1381 return 0; 1382 } 1383 1384 static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid, 1385 union l5cm_specific_data *l5_data) 1386 { 1387 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1388 dma_addr_t map; 1389 1390 map = ctx->kwqe_data_mapping; 1391 l5_data->phy_address.lo = (u64) map & 0xffffffff; 1392 l5_data->phy_address.hi = (u64) map >> 32; 1393 return ctx->kwqe_data; 1394 } 1395 1396 static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid, 1397 u32 type, union l5cm_specific_data *l5_data) 1398 { 1399 struct cnic_local *cp = dev->cnic_priv; 1400 struct bnx2x *bp = netdev_priv(dev->netdev); 1401 struct l5cm_spe kwqe; 1402 struct kwqe_16 *kwq[1]; 1403 u16 type_16; 1404 int ret; 1405 1406 kwqe.hdr.conn_and_cmd_data = 1407 cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) | 1408 BNX2X_HW_CID(bp, cid))); 1409 1410 type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE; 1411 type_16 |= (bp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) & 1412 SPE_HDR_FUNCTION_ID; 1413 1414 kwqe.hdr.type = cpu_to_le16(type_16); 1415 kwqe.hdr.reserved1 = 0; 1416 kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo); 1417 kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi); 1418 1419 kwq[0] = (struct kwqe_16 *) &kwqe; 1420 1421 spin_lock_bh(&cp->cnic_ulp_lock); 1422 ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1); 1423 spin_unlock_bh(&cp->cnic_ulp_lock); 1424 1425 if (ret == 1) 1426 return 0; 1427 1428 return ret; 1429 } 1430 1431 static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type, 1432 struct kcqe *cqes[], u32 num_cqes) 1433 { 1434 struct cnic_local *cp = dev->cnic_priv; 1435 struct cnic_ulp_ops *ulp_ops; 1436 1437 rcu_read_lock(); 1438 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 1439 if (likely(ulp_ops)) { 1440 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 1441 cqes, num_cqes); 1442 } 1443 rcu_read_unlock(); 1444 } 1445 1446 static void cnic_bnx2x_set_tcp_options(struct cnic_dev *dev, int time_stamps, 1447 int en_tcp_dack) 1448 { 1449 struct bnx2x *bp = netdev_priv(dev->netdev); 1450 u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN; 1451 u16 tstorm_flags = 0; 1452 1453 if (time_stamps) { 1454 xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1455 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1456 } 1457 if (en_tcp_dack) 1458 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_DELAYED_ACK_EN; 1459 1460 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1461 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), xstorm_flags); 1462 1463 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1464 TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), tstorm_flags); 1465 } 1466 1467 static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe) 1468 { 1469 struct cnic_local *cp = dev->cnic_priv; 1470 struct bnx2x *bp = netdev_priv(dev->netdev); 1471 struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe; 1472 int hq_bds, pages; 1473 u32 pfid = bp->pfid; 1474 1475 cp->num_iscsi_tasks = req1->num_tasks_per_conn; 1476 cp->num_ccells = req1->num_ccells_per_conn; 1477 cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE * 1478 cp->num_iscsi_tasks; 1479 cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS * 1480 BNX2X_ISCSI_R2TQE_SIZE; 1481 cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE; 1482 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1483 hq_bds = pages * (CNIC_PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE); 1484 cp->num_cqs = req1->num_cqs; 1485 1486 if (!dev->max_iscsi_conn) 1487 return 0; 1488 1489 /* init Tstorm RAM */ 1490 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1491 req1->rq_num_wqes); 1492 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1493 CNIC_PAGE_SIZE); 1494 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 1495 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1496 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1497 TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1498 req1->num_tasks_per_conn); 1499 1500 /* init Ustorm RAM */ 1501 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1502 USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid), 1503 req1->rq_buffer_size); 1504 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1505 CNIC_PAGE_SIZE); 1506 CNIC_WR8(dev, BAR_USTRORM_INTMEM + 1507 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1508 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1509 USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1510 req1->num_tasks_per_conn); 1511 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1512 req1->rq_num_wqes); 1513 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1514 req1->cq_num_wqes); 1515 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1516 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1517 1518 /* init Xstorm RAM */ 1519 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1520 CNIC_PAGE_SIZE); 1521 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1522 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1523 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 1524 XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1525 req1->num_tasks_per_conn); 1526 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1527 hq_bds); 1528 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid), 1529 req1->num_tasks_per_conn); 1530 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1531 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1532 1533 /* init Cstorm RAM */ 1534 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1535 CNIC_PAGE_SIZE); 1536 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 1537 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS); 1538 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1539 CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1540 req1->num_tasks_per_conn); 1541 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1542 req1->cq_num_wqes); 1543 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1544 hq_bds); 1545 1546 cnic_bnx2x_set_tcp_options(dev, 1547 req1->flags & ISCSI_KWQE_INIT1_TIME_STAMPS_ENABLE, 1548 req1->flags & ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE); 1549 1550 return 0; 1551 } 1552 1553 static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe) 1554 { 1555 struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe; 1556 struct bnx2x *bp = netdev_priv(dev->netdev); 1557 u32 pfid = bp->pfid; 1558 struct iscsi_kcqe kcqe; 1559 struct kcqe *cqes[1]; 1560 1561 memset(&kcqe, 0, sizeof(kcqe)); 1562 if (!dev->max_iscsi_conn) { 1563 kcqe.completion_status = 1564 ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED; 1565 goto done; 1566 } 1567 1568 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1569 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1570 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1571 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1572 req2->error_bit_map[1]); 1573 1574 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1575 USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1576 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1577 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1578 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1579 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1580 req2->error_bit_map[1]); 1581 1582 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1583 CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1584 1585 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1586 1587 done: 1588 kcqe.op_code = ISCSI_KCQE_OPCODE_INIT; 1589 cqes[0] = (struct kcqe *) &kcqe; 1590 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1591 1592 return 0; 1593 } 1594 1595 static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1596 { 1597 struct cnic_local *cp = dev->cnic_priv; 1598 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1599 1600 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) { 1601 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1602 1603 cnic_free_dma(dev, &iscsi->hq_info); 1604 cnic_free_dma(dev, &iscsi->r2tq_info); 1605 cnic_free_dma(dev, &iscsi->task_array_info); 1606 cnic_free_id(&cp->cid_tbl, ctx->cid); 1607 } else { 1608 cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid); 1609 } 1610 1611 ctx->cid = 0; 1612 } 1613 1614 static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1615 { 1616 u32 cid; 1617 int ret, pages; 1618 struct cnic_local *cp = dev->cnic_priv; 1619 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1620 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1621 1622 if (ctx->ulp_proto_id == CNIC_ULP_FCOE) { 1623 cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl); 1624 if (cid == -1) { 1625 ret = -ENOMEM; 1626 goto error; 1627 } 1628 ctx->cid = cid; 1629 return 0; 1630 } 1631 1632 cid = cnic_alloc_new_id(&cp->cid_tbl); 1633 if (cid == -1) { 1634 ret = -ENOMEM; 1635 goto error; 1636 } 1637 1638 ctx->cid = cid; 1639 pages = CNIC_PAGE_ALIGN(cp->task_array_size) / CNIC_PAGE_SIZE; 1640 1641 ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1); 1642 if (ret) 1643 goto error; 1644 1645 pages = CNIC_PAGE_ALIGN(cp->r2tq_size) / CNIC_PAGE_SIZE; 1646 ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1); 1647 if (ret) 1648 goto error; 1649 1650 pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE; 1651 ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1); 1652 if (ret) 1653 goto error; 1654 1655 return 0; 1656 1657 error: 1658 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1659 return ret; 1660 } 1661 1662 static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init, 1663 struct regpair *ctx_addr) 1664 { 1665 struct cnic_local *cp = dev->cnic_priv; 1666 struct cnic_eth_dev *ethdev = cp->ethdev; 1667 int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk; 1668 int off = (cid - ethdev->starting_cid) % cp->cids_per_blk; 1669 unsigned long align_off = 0; 1670 dma_addr_t ctx_map; 1671 void *ctx; 1672 1673 if (cp->ctx_align) { 1674 unsigned long mask = cp->ctx_align - 1; 1675 1676 if (cp->ctx_arr[blk].mapping & mask) 1677 align_off = cp->ctx_align - 1678 (cp->ctx_arr[blk].mapping & mask); 1679 } 1680 ctx_map = cp->ctx_arr[blk].mapping + align_off + 1681 (off * BNX2X_CONTEXT_MEM_SIZE); 1682 ctx = cp->ctx_arr[blk].ctx + align_off + 1683 (off * BNX2X_CONTEXT_MEM_SIZE); 1684 if (init) 1685 memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE); 1686 1687 ctx_addr->lo = ctx_map & 0xffffffff; 1688 ctx_addr->hi = (u64) ctx_map >> 32; 1689 return ctx; 1690 } 1691 1692 static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[], 1693 u32 num) 1694 { 1695 struct cnic_local *cp = dev->cnic_priv; 1696 struct bnx2x *bp = netdev_priv(dev->netdev); 1697 struct iscsi_kwqe_conn_offload1 *req1 = 1698 (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1699 struct iscsi_kwqe_conn_offload2 *req2 = 1700 (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1701 struct iscsi_kwqe_conn_offload3 *req3; 1702 struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id]; 1703 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1704 u32 cid = ctx->cid; 1705 u32 hw_cid = BNX2X_HW_CID(bp, cid); 1706 struct iscsi_context *ictx; 1707 struct regpair context_addr; 1708 int i, j, n = 2, n_max; 1709 u8 port = BP_PORT(bp); 1710 1711 ctx->ctx_flags = 0; 1712 if (!req2->num_additional_wqes) 1713 return -EINVAL; 1714 1715 n_max = req2->num_additional_wqes + 2; 1716 1717 ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr); 1718 if (ictx == NULL) 1719 return -ENOMEM; 1720 1721 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1722 1723 ictx->xstorm_ag_context.hq_prod = 1; 1724 1725 ictx->xstorm_st_context.iscsi.first_burst_length = 1726 ISCSI_DEF_FIRST_BURST_LEN; 1727 ictx->xstorm_st_context.iscsi.max_send_pdu_length = 1728 ISCSI_DEF_MAX_RECV_SEG_LEN; 1729 ictx->xstorm_st_context.iscsi.sq_pbl_base.lo = 1730 req1->sq_page_table_addr_lo; 1731 ictx->xstorm_st_context.iscsi.sq_pbl_base.hi = 1732 req1->sq_page_table_addr_hi; 1733 ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi; 1734 ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo; 1735 ictx->xstorm_st_context.iscsi.hq_pbl_base.lo = 1736 iscsi->hq_info.pgtbl_map & 0xffffffff; 1737 ictx->xstorm_st_context.iscsi.hq_pbl_base.hi = 1738 (u64) iscsi->hq_info.pgtbl_map >> 32; 1739 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo = 1740 iscsi->hq_info.pgtbl[0]; 1741 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi = 1742 iscsi->hq_info.pgtbl[1]; 1743 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo = 1744 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1745 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi = 1746 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1747 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo = 1748 iscsi->r2tq_info.pgtbl[0]; 1749 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi = 1750 iscsi->r2tq_info.pgtbl[1]; 1751 ictx->xstorm_st_context.iscsi.task_pbl_base.lo = 1752 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1753 ictx->xstorm_st_context.iscsi.task_pbl_base.hi = 1754 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1755 ictx->xstorm_st_context.iscsi.task_pbl_cache_idx = 1756 BNX2X_ISCSI_PBL_NOT_CACHED; 1757 ictx->xstorm_st_context.iscsi.flags.flags |= 1758 XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA; 1759 ictx->xstorm_st_context.iscsi.flags.flags |= 1760 XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T; 1761 ictx->xstorm_st_context.common.ethernet.reserved_vlan_type = 1762 ETH_P_8021Q; 1763 if (BNX2X_CHIP_IS_E2_PLUS(bp) && 1764 bp->common.chip_port_mode == CHIP_2_PORT_MODE) { 1765 1766 port = 0; 1767 } 1768 ictx->xstorm_st_context.common.flags = 1769 1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT; 1770 ictx->xstorm_st_context.common.flags = 1771 port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT; 1772 1773 ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE; 1774 /* TSTORM requires the base address of RQ DB & not PTE */ 1775 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo = 1776 req2->rq_page_table_addr_lo & CNIC_PAGE_MASK; 1777 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi = 1778 req2->rq_page_table_addr_hi; 1779 ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id; 1780 ictx->tstorm_st_context.tcp.cwnd = 0x5A8; 1781 ictx->tstorm_st_context.tcp.flags2 |= 1782 TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN; 1783 ictx->tstorm_st_context.tcp.ooo_support_mode = 1784 TCP_TSTORM_OOO_DROP_AND_PROC_ACK; 1785 1786 ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG; 1787 1788 ictx->ustorm_st_context.ring.rq.pbl_base.lo = 1789 req2->rq_page_table_addr_lo; 1790 ictx->ustorm_st_context.ring.rq.pbl_base.hi = 1791 req2->rq_page_table_addr_hi; 1792 ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi; 1793 ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo; 1794 ictx->ustorm_st_context.ring.r2tq.pbl_base.lo = 1795 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1796 ictx->ustorm_st_context.ring.r2tq.pbl_base.hi = 1797 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1798 ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo = 1799 iscsi->r2tq_info.pgtbl[0]; 1800 ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi = 1801 iscsi->r2tq_info.pgtbl[1]; 1802 ictx->ustorm_st_context.ring.cq_pbl_base.lo = 1803 req1->cq_page_table_addr_lo; 1804 ictx->ustorm_st_context.ring.cq_pbl_base.hi = 1805 req1->cq_page_table_addr_hi; 1806 ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN; 1807 ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi; 1808 ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo; 1809 ictx->ustorm_st_context.task_pbe_cache_index = 1810 BNX2X_ISCSI_PBL_NOT_CACHED; 1811 ictx->ustorm_st_context.task_pdu_cache_index = 1812 BNX2X_ISCSI_PDU_HEADER_NOT_CACHED; 1813 1814 for (i = 1, j = 1; i < cp->num_cqs; i++, j++) { 1815 if (j == 3) { 1816 if (n >= n_max) 1817 break; 1818 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1819 j = 0; 1820 } 1821 ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN; 1822 ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo = 1823 req3->qp_first_pte[j].hi; 1824 ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi = 1825 req3->qp_first_pte[j].lo; 1826 } 1827 1828 ictx->ustorm_st_context.task_pbl_base.lo = 1829 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1830 ictx->ustorm_st_context.task_pbl_base.hi = 1831 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1832 ictx->ustorm_st_context.tce_phy_addr.lo = 1833 iscsi->task_array_info.pgtbl[0]; 1834 ictx->ustorm_st_context.tce_phy_addr.hi = 1835 iscsi->task_array_info.pgtbl[1]; 1836 ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1837 ictx->ustorm_st_context.num_cqs = cp->num_cqs; 1838 ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN; 1839 ictx->ustorm_st_context.negotiated_rx_and_flags |= 1840 ISCSI_DEF_MAX_BURST_LEN; 1841 ictx->ustorm_st_context.negotiated_rx |= 1842 ISCSI_DEFAULT_MAX_OUTSTANDING_R2T << 1843 USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT; 1844 1845 ictx->cstorm_st_context.hq_pbl_base.lo = 1846 iscsi->hq_info.pgtbl_map & 0xffffffff; 1847 ictx->cstorm_st_context.hq_pbl_base.hi = 1848 (u64) iscsi->hq_info.pgtbl_map >> 32; 1849 ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0]; 1850 ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1]; 1851 ictx->cstorm_st_context.task_pbl_base.lo = 1852 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1853 ictx->cstorm_st_context.task_pbl_base.hi = 1854 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1855 /* CSTORM and USTORM initialization is different, CSTORM requires 1856 * CQ DB base & not PTE addr */ 1857 ictx->cstorm_st_context.cq_db_base.lo = 1858 req1->cq_page_table_addr_lo & CNIC_PAGE_MASK; 1859 ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi; 1860 ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1861 ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1; 1862 for (i = 0; i < cp->num_cqs; i++) { 1863 ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] = 1864 ISCSI_INITIAL_SN; 1865 ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] = 1866 ISCSI_INITIAL_SN; 1867 } 1868 1869 ictx->xstorm_ag_context.cdu_reserved = 1870 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 1871 ISCSI_CONNECTION_TYPE); 1872 ictx->ustorm_ag_context.cdu_usage = 1873 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 1874 ISCSI_CONNECTION_TYPE); 1875 return 0; 1876 1877 } 1878 1879 static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 1880 u32 num, int *work) 1881 { 1882 struct iscsi_kwqe_conn_offload1 *req1; 1883 struct iscsi_kwqe_conn_offload2 *req2; 1884 struct cnic_local *cp = dev->cnic_priv; 1885 struct bnx2x *bp = netdev_priv(dev->netdev); 1886 struct cnic_context *ctx; 1887 struct iscsi_kcqe kcqe; 1888 struct kcqe *cqes[1]; 1889 u32 l5_cid; 1890 int ret = 0; 1891 1892 if (num < 2) { 1893 *work = num; 1894 return -EINVAL; 1895 } 1896 1897 req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1898 req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1899 if ((num - 2) < req2->num_additional_wqes) { 1900 *work = num; 1901 return -EINVAL; 1902 } 1903 *work = 2 + req2->num_additional_wqes; 1904 1905 l5_cid = req1->iscsi_conn_id; 1906 if (l5_cid >= MAX_ISCSI_TBL_SZ) 1907 return -EINVAL; 1908 1909 memset(&kcqe, 0, sizeof(kcqe)); 1910 kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN; 1911 kcqe.iscsi_conn_id = l5_cid; 1912 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 1913 1914 ctx = &cp->ctx_tbl[l5_cid]; 1915 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) { 1916 kcqe.completion_status = 1917 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY; 1918 goto done; 1919 } 1920 1921 if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) { 1922 atomic_dec(&cp->iscsi_conn); 1923 goto done; 1924 } 1925 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 1926 if (ret) { 1927 atomic_dec(&cp->iscsi_conn); 1928 goto done; 1929 } 1930 ret = cnic_setup_bnx2x_ctx(dev, wqes, num); 1931 if (ret < 0) { 1932 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1933 atomic_dec(&cp->iscsi_conn); 1934 goto done; 1935 } 1936 1937 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1938 kcqe.iscsi_conn_context_id = BNX2X_HW_CID(bp, cp->ctx_tbl[l5_cid].cid); 1939 1940 done: 1941 cqes[0] = (struct kcqe *) &kcqe; 1942 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1943 return 0; 1944 } 1945 1946 1947 static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe) 1948 { 1949 struct cnic_local *cp = dev->cnic_priv; 1950 struct iscsi_kwqe_conn_update *req = 1951 (struct iscsi_kwqe_conn_update *) kwqe; 1952 void *data; 1953 union l5cm_specific_data l5_data; 1954 u32 l5_cid, cid = BNX2X_SW_CID(req->context_id); 1955 int ret; 1956 1957 if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0) 1958 return -EINVAL; 1959 1960 data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 1961 if (!data) 1962 return -ENOMEM; 1963 1964 memcpy(data, kwqe, sizeof(struct kwqe)); 1965 1966 ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, 1967 req->context_id, ISCSI_CONNECTION_TYPE, &l5_data); 1968 return ret; 1969 } 1970 1971 static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid) 1972 { 1973 struct cnic_local *cp = dev->cnic_priv; 1974 struct bnx2x *bp = netdev_priv(dev->netdev); 1975 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1976 union l5cm_specific_data l5_data; 1977 int ret; 1978 u32 hw_cid; 1979 1980 init_waitqueue_head(&ctx->waitq); 1981 ctx->wait_cond = 0; 1982 memset(&l5_data, 0, sizeof(l5_data)); 1983 hw_cid = BNX2X_HW_CID(bp, ctx->cid); 1984 1985 ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 1986 hw_cid, NONE_CONNECTION_TYPE, &l5_data); 1987 1988 if (ret == 0) { 1989 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 1990 if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags))) 1991 return -EBUSY; 1992 } 1993 1994 return 0; 1995 } 1996 1997 static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 1998 { 1999 struct cnic_local *cp = dev->cnic_priv; 2000 struct iscsi_kwqe_conn_destroy *req = 2001 (struct iscsi_kwqe_conn_destroy *) kwqe; 2002 u32 l5_cid = req->reserved0; 2003 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2004 int ret = 0; 2005 struct iscsi_kcqe kcqe; 2006 struct kcqe *cqes[1]; 2007 2008 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2009 goto skip_cfc_delete; 2010 2011 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 2012 unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies; 2013 2014 if (delta > (2 * HZ)) 2015 delta = 0; 2016 2017 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2018 queue_delayed_work(cnic_wq, &cp->delete_task, delta); 2019 goto destroy_reply; 2020 } 2021 2022 ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid); 2023 2024 skip_cfc_delete: 2025 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2026 2027 if (!ret) { 2028 atomic_dec(&cp->iscsi_conn); 2029 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2030 } 2031 2032 destroy_reply: 2033 memset(&kcqe, 0, sizeof(kcqe)); 2034 kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN; 2035 kcqe.iscsi_conn_id = l5_cid; 2036 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 2037 kcqe.iscsi_conn_context_id = req->context_id; 2038 2039 cqes[0] = (struct kcqe *) &kcqe; 2040 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 2041 2042 return 0; 2043 } 2044 2045 static void cnic_init_storm_conn_bufs(struct cnic_dev *dev, 2046 struct l4_kwq_connect_req1 *kwqe1, 2047 struct l4_kwq_connect_req3 *kwqe3, 2048 struct l5cm_active_conn_buffer *conn_buf) 2049 { 2050 struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf; 2051 struct l5cm_xstorm_conn_buffer *xstorm_buf = 2052 &conn_buf->xstorm_conn_buffer; 2053 struct l5cm_tstorm_conn_buffer *tstorm_buf = 2054 &conn_buf->tstorm_conn_buffer; 2055 struct regpair context_addr; 2056 u32 cid = BNX2X_SW_CID(kwqe1->cid); 2057 struct in6_addr src_ip, dst_ip; 2058 int i; 2059 u32 *addrp; 2060 2061 addrp = (u32 *) &conn_addr->local_ip_addr; 2062 for (i = 0; i < 4; i++, addrp++) 2063 src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2064 2065 addrp = (u32 *) &conn_addr->remote_ip_addr; 2066 for (i = 0; i < 4; i++, addrp++) 2067 dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2068 2069 cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr); 2070 2071 xstorm_buf->context_addr.hi = context_addr.hi; 2072 xstorm_buf->context_addr.lo = context_addr.lo; 2073 xstorm_buf->mss = 0xffff; 2074 xstorm_buf->rcv_buf = kwqe3->rcv_buf; 2075 if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE) 2076 xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE; 2077 xstorm_buf->pseudo_header_checksum = 2078 swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0)); 2079 2080 if (kwqe3->ka_timeout) { 2081 tstorm_buf->ka_enable = 1; 2082 tstorm_buf->ka_timeout = kwqe3->ka_timeout; 2083 tstorm_buf->ka_interval = kwqe3->ka_interval; 2084 tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count; 2085 } 2086 tstorm_buf->max_rt_time = 0xffffffff; 2087 } 2088 2089 static void cnic_init_bnx2x_mac(struct cnic_dev *dev) 2090 { 2091 struct bnx2x *bp = netdev_priv(dev->netdev); 2092 u32 pfid = bp->pfid; 2093 u8 *mac = dev->mac_addr; 2094 2095 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2096 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]); 2097 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2098 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]); 2099 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2100 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]); 2101 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2102 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]); 2103 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2104 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]); 2105 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2106 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]); 2107 2108 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2109 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]); 2110 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2111 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2112 mac[4]); 2113 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2114 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]); 2115 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2116 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2117 mac[2]); 2118 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2119 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]); 2120 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2121 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2122 mac[0]); 2123 } 2124 2125 static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[], 2126 u32 num, int *work) 2127 { 2128 struct cnic_local *cp = dev->cnic_priv; 2129 struct bnx2x *bp = netdev_priv(dev->netdev); 2130 struct l4_kwq_connect_req1 *kwqe1 = 2131 (struct l4_kwq_connect_req1 *) wqes[0]; 2132 struct l4_kwq_connect_req3 *kwqe3; 2133 struct l5cm_active_conn_buffer *conn_buf; 2134 struct l5cm_conn_addr_params *conn_addr; 2135 union l5cm_specific_data l5_data; 2136 u32 l5_cid = kwqe1->pg_cid; 2137 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 2138 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2139 int ret; 2140 2141 if (num < 2) { 2142 *work = num; 2143 return -EINVAL; 2144 } 2145 2146 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) 2147 *work = 3; 2148 else 2149 *work = 2; 2150 2151 if (num < *work) { 2152 *work = num; 2153 return -EINVAL; 2154 } 2155 2156 if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) { 2157 netdev_err(dev->netdev, "conn_buf size too big\n"); 2158 return -ENOMEM; 2159 } 2160 conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2161 if (!conn_buf) 2162 return -ENOMEM; 2163 2164 memset(conn_buf, 0, sizeof(*conn_buf)); 2165 2166 conn_addr = &conn_buf->conn_addr_buf; 2167 conn_addr->remote_addr_0 = csk->ha[0]; 2168 conn_addr->remote_addr_1 = csk->ha[1]; 2169 conn_addr->remote_addr_2 = csk->ha[2]; 2170 conn_addr->remote_addr_3 = csk->ha[3]; 2171 conn_addr->remote_addr_4 = csk->ha[4]; 2172 conn_addr->remote_addr_5 = csk->ha[5]; 2173 2174 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) { 2175 struct l4_kwq_connect_req2 *kwqe2 = 2176 (struct l4_kwq_connect_req2 *) wqes[1]; 2177 2178 conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4; 2179 conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3; 2180 conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2; 2181 2182 conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4; 2183 conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3; 2184 conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2; 2185 conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION; 2186 } 2187 kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1]; 2188 2189 conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip; 2190 conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip; 2191 conn_addr->local_tcp_port = kwqe1->src_port; 2192 conn_addr->remote_tcp_port = kwqe1->dst_port; 2193 2194 conn_addr->pmtu = kwqe3->pmtu; 2195 cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf); 2196 2197 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 2198 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(bp->pfid), csk->vlan_id); 2199 2200 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT, 2201 kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2202 if (!ret) 2203 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2204 2205 return ret; 2206 } 2207 2208 static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe) 2209 { 2210 struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe; 2211 union l5cm_specific_data l5_data; 2212 int ret; 2213 2214 memset(&l5_data, 0, sizeof(l5_data)); 2215 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE, 2216 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2217 return ret; 2218 } 2219 2220 static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe) 2221 { 2222 struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe; 2223 union l5cm_specific_data l5_data; 2224 int ret; 2225 2226 memset(&l5_data, 0, sizeof(l5_data)); 2227 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT, 2228 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2229 return ret; 2230 } 2231 static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2232 { 2233 struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe; 2234 struct l4_kcq kcqe; 2235 struct kcqe *cqes[1]; 2236 2237 memset(&kcqe, 0, sizeof(kcqe)); 2238 kcqe.pg_host_opaque = req->host_opaque; 2239 kcqe.pg_cid = req->host_opaque; 2240 kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG; 2241 cqes[0] = (struct kcqe *) &kcqe; 2242 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2243 return 0; 2244 } 2245 2246 static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2247 { 2248 struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe; 2249 struct l4_kcq kcqe; 2250 struct kcqe *cqes[1]; 2251 2252 memset(&kcqe, 0, sizeof(kcqe)); 2253 kcqe.pg_host_opaque = req->pg_host_opaque; 2254 kcqe.pg_cid = req->pg_cid; 2255 kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG; 2256 cqes[0] = (struct kcqe *) &kcqe; 2257 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2258 return 0; 2259 } 2260 2261 static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe) 2262 { 2263 struct fcoe_kwqe_stat *req; 2264 struct fcoe_stat_ramrod_params *fcoe_stat; 2265 union l5cm_specific_data l5_data; 2266 struct cnic_local *cp = dev->cnic_priv; 2267 struct bnx2x *bp = netdev_priv(dev->netdev); 2268 int ret; 2269 u32 cid; 2270 2271 req = (struct fcoe_kwqe_stat *) kwqe; 2272 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2273 2274 fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2275 if (!fcoe_stat) 2276 return -ENOMEM; 2277 2278 memset(fcoe_stat, 0, sizeof(*fcoe_stat)); 2279 memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req)); 2280 2281 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid, 2282 FCOE_CONNECTION_TYPE, &l5_data); 2283 return ret; 2284 } 2285 2286 static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[], 2287 u32 num, int *work) 2288 { 2289 int ret; 2290 struct cnic_local *cp = dev->cnic_priv; 2291 struct bnx2x *bp = netdev_priv(dev->netdev); 2292 u32 cid; 2293 struct fcoe_init_ramrod_params *fcoe_init; 2294 struct fcoe_kwqe_init1 *req1; 2295 struct fcoe_kwqe_init2 *req2; 2296 struct fcoe_kwqe_init3 *req3; 2297 union l5cm_specific_data l5_data; 2298 2299 if (num < 3) { 2300 *work = num; 2301 return -EINVAL; 2302 } 2303 req1 = (struct fcoe_kwqe_init1 *) wqes[0]; 2304 req2 = (struct fcoe_kwqe_init2 *) wqes[1]; 2305 req3 = (struct fcoe_kwqe_init3 *) wqes[2]; 2306 if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) { 2307 *work = 1; 2308 return -EINVAL; 2309 } 2310 if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) { 2311 *work = 2; 2312 return -EINVAL; 2313 } 2314 2315 if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) { 2316 netdev_err(dev->netdev, "fcoe_init size too big\n"); 2317 return -ENOMEM; 2318 } 2319 fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2320 if (!fcoe_init) 2321 return -ENOMEM; 2322 2323 memset(fcoe_init, 0, sizeof(*fcoe_init)); 2324 memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1)); 2325 memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2)); 2326 memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3)); 2327 fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff; 2328 fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32; 2329 fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages; 2330 2331 fcoe_init->sb_num = cp->status_blk_num; 2332 fcoe_init->eq_prod = MAX_KCQ_IDX; 2333 fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS; 2334 cp->kcq2.sw_prod_idx = 0; 2335 2336 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2337 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid, 2338 FCOE_CONNECTION_TYPE, &l5_data); 2339 *work = 3; 2340 return ret; 2341 } 2342 2343 static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 2344 u32 num, int *work) 2345 { 2346 int ret = 0; 2347 u32 cid = -1, l5_cid; 2348 struct cnic_local *cp = dev->cnic_priv; 2349 struct bnx2x *bp = netdev_priv(dev->netdev); 2350 struct fcoe_kwqe_conn_offload1 *req1; 2351 struct fcoe_kwqe_conn_offload2 *req2; 2352 struct fcoe_kwqe_conn_offload3 *req3; 2353 struct fcoe_kwqe_conn_offload4 *req4; 2354 struct fcoe_conn_offload_ramrod_params *fcoe_offload; 2355 struct cnic_context *ctx; 2356 struct fcoe_context *fctx; 2357 struct regpair ctx_addr; 2358 union l5cm_specific_data l5_data; 2359 struct fcoe_kcqe kcqe; 2360 struct kcqe *cqes[1]; 2361 2362 if (num < 4) { 2363 *work = num; 2364 return -EINVAL; 2365 } 2366 req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0]; 2367 req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1]; 2368 req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2]; 2369 req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3]; 2370 2371 *work = 4; 2372 2373 l5_cid = req1->fcoe_conn_id; 2374 if (l5_cid >= dev->max_fcoe_conn) 2375 goto err_reply; 2376 2377 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2378 2379 ctx = &cp->ctx_tbl[l5_cid]; 2380 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2381 goto err_reply; 2382 2383 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 2384 if (ret) { 2385 ret = 0; 2386 goto err_reply; 2387 } 2388 cid = ctx->cid; 2389 2390 fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr); 2391 if (fctx) { 2392 u32 hw_cid = BNX2X_HW_CID(bp, cid); 2393 u32 val; 2394 2395 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 2396 FCOE_CONNECTION_TYPE); 2397 fctx->xstorm_ag_context.cdu_reserved = val; 2398 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 2399 FCOE_CONNECTION_TYPE); 2400 fctx->ustorm_ag_context.cdu_usage = val; 2401 } 2402 if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) { 2403 netdev_err(dev->netdev, "fcoe_offload size too big\n"); 2404 goto err_reply; 2405 } 2406 fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2407 if (!fcoe_offload) 2408 goto err_reply; 2409 2410 memset(fcoe_offload, 0, sizeof(*fcoe_offload)); 2411 memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1)); 2412 memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2)); 2413 memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3)); 2414 memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4)); 2415 2416 cid = BNX2X_HW_CID(bp, cid); 2417 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid, 2418 FCOE_CONNECTION_TYPE, &l5_data); 2419 if (!ret) 2420 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2421 2422 return ret; 2423 2424 err_reply: 2425 if (cid != -1) 2426 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2427 2428 memset(&kcqe, 0, sizeof(kcqe)); 2429 kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN; 2430 kcqe.fcoe_conn_id = req1->fcoe_conn_id; 2431 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 2432 2433 cqes[0] = (struct kcqe *) &kcqe; 2434 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2435 return ret; 2436 } 2437 2438 static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe) 2439 { 2440 struct fcoe_kwqe_conn_enable_disable *req; 2441 struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable; 2442 union l5cm_specific_data l5_data; 2443 int ret; 2444 u32 cid, l5_cid; 2445 struct cnic_local *cp = dev->cnic_priv; 2446 2447 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2448 cid = req->context_id; 2449 l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE; 2450 2451 if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) { 2452 netdev_err(dev->netdev, "fcoe_enable size too big\n"); 2453 return -ENOMEM; 2454 } 2455 fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2456 if (!fcoe_enable) 2457 return -ENOMEM; 2458 2459 memset(fcoe_enable, 0, sizeof(*fcoe_enable)); 2460 memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req)); 2461 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid, 2462 FCOE_CONNECTION_TYPE, &l5_data); 2463 return ret; 2464 } 2465 2466 static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe) 2467 { 2468 struct fcoe_kwqe_conn_enable_disable *req; 2469 struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable; 2470 union l5cm_specific_data l5_data; 2471 int ret; 2472 u32 cid, l5_cid; 2473 struct cnic_local *cp = dev->cnic_priv; 2474 2475 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2476 cid = req->context_id; 2477 l5_cid = req->conn_id; 2478 if (l5_cid >= dev->max_fcoe_conn) 2479 return -EINVAL; 2480 2481 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2482 2483 if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) { 2484 netdev_err(dev->netdev, "fcoe_disable size too big\n"); 2485 return -ENOMEM; 2486 } 2487 fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2488 if (!fcoe_disable) 2489 return -ENOMEM; 2490 2491 memset(fcoe_disable, 0, sizeof(*fcoe_disable)); 2492 memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req)); 2493 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid, 2494 FCOE_CONNECTION_TYPE, &l5_data); 2495 return ret; 2496 } 2497 2498 static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2499 { 2500 struct fcoe_kwqe_conn_destroy *req; 2501 union l5cm_specific_data l5_data; 2502 int ret; 2503 u32 cid, l5_cid; 2504 struct cnic_local *cp = dev->cnic_priv; 2505 struct cnic_context *ctx; 2506 struct fcoe_kcqe kcqe; 2507 struct kcqe *cqes[1]; 2508 2509 req = (struct fcoe_kwqe_conn_destroy *) kwqe; 2510 cid = req->context_id; 2511 l5_cid = req->conn_id; 2512 if (l5_cid >= dev->max_fcoe_conn) 2513 return -EINVAL; 2514 2515 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2516 2517 ctx = &cp->ctx_tbl[l5_cid]; 2518 2519 init_waitqueue_head(&ctx->waitq); 2520 ctx->wait_cond = 0; 2521 2522 memset(&kcqe, 0, sizeof(kcqe)); 2523 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR; 2524 memset(&l5_data, 0, sizeof(l5_data)); 2525 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid, 2526 FCOE_CONNECTION_TYPE, &l5_data); 2527 if (ret == 0) { 2528 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 2529 if (ctx->wait_cond) 2530 kcqe.completion_status = 0; 2531 } 2532 2533 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2534 queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000)); 2535 2536 kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN; 2537 kcqe.fcoe_conn_id = req->conn_id; 2538 kcqe.fcoe_conn_context_id = cid; 2539 2540 cqes[0] = (struct kcqe *) &kcqe; 2541 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2542 return ret; 2543 } 2544 2545 static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid) 2546 { 2547 struct cnic_local *cp = dev->cnic_priv; 2548 u32 i; 2549 2550 for (i = start_cid; i < cp->max_cid_space; i++) { 2551 struct cnic_context *ctx = &cp->ctx_tbl[i]; 2552 int j; 2553 2554 while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 2555 msleep(10); 2556 2557 for (j = 0; j < 5; j++) { 2558 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2559 break; 2560 msleep(20); 2561 } 2562 2563 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2564 netdev_warn(dev->netdev, "CID %x not deleted\n", 2565 ctx->cid); 2566 } 2567 } 2568 2569 static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2570 { 2571 union l5cm_specific_data l5_data; 2572 struct cnic_local *cp = dev->cnic_priv; 2573 struct bnx2x *bp = netdev_priv(dev->netdev); 2574 int ret; 2575 u32 cid; 2576 2577 cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ); 2578 2579 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2580 2581 memset(&l5_data, 0, sizeof(l5_data)); 2582 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid, 2583 FCOE_CONNECTION_TYPE, &l5_data); 2584 return ret; 2585 } 2586 2587 static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe) 2588 { 2589 struct cnic_local *cp = dev->cnic_priv; 2590 struct kcqe kcqe; 2591 struct kcqe *cqes[1]; 2592 u32 cid; 2593 u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2594 u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK; 2595 u32 kcqe_op; 2596 int ulp_type; 2597 2598 cid = kwqe->kwqe_info0; 2599 memset(&kcqe, 0, sizeof(kcqe)); 2600 2601 if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) { 2602 u32 l5_cid = 0; 2603 2604 ulp_type = CNIC_ULP_FCOE; 2605 if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) { 2606 struct fcoe_kwqe_conn_enable_disable *req; 2607 2608 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2609 kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN; 2610 cid = req->context_id; 2611 l5_cid = req->conn_id; 2612 } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) { 2613 kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC; 2614 } else { 2615 return; 2616 } 2617 kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT; 2618 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE; 2619 kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2620 kcqe.kcqe_info2 = cid; 2621 kcqe.kcqe_info0 = l5_cid; 2622 2623 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) { 2624 ulp_type = CNIC_ULP_ISCSI; 2625 if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN) 2626 cid = kwqe->kwqe_info1; 2627 2628 kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT; 2629 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI; 2630 kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR; 2631 kcqe.kcqe_info2 = cid; 2632 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0); 2633 2634 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) { 2635 struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe; 2636 2637 ulp_type = CNIC_ULP_L4; 2638 if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1) 2639 kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE; 2640 else if (opcode == L4_KWQE_OPCODE_VALUE_RESET) 2641 kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP; 2642 else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE) 2643 kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 2644 else 2645 return; 2646 2647 kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) | 2648 KCQE_FLAGS_LAYER_MASK_L4; 2649 l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2650 l4kcqe->cid = cid; 2651 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id); 2652 } else { 2653 return; 2654 } 2655 2656 cqes[0] = &kcqe; 2657 cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1); 2658 } 2659 2660 static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev, 2661 struct kwqe *wqes[], u32 num_wqes) 2662 { 2663 int i, work, ret; 2664 u32 opcode; 2665 struct kwqe *kwqe; 2666 2667 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2668 return -EAGAIN; /* bnx2 is down */ 2669 2670 for (i = 0; i < num_wqes; ) { 2671 kwqe = wqes[i]; 2672 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2673 work = 1; 2674 2675 switch (opcode) { 2676 case ISCSI_KWQE_OPCODE_INIT1: 2677 ret = cnic_bnx2x_iscsi_init1(dev, kwqe); 2678 break; 2679 case ISCSI_KWQE_OPCODE_INIT2: 2680 ret = cnic_bnx2x_iscsi_init2(dev, kwqe); 2681 break; 2682 case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1: 2683 ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i], 2684 num_wqes - i, &work); 2685 break; 2686 case ISCSI_KWQE_OPCODE_UPDATE_CONN: 2687 ret = cnic_bnx2x_iscsi_update(dev, kwqe); 2688 break; 2689 case ISCSI_KWQE_OPCODE_DESTROY_CONN: 2690 ret = cnic_bnx2x_iscsi_destroy(dev, kwqe); 2691 break; 2692 case L4_KWQE_OPCODE_VALUE_CONNECT1: 2693 ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i, 2694 &work); 2695 break; 2696 case L4_KWQE_OPCODE_VALUE_CLOSE: 2697 ret = cnic_bnx2x_close(dev, kwqe); 2698 break; 2699 case L4_KWQE_OPCODE_VALUE_RESET: 2700 ret = cnic_bnx2x_reset(dev, kwqe); 2701 break; 2702 case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG: 2703 ret = cnic_bnx2x_offload_pg(dev, kwqe); 2704 break; 2705 case L4_KWQE_OPCODE_VALUE_UPDATE_PG: 2706 ret = cnic_bnx2x_update_pg(dev, kwqe); 2707 break; 2708 case L4_KWQE_OPCODE_VALUE_UPLOAD_PG: 2709 ret = 0; 2710 break; 2711 default: 2712 ret = 0; 2713 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2714 opcode); 2715 break; 2716 } 2717 if (ret < 0) { 2718 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2719 opcode); 2720 2721 /* Possibly bnx2x parity error, send completion 2722 * to ulp drivers with error code to speed up 2723 * cleanup and reset recovery. 2724 */ 2725 if (ret == -EIO || ret == -EAGAIN) 2726 cnic_bnx2x_kwqe_err(dev, kwqe); 2727 } 2728 i += work; 2729 } 2730 return 0; 2731 } 2732 2733 static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev, 2734 struct kwqe *wqes[], u32 num_wqes) 2735 { 2736 struct bnx2x *bp = netdev_priv(dev->netdev); 2737 int i, work, ret; 2738 u32 opcode; 2739 struct kwqe *kwqe; 2740 2741 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2742 return -EAGAIN; /* bnx2 is down */ 2743 2744 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 2745 return -EINVAL; 2746 2747 for (i = 0; i < num_wqes; ) { 2748 kwqe = wqes[i]; 2749 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2750 work = 1; 2751 2752 switch (opcode) { 2753 case FCOE_KWQE_OPCODE_INIT1: 2754 ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i], 2755 num_wqes - i, &work); 2756 break; 2757 case FCOE_KWQE_OPCODE_OFFLOAD_CONN1: 2758 ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i], 2759 num_wqes - i, &work); 2760 break; 2761 case FCOE_KWQE_OPCODE_ENABLE_CONN: 2762 ret = cnic_bnx2x_fcoe_enable(dev, kwqe); 2763 break; 2764 case FCOE_KWQE_OPCODE_DISABLE_CONN: 2765 ret = cnic_bnx2x_fcoe_disable(dev, kwqe); 2766 break; 2767 case FCOE_KWQE_OPCODE_DESTROY_CONN: 2768 ret = cnic_bnx2x_fcoe_destroy(dev, kwqe); 2769 break; 2770 case FCOE_KWQE_OPCODE_DESTROY: 2771 ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe); 2772 break; 2773 case FCOE_KWQE_OPCODE_STAT: 2774 ret = cnic_bnx2x_fcoe_stat(dev, kwqe); 2775 break; 2776 default: 2777 ret = 0; 2778 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2779 opcode); 2780 break; 2781 } 2782 if (ret < 0) { 2783 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2784 opcode); 2785 2786 /* Possibly bnx2x parity error, send completion 2787 * to ulp drivers with error code to speed up 2788 * cleanup and reset recovery. 2789 */ 2790 if (ret == -EIO || ret == -EAGAIN) 2791 cnic_bnx2x_kwqe_err(dev, kwqe); 2792 } 2793 i += work; 2794 } 2795 return 0; 2796 } 2797 2798 static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 2799 u32 num_wqes) 2800 { 2801 int ret = -EINVAL; 2802 u32 layer_code; 2803 2804 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2805 return -EAGAIN; /* bnx2x is down */ 2806 2807 if (!num_wqes) 2808 return 0; 2809 2810 layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK; 2811 switch (layer_code) { 2812 case KWQE_FLAGS_LAYER_MASK_L5_ISCSI: 2813 case KWQE_FLAGS_LAYER_MASK_L4: 2814 case KWQE_FLAGS_LAYER_MASK_L2: 2815 ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes); 2816 break; 2817 2818 case KWQE_FLAGS_LAYER_MASK_L5_FCOE: 2819 ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes); 2820 break; 2821 } 2822 return ret; 2823 } 2824 2825 static inline u32 cnic_get_kcqe_layer_mask(u32 opflag) 2826 { 2827 if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN)) 2828 return KCQE_FLAGS_LAYER_MASK_L4; 2829 2830 return opflag & KCQE_FLAGS_LAYER_MASK; 2831 } 2832 2833 static void service_kcqes(struct cnic_dev *dev, int num_cqes) 2834 { 2835 struct cnic_local *cp = dev->cnic_priv; 2836 int i, j, comp = 0; 2837 2838 i = 0; 2839 j = 1; 2840 while (num_cqes) { 2841 struct cnic_ulp_ops *ulp_ops; 2842 int ulp_type; 2843 u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag; 2844 u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag); 2845 2846 if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION)) 2847 comp++; 2848 2849 while (j < num_cqes) { 2850 u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag; 2851 2852 if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer) 2853 break; 2854 2855 if (unlikely(next_op & KCQE_RAMROD_COMPLETION)) 2856 comp++; 2857 j++; 2858 } 2859 2860 if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA) 2861 ulp_type = CNIC_ULP_RDMA; 2862 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI) 2863 ulp_type = CNIC_ULP_ISCSI; 2864 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE) 2865 ulp_type = CNIC_ULP_FCOE; 2866 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4) 2867 ulp_type = CNIC_ULP_L4; 2868 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2) 2869 goto end; 2870 else { 2871 netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n", 2872 kcqe_op_flag); 2873 goto end; 2874 } 2875 2876 rcu_read_lock(); 2877 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 2878 if (likely(ulp_ops)) { 2879 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 2880 cp->completed_kcq + i, j); 2881 } 2882 rcu_read_unlock(); 2883 end: 2884 num_cqes -= j; 2885 i += j; 2886 j = 1; 2887 } 2888 if (unlikely(comp)) 2889 cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp); 2890 } 2891 2892 static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info) 2893 { 2894 struct cnic_local *cp = dev->cnic_priv; 2895 u16 i, ri, hw_prod, last; 2896 struct kcqe *kcqe; 2897 int kcqe_cnt = 0, last_cnt = 0; 2898 2899 i = ri = last = info->sw_prod_idx; 2900 ri &= MAX_KCQ_IDX; 2901 hw_prod = *info->hw_prod_idx_ptr; 2902 hw_prod = info->hw_idx(hw_prod); 2903 2904 while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) { 2905 kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)]; 2906 cp->completed_kcq[kcqe_cnt++] = kcqe; 2907 i = info->next_idx(i); 2908 ri = i & MAX_KCQ_IDX; 2909 if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) { 2910 last_cnt = kcqe_cnt; 2911 last = i; 2912 } 2913 } 2914 2915 info->sw_prod_idx = last; 2916 return last_cnt; 2917 } 2918 2919 static int cnic_l2_completion(struct cnic_local *cp) 2920 { 2921 u16 hw_cons, sw_cons; 2922 struct cnic_uio_dev *udev = cp->udev; 2923 union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *) 2924 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 2925 u32 cmd; 2926 int comp = 0; 2927 2928 if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags)) 2929 return 0; 2930 2931 hw_cons = *cp->rx_cons_ptr; 2932 if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT) 2933 hw_cons++; 2934 2935 sw_cons = cp->rx_cons; 2936 while (sw_cons != hw_cons) { 2937 u8 cqe_fp_flags; 2938 2939 cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT]; 2940 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 2941 if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) { 2942 cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data); 2943 cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT; 2944 if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP || 2945 cmd == RAMROD_CMD_ID_ETH_HALT) 2946 comp++; 2947 } 2948 sw_cons = BNX2X_NEXT_RCQE(sw_cons); 2949 } 2950 return comp; 2951 } 2952 2953 static void cnic_chk_pkt_rings(struct cnic_local *cp) 2954 { 2955 u16 rx_cons, tx_cons; 2956 int comp = 0; 2957 2958 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 2959 return; 2960 2961 rx_cons = *cp->rx_cons_ptr; 2962 tx_cons = *cp->tx_cons_ptr; 2963 if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) { 2964 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 2965 comp = cnic_l2_completion(cp); 2966 2967 cp->tx_cons = tx_cons; 2968 cp->rx_cons = rx_cons; 2969 2970 if (cp->udev) 2971 uio_event_notify(&cp->udev->cnic_uinfo); 2972 } 2973 if (comp) 2974 clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 2975 } 2976 2977 static u32 cnic_service_bnx2_queues(struct cnic_dev *dev) 2978 { 2979 struct cnic_local *cp = dev->cnic_priv; 2980 u32 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2981 int kcqe_cnt; 2982 2983 /* status block index must be read before reading other fields */ 2984 rmb(); 2985 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2986 2987 while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) { 2988 2989 service_kcqes(dev, kcqe_cnt); 2990 2991 /* Tell compiler that status_blk fields can change. */ 2992 barrier(); 2993 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2994 /* status block index must be read first */ 2995 rmb(); 2996 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2997 } 2998 2999 CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx); 3000 3001 cnic_chk_pkt_rings(cp); 3002 3003 return status_idx; 3004 } 3005 3006 static int cnic_service_bnx2(void *data, void *status_blk) 3007 { 3008 struct cnic_dev *dev = data; 3009 3010 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3011 struct status_block *sblk = status_blk; 3012 3013 return sblk->status_idx; 3014 } 3015 3016 return cnic_service_bnx2_queues(dev); 3017 } 3018 3019 static void cnic_service_bnx2_msix(struct work_struct *work) 3020 { 3021 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3022 struct cnic_dev *dev = cp->dev; 3023 3024 cp->last_status_idx = cnic_service_bnx2_queues(dev); 3025 3026 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 3027 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 3028 } 3029 3030 static void cnic_doirq(struct cnic_dev *dev) 3031 { 3032 struct cnic_local *cp = dev->cnic_priv; 3033 3034 if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3035 u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX; 3036 3037 prefetch(cp->status_blk.gen); 3038 prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]); 3039 3040 queue_work(system_bh_wq, &cp->cnic_irq_bh_work); 3041 } 3042 } 3043 3044 static irqreturn_t cnic_irq(int irq, void *dev_instance) 3045 { 3046 struct cnic_dev *dev = dev_instance; 3047 struct cnic_local *cp = dev->cnic_priv; 3048 3049 if (cp->ack_int) 3050 cp->ack_int(dev); 3051 3052 cnic_doirq(dev); 3053 3054 return IRQ_HANDLED; 3055 } 3056 3057 static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm, 3058 u16 index, u8 op, u8 update) 3059 { 3060 struct bnx2x *bp = netdev_priv(dev->netdev); 3061 u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp) * 32 + 3062 COMMAND_REG_INT_ACK); 3063 struct igu_ack_register igu_ack; 3064 3065 igu_ack.status_block_index = index; 3066 igu_ack.sb_id_and_flags = 3067 ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) | 3068 (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) | 3069 (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) | 3070 (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT)); 3071 3072 CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack)); 3073 } 3074 3075 static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment, 3076 u16 index, u8 op, u8 update) 3077 { 3078 struct igu_regular cmd_data; 3079 u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8; 3080 3081 cmd_data.sb_id_and_flags = 3082 (index << IGU_REGULAR_SB_INDEX_SHIFT) | 3083 (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) | 3084 (update << IGU_REGULAR_BUPDATE_SHIFT) | 3085 (op << IGU_REGULAR_ENABLE_INT_SHIFT); 3086 3087 3088 CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags); 3089 } 3090 3091 static void cnic_ack_bnx2x_msix(struct cnic_dev *dev) 3092 { 3093 struct cnic_local *cp = dev->cnic_priv; 3094 3095 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0, 3096 IGU_INT_DISABLE, 0); 3097 } 3098 3099 static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev) 3100 { 3101 struct cnic_local *cp = dev->cnic_priv; 3102 3103 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0, 3104 IGU_INT_DISABLE, 0); 3105 } 3106 3107 static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx) 3108 { 3109 struct cnic_local *cp = dev->cnic_priv; 3110 3111 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx, 3112 IGU_INT_ENABLE, 1); 3113 } 3114 3115 static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx) 3116 { 3117 struct cnic_local *cp = dev->cnic_priv; 3118 3119 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx, 3120 IGU_INT_ENABLE, 1); 3121 } 3122 3123 static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info) 3124 { 3125 u32 last_status = *info->status_idx_ptr; 3126 int kcqe_cnt; 3127 3128 /* status block index must be read before reading the KCQ */ 3129 rmb(); 3130 while ((kcqe_cnt = cnic_get_kcqes(dev, info))) { 3131 3132 service_kcqes(dev, kcqe_cnt); 3133 3134 /* Tell compiler that sblk fields can change. */ 3135 barrier(); 3136 3137 last_status = *info->status_idx_ptr; 3138 /* status block index must be read before reading the KCQ */ 3139 rmb(); 3140 } 3141 return last_status; 3142 } 3143 3144 static void cnic_service_bnx2x_bh_work(struct work_struct *work) 3145 { 3146 struct cnic_local *cp = from_work(cp, work, cnic_irq_bh_work); 3147 struct cnic_dev *dev = cp->dev; 3148 struct bnx2x *bp = netdev_priv(dev->netdev); 3149 u32 status_idx, new_status_idx; 3150 3151 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) 3152 return; 3153 3154 while (1) { 3155 status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1); 3156 3157 CNIC_WR16(dev, cp->kcq1.io_addr, 3158 cp->kcq1.sw_prod_idx + MAX_KCQ_IDX); 3159 3160 if (!CNIC_SUPPORTS_FCOE(bp)) { 3161 cp->arm_int(dev, status_idx); 3162 break; 3163 } 3164 3165 new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2); 3166 3167 if (new_status_idx != status_idx) 3168 continue; 3169 3170 CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx + 3171 MAX_KCQ_IDX); 3172 3173 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 3174 status_idx, IGU_INT_ENABLE, 1); 3175 3176 break; 3177 } 3178 } 3179 3180 static int cnic_service_bnx2x(void *data, void *status_blk) 3181 { 3182 struct cnic_dev *dev = data; 3183 struct cnic_local *cp = dev->cnic_priv; 3184 3185 if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 3186 cnic_doirq(dev); 3187 3188 cnic_chk_pkt_rings(cp); 3189 3190 return 0; 3191 } 3192 3193 static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type) 3194 { 3195 struct cnic_ulp_ops *ulp_ops; 3196 3197 if (if_type == CNIC_ULP_ISCSI) 3198 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 3199 3200 mutex_lock(&cnic_lock); 3201 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3202 lockdep_is_held(&cnic_lock)); 3203 if (!ulp_ops) { 3204 mutex_unlock(&cnic_lock); 3205 return; 3206 } 3207 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3208 mutex_unlock(&cnic_lock); 3209 3210 if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3211 ulp_ops->cnic_stop(cp->ulp_handle[if_type]); 3212 3213 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3214 } 3215 3216 static void cnic_ulp_stop(struct cnic_dev *dev) 3217 { 3218 struct cnic_local *cp = dev->cnic_priv; 3219 int if_type; 3220 3221 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) 3222 cnic_ulp_stop_one(cp, if_type); 3223 } 3224 3225 static void cnic_ulp_start(struct cnic_dev *dev) 3226 { 3227 struct cnic_local *cp = dev->cnic_priv; 3228 int if_type; 3229 3230 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 3231 struct cnic_ulp_ops *ulp_ops; 3232 3233 mutex_lock(&cnic_lock); 3234 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3235 lockdep_is_held(&cnic_lock)); 3236 if (!ulp_ops || !ulp_ops->cnic_start) { 3237 mutex_unlock(&cnic_lock); 3238 continue; 3239 } 3240 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3241 mutex_unlock(&cnic_lock); 3242 3243 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3244 ulp_ops->cnic_start(cp->ulp_handle[if_type]); 3245 3246 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3247 } 3248 } 3249 3250 static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type) 3251 { 3252 struct cnic_local *cp = dev->cnic_priv; 3253 struct cnic_ulp_ops *ulp_ops; 3254 int rc; 3255 3256 mutex_lock(&cnic_lock); 3257 ulp_ops = rcu_dereference_protected(cp->ulp_ops[ulp_type], 3258 lockdep_is_held(&cnic_lock)); 3259 if (ulp_ops && ulp_ops->cnic_get_stats) 3260 rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]); 3261 else 3262 rc = -ENODEV; 3263 mutex_unlock(&cnic_lock); 3264 return rc; 3265 } 3266 3267 static int cnic_ctl(void *data, struct cnic_ctl_info *info) 3268 { 3269 struct cnic_dev *dev = data; 3270 int ulp_type = CNIC_ULP_ISCSI; 3271 3272 switch (info->cmd) { 3273 case CNIC_CTL_STOP_CMD: 3274 cnic_hold(dev); 3275 3276 cnic_ulp_stop(dev); 3277 cnic_stop_hw(dev); 3278 3279 cnic_put(dev); 3280 break; 3281 case CNIC_CTL_START_CMD: 3282 cnic_hold(dev); 3283 3284 if (!cnic_start_hw(dev)) 3285 cnic_ulp_start(dev); 3286 3287 cnic_put(dev); 3288 break; 3289 case CNIC_CTL_STOP_ISCSI_CMD: { 3290 struct cnic_local *cp = dev->cnic_priv; 3291 set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags); 3292 queue_delayed_work(cnic_wq, &cp->delete_task, 0); 3293 break; 3294 } 3295 case CNIC_CTL_COMPLETION_CMD: { 3296 struct cnic_ctl_completion *comp = &info->data.comp; 3297 u32 cid = BNX2X_SW_CID(comp->cid); 3298 u32 l5_cid; 3299 struct cnic_local *cp = dev->cnic_priv; 3300 3301 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 3302 break; 3303 3304 if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) { 3305 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3306 3307 if (unlikely(comp->error)) { 3308 set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags); 3309 netdev_err(dev->netdev, 3310 "CID %x CFC delete comp error %x\n", 3311 cid, comp->error); 3312 } 3313 3314 ctx->wait_cond = 1; 3315 wake_up(&ctx->waitq); 3316 } 3317 break; 3318 } 3319 case CNIC_CTL_FCOE_STATS_GET_CMD: 3320 ulp_type = CNIC_ULP_FCOE; 3321 fallthrough; 3322 case CNIC_CTL_ISCSI_STATS_GET_CMD: 3323 cnic_hold(dev); 3324 cnic_copy_ulp_stats(dev, ulp_type); 3325 cnic_put(dev); 3326 break; 3327 3328 default: 3329 return -EINVAL; 3330 } 3331 return 0; 3332 } 3333 3334 static void cnic_ulp_init(struct cnic_dev *dev) 3335 { 3336 int i; 3337 struct cnic_local *cp = dev->cnic_priv; 3338 3339 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3340 struct cnic_ulp_ops *ulp_ops; 3341 3342 mutex_lock(&cnic_lock); 3343 ulp_ops = cnic_ulp_tbl_prot(i); 3344 if (!ulp_ops || !ulp_ops->cnic_init) { 3345 mutex_unlock(&cnic_lock); 3346 continue; 3347 } 3348 ulp_get(ulp_ops); 3349 mutex_unlock(&cnic_lock); 3350 3351 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3352 ulp_ops->cnic_init(dev); 3353 3354 ulp_put(ulp_ops); 3355 } 3356 } 3357 3358 static void cnic_ulp_exit(struct cnic_dev *dev) 3359 { 3360 int i; 3361 struct cnic_local *cp = dev->cnic_priv; 3362 3363 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3364 struct cnic_ulp_ops *ulp_ops; 3365 3366 mutex_lock(&cnic_lock); 3367 ulp_ops = cnic_ulp_tbl_prot(i); 3368 if (!ulp_ops || !ulp_ops->cnic_exit) { 3369 mutex_unlock(&cnic_lock); 3370 continue; 3371 } 3372 ulp_get(ulp_ops); 3373 mutex_unlock(&cnic_lock); 3374 3375 if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3376 ulp_ops->cnic_exit(dev); 3377 3378 ulp_put(ulp_ops); 3379 } 3380 } 3381 3382 static int cnic_cm_offload_pg(struct cnic_sock *csk) 3383 { 3384 struct cnic_dev *dev = csk->dev; 3385 struct l4_kwq_offload_pg *l4kwqe; 3386 struct kwqe *wqes[1]; 3387 3388 l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1; 3389 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3390 wqes[0] = (struct kwqe *) l4kwqe; 3391 3392 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG; 3393 l4kwqe->flags = 3394 L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT; 3395 l4kwqe->l2hdr_nbytes = ETH_HLEN; 3396 3397 l4kwqe->da0 = csk->ha[0]; 3398 l4kwqe->da1 = csk->ha[1]; 3399 l4kwqe->da2 = csk->ha[2]; 3400 l4kwqe->da3 = csk->ha[3]; 3401 l4kwqe->da4 = csk->ha[4]; 3402 l4kwqe->da5 = csk->ha[5]; 3403 3404 l4kwqe->sa0 = dev->mac_addr[0]; 3405 l4kwqe->sa1 = dev->mac_addr[1]; 3406 l4kwqe->sa2 = dev->mac_addr[2]; 3407 l4kwqe->sa3 = dev->mac_addr[3]; 3408 l4kwqe->sa4 = dev->mac_addr[4]; 3409 l4kwqe->sa5 = dev->mac_addr[5]; 3410 3411 l4kwqe->etype = ETH_P_IP; 3412 l4kwqe->ipid_start = DEF_IPID_START; 3413 l4kwqe->host_opaque = csk->l5_cid; 3414 3415 if (csk->vlan_id) { 3416 l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING; 3417 l4kwqe->vlan_tag = csk->vlan_id; 3418 l4kwqe->l2hdr_nbytes += 4; 3419 } 3420 3421 return dev->submit_kwqes(dev, wqes, 1); 3422 } 3423 3424 static int cnic_cm_update_pg(struct cnic_sock *csk) 3425 { 3426 struct cnic_dev *dev = csk->dev; 3427 struct l4_kwq_update_pg *l4kwqe; 3428 struct kwqe *wqes[1]; 3429 3430 l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1; 3431 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3432 wqes[0] = (struct kwqe *) l4kwqe; 3433 3434 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG; 3435 l4kwqe->flags = 3436 L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT; 3437 l4kwqe->pg_cid = csk->pg_cid; 3438 3439 l4kwqe->da0 = csk->ha[0]; 3440 l4kwqe->da1 = csk->ha[1]; 3441 l4kwqe->da2 = csk->ha[2]; 3442 l4kwqe->da3 = csk->ha[3]; 3443 l4kwqe->da4 = csk->ha[4]; 3444 l4kwqe->da5 = csk->ha[5]; 3445 3446 l4kwqe->pg_host_opaque = csk->l5_cid; 3447 l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA; 3448 3449 return dev->submit_kwqes(dev, wqes, 1); 3450 } 3451 3452 static int cnic_cm_upload_pg(struct cnic_sock *csk) 3453 { 3454 struct cnic_dev *dev = csk->dev; 3455 struct l4_kwq_upload *l4kwqe; 3456 struct kwqe *wqes[1]; 3457 3458 l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1; 3459 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3460 wqes[0] = (struct kwqe *) l4kwqe; 3461 3462 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG; 3463 l4kwqe->flags = 3464 L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT; 3465 l4kwqe->cid = csk->pg_cid; 3466 3467 return dev->submit_kwqes(dev, wqes, 1); 3468 } 3469 3470 static int cnic_cm_conn_req(struct cnic_sock *csk) 3471 { 3472 struct cnic_dev *dev = csk->dev; 3473 struct l4_kwq_connect_req1 *l4kwqe1; 3474 struct l4_kwq_connect_req2 *l4kwqe2; 3475 struct l4_kwq_connect_req3 *l4kwqe3; 3476 struct kwqe *wqes[3]; 3477 u8 tcp_flags = 0; 3478 int num_wqes = 2; 3479 3480 l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1; 3481 l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2; 3482 l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3; 3483 memset(l4kwqe1, 0, sizeof(*l4kwqe1)); 3484 memset(l4kwqe2, 0, sizeof(*l4kwqe2)); 3485 memset(l4kwqe3, 0, sizeof(*l4kwqe3)); 3486 3487 l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3; 3488 l4kwqe3->flags = 3489 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT; 3490 l4kwqe3->ka_timeout = csk->ka_timeout; 3491 l4kwqe3->ka_interval = csk->ka_interval; 3492 l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count; 3493 l4kwqe3->tos = csk->tos; 3494 l4kwqe3->ttl = csk->ttl; 3495 l4kwqe3->snd_seq_scale = csk->snd_seq_scale; 3496 l4kwqe3->pmtu = csk->mtu; 3497 l4kwqe3->rcv_buf = csk->rcv_buf; 3498 l4kwqe3->snd_buf = csk->snd_buf; 3499 l4kwqe3->seed = csk->seed; 3500 3501 wqes[0] = (struct kwqe *) l4kwqe1; 3502 if (test_bit(SK_F_IPV6, &csk->flags)) { 3503 wqes[1] = (struct kwqe *) l4kwqe2; 3504 wqes[2] = (struct kwqe *) l4kwqe3; 3505 num_wqes = 3; 3506 3507 l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6; 3508 l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2; 3509 l4kwqe2->flags = 3510 L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT | 3511 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT; 3512 l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]); 3513 l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]); 3514 l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]); 3515 l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]); 3516 l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]); 3517 l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]); 3518 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) - 3519 sizeof(struct tcphdr); 3520 } else { 3521 wqes[1] = (struct kwqe *) l4kwqe3; 3522 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) - 3523 sizeof(struct tcphdr); 3524 } 3525 3526 l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1; 3527 l4kwqe1->flags = 3528 (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) | 3529 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT; 3530 l4kwqe1->cid = csk->cid; 3531 l4kwqe1->pg_cid = csk->pg_cid; 3532 l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]); 3533 l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]); 3534 l4kwqe1->src_port = be16_to_cpu(csk->src_port); 3535 l4kwqe1->dst_port = be16_to_cpu(csk->dst_port); 3536 if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK) 3537 tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK; 3538 if (csk->tcp_flags & SK_TCP_KEEP_ALIVE) 3539 tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE; 3540 if (csk->tcp_flags & SK_TCP_NAGLE) 3541 tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE; 3542 if (csk->tcp_flags & SK_TCP_TIMESTAMP) 3543 tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP; 3544 if (csk->tcp_flags & SK_TCP_SACK) 3545 tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK; 3546 if (csk->tcp_flags & SK_TCP_SEG_SCALING) 3547 tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING; 3548 3549 l4kwqe1->tcp_flags = tcp_flags; 3550 3551 return dev->submit_kwqes(dev, wqes, num_wqes); 3552 } 3553 3554 static int cnic_cm_close_req(struct cnic_sock *csk) 3555 { 3556 struct cnic_dev *dev = csk->dev; 3557 struct l4_kwq_close_req *l4kwqe; 3558 struct kwqe *wqes[1]; 3559 3560 l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2; 3561 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3562 wqes[0] = (struct kwqe *) l4kwqe; 3563 3564 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE; 3565 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT; 3566 l4kwqe->cid = csk->cid; 3567 3568 return dev->submit_kwqes(dev, wqes, 1); 3569 } 3570 3571 static int cnic_cm_abort_req(struct cnic_sock *csk) 3572 { 3573 struct cnic_dev *dev = csk->dev; 3574 struct l4_kwq_reset_req *l4kwqe; 3575 struct kwqe *wqes[1]; 3576 3577 l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2; 3578 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3579 wqes[0] = (struct kwqe *) l4kwqe; 3580 3581 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET; 3582 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT; 3583 l4kwqe->cid = csk->cid; 3584 3585 return dev->submit_kwqes(dev, wqes, 1); 3586 } 3587 3588 static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid, 3589 u32 l5_cid, struct cnic_sock **csk, void *context) 3590 { 3591 struct cnic_local *cp = dev->cnic_priv; 3592 struct cnic_sock *csk1; 3593 3594 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3595 return -EINVAL; 3596 3597 if (cp->ctx_tbl) { 3598 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3599 3600 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 3601 return -EAGAIN; 3602 } 3603 3604 csk1 = &cp->csk_tbl[l5_cid]; 3605 if (atomic_read(&csk1->ref_count)) 3606 return -EAGAIN; 3607 3608 if (test_and_set_bit(SK_F_INUSE, &csk1->flags)) 3609 return -EBUSY; 3610 3611 csk1->dev = dev; 3612 csk1->cid = cid; 3613 csk1->l5_cid = l5_cid; 3614 csk1->ulp_type = ulp_type; 3615 csk1->context = context; 3616 3617 csk1->ka_timeout = DEF_KA_TIMEOUT; 3618 csk1->ka_interval = DEF_KA_INTERVAL; 3619 csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT; 3620 csk1->tos = DEF_TOS; 3621 csk1->ttl = DEF_TTL; 3622 csk1->snd_seq_scale = DEF_SND_SEQ_SCALE; 3623 csk1->rcv_buf = DEF_RCV_BUF; 3624 csk1->snd_buf = DEF_SND_BUF; 3625 csk1->seed = DEF_SEED; 3626 csk1->tcp_flags = 0; 3627 3628 *csk = csk1; 3629 return 0; 3630 } 3631 3632 static void cnic_cm_cleanup(struct cnic_sock *csk) 3633 { 3634 if (csk->src_port) { 3635 struct cnic_dev *dev = csk->dev; 3636 struct cnic_local *cp = dev->cnic_priv; 3637 3638 cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port)); 3639 csk->src_port = 0; 3640 } 3641 } 3642 3643 static void cnic_close_conn(struct cnic_sock *csk) 3644 { 3645 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) { 3646 cnic_cm_upload_pg(csk); 3647 clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3648 } 3649 cnic_cm_cleanup(csk); 3650 } 3651 3652 static int cnic_cm_destroy(struct cnic_sock *csk) 3653 { 3654 if (!cnic_in_use(csk)) 3655 return -EINVAL; 3656 3657 csk_hold(csk); 3658 clear_bit(SK_F_INUSE, &csk->flags); 3659 smp_mb__after_atomic(); 3660 while (atomic_read(&csk->ref_count) != 1) 3661 msleep(1); 3662 cnic_cm_cleanup(csk); 3663 3664 csk->flags = 0; 3665 csk_put(csk); 3666 return 0; 3667 } 3668 3669 static inline u16 cnic_get_vlan(struct net_device *dev, 3670 struct net_device **vlan_dev) 3671 { 3672 if (is_vlan_dev(dev)) { 3673 *vlan_dev = vlan_dev_real_dev(dev); 3674 return vlan_dev_vlan_id(dev); 3675 } 3676 *vlan_dev = dev; 3677 return 0; 3678 } 3679 3680 static int cnic_get_v4_route(struct sockaddr_in *dst_addr, 3681 struct dst_entry **dst) 3682 { 3683 #if defined(CONFIG_INET) 3684 struct rtable *rt; 3685 3686 rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0, 3687 RT_SCOPE_UNIVERSE); 3688 if (!IS_ERR(rt)) { 3689 *dst = &rt->dst; 3690 return 0; 3691 } 3692 return PTR_ERR(rt); 3693 #else 3694 return -ENETUNREACH; 3695 #endif 3696 } 3697 3698 static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr, 3699 struct dst_entry **dst) 3700 { 3701 #if IS_ENABLED(CONFIG_IPV6) 3702 struct flowi6 fl6; 3703 3704 memset(&fl6, 0, sizeof(fl6)); 3705 fl6.daddr = dst_addr->sin6_addr; 3706 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 3707 fl6.flowi6_oif = dst_addr->sin6_scope_id; 3708 3709 *dst = ip6_route_output(&init_net, NULL, &fl6); 3710 if ((*dst)->error) { 3711 dst_release(*dst); 3712 *dst = NULL; 3713 return -ENETUNREACH; 3714 } else 3715 return 0; 3716 #endif 3717 3718 return -ENETUNREACH; 3719 } 3720 3721 static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr, 3722 int ulp_type) 3723 { 3724 struct cnic_dev *dev = NULL; 3725 struct dst_entry *dst; 3726 struct net_device *netdev = NULL; 3727 int err = -ENETUNREACH; 3728 3729 if (dst_addr->sin_family == AF_INET) 3730 err = cnic_get_v4_route(dst_addr, &dst); 3731 else if (dst_addr->sin_family == AF_INET6) { 3732 struct sockaddr_in6 *dst_addr6 = 3733 (struct sockaddr_in6 *) dst_addr; 3734 3735 err = cnic_get_v6_route(dst_addr6, &dst); 3736 } else 3737 return NULL; 3738 3739 if (err) 3740 return NULL; 3741 3742 if (!dst->dev) 3743 goto done; 3744 3745 cnic_get_vlan(dst->dev, &netdev); 3746 3747 dev = cnic_from_netdev(netdev); 3748 3749 done: 3750 dst_release(dst); 3751 if (dev) 3752 cnic_put(dev); 3753 return dev; 3754 } 3755 3756 static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3757 { 3758 struct cnic_dev *dev = csk->dev; 3759 struct cnic_local *cp = dev->cnic_priv; 3760 3761 return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk); 3762 } 3763 3764 static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3765 { 3766 struct cnic_dev *dev = csk->dev; 3767 struct cnic_local *cp = dev->cnic_priv; 3768 int is_v6, rc = 0; 3769 struct dst_entry *dst = NULL; 3770 struct net_device *realdev; 3771 __be16 local_port; 3772 u32 port_id; 3773 3774 if (saddr->local.v6.sin6_family == AF_INET6 && 3775 saddr->remote.v6.sin6_family == AF_INET6) 3776 is_v6 = 1; 3777 else if (saddr->local.v4.sin_family == AF_INET && 3778 saddr->remote.v4.sin_family == AF_INET) 3779 is_v6 = 0; 3780 else 3781 return -EINVAL; 3782 3783 clear_bit(SK_F_IPV6, &csk->flags); 3784 3785 if (is_v6) { 3786 set_bit(SK_F_IPV6, &csk->flags); 3787 cnic_get_v6_route(&saddr->remote.v6, &dst); 3788 3789 memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr, 3790 sizeof(struct in6_addr)); 3791 csk->dst_port = saddr->remote.v6.sin6_port; 3792 local_port = saddr->local.v6.sin6_port; 3793 3794 } else { 3795 cnic_get_v4_route(&saddr->remote.v4, &dst); 3796 3797 csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr; 3798 csk->dst_port = saddr->remote.v4.sin_port; 3799 local_port = saddr->local.v4.sin_port; 3800 } 3801 3802 csk->vlan_id = 0; 3803 csk->mtu = dev->netdev->mtu; 3804 if (dst && dst->dev) { 3805 u16 vlan = cnic_get_vlan(dst->dev, &realdev); 3806 if (realdev == dev->netdev) { 3807 csk->vlan_id = vlan; 3808 csk->mtu = dst_mtu(dst); 3809 } 3810 } 3811 3812 port_id = be16_to_cpu(local_port); 3813 if (port_id >= CNIC_LOCAL_PORT_MIN && 3814 port_id < CNIC_LOCAL_PORT_MAX) { 3815 if (cnic_alloc_id(&cp->csk_port_tbl, port_id)) 3816 port_id = 0; 3817 } else 3818 port_id = 0; 3819 3820 if (!port_id) { 3821 port_id = cnic_alloc_new_id(&cp->csk_port_tbl); 3822 if (port_id == -1) { 3823 rc = -ENOMEM; 3824 goto err_out; 3825 } 3826 local_port = cpu_to_be16(port_id); 3827 } 3828 csk->src_port = local_port; 3829 3830 err_out: 3831 dst_release(dst); 3832 return rc; 3833 } 3834 3835 static void cnic_init_csk_state(struct cnic_sock *csk) 3836 { 3837 csk->state = 0; 3838 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3839 clear_bit(SK_F_CLOSING, &csk->flags); 3840 } 3841 3842 static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3843 { 3844 struct cnic_local *cp = csk->dev->cnic_priv; 3845 int err = 0; 3846 3847 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 3848 return -EOPNOTSUPP; 3849 3850 if (!cnic_in_use(csk)) 3851 return -EINVAL; 3852 3853 if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags)) 3854 return -EINVAL; 3855 3856 cnic_init_csk_state(csk); 3857 3858 err = cnic_get_route(csk, saddr); 3859 if (err) 3860 goto err_out; 3861 3862 err = cnic_resolve_addr(csk, saddr); 3863 if (!err) 3864 return 0; 3865 3866 err_out: 3867 clear_bit(SK_F_CONNECT_START, &csk->flags); 3868 return err; 3869 } 3870 3871 static int cnic_cm_abort(struct cnic_sock *csk) 3872 { 3873 struct cnic_local *cp = csk->dev->cnic_priv; 3874 u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP; 3875 3876 if (!cnic_in_use(csk)) 3877 return -EINVAL; 3878 3879 if (cnic_abort_prep(csk)) 3880 return cnic_cm_abort_req(csk); 3881 3882 /* Getting here means that we haven't started connect, or 3883 * connect was not successful, or it has been reset by the target. 3884 */ 3885 3886 cp->close_conn(csk, opcode); 3887 if (csk->state != opcode) { 3888 /* Wait for remote reset sequence to complete */ 3889 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3890 msleep(1); 3891 3892 return -EALREADY; 3893 } 3894 3895 return 0; 3896 } 3897 3898 static int cnic_cm_close(struct cnic_sock *csk) 3899 { 3900 if (!cnic_in_use(csk)) 3901 return -EINVAL; 3902 3903 if (cnic_close_prep(csk)) { 3904 csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 3905 return cnic_cm_close_req(csk); 3906 } else { 3907 /* Wait for remote reset sequence to complete */ 3908 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3909 msleep(1); 3910 3911 return -EALREADY; 3912 } 3913 return 0; 3914 } 3915 3916 static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk, 3917 u8 opcode) 3918 { 3919 struct cnic_ulp_ops *ulp_ops; 3920 int ulp_type = csk->ulp_type; 3921 3922 rcu_read_lock(); 3923 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 3924 if (ulp_ops) { 3925 if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE) 3926 ulp_ops->cm_connect_complete(csk); 3927 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 3928 ulp_ops->cm_close_complete(csk); 3929 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) 3930 ulp_ops->cm_remote_abort(csk); 3931 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP) 3932 ulp_ops->cm_abort_complete(csk); 3933 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED) 3934 ulp_ops->cm_remote_close(csk); 3935 } 3936 rcu_read_unlock(); 3937 } 3938 3939 static int cnic_cm_set_pg(struct cnic_sock *csk) 3940 { 3941 if (cnic_offld_prep(csk)) { 3942 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3943 cnic_cm_update_pg(csk); 3944 else 3945 cnic_cm_offload_pg(csk); 3946 } 3947 return 0; 3948 } 3949 3950 static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe) 3951 { 3952 struct cnic_local *cp = dev->cnic_priv; 3953 u32 l5_cid = kcqe->pg_host_opaque; 3954 u8 opcode = kcqe->op_code; 3955 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 3956 3957 csk_hold(csk); 3958 if (!cnic_in_use(csk)) 3959 goto done; 3960 3961 if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3962 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3963 goto done; 3964 } 3965 /* Possible PG kcqe status: SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */ 3966 if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) { 3967 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3968 cnic_cm_upcall(cp, csk, 3969 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 3970 goto done; 3971 } 3972 3973 csk->pg_cid = kcqe->pg_cid; 3974 set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3975 cnic_cm_conn_req(csk); 3976 3977 done: 3978 csk_put(csk); 3979 } 3980 3981 static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe) 3982 { 3983 struct cnic_local *cp = dev->cnic_priv; 3984 struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe; 3985 u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE; 3986 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3987 3988 ctx->timestamp = jiffies; 3989 ctx->wait_cond = 1; 3990 wake_up(&ctx->waitq); 3991 } 3992 3993 static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe) 3994 { 3995 struct cnic_local *cp = dev->cnic_priv; 3996 struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe; 3997 u8 opcode = l4kcqe->op_code; 3998 u32 l5_cid; 3999 struct cnic_sock *csk; 4000 4001 if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) { 4002 cnic_process_fcoe_term_conn(dev, kcqe); 4003 return; 4004 } 4005 if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG || 4006 opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 4007 cnic_cm_process_offld_pg(dev, l4kcqe); 4008 return; 4009 } 4010 4011 l5_cid = l4kcqe->conn_id; 4012 if (opcode & 0x80) 4013 l5_cid = l4kcqe->cid; 4014 if (l5_cid >= MAX_CM_SK_TBL_SZ) 4015 return; 4016 4017 csk = &cp->csk_tbl[l5_cid]; 4018 csk_hold(csk); 4019 4020 if (!cnic_in_use(csk)) { 4021 csk_put(csk); 4022 return; 4023 } 4024 4025 switch (opcode) { 4026 case L5CM_RAMROD_CMD_ID_TCP_CONNECT: 4027 if (l4kcqe->status != 0) { 4028 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4029 cnic_cm_upcall(cp, csk, 4030 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 4031 } 4032 break; 4033 case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE: 4034 if (l4kcqe->status == 0) 4035 set_bit(SK_F_OFFLD_COMPLETE, &csk->flags); 4036 else if (l4kcqe->status == 4037 L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4038 set_bit(SK_F_HW_ERR, &csk->flags); 4039 4040 smp_mb__before_atomic(); 4041 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4042 cnic_cm_upcall(cp, csk, opcode); 4043 break; 4044 4045 case L5CM_RAMROD_CMD_ID_CLOSE: { 4046 struct iscsi_kcqe *l5kcqe = (struct iscsi_kcqe *) kcqe; 4047 4048 if (l4kcqe->status == 0 && l5kcqe->completion_status == 0) 4049 break; 4050 4051 netdev_warn(dev->netdev, "RAMROD CLOSE compl with status 0x%x completion status 0x%x\n", 4052 l4kcqe->status, l5kcqe->completion_status); 4053 opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 4054 } 4055 fallthrough; 4056 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4057 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4058 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4059 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4060 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4061 if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4062 set_bit(SK_F_HW_ERR, &csk->flags); 4063 4064 cp->close_conn(csk, opcode); 4065 break; 4066 4067 case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED: 4068 /* after we already sent CLOSE_REQ */ 4069 if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) && 4070 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) && 4071 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 4072 cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP); 4073 else 4074 cnic_cm_upcall(cp, csk, opcode); 4075 break; 4076 } 4077 csk_put(csk); 4078 } 4079 4080 static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num) 4081 { 4082 struct cnic_dev *dev = data; 4083 int i; 4084 4085 for (i = 0; i < num; i++) 4086 cnic_cm_process_kcqe(dev, kcqe[i]); 4087 } 4088 4089 static struct cnic_ulp_ops cm_ulp_ops = { 4090 .indicate_kcqes = cnic_cm_indicate_kcqe, 4091 }; 4092 4093 static void cnic_cm_free_mem(struct cnic_dev *dev) 4094 { 4095 struct cnic_local *cp = dev->cnic_priv; 4096 4097 kvfree(cp->csk_tbl); 4098 cp->csk_tbl = NULL; 4099 cnic_free_id_tbl(&cp->csk_port_tbl); 4100 } 4101 4102 static int cnic_cm_alloc_mem(struct cnic_dev *dev) 4103 { 4104 struct cnic_local *cp = dev->cnic_priv; 4105 u32 port_id; 4106 int i; 4107 4108 cp->csk_tbl = kvcalloc(MAX_CM_SK_TBL_SZ, sizeof(struct cnic_sock), 4109 GFP_KERNEL); 4110 if (!cp->csk_tbl) 4111 return -ENOMEM; 4112 4113 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) 4114 atomic_set(&cp->csk_tbl[i].ref_count, 0); 4115 4116 port_id = get_random_u32_below(CNIC_LOCAL_PORT_RANGE); 4117 if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, 4118 CNIC_LOCAL_PORT_MIN, port_id)) { 4119 cnic_cm_free_mem(dev); 4120 return -ENOMEM; 4121 } 4122 return 0; 4123 } 4124 4125 static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode) 4126 { 4127 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 4128 /* Unsolicited RESET_COMP or RESET_RECEIVED */ 4129 opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED; 4130 csk->state = opcode; 4131 } 4132 4133 /* 1. If event opcode matches the expected event in csk->state 4134 * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any 4135 * event 4136 * 3. If the expected event is 0, meaning the connection was never 4137 * never established, we accept the opcode from cm_abort. 4138 */ 4139 if (opcode == csk->state || csk->state == 0 || 4140 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP || 4141 csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) { 4142 if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) { 4143 if (csk->state == 0) 4144 csk->state = opcode; 4145 return 1; 4146 } 4147 } 4148 return 0; 4149 } 4150 4151 static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode) 4152 { 4153 struct cnic_dev *dev = csk->dev; 4154 struct cnic_local *cp = dev->cnic_priv; 4155 4156 if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) { 4157 cnic_cm_upcall(cp, csk, opcode); 4158 return; 4159 } 4160 4161 clear_bit(SK_F_CONNECT_START, &csk->flags); 4162 cnic_close_conn(csk); 4163 csk->state = opcode; 4164 cnic_cm_upcall(cp, csk, opcode); 4165 } 4166 4167 static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev) 4168 { 4169 } 4170 4171 static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) 4172 { 4173 u32 seed; 4174 4175 seed = get_random_u32(); 4176 cnic_ctx_wr(dev, 45, 0, seed); 4177 return 0; 4178 } 4179 4180 static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode) 4181 { 4182 struct cnic_dev *dev = csk->dev; 4183 struct cnic_local *cp = dev->cnic_priv; 4184 struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid]; 4185 union l5cm_specific_data l5_data; 4186 u32 cmd = 0; 4187 int close_complete = 0; 4188 4189 switch (opcode) { 4190 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4191 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4192 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4193 if (cnic_ready_to_close(csk, opcode)) { 4194 if (test_bit(SK_F_HW_ERR, &csk->flags)) 4195 close_complete = 1; 4196 else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 4197 cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE; 4198 else 4199 close_complete = 1; 4200 } 4201 break; 4202 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4203 cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD; 4204 break; 4205 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4206 close_complete = 1; 4207 break; 4208 } 4209 if (cmd) { 4210 memset(&l5_data, 0, sizeof(l5_data)); 4211 4212 cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE, 4213 &l5_data); 4214 } else if (close_complete) { 4215 ctx->timestamp = jiffies; 4216 cnic_close_conn(csk); 4217 cnic_cm_upcall(cp, csk, csk->state); 4218 } 4219 } 4220 4221 static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) 4222 { 4223 struct cnic_local *cp = dev->cnic_priv; 4224 4225 if (!cp->ctx_tbl) 4226 return; 4227 4228 if (!netif_running(dev->netdev)) 4229 return; 4230 4231 cnic_bnx2x_delete_wait(dev, 0); 4232 4233 cancel_delayed_work_sync(&cp->delete_task); 4234 4235 if (atomic_read(&cp->iscsi_conn) != 0) 4236 netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", 4237 atomic_read(&cp->iscsi_conn)); 4238 } 4239 4240 static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev) 4241 { 4242 struct bnx2x *bp = netdev_priv(dev->netdev); 4243 u32 pfid = bp->pfid; 4244 u32 port = BP_PORT(bp); 4245 4246 cnic_init_bnx2x_mac(dev); 4247 cnic_bnx2x_set_tcp_options(dev, 0, 1); 4248 4249 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 4250 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0); 4251 4252 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4253 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1); 4254 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4255 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port), 4256 DEF_MAX_DA_COUNT); 4257 4258 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4259 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL); 4260 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4261 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS); 4262 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4263 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2); 4264 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4265 XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER); 4266 4267 CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid), 4268 DEF_MAX_CWND); 4269 return 0; 4270 } 4271 4272 static void cnic_delete_task(struct work_struct *work) 4273 { 4274 struct cnic_local *cp; 4275 struct cnic_dev *dev; 4276 u32 i; 4277 int need_resched = 0; 4278 4279 cp = container_of(work, struct cnic_local, delete_task.work); 4280 dev = cp->dev; 4281 4282 if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) { 4283 struct drv_ctl_info info; 4284 4285 cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI); 4286 4287 memset(&info, 0, sizeof(struct drv_ctl_info)); 4288 info.cmd = DRV_CTL_ISCSI_STOPPED_CMD; 4289 cp->ethdev->drv_ctl(dev->netdev, &info); 4290 } 4291 4292 for (i = 0; i < cp->max_cid_space; i++) { 4293 struct cnic_context *ctx = &cp->ctx_tbl[i]; 4294 int err; 4295 4296 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) || 4297 !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4298 continue; 4299 4300 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 4301 need_resched = 1; 4302 continue; 4303 } 4304 4305 if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4306 continue; 4307 4308 err = cnic_bnx2x_destroy_ramrod(dev, i); 4309 4310 cnic_free_bnx2x_conn_resc(dev, i); 4311 if (!err) { 4312 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) 4313 atomic_dec(&cp->iscsi_conn); 4314 4315 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 4316 } 4317 } 4318 4319 if (need_resched) 4320 queue_delayed_work(cnic_wq, &cp->delete_task, 4321 msecs_to_jiffies(10)); 4322 4323 } 4324 4325 static int cnic_cm_open(struct cnic_dev *dev) 4326 { 4327 struct cnic_local *cp = dev->cnic_priv; 4328 int err; 4329 4330 err = cnic_cm_alloc_mem(dev); 4331 if (err) 4332 return err; 4333 4334 err = cp->start_cm(dev); 4335 4336 if (err) 4337 goto err_out; 4338 4339 INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task); 4340 4341 dev->cm_create = cnic_cm_create; 4342 dev->cm_destroy = cnic_cm_destroy; 4343 dev->cm_connect = cnic_cm_connect; 4344 dev->cm_abort = cnic_cm_abort; 4345 dev->cm_close = cnic_cm_close; 4346 dev->cm_select_dev = cnic_cm_select_dev; 4347 4348 cp->ulp_handle[CNIC_ULP_L4] = dev; 4349 rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops); 4350 return 0; 4351 4352 err_out: 4353 cnic_cm_free_mem(dev); 4354 return err; 4355 } 4356 4357 static int cnic_cm_shutdown(struct cnic_dev *dev) 4358 { 4359 struct cnic_local *cp = dev->cnic_priv; 4360 int i; 4361 4362 if (!cp->csk_tbl) 4363 return 0; 4364 4365 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) { 4366 struct cnic_sock *csk = &cp->csk_tbl[i]; 4367 4368 clear_bit(SK_F_INUSE, &csk->flags); 4369 cnic_cm_cleanup(csk); 4370 } 4371 cnic_cm_free_mem(dev); 4372 4373 return 0; 4374 } 4375 4376 static void cnic_init_context(struct cnic_dev *dev, u32 cid) 4377 { 4378 u32 cid_addr; 4379 int i; 4380 4381 cid_addr = GET_CID_ADDR(cid); 4382 4383 for (i = 0; i < CTX_SIZE; i += 4) 4384 cnic_ctx_wr(dev, cid_addr, i, 0); 4385 } 4386 4387 static int cnic_setup_5709_context(struct cnic_dev *dev, int valid) 4388 { 4389 struct cnic_local *cp = dev->cnic_priv; 4390 int ret = 0, i; 4391 u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0; 4392 4393 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4394 return 0; 4395 4396 for (i = 0; i < cp->ctx_blks; i++) { 4397 int j; 4398 u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk; 4399 u32 val; 4400 4401 memset(cp->ctx_arr[i].ctx, 0, CNIC_PAGE_SIZE); 4402 4403 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0, 4404 (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit); 4405 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1, 4406 (u64) cp->ctx_arr[i].mapping >> 32); 4407 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx | 4408 BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ); 4409 for (j = 0; j < 10; j++) { 4410 4411 val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL); 4412 if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ)) 4413 break; 4414 udelay(5); 4415 } 4416 if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) { 4417 ret = -EBUSY; 4418 break; 4419 } 4420 } 4421 return ret; 4422 } 4423 4424 static void cnic_free_irq(struct cnic_dev *dev) 4425 { 4426 struct cnic_local *cp = dev->cnic_priv; 4427 struct cnic_eth_dev *ethdev = cp->ethdev; 4428 4429 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4430 cp->disable_int_sync(dev); 4431 cancel_work_sync(&cp->cnic_irq_bh_work); 4432 free_irq(ethdev->irq_arr[0].vector, dev); 4433 } 4434 } 4435 4436 static int cnic_request_irq(struct cnic_dev *dev) 4437 { 4438 struct cnic_local *cp = dev->cnic_priv; 4439 struct cnic_eth_dev *ethdev = cp->ethdev; 4440 int err; 4441 4442 err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev); 4443 if (err) 4444 disable_work_sync(&cp->cnic_irq_bh_work); 4445 4446 return err; 4447 } 4448 4449 static int cnic_init_bnx2_irq(struct cnic_dev *dev) 4450 { 4451 struct cnic_local *cp = dev->cnic_priv; 4452 struct cnic_eth_dev *ethdev = cp->ethdev; 4453 4454 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4455 int err, i = 0; 4456 int sblk_num = cp->status_blk_num; 4457 u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) + 4458 BNX2_HC_SB_CONFIG_1; 4459 4460 CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT); 4461 4462 CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8); 4463 CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220); 4464 CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220); 4465 4466 cp->last_status_idx = cp->status_blk.bnx2->status_idx; 4467 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2_msix); 4468 err = cnic_request_irq(dev); 4469 if (err) 4470 return err; 4471 4472 while (cp->status_blk.bnx2->status_completion_producer_index && 4473 i < 10) { 4474 CNIC_WR(dev, BNX2_HC_COALESCE_NOW, 4475 1 << (11 + sblk_num)); 4476 udelay(10); 4477 i++; 4478 barrier(); 4479 } 4480 if (cp->status_blk.bnx2->status_completion_producer_index) { 4481 cnic_free_irq(dev); 4482 goto failed; 4483 } 4484 4485 } else { 4486 struct status_block *sblk = cp->status_blk.gen; 4487 u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND); 4488 int i = 0; 4489 4490 while (sblk->status_completion_producer_index && i < 10) { 4491 CNIC_WR(dev, BNX2_HC_COMMAND, 4492 hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT); 4493 udelay(10); 4494 i++; 4495 barrier(); 4496 } 4497 if (sblk->status_completion_producer_index) 4498 goto failed; 4499 4500 } 4501 return 0; 4502 4503 failed: 4504 netdev_err(dev->netdev, "KCQ index not resetting to 0\n"); 4505 return -EBUSY; 4506 } 4507 4508 static void cnic_enable_bnx2_int(struct cnic_dev *dev) 4509 { 4510 struct cnic_local *cp = dev->cnic_priv; 4511 struct cnic_eth_dev *ethdev = cp->ethdev; 4512 4513 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4514 return; 4515 4516 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4517 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 4518 } 4519 4520 static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev) 4521 { 4522 struct cnic_local *cp = dev->cnic_priv; 4523 struct cnic_eth_dev *ethdev = cp->ethdev; 4524 4525 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4526 return; 4527 4528 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4529 BNX2_PCICFG_INT_ACK_CMD_MASK_INT); 4530 CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD); 4531 synchronize_irq(ethdev->irq_arr[0].vector); 4532 } 4533 4534 static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev) 4535 { 4536 struct cnic_local *cp = dev->cnic_priv; 4537 struct cnic_eth_dev *ethdev = cp->ethdev; 4538 struct cnic_uio_dev *udev = cp->udev; 4539 u32 cid_addr, tx_cid, sb_id; 4540 u32 val, offset0, offset1, offset2, offset3; 4541 int i; 4542 struct bnx2_tx_bd *txbd; 4543 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4544 struct status_block *s_blk = cp->status_blk.gen; 4545 4546 sb_id = cp->status_blk_num; 4547 tx_cid = 20; 4548 cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2; 4549 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4550 struct status_block_msix *sblk = cp->status_blk.bnx2; 4551 4552 tx_cid = TX_TSS_CID + sb_id - 1; 4553 CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) | 4554 (TX_TSS_CID << 7)); 4555 cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index; 4556 } 4557 cp->tx_cons = *cp->tx_cons_ptr; 4558 4559 cid_addr = GET_CID_ADDR(tx_cid); 4560 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 4561 u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40; 4562 4563 for (i = 0; i < PHY_CTX_SIZE; i += 4) 4564 cnic_ctx_wr(dev, cid_addr2, i, 0); 4565 4566 offset0 = BNX2_L2CTX_TYPE_XI; 4567 offset1 = BNX2_L2CTX_CMD_TYPE_XI; 4568 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI; 4569 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI; 4570 } else { 4571 cnic_init_context(dev, tx_cid); 4572 cnic_init_context(dev, tx_cid + 1); 4573 4574 offset0 = BNX2_L2CTX_TYPE; 4575 offset1 = BNX2_L2CTX_CMD_TYPE; 4576 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI; 4577 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO; 4578 } 4579 val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2; 4580 cnic_ctx_wr(dev, cid_addr, offset0, val); 4581 4582 val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16); 4583 cnic_ctx_wr(dev, cid_addr, offset1, val); 4584 4585 txbd = udev->l2_ring; 4586 4587 buf_map = udev->l2_buf_map; 4588 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) { 4589 txbd->tx_bd_haddr_hi = (u64) buf_map >> 32; 4590 txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4591 } 4592 val = (u64) ring_map >> 32; 4593 cnic_ctx_wr(dev, cid_addr, offset2, val); 4594 txbd->tx_bd_haddr_hi = val; 4595 4596 val = (u64) ring_map & 0xffffffff; 4597 cnic_ctx_wr(dev, cid_addr, offset3, val); 4598 txbd->tx_bd_haddr_lo = val; 4599 } 4600 4601 static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev) 4602 { 4603 struct cnic_local *cp = dev->cnic_priv; 4604 struct cnic_eth_dev *ethdev = cp->ethdev; 4605 struct cnic_uio_dev *udev = cp->udev; 4606 u32 cid_addr, sb_id, val, coal_reg, coal_val; 4607 int i; 4608 struct bnx2_rx_bd *rxbd; 4609 struct status_block *s_blk = cp->status_blk.gen; 4610 dma_addr_t ring_map = udev->l2_ring_map; 4611 4612 sb_id = cp->status_blk_num; 4613 cnic_init_context(dev, 2); 4614 cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2; 4615 coal_reg = BNX2_HC_COMMAND; 4616 coal_val = CNIC_RD(dev, coal_reg); 4617 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4618 struct status_block_msix *sblk = cp->status_blk.bnx2; 4619 4620 cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index; 4621 coal_reg = BNX2_HC_COALESCE_NOW; 4622 coal_val = 1 << (11 + sb_id); 4623 } 4624 i = 0; 4625 while (!(*cp->rx_cons_ptr != 0) && i < 10) { 4626 CNIC_WR(dev, coal_reg, coal_val); 4627 udelay(10); 4628 i++; 4629 barrier(); 4630 } 4631 cp->rx_cons = *cp->rx_cons_ptr; 4632 4633 cid_addr = GET_CID_ADDR(2); 4634 val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE | 4635 BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8); 4636 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val); 4637 4638 if (sb_id == 0) 4639 val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT; 4640 else 4641 val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id); 4642 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val); 4643 4644 rxbd = udev->l2_ring + CNIC_PAGE_SIZE; 4645 for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) { 4646 dma_addr_t buf_map; 4647 int n = (i % cp->l2_rx_ring_size) + 1; 4648 4649 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4650 rxbd->rx_bd_len = cp->l2_single_buf_size; 4651 rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END; 4652 rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32; 4653 rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4654 } 4655 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 4656 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val); 4657 rxbd->rx_bd_haddr_hi = val; 4658 4659 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 4660 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val); 4661 rxbd->rx_bd_haddr_lo = val; 4662 4663 val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD); 4664 cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2)); 4665 } 4666 4667 static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev) 4668 { 4669 struct kwqe *wqes[1], l2kwqe; 4670 4671 memset(&l2kwqe, 0, sizeof(l2kwqe)); 4672 wqes[0] = &l2kwqe; 4673 l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) | 4674 (L2_KWQE_OPCODE_VALUE_FLUSH << 4675 KWQE_OPCODE_SHIFT) | 2; 4676 dev->submit_kwqes(dev, wqes, 1); 4677 } 4678 4679 static void cnic_set_bnx2_mac(struct cnic_dev *dev) 4680 { 4681 struct cnic_local *cp = dev->cnic_priv; 4682 u32 val; 4683 4684 val = cp->func << 2; 4685 4686 cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val); 4687 4688 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4689 BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER); 4690 dev->mac_addr[0] = (u8) (val >> 8); 4691 dev->mac_addr[1] = (u8) val; 4692 4693 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val); 4694 4695 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4696 BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER); 4697 dev->mac_addr[2] = (u8) (val >> 24); 4698 dev->mac_addr[3] = (u8) (val >> 16); 4699 dev->mac_addr[4] = (u8) (val >> 8); 4700 dev->mac_addr[5] = (u8) val; 4701 4702 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val); 4703 4704 val = 4 | BNX2_RPM_SORT_USER2_BC_EN; 4705 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4706 val |= BNX2_RPM_SORT_USER2_PROM_VLAN; 4707 4708 CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0); 4709 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val); 4710 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA); 4711 } 4712 4713 static int cnic_start_bnx2_hw(struct cnic_dev *dev) 4714 { 4715 struct cnic_local *cp = dev->cnic_priv; 4716 struct cnic_eth_dev *ethdev = cp->ethdev; 4717 struct status_block *sblk = cp->status_blk.gen; 4718 u32 val, kcq_cid_addr, kwq_cid_addr; 4719 int err; 4720 4721 cnic_set_bnx2_mac(dev); 4722 4723 val = CNIC_RD(dev, BNX2_MQ_CONFIG); 4724 val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE; 4725 if (CNIC_PAGE_BITS > 12) 4726 val |= (12 - 8) << 4; 4727 else 4728 val |= (CNIC_PAGE_BITS - 8) << 4; 4729 4730 CNIC_WR(dev, BNX2_MQ_CONFIG, val); 4731 4732 CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8); 4733 CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220); 4734 CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220); 4735 4736 err = cnic_setup_5709_context(dev, 1); 4737 if (err) 4738 return err; 4739 4740 cnic_init_context(dev, KWQ_CID); 4741 cnic_init_context(dev, KCQ_CID); 4742 4743 kwq_cid_addr = GET_CID_ADDR(KWQ_CID); 4744 cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX; 4745 4746 cp->max_kwq_idx = MAX_KWQ_IDX; 4747 cp->kwq_prod_idx = 0; 4748 cp->kwq_con_idx = 0; 4749 set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 4750 4751 if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708) 4752 cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15; 4753 else 4754 cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index; 4755 4756 /* Initialize the kernel work queue context. */ 4757 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4758 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4759 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val); 4760 4761 val = (CNIC_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16; 4762 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4763 4764 val = ((CNIC_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT; 4765 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4766 4767 val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32); 4768 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4769 4770 val = (u32) cp->kwq_info.pgtbl_map; 4771 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4772 4773 kcq_cid_addr = GET_CID_ADDR(KCQ_CID); 4774 cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX; 4775 4776 cp->kcq1.sw_prod_idx = 0; 4777 cp->kcq1.hw_prod_idx_ptr = 4778 &sblk->status_completion_producer_index; 4779 4780 cp->kcq1.status_idx_ptr = &sblk->status_idx; 4781 4782 /* Initialize the kernel complete queue context. */ 4783 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4784 (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4785 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val); 4786 4787 val = (CNIC_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16; 4788 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4789 4790 val = ((CNIC_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT; 4791 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4792 4793 val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32); 4794 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4795 4796 val = (u32) cp->kcq1.dma.pgtbl_map; 4797 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4798 4799 cp->int_num = 0; 4800 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4801 struct status_block_msix *msblk = cp->status_blk.bnx2; 4802 u32 sb_id = cp->status_blk_num; 4803 u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id); 4804 4805 cp->kcq1.hw_prod_idx_ptr = 4806 &msblk->status_completion_producer_index; 4807 cp->kcq1.status_idx_ptr = &msblk->status_idx; 4808 cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index; 4809 cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT; 4810 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4811 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4812 } 4813 4814 /* Enable Commnad Scheduler notification when we write to the 4815 * host producer index of the kernel contexts. */ 4816 CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2); 4817 4818 /* Enable Command Scheduler notification when we write to either 4819 * the Send Queue or Receive Queue producer indexes of the kernel 4820 * bypass contexts. */ 4821 CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7); 4822 CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7); 4823 4824 /* Notify COM when the driver post an application buffer. */ 4825 CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000); 4826 4827 /* Set the CP and COM doorbells. These two processors polls the 4828 * doorbell for a non zero value before running. This must be done 4829 * after setting up the kernel queue contexts. */ 4830 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1); 4831 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1); 4832 4833 cnic_init_bnx2_tx_ring(dev); 4834 cnic_init_bnx2_rx_ring(dev); 4835 4836 err = cnic_init_bnx2_irq(dev); 4837 if (err) { 4838 netdev_err(dev->netdev, "cnic_init_irq failed\n"); 4839 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 4840 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 4841 return err; 4842 } 4843 4844 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 4845 4846 return 0; 4847 } 4848 4849 static void cnic_setup_bnx2x_context(struct cnic_dev *dev) 4850 { 4851 struct cnic_local *cp = dev->cnic_priv; 4852 struct cnic_eth_dev *ethdev = cp->ethdev; 4853 u32 start_offset = ethdev->ctx_tbl_offset; 4854 int i; 4855 4856 for (i = 0; i < cp->ctx_blks; i++) { 4857 struct cnic_ctx *ctx = &cp->ctx_arr[i]; 4858 dma_addr_t map = ctx->mapping; 4859 4860 if (cp->ctx_align) { 4861 unsigned long mask = cp->ctx_align - 1; 4862 4863 map = (map + mask) & ~mask; 4864 } 4865 4866 cnic_ctx_tbl_wr(dev, start_offset + i, map); 4867 } 4868 } 4869 4870 static int cnic_init_bnx2x_irq(struct cnic_dev *dev) 4871 { 4872 struct cnic_local *cp = dev->cnic_priv; 4873 struct cnic_eth_dev *ethdev = cp->ethdev; 4874 int err = 0; 4875 4876 INIT_WORK(&cp->cnic_irq_bh_work, cnic_service_bnx2x_bh_work); 4877 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 4878 err = cnic_request_irq(dev); 4879 4880 return err; 4881 } 4882 4883 static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev, 4884 u16 sb_id, u8 sb_index, 4885 u8 disable) 4886 { 4887 struct bnx2x *bp = netdev_priv(dev->netdev); 4888 4889 u32 addr = BAR_CSTRORM_INTMEM + 4890 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4891 offsetof(struct hc_status_block_data_e1x, index_data) + 4892 sizeof(struct hc_index_data)*sb_index + 4893 offsetof(struct hc_index_data, flags); 4894 u16 flags = CNIC_RD16(dev, addr); 4895 /* clear and set */ 4896 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4897 flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) & 4898 HC_INDEX_DATA_HC_ENABLED); 4899 CNIC_WR16(dev, addr, flags); 4900 } 4901 4902 static void cnic_enable_bnx2x_int(struct cnic_dev *dev) 4903 { 4904 struct cnic_local *cp = dev->cnic_priv; 4905 struct bnx2x *bp = netdev_priv(dev->netdev); 4906 u8 sb_id = cp->status_blk_num; 4907 4908 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 4909 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4910 offsetof(struct hc_status_block_data_e1x, index_data) + 4911 sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS + 4912 offsetof(struct hc_index_data, timeout), 64 / 4); 4913 cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0); 4914 } 4915 4916 static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev) 4917 { 4918 } 4919 4920 static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev, 4921 struct client_init_ramrod_data *data) 4922 { 4923 struct cnic_local *cp = dev->cnic_priv; 4924 struct bnx2x *bp = netdev_priv(dev->netdev); 4925 struct cnic_uio_dev *udev = cp->udev; 4926 union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring; 4927 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4928 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4929 int i; 4930 u32 cli = cp->ethdev->iscsi_l2_client_id; 4931 u32 val; 4932 4933 memset(txbd, 0, CNIC_PAGE_SIZE); 4934 4935 buf_map = udev->l2_buf_map; 4936 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) { 4937 struct eth_tx_start_bd *start_bd = &txbd->start_bd; 4938 struct eth_tx_parse_bd_e1x *pbd_e1x = 4939 &((txbd + 1)->parse_bd_e1x); 4940 struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2); 4941 struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd); 4942 4943 start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4944 start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4945 reg_bd->addr_hi = start_bd->addr_hi; 4946 reg_bd->addr_lo = start_bd->addr_lo + 0x10; 4947 start_bd->nbytes = cpu_to_le16(0x10); 4948 start_bd->nbd = cpu_to_le16(3); 4949 start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 4950 start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS; 4951 start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 4952 4953 if (BNX2X_CHIP_IS_E2_PLUS(bp)) 4954 pbd_e2->parsing_data = (UNICAST_ADDRESS << 4955 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT); 4956 else 4957 pbd_e1x->global_data = (UNICAST_ADDRESS << 4958 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT); 4959 } 4960 4961 val = (u64) ring_map >> 32; 4962 txbd->next_bd.addr_hi = cpu_to_le32(val); 4963 4964 data->tx.tx_bd_page_base.hi = cpu_to_le32(val); 4965 4966 val = (u64) ring_map & 0xffffffff; 4967 txbd->next_bd.addr_lo = cpu_to_le32(val); 4968 4969 data->tx.tx_bd_page_base.lo = cpu_to_le32(val); 4970 4971 /* Other ramrod params */ 4972 data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS; 4973 data->tx.tx_status_block_id = BNX2X_DEF_SB_ID; 4974 4975 /* reset xstorm per client statistics */ 4976 if (cli < MAX_STAT_COUNTER_ID) { 4977 data->general.statistics_zero_flg = 1; 4978 data->general.statistics_en_flg = 1; 4979 data->general.statistics_counter_id = cli; 4980 } 4981 4982 cp->tx_cons_ptr = 4983 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS]; 4984 } 4985 4986 static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev, 4987 struct client_init_ramrod_data *data) 4988 { 4989 struct cnic_local *cp = dev->cnic_priv; 4990 struct bnx2x *bp = netdev_priv(dev->netdev); 4991 struct cnic_uio_dev *udev = cp->udev; 4992 struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring + 4993 CNIC_PAGE_SIZE); 4994 struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *) 4995 (udev->l2_ring + (2 * CNIC_PAGE_SIZE)); 4996 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4997 int i; 4998 u32 cli = cp->ethdev->iscsi_l2_client_id; 4999 int cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5000 u32 val; 5001 dma_addr_t ring_map = udev->l2_ring_map; 5002 5003 /* General data */ 5004 data->general.client_id = cli; 5005 data->general.activate_flg = 1; 5006 data->general.sp_client_id = cli; 5007 data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14); 5008 data->general.func_id = bp->pfid; 5009 5010 for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) { 5011 dma_addr_t buf_map; 5012 int n = (i % cp->l2_rx_ring_size) + 1; 5013 5014 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 5015 rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 5016 rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 5017 } 5018 5019 val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32; 5020 rxbd->addr_hi = cpu_to_le32(val); 5021 data->rx.bd_page_base.hi = cpu_to_le32(val); 5022 5023 val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff; 5024 rxbd->addr_lo = cpu_to_le32(val); 5025 data->rx.bd_page_base.lo = cpu_to_le32(val); 5026 5027 rxcqe += BNX2X_MAX_RCQ_DESC_CNT; 5028 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) >> 32; 5029 rxcqe->addr_hi = cpu_to_le32(val); 5030 data->rx.cqe_page_base.hi = cpu_to_le32(val); 5031 5032 val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) & 0xffffffff; 5033 rxcqe->addr_lo = cpu_to_le32(val); 5034 data->rx.cqe_page_base.lo = cpu_to_le32(val); 5035 5036 /* Other ramrod params */ 5037 data->rx.client_qzone_id = cl_qzone_id; 5038 data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS; 5039 data->rx.status_block_id = BNX2X_DEF_SB_ID; 5040 5041 data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT; 5042 5043 data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size); 5044 data->rx.outer_vlan_removal_enable_flg = 1; 5045 data->rx.silent_vlan_removal_flg = 1; 5046 data->rx.silent_vlan_value = 0; 5047 data->rx.silent_vlan_mask = 0xffff; 5048 5049 cp->rx_cons_ptr = 5050 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS]; 5051 cp->rx_cons = *cp->rx_cons_ptr; 5052 } 5053 5054 static void cnic_init_bnx2x_kcq(struct cnic_dev *dev) 5055 { 5056 struct cnic_local *cp = dev->cnic_priv; 5057 struct bnx2x *bp = netdev_priv(dev->netdev); 5058 u32 pfid = bp->pfid; 5059 5060 cp->kcq1.io_addr = BAR_CSTRORM_INTMEM + 5061 CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0); 5062 cp->kcq1.sw_prod_idx = 0; 5063 5064 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5065 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5066 5067 cp->kcq1.hw_prod_idx_ptr = 5068 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5069 cp->kcq1.status_idx_ptr = 5070 &sb->sb.running_index[SM_RX_ID]; 5071 } else { 5072 struct host_hc_status_block_e1x *sb = cp->status_blk.gen; 5073 5074 cp->kcq1.hw_prod_idx_ptr = 5075 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5076 cp->kcq1.status_idx_ptr = 5077 &sb->sb.running_index[SM_RX_ID]; 5078 } 5079 5080 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5081 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5082 5083 cp->kcq2.io_addr = BAR_USTRORM_INTMEM + 5084 USTORM_FCOE_EQ_PROD_OFFSET(pfid); 5085 cp->kcq2.sw_prod_idx = 0; 5086 cp->kcq2.hw_prod_idx_ptr = 5087 &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS]; 5088 cp->kcq2.status_idx_ptr = 5089 &sb->sb.running_index[SM_RX_ID]; 5090 } 5091 } 5092 5093 static int cnic_start_bnx2x_hw(struct cnic_dev *dev) 5094 { 5095 struct cnic_local *cp = dev->cnic_priv; 5096 struct bnx2x *bp = netdev_priv(dev->netdev); 5097 struct cnic_eth_dev *ethdev = cp->ethdev; 5098 int ret; 5099 u32 pfid; 5100 5101 dev->stats_addr = ethdev->addr_drv_info_to_mcp; 5102 cp->func = bp->pf_num; 5103 5104 pfid = bp->pfid; 5105 5106 ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ, 5107 cp->iscsi_start_cid, 0); 5108 5109 if (ret) 5110 return -ENOMEM; 5111 5112 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5113 ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn, 5114 cp->fcoe_start_cid, 0); 5115 5116 if (ret) 5117 return -ENOMEM; 5118 } 5119 5120 cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2; 5121 5122 cnic_init_bnx2x_kcq(dev); 5123 5124 /* Only 1 EQ */ 5125 CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX); 5126 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5127 CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0); 5128 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5129 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0), 5130 cp->kcq1.dma.pg_map_arr[1] & 0xffffffff); 5131 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5132 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4, 5133 (u64) cp->kcq1.dma.pg_map_arr[1] >> 32); 5134 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5135 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0), 5136 cp->kcq1.dma.pg_map_arr[0] & 0xffffffff); 5137 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5138 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4, 5139 (u64) cp->kcq1.dma.pg_map_arr[0] >> 32); 5140 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5141 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1); 5142 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 5143 CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num); 5144 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5145 CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0), 5146 HC_INDEX_ISCSI_EQ_CONS); 5147 5148 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5149 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid), 5150 cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff); 5151 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5152 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4, 5153 (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32); 5154 5155 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 5156 TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF); 5157 5158 cnic_setup_bnx2x_context(dev); 5159 5160 ret = cnic_init_bnx2x_irq(dev); 5161 if (ret) 5162 return ret; 5163 5164 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 5165 return 0; 5166 } 5167 5168 static void cnic_init_rings(struct cnic_dev *dev) 5169 { 5170 struct cnic_local *cp = dev->cnic_priv; 5171 struct bnx2x *bp = netdev_priv(dev->netdev); 5172 struct cnic_uio_dev *udev = cp->udev; 5173 5174 if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5175 return; 5176 5177 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5178 cnic_init_bnx2_tx_ring(dev); 5179 cnic_init_bnx2_rx_ring(dev); 5180 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5181 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5182 u32 cli = cp->ethdev->iscsi_l2_client_id; 5183 u32 cid = cp->ethdev->iscsi_l2_cid; 5184 u32 cl_qzone_id; 5185 struct client_init_ramrod_data *data; 5186 union l5cm_specific_data l5_data; 5187 struct ustorm_eth_rx_producers rx_prods = {0}; 5188 u32 off, i, *cid_ptr; 5189 5190 rx_prods.bd_prod = 0; 5191 rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT; 5192 barrier(); 5193 5194 cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5195 5196 off = BAR_USTRORM_INTMEM + 5197 (BNX2X_CHIP_IS_E2_PLUS(bp) ? 5198 USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) : 5199 USTORM_RX_PRODS_E1X_OFFSET(BP_PORT(bp), cli)); 5200 5201 for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++) 5202 CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]); 5203 5204 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5205 5206 data = udev->l2_buf; 5207 cid_ptr = udev->l2_buf + 12; 5208 5209 memset(data, 0, sizeof(*data)); 5210 5211 cnic_init_bnx2x_tx_ring(dev, data); 5212 cnic_init_bnx2x_rx_ring(dev, data); 5213 5214 data->general.fp_hsi_ver = ETH_FP_HSI_VERSION; 5215 5216 l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff; 5217 l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32; 5218 5219 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5220 5221 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP, 5222 cid, ETH_CONNECTION_TYPE, &l5_data); 5223 5224 i = 0; 5225 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5226 ++i < 10) 5227 msleep(1); 5228 5229 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5230 netdev_err(dev->netdev, 5231 "iSCSI CLIENT_SETUP did not complete\n"); 5232 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5233 cnic_ring_ctl(dev, cid, cli, 1); 5234 *cid_ptr = cid >> 4; 5235 *(cid_ptr + 1) = cid * bp->db_size; 5236 *(cid_ptr + 2) = UIO_USE_TX_DOORBELL; 5237 } 5238 } 5239 5240 static void cnic_shutdown_rings(struct cnic_dev *dev) 5241 { 5242 struct cnic_local *cp = dev->cnic_priv; 5243 struct cnic_uio_dev *udev = cp->udev; 5244 void *rx_ring; 5245 5246 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5247 return; 5248 5249 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5250 cnic_shutdown_bnx2_rx_ring(dev); 5251 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5252 u32 cli = cp->ethdev->iscsi_l2_client_id; 5253 u32 cid = cp->ethdev->iscsi_l2_cid; 5254 union l5cm_specific_data l5_data; 5255 int i; 5256 5257 cnic_ring_ctl(dev, cid, cli, 0); 5258 5259 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5260 5261 l5_data.phy_address.lo = cli; 5262 l5_data.phy_address.hi = 0; 5263 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT, 5264 cid, ETH_CONNECTION_TYPE, &l5_data); 5265 i = 0; 5266 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5267 ++i < 10) 5268 msleep(1); 5269 5270 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5271 netdev_err(dev->netdev, 5272 "iSCSI CLIENT_HALT did not complete\n"); 5273 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5274 5275 memset(&l5_data, 0, sizeof(l5_data)); 5276 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 5277 cid, NONE_CONNECTION_TYPE, &l5_data); 5278 msleep(10); 5279 } 5280 clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5281 rx_ring = udev->l2_ring + CNIC_PAGE_SIZE; 5282 memset(rx_ring, 0, CNIC_PAGE_SIZE); 5283 } 5284 5285 static int cnic_register_netdev(struct cnic_dev *dev) 5286 { 5287 struct cnic_local *cp = dev->cnic_priv; 5288 struct cnic_eth_dev *ethdev = cp->ethdev; 5289 int err; 5290 5291 if (!ethdev) 5292 return -ENODEV; 5293 5294 if (ethdev->drv_state & CNIC_DRV_STATE_REGD) 5295 return 0; 5296 5297 err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev); 5298 if (err) 5299 netdev_err(dev->netdev, "register_cnic failed\n"); 5300 5301 /* Read iSCSI config again. On some bnx2x device, iSCSI config 5302 * can change after firmware is downloaded. 5303 */ 5304 dev->max_iscsi_conn = ethdev->max_iscsi_conn; 5305 if (ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 5306 dev->max_iscsi_conn = 0; 5307 5308 return err; 5309 } 5310 5311 static void cnic_unregister_netdev(struct cnic_dev *dev) 5312 { 5313 struct cnic_local *cp = dev->cnic_priv; 5314 struct cnic_eth_dev *ethdev = cp->ethdev; 5315 5316 if (!ethdev) 5317 return; 5318 5319 ethdev->drv_unregister_cnic(dev->netdev); 5320 } 5321 5322 static int cnic_start_hw(struct cnic_dev *dev) 5323 { 5324 struct cnic_local *cp = dev->cnic_priv; 5325 struct cnic_eth_dev *ethdev = cp->ethdev; 5326 int err; 5327 5328 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5329 return -EALREADY; 5330 5331 dev->regview = ethdev->io_base; 5332 pci_dev_get(dev->pcidev); 5333 cp->func = PCI_FUNC(dev->pcidev->devfn); 5334 cp->status_blk.gen = ethdev->irq_arr[0].status_blk; 5335 cp->status_blk_map = ethdev->irq_arr[0].status_blk_map; 5336 cp->status_blk_num = ethdev->irq_arr[0].status_blk_num; 5337 5338 err = cp->alloc_resc(dev); 5339 if (err) { 5340 netdev_err(dev->netdev, "allocate resource failure\n"); 5341 goto err1; 5342 } 5343 5344 err = cp->start_hw(dev); 5345 if (err) 5346 goto err1; 5347 5348 err = cnic_cm_open(dev); 5349 if (err) 5350 goto err1; 5351 5352 set_bit(CNIC_F_CNIC_UP, &dev->flags); 5353 5354 cp->enable_int(dev); 5355 5356 return 0; 5357 5358 err1: 5359 if (ethdev->drv_state & CNIC_DRV_STATE_HANDLES_IRQ) 5360 cp->stop_hw(dev); 5361 else 5362 cp->free_resc(dev); 5363 pci_dev_put(dev->pcidev); 5364 return err; 5365 } 5366 5367 static void cnic_stop_bnx2_hw(struct cnic_dev *dev) 5368 { 5369 cnic_disable_bnx2_int_sync(dev); 5370 5371 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 5372 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 5373 5374 cnic_init_context(dev, KWQ_CID); 5375 cnic_init_context(dev, KCQ_CID); 5376 5377 cnic_setup_5709_context(dev, 0); 5378 cnic_free_irq(dev); 5379 5380 cnic_free_resc(dev); 5381 } 5382 5383 5384 static void cnic_stop_bnx2x_hw(struct cnic_dev *dev) 5385 { 5386 struct cnic_local *cp = dev->cnic_priv; 5387 struct bnx2x *bp = netdev_priv(dev->netdev); 5388 u32 hc_index = HC_INDEX_ISCSI_EQ_CONS; 5389 u32 sb_id = cp->status_blk_num; 5390 u32 idx_off, syn_off; 5391 5392 cnic_free_irq(dev); 5393 5394 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5395 idx_off = offsetof(struct hc_status_block_e2, index_values) + 5396 (hc_index * sizeof(u16)); 5397 5398 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id); 5399 } else { 5400 idx_off = offsetof(struct hc_status_block_e1x, index_values) + 5401 (hc_index * sizeof(u16)); 5402 5403 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id); 5404 } 5405 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0); 5406 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) + 5407 idx_off, 0); 5408 5409 *cp->kcq1.hw_prod_idx_ptr = 0; 5410 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5411 CSTORM_ISCSI_EQ_CONS_OFFSET(bp->pfid, 0), 0); 5412 CNIC_WR16(dev, cp->kcq1.io_addr, 0); 5413 cnic_free_resc(dev); 5414 } 5415 5416 static void cnic_stop_hw(struct cnic_dev *dev) 5417 { 5418 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5419 struct cnic_local *cp = dev->cnic_priv; 5420 int i = 0; 5421 5422 /* Need to wait for the ring shutdown event to complete 5423 * before clearing the CNIC_UP flag. 5424 */ 5425 while (cp->udev && cp->udev->uio_dev != -1 && i < 15) { 5426 msleep(100); 5427 i++; 5428 } 5429 cnic_shutdown_rings(dev); 5430 cp->stop_cm(dev); 5431 cp->ethdev->drv_state &= ~CNIC_DRV_STATE_HANDLES_IRQ; 5432 clear_bit(CNIC_F_CNIC_UP, &dev->flags); 5433 RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL); 5434 synchronize_rcu(); 5435 cnic_cm_shutdown(dev); 5436 cp->stop_hw(dev); 5437 pci_dev_put(dev->pcidev); 5438 } 5439 } 5440 5441 static void cnic_free_dev(struct cnic_dev *dev) 5442 { 5443 int i = 0; 5444 5445 while ((atomic_read(&dev->ref_count) != 0) && i < 10) { 5446 msleep(100); 5447 i++; 5448 } 5449 if (atomic_read(&dev->ref_count) != 0) 5450 netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n"); 5451 5452 netdev_info(dev->netdev, "Removed CNIC device\n"); 5453 dev_put(dev->netdev); 5454 kfree(dev); 5455 } 5456 5457 static int cnic_get_fc_npiv_tbl(struct cnic_dev *dev, 5458 struct cnic_fc_npiv_tbl *npiv_tbl) 5459 { 5460 struct cnic_local *cp = dev->cnic_priv; 5461 struct bnx2x *bp = netdev_priv(dev->netdev); 5462 int ret; 5463 5464 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5465 return -EAGAIN; /* bnx2x is down */ 5466 5467 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 5468 return -EINVAL; 5469 5470 ret = cp->ethdev->drv_get_fc_npiv_tbl(dev->netdev, npiv_tbl); 5471 return ret; 5472 } 5473 5474 static struct cnic_dev *cnic_alloc_dev(struct net_device *dev, 5475 struct pci_dev *pdev) 5476 { 5477 struct cnic_dev *cdev; 5478 struct cnic_local *cp; 5479 int alloc_size; 5480 5481 alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local); 5482 5483 cdev = kzalloc(alloc_size, GFP_KERNEL); 5484 if (cdev == NULL) 5485 return NULL; 5486 5487 cdev->netdev = dev; 5488 cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev); 5489 cdev->register_device = cnic_register_device; 5490 cdev->unregister_device = cnic_unregister_device; 5491 cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv; 5492 cdev->get_fc_npiv_tbl = cnic_get_fc_npiv_tbl; 5493 atomic_set(&cdev->ref_count, 0); 5494 5495 cp = cdev->cnic_priv; 5496 cp->dev = cdev; 5497 cp->l2_single_buf_size = 0x400; 5498 cp->l2_rx_ring_size = 3; 5499 5500 spin_lock_init(&cp->cnic_ulp_lock); 5501 5502 netdev_info(dev, "Added CNIC device\n"); 5503 5504 return cdev; 5505 } 5506 5507 static struct cnic_dev *init_bnx2_cnic(struct net_device *dev) 5508 { 5509 struct pci_dev *pdev; 5510 struct cnic_dev *cdev; 5511 struct cnic_local *cp; 5512 struct bnx2 *bp = netdev_priv(dev); 5513 struct cnic_eth_dev *ethdev = NULL; 5514 5515 if (bp->cnic_probe) 5516 ethdev = (bp->cnic_probe)(dev); 5517 5518 if (!ethdev) 5519 return NULL; 5520 5521 pdev = ethdev->pdev; 5522 if (!pdev) 5523 return NULL; 5524 5525 dev_hold(dev); 5526 pci_dev_get(pdev); 5527 if ((pdev->device == PCI_DEVICE_ID_NX2_5709 || 5528 pdev->device == PCI_DEVICE_ID_NX2_5709S) && 5529 (pdev->revision < 0x10)) { 5530 pci_dev_put(pdev); 5531 goto cnic_err; 5532 } 5533 pci_dev_put(pdev); 5534 5535 cdev = cnic_alloc_dev(dev, pdev); 5536 if (cdev == NULL) 5537 goto cnic_err; 5538 5539 set_bit(CNIC_F_BNX2_CLASS, &cdev->flags); 5540 cdev->submit_kwqes = cnic_submit_bnx2_kwqes; 5541 5542 cp = cdev->cnic_priv; 5543 cp->ethdev = ethdev; 5544 cdev->pcidev = pdev; 5545 cp->chip_id = ethdev->chip_id; 5546 5547 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5548 5549 cp->cnic_ops = &cnic_bnx2_ops; 5550 cp->start_hw = cnic_start_bnx2_hw; 5551 cp->stop_hw = cnic_stop_bnx2_hw; 5552 cp->setup_pgtbl = cnic_setup_page_tbl; 5553 cp->alloc_resc = cnic_alloc_bnx2_resc; 5554 cp->free_resc = cnic_free_resc; 5555 cp->start_cm = cnic_cm_init_bnx2_hw; 5556 cp->stop_cm = cnic_cm_stop_bnx2_hw; 5557 cp->enable_int = cnic_enable_bnx2_int; 5558 cp->disable_int_sync = cnic_disable_bnx2_int_sync; 5559 cp->close_conn = cnic_close_bnx2_conn; 5560 return cdev; 5561 5562 cnic_err: 5563 dev_put(dev); 5564 return NULL; 5565 } 5566 5567 static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev) 5568 { 5569 struct pci_dev *pdev; 5570 struct cnic_dev *cdev; 5571 struct cnic_local *cp; 5572 struct bnx2x *bp = netdev_priv(dev); 5573 struct cnic_eth_dev *ethdev = NULL; 5574 5575 if (bp->cnic_probe) 5576 ethdev = bp->cnic_probe(dev); 5577 5578 if (!ethdev) 5579 return NULL; 5580 5581 pdev = ethdev->pdev; 5582 if (!pdev) 5583 return NULL; 5584 5585 dev_hold(dev); 5586 cdev = cnic_alloc_dev(dev, pdev); 5587 if (cdev == NULL) { 5588 dev_put(dev); 5589 return NULL; 5590 } 5591 5592 set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags); 5593 cdev->submit_kwqes = cnic_submit_bnx2x_kwqes; 5594 5595 cp = cdev->cnic_priv; 5596 cp->ethdev = ethdev; 5597 cdev->pcidev = pdev; 5598 cp->chip_id = ethdev->chip_id; 5599 5600 cdev->stats_addr = ethdev->addr_drv_info_to_mcp; 5601 5602 if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)) 5603 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5604 if (CNIC_SUPPORTS_FCOE(bp)) { 5605 cdev->max_fcoe_conn = ethdev->max_fcoe_conn; 5606 cdev->max_fcoe_exchanges = ethdev->max_fcoe_exchanges; 5607 } 5608 5609 if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS) 5610 cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS; 5611 5612 memcpy(cdev->mac_addr, ethdev->iscsi_mac, ETH_ALEN); 5613 5614 cp->cnic_ops = &cnic_bnx2x_ops; 5615 cp->start_hw = cnic_start_bnx2x_hw; 5616 cp->stop_hw = cnic_stop_bnx2x_hw; 5617 cp->setup_pgtbl = cnic_setup_page_tbl_le; 5618 cp->alloc_resc = cnic_alloc_bnx2x_resc; 5619 cp->free_resc = cnic_free_resc; 5620 cp->start_cm = cnic_cm_init_bnx2x_hw; 5621 cp->stop_cm = cnic_cm_stop_bnx2x_hw; 5622 cp->enable_int = cnic_enable_bnx2x_int; 5623 cp->disable_int_sync = cnic_disable_bnx2x_int_sync; 5624 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5625 cp->ack_int = cnic_ack_bnx2x_e2_msix; 5626 cp->arm_int = cnic_arm_bnx2x_e2_msix; 5627 } else { 5628 cp->ack_int = cnic_ack_bnx2x_msix; 5629 cp->arm_int = cnic_arm_bnx2x_msix; 5630 } 5631 cp->close_conn = cnic_close_bnx2x_conn; 5632 return cdev; 5633 } 5634 5635 static struct cnic_dev *is_cnic_dev(struct net_device *dev) 5636 { 5637 struct ethtool_drvinfo drvinfo; 5638 struct cnic_dev *cdev = NULL; 5639 5640 if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) { 5641 memset(&drvinfo, 0, sizeof(drvinfo)); 5642 dev->ethtool_ops->get_drvinfo(dev, &drvinfo); 5643 5644 if (!strcmp(drvinfo.driver, "bnx2")) 5645 cdev = init_bnx2_cnic(dev); 5646 if (!strcmp(drvinfo.driver, "bnx2x")) 5647 cdev = init_bnx2x_cnic(dev); 5648 if (cdev) { 5649 write_lock(&cnic_dev_lock); 5650 list_add(&cdev->list, &cnic_dev_list); 5651 write_unlock(&cnic_dev_lock); 5652 } 5653 } 5654 return cdev; 5655 } 5656 5657 static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event, 5658 u16 vlan_id) 5659 { 5660 int if_type; 5661 5662 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 5663 struct cnic_ulp_ops *ulp_ops; 5664 void *ctx; 5665 5666 mutex_lock(&cnic_lock); 5667 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 5668 lockdep_is_held(&cnic_lock)); 5669 if (!ulp_ops || !ulp_ops->indicate_netevent) { 5670 mutex_unlock(&cnic_lock); 5671 continue; 5672 } 5673 5674 ctx = cp->ulp_handle[if_type]; 5675 5676 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5677 mutex_unlock(&cnic_lock); 5678 5679 ulp_ops->indicate_netevent(ctx, event, vlan_id); 5680 5681 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 5682 } 5683 } 5684 5685 /* netdev event handler */ 5686 static int cnic_netdev_event(struct notifier_block *this, unsigned long event, 5687 void *ptr) 5688 { 5689 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 5690 struct cnic_dev *dev; 5691 int new_dev = 0; 5692 5693 dev = cnic_from_netdev(netdev); 5694 5695 if (!dev && event == NETDEV_REGISTER) { 5696 /* Check for the hot-plug device */ 5697 dev = is_cnic_dev(netdev); 5698 if (dev) { 5699 new_dev = 1; 5700 cnic_hold(dev); 5701 } 5702 } 5703 if (dev) { 5704 struct cnic_local *cp = dev->cnic_priv; 5705 5706 if (new_dev) 5707 cnic_ulp_init(dev); 5708 else if (event == NETDEV_UNREGISTER) 5709 cnic_ulp_exit(dev); 5710 5711 if (event == NETDEV_UP) { 5712 if (cnic_register_netdev(dev) != 0) { 5713 cnic_put(dev); 5714 goto done; 5715 } 5716 if (!cnic_start_hw(dev)) 5717 cnic_ulp_start(dev); 5718 } 5719 5720 cnic_rcv_netevent(cp, event, 0); 5721 5722 if (event == NETDEV_GOING_DOWN) { 5723 cnic_ulp_stop(dev); 5724 cnic_stop_hw(dev); 5725 cnic_unregister_netdev(dev); 5726 } else if (event == NETDEV_UNREGISTER) { 5727 write_lock(&cnic_dev_lock); 5728 list_del_init(&dev->list); 5729 write_unlock(&cnic_dev_lock); 5730 5731 cnic_put(dev); 5732 cnic_free_dev(dev); 5733 goto done; 5734 } 5735 cnic_put(dev); 5736 } else { 5737 struct net_device *realdev; 5738 u16 vid; 5739 5740 vid = cnic_get_vlan(netdev, &realdev); 5741 if (realdev) { 5742 dev = cnic_from_netdev(realdev); 5743 if (dev) { 5744 vid |= VLAN_CFI_MASK; /* make non-zero */ 5745 cnic_rcv_netevent(dev->cnic_priv, event, vid); 5746 cnic_put(dev); 5747 } 5748 } 5749 } 5750 done: 5751 return NOTIFY_DONE; 5752 } 5753 5754 static struct notifier_block cnic_netdev_notifier = { 5755 .notifier_call = cnic_netdev_event 5756 }; 5757 5758 static void cnic_release(void) 5759 { 5760 struct cnic_uio_dev *udev; 5761 5762 while (!list_empty(&cnic_udev_list)) { 5763 udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev, 5764 list); 5765 cnic_free_uio(udev); 5766 } 5767 } 5768 5769 static int __init cnic_init(void) 5770 { 5771 int rc = 0; 5772 5773 pr_info("%s", version); 5774 5775 rc = register_netdevice_notifier(&cnic_netdev_notifier); 5776 if (rc) { 5777 cnic_release(); 5778 return rc; 5779 } 5780 5781 cnic_wq = create_singlethread_workqueue("cnic_wq"); 5782 if (!cnic_wq) { 5783 cnic_release(); 5784 unregister_netdevice_notifier(&cnic_netdev_notifier); 5785 return -ENOMEM; 5786 } 5787 5788 return 0; 5789 } 5790 5791 static void __exit cnic_exit(void) 5792 { 5793 unregister_netdevice_notifier(&cnic_netdev_notifier); 5794 cnic_release(); 5795 destroy_workqueue(cnic_wq); 5796 } 5797 5798 module_init(cnic_init); 5799 module_exit(cnic_exit); 5800