1 /* cnic.c: Broadcom CNIC core network driver. 2 * 3 * Copyright (c) 2006-2013 Broadcom Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 * 9 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com) 10 * Modified and maintained by: Michael Chan <mchan@broadcom.com> 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/module.h> 16 17 #include <linux/kernel.h> 18 #include <linux/errno.h> 19 #include <linux/list.h> 20 #include <linux/slab.h> 21 #include <linux/pci.h> 22 #include <linux/init.h> 23 #include <linux/netdevice.h> 24 #include <linux/uio_driver.h> 25 #include <linux/in.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/delay.h> 28 #include <linux/ethtool.h> 29 #include <linux/if_vlan.h> 30 #include <linux/prefetch.h> 31 #include <linux/random.h> 32 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE) 33 #define BCM_VLAN 1 34 #endif 35 #include <net/ip.h> 36 #include <net/tcp.h> 37 #include <net/route.h> 38 #include <net/ipv6.h> 39 #include <net/ip6_route.h> 40 #include <net/ip6_checksum.h> 41 #include <scsi/iscsi_if.h> 42 43 #define BCM_CNIC 1 44 #include "cnic_if.h" 45 #include "bnx2.h" 46 #include "bnx2x/bnx2x.h" 47 #include "bnx2x/bnx2x_reg.h" 48 #include "bnx2x/bnx2x_fw_defs.h" 49 #include "bnx2x/bnx2x_hsi.h" 50 #include "../../../scsi/bnx2i/57xx_iscsi_constants.h" 51 #include "../../../scsi/bnx2i/57xx_iscsi_hsi.h" 52 #include "../../../scsi/bnx2fc/bnx2fc_constants.h" 53 #include "cnic.h" 54 #include "cnic_defs.h" 55 56 #define CNIC_MODULE_NAME "cnic" 57 58 static char version[] = 59 "Broadcom NetXtreme II CNIC Driver " CNIC_MODULE_NAME " v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n"; 60 61 MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) " 62 "Chen (zongxi@broadcom.com"); 63 MODULE_DESCRIPTION("Broadcom NetXtreme II CNIC Driver"); 64 MODULE_LICENSE("GPL"); 65 MODULE_VERSION(CNIC_MODULE_VERSION); 66 67 /* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */ 68 static LIST_HEAD(cnic_dev_list); 69 static LIST_HEAD(cnic_udev_list); 70 static DEFINE_RWLOCK(cnic_dev_lock); 71 static DEFINE_MUTEX(cnic_lock); 72 73 static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE]; 74 75 /* helper function, assuming cnic_lock is held */ 76 static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type) 77 { 78 return rcu_dereference_protected(cnic_ulp_tbl[type], 79 lockdep_is_held(&cnic_lock)); 80 } 81 82 static int cnic_service_bnx2(void *, void *); 83 static int cnic_service_bnx2x(void *, void *); 84 static int cnic_ctl(void *, struct cnic_ctl_info *); 85 86 static struct cnic_ops cnic_bnx2_ops = { 87 .cnic_owner = THIS_MODULE, 88 .cnic_handler = cnic_service_bnx2, 89 .cnic_ctl = cnic_ctl, 90 }; 91 92 static struct cnic_ops cnic_bnx2x_ops = { 93 .cnic_owner = THIS_MODULE, 94 .cnic_handler = cnic_service_bnx2x, 95 .cnic_ctl = cnic_ctl, 96 }; 97 98 static struct workqueue_struct *cnic_wq; 99 100 static void cnic_shutdown_rings(struct cnic_dev *); 101 static void cnic_init_rings(struct cnic_dev *); 102 static int cnic_cm_set_pg(struct cnic_sock *); 103 104 static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode) 105 { 106 struct cnic_uio_dev *udev = uinfo->priv; 107 struct cnic_dev *dev; 108 109 if (!capable(CAP_NET_ADMIN)) 110 return -EPERM; 111 112 if (udev->uio_dev != -1) 113 return -EBUSY; 114 115 rtnl_lock(); 116 dev = udev->dev; 117 118 if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 119 rtnl_unlock(); 120 return -ENODEV; 121 } 122 123 udev->uio_dev = iminor(inode); 124 125 cnic_shutdown_rings(dev); 126 cnic_init_rings(dev); 127 rtnl_unlock(); 128 129 return 0; 130 } 131 132 static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode) 133 { 134 struct cnic_uio_dev *udev = uinfo->priv; 135 136 udev->uio_dev = -1; 137 return 0; 138 } 139 140 static inline void cnic_hold(struct cnic_dev *dev) 141 { 142 atomic_inc(&dev->ref_count); 143 } 144 145 static inline void cnic_put(struct cnic_dev *dev) 146 { 147 atomic_dec(&dev->ref_count); 148 } 149 150 static inline void csk_hold(struct cnic_sock *csk) 151 { 152 atomic_inc(&csk->ref_count); 153 } 154 155 static inline void csk_put(struct cnic_sock *csk) 156 { 157 atomic_dec(&csk->ref_count); 158 } 159 160 static struct cnic_dev *cnic_from_netdev(struct net_device *netdev) 161 { 162 struct cnic_dev *cdev; 163 164 read_lock(&cnic_dev_lock); 165 list_for_each_entry(cdev, &cnic_dev_list, list) { 166 if (netdev == cdev->netdev) { 167 cnic_hold(cdev); 168 read_unlock(&cnic_dev_lock); 169 return cdev; 170 } 171 } 172 read_unlock(&cnic_dev_lock); 173 return NULL; 174 } 175 176 static inline void ulp_get(struct cnic_ulp_ops *ulp_ops) 177 { 178 atomic_inc(&ulp_ops->ref_count); 179 } 180 181 static inline void ulp_put(struct cnic_ulp_ops *ulp_ops) 182 { 183 atomic_dec(&ulp_ops->ref_count); 184 } 185 186 static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val) 187 { 188 struct cnic_local *cp = dev->cnic_priv; 189 struct cnic_eth_dev *ethdev = cp->ethdev; 190 struct drv_ctl_info info; 191 struct drv_ctl_io *io = &info.data.io; 192 193 info.cmd = DRV_CTL_CTX_WR_CMD; 194 io->cid_addr = cid_addr; 195 io->offset = off; 196 io->data = val; 197 ethdev->drv_ctl(dev->netdev, &info); 198 } 199 200 static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr) 201 { 202 struct cnic_local *cp = dev->cnic_priv; 203 struct cnic_eth_dev *ethdev = cp->ethdev; 204 struct drv_ctl_info info; 205 struct drv_ctl_io *io = &info.data.io; 206 207 info.cmd = DRV_CTL_CTXTBL_WR_CMD; 208 io->offset = off; 209 io->dma_addr = addr; 210 ethdev->drv_ctl(dev->netdev, &info); 211 } 212 213 static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start) 214 { 215 struct cnic_local *cp = dev->cnic_priv; 216 struct cnic_eth_dev *ethdev = cp->ethdev; 217 struct drv_ctl_info info; 218 struct drv_ctl_l2_ring *ring = &info.data.ring; 219 220 if (start) 221 info.cmd = DRV_CTL_START_L2_CMD; 222 else 223 info.cmd = DRV_CTL_STOP_L2_CMD; 224 225 ring->cid = cid; 226 ring->client_id = cl_id; 227 ethdev->drv_ctl(dev->netdev, &info); 228 } 229 230 static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val) 231 { 232 struct cnic_local *cp = dev->cnic_priv; 233 struct cnic_eth_dev *ethdev = cp->ethdev; 234 struct drv_ctl_info info; 235 struct drv_ctl_io *io = &info.data.io; 236 237 info.cmd = DRV_CTL_IO_WR_CMD; 238 io->offset = off; 239 io->data = val; 240 ethdev->drv_ctl(dev->netdev, &info); 241 } 242 243 static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off) 244 { 245 struct cnic_local *cp = dev->cnic_priv; 246 struct cnic_eth_dev *ethdev = cp->ethdev; 247 struct drv_ctl_info info; 248 struct drv_ctl_io *io = &info.data.io; 249 250 info.cmd = DRV_CTL_IO_RD_CMD; 251 io->offset = off; 252 ethdev->drv_ctl(dev->netdev, &info); 253 return io->data; 254 } 255 256 static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg) 257 { 258 struct cnic_local *cp = dev->cnic_priv; 259 struct cnic_eth_dev *ethdev = cp->ethdev; 260 struct drv_ctl_info info; 261 struct fcoe_capabilities *fcoe_cap = 262 &info.data.register_data.fcoe_features; 263 264 if (reg) { 265 info.cmd = DRV_CTL_ULP_REGISTER_CMD; 266 if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap) 267 memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap)); 268 } else { 269 info.cmd = DRV_CTL_ULP_UNREGISTER_CMD; 270 } 271 272 info.data.ulp_type = ulp_type; 273 ethdev->drv_ctl(dev->netdev, &info); 274 } 275 276 static int cnic_in_use(struct cnic_sock *csk) 277 { 278 return test_bit(SK_F_INUSE, &csk->flags); 279 } 280 281 static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count) 282 { 283 struct cnic_local *cp = dev->cnic_priv; 284 struct cnic_eth_dev *ethdev = cp->ethdev; 285 struct drv_ctl_info info; 286 287 info.cmd = cmd; 288 info.data.credit.credit_count = count; 289 ethdev->drv_ctl(dev->netdev, &info); 290 } 291 292 static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid) 293 { 294 u32 i; 295 296 if (!cp->ctx_tbl) 297 return -EINVAL; 298 299 for (i = 0; i < cp->max_cid_space; i++) { 300 if (cp->ctx_tbl[i].cid == cid) { 301 *l5_cid = i; 302 return 0; 303 } 304 } 305 return -EINVAL; 306 } 307 308 static int cnic_send_nlmsg(struct cnic_local *cp, u32 type, 309 struct cnic_sock *csk) 310 { 311 struct iscsi_path path_req; 312 char *buf = NULL; 313 u16 len = 0; 314 u32 msg_type = ISCSI_KEVENT_IF_DOWN; 315 struct cnic_ulp_ops *ulp_ops; 316 struct cnic_uio_dev *udev = cp->udev; 317 int rc = 0, retry = 0; 318 319 if (!udev || udev->uio_dev == -1) 320 return -ENODEV; 321 322 if (csk) { 323 len = sizeof(path_req); 324 buf = (char *) &path_req; 325 memset(&path_req, 0, len); 326 327 msg_type = ISCSI_KEVENT_PATH_REQ; 328 path_req.handle = (u64) csk->l5_cid; 329 if (test_bit(SK_F_IPV6, &csk->flags)) { 330 memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0], 331 sizeof(struct in6_addr)); 332 path_req.ip_addr_len = 16; 333 } else { 334 memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0], 335 sizeof(struct in_addr)); 336 path_req.ip_addr_len = 4; 337 } 338 path_req.vlan_id = csk->vlan_id; 339 path_req.pmtu = csk->mtu; 340 } 341 342 while (retry < 3) { 343 rc = 0; 344 rcu_read_lock(); 345 ulp_ops = rcu_dereference(cnic_ulp_tbl[CNIC_ULP_ISCSI]); 346 if (ulp_ops) 347 rc = ulp_ops->iscsi_nl_send_msg( 348 cp->ulp_handle[CNIC_ULP_ISCSI], 349 msg_type, buf, len); 350 rcu_read_unlock(); 351 if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ) 352 break; 353 354 msleep(100); 355 retry++; 356 } 357 return rc; 358 } 359 360 static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8); 361 362 static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type, 363 char *buf, u16 len) 364 { 365 int rc = -EINVAL; 366 367 switch (msg_type) { 368 case ISCSI_UEVENT_PATH_UPDATE: { 369 struct cnic_local *cp; 370 u32 l5_cid; 371 struct cnic_sock *csk; 372 struct iscsi_path *path_resp; 373 374 if (len < sizeof(*path_resp)) 375 break; 376 377 path_resp = (struct iscsi_path *) buf; 378 cp = dev->cnic_priv; 379 l5_cid = (u32) path_resp->handle; 380 if (l5_cid >= MAX_CM_SK_TBL_SZ) 381 break; 382 383 rcu_read_lock(); 384 if (!rcu_dereference(cp->ulp_ops[CNIC_ULP_L4])) { 385 rc = -ENODEV; 386 rcu_read_unlock(); 387 break; 388 } 389 csk = &cp->csk_tbl[l5_cid]; 390 csk_hold(csk); 391 if (cnic_in_use(csk) && 392 test_bit(SK_F_CONNECT_START, &csk->flags)) { 393 394 csk->vlan_id = path_resp->vlan_id; 395 396 memcpy(csk->ha, path_resp->mac_addr, ETH_ALEN); 397 if (test_bit(SK_F_IPV6, &csk->flags)) 398 memcpy(&csk->src_ip[0], &path_resp->src.v6_addr, 399 sizeof(struct in6_addr)); 400 else 401 memcpy(&csk->src_ip[0], &path_resp->src.v4_addr, 402 sizeof(struct in_addr)); 403 404 if (is_valid_ether_addr(csk->ha)) { 405 cnic_cm_set_pg(csk); 406 } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) && 407 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 408 409 cnic_cm_upcall(cp, csk, 410 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 411 clear_bit(SK_F_CONNECT_START, &csk->flags); 412 } 413 } 414 csk_put(csk); 415 rcu_read_unlock(); 416 rc = 0; 417 } 418 } 419 420 return rc; 421 } 422 423 static int cnic_offld_prep(struct cnic_sock *csk) 424 { 425 if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 426 return 0; 427 428 if (!test_bit(SK_F_CONNECT_START, &csk->flags)) { 429 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 430 return 0; 431 } 432 433 return 1; 434 } 435 436 static int cnic_close_prep(struct cnic_sock *csk) 437 { 438 clear_bit(SK_F_CONNECT_START, &csk->flags); 439 smp_mb__after_clear_bit(); 440 441 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 442 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 443 msleep(1); 444 445 return 1; 446 } 447 return 0; 448 } 449 450 static int cnic_abort_prep(struct cnic_sock *csk) 451 { 452 clear_bit(SK_F_CONNECT_START, &csk->flags); 453 smp_mb__after_clear_bit(); 454 455 while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags)) 456 msleep(1); 457 458 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 459 csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP; 460 return 1; 461 } 462 463 return 0; 464 } 465 466 int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops) 467 { 468 struct cnic_dev *dev; 469 470 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 471 pr_err("%s: Bad type %d\n", __func__, ulp_type); 472 return -EINVAL; 473 } 474 mutex_lock(&cnic_lock); 475 if (cnic_ulp_tbl_prot(ulp_type)) { 476 pr_err("%s: Type %d has already been registered\n", 477 __func__, ulp_type); 478 mutex_unlock(&cnic_lock); 479 return -EBUSY; 480 } 481 482 read_lock(&cnic_dev_lock); 483 list_for_each_entry(dev, &cnic_dev_list, list) { 484 struct cnic_local *cp = dev->cnic_priv; 485 486 clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]); 487 } 488 read_unlock(&cnic_dev_lock); 489 490 atomic_set(&ulp_ops->ref_count, 0); 491 rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops); 492 mutex_unlock(&cnic_lock); 493 494 /* Prevent race conditions with netdev_event */ 495 rtnl_lock(); 496 list_for_each_entry(dev, &cnic_dev_list, list) { 497 struct cnic_local *cp = dev->cnic_priv; 498 499 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type])) 500 ulp_ops->cnic_init(dev); 501 } 502 rtnl_unlock(); 503 504 return 0; 505 } 506 507 int cnic_unregister_driver(int ulp_type) 508 { 509 struct cnic_dev *dev; 510 struct cnic_ulp_ops *ulp_ops; 511 int i = 0; 512 513 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 514 pr_err("%s: Bad type %d\n", __func__, ulp_type); 515 return -EINVAL; 516 } 517 mutex_lock(&cnic_lock); 518 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 519 if (!ulp_ops) { 520 pr_err("%s: Type %d has not been registered\n", 521 __func__, ulp_type); 522 goto out_unlock; 523 } 524 read_lock(&cnic_dev_lock); 525 list_for_each_entry(dev, &cnic_dev_list, list) { 526 struct cnic_local *cp = dev->cnic_priv; 527 528 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 529 pr_err("%s: Type %d still has devices registered\n", 530 __func__, ulp_type); 531 read_unlock(&cnic_dev_lock); 532 goto out_unlock; 533 } 534 } 535 read_unlock(&cnic_dev_lock); 536 537 RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL); 538 539 mutex_unlock(&cnic_lock); 540 synchronize_rcu(); 541 while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) { 542 msleep(100); 543 i++; 544 } 545 546 if (atomic_read(&ulp_ops->ref_count) != 0) 547 pr_warn("%s: Failed waiting for ref count to go to zero\n", 548 __func__); 549 return 0; 550 551 out_unlock: 552 mutex_unlock(&cnic_lock); 553 return -EINVAL; 554 } 555 556 static int cnic_start_hw(struct cnic_dev *); 557 static void cnic_stop_hw(struct cnic_dev *); 558 559 static int cnic_register_device(struct cnic_dev *dev, int ulp_type, 560 void *ulp_ctx) 561 { 562 struct cnic_local *cp = dev->cnic_priv; 563 struct cnic_ulp_ops *ulp_ops; 564 565 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 566 pr_err("%s: Bad type %d\n", __func__, ulp_type); 567 return -EINVAL; 568 } 569 mutex_lock(&cnic_lock); 570 if (cnic_ulp_tbl_prot(ulp_type) == NULL) { 571 pr_err("%s: Driver with type %d has not been registered\n", 572 __func__, ulp_type); 573 mutex_unlock(&cnic_lock); 574 return -EAGAIN; 575 } 576 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 577 pr_err("%s: Type %d has already been registered to this device\n", 578 __func__, ulp_type); 579 mutex_unlock(&cnic_lock); 580 return -EBUSY; 581 } 582 583 clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]); 584 cp->ulp_handle[ulp_type] = ulp_ctx; 585 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 586 rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops); 587 cnic_hold(dev); 588 589 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 590 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type])) 591 ulp_ops->cnic_start(cp->ulp_handle[ulp_type]); 592 593 mutex_unlock(&cnic_lock); 594 595 cnic_ulp_ctl(dev, ulp_type, true); 596 597 return 0; 598 599 } 600 EXPORT_SYMBOL(cnic_register_driver); 601 602 static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type) 603 { 604 struct cnic_local *cp = dev->cnic_priv; 605 int i = 0; 606 607 if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) { 608 pr_err("%s: Bad type %d\n", __func__, ulp_type); 609 return -EINVAL; 610 } 611 mutex_lock(&cnic_lock); 612 if (rcu_dereference(cp->ulp_ops[ulp_type])) { 613 RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL); 614 cnic_put(dev); 615 } else { 616 pr_err("%s: device not registered to this ulp type %d\n", 617 __func__, ulp_type); 618 mutex_unlock(&cnic_lock); 619 return -EINVAL; 620 } 621 mutex_unlock(&cnic_lock); 622 623 if (ulp_type == CNIC_ULP_ISCSI) 624 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 625 else if (ulp_type == CNIC_ULP_FCOE) 626 dev->fcoe_cap = NULL; 627 628 synchronize_rcu(); 629 630 while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) && 631 i < 20) { 632 msleep(100); 633 i++; 634 } 635 if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type])) 636 netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n"); 637 638 cnic_ulp_ctl(dev, ulp_type, false); 639 640 return 0; 641 } 642 EXPORT_SYMBOL(cnic_unregister_driver); 643 644 static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id, 645 u32 next) 646 { 647 id_tbl->start = start_id; 648 id_tbl->max = size; 649 id_tbl->next = next; 650 spin_lock_init(&id_tbl->lock); 651 id_tbl->table = kzalloc(DIV_ROUND_UP(size, 32) * 4, GFP_KERNEL); 652 if (!id_tbl->table) 653 return -ENOMEM; 654 655 return 0; 656 } 657 658 static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl) 659 { 660 kfree(id_tbl->table); 661 id_tbl->table = NULL; 662 } 663 664 static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id) 665 { 666 int ret = -1; 667 668 id -= id_tbl->start; 669 if (id >= id_tbl->max) 670 return ret; 671 672 spin_lock(&id_tbl->lock); 673 if (!test_bit(id, id_tbl->table)) { 674 set_bit(id, id_tbl->table); 675 ret = 0; 676 } 677 spin_unlock(&id_tbl->lock); 678 return ret; 679 } 680 681 /* Returns -1 if not successful */ 682 static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl) 683 { 684 u32 id; 685 686 spin_lock(&id_tbl->lock); 687 id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next); 688 if (id >= id_tbl->max) { 689 id = -1; 690 if (id_tbl->next != 0) { 691 id = find_first_zero_bit(id_tbl->table, id_tbl->next); 692 if (id >= id_tbl->next) 693 id = -1; 694 } 695 } 696 697 if (id < id_tbl->max) { 698 set_bit(id, id_tbl->table); 699 id_tbl->next = (id + 1) & (id_tbl->max - 1); 700 id += id_tbl->start; 701 } 702 703 spin_unlock(&id_tbl->lock); 704 705 return id; 706 } 707 708 static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id) 709 { 710 if (id == -1) 711 return; 712 713 id -= id_tbl->start; 714 if (id >= id_tbl->max) 715 return; 716 717 clear_bit(id, id_tbl->table); 718 } 719 720 static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma) 721 { 722 int i; 723 724 if (!dma->pg_arr) 725 return; 726 727 for (i = 0; i < dma->num_pages; i++) { 728 if (dma->pg_arr[i]) { 729 dma_free_coherent(&dev->pcidev->dev, BNX2_PAGE_SIZE, 730 dma->pg_arr[i], dma->pg_map_arr[i]); 731 dma->pg_arr[i] = NULL; 732 } 733 } 734 if (dma->pgtbl) { 735 dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size, 736 dma->pgtbl, dma->pgtbl_map); 737 dma->pgtbl = NULL; 738 } 739 kfree(dma->pg_arr); 740 dma->pg_arr = NULL; 741 dma->num_pages = 0; 742 } 743 744 static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma) 745 { 746 int i; 747 __le32 *page_table = (__le32 *) dma->pgtbl; 748 749 for (i = 0; i < dma->num_pages; i++) { 750 /* Each entry needs to be in big endian format. */ 751 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 752 page_table++; 753 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 754 page_table++; 755 } 756 } 757 758 static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma) 759 { 760 int i; 761 __le32 *page_table = (__le32 *) dma->pgtbl; 762 763 for (i = 0; i < dma->num_pages; i++) { 764 /* Each entry needs to be in little endian format. */ 765 *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff); 766 page_table++; 767 *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32); 768 page_table++; 769 } 770 } 771 772 static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma, 773 int pages, int use_pg_tbl) 774 { 775 int i, size; 776 struct cnic_local *cp = dev->cnic_priv; 777 778 size = pages * (sizeof(void *) + sizeof(dma_addr_t)); 779 dma->pg_arr = kzalloc(size, GFP_ATOMIC); 780 if (dma->pg_arr == NULL) 781 return -ENOMEM; 782 783 dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages); 784 dma->num_pages = pages; 785 786 for (i = 0; i < pages; i++) { 787 dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev, 788 BNX2_PAGE_SIZE, 789 &dma->pg_map_arr[i], 790 GFP_ATOMIC); 791 if (dma->pg_arr[i] == NULL) 792 goto error; 793 } 794 if (!use_pg_tbl) 795 return 0; 796 797 dma->pgtbl_size = ((pages * 8) + BNX2_PAGE_SIZE - 1) & 798 ~(BNX2_PAGE_SIZE - 1); 799 dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size, 800 &dma->pgtbl_map, GFP_ATOMIC); 801 if (dma->pgtbl == NULL) 802 goto error; 803 804 cp->setup_pgtbl(dev, dma); 805 806 return 0; 807 808 error: 809 cnic_free_dma(dev, dma); 810 return -ENOMEM; 811 } 812 813 static void cnic_free_context(struct cnic_dev *dev) 814 { 815 struct cnic_local *cp = dev->cnic_priv; 816 int i; 817 818 for (i = 0; i < cp->ctx_blks; i++) { 819 if (cp->ctx_arr[i].ctx) { 820 dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 821 cp->ctx_arr[i].ctx, 822 cp->ctx_arr[i].mapping); 823 cp->ctx_arr[i].ctx = NULL; 824 } 825 } 826 } 827 828 static void __cnic_free_uio_rings(struct cnic_uio_dev *udev) 829 { 830 if (udev->l2_buf) { 831 dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size, 832 udev->l2_buf, udev->l2_buf_map); 833 udev->l2_buf = NULL; 834 } 835 836 if (udev->l2_ring) { 837 dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size, 838 udev->l2_ring, udev->l2_ring_map); 839 udev->l2_ring = NULL; 840 } 841 842 } 843 844 static void __cnic_free_uio(struct cnic_uio_dev *udev) 845 { 846 uio_unregister_device(&udev->cnic_uinfo); 847 848 __cnic_free_uio_rings(udev); 849 850 pci_dev_put(udev->pdev); 851 kfree(udev); 852 } 853 854 static void cnic_free_uio(struct cnic_uio_dev *udev) 855 { 856 if (!udev) 857 return; 858 859 write_lock(&cnic_dev_lock); 860 list_del_init(&udev->list); 861 write_unlock(&cnic_dev_lock); 862 __cnic_free_uio(udev); 863 } 864 865 static void cnic_free_resc(struct cnic_dev *dev) 866 { 867 struct cnic_local *cp = dev->cnic_priv; 868 struct cnic_uio_dev *udev = cp->udev; 869 870 if (udev) { 871 udev->dev = NULL; 872 cp->udev = NULL; 873 if (udev->uio_dev == -1) 874 __cnic_free_uio_rings(udev); 875 } 876 877 cnic_free_context(dev); 878 kfree(cp->ctx_arr); 879 cp->ctx_arr = NULL; 880 cp->ctx_blks = 0; 881 882 cnic_free_dma(dev, &cp->gbl_buf_info); 883 cnic_free_dma(dev, &cp->kwq_info); 884 cnic_free_dma(dev, &cp->kwq_16_data_info); 885 cnic_free_dma(dev, &cp->kcq2.dma); 886 cnic_free_dma(dev, &cp->kcq1.dma); 887 kfree(cp->iscsi_tbl); 888 cp->iscsi_tbl = NULL; 889 kfree(cp->ctx_tbl); 890 cp->ctx_tbl = NULL; 891 892 cnic_free_id_tbl(&cp->fcoe_cid_tbl); 893 cnic_free_id_tbl(&cp->cid_tbl); 894 } 895 896 static int cnic_alloc_context(struct cnic_dev *dev) 897 { 898 struct cnic_local *cp = dev->cnic_priv; 899 900 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 901 int i, k, arr_size; 902 903 cp->ctx_blk_size = BNX2_PAGE_SIZE; 904 cp->cids_per_blk = BNX2_PAGE_SIZE / 128; 905 arr_size = BNX2_MAX_CID / cp->cids_per_blk * 906 sizeof(struct cnic_ctx); 907 cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL); 908 if (cp->ctx_arr == NULL) 909 return -ENOMEM; 910 911 k = 0; 912 for (i = 0; i < 2; i++) { 913 u32 j, reg, off, lo, hi; 914 915 if (i == 0) 916 off = BNX2_PG_CTX_MAP; 917 else 918 off = BNX2_ISCSI_CTX_MAP; 919 920 reg = cnic_reg_rd_ind(dev, off); 921 lo = reg >> 16; 922 hi = reg & 0xffff; 923 for (j = lo; j < hi; j += cp->cids_per_blk, k++) 924 cp->ctx_arr[k].cid = j; 925 } 926 927 cp->ctx_blks = k; 928 if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) { 929 cp->ctx_blks = 0; 930 return -ENOMEM; 931 } 932 933 for (i = 0; i < cp->ctx_blks; i++) { 934 cp->ctx_arr[i].ctx = 935 dma_alloc_coherent(&dev->pcidev->dev, 936 BNX2_PAGE_SIZE, 937 &cp->ctx_arr[i].mapping, 938 GFP_KERNEL); 939 if (cp->ctx_arr[i].ctx == NULL) 940 return -ENOMEM; 941 } 942 } 943 return 0; 944 } 945 946 static u16 cnic_bnx2_next_idx(u16 idx) 947 { 948 return idx + 1; 949 } 950 951 static u16 cnic_bnx2_hw_idx(u16 idx) 952 { 953 return idx; 954 } 955 956 static u16 cnic_bnx2x_next_idx(u16 idx) 957 { 958 idx++; 959 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 960 idx++; 961 962 return idx; 963 } 964 965 static u16 cnic_bnx2x_hw_idx(u16 idx) 966 { 967 if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT) 968 idx++; 969 return idx; 970 } 971 972 static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info, 973 bool use_pg_tbl) 974 { 975 int err, i, use_page_tbl = 0; 976 struct kcqe **kcq; 977 978 if (use_pg_tbl) 979 use_page_tbl = 1; 980 981 err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl); 982 if (err) 983 return err; 984 985 kcq = (struct kcqe **) info->dma.pg_arr; 986 info->kcq = kcq; 987 988 info->next_idx = cnic_bnx2_next_idx; 989 info->hw_idx = cnic_bnx2_hw_idx; 990 if (use_pg_tbl) 991 return 0; 992 993 info->next_idx = cnic_bnx2x_next_idx; 994 info->hw_idx = cnic_bnx2x_hw_idx; 995 996 for (i = 0; i < KCQ_PAGE_CNT; i++) { 997 struct bnx2x_bd_chain_next *next = 998 (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT]; 999 int j = i + 1; 1000 1001 if (j >= KCQ_PAGE_CNT) 1002 j = 0; 1003 next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32; 1004 next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff; 1005 } 1006 return 0; 1007 } 1008 1009 static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages) 1010 { 1011 struct cnic_local *cp = udev->dev->cnic_priv; 1012 1013 if (udev->l2_ring) 1014 return 0; 1015 1016 udev->l2_ring_size = pages * BNX2_PAGE_SIZE; 1017 udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size, 1018 &udev->l2_ring_map, 1019 GFP_KERNEL | __GFP_COMP); 1020 if (!udev->l2_ring) 1021 return -ENOMEM; 1022 1023 udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size; 1024 udev->l2_buf_size = PAGE_ALIGN(udev->l2_buf_size); 1025 udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size, 1026 &udev->l2_buf_map, 1027 GFP_KERNEL | __GFP_COMP); 1028 if (!udev->l2_buf) { 1029 __cnic_free_uio_rings(udev); 1030 return -ENOMEM; 1031 } 1032 1033 return 0; 1034 1035 } 1036 1037 static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages) 1038 { 1039 struct cnic_local *cp = dev->cnic_priv; 1040 struct cnic_uio_dev *udev; 1041 1042 read_lock(&cnic_dev_lock); 1043 list_for_each_entry(udev, &cnic_udev_list, list) { 1044 if (udev->pdev == dev->pcidev) { 1045 udev->dev = dev; 1046 if (__cnic_alloc_uio_rings(udev, pages)) { 1047 udev->dev = NULL; 1048 read_unlock(&cnic_dev_lock); 1049 return -ENOMEM; 1050 } 1051 cp->udev = udev; 1052 read_unlock(&cnic_dev_lock); 1053 return 0; 1054 } 1055 } 1056 read_unlock(&cnic_dev_lock); 1057 1058 udev = kzalloc(sizeof(struct cnic_uio_dev), GFP_ATOMIC); 1059 if (!udev) 1060 return -ENOMEM; 1061 1062 udev->uio_dev = -1; 1063 1064 udev->dev = dev; 1065 udev->pdev = dev->pcidev; 1066 1067 if (__cnic_alloc_uio_rings(udev, pages)) 1068 goto err_udev; 1069 1070 write_lock(&cnic_dev_lock); 1071 list_add(&udev->list, &cnic_udev_list); 1072 write_unlock(&cnic_dev_lock); 1073 1074 pci_dev_get(udev->pdev); 1075 1076 cp->udev = udev; 1077 1078 return 0; 1079 1080 err_udev: 1081 kfree(udev); 1082 return -ENOMEM; 1083 } 1084 1085 static int cnic_init_uio(struct cnic_dev *dev) 1086 { 1087 struct cnic_local *cp = dev->cnic_priv; 1088 struct cnic_uio_dev *udev = cp->udev; 1089 struct uio_info *uinfo; 1090 int ret = 0; 1091 1092 if (!udev) 1093 return -ENOMEM; 1094 1095 uinfo = &udev->cnic_uinfo; 1096 1097 uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0); 1098 uinfo->mem[0].internal_addr = dev->regview; 1099 uinfo->mem[0].memtype = UIO_MEM_PHYS; 1100 1101 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 1102 uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID + 1103 TX_MAX_TSS_RINGS + 1); 1104 uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen & 1105 PAGE_MASK; 1106 if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 1107 uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE * 9; 1108 else 1109 uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE; 1110 1111 uinfo->name = "bnx2_cnic"; 1112 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 1113 uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0); 1114 1115 uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk & 1116 PAGE_MASK; 1117 uinfo->mem[1].size = sizeof(*cp->bnx2x_def_status_blk); 1118 1119 uinfo->name = "bnx2x_cnic"; 1120 } 1121 1122 uinfo->mem[1].memtype = UIO_MEM_LOGICAL; 1123 1124 uinfo->mem[2].addr = (unsigned long) udev->l2_ring; 1125 uinfo->mem[2].size = udev->l2_ring_size; 1126 uinfo->mem[2].memtype = UIO_MEM_LOGICAL; 1127 1128 uinfo->mem[3].addr = (unsigned long) udev->l2_buf; 1129 uinfo->mem[3].size = udev->l2_buf_size; 1130 uinfo->mem[3].memtype = UIO_MEM_LOGICAL; 1131 1132 uinfo->version = CNIC_MODULE_VERSION; 1133 uinfo->irq = UIO_IRQ_CUSTOM; 1134 1135 uinfo->open = cnic_uio_open; 1136 uinfo->release = cnic_uio_close; 1137 1138 if (udev->uio_dev == -1) { 1139 if (!uinfo->priv) { 1140 uinfo->priv = udev; 1141 1142 ret = uio_register_device(&udev->pdev->dev, uinfo); 1143 } 1144 } else { 1145 cnic_init_rings(dev); 1146 } 1147 1148 return ret; 1149 } 1150 1151 static int cnic_alloc_bnx2_resc(struct cnic_dev *dev) 1152 { 1153 struct cnic_local *cp = dev->cnic_priv; 1154 int ret; 1155 1156 ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1); 1157 if (ret) 1158 goto error; 1159 cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr; 1160 1161 ret = cnic_alloc_kcq(dev, &cp->kcq1, true); 1162 if (ret) 1163 goto error; 1164 1165 ret = cnic_alloc_context(dev); 1166 if (ret) 1167 goto error; 1168 1169 ret = cnic_alloc_uio_rings(dev, 2); 1170 if (ret) 1171 goto error; 1172 1173 ret = cnic_init_uio(dev); 1174 if (ret) 1175 goto error; 1176 1177 return 0; 1178 1179 error: 1180 cnic_free_resc(dev); 1181 return ret; 1182 } 1183 1184 static int cnic_alloc_bnx2x_context(struct cnic_dev *dev) 1185 { 1186 struct cnic_local *cp = dev->cnic_priv; 1187 struct bnx2x *bp = netdev_priv(dev->netdev); 1188 int ctx_blk_size = cp->ethdev->ctx_blk_size; 1189 int total_mem, blks, i; 1190 1191 total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space; 1192 blks = total_mem / ctx_blk_size; 1193 if (total_mem % ctx_blk_size) 1194 blks++; 1195 1196 if (blks > cp->ethdev->ctx_tbl_len) 1197 return -ENOMEM; 1198 1199 cp->ctx_arr = kcalloc(blks, sizeof(struct cnic_ctx), GFP_KERNEL); 1200 if (cp->ctx_arr == NULL) 1201 return -ENOMEM; 1202 1203 cp->ctx_blks = blks; 1204 cp->ctx_blk_size = ctx_blk_size; 1205 if (!CHIP_IS_E1(bp)) 1206 cp->ctx_align = 0; 1207 else 1208 cp->ctx_align = ctx_blk_size; 1209 1210 cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE; 1211 1212 for (i = 0; i < blks; i++) { 1213 cp->ctx_arr[i].ctx = 1214 dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size, 1215 &cp->ctx_arr[i].mapping, 1216 GFP_KERNEL); 1217 if (cp->ctx_arr[i].ctx == NULL) 1218 return -ENOMEM; 1219 1220 if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) { 1221 if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) { 1222 cnic_free_context(dev); 1223 cp->ctx_blk_size += cp->ctx_align; 1224 i = -1; 1225 continue; 1226 } 1227 } 1228 } 1229 return 0; 1230 } 1231 1232 static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev) 1233 { 1234 struct cnic_local *cp = dev->cnic_priv; 1235 struct bnx2x *bp = netdev_priv(dev->netdev); 1236 struct cnic_eth_dev *ethdev = cp->ethdev; 1237 u32 start_cid = ethdev->starting_cid; 1238 int i, j, n, ret, pages; 1239 struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info; 1240 1241 cp->max_cid_space = MAX_ISCSI_TBL_SZ; 1242 cp->iscsi_start_cid = start_cid; 1243 cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ; 1244 1245 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 1246 cp->max_cid_space += dev->max_fcoe_conn; 1247 cp->fcoe_init_cid = ethdev->fcoe_init_cid; 1248 if (!cp->fcoe_init_cid) 1249 cp->fcoe_init_cid = 0x10; 1250 } 1251 1252 cp->iscsi_tbl = kzalloc(sizeof(struct cnic_iscsi) * MAX_ISCSI_TBL_SZ, 1253 GFP_KERNEL); 1254 if (!cp->iscsi_tbl) 1255 goto error; 1256 1257 cp->ctx_tbl = kzalloc(sizeof(struct cnic_context) * 1258 cp->max_cid_space, GFP_KERNEL); 1259 if (!cp->ctx_tbl) 1260 goto error; 1261 1262 for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) { 1263 cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i]; 1264 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI; 1265 } 1266 1267 for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++) 1268 cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE; 1269 1270 pages = PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) / 1271 PAGE_SIZE; 1272 1273 ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0); 1274 if (ret) 1275 return -ENOMEM; 1276 1277 n = PAGE_SIZE / CNIC_KWQ16_DATA_SIZE; 1278 for (i = 0, j = 0; i < cp->max_cid_space; i++) { 1279 long off = CNIC_KWQ16_DATA_SIZE * (i % n); 1280 1281 cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off; 1282 cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] + 1283 off; 1284 1285 if ((i % n) == (n - 1)) 1286 j++; 1287 } 1288 1289 ret = cnic_alloc_kcq(dev, &cp->kcq1, false); 1290 if (ret) 1291 goto error; 1292 1293 if (CNIC_SUPPORTS_FCOE(bp)) { 1294 ret = cnic_alloc_kcq(dev, &cp->kcq2, true); 1295 if (ret) 1296 goto error; 1297 } 1298 1299 pages = PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / PAGE_SIZE; 1300 ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0); 1301 if (ret) 1302 goto error; 1303 1304 ret = cnic_alloc_bnx2x_context(dev); 1305 if (ret) 1306 goto error; 1307 1308 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 1309 return 0; 1310 1311 cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk; 1312 1313 cp->l2_rx_ring_size = 15; 1314 1315 ret = cnic_alloc_uio_rings(dev, 4); 1316 if (ret) 1317 goto error; 1318 1319 ret = cnic_init_uio(dev); 1320 if (ret) 1321 goto error; 1322 1323 return 0; 1324 1325 error: 1326 cnic_free_resc(dev); 1327 return -ENOMEM; 1328 } 1329 1330 static inline u32 cnic_kwq_avail(struct cnic_local *cp) 1331 { 1332 return cp->max_kwq_idx - 1333 ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx); 1334 } 1335 1336 static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 1337 u32 num_wqes) 1338 { 1339 struct cnic_local *cp = dev->cnic_priv; 1340 struct kwqe *prod_qe; 1341 u16 prod, sw_prod, i; 1342 1343 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 1344 return -EAGAIN; /* bnx2 is down */ 1345 1346 spin_lock_bh(&cp->cnic_ulp_lock); 1347 if (num_wqes > cnic_kwq_avail(cp) && 1348 !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) { 1349 spin_unlock_bh(&cp->cnic_ulp_lock); 1350 return -EAGAIN; 1351 } 1352 1353 clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 1354 1355 prod = cp->kwq_prod_idx; 1356 sw_prod = prod & MAX_KWQ_IDX; 1357 for (i = 0; i < num_wqes; i++) { 1358 prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)]; 1359 memcpy(prod_qe, wqes[i], sizeof(struct kwqe)); 1360 prod++; 1361 sw_prod = prod & MAX_KWQ_IDX; 1362 } 1363 cp->kwq_prod_idx = prod; 1364 1365 CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx); 1366 1367 spin_unlock_bh(&cp->cnic_ulp_lock); 1368 return 0; 1369 } 1370 1371 static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid, 1372 union l5cm_specific_data *l5_data) 1373 { 1374 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1375 dma_addr_t map; 1376 1377 map = ctx->kwqe_data_mapping; 1378 l5_data->phy_address.lo = (u64) map & 0xffffffff; 1379 l5_data->phy_address.hi = (u64) map >> 32; 1380 return ctx->kwqe_data; 1381 } 1382 1383 static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid, 1384 u32 type, union l5cm_specific_data *l5_data) 1385 { 1386 struct cnic_local *cp = dev->cnic_priv; 1387 struct bnx2x *bp = netdev_priv(dev->netdev); 1388 struct l5cm_spe kwqe; 1389 struct kwqe_16 *kwq[1]; 1390 u16 type_16; 1391 int ret; 1392 1393 kwqe.hdr.conn_and_cmd_data = 1394 cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) | 1395 BNX2X_HW_CID(bp, cid))); 1396 1397 type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE; 1398 type_16 |= (bp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) & 1399 SPE_HDR_FUNCTION_ID; 1400 1401 kwqe.hdr.type = cpu_to_le16(type_16); 1402 kwqe.hdr.reserved1 = 0; 1403 kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo); 1404 kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi); 1405 1406 kwq[0] = (struct kwqe_16 *) &kwqe; 1407 1408 spin_lock_bh(&cp->cnic_ulp_lock); 1409 ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1); 1410 spin_unlock_bh(&cp->cnic_ulp_lock); 1411 1412 if (ret == 1) 1413 return 0; 1414 1415 return ret; 1416 } 1417 1418 static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type, 1419 struct kcqe *cqes[], u32 num_cqes) 1420 { 1421 struct cnic_local *cp = dev->cnic_priv; 1422 struct cnic_ulp_ops *ulp_ops; 1423 1424 rcu_read_lock(); 1425 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 1426 if (likely(ulp_ops)) { 1427 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 1428 cqes, num_cqes); 1429 } 1430 rcu_read_unlock(); 1431 } 1432 1433 static void cnic_bnx2x_set_tcp_options(struct cnic_dev *dev, int time_stamps, 1434 int en_tcp_dack) 1435 { 1436 struct bnx2x *bp = netdev_priv(dev->netdev); 1437 u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN; 1438 u16 tstorm_flags = 0; 1439 1440 if (time_stamps) { 1441 xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1442 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED; 1443 } 1444 if (en_tcp_dack) 1445 tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_DELAYED_ACK_EN; 1446 1447 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1448 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), xstorm_flags); 1449 1450 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1451 TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), tstorm_flags); 1452 } 1453 1454 static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe) 1455 { 1456 struct cnic_local *cp = dev->cnic_priv; 1457 struct bnx2x *bp = netdev_priv(dev->netdev); 1458 struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe; 1459 int hq_bds, pages; 1460 u32 pfid = bp->pfid; 1461 1462 cp->num_iscsi_tasks = req1->num_tasks_per_conn; 1463 cp->num_ccells = req1->num_ccells_per_conn; 1464 cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE * 1465 cp->num_iscsi_tasks; 1466 cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS * 1467 BNX2X_ISCSI_R2TQE_SIZE; 1468 cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE; 1469 pages = PAGE_ALIGN(cp->hq_size) / PAGE_SIZE; 1470 hq_bds = pages * (PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE); 1471 cp->num_cqs = req1->num_cqs; 1472 1473 if (!dev->max_iscsi_conn) 1474 return 0; 1475 1476 /* init Tstorm RAM */ 1477 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1478 req1->rq_num_wqes); 1479 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1480 PAGE_SIZE); 1481 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 1482 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1483 CNIC_WR16(dev, BAR_TSTRORM_INTMEM + 1484 TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1485 req1->num_tasks_per_conn); 1486 1487 /* init Ustorm RAM */ 1488 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1489 USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid), 1490 req1->rq_buffer_size); 1491 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1492 PAGE_SIZE); 1493 CNIC_WR8(dev, BAR_USTRORM_INTMEM + 1494 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1495 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1496 USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1497 req1->num_tasks_per_conn); 1498 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid), 1499 req1->rq_num_wqes); 1500 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1501 req1->cq_num_wqes); 1502 CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1503 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1504 1505 /* init Xstorm RAM */ 1506 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1507 PAGE_SIZE); 1508 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 1509 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1510 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 1511 XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1512 req1->num_tasks_per_conn); 1513 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1514 hq_bds); 1515 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid), 1516 req1->num_tasks_per_conn); 1517 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid), 1518 cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS); 1519 1520 /* init Cstorm RAM */ 1521 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid), 1522 PAGE_SIZE); 1523 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 1524 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), PAGE_SHIFT); 1525 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1526 CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid), 1527 req1->num_tasks_per_conn); 1528 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid), 1529 req1->cq_num_wqes); 1530 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid), 1531 hq_bds); 1532 1533 cnic_bnx2x_set_tcp_options(dev, 1534 req1->flags & ISCSI_KWQE_INIT1_TIME_STAMPS_ENABLE, 1535 req1->flags & ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE); 1536 1537 return 0; 1538 } 1539 1540 static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe) 1541 { 1542 struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe; 1543 struct bnx2x *bp = netdev_priv(dev->netdev); 1544 u32 pfid = bp->pfid; 1545 struct iscsi_kcqe kcqe; 1546 struct kcqe *cqes[1]; 1547 1548 memset(&kcqe, 0, sizeof(kcqe)); 1549 if (!dev->max_iscsi_conn) { 1550 kcqe.completion_status = 1551 ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED; 1552 goto done; 1553 } 1554 1555 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1556 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1557 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 1558 TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1559 req2->error_bit_map[1]); 1560 1561 CNIC_WR16(dev, BAR_USTRORM_INTMEM + 1562 USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1563 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1564 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]); 1565 CNIC_WR(dev, BAR_USTRORM_INTMEM + 1566 USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4, 1567 req2->error_bit_map[1]); 1568 1569 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 1570 CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn); 1571 1572 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1573 1574 done: 1575 kcqe.op_code = ISCSI_KCQE_OPCODE_INIT; 1576 cqes[0] = (struct kcqe *) &kcqe; 1577 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1578 1579 return 0; 1580 } 1581 1582 static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1583 { 1584 struct cnic_local *cp = dev->cnic_priv; 1585 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1586 1587 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) { 1588 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1589 1590 cnic_free_dma(dev, &iscsi->hq_info); 1591 cnic_free_dma(dev, &iscsi->r2tq_info); 1592 cnic_free_dma(dev, &iscsi->task_array_info); 1593 cnic_free_id(&cp->cid_tbl, ctx->cid); 1594 } else { 1595 cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid); 1596 } 1597 1598 ctx->cid = 0; 1599 } 1600 1601 static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid) 1602 { 1603 u32 cid; 1604 int ret, pages; 1605 struct cnic_local *cp = dev->cnic_priv; 1606 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1607 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1608 1609 if (ctx->ulp_proto_id == CNIC_ULP_FCOE) { 1610 cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl); 1611 if (cid == -1) { 1612 ret = -ENOMEM; 1613 goto error; 1614 } 1615 ctx->cid = cid; 1616 return 0; 1617 } 1618 1619 cid = cnic_alloc_new_id(&cp->cid_tbl); 1620 if (cid == -1) { 1621 ret = -ENOMEM; 1622 goto error; 1623 } 1624 1625 ctx->cid = cid; 1626 pages = PAGE_ALIGN(cp->task_array_size) / PAGE_SIZE; 1627 1628 ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1); 1629 if (ret) 1630 goto error; 1631 1632 pages = PAGE_ALIGN(cp->r2tq_size) / PAGE_SIZE; 1633 ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1); 1634 if (ret) 1635 goto error; 1636 1637 pages = PAGE_ALIGN(cp->hq_size) / PAGE_SIZE; 1638 ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1); 1639 if (ret) 1640 goto error; 1641 1642 return 0; 1643 1644 error: 1645 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1646 return ret; 1647 } 1648 1649 static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init, 1650 struct regpair *ctx_addr) 1651 { 1652 struct cnic_local *cp = dev->cnic_priv; 1653 struct cnic_eth_dev *ethdev = cp->ethdev; 1654 int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk; 1655 int off = (cid - ethdev->starting_cid) % cp->cids_per_blk; 1656 unsigned long align_off = 0; 1657 dma_addr_t ctx_map; 1658 void *ctx; 1659 1660 if (cp->ctx_align) { 1661 unsigned long mask = cp->ctx_align - 1; 1662 1663 if (cp->ctx_arr[blk].mapping & mask) 1664 align_off = cp->ctx_align - 1665 (cp->ctx_arr[blk].mapping & mask); 1666 } 1667 ctx_map = cp->ctx_arr[blk].mapping + align_off + 1668 (off * BNX2X_CONTEXT_MEM_SIZE); 1669 ctx = cp->ctx_arr[blk].ctx + align_off + 1670 (off * BNX2X_CONTEXT_MEM_SIZE); 1671 if (init) 1672 memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE); 1673 1674 ctx_addr->lo = ctx_map & 0xffffffff; 1675 ctx_addr->hi = (u64) ctx_map >> 32; 1676 return ctx; 1677 } 1678 1679 static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[], 1680 u32 num) 1681 { 1682 struct cnic_local *cp = dev->cnic_priv; 1683 struct bnx2x *bp = netdev_priv(dev->netdev); 1684 struct iscsi_kwqe_conn_offload1 *req1 = 1685 (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1686 struct iscsi_kwqe_conn_offload2 *req2 = 1687 (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1688 struct iscsi_kwqe_conn_offload3 *req3; 1689 struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id]; 1690 struct cnic_iscsi *iscsi = ctx->proto.iscsi; 1691 u32 cid = ctx->cid; 1692 u32 hw_cid = BNX2X_HW_CID(bp, cid); 1693 struct iscsi_context *ictx; 1694 struct regpair context_addr; 1695 int i, j, n = 2, n_max; 1696 u8 port = BP_PORT(bp); 1697 1698 ctx->ctx_flags = 0; 1699 if (!req2->num_additional_wqes) 1700 return -EINVAL; 1701 1702 n_max = req2->num_additional_wqes + 2; 1703 1704 ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr); 1705 if (ictx == NULL) 1706 return -ENOMEM; 1707 1708 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1709 1710 ictx->xstorm_ag_context.hq_prod = 1; 1711 1712 ictx->xstorm_st_context.iscsi.first_burst_length = 1713 ISCSI_DEF_FIRST_BURST_LEN; 1714 ictx->xstorm_st_context.iscsi.max_send_pdu_length = 1715 ISCSI_DEF_MAX_RECV_SEG_LEN; 1716 ictx->xstorm_st_context.iscsi.sq_pbl_base.lo = 1717 req1->sq_page_table_addr_lo; 1718 ictx->xstorm_st_context.iscsi.sq_pbl_base.hi = 1719 req1->sq_page_table_addr_hi; 1720 ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi; 1721 ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo; 1722 ictx->xstorm_st_context.iscsi.hq_pbl_base.lo = 1723 iscsi->hq_info.pgtbl_map & 0xffffffff; 1724 ictx->xstorm_st_context.iscsi.hq_pbl_base.hi = 1725 (u64) iscsi->hq_info.pgtbl_map >> 32; 1726 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo = 1727 iscsi->hq_info.pgtbl[0]; 1728 ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi = 1729 iscsi->hq_info.pgtbl[1]; 1730 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo = 1731 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1732 ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi = 1733 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1734 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo = 1735 iscsi->r2tq_info.pgtbl[0]; 1736 ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi = 1737 iscsi->r2tq_info.pgtbl[1]; 1738 ictx->xstorm_st_context.iscsi.task_pbl_base.lo = 1739 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1740 ictx->xstorm_st_context.iscsi.task_pbl_base.hi = 1741 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1742 ictx->xstorm_st_context.iscsi.task_pbl_cache_idx = 1743 BNX2X_ISCSI_PBL_NOT_CACHED; 1744 ictx->xstorm_st_context.iscsi.flags.flags |= 1745 XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA; 1746 ictx->xstorm_st_context.iscsi.flags.flags |= 1747 XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T; 1748 ictx->xstorm_st_context.common.ethernet.reserved_vlan_type = 1749 ETH_P_8021Q; 1750 if (BNX2X_CHIP_IS_E2_PLUS(bp) && 1751 bp->common.chip_port_mode == CHIP_2_PORT_MODE) { 1752 1753 port = 0; 1754 } 1755 ictx->xstorm_st_context.common.flags = 1756 1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT; 1757 ictx->xstorm_st_context.common.flags = 1758 port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT; 1759 1760 ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE; 1761 /* TSTORM requires the base address of RQ DB & not PTE */ 1762 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo = 1763 req2->rq_page_table_addr_lo & PAGE_MASK; 1764 ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi = 1765 req2->rq_page_table_addr_hi; 1766 ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id; 1767 ictx->tstorm_st_context.tcp.cwnd = 0x5A8; 1768 ictx->tstorm_st_context.tcp.flags2 |= 1769 TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN; 1770 ictx->tstorm_st_context.tcp.ooo_support_mode = 1771 TCP_TSTORM_OOO_DROP_AND_PROC_ACK; 1772 1773 ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG; 1774 1775 ictx->ustorm_st_context.ring.rq.pbl_base.lo = 1776 req2->rq_page_table_addr_lo; 1777 ictx->ustorm_st_context.ring.rq.pbl_base.hi = 1778 req2->rq_page_table_addr_hi; 1779 ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi; 1780 ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo; 1781 ictx->ustorm_st_context.ring.r2tq.pbl_base.lo = 1782 iscsi->r2tq_info.pgtbl_map & 0xffffffff; 1783 ictx->ustorm_st_context.ring.r2tq.pbl_base.hi = 1784 (u64) iscsi->r2tq_info.pgtbl_map >> 32; 1785 ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo = 1786 iscsi->r2tq_info.pgtbl[0]; 1787 ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi = 1788 iscsi->r2tq_info.pgtbl[1]; 1789 ictx->ustorm_st_context.ring.cq_pbl_base.lo = 1790 req1->cq_page_table_addr_lo; 1791 ictx->ustorm_st_context.ring.cq_pbl_base.hi = 1792 req1->cq_page_table_addr_hi; 1793 ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN; 1794 ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi; 1795 ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo; 1796 ictx->ustorm_st_context.task_pbe_cache_index = 1797 BNX2X_ISCSI_PBL_NOT_CACHED; 1798 ictx->ustorm_st_context.task_pdu_cache_index = 1799 BNX2X_ISCSI_PDU_HEADER_NOT_CACHED; 1800 1801 for (i = 1, j = 1; i < cp->num_cqs; i++, j++) { 1802 if (j == 3) { 1803 if (n >= n_max) 1804 break; 1805 req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++]; 1806 j = 0; 1807 } 1808 ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN; 1809 ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo = 1810 req3->qp_first_pte[j].hi; 1811 ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi = 1812 req3->qp_first_pte[j].lo; 1813 } 1814 1815 ictx->ustorm_st_context.task_pbl_base.lo = 1816 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1817 ictx->ustorm_st_context.task_pbl_base.hi = 1818 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1819 ictx->ustorm_st_context.tce_phy_addr.lo = 1820 iscsi->task_array_info.pgtbl[0]; 1821 ictx->ustorm_st_context.tce_phy_addr.hi = 1822 iscsi->task_array_info.pgtbl[1]; 1823 ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1824 ictx->ustorm_st_context.num_cqs = cp->num_cqs; 1825 ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN; 1826 ictx->ustorm_st_context.negotiated_rx_and_flags |= 1827 ISCSI_DEF_MAX_BURST_LEN; 1828 ictx->ustorm_st_context.negotiated_rx |= 1829 ISCSI_DEFAULT_MAX_OUTSTANDING_R2T << 1830 USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT; 1831 1832 ictx->cstorm_st_context.hq_pbl_base.lo = 1833 iscsi->hq_info.pgtbl_map & 0xffffffff; 1834 ictx->cstorm_st_context.hq_pbl_base.hi = 1835 (u64) iscsi->hq_info.pgtbl_map >> 32; 1836 ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0]; 1837 ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1]; 1838 ictx->cstorm_st_context.task_pbl_base.lo = 1839 iscsi->task_array_info.pgtbl_map & 0xffffffff; 1840 ictx->cstorm_st_context.task_pbl_base.hi = 1841 (u64) iscsi->task_array_info.pgtbl_map >> 32; 1842 /* CSTORM and USTORM initialization is different, CSTORM requires 1843 * CQ DB base & not PTE addr */ 1844 ictx->cstorm_st_context.cq_db_base.lo = 1845 req1->cq_page_table_addr_lo & PAGE_MASK; 1846 ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi; 1847 ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id; 1848 ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1; 1849 for (i = 0; i < cp->num_cqs; i++) { 1850 ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] = 1851 ISCSI_INITIAL_SN; 1852 ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] = 1853 ISCSI_INITIAL_SN; 1854 } 1855 1856 ictx->xstorm_ag_context.cdu_reserved = 1857 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 1858 ISCSI_CONNECTION_TYPE); 1859 ictx->ustorm_ag_context.cdu_usage = 1860 CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 1861 ISCSI_CONNECTION_TYPE); 1862 return 0; 1863 1864 } 1865 1866 static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 1867 u32 num, int *work) 1868 { 1869 struct iscsi_kwqe_conn_offload1 *req1; 1870 struct iscsi_kwqe_conn_offload2 *req2; 1871 struct cnic_local *cp = dev->cnic_priv; 1872 struct bnx2x *bp = netdev_priv(dev->netdev); 1873 struct cnic_context *ctx; 1874 struct iscsi_kcqe kcqe; 1875 struct kcqe *cqes[1]; 1876 u32 l5_cid; 1877 int ret = 0; 1878 1879 if (num < 2) { 1880 *work = num; 1881 return -EINVAL; 1882 } 1883 1884 req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0]; 1885 req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1]; 1886 if ((num - 2) < req2->num_additional_wqes) { 1887 *work = num; 1888 return -EINVAL; 1889 } 1890 *work = 2 + req2->num_additional_wqes; 1891 1892 l5_cid = req1->iscsi_conn_id; 1893 if (l5_cid >= MAX_ISCSI_TBL_SZ) 1894 return -EINVAL; 1895 1896 memset(&kcqe, 0, sizeof(kcqe)); 1897 kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN; 1898 kcqe.iscsi_conn_id = l5_cid; 1899 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 1900 1901 ctx = &cp->ctx_tbl[l5_cid]; 1902 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) { 1903 kcqe.completion_status = 1904 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY; 1905 goto done; 1906 } 1907 1908 if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) { 1909 atomic_dec(&cp->iscsi_conn); 1910 goto done; 1911 } 1912 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 1913 if (ret) { 1914 atomic_dec(&cp->iscsi_conn); 1915 ret = 0; 1916 goto done; 1917 } 1918 ret = cnic_setup_bnx2x_ctx(dev, wqes, num); 1919 if (ret < 0) { 1920 cnic_free_bnx2x_conn_resc(dev, l5_cid); 1921 atomic_dec(&cp->iscsi_conn); 1922 goto done; 1923 } 1924 1925 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 1926 kcqe.iscsi_conn_context_id = BNX2X_HW_CID(bp, cp->ctx_tbl[l5_cid].cid); 1927 1928 done: 1929 cqes[0] = (struct kcqe *) &kcqe; 1930 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 1931 return 0; 1932 } 1933 1934 1935 static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe) 1936 { 1937 struct cnic_local *cp = dev->cnic_priv; 1938 struct iscsi_kwqe_conn_update *req = 1939 (struct iscsi_kwqe_conn_update *) kwqe; 1940 void *data; 1941 union l5cm_specific_data l5_data; 1942 u32 l5_cid, cid = BNX2X_SW_CID(req->context_id); 1943 int ret; 1944 1945 if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0) 1946 return -EINVAL; 1947 1948 data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 1949 if (!data) 1950 return -ENOMEM; 1951 1952 memcpy(data, kwqe, sizeof(struct kwqe)); 1953 1954 ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, 1955 req->context_id, ISCSI_CONNECTION_TYPE, &l5_data); 1956 return ret; 1957 } 1958 1959 static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid) 1960 { 1961 struct cnic_local *cp = dev->cnic_priv; 1962 struct bnx2x *bp = netdev_priv(dev->netdev); 1963 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1964 union l5cm_specific_data l5_data; 1965 int ret; 1966 u32 hw_cid; 1967 1968 init_waitqueue_head(&ctx->waitq); 1969 ctx->wait_cond = 0; 1970 memset(&l5_data, 0, sizeof(l5_data)); 1971 hw_cid = BNX2X_HW_CID(bp, ctx->cid); 1972 1973 ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 1974 hw_cid, NONE_CONNECTION_TYPE, &l5_data); 1975 1976 if (ret == 0) { 1977 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 1978 if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags))) 1979 return -EBUSY; 1980 } 1981 1982 return 0; 1983 } 1984 1985 static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 1986 { 1987 struct cnic_local *cp = dev->cnic_priv; 1988 struct iscsi_kwqe_conn_destroy *req = 1989 (struct iscsi_kwqe_conn_destroy *) kwqe; 1990 u32 l5_cid = req->reserved0; 1991 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 1992 int ret = 0; 1993 struct iscsi_kcqe kcqe; 1994 struct kcqe *cqes[1]; 1995 1996 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 1997 goto skip_cfc_delete; 1998 1999 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 2000 unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies; 2001 2002 if (delta > (2 * HZ)) 2003 delta = 0; 2004 2005 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2006 queue_delayed_work(cnic_wq, &cp->delete_task, delta); 2007 goto destroy_reply; 2008 } 2009 2010 ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid); 2011 2012 skip_cfc_delete: 2013 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2014 2015 if (!ret) { 2016 atomic_dec(&cp->iscsi_conn); 2017 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2018 } 2019 2020 destroy_reply: 2021 memset(&kcqe, 0, sizeof(kcqe)); 2022 kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN; 2023 kcqe.iscsi_conn_id = l5_cid; 2024 kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS; 2025 kcqe.iscsi_conn_context_id = req->context_id; 2026 2027 cqes[0] = (struct kcqe *) &kcqe; 2028 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1); 2029 2030 return 0; 2031 } 2032 2033 static void cnic_init_storm_conn_bufs(struct cnic_dev *dev, 2034 struct l4_kwq_connect_req1 *kwqe1, 2035 struct l4_kwq_connect_req3 *kwqe3, 2036 struct l5cm_active_conn_buffer *conn_buf) 2037 { 2038 struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf; 2039 struct l5cm_xstorm_conn_buffer *xstorm_buf = 2040 &conn_buf->xstorm_conn_buffer; 2041 struct l5cm_tstorm_conn_buffer *tstorm_buf = 2042 &conn_buf->tstorm_conn_buffer; 2043 struct regpair context_addr; 2044 u32 cid = BNX2X_SW_CID(kwqe1->cid); 2045 struct in6_addr src_ip, dst_ip; 2046 int i; 2047 u32 *addrp; 2048 2049 addrp = (u32 *) &conn_addr->local_ip_addr; 2050 for (i = 0; i < 4; i++, addrp++) 2051 src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2052 2053 addrp = (u32 *) &conn_addr->remote_ip_addr; 2054 for (i = 0; i < 4; i++, addrp++) 2055 dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp); 2056 2057 cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr); 2058 2059 xstorm_buf->context_addr.hi = context_addr.hi; 2060 xstorm_buf->context_addr.lo = context_addr.lo; 2061 xstorm_buf->mss = 0xffff; 2062 xstorm_buf->rcv_buf = kwqe3->rcv_buf; 2063 if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE) 2064 xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE; 2065 xstorm_buf->pseudo_header_checksum = 2066 swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0)); 2067 2068 if (kwqe3->ka_timeout) { 2069 tstorm_buf->ka_enable = 1; 2070 tstorm_buf->ka_timeout = kwqe3->ka_timeout; 2071 tstorm_buf->ka_interval = kwqe3->ka_interval; 2072 tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count; 2073 } 2074 tstorm_buf->max_rt_time = 0xffffffff; 2075 } 2076 2077 static void cnic_init_bnx2x_mac(struct cnic_dev *dev) 2078 { 2079 struct bnx2x *bp = netdev_priv(dev->netdev); 2080 u32 pfid = bp->pfid; 2081 u8 *mac = dev->mac_addr; 2082 2083 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2084 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]); 2085 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2086 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]); 2087 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2088 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]); 2089 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2090 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]); 2091 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2092 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]); 2093 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 2094 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]); 2095 2096 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2097 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]); 2098 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2099 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2100 mac[4]); 2101 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2102 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]); 2103 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2104 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2105 mac[2]); 2106 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2107 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]); 2108 CNIC_WR8(dev, BAR_TSTRORM_INTMEM + 2109 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1, 2110 mac[0]); 2111 } 2112 2113 static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[], 2114 u32 num, int *work) 2115 { 2116 struct cnic_local *cp = dev->cnic_priv; 2117 struct bnx2x *bp = netdev_priv(dev->netdev); 2118 struct l4_kwq_connect_req1 *kwqe1 = 2119 (struct l4_kwq_connect_req1 *) wqes[0]; 2120 struct l4_kwq_connect_req3 *kwqe3; 2121 struct l5cm_active_conn_buffer *conn_buf; 2122 struct l5cm_conn_addr_params *conn_addr; 2123 union l5cm_specific_data l5_data; 2124 u32 l5_cid = kwqe1->pg_cid; 2125 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 2126 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 2127 int ret; 2128 2129 if (num < 2) { 2130 *work = num; 2131 return -EINVAL; 2132 } 2133 2134 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) 2135 *work = 3; 2136 else 2137 *work = 2; 2138 2139 if (num < *work) { 2140 *work = num; 2141 return -EINVAL; 2142 } 2143 2144 if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) { 2145 netdev_err(dev->netdev, "conn_buf size too big\n"); 2146 return -ENOMEM; 2147 } 2148 conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2149 if (!conn_buf) 2150 return -ENOMEM; 2151 2152 memset(conn_buf, 0, sizeof(*conn_buf)); 2153 2154 conn_addr = &conn_buf->conn_addr_buf; 2155 conn_addr->remote_addr_0 = csk->ha[0]; 2156 conn_addr->remote_addr_1 = csk->ha[1]; 2157 conn_addr->remote_addr_2 = csk->ha[2]; 2158 conn_addr->remote_addr_3 = csk->ha[3]; 2159 conn_addr->remote_addr_4 = csk->ha[4]; 2160 conn_addr->remote_addr_5 = csk->ha[5]; 2161 2162 if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) { 2163 struct l4_kwq_connect_req2 *kwqe2 = 2164 (struct l4_kwq_connect_req2 *) wqes[1]; 2165 2166 conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4; 2167 conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3; 2168 conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2; 2169 2170 conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4; 2171 conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3; 2172 conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2; 2173 conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION; 2174 } 2175 kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1]; 2176 2177 conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip; 2178 conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip; 2179 conn_addr->local_tcp_port = kwqe1->src_port; 2180 conn_addr->remote_tcp_port = kwqe1->dst_port; 2181 2182 conn_addr->pmtu = kwqe3->pmtu; 2183 cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf); 2184 2185 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 2186 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(bp->pfid), csk->vlan_id); 2187 2188 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT, 2189 kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2190 if (!ret) 2191 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2192 2193 return ret; 2194 } 2195 2196 static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe) 2197 { 2198 struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe; 2199 union l5cm_specific_data l5_data; 2200 int ret; 2201 2202 memset(&l5_data, 0, sizeof(l5_data)); 2203 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE, 2204 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2205 return ret; 2206 } 2207 2208 static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe) 2209 { 2210 struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe; 2211 union l5cm_specific_data l5_data; 2212 int ret; 2213 2214 memset(&l5_data, 0, sizeof(l5_data)); 2215 ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT, 2216 req->cid, ISCSI_CONNECTION_TYPE, &l5_data); 2217 return ret; 2218 } 2219 static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2220 { 2221 struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe; 2222 struct l4_kcq kcqe; 2223 struct kcqe *cqes[1]; 2224 2225 memset(&kcqe, 0, sizeof(kcqe)); 2226 kcqe.pg_host_opaque = req->host_opaque; 2227 kcqe.pg_cid = req->host_opaque; 2228 kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG; 2229 cqes[0] = (struct kcqe *) &kcqe; 2230 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2231 return 0; 2232 } 2233 2234 static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe) 2235 { 2236 struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe; 2237 struct l4_kcq kcqe; 2238 struct kcqe *cqes[1]; 2239 2240 memset(&kcqe, 0, sizeof(kcqe)); 2241 kcqe.pg_host_opaque = req->pg_host_opaque; 2242 kcqe.pg_cid = req->pg_cid; 2243 kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG; 2244 cqes[0] = (struct kcqe *) &kcqe; 2245 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1); 2246 return 0; 2247 } 2248 2249 static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe) 2250 { 2251 struct fcoe_kwqe_stat *req; 2252 struct fcoe_stat_ramrod_params *fcoe_stat; 2253 union l5cm_specific_data l5_data; 2254 struct cnic_local *cp = dev->cnic_priv; 2255 struct bnx2x *bp = netdev_priv(dev->netdev); 2256 int ret; 2257 u32 cid; 2258 2259 req = (struct fcoe_kwqe_stat *) kwqe; 2260 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2261 2262 fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2263 if (!fcoe_stat) 2264 return -ENOMEM; 2265 2266 memset(fcoe_stat, 0, sizeof(*fcoe_stat)); 2267 memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req)); 2268 2269 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid, 2270 FCOE_CONNECTION_TYPE, &l5_data); 2271 return ret; 2272 } 2273 2274 static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[], 2275 u32 num, int *work) 2276 { 2277 int ret; 2278 struct cnic_local *cp = dev->cnic_priv; 2279 struct bnx2x *bp = netdev_priv(dev->netdev); 2280 u32 cid; 2281 struct fcoe_init_ramrod_params *fcoe_init; 2282 struct fcoe_kwqe_init1 *req1; 2283 struct fcoe_kwqe_init2 *req2; 2284 struct fcoe_kwqe_init3 *req3; 2285 union l5cm_specific_data l5_data; 2286 2287 if (num < 3) { 2288 *work = num; 2289 return -EINVAL; 2290 } 2291 req1 = (struct fcoe_kwqe_init1 *) wqes[0]; 2292 req2 = (struct fcoe_kwqe_init2 *) wqes[1]; 2293 req3 = (struct fcoe_kwqe_init3 *) wqes[2]; 2294 if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) { 2295 *work = 1; 2296 return -EINVAL; 2297 } 2298 if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) { 2299 *work = 2; 2300 return -EINVAL; 2301 } 2302 2303 if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) { 2304 netdev_err(dev->netdev, "fcoe_init size too big\n"); 2305 return -ENOMEM; 2306 } 2307 fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data); 2308 if (!fcoe_init) 2309 return -ENOMEM; 2310 2311 memset(fcoe_init, 0, sizeof(*fcoe_init)); 2312 memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1)); 2313 memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2)); 2314 memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3)); 2315 fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff; 2316 fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32; 2317 fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages; 2318 2319 fcoe_init->sb_num = cp->status_blk_num; 2320 fcoe_init->eq_prod = MAX_KCQ_IDX; 2321 fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS; 2322 cp->kcq2.sw_prod_idx = 0; 2323 2324 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2325 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid, 2326 FCOE_CONNECTION_TYPE, &l5_data); 2327 *work = 3; 2328 return ret; 2329 } 2330 2331 static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[], 2332 u32 num, int *work) 2333 { 2334 int ret = 0; 2335 u32 cid = -1, l5_cid; 2336 struct cnic_local *cp = dev->cnic_priv; 2337 struct bnx2x *bp = netdev_priv(dev->netdev); 2338 struct fcoe_kwqe_conn_offload1 *req1; 2339 struct fcoe_kwqe_conn_offload2 *req2; 2340 struct fcoe_kwqe_conn_offload3 *req3; 2341 struct fcoe_kwqe_conn_offload4 *req4; 2342 struct fcoe_conn_offload_ramrod_params *fcoe_offload; 2343 struct cnic_context *ctx; 2344 struct fcoe_context *fctx; 2345 struct regpair ctx_addr; 2346 union l5cm_specific_data l5_data; 2347 struct fcoe_kcqe kcqe; 2348 struct kcqe *cqes[1]; 2349 2350 if (num < 4) { 2351 *work = num; 2352 return -EINVAL; 2353 } 2354 req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0]; 2355 req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1]; 2356 req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2]; 2357 req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3]; 2358 2359 *work = 4; 2360 2361 l5_cid = req1->fcoe_conn_id; 2362 if (l5_cid >= dev->max_fcoe_conn) 2363 goto err_reply; 2364 2365 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2366 2367 ctx = &cp->ctx_tbl[l5_cid]; 2368 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2369 goto err_reply; 2370 2371 ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid); 2372 if (ret) { 2373 ret = 0; 2374 goto err_reply; 2375 } 2376 cid = ctx->cid; 2377 2378 fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr); 2379 if (fctx) { 2380 u32 hw_cid = BNX2X_HW_CID(bp, cid); 2381 u32 val; 2382 2383 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG, 2384 FCOE_CONNECTION_TYPE); 2385 fctx->xstorm_ag_context.cdu_reserved = val; 2386 val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG, 2387 FCOE_CONNECTION_TYPE); 2388 fctx->ustorm_ag_context.cdu_usage = val; 2389 } 2390 if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) { 2391 netdev_err(dev->netdev, "fcoe_offload size too big\n"); 2392 goto err_reply; 2393 } 2394 fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2395 if (!fcoe_offload) 2396 goto err_reply; 2397 2398 memset(fcoe_offload, 0, sizeof(*fcoe_offload)); 2399 memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1)); 2400 memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2)); 2401 memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3)); 2402 memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4)); 2403 2404 cid = BNX2X_HW_CID(bp, cid); 2405 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid, 2406 FCOE_CONNECTION_TYPE, &l5_data); 2407 if (!ret) 2408 set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 2409 2410 return ret; 2411 2412 err_reply: 2413 if (cid != -1) 2414 cnic_free_bnx2x_conn_resc(dev, l5_cid); 2415 2416 memset(&kcqe, 0, sizeof(kcqe)); 2417 kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN; 2418 kcqe.fcoe_conn_id = req1->fcoe_conn_id; 2419 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE; 2420 2421 cqes[0] = (struct kcqe *) &kcqe; 2422 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2423 return ret; 2424 } 2425 2426 static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe) 2427 { 2428 struct fcoe_kwqe_conn_enable_disable *req; 2429 struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable; 2430 union l5cm_specific_data l5_data; 2431 int ret; 2432 u32 cid, l5_cid; 2433 struct cnic_local *cp = dev->cnic_priv; 2434 2435 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2436 cid = req->context_id; 2437 l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE; 2438 2439 if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) { 2440 netdev_err(dev->netdev, "fcoe_enable size too big\n"); 2441 return -ENOMEM; 2442 } 2443 fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2444 if (!fcoe_enable) 2445 return -ENOMEM; 2446 2447 memset(fcoe_enable, 0, sizeof(*fcoe_enable)); 2448 memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req)); 2449 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid, 2450 FCOE_CONNECTION_TYPE, &l5_data); 2451 return ret; 2452 } 2453 2454 static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe) 2455 { 2456 struct fcoe_kwqe_conn_enable_disable *req; 2457 struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable; 2458 union l5cm_specific_data l5_data; 2459 int ret; 2460 u32 cid, l5_cid; 2461 struct cnic_local *cp = dev->cnic_priv; 2462 2463 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2464 cid = req->context_id; 2465 l5_cid = req->conn_id; 2466 if (l5_cid >= dev->max_fcoe_conn) 2467 return -EINVAL; 2468 2469 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2470 2471 if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) { 2472 netdev_err(dev->netdev, "fcoe_disable size too big\n"); 2473 return -ENOMEM; 2474 } 2475 fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data); 2476 if (!fcoe_disable) 2477 return -ENOMEM; 2478 2479 memset(fcoe_disable, 0, sizeof(*fcoe_disable)); 2480 memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req)); 2481 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid, 2482 FCOE_CONNECTION_TYPE, &l5_data); 2483 return ret; 2484 } 2485 2486 static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2487 { 2488 struct fcoe_kwqe_conn_destroy *req; 2489 union l5cm_specific_data l5_data; 2490 int ret; 2491 u32 cid, l5_cid; 2492 struct cnic_local *cp = dev->cnic_priv; 2493 struct cnic_context *ctx; 2494 struct fcoe_kcqe kcqe; 2495 struct kcqe *cqes[1]; 2496 2497 req = (struct fcoe_kwqe_conn_destroy *) kwqe; 2498 cid = req->context_id; 2499 l5_cid = req->conn_id; 2500 if (l5_cid >= dev->max_fcoe_conn) 2501 return -EINVAL; 2502 2503 l5_cid += BNX2X_FCOE_L5_CID_BASE; 2504 2505 ctx = &cp->ctx_tbl[l5_cid]; 2506 2507 init_waitqueue_head(&ctx->waitq); 2508 ctx->wait_cond = 0; 2509 2510 memset(&kcqe, 0, sizeof(kcqe)); 2511 kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR; 2512 memset(&l5_data, 0, sizeof(l5_data)); 2513 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid, 2514 FCOE_CONNECTION_TYPE, &l5_data); 2515 if (ret == 0) { 2516 wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO); 2517 if (ctx->wait_cond) 2518 kcqe.completion_status = 0; 2519 } 2520 2521 set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags); 2522 queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000)); 2523 2524 kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN; 2525 kcqe.fcoe_conn_id = req->conn_id; 2526 kcqe.fcoe_conn_context_id = cid; 2527 2528 cqes[0] = (struct kcqe *) &kcqe; 2529 cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1); 2530 return ret; 2531 } 2532 2533 static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid) 2534 { 2535 struct cnic_local *cp = dev->cnic_priv; 2536 u32 i; 2537 2538 for (i = start_cid; i < cp->max_cid_space; i++) { 2539 struct cnic_context *ctx = &cp->ctx_tbl[i]; 2540 int j; 2541 2542 while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 2543 msleep(10); 2544 2545 for (j = 0; j < 5; j++) { 2546 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2547 break; 2548 msleep(20); 2549 } 2550 2551 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 2552 netdev_warn(dev->netdev, "CID %x not deleted\n", 2553 ctx->cid); 2554 } 2555 } 2556 2557 static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe) 2558 { 2559 struct fcoe_kwqe_destroy *req; 2560 union l5cm_specific_data l5_data; 2561 struct cnic_local *cp = dev->cnic_priv; 2562 struct bnx2x *bp = netdev_priv(dev->netdev); 2563 int ret; 2564 u32 cid; 2565 2566 cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ); 2567 2568 req = (struct fcoe_kwqe_destroy *) kwqe; 2569 cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid); 2570 2571 memset(&l5_data, 0, sizeof(l5_data)); 2572 ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid, 2573 FCOE_CONNECTION_TYPE, &l5_data); 2574 return ret; 2575 } 2576 2577 static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe) 2578 { 2579 struct cnic_local *cp = dev->cnic_priv; 2580 struct kcqe kcqe; 2581 struct kcqe *cqes[1]; 2582 u32 cid; 2583 u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2584 u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK; 2585 u32 kcqe_op; 2586 int ulp_type; 2587 2588 cid = kwqe->kwqe_info0; 2589 memset(&kcqe, 0, sizeof(kcqe)); 2590 2591 if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) { 2592 u32 l5_cid = 0; 2593 2594 ulp_type = CNIC_ULP_FCOE; 2595 if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) { 2596 struct fcoe_kwqe_conn_enable_disable *req; 2597 2598 req = (struct fcoe_kwqe_conn_enable_disable *) kwqe; 2599 kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN; 2600 cid = req->context_id; 2601 l5_cid = req->conn_id; 2602 } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) { 2603 kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC; 2604 } else { 2605 return; 2606 } 2607 kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT; 2608 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE; 2609 kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2610 kcqe.kcqe_info2 = cid; 2611 kcqe.kcqe_info0 = l5_cid; 2612 2613 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) { 2614 ulp_type = CNIC_ULP_ISCSI; 2615 if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN) 2616 cid = kwqe->kwqe_info1; 2617 2618 kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT; 2619 kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI; 2620 kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR; 2621 kcqe.kcqe_info2 = cid; 2622 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0); 2623 2624 } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) { 2625 struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe; 2626 2627 ulp_type = CNIC_ULP_L4; 2628 if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1) 2629 kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE; 2630 else if (opcode == L4_KWQE_OPCODE_VALUE_RESET) 2631 kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP; 2632 else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE) 2633 kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 2634 else 2635 return; 2636 2637 kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) | 2638 KCQE_FLAGS_LAYER_MASK_L4; 2639 l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR; 2640 l4kcqe->cid = cid; 2641 cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id); 2642 } else { 2643 return; 2644 } 2645 2646 cqes[0] = &kcqe; 2647 cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1); 2648 } 2649 2650 static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev, 2651 struct kwqe *wqes[], u32 num_wqes) 2652 { 2653 int i, work, ret; 2654 u32 opcode; 2655 struct kwqe *kwqe; 2656 2657 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2658 return -EAGAIN; /* bnx2 is down */ 2659 2660 for (i = 0; i < num_wqes; ) { 2661 kwqe = wqes[i]; 2662 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2663 work = 1; 2664 2665 switch (opcode) { 2666 case ISCSI_KWQE_OPCODE_INIT1: 2667 ret = cnic_bnx2x_iscsi_init1(dev, kwqe); 2668 break; 2669 case ISCSI_KWQE_OPCODE_INIT2: 2670 ret = cnic_bnx2x_iscsi_init2(dev, kwqe); 2671 break; 2672 case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1: 2673 ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i], 2674 num_wqes - i, &work); 2675 break; 2676 case ISCSI_KWQE_OPCODE_UPDATE_CONN: 2677 ret = cnic_bnx2x_iscsi_update(dev, kwqe); 2678 break; 2679 case ISCSI_KWQE_OPCODE_DESTROY_CONN: 2680 ret = cnic_bnx2x_iscsi_destroy(dev, kwqe); 2681 break; 2682 case L4_KWQE_OPCODE_VALUE_CONNECT1: 2683 ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i, 2684 &work); 2685 break; 2686 case L4_KWQE_OPCODE_VALUE_CLOSE: 2687 ret = cnic_bnx2x_close(dev, kwqe); 2688 break; 2689 case L4_KWQE_OPCODE_VALUE_RESET: 2690 ret = cnic_bnx2x_reset(dev, kwqe); 2691 break; 2692 case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG: 2693 ret = cnic_bnx2x_offload_pg(dev, kwqe); 2694 break; 2695 case L4_KWQE_OPCODE_VALUE_UPDATE_PG: 2696 ret = cnic_bnx2x_update_pg(dev, kwqe); 2697 break; 2698 case L4_KWQE_OPCODE_VALUE_UPLOAD_PG: 2699 ret = 0; 2700 break; 2701 default: 2702 ret = 0; 2703 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2704 opcode); 2705 break; 2706 } 2707 if (ret < 0) { 2708 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2709 opcode); 2710 2711 /* Possibly bnx2x parity error, send completion 2712 * to ulp drivers with error code to speed up 2713 * cleanup and reset recovery. 2714 */ 2715 if (ret == -EIO || ret == -EAGAIN) 2716 cnic_bnx2x_kwqe_err(dev, kwqe); 2717 } 2718 i += work; 2719 } 2720 return 0; 2721 } 2722 2723 static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev, 2724 struct kwqe *wqes[], u32 num_wqes) 2725 { 2726 struct bnx2x *bp = netdev_priv(dev->netdev); 2727 int i, work, ret; 2728 u32 opcode; 2729 struct kwqe *kwqe; 2730 2731 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2732 return -EAGAIN; /* bnx2 is down */ 2733 2734 if (!BNX2X_CHIP_IS_E2_PLUS(bp)) 2735 return -EINVAL; 2736 2737 for (i = 0; i < num_wqes; ) { 2738 kwqe = wqes[i]; 2739 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag); 2740 work = 1; 2741 2742 switch (opcode) { 2743 case FCOE_KWQE_OPCODE_INIT1: 2744 ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i], 2745 num_wqes - i, &work); 2746 break; 2747 case FCOE_KWQE_OPCODE_OFFLOAD_CONN1: 2748 ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i], 2749 num_wqes - i, &work); 2750 break; 2751 case FCOE_KWQE_OPCODE_ENABLE_CONN: 2752 ret = cnic_bnx2x_fcoe_enable(dev, kwqe); 2753 break; 2754 case FCOE_KWQE_OPCODE_DISABLE_CONN: 2755 ret = cnic_bnx2x_fcoe_disable(dev, kwqe); 2756 break; 2757 case FCOE_KWQE_OPCODE_DESTROY_CONN: 2758 ret = cnic_bnx2x_fcoe_destroy(dev, kwqe); 2759 break; 2760 case FCOE_KWQE_OPCODE_DESTROY: 2761 ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe); 2762 break; 2763 case FCOE_KWQE_OPCODE_STAT: 2764 ret = cnic_bnx2x_fcoe_stat(dev, kwqe); 2765 break; 2766 default: 2767 ret = 0; 2768 netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n", 2769 opcode); 2770 break; 2771 } 2772 if (ret < 0) { 2773 netdev_err(dev->netdev, "KWQE(0x%x) failed\n", 2774 opcode); 2775 2776 /* Possibly bnx2x parity error, send completion 2777 * to ulp drivers with error code to speed up 2778 * cleanup and reset recovery. 2779 */ 2780 if (ret == -EIO || ret == -EAGAIN) 2781 cnic_bnx2x_kwqe_err(dev, kwqe); 2782 } 2783 i += work; 2784 } 2785 return 0; 2786 } 2787 2788 static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[], 2789 u32 num_wqes) 2790 { 2791 int ret = -EINVAL; 2792 u32 layer_code; 2793 2794 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 2795 return -EAGAIN; /* bnx2x is down */ 2796 2797 if (!num_wqes) 2798 return 0; 2799 2800 layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK; 2801 switch (layer_code) { 2802 case KWQE_FLAGS_LAYER_MASK_L5_ISCSI: 2803 case KWQE_FLAGS_LAYER_MASK_L4: 2804 case KWQE_FLAGS_LAYER_MASK_L2: 2805 ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes); 2806 break; 2807 2808 case KWQE_FLAGS_LAYER_MASK_L5_FCOE: 2809 ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes); 2810 break; 2811 } 2812 return ret; 2813 } 2814 2815 static inline u32 cnic_get_kcqe_layer_mask(u32 opflag) 2816 { 2817 if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN)) 2818 return KCQE_FLAGS_LAYER_MASK_L4; 2819 2820 return opflag & KCQE_FLAGS_LAYER_MASK; 2821 } 2822 2823 static void service_kcqes(struct cnic_dev *dev, int num_cqes) 2824 { 2825 struct cnic_local *cp = dev->cnic_priv; 2826 int i, j, comp = 0; 2827 2828 i = 0; 2829 j = 1; 2830 while (num_cqes) { 2831 struct cnic_ulp_ops *ulp_ops; 2832 int ulp_type; 2833 u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag; 2834 u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag); 2835 2836 if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION)) 2837 comp++; 2838 2839 while (j < num_cqes) { 2840 u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag; 2841 2842 if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer) 2843 break; 2844 2845 if (unlikely(next_op & KCQE_RAMROD_COMPLETION)) 2846 comp++; 2847 j++; 2848 } 2849 2850 if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA) 2851 ulp_type = CNIC_ULP_RDMA; 2852 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI) 2853 ulp_type = CNIC_ULP_ISCSI; 2854 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE) 2855 ulp_type = CNIC_ULP_FCOE; 2856 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4) 2857 ulp_type = CNIC_ULP_L4; 2858 else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2) 2859 goto end; 2860 else { 2861 netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n", 2862 kcqe_op_flag); 2863 goto end; 2864 } 2865 2866 rcu_read_lock(); 2867 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 2868 if (likely(ulp_ops)) { 2869 ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type], 2870 cp->completed_kcq + i, j); 2871 } 2872 rcu_read_unlock(); 2873 end: 2874 num_cqes -= j; 2875 i += j; 2876 j = 1; 2877 } 2878 if (unlikely(comp)) 2879 cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp); 2880 } 2881 2882 static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info) 2883 { 2884 struct cnic_local *cp = dev->cnic_priv; 2885 u16 i, ri, hw_prod, last; 2886 struct kcqe *kcqe; 2887 int kcqe_cnt = 0, last_cnt = 0; 2888 2889 i = ri = last = info->sw_prod_idx; 2890 ri &= MAX_KCQ_IDX; 2891 hw_prod = *info->hw_prod_idx_ptr; 2892 hw_prod = info->hw_idx(hw_prod); 2893 2894 while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) { 2895 kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)]; 2896 cp->completed_kcq[kcqe_cnt++] = kcqe; 2897 i = info->next_idx(i); 2898 ri = i & MAX_KCQ_IDX; 2899 if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) { 2900 last_cnt = kcqe_cnt; 2901 last = i; 2902 } 2903 } 2904 2905 info->sw_prod_idx = last; 2906 return last_cnt; 2907 } 2908 2909 static int cnic_l2_completion(struct cnic_local *cp) 2910 { 2911 u16 hw_cons, sw_cons; 2912 struct cnic_uio_dev *udev = cp->udev; 2913 union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *) 2914 (udev->l2_ring + (2 * BNX2_PAGE_SIZE)); 2915 u32 cmd; 2916 int comp = 0; 2917 2918 if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags)) 2919 return 0; 2920 2921 hw_cons = *cp->rx_cons_ptr; 2922 if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT) 2923 hw_cons++; 2924 2925 sw_cons = cp->rx_cons; 2926 while (sw_cons != hw_cons) { 2927 u8 cqe_fp_flags; 2928 2929 cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT]; 2930 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 2931 if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) { 2932 cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data); 2933 cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT; 2934 if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP || 2935 cmd == RAMROD_CMD_ID_ETH_HALT) 2936 comp++; 2937 } 2938 sw_cons = BNX2X_NEXT_RCQE(sw_cons); 2939 } 2940 return comp; 2941 } 2942 2943 static void cnic_chk_pkt_rings(struct cnic_local *cp) 2944 { 2945 u16 rx_cons, tx_cons; 2946 int comp = 0; 2947 2948 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 2949 return; 2950 2951 rx_cons = *cp->rx_cons_ptr; 2952 tx_cons = *cp->tx_cons_ptr; 2953 if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) { 2954 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 2955 comp = cnic_l2_completion(cp); 2956 2957 cp->tx_cons = tx_cons; 2958 cp->rx_cons = rx_cons; 2959 2960 if (cp->udev) 2961 uio_event_notify(&cp->udev->cnic_uinfo); 2962 } 2963 if (comp) 2964 clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 2965 } 2966 2967 static u32 cnic_service_bnx2_queues(struct cnic_dev *dev) 2968 { 2969 struct cnic_local *cp = dev->cnic_priv; 2970 u32 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2971 int kcqe_cnt; 2972 2973 /* status block index must be read before reading other fields */ 2974 rmb(); 2975 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2976 2977 while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) { 2978 2979 service_kcqes(dev, kcqe_cnt); 2980 2981 /* Tell compiler that status_blk fields can change. */ 2982 barrier(); 2983 status_idx = (u16) *cp->kcq1.status_idx_ptr; 2984 /* status block index must be read first */ 2985 rmb(); 2986 cp->kwq_con_idx = *cp->kwq_con_idx_ptr; 2987 } 2988 2989 CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx); 2990 2991 cnic_chk_pkt_rings(cp); 2992 2993 return status_idx; 2994 } 2995 2996 static int cnic_service_bnx2(void *data, void *status_blk) 2997 { 2998 struct cnic_dev *dev = data; 2999 3000 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3001 struct status_block *sblk = status_blk; 3002 3003 return sblk->status_idx; 3004 } 3005 3006 return cnic_service_bnx2_queues(dev); 3007 } 3008 3009 static void cnic_service_bnx2_msix(unsigned long data) 3010 { 3011 struct cnic_dev *dev = (struct cnic_dev *) data; 3012 struct cnic_local *cp = dev->cnic_priv; 3013 3014 cp->last_status_idx = cnic_service_bnx2_queues(dev); 3015 3016 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 3017 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 3018 } 3019 3020 static void cnic_doirq(struct cnic_dev *dev) 3021 { 3022 struct cnic_local *cp = dev->cnic_priv; 3023 3024 if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) { 3025 u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX; 3026 3027 prefetch(cp->status_blk.gen); 3028 prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]); 3029 3030 tasklet_schedule(&cp->cnic_irq_task); 3031 } 3032 } 3033 3034 static irqreturn_t cnic_irq(int irq, void *dev_instance) 3035 { 3036 struct cnic_dev *dev = dev_instance; 3037 struct cnic_local *cp = dev->cnic_priv; 3038 3039 if (cp->ack_int) 3040 cp->ack_int(dev); 3041 3042 cnic_doirq(dev); 3043 3044 return IRQ_HANDLED; 3045 } 3046 3047 static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm, 3048 u16 index, u8 op, u8 update) 3049 { 3050 struct bnx2x *bp = netdev_priv(dev->netdev); 3051 u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp) * 32 + 3052 COMMAND_REG_INT_ACK); 3053 struct igu_ack_register igu_ack; 3054 3055 igu_ack.status_block_index = index; 3056 igu_ack.sb_id_and_flags = 3057 ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) | 3058 (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) | 3059 (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) | 3060 (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT)); 3061 3062 CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack)); 3063 } 3064 3065 static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment, 3066 u16 index, u8 op, u8 update) 3067 { 3068 struct igu_regular cmd_data; 3069 u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8; 3070 3071 cmd_data.sb_id_and_flags = 3072 (index << IGU_REGULAR_SB_INDEX_SHIFT) | 3073 (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) | 3074 (update << IGU_REGULAR_BUPDATE_SHIFT) | 3075 (op << IGU_REGULAR_ENABLE_INT_SHIFT); 3076 3077 3078 CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags); 3079 } 3080 3081 static void cnic_ack_bnx2x_msix(struct cnic_dev *dev) 3082 { 3083 struct cnic_local *cp = dev->cnic_priv; 3084 3085 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0, 3086 IGU_INT_DISABLE, 0); 3087 } 3088 3089 static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev) 3090 { 3091 struct cnic_local *cp = dev->cnic_priv; 3092 3093 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0, 3094 IGU_INT_DISABLE, 0); 3095 } 3096 3097 static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx) 3098 { 3099 struct cnic_local *cp = dev->cnic_priv; 3100 3101 cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx, 3102 IGU_INT_ENABLE, 1); 3103 } 3104 3105 static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx) 3106 { 3107 struct cnic_local *cp = dev->cnic_priv; 3108 3109 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx, 3110 IGU_INT_ENABLE, 1); 3111 } 3112 3113 static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info) 3114 { 3115 u32 last_status = *info->status_idx_ptr; 3116 int kcqe_cnt; 3117 3118 /* status block index must be read before reading the KCQ */ 3119 rmb(); 3120 while ((kcqe_cnt = cnic_get_kcqes(dev, info))) { 3121 3122 service_kcqes(dev, kcqe_cnt); 3123 3124 /* Tell compiler that sblk fields can change. */ 3125 barrier(); 3126 3127 last_status = *info->status_idx_ptr; 3128 /* status block index must be read before reading the KCQ */ 3129 rmb(); 3130 } 3131 return last_status; 3132 } 3133 3134 static void cnic_service_bnx2x_bh(unsigned long data) 3135 { 3136 struct cnic_dev *dev = (struct cnic_dev *) data; 3137 struct cnic_local *cp = dev->cnic_priv; 3138 struct bnx2x *bp = netdev_priv(dev->netdev); 3139 u32 status_idx, new_status_idx; 3140 3141 if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) 3142 return; 3143 3144 while (1) { 3145 status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1); 3146 3147 CNIC_WR16(dev, cp->kcq1.io_addr, 3148 cp->kcq1.sw_prod_idx + MAX_KCQ_IDX); 3149 3150 if (!CNIC_SUPPORTS_FCOE(bp)) { 3151 cp->arm_int(dev, status_idx); 3152 break; 3153 } 3154 3155 new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2); 3156 3157 if (new_status_idx != status_idx) 3158 continue; 3159 3160 CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx + 3161 MAX_KCQ_IDX); 3162 3163 cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 3164 status_idx, IGU_INT_ENABLE, 1); 3165 3166 break; 3167 } 3168 } 3169 3170 static int cnic_service_bnx2x(void *data, void *status_blk) 3171 { 3172 struct cnic_dev *dev = data; 3173 struct cnic_local *cp = dev->cnic_priv; 3174 3175 if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 3176 cnic_doirq(dev); 3177 3178 cnic_chk_pkt_rings(cp); 3179 3180 return 0; 3181 } 3182 3183 static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type) 3184 { 3185 struct cnic_ulp_ops *ulp_ops; 3186 3187 if (if_type == CNIC_ULP_ISCSI) 3188 cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL); 3189 3190 mutex_lock(&cnic_lock); 3191 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3192 lockdep_is_held(&cnic_lock)); 3193 if (!ulp_ops) { 3194 mutex_unlock(&cnic_lock); 3195 return; 3196 } 3197 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3198 mutex_unlock(&cnic_lock); 3199 3200 if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3201 ulp_ops->cnic_stop(cp->ulp_handle[if_type]); 3202 3203 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3204 } 3205 3206 static void cnic_ulp_stop(struct cnic_dev *dev) 3207 { 3208 struct cnic_local *cp = dev->cnic_priv; 3209 int if_type; 3210 3211 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) 3212 cnic_ulp_stop_one(cp, if_type); 3213 } 3214 3215 static void cnic_ulp_start(struct cnic_dev *dev) 3216 { 3217 struct cnic_local *cp = dev->cnic_priv; 3218 int if_type; 3219 3220 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 3221 struct cnic_ulp_ops *ulp_ops; 3222 3223 mutex_lock(&cnic_lock); 3224 ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type], 3225 lockdep_is_held(&cnic_lock)); 3226 if (!ulp_ops || !ulp_ops->cnic_start) { 3227 mutex_unlock(&cnic_lock); 3228 continue; 3229 } 3230 set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3231 mutex_unlock(&cnic_lock); 3232 3233 if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type])) 3234 ulp_ops->cnic_start(cp->ulp_handle[if_type]); 3235 3236 clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]); 3237 } 3238 } 3239 3240 static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type) 3241 { 3242 struct cnic_local *cp = dev->cnic_priv; 3243 struct cnic_ulp_ops *ulp_ops; 3244 int rc; 3245 3246 mutex_lock(&cnic_lock); 3247 ulp_ops = cnic_ulp_tbl_prot(ulp_type); 3248 if (ulp_ops && ulp_ops->cnic_get_stats) 3249 rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]); 3250 else 3251 rc = -ENODEV; 3252 mutex_unlock(&cnic_lock); 3253 return rc; 3254 } 3255 3256 static int cnic_ctl(void *data, struct cnic_ctl_info *info) 3257 { 3258 struct cnic_dev *dev = data; 3259 int ulp_type = CNIC_ULP_ISCSI; 3260 3261 switch (info->cmd) { 3262 case CNIC_CTL_STOP_CMD: 3263 cnic_hold(dev); 3264 3265 cnic_ulp_stop(dev); 3266 cnic_stop_hw(dev); 3267 3268 cnic_put(dev); 3269 break; 3270 case CNIC_CTL_START_CMD: 3271 cnic_hold(dev); 3272 3273 if (!cnic_start_hw(dev)) 3274 cnic_ulp_start(dev); 3275 3276 cnic_put(dev); 3277 break; 3278 case CNIC_CTL_STOP_ISCSI_CMD: { 3279 struct cnic_local *cp = dev->cnic_priv; 3280 set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags); 3281 queue_delayed_work(cnic_wq, &cp->delete_task, 0); 3282 break; 3283 } 3284 case CNIC_CTL_COMPLETION_CMD: { 3285 struct cnic_ctl_completion *comp = &info->data.comp; 3286 u32 cid = BNX2X_SW_CID(comp->cid); 3287 u32 l5_cid; 3288 struct cnic_local *cp = dev->cnic_priv; 3289 3290 if (!test_bit(CNIC_F_CNIC_UP, &dev->flags)) 3291 break; 3292 3293 if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) { 3294 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3295 3296 if (unlikely(comp->error)) { 3297 set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags); 3298 netdev_err(dev->netdev, 3299 "CID %x CFC delete comp error %x\n", 3300 cid, comp->error); 3301 } 3302 3303 ctx->wait_cond = 1; 3304 wake_up(&ctx->waitq); 3305 } 3306 break; 3307 } 3308 case CNIC_CTL_FCOE_STATS_GET_CMD: 3309 ulp_type = CNIC_ULP_FCOE; 3310 /* fall through */ 3311 case CNIC_CTL_ISCSI_STATS_GET_CMD: 3312 cnic_hold(dev); 3313 cnic_copy_ulp_stats(dev, ulp_type); 3314 cnic_put(dev); 3315 break; 3316 3317 default: 3318 return -EINVAL; 3319 } 3320 return 0; 3321 } 3322 3323 static void cnic_ulp_init(struct cnic_dev *dev) 3324 { 3325 int i; 3326 struct cnic_local *cp = dev->cnic_priv; 3327 3328 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3329 struct cnic_ulp_ops *ulp_ops; 3330 3331 mutex_lock(&cnic_lock); 3332 ulp_ops = cnic_ulp_tbl_prot(i); 3333 if (!ulp_ops || !ulp_ops->cnic_init) { 3334 mutex_unlock(&cnic_lock); 3335 continue; 3336 } 3337 ulp_get(ulp_ops); 3338 mutex_unlock(&cnic_lock); 3339 3340 if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3341 ulp_ops->cnic_init(dev); 3342 3343 ulp_put(ulp_ops); 3344 } 3345 } 3346 3347 static void cnic_ulp_exit(struct cnic_dev *dev) 3348 { 3349 int i; 3350 struct cnic_local *cp = dev->cnic_priv; 3351 3352 for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) { 3353 struct cnic_ulp_ops *ulp_ops; 3354 3355 mutex_lock(&cnic_lock); 3356 ulp_ops = cnic_ulp_tbl_prot(i); 3357 if (!ulp_ops || !ulp_ops->cnic_exit) { 3358 mutex_unlock(&cnic_lock); 3359 continue; 3360 } 3361 ulp_get(ulp_ops); 3362 mutex_unlock(&cnic_lock); 3363 3364 if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i])) 3365 ulp_ops->cnic_exit(dev); 3366 3367 ulp_put(ulp_ops); 3368 } 3369 } 3370 3371 static int cnic_cm_offload_pg(struct cnic_sock *csk) 3372 { 3373 struct cnic_dev *dev = csk->dev; 3374 struct l4_kwq_offload_pg *l4kwqe; 3375 struct kwqe *wqes[1]; 3376 3377 l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1; 3378 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3379 wqes[0] = (struct kwqe *) l4kwqe; 3380 3381 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG; 3382 l4kwqe->flags = 3383 L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT; 3384 l4kwqe->l2hdr_nbytes = ETH_HLEN; 3385 3386 l4kwqe->da0 = csk->ha[0]; 3387 l4kwqe->da1 = csk->ha[1]; 3388 l4kwqe->da2 = csk->ha[2]; 3389 l4kwqe->da3 = csk->ha[3]; 3390 l4kwqe->da4 = csk->ha[4]; 3391 l4kwqe->da5 = csk->ha[5]; 3392 3393 l4kwqe->sa0 = dev->mac_addr[0]; 3394 l4kwqe->sa1 = dev->mac_addr[1]; 3395 l4kwqe->sa2 = dev->mac_addr[2]; 3396 l4kwqe->sa3 = dev->mac_addr[3]; 3397 l4kwqe->sa4 = dev->mac_addr[4]; 3398 l4kwqe->sa5 = dev->mac_addr[5]; 3399 3400 l4kwqe->etype = ETH_P_IP; 3401 l4kwqe->ipid_start = DEF_IPID_START; 3402 l4kwqe->host_opaque = csk->l5_cid; 3403 3404 if (csk->vlan_id) { 3405 l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING; 3406 l4kwqe->vlan_tag = csk->vlan_id; 3407 l4kwqe->l2hdr_nbytes += 4; 3408 } 3409 3410 return dev->submit_kwqes(dev, wqes, 1); 3411 } 3412 3413 static int cnic_cm_update_pg(struct cnic_sock *csk) 3414 { 3415 struct cnic_dev *dev = csk->dev; 3416 struct l4_kwq_update_pg *l4kwqe; 3417 struct kwqe *wqes[1]; 3418 3419 l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1; 3420 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3421 wqes[0] = (struct kwqe *) l4kwqe; 3422 3423 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG; 3424 l4kwqe->flags = 3425 L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT; 3426 l4kwqe->pg_cid = csk->pg_cid; 3427 3428 l4kwqe->da0 = csk->ha[0]; 3429 l4kwqe->da1 = csk->ha[1]; 3430 l4kwqe->da2 = csk->ha[2]; 3431 l4kwqe->da3 = csk->ha[3]; 3432 l4kwqe->da4 = csk->ha[4]; 3433 l4kwqe->da5 = csk->ha[5]; 3434 3435 l4kwqe->pg_host_opaque = csk->l5_cid; 3436 l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA; 3437 3438 return dev->submit_kwqes(dev, wqes, 1); 3439 } 3440 3441 static int cnic_cm_upload_pg(struct cnic_sock *csk) 3442 { 3443 struct cnic_dev *dev = csk->dev; 3444 struct l4_kwq_upload *l4kwqe; 3445 struct kwqe *wqes[1]; 3446 3447 l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1; 3448 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3449 wqes[0] = (struct kwqe *) l4kwqe; 3450 3451 l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG; 3452 l4kwqe->flags = 3453 L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT; 3454 l4kwqe->cid = csk->pg_cid; 3455 3456 return dev->submit_kwqes(dev, wqes, 1); 3457 } 3458 3459 static int cnic_cm_conn_req(struct cnic_sock *csk) 3460 { 3461 struct cnic_dev *dev = csk->dev; 3462 struct l4_kwq_connect_req1 *l4kwqe1; 3463 struct l4_kwq_connect_req2 *l4kwqe2; 3464 struct l4_kwq_connect_req3 *l4kwqe3; 3465 struct kwqe *wqes[3]; 3466 u8 tcp_flags = 0; 3467 int num_wqes = 2; 3468 3469 l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1; 3470 l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2; 3471 l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3; 3472 memset(l4kwqe1, 0, sizeof(*l4kwqe1)); 3473 memset(l4kwqe2, 0, sizeof(*l4kwqe2)); 3474 memset(l4kwqe3, 0, sizeof(*l4kwqe3)); 3475 3476 l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3; 3477 l4kwqe3->flags = 3478 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT; 3479 l4kwqe3->ka_timeout = csk->ka_timeout; 3480 l4kwqe3->ka_interval = csk->ka_interval; 3481 l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count; 3482 l4kwqe3->tos = csk->tos; 3483 l4kwqe3->ttl = csk->ttl; 3484 l4kwqe3->snd_seq_scale = csk->snd_seq_scale; 3485 l4kwqe3->pmtu = csk->mtu; 3486 l4kwqe3->rcv_buf = csk->rcv_buf; 3487 l4kwqe3->snd_buf = csk->snd_buf; 3488 l4kwqe3->seed = csk->seed; 3489 3490 wqes[0] = (struct kwqe *) l4kwqe1; 3491 if (test_bit(SK_F_IPV6, &csk->flags)) { 3492 wqes[1] = (struct kwqe *) l4kwqe2; 3493 wqes[2] = (struct kwqe *) l4kwqe3; 3494 num_wqes = 3; 3495 3496 l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6; 3497 l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2; 3498 l4kwqe2->flags = 3499 L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT | 3500 L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT; 3501 l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]); 3502 l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]); 3503 l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]); 3504 l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]); 3505 l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]); 3506 l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]); 3507 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) - 3508 sizeof(struct tcphdr); 3509 } else { 3510 wqes[1] = (struct kwqe *) l4kwqe3; 3511 l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) - 3512 sizeof(struct tcphdr); 3513 } 3514 3515 l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1; 3516 l4kwqe1->flags = 3517 (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) | 3518 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT; 3519 l4kwqe1->cid = csk->cid; 3520 l4kwqe1->pg_cid = csk->pg_cid; 3521 l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]); 3522 l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]); 3523 l4kwqe1->src_port = be16_to_cpu(csk->src_port); 3524 l4kwqe1->dst_port = be16_to_cpu(csk->dst_port); 3525 if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK) 3526 tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK; 3527 if (csk->tcp_flags & SK_TCP_KEEP_ALIVE) 3528 tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE; 3529 if (csk->tcp_flags & SK_TCP_NAGLE) 3530 tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE; 3531 if (csk->tcp_flags & SK_TCP_TIMESTAMP) 3532 tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP; 3533 if (csk->tcp_flags & SK_TCP_SACK) 3534 tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK; 3535 if (csk->tcp_flags & SK_TCP_SEG_SCALING) 3536 tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING; 3537 3538 l4kwqe1->tcp_flags = tcp_flags; 3539 3540 return dev->submit_kwqes(dev, wqes, num_wqes); 3541 } 3542 3543 static int cnic_cm_close_req(struct cnic_sock *csk) 3544 { 3545 struct cnic_dev *dev = csk->dev; 3546 struct l4_kwq_close_req *l4kwqe; 3547 struct kwqe *wqes[1]; 3548 3549 l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2; 3550 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3551 wqes[0] = (struct kwqe *) l4kwqe; 3552 3553 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE; 3554 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT; 3555 l4kwqe->cid = csk->cid; 3556 3557 return dev->submit_kwqes(dev, wqes, 1); 3558 } 3559 3560 static int cnic_cm_abort_req(struct cnic_sock *csk) 3561 { 3562 struct cnic_dev *dev = csk->dev; 3563 struct l4_kwq_reset_req *l4kwqe; 3564 struct kwqe *wqes[1]; 3565 3566 l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2; 3567 memset(l4kwqe, 0, sizeof(*l4kwqe)); 3568 wqes[0] = (struct kwqe *) l4kwqe; 3569 3570 l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET; 3571 l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT; 3572 l4kwqe->cid = csk->cid; 3573 3574 return dev->submit_kwqes(dev, wqes, 1); 3575 } 3576 3577 static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid, 3578 u32 l5_cid, struct cnic_sock **csk, void *context) 3579 { 3580 struct cnic_local *cp = dev->cnic_priv; 3581 struct cnic_sock *csk1; 3582 3583 if (l5_cid >= MAX_CM_SK_TBL_SZ) 3584 return -EINVAL; 3585 3586 if (cp->ctx_tbl) { 3587 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3588 3589 if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) 3590 return -EAGAIN; 3591 } 3592 3593 csk1 = &cp->csk_tbl[l5_cid]; 3594 if (atomic_read(&csk1->ref_count)) 3595 return -EAGAIN; 3596 3597 if (test_and_set_bit(SK_F_INUSE, &csk1->flags)) 3598 return -EBUSY; 3599 3600 csk1->dev = dev; 3601 csk1->cid = cid; 3602 csk1->l5_cid = l5_cid; 3603 csk1->ulp_type = ulp_type; 3604 csk1->context = context; 3605 3606 csk1->ka_timeout = DEF_KA_TIMEOUT; 3607 csk1->ka_interval = DEF_KA_INTERVAL; 3608 csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT; 3609 csk1->tos = DEF_TOS; 3610 csk1->ttl = DEF_TTL; 3611 csk1->snd_seq_scale = DEF_SND_SEQ_SCALE; 3612 csk1->rcv_buf = DEF_RCV_BUF; 3613 csk1->snd_buf = DEF_SND_BUF; 3614 csk1->seed = DEF_SEED; 3615 csk1->tcp_flags = 0; 3616 3617 *csk = csk1; 3618 return 0; 3619 } 3620 3621 static void cnic_cm_cleanup(struct cnic_sock *csk) 3622 { 3623 if (csk->src_port) { 3624 struct cnic_dev *dev = csk->dev; 3625 struct cnic_local *cp = dev->cnic_priv; 3626 3627 cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port)); 3628 csk->src_port = 0; 3629 } 3630 } 3631 3632 static void cnic_close_conn(struct cnic_sock *csk) 3633 { 3634 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) { 3635 cnic_cm_upload_pg(csk); 3636 clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3637 } 3638 cnic_cm_cleanup(csk); 3639 } 3640 3641 static int cnic_cm_destroy(struct cnic_sock *csk) 3642 { 3643 if (!cnic_in_use(csk)) 3644 return -EINVAL; 3645 3646 csk_hold(csk); 3647 clear_bit(SK_F_INUSE, &csk->flags); 3648 smp_mb__after_clear_bit(); 3649 while (atomic_read(&csk->ref_count) != 1) 3650 msleep(1); 3651 cnic_cm_cleanup(csk); 3652 3653 csk->flags = 0; 3654 csk_put(csk); 3655 return 0; 3656 } 3657 3658 static inline u16 cnic_get_vlan(struct net_device *dev, 3659 struct net_device **vlan_dev) 3660 { 3661 if (dev->priv_flags & IFF_802_1Q_VLAN) { 3662 *vlan_dev = vlan_dev_real_dev(dev); 3663 return vlan_dev_vlan_id(dev); 3664 } 3665 *vlan_dev = dev; 3666 return 0; 3667 } 3668 3669 static int cnic_get_v4_route(struct sockaddr_in *dst_addr, 3670 struct dst_entry **dst) 3671 { 3672 #if defined(CONFIG_INET) 3673 struct rtable *rt; 3674 3675 rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0); 3676 if (!IS_ERR(rt)) { 3677 *dst = &rt->dst; 3678 return 0; 3679 } 3680 return PTR_ERR(rt); 3681 #else 3682 return -ENETUNREACH; 3683 #endif 3684 } 3685 3686 static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr, 3687 struct dst_entry **dst) 3688 { 3689 #if defined(CONFIG_IPV6) || (defined(CONFIG_IPV6_MODULE) && defined(MODULE)) 3690 struct flowi6 fl6; 3691 3692 memset(&fl6, 0, sizeof(fl6)); 3693 fl6.daddr = dst_addr->sin6_addr; 3694 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 3695 fl6.flowi6_oif = dst_addr->sin6_scope_id; 3696 3697 *dst = ip6_route_output(&init_net, NULL, &fl6); 3698 if ((*dst)->error) { 3699 dst_release(*dst); 3700 *dst = NULL; 3701 return -ENETUNREACH; 3702 } else 3703 return 0; 3704 #endif 3705 3706 return -ENETUNREACH; 3707 } 3708 3709 static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr, 3710 int ulp_type) 3711 { 3712 struct cnic_dev *dev = NULL; 3713 struct dst_entry *dst; 3714 struct net_device *netdev = NULL; 3715 int err = -ENETUNREACH; 3716 3717 if (dst_addr->sin_family == AF_INET) 3718 err = cnic_get_v4_route(dst_addr, &dst); 3719 else if (dst_addr->sin_family == AF_INET6) { 3720 struct sockaddr_in6 *dst_addr6 = 3721 (struct sockaddr_in6 *) dst_addr; 3722 3723 err = cnic_get_v6_route(dst_addr6, &dst); 3724 } else 3725 return NULL; 3726 3727 if (err) 3728 return NULL; 3729 3730 if (!dst->dev) 3731 goto done; 3732 3733 cnic_get_vlan(dst->dev, &netdev); 3734 3735 dev = cnic_from_netdev(netdev); 3736 3737 done: 3738 dst_release(dst); 3739 if (dev) 3740 cnic_put(dev); 3741 return dev; 3742 } 3743 3744 static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3745 { 3746 struct cnic_dev *dev = csk->dev; 3747 struct cnic_local *cp = dev->cnic_priv; 3748 3749 return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk); 3750 } 3751 3752 static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3753 { 3754 struct cnic_dev *dev = csk->dev; 3755 struct cnic_local *cp = dev->cnic_priv; 3756 int is_v6, rc = 0; 3757 struct dst_entry *dst = NULL; 3758 struct net_device *realdev; 3759 __be16 local_port; 3760 u32 port_id; 3761 3762 if (saddr->local.v6.sin6_family == AF_INET6 && 3763 saddr->remote.v6.sin6_family == AF_INET6) 3764 is_v6 = 1; 3765 else if (saddr->local.v4.sin_family == AF_INET && 3766 saddr->remote.v4.sin_family == AF_INET) 3767 is_v6 = 0; 3768 else 3769 return -EINVAL; 3770 3771 clear_bit(SK_F_IPV6, &csk->flags); 3772 3773 if (is_v6) { 3774 set_bit(SK_F_IPV6, &csk->flags); 3775 cnic_get_v6_route(&saddr->remote.v6, &dst); 3776 3777 memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr, 3778 sizeof(struct in6_addr)); 3779 csk->dst_port = saddr->remote.v6.sin6_port; 3780 local_port = saddr->local.v6.sin6_port; 3781 3782 } else { 3783 cnic_get_v4_route(&saddr->remote.v4, &dst); 3784 3785 csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr; 3786 csk->dst_port = saddr->remote.v4.sin_port; 3787 local_port = saddr->local.v4.sin_port; 3788 } 3789 3790 csk->vlan_id = 0; 3791 csk->mtu = dev->netdev->mtu; 3792 if (dst && dst->dev) { 3793 u16 vlan = cnic_get_vlan(dst->dev, &realdev); 3794 if (realdev == dev->netdev) { 3795 csk->vlan_id = vlan; 3796 csk->mtu = dst_mtu(dst); 3797 } 3798 } 3799 3800 port_id = be16_to_cpu(local_port); 3801 if (port_id >= CNIC_LOCAL_PORT_MIN && 3802 port_id < CNIC_LOCAL_PORT_MAX) { 3803 if (cnic_alloc_id(&cp->csk_port_tbl, port_id)) 3804 port_id = 0; 3805 } else 3806 port_id = 0; 3807 3808 if (!port_id) { 3809 port_id = cnic_alloc_new_id(&cp->csk_port_tbl); 3810 if (port_id == -1) { 3811 rc = -ENOMEM; 3812 goto err_out; 3813 } 3814 local_port = cpu_to_be16(port_id); 3815 } 3816 csk->src_port = local_port; 3817 3818 err_out: 3819 dst_release(dst); 3820 return rc; 3821 } 3822 3823 static void cnic_init_csk_state(struct cnic_sock *csk) 3824 { 3825 csk->state = 0; 3826 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3827 clear_bit(SK_F_CLOSING, &csk->flags); 3828 } 3829 3830 static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr) 3831 { 3832 struct cnic_local *cp = csk->dev->cnic_priv; 3833 int err = 0; 3834 3835 if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 3836 return -EOPNOTSUPP; 3837 3838 if (!cnic_in_use(csk)) 3839 return -EINVAL; 3840 3841 if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags)) 3842 return -EINVAL; 3843 3844 cnic_init_csk_state(csk); 3845 3846 err = cnic_get_route(csk, saddr); 3847 if (err) 3848 goto err_out; 3849 3850 err = cnic_resolve_addr(csk, saddr); 3851 if (!err) 3852 return 0; 3853 3854 err_out: 3855 clear_bit(SK_F_CONNECT_START, &csk->flags); 3856 return err; 3857 } 3858 3859 static int cnic_cm_abort(struct cnic_sock *csk) 3860 { 3861 struct cnic_local *cp = csk->dev->cnic_priv; 3862 u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP; 3863 3864 if (!cnic_in_use(csk)) 3865 return -EINVAL; 3866 3867 if (cnic_abort_prep(csk)) 3868 return cnic_cm_abort_req(csk); 3869 3870 /* Getting here means that we haven't started connect, or 3871 * connect was not successful, or it has been reset by the target. 3872 */ 3873 3874 cp->close_conn(csk, opcode); 3875 if (csk->state != opcode) { 3876 /* Wait for remote reset sequence to complete */ 3877 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3878 msleep(1); 3879 3880 return -EALREADY; 3881 } 3882 3883 return 0; 3884 } 3885 3886 static int cnic_cm_close(struct cnic_sock *csk) 3887 { 3888 if (!cnic_in_use(csk)) 3889 return -EINVAL; 3890 3891 if (cnic_close_prep(csk)) { 3892 csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 3893 return cnic_cm_close_req(csk); 3894 } else { 3895 /* Wait for remote reset sequence to complete */ 3896 while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3897 msleep(1); 3898 3899 return -EALREADY; 3900 } 3901 return 0; 3902 } 3903 3904 static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk, 3905 u8 opcode) 3906 { 3907 struct cnic_ulp_ops *ulp_ops; 3908 int ulp_type = csk->ulp_type; 3909 3910 rcu_read_lock(); 3911 ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]); 3912 if (ulp_ops) { 3913 if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE) 3914 ulp_ops->cm_connect_complete(csk); 3915 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 3916 ulp_ops->cm_close_complete(csk); 3917 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) 3918 ulp_ops->cm_remote_abort(csk); 3919 else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP) 3920 ulp_ops->cm_abort_complete(csk); 3921 else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED) 3922 ulp_ops->cm_remote_close(csk); 3923 } 3924 rcu_read_unlock(); 3925 } 3926 3927 static int cnic_cm_set_pg(struct cnic_sock *csk) 3928 { 3929 if (cnic_offld_prep(csk)) { 3930 if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 3931 cnic_cm_update_pg(csk); 3932 else 3933 cnic_cm_offload_pg(csk); 3934 } 3935 return 0; 3936 } 3937 3938 static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe) 3939 { 3940 struct cnic_local *cp = dev->cnic_priv; 3941 u32 l5_cid = kcqe->pg_host_opaque; 3942 u8 opcode = kcqe->op_code; 3943 struct cnic_sock *csk = &cp->csk_tbl[l5_cid]; 3944 3945 csk_hold(csk); 3946 if (!cnic_in_use(csk)) 3947 goto done; 3948 3949 if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3950 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3951 goto done; 3952 } 3953 /* Possible PG kcqe status: SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */ 3954 if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) { 3955 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 3956 cnic_cm_upcall(cp, csk, 3957 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 3958 goto done; 3959 } 3960 3961 csk->pg_cid = kcqe->pg_cid; 3962 set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags); 3963 cnic_cm_conn_req(csk); 3964 3965 done: 3966 csk_put(csk); 3967 } 3968 3969 static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe) 3970 { 3971 struct cnic_local *cp = dev->cnic_priv; 3972 struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe; 3973 u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE; 3974 struct cnic_context *ctx = &cp->ctx_tbl[l5_cid]; 3975 3976 ctx->timestamp = jiffies; 3977 ctx->wait_cond = 1; 3978 wake_up(&ctx->waitq); 3979 } 3980 3981 static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe) 3982 { 3983 struct cnic_local *cp = dev->cnic_priv; 3984 struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe; 3985 u8 opcode = l4kcqe->op_code; 3986 u32 l5_cid; 3987 struct cnic_sock *csk; 3988 3989 if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) { 3990 cnic_process_fcoe_term_conn(dev, kcqe); 3991 return; 3992 } 3993 if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG || 3994 opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) { 3995 cnic_cm_process_offld_pg(dev, l4kcqe); 3996 return; 3997 } 3998 3999 l5_cid = l4kcqe->conn_id; 4000 if (opcode & 0x80) 4001 l5_cid = l4kcqe->cid; 4002 if (l5_cid >= MAX_CM_SK_TBL_SZ) 4003 return; 4004 4005 csk = &cp->csk_tbl[l5_cid]; 4006 csk_hold(csk); 4007 4008 if (!cnic_in_use(csk)) { 4009 csk_put(csk); 4010 return; 4011 } 4012 4013 switch (opcode) { 4014 case L5CM_RAMROD_CMD_ID_TCP_CONNECT: 4015 if (l4kcqe->status != 0) { 4016 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4017 cnic_cm_upcall(cp, csk, 4018 L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE); 4019 } 4020 break; 4021 case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE: 4022 if (l4kcqe->status == 0) 4023 set_bit(SK_F_OFFLD_COMPLETE, &csk->flags); 4024 else if (l4kcqe->status == 4025 L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4026 set_bit(SK_F_HW_ERR, &csk->flags); 4027 4028 smp_mb__before_clear_bit(); 4029 clear_bit(SK_F_OFFLD_SCHED, &csk->flags); 4030 cnic_cm_upcall(cp, csk, opcode); 4031 break; 4032 4033 case L5CM_RAMROD_CMD_ID_CLOSE: { 4034 struct iscsi_kcqe *l5kcqe = (struct iscsi_kcqe *) kcqe; 4035 4036 if (l4kcqe->status != 0 || l5kcqe->completion_status != 0) { 4037 netdev_warn(dev->netdev, "RAMROD CLOSE compl with status 0x%x completion status 0x%x\n", 4038 l4kcqe->status, l5kcqe->completion_status); 4039 opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP; 4040 /* Fall through */ 4041 } else { 4042 break; 4043 } 4044 } 4045 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4046 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4047 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4048 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4049 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4050 if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR) 4051 set_bit(SK_F_HW_ERR, &csk->flags); 4052 4053 cp->close_conn(csk, opcode); 4054 break; 4055 4056 case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED: 4057 /* after we already sent CLOSE_REQ */ 4058 if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) && 4059 !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) && 4060 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP) 4061 cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP); 4062 else 4063 cnic_cm_upcall(cp, csk, opcode); 4064 break; 4065 } 4066 csk_put(csk); 4067 } 4068 4069 static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num) 4070 { 4071 struct cnic_dev *dev = data; 4072 int i; 4073 4074 for (i = 0; i < num; i++) 4075 cnic_cm_process_kcqe(dev, kcqe[i]); 4076 } 4077 4078 static struct cnic_ulp_ops cm_ulp_ops = { 4079 .indicate_kcqes = cnic_cm_indicate_kcqe, 4080 }; 4081 4082 static void cnic_cm_free_mem(struct cnic_dev *dev) 4083 { 4084 struct cnic_local *cp = dev->cnic_priv; 4085 4086 kfree(cp->csk_tbl); 4087 cp->csk_tbl = NULL; 4088 cnic_free_id_tbl(&cp->csk_port_tbl); 4089 } 4090 4091 static int cnic_cm_alloc_mem(struct cnic_dev *dev) 4092 { 4093 struct cnic_local *cp = dev->cnic_priv; 4094 u32 port_id; 4095 4096 cp->csk_tbl = kzalloc(sizeof(struct cnic_sock) * MAX_CM_SK_TBL_SZ, 4097 GFP_KERNEL); 4098 if (!cp->csk_tbl) 4099 return -ENOMEM; 4100 4101 port_id = prandom_u32(); 4102 port_id %= CNIC_LOCAL_PORT_RANGE; 4103 if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, 4104 CNIC_LOCAL_PORT_MIN, port_id)) { 4105 cnic_cm_free_mem(dev); 4106 return -ENOMEM; 4107 } 4108 return 0; 4109 } 4110 4111 static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode) 4112 { 4113 if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) { 4114 /* Unsolicited RESET_COMP or RESET_RECEIVED */ 4115 opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED; 4116 csk->state = opcode; 4117 } 4118 4119 /* 1. If event opcode matches the expected event in csk->state 4120 * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any 4121 * event 4122 * 3. If the expected event is 0, meaning the connection was never 4123 * never established, we accept the opcode from cm_abort. 4124 */ 4125 if (opcode == csk->state || csk->state == 0 || 4126 csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP || 4127 csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) { 4128 if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) { 4129 if (csk->state == 0) 4130 csk->state = opcode; 4131 return 1; 4132 } 4133 } 4134 return 0; 4135 } 4136 4137 static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode) 4138 { 4139 struct cnic_dev *dev = csk->dev; 4140 struct cnic_local *cp = dev->cnic_priv; 4141 4142 if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) { 4143 cnic_cm_upcall(cp, csk, opcode); 4144 return; 4145 } 4146 4147 clear_bit(SK_F_CONNECT_START, &csk->flags); 4148 cnic_close_conn(csk); 4149 csk->state = opcode; 4150 cnic_cm_upcall(cp, csk, opcode); 4151 } 4152 4153 static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev) 4154 { 4155 } 4156 4157 static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) 4158 { 4159 u32 seed; 4160 4161 seed = prandom_u32(); 4162 cnic_ctx_wr(dev, 45, 0, seed); 4163 return 0; 4164 } 4165 4166 static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode) 4167 { 4168 struct cnic_dev *dev = csk->dev; 4169 struct cnic_local *cp = dev->cnic_priv; 4170 struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid]; 4171 union l5cm_specific_data l5_data; 4172 u32 cmd = 0; 4173 int close_complete = 0; 4174 4175 switch (opcode) { 4176 case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED: 4177 case L4_KCQE_OPCODE_VALUE_CLOSE_COMP: 4178 case L4_KCQE_OPCODE_VALUE_RESET_COMP: 4179 if (cnic_ready_to_close(csk, opcode)) { 4180 if (test_bit(SK_F_HW_ERR, &csk->flags)) 4181 close_complete = 1; 4182 else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) 4183 cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE; 4184 else 4185 close_complete = 1; 4186 } 4187 break; 4188 case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE: 4189 cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD; 4190 break; 4191 case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD: 4192 close_complete = 1; 4193 break; 4194 } 4195 if (cmd) { 4196 memset(&l5_data, 0, sizeof(l5_data)); 4197 4198 cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE, 4199 &l5_data); 4200 } else if (close_complete) { 4201 ctx->timestamp = jiffies; 4202 cnic_close_conn(csk); 4203 cnic_cm_upcall(cp, csk, csk->state); 4204 } 4205 } 4206 4207 static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) 4208 { 4209 struct cnic_local *cp = dev->cnic_priv; 4210 4211 if (!cp->ctx_tbl) 4212 return; 4213 4214 if (!netif_running(dev->netdev)) 4215 return; 4216 4217 cnic_bnx2x_delete_wait(dev, 0); 4218 4219 cancel_delayed_work(&cp->delete_task); 4220 flush_workqueue(cnic_wq); 4221 4222 if (atomic_read(&cp->iscsi_conn) != 0) 4223 netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", 4224 atomic_read(&cp->iscsi_conn)); 4225 } 4226 4227 static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev) 4228 { 4229 struct bnx2x *bp = netdev_priv(dev->netdev); 4230 u32 pfid = bp->pfid; 4231 u32 port = BP_PORT(bp); 4232 4233 cnic_init_bnx2x_mac(dev); 4234 cnic_bnx2x_set_tcp_options(dev, 0, 1); 4235 4236 CNIC_WR16(dev, BAR_XSTRORM_INTMEM + 4237 XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0); 4238 4239 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4240 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1); 4241 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4242 XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port), 4243 DEF_MAX_DA_COUNT); 4244 4245 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4246 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL); 4247 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4248 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS); 4249 CNIC_WR8(dev, BAR_XSTRORM_INTMEM + 4250 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2); 4251 CNIC_WR(dev, BAR_XSTRORM_INTMEM + 4252 XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER); 4253 4254 CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid), 4255 DEF_MAX_CWND); 4256 return 0; 4257 } 4258 4259 static void cnic_delete_task(struct work_struct *work) 4260 { 4261 struct cnic_local *cp; 4262 struct cnic_dev *dev; 4263 u32 i; 4264 int need_resched = 0; 4265 4266 cp = container_of(work, struct cnic_local, delete_task.work); 4267 dev = cp->dev; 4268 4269 if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) { 4270 struct drv_ctl_info info; 4271 4272 cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI); 4273 4274 info.cmd = DRV_CTL_ISCSI_STOPPED_CMD; 4275 cp->ethdev->drv_ctl(dev->netdev, &info); 4276 } 4277 4278 for (i = 0; i < cp->max_cid_space; i++) { 4279 struct cnic_context *ctx = &cp->ctx_tbl[i]; 4280 int err; 4281 4282 if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) || 4283 !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4284 continue; 4285 4286 if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) { 4287 need_resched = 1; 4288 continue; 4289 } 4290 4291 if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags)) 4292 continue; 4293 4294 err = cnic_bnx2x_destroy_ramrod(dev, i); 4295 4296 cnic_free_bnx2x_conn_resc(dev, i); 4297 if (!err) { 4298 if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) 4299 atomic_dec(&cp->iscsi_conn); 4300 4301 clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags); 4302 } 4303 } 4304 4305 if (need_resched) 4306 queue_delayed_work(cnic_wq, &cp->delete_task, 4307 msecs_to_jiffies(10)); 4308 4309 } 4310 4311 static int cnic_cm_open(struct cnic_dev *dev) 4312 { 4313 struct cnic_local *cp = dev->cnic_priv; 4314 int err; 4315 4316 err = cnic_cm_alloc_mem(dev); 4317 if (err) 4318 return err; 4319 4320 err = cp->start_cm(dev); 4321 4322 if (err) 4323 goto err_out; 4324 4325 INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task); 4326 4327 dev->cm_create = cnic_cm_create; 4328 dev->cm_destroy = cnic_cm_destroy; 4329 dev->cm_connect = cnic_cm_connect; 4330 dev->cm_abort = cnic_cm_abort; 4331 dev->cm_close = cnic_cm_close; 4332 dev->cm_select_dev = cnic_cm_select_dev; 4333 4334 cp->ulp_handle[CNIC_ULP_L4] = dev; 4335 rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops); 4336 return 0; 4337 4338 err_out: 4339 cnic_cm_free_mem(dev); 4340 return err; 4341 } 4342 4343 static int cnic_cm_shutdown(struct cnic_dev *dev) 4344 { 4345 struct cnic_local *cp = dev->cnic_priv; 4346 int i; 4347 4348 if (!cp->csk_tbl) 4349 return 0; 4350 4351 for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) { 4352 struct cnic_sock *csk = &cp->csk_tbl[i]; 4353 4354 clear_bit(SK_F_INUSE, &csk->flags); 4355 cnic_cm_cleanup(csk); 4356 } 4357 cnic_cm_free_mem(dev); 4358 4359 return 0; 4360 } 4361 4362 static void cnic_init_context(struct cnic_dev *dev, u32 cid) 4363 { 4364 u32 cid_addr; 4365 int i; 4366 4367 cid_addr = GET_CID_ADDR(cid); 4368 4369 for (i = 0; i < CTX_SIZE; i += 4) 4370 cnic_ctx_wr(dev, cid_addr, i, 0); 4371 } 4372 4373 static int cnic_setup_5709_context(struct cnic_dev *dev, int valid) 4374 { 4375 struct cnic_local *cp = dev->cnic_priv; 4376 int ret = 0, i; 4377 u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0; 4378 4379 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4380 return 0; 4381 4382 for (i = 0; i < cp->ctx_blks; i++) { 4383 int j; 4384 u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk; 4385 u32 val; 4386 4387 memset(cp->ctx_arr[i].ctx, 0, BNX2_PAGE_SIZE); 4388 4389 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0, 4390 (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit); 4391 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1, 4392 (u64) cp->ctx_arr[i].mapping >> 32); 4393 CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx | 4394 BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ); 4395 for (j = 0; j < 10; j++) { 4396 4397 val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL); 4398 if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ)) 4399 break; 4400 udelay(5); 4401 } 4402 if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) { 4403 ret = -EBUSY; 4404 break; 4405 } 4406 } 4407 return ret; 4408 } 4409 4410 static void cnic_free_irq(struct cnic_dev *dev) 4411 { 4412 struct cnic_local *cp = dev->cnic_priv; 4413 struct cnic_eth_dev *ethdev = cp->ethdev; 4414 4415 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4416 cp->disable_int_sync(dev); 4417 tasklet_kill(&cp->cnic_irq_task); 4418 free_irq(ethdev->irq_arr[0].vector, dev); 4419 } 4420 } 4421 4422 static int cnic_request_irq(struct cnic_dev *dev) 4423 { 4424 struct cnic_local *cp = dev->cnic_priv; 4425 struct cnic_eth_dev *ethdev = cp->ethdev; 4426 int err; 4427 4428 err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev); 4429 if (err) 4430 tasklet_disable(&cp->cnic_irq_task); 4431 4432 return err; 4433 } 4434 4435 static int cnic_init_bnx2_irq(struct cnic_dev *dev) 4436 { 4437 struct cnic_local *cp = dev->cnic_priv; 4438 struct cnic_eth_dev *ethdev = cp->ethdev; 4439 4440 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4441 int err, i = 0; 4442 int sblk_num = cp->status_blk_num; 4443 u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) + 4444 BNX2_HC_SB_CONFIG_1; 4445 4446 CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT); 4447 4448 CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8); 4449 CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220); 4450 CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220); 4451 4452 cp->last_status_idx = cp->status_blk.bnx2->status_idx; 4453 tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2_msix, 4454 (unsigned long) dev); 4455 err = cnic_request_irq(dev); 4456 if (err) 4457 return err; 4458 4459 while (cp->status_blk.bnx2->status_completion_producer_index && 4460 i < 10) { 4461 CNIC_WR(dev, BNX2_HC_COALESCE_NOW, 4462 1 << (11 + sblk_num)); 4463 udelay(10); 4464 i++; 4465 barrier(); 4466 } 4467 if (cp->status_blk.bnx2->status_completion_producer_index) { 4468 cnic_free_irq(dev); 4469 goto failed; 4470 } 4471 4472 } else { 4473 struct status_block *sblk = cp->status_blk.gen; 4474 u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND); 4475 int i = 0; 4476 4477 while (sblk->status_completion_producer_index && i < 10) { 4478 CNIC_WR(dev, BNX2_HC_COMMAND, 4479 hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT); 4480 udelay(10); 4481 i++; 4482 barrier(); 4483 } 4484 if (sblk->status_completion_producer_index) 4485 goto failed; 4486 4487 } 4488 return 0; 4489 4490 failed: 4491 netdev_err(dev->netdev, "KCQ index not resetting to 0\n"); 4492 return -EBUSY; 4493 } 4494 4495 static void cnic_enable_bnx2_int(struct cnic_dev *dev) 4496 { 4497 struct cnic_local *cp = dev->cnic_priv; 4498 struct cnic_eth_dev *ethdev = cp->ethdev; 4499 4500 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4501 return; 4502 4503 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4504 BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx); 4505 } 4506 4507 static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev) 4508 { 4509 struct cnic_local *cp = dev->cnic_priv; 4510 struct cnic_eth_dev *ethdev = cp->ethdev; 4511 4512 if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)) 4513 return; 4514 4515 CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num | 4516 BNX2_PCICFG_INT_ACK_CMD_MASK_INT); 4517 CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD); 4518 synchronize_irq(ethdev->irq_arr[0].vector); 4519 } 4520 4521 static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev) 4522 { 4523 struct cnic_local *cp = dev->cnic_priv; 4524 struct cnic_eth_dev *ethdev = cp->ethdev; 4525 struct cnic_uio_dev *udev = cp->udev; 4526 u32 cid_addr, tx_cid, sb_id; 4527 u32 val, offset0, offset1, offset2, offset3; 4528 int i; 4529 struct bnx2_tx_bd *txbd; 4530 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4531 struct status_block *s_blk = cp->status_blk.gen; 4532 4533 sb_id = cp->status_blk_num; 4534 tx_cid = 20; 4535 cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2; 4536 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4537 struct status_block_msix *sblk = cp->status_blk.bnx2; 4538 4539 tx_cid = TX_TSS_CID + sb_id - 1; 4540 CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) | 4541 (TX_TSS_CID << 7)); 4542 cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index; 4543 } 4544 cp->tx_cons = *cp->tx_cons_ptr; 4545 4546 cid_addr = GET_CID_ADDR(tx_cid); 4547 if (BNX2_CHIP(cp) == BNX2_CHIP_5709) { 4548 u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40; 4549 4550 for (i = 0; i < PHY_CTX_SIZE; i += 4) 4551 cnic_ctx_wr(dev, cid_addr2, i, 0); 4552 4553 offset0 = BNX2_L2CTX_TYPE_XI; 4554 offset1 = BNX2_L2CTX_CMD_TYPE_XI; 4555 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI; 4556 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI; 4557 } else { 4558 cnic_init_context(dev, tx_cid); 4559 cnic_init_context(dev, tx_cid + 1); 4560 4561 offset0 = BNX2_L2CTX_TYPE; 4562 offset1 = BNX2_L2CTX_CMD_TYPE; 4563 offset2 = BNX2_L2CTX_TBDR_BHADDR_HI; 4564 offset3 = BNX2_L2CTX_TBDR_BHADDR_LO; 4565 } 4566 val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2; 4567 cnic_ctx_wr(dev, cid_addr, offset0, val); 4568 4569 val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16); 4570 cnic_ctx_wr(dev, cid_addr, offset1, val); 4571 4572 txbd = udev->l2_ring; 4573 4574 buf_map = udev->l2_buf_map; 4575 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) { 4576 txbd->tx_bd_haddr_hi = (u64) buf_map >> 32; 4577 txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4578 } 4579 val = (u64) ring_map >> 32; 4580 cnic_ctx_wr(dev, cid_addr, offset2, val); 4581 txbd->tx_bd_haddr_hi = val; 4582 4583 val = (u64) ring_map & 0xffffffff; 4584 cnic_ctx_wr(dev, cid_addr, offset3, val); 4585 txbd->tx_bd_haddr_lo = val; 4586 } 4587 4588 static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev) 4589 { 4590 struct cnic_local *cp = dev->cnic_priv; 4591 struct cnic_eth_dev *ethdev = cp->ethdev; 4592 struct cnic_uio_dev *udev = cp->udev; 4593 u32 cid_addr, sb_id, val, coal_reg, coal_val; 4594 int i; 4595 struct bnx2_rx_bd *rxbd; 4596 struct status_block *s_blk = cp->status_blk.gen; 4597 dma_addr_t ring_map = udev->l2_ring_map; 4598 4599 sb_id = cp->status_blk_num; 4600 cnic_init_context(dev, 2); 4601 cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2; 4602 coal_reg = BNX2_HC_COMMAND; 4603 coal_val = CNIC_RD(dev, coal_reg); 4604 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4605 struct status_block_msix *sblk = cp->status_blk.bnx2; 4606 4607 cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index; 4608 coal_reg = BNX2_HC_COALESCE_NOW; 4609 coal_val = 1 << (11 + sb_id); 4610 } 4611 i = 0; 4612 while (!(*cp->rx_cons_ptr != 0) && i < 10) { 4613 CNIC_WR(dev, coal_reg, coal_val); 4614 udelay(10); 4615 i++; 4616 barrier(); 4617 } 4618 cp->rx_cons = *cp->rx_cons_ptr; 4619 4620 cid_addr = GET_CID_ADDR(2); 4621 val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE | 4622 BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8); 4623 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val); 4624 4625 if (sb_id == 0) 4626 val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT; 4627 else 4628 val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id); 4629 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val); 4630 4631 rxbd = udev->l2_ring + BNX2_PAGE_SIZE; 4632 for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) { 4633 dma_addr_t buf_map; 4634 int n = (i % cp->l2_rx_ring_size) + 1; 4635 4636 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 4637 rxbd->rx_bd_len = cp->l2_single_buf_size; 4638 rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END; 4639 rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32; 4640 rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff; 4641 } 4642 val = (u64) (ring_map + BNX2_PAGE_SIZE) >> 32; 4643 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val); 4644 rxbd->rx_bd_haddr_hi = val; 4645 4646 val = (u64) (ring_map + BNX2_PAGE_SIZE) & 0xffffffff; 4647 cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val); 4648 rxbd->rx_bd_haddr_lo = val; 4649 4650 val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD); 4651 cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2)); 4652 } 4653 4654 static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev) 4655 { 4656 struct kwqe *wqes[1], l2kwqe; 4657 4658 memset(&l2kwqe, 0, sizeof(l2kwqe)); 4659 wqes[0] = &l2kwqe; 4660 l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) | 4661 (L2_KWQE_OPCODE_VALUE_FLUSH << 4662 KWQE_OPCODE_SHIFT) | 2; 4663 dev->submit_kwqes(dev, wqes, 1); 4664 } 4665 4666 static void cnic_set_bnx2_mac(struct cnic_dev *dev) 4667 { 4668 struct cnic_local *cp = dev->cnic_priv; 4669 u32 val; 4670 4671 val = cp->func << 2; 4672 4673 cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val); 4674 4675 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4676 BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER); 4677 dev->mac_addr[0] = (u8) (val >> 8); 4678 dev->mac_addr[1] = (u8) val; 4679 4680 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val); 4681 4682 val = cnic_reg_rd_ind(dev, cp->shmem_base + 4683 BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER); 4684 dev->mac_addr[2] = (u8) (val >> 24); 4685 dev->mac_addr[3] = (u8) (val >> 16); 4686 dev->mac_addr[4] = (u8) (val >> 8); 4687 dev->mac_addr[5] = (u8) val; 4688 4689 CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val); 4690 4691 val = 4 | BNX2_RPM_SORT_USER2_BC_EN; 4692 if (BNX2_CHIP(cp) != BNX2_CHIP_5709) 4693 val |= BNX2_RPM_SORT_USER2_PROM_VLAN; 4694 4695 CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0); 4696 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val); 4697 CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA); 4698 } 4699 4700 static int cnic_start_bnx2_hw(struct cnic_dev *dev) 4701 { 4702 struct cnic_local *cp = dev->cnic_priv; 4703 struct cnic_eth_dev *ethdev = cp->ethdev; 4704 struct status_block *sblk = cp->status_blk.gen; 4705 u32 val, kcq_cid_addr, kwq_cid_addr; 4706 int err; 4707 4708 cnic_set_bnx2_mac(dev); 4709 4710 val = CNIC_RD(dev, BNX2_MQ_CONFIG); 4711 val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE; 4712 if (BNX2_PAGE_BITS > 12) 4713 val |= (12 - 8) << 4; 4714 else 4715 val |= (BNX2_PAGE_BITS - 8) << 4; 4716 4717 CNIC_WR(dev, BNX2_MQ_CONFIG, val); 4718 4719 CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8); 4720 CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220); 4721 CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220); 4722 4723 err = cnic_setup_5709_context(dev, 1); 4724 if (err) 4725 return err; 4726 4727 cnic_init_context(dev, KWQ_CID); 4728 cnic_init_context(dev, KCQ_CID); 4729 4730 kwq_cid_addr = GET_CID_ADDR(KWQ_CID); 4731 cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX; 4732 4733 cp->max_kwq_idx = MAX_KWQ_IDX; 4734 cp->kwq_prod_idx = 0; 4735 cp->kwq_con_idx = 0; 4736 set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags); 4737 4738 if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708) 4739 cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15; 4740 else 4741 cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index; 4742 4743 /* Initialize the kernel work queue context. */ 4744 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4745 (BNX2_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4746 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val); 4747 4748 val = (BNX2_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16; 4749 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4750 4751 val = ((BNX2_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT; 4752 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4753 4754 val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32); 4755 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4756 4757 val = (u32) cp->kwq_info.pgtbl_map; 4758 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4759 4760 kcq_cid_addr = GET_CID_ADDR(KCQ_CID); 4761 cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX; 4762 4763 cp->kcq1.sw_prod_idx = 0; 4764 cp->kcq1.hw_prod_idx_ptr = 4765 &sblk->status_completion_producer_index; 4766 4767 cp->kcq1.status_idx_ptr = &sblk->status_idx; 4768 4769 /* Initialize the kernel complete queue context. */ 4770 val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE | 4771 (BNX2_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ; 4772 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val); 4773 4774 val = (BNX2_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16; 4775 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val); 4776 4777 val = ((BNX2_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT; 4778 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val); 4779 4780 val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32); 4781 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val); 4782 4783 val = (u32) cp->kcq1.dma.pgtbl_map; 4784 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val); 4785 4786 cp->int_num = 0; 4787 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) { 4788 struct status_block_msix *msblk = cp->status_blk.bnx2; 4789 u32 sb_id = cp->status_blk_num; 4790 u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id); 4791 4792 cp->kcq1.hw_prod_idx_ptr = 4793 &msblk->status_completion_producer_index; 4794 cp->kcq1.status_idx_ptr = &msblk->status_idx; 4795 cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index; 4796 cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT; 4797 cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4798 cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb); 4799 } 4800 4801 /* Enable Commnad Scheduler notification when we write to the 4802 * host producer index of the kernel contexts. */ 4803 CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2); 4804 4805 /* Enable Command Scheduler notification when we write to either 4806 * the Send Queue or Receive Queue producer indexes of the kernel 4807 * bypass contexts. */ 4808 CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7); 4809 CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7); 4810 4811 /* Notify COM when the driver post an application buffer. */ 4812 CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000); 4813 4814 /* Set the CP and COM doorbells. These two processors polls the 4815 * doorbell for a non zero value before running. This must be done 4816 * after setting up the kernel queue contexts. */ 4817 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1); 4818 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1); 4819 4820 cnic_init_bnx2_tx_ring(dev); 4821 cnic_init_bnx2_rx_ring(dev); 4822 4823 err = cnic_init_bnx2_irq(dev); 4824 if (err) { 4825 netdev_err(dev->netdev, "cnic_init_irq failed\n"); 4826 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 4827 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 4828 return err; 4829 } 4830 4831 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 4832 4833 return 0; 4834 } 4835 4836 static void cnic_setup_bnx2x_context(struct cnic_dev *dev) 4837 { 4838 struct cnic_local *cp = dev->cnic_priv; 4839 struct cnic_eth_dev *ethdev = cp->ethdev; 4840 u32 start_offset = ethdev->ctx_tbl_offset; 4841 int i; 4842 4843 for (i = 0; i < cp->ctx_blks; i++) { 4844 struct cnic_ctx *ctx = &cp->ctx_arr[i]; 4845 dma_addr_t map = ctx->mapping; 4846 4847 if (cp->ctx_align) { 4848 unsigned long mask = cp->ctx_align - 1; 4849 4850 map = (map + mask) & ~mask; 4851 } 4852 4853 cnic_ctx_tbl_wr(dev, start_offset + i, map); 4854 } 4855 } 4856 4857 static int cnic_init_bnx2x_irq(struct cnic_dev *dev) 4858 { 4859 struct cnic_local *cp = dev->cnic_priv; 4860 struct cnic_eth_dev *ethdev = cp->ethdev; 4861 int err = 0; 4862 4863 tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2x_bh, 4864 (unsigned long) dev); 4865 if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) 4866 err = cnic_request_irq(dev); 4867 4868 return err; 4869 } 4870 4871 static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev, 4872 u16 sb_id, u8 sb_index, 4873 u8 disable) 4874 { 4875 struct bnx2x *bp = netdev_priv(dev->netdev); 4876 4877 u32 addr = BAR_CSTRORM_INTMEM + 4878 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4879 offsetof(struct hc_status_block_data_e1x, index_data) + 4880 sizeof(struct hc_index_data)*sb_index + 4881 offsetof(struct hc_index_data, flags); 4882 u16 flags = CNIC_RD16(dev, addr); 4883 /* clear and set */ 4884 flags &= ~HC_INDEX_DATA_HC_ENABLED; 4885 flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) & 4886 HC_INDEX_DATA_HC_ENABLED); 4887 CNIC_WR16(dev, addr, flags); 4888 } 4889 4890 static void cnic_enable_bnx2x_int(struct cnic_dev *dev) 4891 { 4892 struct cnic_local *cp = dev->cnic_priv; 4893 struct bnx2x *bp = netdev_priv(dev->netdev); 4894 u8 sb_id = cp->status_blk_num; 4895 4896 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 4897 CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) + 4898 offsetof(struct hc_status_block_data_e1x, index_data) + 4899 sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS + 4900 offsetof(struct hc_index_data, timeout), 64 / 4); 4901 cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0); 4902 } 4903 4904 static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev) 4905 { 4906 } 4907 4908 static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev, 4909 struct client_init_ramrod_data *data) 4910 { 4911 struct cnic_local *cp = dev->cnic_priv; 4912 struct bnx2x *bp = netdev_priv(dev->netdev); 4913 struct cnic_uio_dev *udev = cp->udev; 4914 union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring; 4915 dma_addr_t buf_map, ring_map = udev->l2_ring_map; 4916 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4917 int i; 4918 u32 cli = cp->ethdev->iscsi_l2_client_id; 4919 u32 val; 4920 4921 memset(txbd, 0, BNX2_PAGE_SIZE); 4922 4923 buf_map = udev->l2_buf_map; 4924 for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) { 4925 struct eth_tx_start_bd *start_bd = &txbd->start_bd; 4926 struct eth_tx_parse_bd_e1x *pbd_e1x = 4927 &((txbd + 1)->parse_bd_e1x); 4928 struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2); 4929 struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd); 4930 4931 start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 4932 start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 4933 reg_bd->addr_hi = start_bd->addr_hi; 4934 reg_bd->addr_lo = start_bd->addr_lo + 0x10; 4935 start_bd->nbytes = cpu_to_le16(0x10); 4936 start_bd->nbd = cpu_to_le16(3); 4937 start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 4938 start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS; 4939 start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 4940 4941 if (BNX2X_CHIP_IS_E2_PLUS(bp)) 4942 pbd_e2->parsing_data = (UNICAST_ADDRESS << 4943 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT); 4944 else 4945 pbd_e1x->global_data = (UNICAST_ADDRESS << 4946 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT); 4947 } 4948 4949 val = (u64) ring_map >> 32; 4950 txbd->next_bd.addr_hi = cpu_to_le32(val); 4951 4952 data->tx.tx_bd_page_base.hi = cpu_to_le32(val); 4953 4954 val = (u64) ring_map & 0xffffffff; 4955 txbd->next_bd.addr_lo = cpu_to_le32(val); 4956 4957 data->tx.tx_bd_page_base.lo = cpu_to_le32(val); 4958 4959 /* Other ramrod params */ 4960 data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS; 4961 data->tx.tx_status_block_id = BNX2X_DEF_SB_ID; 4962 4963 /* reset xstorm per client statistics */ 4964 if (cli < MAX_STAT_COUNTER_ID) { 4965 data->general.statistics_zero_flg = 1; 4966 data->general.statistics_en_flg = 1; 4967 data->general.statistics_counter_id = cli; 4968 } 4969 4970 cp->tx_cons_ptr = 4971 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS]; 4972 } 4973 4974 static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev, 4975 struct client_init_ramrod_data *data) 4976 { 4977 struct cnic_local *cp = dev->cnic_priv; 4978 struct bnx2x *bp = netdev_priv(dev->netdev); 4979 struct cnic_uio_dev *udev = cp->udev; 4980 struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring + 4981 BNX2_PAGE_SIZE); 4982 struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *) 4983 (udev->l2_ring + (2 * BNX2_PAGE_SIZE)); 4984 struct host_sp_status_block *sb = cp->bnx2x_def_status_blk; 4985 int i; 4986 u32 cli = cp->ethdev->iscsi_l2_client_id; 4987 int cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 4988 u32 val; 4989 dma_addr_t ring_map = udev->l2_ring_map; 4990 4991 /* General data */ 4992 data->general.client_id = cli; 4993 data->general.activate_flg = 1; 4994 data->general.sp_client_id = cli; 4995 data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14); 4996 data->general.func_id = bp->pfid; 4997 4998 for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) { 4999 dma_addr_t buf_map; 5000 int n = (i % cp->l2_rx_ring_size) + 1; 5001 5002 buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size); 5003 rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32); 5004 rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff); 5005 } 5006 5007 val = (u64) (ring_map + BNX2_PAGE_SIZE) >> 32; 5008 rxbd->addr_hi = cpu_to_le32(val); 5009 data->rx.bd_page_base.hi = cpu_to_le32(val); 5010 5011 val = (u64) (ring_map + BNX2_PAGE_SIZE) & 0xffffffff; 5012 rxbd->addr_lo = cpu_to_le32(val); 5013 data->rx.bd_page_base.lo = cpu_to_le32(val); 5014 5015 rxcqe += BNX2X_MAX_RCQ_DESC_CNT; 5016 val = (u64) (ring_map + (2 * BNX2_PAGE_SIZE)) >> 32; 5017 rxcqe->addr_hi = cpu_to_le32(val); 5018 data->rx.cqe_page_base.hi = cpu_to_le32(val); 5019 5020 val = (u64) (ring_map + (2 * BNX2_PAGE_SIZE)) & 0xffffffff; 5021 rxcqe->addr_lo = cpu_to_le32(val); 5022 data->rx.cqe_page_base.lo = cpu_to_le32(val); 5023 5024 /* Other ramrod params */ 5025 data->rx.client_qzone_id = cl_qzone_id; 5026 data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS; 5027 data->rx.status_block_id = BNX2X_DEF_SB_ID; 5028 5029 data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT; 5030 5031 data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size); 5032 data->rx.outer_vlan_removal_enable_flg = 1; 5033 data->rx.silent_vlan_removal_flg = 1; 5034 data->rx.silent_vlan_value = 0; 5035 data->rx.silent_vlan_mask = 0xffff; 5036 5037 cp->rx_cons_ptr = 5038 &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS]; 5039 cp->rx_cons = *cp->rx_cons_ptr; 5040 } 5041 5042 static void cnic_init_bnx2x_kcq(struct cnic_dev *dev) 5043 { 5044 struct cnic_local *cp = dev->cnic_priv; 5045 struct bnx2x *bp = netdev_priv(dev->netdev); 5046 u32 pfid = bp->pfid; 5047 5048 cp->kcq1.io_addr = BAR_CSTRORM_INTMEM + 5049 CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0); 5050 cp->kcq1.sw_prod_idx = 0; 5051 5052 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5053 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5054 5055 cp->kcq1.hw_prod_idx_ptr = 5056 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5057 cp->kcq1.status_idx_ptr = 5058 &sb->sb.running_index[SM_RX_ID]; 5059 } else { 5060 struct host_hc_status_block_e1x *sb = cp->status_blk.gen; 5061 5062 cp->kcq1.hw_prod_idx_ptr = 5063 &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS]; 5064 cp->kcq1.status_idx_ptr = 5065 &sb->sb.running_index[SM_RX_ID]; 5066 } 5067 5068 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5069 struct host_hc_status_block_e2 *sb = cp->status_blk.gen; 5070 5071 cp->kcq2.io_addr = BAR_USTRORM_INTMEM + 5072 USTORM_FCOE_EQ_PROD_OFFSET(pfid); 5073 cp->kcq2.sw_prod_idx = 0; 5074 cp->kcq2.hw_prod_idx_ptr = 5075 &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS]; 5076 cp->kcq2.status_idx_ptr = 5077 &sb->sb.running_index[SM_RX_ID]; 5078 } 5079 } 5080 5081 static int cnic_start_bnx2x_hw(struct cnic_dev *dev) 5082 { 5083 struct cnic_local *cp = dev->cnic_priv; 5084 struct bnx2x *bp = netdev_priv(dev->netdev); 5085 struct cnic_eth_dev *ethdev = cp->ethdev; 5086 int func, ret; 5087 u32 pfid; 5088 5089 dev->stats_addr = ethdev->addr_drv_info_to_mcp; 5090 cp->func = bp->pf_num; 5091 5092 func = CNIC_FUNC(cp); 5093 pfid = bp->pfid; 5094 5095 ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ, 5096 cp->iscsi_start_cid, 0); 5097 5098 if (ret) 5099 return -ENOMEM; 5100 5101 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5102 ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn, 5103 cp->fcoe_start_cid, 0); 5104 5105 if (ret) 5106 return -ENOMEM; 5107 } 5108 5109 cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2; 5110 5111 cnic_init_bnx2x_kcq(dev); 5112 5113 /* Only 1 EQ */ 5114 CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX); 5115 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5116 CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0); 5117 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5118 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0), 5119 cp->kcq1.dma.pg_map_arr[1] & 0xffffffff); 5120 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5121 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4, 5122 (u64) cp->kcq1.dma.pg_map_arr[1] >> 32); 5123 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5124 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0), 5125 cp->kcq1.dma.pg_map_arr[0] & 0xffffffff); 5126 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5127 CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4, 5128 (u64) cp->kcq1.dma.pg_map_arr[0] >> 32); 5129 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5130 CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1); 5131 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + 5132 CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num); 5133 CNIC_WR8(dev, BAR_CSTRORM_INTMEM + 5134 CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0), 5135 HC_INDEX_ISCSI_EQ_CONS); 5136 5137 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5138 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid), 5139 cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff); 5140 CNIC_WR(dev, BAR_USTRORM_INTMEM + 5141 USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4, 5142 (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32); 5143 5144 CNIC_WR(dev, BAR_TSTRORM_INTMEM + 5145 TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF); 5146 5147 cnic_setup_bnx2x_context(dev); 5148 5149 ret = cnic_init_bnx2x_irq(dev); 5150 if (ret) 5151 return ret; 5152 5153 ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ; 5154 return 0; 5155 } 5156 5157 static void cnic_init_rings(struct cnic_dev *dev) 5158 { 5159 struct cnic_local *cp = dev->cnic_priv; 5160 struct bnx2x *bp = netdev_priv(dev->netdev); 5161 struct cnic_uio_dev *udev = cp->udev; 5162 5163 if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5164 return; 5165 5166 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5167 cnic_init_bnx2_tx_ring(dev); 5168 cnic_init_bnx2_rx_ring(dev); 5169 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5170 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5171 u32 cli = cp->ethdev->iscsi_l2_client_id; 5172 u32 cid = cp->ethdev->iscsi_l2_cid; 5173 u32 cl_qzone_id; 5174 struct client_init_ramrod_data *data; 5175 union l5cm_specific_data l5_data; 5176 struct ustorm_eth_rx_producers rx_prods = {0}; 5177 u32 off, i, *cid_ptr; 5178 5179 rx_prods.bd_prod = 0; 5180 rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT; 5181 barrier(); 5182 5183 cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli); 5184 5185 off = BAR_USTRORM_INTMEM + 5186 (BNX2X_CHIP_IS_E2_PLUS(bp) ? 5187 USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) : 5188 USTORM_RX_PRODS_E1X_OFFSET(BP_PORT(bp), cli)); 5189 5190 for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++) 5191 CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]); 5192 5193 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5194 5195 data = udev->l2_buf; 5196 cid_ptr = udev->l2_buf + 12; 5197 5198 memset(data, 0, sizeof(*data)); 5199 5200 cnic_init_bnx2x_tx_ring(dev, data); 5201 cnic_init_bnx2x_rx_ring(dev, data); 5202 5203 l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff; 5204 l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32; 5205 5206 set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5207 5208 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP, 5209 cid, ETH_CONNECTION_TYPE, &l5_data); 5210 5211 i = 0; 5212 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5213 ++i < 10) 5214 msleep(1); 5215 5216 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5217 netdev_err(dev->netdev, 5218 "iSCSI CLIENT_SETUP did not complete\n"); 5219 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5220 cnic_ring_ctl(dev, cid, cli, 1); 5221 *cid_ptr = cid >> 4; 5222 *(cid_ptr + 1) = cid * bp->db_size; 5223 } 5224 } 5225 5226 static void cnic_shutdown_rings(struct cnic_dev *dev) 5227 { 5228 struct cnic_local *cp = dev->cnic_priv; 5229 struct cnic_uio_dev *udev = cp->udev; 5230 void *rx_ring; 5231 5232 if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags)) 5233 return; 5234 5235 if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) { 5236 cnic_shutdown_bnx2_rx_ring(dev); 5237 } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) { 5238 u32 cli = cp->ethdev->iscsi_l2_client_id; 5239 u32 cid = cp->ethdev->iscsi_l2_cid; 5240 union l5cm_specific_data l5_data; 5241 int i; 5242 5243 cnic_ring_ctl(dev, cid, cli, 0); 5244 5245 set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags); 5246 5247 l5_data.phy_address.lo = cli; 5248 l5_data.phy_address.hi = 0; 5249 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT, 5250 cid, ETH_CONNECTION_TYPE, &l5_data); 5251 i = 0; 5252 while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) && 5253 ++i < 10) 5254 msleep(1); 5255 5256 if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags)) 5257 netdev_err(dev->netdev, 5258 "iSCSI CLIENT_HALT did not complete\n"); 5259 cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1); 5260 5261 memset(&l5_data, 0, sizeof(l5_data)); 5262 cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL, 5263 cid, NONE_CONNECTION_TYPE, &l5_data); 5264 msleep(10); 5265 } 5266 clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags); 5267 rx_ring = udev->l2_ring + BNX2_PAGE_SIZE; 5268 memset(rx_ring, 0, BNX2_PAGE_SIZE); 5269 } 5270 5271 static int cnic_register_netdev(struct cnic_dev *dev) 5272 { 5273 struct cnic_local *cp = dev->cnic_priv; 5274 struct cnic_eth_dev *ethdev = cp->ethdev; 5275 int err; 5276 5277 if (!ethdev) 5278 return -ENODEV; 5279 5280 if (ethdev->drv_state & CNIC_DRV_STATE_REGD) 5281 return 0; 5282 5283 err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev); 5284 if (err) 5285 netdev_err(dev->netdev, "register_cnic failed\n"); 5286 5287 /* Read iSCSI config again. On some bnx2x device, iSCSI config 5288 * can change after firmware is downloaded. 5289 */ 5290 dev->max_iscsi_conn = ethdev->max_iscsi_conn; 5291 if (ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI) 5292 dev->max_iscsi_conn = 0; 5293 5294 return err; 5295 } 5296 5297 static void cnic_unregister_netdev(struct cnic_dev *dev) 5298 { 5299 struct cnic_local *cp = dev->cnic_priv; 5300 struct cnic_eth_dev *ethdev = cp->ethdev; 5301 5302 if (!ethdev) 5303 return; 5304 5305 ethdev->drv_unregister_cnic(dev->netdev); 5306 } 5307 5308 static int cnic_start_hw(struct cnic_dev *dev) 5309 { 5310 struct cnic_local *cp = dev->cnic_priv; 5311 struct cnic_eth_dev *ethdev = cp->ethdev; 5312 int err; 5313 5314 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) 5315 return -EALREADY; 5316 5317 dev->regview = ethdev->io_base; 5318 pci_dev_get(dev->pcidev); 5319 cp->func = PCI_FUNC(dev->pcidev->devfn); 5320 cp->status_blk.gen = ethdev->irq_arr[0].status_blk; 5321 cp->status_blk_num = ethdev->irq_arr[0].status_blk_num; 5322 5323 err = cp->alloc_resc(dev); 5324 if (err) { 5325 netdev_err(dev->netdev, "allocate resource failure\n"); 5326 goto err1; 5327 } 5328 5329 err = cp->start_hw(dev); 5330 if (err) 5331 goto err1; 5332 5333 err = cnic_cm_open(dev); 5334 if (err) 5335 goto err1; 5336 5337 set_bit(CNIC_F_CNIC_UP, &dev->flags); 5338 5339 cp->enable_int(dev); 5340 5341 return 0; 5342 5343 err1: 5344 cp->free_resc(dev); 5345 pci_dev_put(dev->pcidev); 5346 return err; 5347 } 5348 5349 static void cnic_stop_bnx2_hw(struct cnic_dev *dev) 5350 { 5351 cnic_disable_bnx2_int_sync(dev); 5352 5353 cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0); 5354 cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0); 5355 5356 cnic_init_context(dev, KWQ_CID); 5357 cnic_init_context(dev, KCQ_CID); 5358 5359 cnic_setup_5709_context(dev, 0); 5360 cnic_free_irq(dev); 5361 5362 cnic_free_resc(dev); 5363 } 5364 5365 5366 static void cnic_stop_bnx2x_hw(struct cnic_dev *dev) 5367 { 5368 struct cnic_local *cp = dev->cnic_priv; 5369 struct bnx2x *bp = netdev_priv(dev->netdev); 5370 u32 hc_index = HC_INDEX_ISCSI_EQ_CONS; 5371 u32 sb_id = cp->status_blk_num; 5372 u32 idx_off, syn_off; 5373 5374 cnic_free_irq(dev); 5375 5376 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5377 idx_off = offsetof(struct hc_status_block_e2, index_values) + 5378 (hc_index * sizeof(u16)); 5379 5380 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id); 5381 } else { 5382 idx_off = offsetof(struct hc_status_block_e1x, index_values) + 5383 (hc_index * sizeof(u16)); 5384 5385 syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id); 5386 } 5387 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0); 5388 CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) + 5389 idx_off, 0); 5390 5391 *cp->kcq1.hw_prod_idx_ptr = 0; 5392 CNIC_WR(dev, BAR_CSTRORM_INTMEM + 5393 CSTORM_ISCSI_EQ_CONS_OFFSET(bp->pfid, 0), 0); 5394 CNIC_WR16(dev, cp->kcq1.io_addr, 0); 5395 cnic_free_resc(dev); 5396 } 5397 5398 static void cnic_stop_hw(struct cnic_dev *dev) 5399 { 5400 if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) { 5401 struct cnic_local *cp = dev->cnic_priv; 5402 int i = 0; 5403 5404 /* Need to wait for the ring shutdown event to complete 5405 * before clearing the CNIC_UP flag. 5406 */ 5407 while (cp->udev && cp->udev->uio_dev != -1 && i < 15) { 5408 msleep(100); 5409 i++; 5410 } 5411 cnic_shutdown_rings(dev); 5412 cp->stop_cm(dev); 5413 cp->ethdev->drv_state &= ~CNIC_DRV_STATE_HANDLES_IRQ; 5414 clear_bit(CNIC_F_CNIC_UP, &dev->flags); 5415 RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL); 5416 synchronize_rcu(); 5417 cnic_cm_shutdown(dev); 5418 cp->stop_hw(dev); 5419 pci_dev_put(dev->pcidev); 5420 } 5421 } 5422 5423 static void cnic_free_dev(struct cnic_dev *dev) 5424 { 5425 int i = 0; 5426 5427 while ((atomic_read(&dev->ref_count) != 0) && i < 10) { 5428 msleep(100); 5429 i++; 5430 } 5431 if (atomic_read(&dev->ref_count) != 0) 5432 netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n"); 5433 5434 netdev_info(dev->netdev, "Removed CNIC device\n"); 5435 dev_put(dev->netdev); 5436 kfree(dev); 5437 } 5438 5439 static struct cnic_dev *cnic_alloc_dev(struct net_device *dev, 5440 struct pci_dev *pdev) 5441 { 5442 struct cnic_dev *cdev; 5443 struct cnic_local *cp; 5444 int alloc_size; 5445 5446 alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local); 5447 5448 cdev = kzalloc(alloc_size, GFP_KERNEL); 5449 if (cdev == NULL) 5450 return NULL; 5451 5452 cdev->netdev = dev; 5453 cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev); 5454 cdev->register_device = cnic_register_device; 5455 cdev->unregister_device = cnic_unregister_device; 5456 cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv; 5457 5458 cp = cdev->cnic_priv; 5459 cp->dev = cdev; 5460 cp->l2_single_buf_size = 0x400; 5461 cp->l2_rx_ring_size = 3; 5462 5463 spin_lock_init(&cp->cnic_ulp_lock); 5464 5465 netdev_info(dev, "Added CNIC device\n"); 5466 5467 return cdev; 5468 } 5469 5470 static struct cnic_dev *init_bnx2_cnic(struct net_device *dev) 5471 { 5472 struct pci_dev *pdev; 5473 struct cnic_dev *cdev; 5474 struct cnic_local *cp; 5475 struct bnx2 *bp = netdev_priv(dev); 5476 struct cnic_eth_dev *ethdev = NULL; 5477 5478 if (bp->cnic_probe) 5479 ethdev = (bp->cnic_probe)(dev); 5480 5481 if (!ethdev) 5482 return NULL; 5483 5484 pdev = ethdev->pdev; 5485 if (!pdev) 5486 return NULL; 5487 5488 dev_hold(dev); 5489 pci_dev_get(pdev); 5490 if ((pdev->device == PCI_DEVICE_ID_NX2_5709 || 5491 pdev->device == PCI_DEVICE_ID_NX2_5709S) && 5492 (pdev->revision < 0x10)) { 5493 pci_dev_put(pdev); 5494 goto cnic_err; 5495 } 5496 pci_dev_put(pdev); 5497 5498 cdev = cnic_alloc_dev(dev, pdev); 5499 if (cdev == NULL) 5500 goto cnic_err; 5501 5502 set_bit(CNIC_F_BNX2_CLASS, &cdev->flags); 5503 cdev->submit_kwqes = cnic_submit_bnx2_kwqes; 5504 5505 cp = cdev->cnic_priv; 5506 cp->ethdev = ethdev; 5507 cdev->pcidev = pdev; 5508 cp->chip_id = ethdev->chip_id; 5509 5510 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5511 5512 cp->cnic_ops = &cnic_bnx2_ops; 5513 cp->start_hw = cnic_start_bnx2_hw; 5514 cp->stop_hw = cnic_stop_bnx2_hw; 5515 cp->setup_pgtbl = cnic_setup_page_tbl; 5516 cp->alloc_resc = cnic_alloc_bnx2_resc; 5517 cp->free_resc = cnic_free_resc; 5518 cp->start_cm = cnic_cm_init_bnx2_hw; 5519 cp->stop_cm = cnic_cm_stop_bnx2_hw; 5520 cp->enable_int = cnic_enable_bnx2_int; 5521 cp->disable_int_sync = cnic_disable_bnx2_int_sync; 5522 cp->close_conn = cnic_close_bnx2_conn; 5523 return cdev; 5524 5525 cnic_err: 5526 dev_put(dev); 5527 return NULL; 5528 } 5529 5530 static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev) 5531 { 5532 struct pci_dev *pdev; 5533 struct cnic_dev *cdev; 5534 struct cnic_local *cp; 5535 struct bnx2x *bp = netdev_priv(dev); 5536 struct cnic_eth_dev *ethdev = NULL; 5537 5538 if (bp->cnic_probe) 5539 ethdev = bp->cnic_probe(dev); 5540 5541 if (!ethdev) 5542 return NULL; 5543 5544 pdev = ethdev->pdev; 5545 if (!pdev) 5546 return NULL; 5547 5548 dev_hold(dev); 5549 cdev = cnic_alloc_dev(dev, pdev); 5550 if (cdev == NULL) { 5551 dev_put(dev); 5552 return NULL; 5553 } 5554 5555 set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags); 5556 cdev->submit_kwqes = cnic_submit_bnx2x_kwqes; 5557 5558 cp = cdev->cnic_priv; 5559 cp->ethdev = ethdev; 5560 cdev->pcidev = pdev; 5561 cp->chip_id = ethdev->chip_id; 5562 5563 cdev->stats_addr = ethdev->addr_drv_info_to_mcp; 5564 5565 if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)) 5566 cdev->max_iscsi_conn = ethdev->max_iscsi_conn; 5567 if (CNIC_SUPPORTS_FCOE(bp)) { 5568 cdev->max_fcoe_conn = ethdev->max_fcoe_conn; 5569 cdev->max_fcoe_exchanges = ethdev->max_fcoe_exchanges; 5570 } 5571 5572 if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS) 5573 cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS; 5574 5575 memcpy(cdev->mac_addr, ethdev->iscsi_mac, ETH_ALEN); 5576 5577 cp->cnic_ops = &cnic_bnx2x_ops; 5578 cp->start_hw = cnic_start_bnx2x_hw; 5579 cp->stop_hw = cnic_stop_bnx2x_hw; 5580 cp->setup_pgtbl = cnic_setup_page_tbl_le; 5581 cp->alloc_resc = cnic_alloc_bnx2x_resc; 5582 cp->free_resc = cnic_free_resc; 5583 cp->start_cm = cnic_cm_init_bnx2x_hw; 5584 cp->stop_cm = cnic_cm_stop_bnx2x_hw; 5585 cp->enable_int = cnic_enable_bnx2x_int; 5586 cp->disable_int_sync = cnic_disable_bnx2x_int_sync; 5587 if (BNX2X_CHIP_IS_E2_PLUS(bp)) { 5588 cp->ack_int = cnic_ack_bnx2x_e2_msix; 5589 cp->arm_int = cnic_arm_bnx2x_e2_msix; 5590 } else { 5591 cp->ack_int = cnic_ack_bnx2x_msix; 5592 cp->arm_int = cnic_arm_bnx2x_msix; 5593 } 5594 cp->close_conn = cnic_close_bnx2x_conn; 5595 return cdev; 5596 } 5597 5598 static struct cnic_dev *is_cnic_dev(struct net_device *dev) 5599 { 5600 struct ethtool_drvinfo drvinfo; 5601 struct cnic_dev *cdev = NULL; 5602 5603 if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) { 5604 memset(&drvinfo, 0, sizeof(drvinfo)); 5605 dev->ethtool_ops->get_drvinfo(dev, &drvinfo); 5606 5607 if (!strcmp(drvinfo.driver, "bnx2")) 5608 cdev = init_bnx2_cnic(dev); 5609 if (!strcmp(drvinfo.driver, "bnx2x")) 5610 cdev = init_bnx2x_cnic(dev); 5611 if (cdev) { 5612 write_lock(&cnic_dev_lock); 5613 list_add(&cdev->list, &cnic_dev_list); 5614 write_unlock(&cnic_dev_lock); 5615 } 5616 } 5617 return cdev; 5618 } 5619 5620 static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event, 5621 u16 vlan_id) 5622 { 5623 int if_type; 5624 5625 rcu_read_lock(); 5626 for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) { 5627 struct cnic_ulp_ops *ulp_ops; 5628 void *ctx; 5629 5630 ulp_ops = rcu_dereference(cp->ulp_ops[if_type]); 5631 if (!ulp_ops || !ulp_ops->indicate_netevent) 5632 continue; 5633 5634 ctx = cp->ulp_handle[if_type]; 5635 5636 ulp_ops->indicate_netevent(ctx, event, vlan_id); 5637 } 5638 rcu_read_unlock(); 5639 } 5640 5641 /* netdev event handler */ 5642 static int cnic_netdev_event(struct notifier_block *this, unsigned long event, 5643 void *ptr) 5644 { 5645 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 5646 struct cnic_dev *dev; 5647 int new_dev = 0; 5648 5649 dev = cnic_from_netdev(netdev); 5650 5651 if (!dev && event == NETDEV_REGISTER) { 5652 /* Check for the hot-plug device */ 5653 dev = is_cnic_dev(netdev); 5654 if (dev) { 5655 new_dev = 1; 5656 cnic_hold(dev); 5657 } 5658 } 5659 if (dev) { 5660 struct cnic_local *cp = dev->cnic_priv; 5661 5662 if (new_dev) 5663 cnic_ulp_init(dev); 5664 else if (event == NETDEV_UNREGISTER) 5665 cnic_ulp_exit(dev); 5666 5667 if (event == NETDEV_UP) { 5668 if (cnic_register_netdev(dev) != 0) { 5669 cnic_put(dev); 5670 goto done; 5671 } 5672 if (!cnic_start_hw(dev)) 5673 cnic_ulp_start(dev); 5674 } 5675 5676 cnic_rcv_netevent(cp, event, 0); 5677 5678 if (event == NETDEV_GOING_DOWN) { 5679 cnic_ulp_stop(dev); 5680 cnic_stop_hw(dev); 5681 cnic_unregister_netdev(dev); 5682 } else if (event == NETDEV_UNREGISTER) { 5683 write_lock(&cnic_dev_lock); 5684 list_del_init(&dev->list); 5685 write_unlock(&cnic_dev_lock); 5686 5687 cnic_put(dev); 5688 cnic_free_dev(dev); 5689 goto done; 5690 } 5691 cnic_put(dev); 5692 } else { 5693 struct net_device *realdev; 5694 u16 vid; 5695 5696 vid = cnic_get_vlan(netdev, &realdev); 5697 if (realdev) { 5698 dev = cnic_from_netdev(realdev); 5699 if (dev) { 5700 vid |= VLAN_TAG_PRESENT; 5701 cnic_rcv_netevent(dev->cnic_priv, event, vid); 5702 cnic_put(dev); 5703 } 5704 } 5705 } 5706 done: 5707 return NOTIFY_DONE; 5708 } 5709 5710 static struct notifier_block cnic_netdev_notifier = { 5711 .notifier_call = cnic_netdev_event 5712 }; 5713 5714 static void cnic_release(void) 5715 { 5716 struct cnic_uio_dev *udev; 5717 5718 while (!list_empty(&cnic_udev_list)) { 5719 udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev, 5720 list); 5721 cnic_free_uio(udev); 5722 } 5723 } 5724 5725 static int __init cnic_init(void) 5726 { 5727 int rc = 0; 5728 5729 pr_info("%s", version); 5730 5731 rc = register_netdevice_notifier(&cnic_netdev_notifier); 5732 if (rc) { 5733 cnic_release(); 5734 return rc; 5735 } 5736 5737 cnic_wq = create_singlethread_workqueue("cnic_wq"); 5738 if (!cnic_wq) { 5739 cnic_release(); 5740 unregister_netdevice_notifier(&cnic_netdev_notifier); 5741 return -ENOMEM; 5742 } 5743 5744 return 0; 5745 } 5746 5747 static void __exit cnic_exit(void) 5748 { 5749 unregister_netdevice_notifier(&cnic_netdev_notifier); 5750 cnic_release(); 5751 destroy_workqueue(cnic_wq); 5752 } 5753 5754 module_init(cnic_init); 5755 module_exit(cnic_exit); 5756