1 /* 2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver. 3 * 4 * Copyright (c) 2010-2015 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Written by: Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/skbuff.h> 17 #include <linux/crypto.h> 18 #include <linux/scatterlist.h> 19 #include <linux/pci.h> 20 #include <scsi/scsi.h> 21 #include <scsi/scsi_cmnd.h> 22 #include <scsi/scsi_host.h> 23 #include <linux/if_vlan.h> 24 #include <linux/inet.h> 25 #include <net/dst.h> 26 #include <net/route.h> 27 #include <net/ipv6.h> 28 #include <net/ip6_route.h> 29 #include <net/addrconf.h> 30 31 #include <linux/inetdevice.h> /* ip_dev_find */ 32 #include <linux/module.h> 33 #include <net/tcp.h> 34 35 static unsigned int dbg_level; 36 37 #include "libcxgbi.h" 38 39 #define DRV_MODULE_NAME "libcxgbi" 40 #define DRV_MODULE_DESC "Chelsio iSCSI driver library" 41 #define DRV_MODULE_VERSION "0.9.1-ko" 42 #define DRV_MODULE_RELDATE "Apr. 2015" 43 44 static char version[] = 45 DRV_MODULE_DESC " " DRV_MODULE_NAME 46 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 47 48 MODULE_AUTHOR("Chelsio Communications, Inc."); 49 MODULE_DESCRIPTION(DRV_MODULE_DESC); 50 MODULE_VERSION(DRV_MODULE_VERSION); 51 MODULE_LICENSE("GPL"); 52 53 module_param(dbg_level, uint, 0644); 54 MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)"); 55 56 57 /* 58 * cxgbi device management 59 * maintains a list of the cxgbi devices 60 */ 61 static LIST_HEAD(cdev_list); 62 static DEFINE_MUTEX(cdev_mutex); 63 64 static LIST_HEAD(cdev_rcu_list); 65 static DEFINE_SPINLOCK(cdev_rcu_lock); 66 67 int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base, 68 unsigned int max_conn) 69 { 70 struct cxgbi_ports_map *pmap = &cdev->pmap; 71 72 pmap->port_csk = cxgbi_alloc_big_mem(max_conn * 73 sizeof(struct cxgbi_sock *), 74 GFP_KERNEL); 75 if (!pmap->port_csk) { 76 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn); 77 return -ENOMEM; 78 } 79 80 pmap->max_connect = max_conn; 81 pmap->sport_base = base; 82 spin_lock_init(&pmap->lock); 83 return 0; 84 } 85 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create); 86 87 void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev) 88 { 89 struct cxgbi_ports_map *pmap = &cdev->pmap; 90 struct cxgbi_sock *csk; 91 int i; 92 93 for (i = 0; i < pmap->max_connect; i++) { 94 if (pmap->port_csk[i]) { 95 csk = pmap->port_csk[i]; 96 pmap->port_csk[i] = NULL; 97 log_debug(1 << CXGBI_DBG_SOCK, 98 "csk 0x%p, cdev 0x%p, offload down.\n", 99 csk, cdev); 100 spin_lock_bh(&csk->lock); 101 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN); 102 cxgbi_sock_closed(csk); 103 spin_unlock_bh(&csk->lock); 104 cxgbi_sock_put(csk); 105 } 106 } 107 } 108 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup); 109 110 static inline void cxgbi_device_destroy(struct cxgbi_device *cdev) 111 { 112 log_debug(1 << CXGBI_DBG_DEV, 113 "cdev 0x%p, p# %u.\n", cdev, cdev->nports); 114 cxgbi_hbas_remove(cdev); 115 cxgbi_device_portmap_cleanup(cdev); 116 if (cdev->dev_ddp_cleanup) 117 cdev->dev_ddp_cleanup(cdev); 118 else 119 cxgbi_ddp_cleanup(cdev); 120 if (cdev->ddp) 121 cxgbi_ddp_cleanup(cdev); 122 if (cdev->pmap.max_connect) 123 cxgbi_free_big_mem(cdev->pmap.port_csk); 124 kfree(cdev); 125 } 126 127 struct cxgbi_device *cxgbi_device_register(unsigned int extra, 128 unsigned int nports) 129 { 130 struct cxgbi_device *cdev; 131 132 cdev = kzalloc(sizeof(*cdev) + extra + nports * 133 (sizeof(struct cxgbi_hba *) + 134 sizeof(struct net_device *)), 135 GFP_KERNEL); 136 if (!cdev) { 137 pr_warn("nport %d, OOM.\n", nports); 138 return NULL; 139 } 140 cdev->ports = (struct net_device **)(cdev + 1); 141 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports * 142 sizeof(struct net_device *)); 143 if (extra) 144 cdev->dd_data = ((char *)cdev->hbas) + 145 nports * sizeof(struct cxgbi_hba *); 146 spin_lock_init(&cdev->pmap.lock); 147 148 mutex_lock(&cdev_mutex); 149 list_add_tail(&cdev->list_head, &cdev_list); 150 mutex_unlock(&cdev_mutex); 151 152 spin_lock(&cdev_rcu_lock); 153 list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list); 154 spin_unlock(&cdev_rcu_lock); 155 156 log_debug(1 << CXGBI_DBG_DEV, 157 "cdev 0x%p, p# %u.\n", cdev, nports); 158 return cdev; 159 } 160 EXPORT_SYMBOL_GPL(cxgbi_device_register); 161 162 void cxgbi_device_unregister(struct cxgbi_device *cdev) 163 { 164 log_debug(1 << CXGBI_DBG_DEV, 165 "cdev 0x%p, p# %u,%s.\n", 166 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : ""); 167 168 mutex_lock(&cdev_mutex); 169 list_del(&cdev->list_head); 170 mutex_unlock(&cdev_mutex); 171 172 spin_lock(&cdev_rcu_lock); 173 list_del_rcu(&cdev->rcu_node); 174 spin_unlock(&cdev_rcu_lock); 175 synchronize_rcu(); 176 177 cxgbi_device_destroy(cdev); 178 } 179 EXPORT_SYMBOL_GPL(cxgbi_device_unregister); 180 181 void cxgbi_device_unregister_all(unsigned int flag) 182 { 183 struct cxgbi_device *cdev, *tmp; 184 185 mutex_lock(&cdev_mutex); 186 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 187 if ((cdev->flags & flag) == flag) { 188 mutex_unlock(&cdev_mutex); 189 cxgbi_device_unregister(cdev); 190 mutex_lock(&cdev_mutex); 191 } 192 } 193 mutex_unlock(&cdev_mutex); 194 } 195 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all); 196 197 struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev) 198 { 199 struct cxgbi_device *cdev, *tmp; 200 201 mutex_lock(&cdev_mutex); 202 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 203 if (cdev->lldev == lldev) { 204 mutex_unlock(&cdev_mutex); 205 return cdev; 206 } 207 } 208 mutex_unlock(&cdev_mutex); 209 210 log_debug(1 << CXGBI_DBG_DEV, 211 "lldev 0x%p, NO match found.\n", lldev); 212 return NULL; 213 } 214 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev); 215 216 struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev, 217 int *port) 218 { 219 struct net_device *vdev = NULL; 220 struct cxgbi_device *cdev, *tmp; 221 int i; 222 223 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 224 vdev = ndev; 225 ndev = vlan_dev_real_dev(ndev); 226 log_debug(1 << CXGBI_DBG_DEV, 227 "vlan dev %s -> %s.\n", vdev->name, ndev->name); 228 } 229 230 mutex_lock(&cdev_mutex); 231 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 232 for (i = 0; i < cdev->nports; i++) { 233 if (ndev == cdev->ports[i]) { 234 cdev->hbas[i]->vdev = vdev; 235 mutex_unlock(&cdev_mutex); 236 if (port) 237 *port = i; 238 return cdev; 239 } 240 } 241 } 242 mutex_unlock(&cdev_mutex); 243 log_debug(1 << CXGBI_DBG_DEV, 244 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 245 return NULL; 246 } 247 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev); 248 249 struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev, 250 int *port) 251 { 252 struct net_device *vdev = NULL; 253 struct cxgbi_device *cdev; 254 int i; 255 256 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 257 vdev = ndev; 258 ndev = vlan_dev_real_dev(ndev); 259 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 260 } 261 262 rcu_read_lock(); 263 list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) { 264 for (i = 0; i < cdev->nports; i++) { 265 if (ndev == cdev->ports[i]) { 266 cdev->hbas[i]->vdev = vdev; 267 rcu_read_unlock(); 268 if (port) 269 *port = i; 270 return cdev; 271 } 272 } 273 } 274 rcu_read_unlock(); 275 276 log_debug(1 << CXGBI_DBG_DEV, 277 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 278 return NULL; 279 } 280 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu); 281 282 #if IS_ENABLED(CONFIG_IPV6) 283 static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev, 284 int *port) 285 { 286 struct net_device *vdev = NULL; 287 struct cxgbi_device *cdev, *tmp; 288 int i; 289 290 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 291 vdev = ndev; 292 ndev = vlan_dev_real_dev(ndev); 293 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 294 } 295 296 mutex_lock(&cdev_mutex); 297 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 298 for (i = 0; i < cdev->nports; i++) { 299 if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr, 300 MAX_ADDR_LEN)) { 301 cdev->hbas[i]->vdev = vdev; 302 mutex_unlock(&cdev_mutex); 303 if (port) 304 *port = i; 305 return cdev; 306 } 307 } 308 } 309 mutex_unlock(&cdev_mutex); 310 log_debug(1 << CXGBI_DBG_DEV, 311 "ndev 0x%p, %s, NO match mac found.\n", 312 ndev, ndev->name); 313 return NULL; 314 } 315 #endif 316 317 void cxgbi_hbas_remove(struct cxgbi_device *cdev) 318 { 319 int i; 320 struct cxgbi_hba *chba; 321 322 log_debug(1 << CXGBI_DBG_DEV, 323 "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 324 325 for (i = 0; i < cdev->nports; i++) { 326 chba = cdev->hbas[i]; 327 if (chba) { 328 cdev->hbas[i] = NULL; 329 iscsi_host_remove(chba->shost); 330 pci_dev_put(cdev->pdev); 331 iscsi_host_free(chba->shost); 332 } 333 } 334 } 335 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove); 336 337 int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun, 338 unsigned int max_id, struct scsi_host_template *sht, 339 struct scsi_transport_template *stt) 340 { 341 struct cxgbi_hba *chba; 342 struct Scsi_Host *shost; 343 int i, err; 344 345 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 346 347 for (i = 0; i < cdev->nports; i++) { 348 shost = iscsi_host_alloc(sht, sizeof(*chba), 1); 349 if (!shost) { 350 pr_info("0x%p, p%d, %s, host alloc failed.\n", 351 cdev, i, cdev->ports[i]->name); 352 err = -ENOMEM; 353 goto err_out; 354 } 355 356 shost->transportt = stt; 357 shost->max_lun = max_lun; 358 shost->max_id = max_id; 359 shost->max_channel = 0; 360 shost->max_cmd_len = 16; 361 362 chba = iscsi_host_priv(shost); 363 chba->cdev = cdev; 364 chba->ndev = cdev->ports[i]; 365 chba->shost = shost; 366 367 log_debug(1 << CXGBI_DBG_DEV, 368 "cdev 0x%p, p#%d %s: chba 0x%p.\n", 369 cdev, i, cdev->ports[i]->name, chba); 370 371 pci_dev_get(cdev->pdev); 372 err = iscsi_host_add(shost, &cdev->pdev->dev); 373 if (err) { 374 pr_info("cdev 0x%p, p#%d %s, host add failed.\n", 375 cdev, i, cdev->ports[i]->name); 376 pci_dev_put(cdev->pdev); 377 scsi_host_put(shost); 378 goto err_out; 379 } 380 381 cdev->hbas[i] = chba; 382 } 383 384 return 0; 385 386 err_out: 387 cxgbi_hbas_remove(cdev); 388 return err; 389 } 390 EXPORT_SYMBOL_GPL(cxgbi_hbas_add); 391 392 /* 393 * iSCSI offload 394 * 395 * - source port management 396 * To find a free source port in the port allocation map we use a very simple 397 * rotor scheme to look for the next free port. 398 * 399 * If a source port has been specified make sure that it doesn't collide with 400 * our normal source port allocation map. If it's outside the range of our 401 * allocation/deallocation scheme just let them use it. 402 * 403 * If the source port is outside our allocation range, the caller is 404 * responsible for keeping track of their port usage. 405 */ 406 407 static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev, 408 unsigned char port_id) 409 { 410 struct cxgbi_ports_map *pmap = &cdev->pmap; 411 unsigned int i; 412 unsigned int used; 413 414 if (!pmap->max_connect || !pmap->used) 415 return NULL; 416 417 spin_lock_bh(&pmap->lock); 418 used = pmap->used; 419 for (i = 0; used && i < pmap->max_connect; i++) { 420 struct cxgbi_sock *csk = pmap->port_csk[i]; 421 422 if (csk) { 423 if (csk->port_id == port_id) { 424 spin_unlock_bh(&pmap->lock); 425 return csk; 426 } 427 used--; 428 } 429 } 430 spin_unlock_bh(&pmap->lock); 431 432 return NULL; 433 } 434 435 static int sock_get_port(struct cxgbi_sock *csk) 436 { 437 struct cxgbi_device *cdev = csk->cdev; 438 struct cxgbi_ports_map *pmap = &cdev->pmap; 439 unsigned int start; 440 int idx; 441 __be16 *port; 442 443 if (!pmap->max_connect) { 444 pr_err("cdev 0x%p, p#%u %s, NO port map.\n", 445 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 446 return -EADDRNOTAVAIL; 447 } 448 449 if (csk->csk_family == AF_INET) 450 port = &csk->saddr.sin_port; 451 else /* ipv6 */ 452 port = &csk->saddr6.sin6_port; 453 454 if (*port) { 455 pr_err("source port NON-ZERO %u.\n", 456 ntohs(*port)); 457 return -EADDRINUSE; 458 } 459 460 spin_lock_bh(&pmap->lock); 461 if (pmap->used >= pmap->max_connect) { 462 spin_unlock_bh(&pmap->lock); 463 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n", 464 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 465 return -EADDRNOTAVAIL; 466 } 467 468 start = idx = pmap->next; 469 do { 470 if (++idx >= pmap->max_connect) 471 idx = 0; 472 if (!pmap->port_csk[idx]) { 473 pmap->used++; 474 *port = htons(pmap->sport_base + idx); 475 pmap->next = idx; 476 pmap->port_csk[idx] = csk; 477 spin_unlock_bh(&pmap->lock); 478 cxgbi_sock_get(csk); 479 log_debug(1 << CXGBI_DBG_SOCK, 480 "cdev 0x%p, p#%u %s, p %u, %u.\n", 481 cdev, csk->port_id, 482 cdev->ports[csk->port_id]->name, 483 pmap->sport_base + idx, pmap->next); 484 return 0; 485 } 486 } while (idx != start); 487 spin_unlock_bh(&pmap->lock); 488 489 /* should not happen */ 490 pr_warn("cdev 0x%p, p#%u %s, next %u?\n", 491 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 492 pmap->next); 493 return -EADDRNOTAVAIL; 494 } 495 496 static void sock_put_port(struct cxgbi_sock *csk) 497 { 498 struct cxgbi_device *cdev = csk->cdev; 499 struct cxgbi_ports_map *pmap = &cdev->pmap; 500 __be16 *port; 501 502 if (csk->csk_family == AF_INET) 503 port = &csk->saddr.sin_port; 504 else /* ipv6 */ 505 port = &csk->saddr6.sin6_port; 506 507 if (*port) { 508 int idx = ntohs(*port) - pmap->sport_base; 509 510 *port = 0; 511 if (idx < 0 || idx >= pmap->max_connect) { 512 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n", 513 cdev, csk->port_id, 514 cdev->ports[csk->port_id]->name, 515 ntohs(*port)); 516 return; 517 } 518 519 spin_lock_bh(&pmap->lock); 520 pmap->port_csk[idx] = NULL; 521 pmap->used--; 522 spin_unlock_bh(&pmap->lock); 523 524 log_debug(1 << CXGBI_DBG_SOCK, 525 "cdev 0x%p, p#%u %s, release %u.\n", 526 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 527 pmap->sport_base + idx); 528 529 cxgbi_sock_put(csk); 530 } 531 } 532 533 /* 534 * iscsi tcp connection 535 */ 536 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk) 537 { 538 if (csk->cpl_close) { 539 kfree_skb(csk->cpl_close); 540 csk->cpl_close = NULL; 541 } 542 if (csk->cpl_abort_req) { 543 kfree_skb(csk->cpl_abort_req); 544 csk->cpl_abort_req = NULL; 545 } 546 if (csk->cpl_abort_rpl) { 547 kfree_skb(csk->cpl_abort_rpl); 548 csk->cpl_abort_rpl = NULL; 549 } 550 } 551 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs); 552 553 static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev) 554 { 555 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO); 556 557 if (!csk) { 558 pr_info("alloc csk %zu failed.\n", sizeof(*csk)); 559 return NULL; 560 } 561 562 if (cdev->csk_alloc_cpls(csk) < 0) { 563 pr_info("csk 0x%p, alloc cpls failed.\n", csk); 564 kfree(csk); 565 return NULL; 566 } 567 568 spin_lock_init(&csk->lock); 569 kref_init(&csk->refcnt); 570 skb_queue_head_init(&csk->receive_queue); 571 skb_queue_head_init(&csk->write_queue); 572 setup_timer(&csk->retry_timer, NULL, (unsigned long)csk); 573 rwlock_init(&csk->callback_lock); 574 csk->cdev = cdev; 575 csk->flags = 0; 576 cxgbi_sock_set_state(csk, CTP_CLOSED); 577 578 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk); 579 580 return csk; 581 } 582 583 static struct rtable *find_route_ipv4(struct flowi4 *fl4, 584 __be32 saddr, __be32 daddr, 585 __be16 sport, __be16 dport, u8 tos) 586 { 587 struct rtable *rt; 588 589 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr, 590 dport, sport, IPPROTO_TCP, tos, 0); 591 if (IS_ERR(rt)) 592 return NULL; 593 594 return rt; 595 } 596 597 static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr) 598 { 599 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr; 600 struct dst_entry *dst; 601 struct net_device *ndev; 602 struct cxgbi_device *cdev; 603 struct rtable *rt = NULL; 604 struct neighbour *n; 605 struct flowi4 fl4; 606 struct cxgbi_sock *csk = NULL; 607 unsigned int mtu = 0; 608 int port = 0xFFFF; 609 int err = 0; 610 611 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0); 612 if (!rt) { 613 pr_info("no route to ipv4 0x%x, port %u.\n", 614 be32_to_cpu(daddr->sin_addr.s_addr), 615 be16_to_cpu(daddr->sin_port)); 616 err = -ENETUNREACH; 617 goto err_out; 618 } 619 dst = &rt->dst; 620 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr); 621 if (!n) { 622 err = -ENODEV; 623 goto rel_rt; 624 } 625 ndev = n->dev; 626 627 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { 628 pr_info("multi-cast route %pI4, port %u, dev %s.\n", 629 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 630 ndev->name); 631 err = -ENETUNREACH; 632 goto rel_neigh; 633 } 634 635 if (ndev->flags & IFF_LOOPBACK) { 636 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr); 637 mtu = ndev->mtu; 638 pr_info("rt dev %s, loopback -> %s, mtu %u.\n", 639 n->dev->name, ndev->name, mtu); 640 } 641 642 cdev = cxgbi_device_find_by_netdev(ndev, &port); 643 if (!cdev) { 644 pr_info("dst %pI4, %s, NOT cxgbi device.\n", 645 &daddr->sin_addr.s_addr, ndev->name); 646 err = -ENETUNREACH; 647 goto rel_neigh; 648 } 649 log_debug(1 << CXGBI_DBG_SOCK, 650 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n", 651 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 652 port, ndev->name, cdev); 653 654 csk = cxgbi_sock_create(cdev); 655 if (!csk) { 656 err = -ENOMEM; 657 goto rel_neigh; 658 } 659 csk->cdev = cdev; 660 csk->port_id = port; 661 csk->mtu = mtu; 662 csk->dst = dst; 663 664 csk->csk_family = AF_INET; 665 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr; 666 csk->daddr.sin_port = daddr->sin_port; 667 csk->daddr.sin_family = daddr->sin_family; 668 csk->saddr.sin_family = daddr->sin_family; 669 csk->saddr.sin_addr.s_addr = fl4.saddr; 670 neigh_release(n); 671 672 return csk; 673 674 rel_neigh: 675 neigh_release(n); 676 677 rel_rt: 678 ip_rt_put(rt); 679 if (csk) 680 cxgbi_sock_closed(csk); 681 err_out: 682 return ERR_PTR(err); 683 } 684 685 #if IS_ENABLED(CONFIG_IPV6) 686 static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr, 687 const struct in6_addr *daddr) 688 { 689 struct flowi6 fl; 690 691 memset(&fl, 0, sizeof(fl)); 692 if (saddr) 693 memcpy(&fl.saddr, saddr, sizeof(struct in6_addr)); 694 if (daddr) 695 memcpy(&fl.daddr, daddr, sizeof(struct in6_addr)); 696 return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl); 697 } 698 699 static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr) 700 { 701 struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr; 702 struct dst_entry *dst; 703 struct net_device *ndev; 704 struct cxgbi_device *cdev; 705 struct rt6_info *rt = NULL; 706 struct neighbour *n; 707 struct in6_addr pref_saddr; 708 struct cxgbi_sock *csk = NULL; 709 unsigned int mtu = 0; 710 int port = 0xFFFF; 711 int err = 0; 712 713 rt = find_route_ipv6(NULL, &daddr6->sin6_addr); 714 715 if (!rt) { 716 pr_info("no route to ipv6 %pI6 port %u\n", 717 daddr6->sin6_addr.s6_addr, 718 be16_to_cpu(daddr6->sin6_port)); 719 err = -ENETUNREACH; 720 goto err_out; 721 } 722 723 dst = &rt->dst; 724 725 n = dst_neigh_lookup(dst, &daddr6->sin6_addr); 726 727 if (!n) { 728 pr_info("%pI6, port %u, dst no neighbour.\n", 729 daddr6->sin6_addr.s6_addr, 730 be16_to_cpu(daddr6->sin6_port)); 731 err = -ENETUNREACH; 732 goto rel_rt; 733 } 734 ndev = n->dev; 735 736 if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) { 737 pr_info("multi-cast route %pI6 port %u, dev %s.\n", 738 daddr6->sin6_addr.s6_addr, 739 ntohs(daddr6->sin6_port), ndev->name); 740 err = -ENETUNREACH; 741 goto rel_rt; 742 } 743 744 cdev = cxgbi_device_find_by_netdev(ndev, &port); 745 if (!cdev) 746 cdev = cxgbi_device_find_by_mac(ndev, &port); 747 if (!cdev) { 748 pr_info("dst %pI6 %s, NOT cxgbi device.\n", 749 daddr6->sin6_addr.s6_addr, ndev->name); 750 err = -ENETUNREACH; 751 goto rel_rt; 752 } 753 log_debug(1 << CXGBI_DBG_SOCK, 754 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n", 755 daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port, 756 ndev->name, cdev); 757 758 csk = cxgbi_sock_create(cdev); 759 if (!csk) { 760 err = -ENOMEM; 761 goto rel_rt; 762 } 763 csk->cdev = cdev; 764 csk->port_id = port; 765 csk->mtu = mtu; 766 csk->dst = dst; 767 768 if (ipv6_addr_any(&rt->rt6i_prefsrc.addr)) { 769 struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt); 770 771 err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL, 772 &daddr6->sin6_addr, 0, &pref_saddr); 773 if (err) { 774 pr_info("failed to get source address to reach %pI6\n", 775 &daddr6->sin6_addr); 776 goto rel_rt; 777 } 778 } else { 779 pref_saddr = rt->rt6i_prefsrc.addr; 780 } 781 782 csk->csk_family = AF_INET6; 783 csk->daddr6.sin6_addr = daddr6->sin6_addr; 784 csk->daddr6.sin6_port = daddr6->sin6_port; 785 csk->daddr6.sin6_family = daddr6->sin6_family; 786 csk->saddr6.sin6_family = daddr6->sin6_family; 787 csk->saddr6.sin6_addr = pref_saddr; 788 789 neigh_release(n); 790 return csk; 791 792 rel_rt: 793 if (n) 794 neigh_release(n); 795 796 ip6_rt_put(rt); 797 if (csk) 798 cxgbi_sock_closed(csk); 799 err_out: 800 return ERR_PTR(err); 801 } 802 #endif /* IS_ENABLED(CONFIG_IPV6) */ 803 804 void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn, 805 unsigned int opt) 806 { 807 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn; 808 dst_confirm(csk->dst); 809 smp_mb(); 810 cxgbi_sock_set_state(csk, CTP_ESTABLISHED); 811 } 812 EXPORT_SYMBOL_GPL(cxgbi_sock_established); 813 814 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk) 815 { 816 log_debug(1 << CXGBI_DBG_SOCK, 817 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n", 818 csk, csk->state, csk->flags, csk->user_data); 819 820 if (csk->state != CTP_ESTABLISHED) { 821 read_lock_bh(&csk->callback_lock); 822 if (csk->user_data) 823 iscsi_conn_failure(csk->user_data, 824 ISCSI_ERR_TCP_CONN_CLOSE); 825 read_unlock_bh(&csk->callback_lock); 826 } 827 } 828 829 void cxgbi_sock_closed(struct cxgbi_sock *csk) 830 { 831 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 832 csk, (csk)->state, (csk)->flags, (csk)->tid); 833 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 834 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED) 835 return; 836 if (csk->saddr.sin_port) 837 sock_put_port(csk); 838 if (csk->dst) 839 dst_release(csk->dst); 840 csk->cdev->csk_release_offload_resources(csk); 841 cxgbi_sock_set_state(csk, CTP_CLOSED); 842 cxgbi_inform_iscsi_conn_closing(csk); 843 cxgbi_sock_put(csk); 844 } 845 EXPORT_SYMBOL_GPL(cxgbi_sock_closed); 846 847 static void need_active_close(struct cxgbi_sock *csk) 848 { 849 int data_lost; 850 int close_req = 0; 851 852 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 853 csk, (csk)->state, (csk)->flags, (csk)->tid); 854 spin_lock_bh(&csk->lock); 855 dst_confirm(csk->dst); 856 data_lost = skb_queue_len(&csk->receive_queue); 857 __skb_queue_purge(&csk->receive_queue); 858 859 if (csk->state == CTP_ACTIVE_OPEN) 860 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 861 else if (csk->state == CTP_ESTABLISHED) { 862 close_req = 1; 863 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE); 864 } else if (csk->state == CTP_PASSIVE_CLOSE) { 865 close_req = 1; 866 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 867 } 868 869 if (close_req) { 870 if (data_lost) 871 csk->cdev->csk_send_abort_req(csk); 872 else 873 csk->cdev->csk_send_close_req(csk); 874 } 875 876 spin_unlock_bh(&csk->lock); 877 } 878 879 void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno) 880 { 881 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n", 882 csk, csk->state, csk->flags, 883 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port, 884 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port, 885 errno); 886 887 cxgbi_sock_set_state(csk, CTP_CONNECTING); 888 csk->err = errno; 889 cxgbi_sock_closed(csk); 890 } 891 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open); 892 893 void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb) 894 { 895 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk; 896 897 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 898 csk, (csk)->state, (csk)->flags, (csk)->tid); 899 cxgbi_sock_get(csk); 900 spin_lock_bh(&csk->lock); 901 if (csk->state == CTP_ACTIVE_OPEN) 902 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH); 903 spin_unlock_bh(&csk->lock); 904 cxgbi_sock_put(csk); 905 __kfree_skb(skb); 906 } 907 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure); 908 909 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk) 910 { 911 cxgbi_sock_get(csk); 912 spin_lock_bh(&csk->lock); 913 914 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD); 915 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 916 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING); 917 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) 918 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n", 919 csk, csk->state, csk->flags, csk->tid); 920 cxgbi_sock_closed(csk); 921 } 922 923 spin_unlock_bh(&csk->lock); 924 cxgbi_sock_put(csk); 925 } 926 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl); 927 928 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk) 929 { 930 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 931 csk, (csk)->state, (csk)->flags, (csk)->tid); 932 cxgbi_sock_get(csk); 933 spin_lock_bh(&csk->lock); 934 935 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 936 goto done; 937 938 switch (csk->state) { 939 case CTP_ESTABLISHED: 940 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE); 941 break; 942 case CTP_ACTIVE_CLOSE: 943 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 944 break; 945 case CTP_CLOSE_WAIT_1: 946 cxgbi_sock_closed(csk); 947 break; 948 case CTP_ABORTING: 949 break; 950 default: 951 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 952 csk, csk->state, csk->flags, csk->tid); 953 } 954 cxgbi_inform_iscsi_conn_closing(csk); 955 done: 956 spin_unlock_bh(&csk->lock); 957 cxgbi_sock_put(csk); 958 } 959 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close); 960 961 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt) 962 { 963 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 964 csk, (csk)->state, (csk)->flags, (csk)->tid); 965 cxgbi_sock_get(csk); 966 spin_lock_bh(&csk->lock); 967 968 csk->snd_una = snd_nxt - 1; 969 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 970 goto done; 971 972 switch (csk->state) { 973 case CTP_ACTIVE_CLOSE: 974 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1); 975 break; 976 case CTP_CLOSE_WAIT_1: 977 case CTP_CLOSE_WAIT_2: 978 cxgbi_sock_closed(csk); 979 break; 980 case CTP_ABORTING: 981 break; 982 default: 983 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 984 csk, csk->state, csk->flags, csk->tid); 985 } 986 done: 987 spin_unlock_bh(&csk->lock); 988 cxgbi_sock_put(csk); 989 } 990 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl); 991 992 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits, 993 unsigned int snd_una, int seq_chk) 994 { 995 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 996 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n", 997 csk, csk->state, csk->flags, csk->tid, credits, 998 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk); 999 1000 spin_lock_bh(&csk->lock); 1001 1002 csk->wr_cred += credits; 1003 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred) 1004 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 1005 1006 while (credits) { 1007 struct sk_buff *p = cxgbi_sock_peek_wr(csk); 1008 1009 if (unlikely(!p)) { 1010 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n", 1011 csk, csk->state, csk->flags, csk->tid, credits, 1012 csk->wr_cred, csk->wr_una_cred); 1013 break; 1014 } 1015 1016 if (unlikely(credits < p->csum)) { 1017 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n", 1018 csk, csk->state, csk->flags, csk->tid, 1019 credits, csk->wr_cred, csk->wr_una_cred, 1020 p->csum); 1021 p->csum -= credits; 1022 break; 1023 } else { 1024 cxgbi_sock_dequeue_wr(csk); 1025 credits -= p->csum; 1026 kfree_skb(p); 1027 } 1028 } 1029 1030 cxgbi_sock_check_wr_invariants(csk); 1031 1032 if (seq_chk) { 1033 if (unlikely(before(snd_una, csk->snd_una))) { 1034 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.", 1035 csk, csk->state, csk->flags, csk->tid, snd_una, 1036 csk->snd_una); 1037 goto done; 1038 } 1039 1040 if (csk->snd_una != snd_una) { 1041 csk->snd_una = snd_una; 1042 dst_confirm(csk->dst); 1043 } 1044 } 1045 1046 if (skb_queue_len(&csk->write_queue)) { 1047 if (csk->cdev->csk_push_tx_frames(csk, 0)) 1048 cxgbi_conn_tx_open(csk); 1049 } else 1050 cxgbi_conn_tx_open(csk); 1051 done: 1052 spin_unlock_bh(&csk->lock); 1053 } 1054 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack); 1055 1056 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk, 1057 unsigned short mtu) 1058 { 1059 int i = 0; 1060 1061 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu) 1062 ++i; 1063 1064 return i; 1065 } 1066 1067 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu) 1068 { 1069 unsigned int idx; 1070 struct dst_entry *dst = csk->dst; 1071 1072 csk->advmss = dst_metric_advmss(dst); 1073 1074 if (csk->advmss > pmtu - 40) 1075 csk->advmss = pmtu - 40; 1076 if (csk->advmss < csk->cdev->mtus[0] - 40) 1077 csk->advmss = csk->cdev->mtus[0] - 40; 1078 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40); 1079 1080 return idx; 1081 } 1082 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss); 1083 1084 void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb) 1085 { 1086 cxgbi_skcb_tcp_seq(skb) = csk->write_seq; 1087 __skb_queue_tail(&csk->write_queue, skb); 1088 } 1089 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail); 1090 1091 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk) 1092 { 1093 struct sk_buff *skb; 1094 1095 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL) 1096 kfree_skb(skb); 1097 } 1098 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue); 1099 1100 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk) 1101 { 1102 int pending = cxgbi_sock_count_pending_wrs(csk); 1103 1104 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred)) 1105 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 1106 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred); 1107 } 1108 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants); 1109 1110 static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb) 1111 { 1112 struct cxgbi_device *cdev = csk->cdev; 1113 struct sk_buff *next; 1114 int err, copied = 0; 1115 1116 spin_lock_bh(&csk->lock); 1117 1118 if (csk->state != CTP_ESTABLISHED) { 1119 log_debug(1 << CXGBI_DBG_PDU_TX, 1120 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n", 1121 csk, csk->state, csk->flags, csk->tid); 1122 err = -EAGAIN; 1123 goto out_err; 1124 } 1125 1126 if (csk->err) { 1127 log_debug(1 << CXGBI_DBG_PDU_TX, 1128 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n", 1129 csk, csk->state, csk->flags, csk->tid, csk->err); 1130 err = -EPIPE; 1131 goto out_err; 1132 } 1133 1134 if (csk->write_seq - csk->snd_una >= csk->snd_win) { 1135 log_debug(1 << CXGBI_DBG_PDU_TX, 1136 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n", 1137 csk, csk->state, csk->flags, csk->tid, csk->write_seq, 1138 csk->snd_una, csk->snd_win); 1139 err = -ENOBUFS; 1140 goto out_err; 1141 } 1142 1143 while (skb) { 1144 int frags = skb_shinfo(skb)->nr_frags + 1145 (skb->len != skb->data_len); 1146 1147 if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) { 1148 pr_err("csk 0x%p, skb head %u < %u.\n", 1149 csk, skb_headroom(skb), cdev->skb_tx_rsvd); 1150 err = -EINVAL; 1151 goto out_err; 1152 } 1153 1154 if (frags >= SKB_WR_LIST_SIZE) { 1155 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n", 1156 csk, skb_shinfo(skb)->nr_frags, skb->len, 1157 skb->data_len, (uint)(SKB_WR_LIST_SIZE)); 1158 err = -EINVAL; 1159 goto out_err; 1160 } 1161 1162 next = skb->next; 1163 skb->next = NULL; 1164 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR); 1165 cxgbi_sock_skb_entail(csk, skb); 1166 copied += skb->len; 1167 csk->write_seq += skb->len + 1168 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 1169 skb = next; 1170 } 1171 done: 1172 if (likely(skb_queue_len(&csk->write_queue))) 1173 cdev->csk_push_tx_frames(csk, 1); 1174 spin_unlock_bh(&csk->lock); 1175 return copied; 1176 1177 out_err: 1178 if (copied == 0 && err == -EPIPE) 1179 copied = csk->err ? csk->err : -EPIPE; 1180 else 1181 copied = err; 1182 goto done; 1183 } 1184 1185 /* 1186 * Direct Data Placement - 1187 * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted 1188 * final destination host-memory buffers based on the Initiator Task Tag (ITT) 1189 * in Data-In or Target Task Tag (TTT) in Data-Out PDUs. 1190 * The host memory address is programmed into h/w in the format of pagepod 1191 * entries. 1192 * The location of the pagepod entry is encoded into ddp tag which is used as 1193 * the base for ITT/TTT. 1194 */ 1195 1196 static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4}; 1197 static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16}; 1198 static unsigned char page_idx = DDP_PGIDX_MAX; 1199 1200 static unsigned char sw_tag_idx_bits; 1201 static unsigned char sw_tag_age_bits; 1202 1203 /* 1204 * Direct-Data Placement page size adjustment 1205 */ 1206 static int ddp_adjust_page_table(void) 1207 { 1208 int i; 1209 unsigned int base_order, order; 1210 1211 if (PAGE_SIZE < (1UL << ddp_page_shift[0])) { 1212 pr_info("PAGE_SIZE 0x%lx too small, min 0x%lx\n", 1213 PAGE_SIZE, 1UL << ddp_page_shift[0]); 1214 return -EINVAL; 1215 } 1216 1217 base_order = get_order(1UL << ddp_page_shift[0]); 1218 order = get_order(1UL << PAGE_SHIFT); 1219 1220 for (i = 0; i < DDP_PGIDX_MAX; i++) { 1221 /* first is the kernel page size, then just doubling */ 1222 ddp_page_order[i] = order - base_order + i; 1223 ddp_page_shift[i] = PAGE_SHIFT + i; 1224 } 1225 return 0; 1226 } 1227 1228 static int ddp_find_page_index(unsigned long pgsz) 1229 { 1230 int i; 1231 1232 for (i = 0; i < DDP_PGIDX_MAX; i++) { 1233 if (pgsz == (1UL << ddp_page_shift[i])) 1234 return i; 1235 } 1236 pr_info("ddp page size %lu not supported.\n", pgsz); 1237 return DDP_PGIDX_MAX; 1238 } 1239 1240 static void ddp_setup_host_page_size(void) 1241 { 1242 if (page_idx == DDP_PGIDX_MAX) { 1243 page_idx = ddp_find_page_index(PAGE_SIZE); 1244 1245 if (page_idx == DDP_PGIDX_MAX) { 1246 pr_info("system PAGE %lu, update hw.\n", PAGE_SIZE); 1247 if (ddp_adjust_page_table() < 0) { 1248 pr_info("PAGE %lu, disable ddp.\n", PAGE_SIZE); 1249 return; 1250 } 1251 page_idx = ddp_find_page_index(PAGE_SIZE); 1252 } 1253 pr_info("system PAGE %lu, ddp idx %u.\n", PAGE_SIZE, page_idx); 1254 } 1255 } 1256 1257 void cxgbi_ddp_page_size_factor(int *pgsz_factor) 1258 { 1259 int i; 1260 1261 for (i = 0; i < DDP_PGIDX_MAX; i++) 1262 pgsz_factor[i] = ddp_page_order[i]; 1263 } 1264 EXPORT_SYMBOL_GPL(cxgbi_ddp_page_size_factor); 1265 1266 /* 1267 * DDP setup & teardown 1268 */ 1269 1270 void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *ppod, 1271 struct cxgbi_pagepod_hdr *hdr, 1272 struct cxgbi_gather_list *gl, unsigned int gidx) 1273 { 1274 int i; 1275 1276 memcpy(ppod, hdr, sizeof(*hdr)); 1277 for (i = 0; i < (PPOD_PAGES_MAX + 1); i++, gidx++) { 1278 ppod->addr[i] = gidx < gl->nelem ? 1279 cpu_to_be64(gl->phys_addr[gidx]) : 0ULL; 1280 } 1281 } 1282 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_set); 1283 1284 void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *ppod) 1285 { 1286 memset(ppod, 0, sizeof(*ppod)); 1287 } 1288 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_clear); 1289 1290 static inline int ddp_find_unused_entries(struct cxgbi_ddp_info *ddp, 1291 unsigned int start, unsigned int max, 1292 unsigned int count, 1293 struct cxgbi_gather_list *gl) 1294 { 1295 unsigned int i, j, k; 1296 1297 /* not enough entries */ 1298 if ((max - start) < count) { 1299 log_debug(1 << CXGBI_DBG_DDP, 1300 "NOT enough entries %u+%u < %u.\n", start, count, max); 1301 return -EBUSY; 1302 } 1303 1304 max -= count; 1305 spin_lock(&ddp->map_lock); 1306 for (i = start; i < max;) { 1307 for (j = 0, k = i; j < count; j++, k++) { 1308 if (ddp->gl_map[k]) 1309 break; 1310 } 1311 if (j == count) { 1312 for (j = 0, k = i; j < count; j++, k++) 1313 ddp->gl_map[k] = gl; 1314 spin_unlock(&ddp->map_lock); 1315 return i; 1316 } 1317 i += j + 1; 1318 } 1319 spin_unlock(&ddp->map_lock); 1320 log_debug(1 << CXGBI_DBG_DDP, 1321 "NO suitable entries %u available.\n", count); 1322 return -EBUSY; 1323 } 1324 1325 static inline void ddp_unmark_entries(struct cxgbi_ddp_info *ddp, 1326 int start, int count) 1327 { 1328 spin_lock(&ddp->map_lock); 1329 memset(&ddp->gl_map[start], 0, 1330 count * sizeof(struct cxgbi_gather_list *)); 1331 spin_unlock(&ddp->map_lock); 1332 } 1333 1334 static inline void ddp_gl_unmap(struct pci_dev *pdev, 1335 struct cxgbi_gather_list *gl) 1336 { 1337 int i; 1338 1339 for (i = 0; i < gl->nelem; i++) 1340 dma_unmap_page(&pdev->dev, gl->phys_addr[i], PAGE_SIZE, 1341 PCI_DMA_FROMDEVICE); 1342 } 1343 1344 static inline int ddp_gl_map(struct pci_dev *pdev, 1345 struct cxgbi_gather_list *gl) 1346 { 1347 int i; 1348 1349 for (i = 0; i < gl->nelem; i++) { 1350 gl->phys_addr[i] = dma_map_page(&pdev->dev, gl->pages[i], 0, 1351 PAGE_SIZE, 1352 PCI_DMA_FROMDEVICE); 1353 if (unlikely(dma_mapping_error(&pdev->dev, gl->phys_addr[i]))) { 1354 log_debug(1 << CXGBI_DBG_DDP, 1355 "page %d 0x%p, 0x%p dma mapping err.\n", 1356 i, gl->pages[i], pdev); 1357 goto unmap; 1358 } 1359 } 1360 return i; 1361 unmap: 1362 if (i) { 1363 unsigned int nelem = gl->nelem; 1364 1365 gl->nelem = i; 1366 ddp_gl_unmap(pdev, gl); 1367 gl->nelem = nelem; 1368 } 1369 return -EINVAL; 1370 } 1371 1372 static void ddp_release_gl(struct cxgbi_gather_list *gl, 1373 struct pci_dev *pdev) 1374 { 1375 ddp_gl_unmap(pdev, gl); 1376 kfree(gl); 1377 } 1378 1379 static struct cxgbi_gather_list *ddp_make_gl(unsigned int xferlen, 1380 struct scatterlist *sgl, 1381 unsigned int sgcnt, 1382 struct pci_dev *pdev, 1383 gfp_t gfp) 1384 { 1385 struct cxgbi_gather_list *gl; 1386 struct scatterlist *sg = sgl; 1387 struct page *sgpage = sg_page(sg); 1388 unsigned int sglen = sg->length; 1389 unsigned int sgoffset = sg->offset; 1390 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >> 1391 PAGE_SHIFT; 1392 int i = 1, j = 0; 1393 1394 if (xferlen < DDP_THRESHOLD) { 1395 log_debug(1 << CXGBI_DBG_DDP, 1396 "xfer %u < threshold %u, no ddp.\n", 1397 xferlen, DDP_THRESHOLD); 1398 return NULL; 1399 } 1400 1401 gl = kzalloc(sizeof(struct cxgbi_gather_list) + 1402 npages * (sizeof(dma_addr_t) + 1403 sizeof(struct page *)), gfp); 1404 if (!gl) { 1405 log_debug(1 << CXGBI_DBG_DDP, 1406 "xfer %u, %u pages, OOM.\n", xferlen, npages); 1407 return NULL; 1408 } 1409 1410 log_debug(1 << CXGBI_DBG_DDP, 1411 "xfer %u, sgl %u, gl max %u.\n", xferlen, sgcnt, npages); 1412 1413 gl->pages = (struct page **)&gl->phys_addr[npages]; 1414 gl->nelem = npages; 1415 gl->length = xferlen; 1416 gl->offset = sgoffset; 1417 gl->pages[0] = sgpage; 1418 1419 for (i = 1, sg = sg_next(sgl), j = 0; i < sgcnt; 1420 i++, sg = sg_next(sg)) { 1421 struct page *page = sg_page(sg); 1422 1423 if (sgpage == page && sg->offset == sgoffset + sglen) 1424 sglen += sg->length; 1425 else { 1426 /* make sure the sgl is fit for ddp: 1427 * each has the same page size, and 1428 * all of the middle pages are used completely 1429 */ 1430 if ((j && sgoffset) || ((i != sgcnt - 1) && 1431 ((sglen + sgoffset) & ~PAGE_MASK))) { 1432 log_debug(1 << CXGBI_DBG_DDP, 1433 "page %d/%u, %u + %u.\n", 1434 i, sgcnt, sgoffset, sglen); 1435 goto error_out; 1436 } 1437 1438 j++; 1439 if (j == gl->nelem || sg->offset) { 1440 log_debug(1 << CXGBI_DBG_DDP, 1441 "page %d/%u, offset %u.\n", 1442 j, gl->nelem, sg->offset); 1443 goto error_out; 1444 } 1445 gl->pages[j] = page; 1446 sglen = sg->length; 1447 sgoffset = sg->offset; 1448 sgpage = page; 1449 } 1450 } 1451 gl->nelem = ++j; 1452 1453 if (ddp_gl_map(pdev, gl) < 0) 1454 goto error_out; 1455 1456 return gl; 1457 1458 error_out: 1459 kfree(gl); 1460 return NULL; 1461 } 1462 1463 static void ddp_tag_release(struct cxgbi_hba *chba, u32 tag) 1464 { 1465 struct cxgbi_device *cdev = chba->cdev; 1466 struct cxgbi_ddp_info *ddp = cdev->ddp; 1467 u32 idx; 1468 1469 idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask; 1470 if (idx < ddp->nppods) { 1471 struct cxgbi_gather_list *gl = ddp->gl_map[idx]; 1472 unsigned int npods; 1473 1474 if (!gl || !gl->nelem) { 1475 pr_warn("tag 0x%x, idx %u, gl 0x%p, %u.\n", 1476 tag, idx, gl, gl ? gl->nelem : 0); 1477 return; 1478 } 1479 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT; 1480 log_debug(1 << CXGBI_DBG_DDP, 1481 "tag 0x%x, release idx %u, npods %u.\n", 1482 tag, idx, npods); 1483 cdev->csk_ddp_clear(chba, tag, idx, npods); 1484 ddp_unmark_entries(ddp, idx, npods); 1485 ddp_release_gl(gl, ddp->pdev); 1486 } else 1487 pr_warn("tag 0x%x, idx %u > max %u.\n", tag, idx, ddp->nppods); 1488 } 1489 1490 static int ddp_tag_reserve(struct cxgbi_sock *csk, unsigned int tid, 1491 u32 sw_tag, u32 *tagp, struct cxgbi_gather_list *gl, 1492 gfp_t gfp) 1493 { 1494 struct cxgbi_device *cdev = csk->cdev; 1495 struct cxgbi_ddp_info *ddp = cdev->ddp; 1496 struct cxgbi_tag_format *tformat = &cdev->tag_format; 1497 struct cxgbi_pagepod_hdr hdr; 1498 unsigned int npods; 1499 int idx = -1; 1500 int err = -ENOMEM; 1501 u32 tag; 1502 1503 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT; 1504 if (ddp->idx_last == ddp->nppods) 1505 idx = ddp_find_unused_entries(ddp, 0, ddp->nppods, 1506 npods, gl); 1507 else { 1508 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, 1509 ddp->nppods, npods, 1510 gl); 1511 if (idx < 0 && ddp->idx_last >= npods) { 1512 idx = ddp_find_unused_entries(ddp, 0, 1513 min(ddp->idx_last + npods, ddp->nppods), 1514 npods, gl); 1515 } 1516 } 1517 if (idx < 0) { 1518 log_debug(1 << CXGBI_DBG_DDP, 1519 "xferlen %u, gl %u, npods %u NO DDP.\n", 1520 gl->length, gl->nelem, npods); 1521 return idx; 1522 } 1523 1524 tag = cxgbi_ddp_tag_base(tformat, sw_tag); 1525 tag |= idx << PPOD_IDX_SHIFT; 1526 1527 hdr.rsvd = 0; 1528 hdr.vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid)); 1529 hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask); 1530 hdr.max_offset = htonl(gl->length); 1531 hdr.page_offset = htonl(gl->offset); 1532 1533 err = cdev->csk_ddp_set(csk, &hdr, idx, npods, gl); 1534 if (err < 0) 1535 goto unmark_entries; 1536 1537 ddp->idx_last = idx; 1538 log_debug(1 << CXGBI_DBG_DDP, 1539 "xfer %u, gl %u,%u, tid 0x%x, tag 0x%x->0x%x(%u,%u).\n", 1540 gl->length, gl->nelem, gl->offset, tid, sw_tag, tag, idx, 1541 npods); 1542 *tagp = tag; 1543 return 0; 1544 1545 unmark_entries: 1546 ddp_unmark_entries(ddp, idx, npods); 1547 return err; 1548 } 1549 1550 int cxgbi_ddp_reserve(struct cxgbi_sock *csk, unsigned int *tagp, 1551 unsigned int sw_tag, unsigned int xferlen, 1552 struct scatterlist *sgl, unsigned int sgcnt, gfp_t gfp) 1553 { 1554 struct cxgbi_device *cdev = csk->cdev; 1555 struct cxgbi_tag_format *tformat = &cdev->tag_format; 1556 struct cxgbi_gather_list *gl; 1557 int err; 1558 1559 if (page_idx >= DDP_PGIDX_MAX || !cdev->ddp || 1560 xferlen < DDP_THRESHOLD) { 1561 log_debug(1 << CXGBI_DBG_DDP, 1562 "pgidx %u, xfer %u, NO ddp.\n", page_idx, xferlen); 1563 return -EINVAL; 1564 } 1565 1566 if (!cxgbi_sw_tag_usable(tformat, sw_tag)) { 1567 log_debug(1 << CXGBI_DBG_DDP, 1568 "sw_tag 0x%x NOT usable.\n", sw_tag); 1569 return -EINVAL; 1570 } 1571 1572 gl = ddp_make_gl(xferlen, sgl, sgcnt, cdev->pdev, gfp); 1573 if (!gl) 1574 return -ENOMEM; 1575 1576 err = ddp_tag_reserve(csk, csk->tid, sw_tag, tagp, gl, gfp); 1577 if (err < 0) 1578 ddp_release_gl(gl, cdev->pdev); 1579 1580 return err; 1581 } 1582 1583 static void ddp_destroy(struct kref *kref) 1584 { 1585 struct cxgbi_ddp_info *ddp = container_of(kref, 1586 struct cxgbi_ddp_info, 1587 refcnt); 1588 struct cxgbi_device *cdev = ddp->cdev; 1589 int i = 0; 1590 1591 pr_info("kref 0, destroy ddp 0x%p, cdev 0x%p.\n", ddp, cdev); 1592 1593 while (i < ddp->nppods) { 1594 struct cxgbi_gather_list *gl = ddp->gl_map[i]; 1595 1596 if (gl) { 1597 int npods = (gl->nelem + PPOD_PAGES_MAX - 1) 1598 >> PPOD_PAGES_SHIFT; 1599 pr_info("cdev 0x%p, ddp %d + %d.\n", cdev, i, npods); 1600 kfree(gl); 1601 i += npods; 1602 } else 1603 i++; 1604 } 1605 cxgbi_free_big_mem(ddp); 1606 } 1607 1608 int cxgbi_ddp_cleanup(struct cxgbi_device *cdev) 1609 { 1610 struct cxgbi_ddp_info *ddp = cdev->ddp; 1611 1612 log_debug(1 << CXGBI_DBG_DDP, 1613 "cdev 0x%p, release ddp 0x%p.\n", cdev, ddp); 1614 cdev->ddp = NULL; 1615 if (ddp) 1616 return kref_put(&ddp->refcnt, ddp_destroy); 1617 return 0; 1618 } 1619 EXPORT_SYMBOL_GPL(cxgbi_ddp_cleanup); 1620 1621 int cxgbi_ddp_init(struct cxgbi_device *cdev, 1622 unsigned int llimit, unsigned int ulimit, 1623 unsigned int max_txsz, unsigned int max_rxsz) 1624 { 1625 struct cxgbi_ddp_info *ddp; 1626 unsigned int ppmax, bits; 1627 1628 ppmax = (ulimit - llimit + 1) >> PPOD_SIZE_SHIFT; 1629 bits = __ilog2_u32(ppmax) + 1; 1630 if (bits > PPOD_IDX_MAX_SIZE) 1631 bits = PPOD_IDX_MAX_SIZE; 1632 ppmax = (1 << (bits - 1)) - 1; 1633 1634 ddp = cxgbi_alloc_big_mem(sizeof(struct cxgbi_ddp_info) + 1635 ppmax * (sizeof(struct cxgbi_gather_list *) + 1636 sizeof(struct sk_buff *)), 1637 GFP_KERNEL); 1638 if (!ddp) { 1639 pr_warn("cdev 0x%p, ddp ppmax %u OOM.\n", cdev, ppmax); 1640 return -ENOMEM; 1641 } 1642 ddp->gl_map = (struct cxgbi_gather_list **)(ddp + 1); 1643 cdev->ddp = ddp; 1644 1645 spin_lock_init(&ddp->map_lock); 1646 kref_init(&ddp->refcnt); 1647 1648 ddp->cdev = cdev; 1649 ddp->pdev = cdev->pdev; 1650 ddp->llimit = llimit; 1651 ddp->ulimit = ulimit; 1652 ddp->max_txsz = min_t(unsigned int, max_txsz, ULP2_MAX_PKT_SIZE); 1653 ddp->max_rxsz = min_t(unsigned int, max_rxsz, ULP2_MAX_PKT_SIZE); 1654 ddp->nppods = ppmax; 1655 ddp->idx_last = ppmax; 1656 ddp->idx_bits = bits; 1657 ddp->idx_mask = (1 << bits) - 1; 1658 ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1; 1659 1660 cdev->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits; 1661 cdev->tag_format.rsvd_bits = ddp->idx_bits; 1662 cdev->tag_format.rsvd_shift = PPOD_IDX_SHIFT; 1663 cdev->tag_format.rsvd_mask = (1 << cdev->tag_format.rsvd_bits) - 1; 1664 1665 pr_info("%s tag format, sw %u, rsvd %u,%u, mask 0x%x.\n", 1666 cdev->ports[0]->name, cdev->tag_format.sw_bits, 1667 cdev->tag_format.rsvd_bits, cdev->tag_format.rsvd_shift, 1668 cdev->tag_format.rsvd_mask); 1669 1670 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1671 ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN); 1672 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1673 ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN); 1674 1675 log_debug(1 << CXGBI_DBG_DDP, 1676 "%s max payload size: %u/%u, %u/%u.\n", 1677 cdev->ports[0]->name, cdev->tx_max_size, ddp->max_txsz, 1678 cdev->rx_max_size, ddp->max_rxsz); 1679 return 0; 1680 } 1681 EXPORT_SYMBOL_GPL(cxgbi_ddp_init); 1682 1683 /* 1684 * APIs interacting with open-iscsi libraries 1685 */ 1686 1687 static unsigned char padding[4]; 1688 1689 static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt) 1690 { 1691 struct scsi_cmnd *sc = task->sc; 1692 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1693 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1694 struct cxgbi_hba *chba = cconn->chba; 1695 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format; 1696 u32 tag = ntohl((__force u32)hdr_itt); 1697 1698 log_debug(1 << CXGBI_DBG_DDP, 1699 "cdev 0x%p, release tag 0x%x.\n", chba->cdev, tag); 1700 if (sc && 1701 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) && 1702 cxgbi_is_ddp_tag(tformat, tag)) 1703 ddp_tag_release(chba, tag); 1704 } 1705 1706 static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) 1707 { 1708 struct scsi_cmnd *sc = task->sc; 1709 struct iscsi_conn *conn = task->conn; 1710 struct iscsi_session *sess = conn->session; 1711 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1712 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1713 struct cxgbi_hba *chba = cconn->chba; 1714 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format; 1715 u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt; 1716 u32 tag = 0; 1717 int err = -EINVAL; 1718 1719 if (sc && 1720 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE)) { 1721 err = cxgbi_ddp_reserve(cconn->cep->csk, &tag, sw_tag, 1722 scsi_in(sc)->length, 1723 scsi_in(sc)->table.sgl, 1724 scsi_in(sc)->table.nents, 1725 GFP_ATOMIC); 1726 if (err < 0) 1727 log_debug(1 << CXGBI_DBG_DDP, 1728 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n", 1729 cconn->cep->csk, task, scsi_in(sc)->length, 1730 scsi_in(sc)->table.nents); 1731 } 1732 1733 if (err < 0) 1734 tag = cxgbi_set_non_ddp_tag(tformat, sw_tag); 1735 /* the itt need to sent in big-endian order */ 1736 *hdr_itt = (__force itt_t)htonl(tag); 1737 1738 log_debug(1 << CXGBI_DBG_DDP, 1739 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n", 1740 chba->cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt); 1741 return 0; 1742 } 1743 1744 void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age) 1745 { 1746 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1747 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1748 struct cxgbi_device *cdev = cconn->chba->cdev; 1749 u32 tag = ntohl((__force u32) itt); 1750 u32 sw_bits; 1751 1752 sw_bits = cxgbi_tag_nonrsvd_bits(&cdev->tag_format, tag); 1753 if (idx) 1754 *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1); 1755 if (age) 1756 *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK; 1757 1758 log_debug(1 << CXGBI_DBG_DDP, 1759 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n", 1760 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF, 1761 age ? *age : 0xFF); 1762 } 1763 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt); 1764 1765 void cxgbi_conn_tx_open(struct cxgbi_sock *csk) 1766 { 1767 struct iscsi_conn *conn = csk->user_data; 1768 1769 if (conn) { 1770 log_debug(1 << CXGBI_DBG_SOCK, 1771 "csk 0x%p, cid %d.\n", csk, conn->id); 1772 iscsi_conn_queue_work(conn); 1773 } 1774 } 1775 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open); 1776 1777 /* 1778 * pdu receive, interact with libiscsi_tcp 1779 */ 1780 static inline int read_pdu_skb(struct iscsi_conn *conn, 1781 struct sk_buff *skb, 1782 unsigned int offset, 1783 int offloaded) 1784 { 1785 int status = 0; 1786 int bytes_read; 1787 1788 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status); 1789 switch (status) { 1790 case ISCSI_TCP_CONN_ERR: 1791 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n", 1792 skb, offset, offloaded); 1793 return -EIO; 1794 case ISCSI_TCP_SUSPENDED: 1795 log_debug(1 << CXGBI_DBG_PDU_RX, 1796 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n", 1797 skb, offset, offloaded, bytes_read); 1798 /* no transfer - just have caller flush queue */ 1799 return bytes_read; 1800 case ISCSI_TCP_SKB_DONE: 1801 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n", 1802 skb, offset, offloaded); 1803 /* 1804 * pdus should always fit in the skb and we should get 1805 * segment done notifcation. 1806 */ 1807 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb."); 1808 return -EFAULT; 1809 case ISCSI_TCP_SEGMENT_DONE: 1810 log_debug(1 << CXGBI_DBG_PDU_RX, 1811 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n", 1812 skb, offset, offloaded, bytes_read); 1813 return bytes_read; 1814 default: 1815 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n", 1816 skb, offset, offloaded, status); 1817 return -EINVAL; 1818 } 1819 } 1820 1821 static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb) 1822 { 1823 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1824 1825 log_debug(1 << CXGBI_DBG_PDU_RX, 1826 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1827 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1828 1829 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) { 1830 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb); 1831 iscsi_conn_failure(conn, ISCSI_ERR_PROTO); 1832 return -EIO; 1833 } 1834 1835 if (conn->hdrdgst_en && 1836 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) { 1837 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb); 1838 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST); 1839 return -EIO; 1840 } 1841 1842 return read_pdu_skb(conn, skb, 0, 0); 1843 } 1844 1845 static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb, 1846 struct sk_buff *skb, unsigned int offset) 1847 { 1848 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1849 bool offloaded = 0; 1850 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK; 1851 1852 log_debug(1 << CXGBI_DBG_PDU_RX, 1853 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1854 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1855 1856 if (conn->datadgst_en && 1857 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) { 1858 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n", 1859 conn, lskb, cxgbi_skcb_flags(lskb)); 1860 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1861 return -EIO; 1862 } 1863 1864 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn)) 1865 return 0; 1866 1867 /* coalesced, add header digest length */ 1868 if (lskb == skb && conn->hdrdgst_en) 1869 offset += ISCSI_DIGEST_SIZE; 1870 1871 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD)) 1872 offloaded = 1; 1873 1874 if (opcode == ISCSI_OP_SCSI_DATA_IN) 1875 log_debug(1 << CXGBI_DBG_PDU_RX, 1876 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n", 1877 skb, opcode, ntohl(tcp_conn->in.hdr->itt), 1878 tcp_conn->in.datalen, offloaded ? "is" : "not"); 1879 1880 return read_pdu_skb(conn, skb, offset, offloaded); 1881 } 1882 1883 static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied) 1884 { 1885 struct cxgbi_device *cdev = csk->cdev; 1886 int must_send; 1887 u32 credits; 1888 1889 log_debug(1 << CXGBI_DBG_PDU_RX, 1890 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n", 1891 csk, csk->state, csk->flags, csk->tid, csk->copied_seq, 1892 csk->rcv_wup, cdev->rx_credit_thres, 1893 csk->rcv_win); 1894 1895 if (csk->state != CTP_ESTABLISHED) 1896 return; 1897 1898 credits = csk->copied_seq - csk->rcv_wup; 1899 if (unlikely(!credits)) 1900 return; 1901 if (unlikely(cdev->rx_credit_thres == 0)) 1902 return; 1903 1904 must_send = credits + 16384 >= csk->rcv_win; 1905 if (must_send || credits >= cdev->rx_credit_thres) 1906 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits); 1907 } 1908 1909 void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk) 1910 { 1911 struct cxgbi_device *cdev = csk->cdev; 1912 struct iscsi_conn *conn = csk->user_data; 1913 struct sk_buff *skb; 1914 unsigned int read = 0; 1915 int err = 0; 1916 1917 log_debug(1 << CXGBI_DBG_PDU_RX, 1918 "csk 0x%p, conn 0x%p.\n", csk, conn); 1919 1920 if (unlikely(!conn || conn->suspend_rx)) { 1921 log_debug(1 << CXGBI_DBG_PDU_RX, 1922 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n", 1923 csk, conn, conn ? conn->id : 0xFF, 1924 conn ? conn->suspend_rx : 0xFF); 1925 return; 1926 } 1927 1928 while (!err) { 1929 skb = skb_peek(&csk->receive_queue); 1930 if (!skb || 1931 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) { 1932 if (skb) 1933 log_debug(1 << CXGBI_DBG_PDU_RX, 1934 "skb 0x%p, NOT ready 0x%lx.\n", 1935 skb, cxgbi_skcb_flags(skb)); 1936 break; 1937 } 1938 __skb_unlink(skb, &csk->receive_queue); 1939 1940 read += cxgbi_skcb_rx_pdulen(skb); 1941 log_debug(1 << CXGBI_DBG_PDU_RX, 1942 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n", 1943 csk, skb, skb->len, cxgbi_skcb_flags(skb), 1944 cxgbi_skcb_rx_pdulen(skb)); 1945 1946 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) { 1947 err = skb_read_pdu_bhs(conn, skb); 1948 if (err < 0) { 1949 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, " 1950 "f 0x%lx, plen %u.\n", 1951 csk, skb, skb->len, 1952 cxgbi_skcb_flags(skb), 1953 cxgbi_skcb_rx_pdulen(skb)); 1954 goto skb_done; 1955 } 1956 err = skb_read_pdu_data(conn, skb, skb, 1957 err + cdev->skb_rx_extra); 1958 if (err < 0) 1959 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, " 1960 "f 0x%lx, plen %u.\n", 1961 csk, skb, skb->len, 1962 cxgbi_skcb_flags(skb), 1963 cxgbi_skcb_rx_pdulen(skb)); 1964 } else { 1965 err = skb_read_pdu_bhs(conn, skb); 1966 if (err < 0) { 1967 pr_err("bhs, csk 0x%p, skb 0x%p,%u, " 1968 "f 0x%lx, plen %u.\n", 1969 csk, skb, skb->len, 1970 cxgbi_skcb_flags(skb), 1971 cxgbi_skcb_rx_pdulen(skb)); 1972 goto skb_done; 1973 } 1974 1975 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { 1976 struct sk_buff *dskb; 1977 1978 dskb = skb_peek(&csk->receive_queue); 1979 if (!dskb) { 1980 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx," 1981 " plen %u, NO data.\n", 1982 csk, skb, skb->len, 1983 cxgbi_skcb_flags(skb), 1984 cxgbi_skcb_rx_pdulen(skb)); 1985 err = -EIO; 1986 goto skb_done; 1987 } 1988 __skb_unlink(dskb, &csk->receive_queue); 1989 1990 err = skb_read_pdu_data(conn, skb, dskb, 0); 1991 if (err < 0) 1992 pr_err("data, csk 0x%p, skb 0x%p,%u, " 1993 "f 0x%lx, plen %u, dskb 0x%p," 1994 "%u.\n", 1995 csk, skb, skb->len, 1996 cxgbi_skcb_flags(skb), 1997 cxgbi_skcb_rx_pdulen(skb), 1998 dskb, dskb->len); 1999 __kfree_skb(dskb); 2000 } else 2001 err = skb_read_pdu_data(conn, skb, skb, 0); 2002 } 2003 skb_done: 2004 __kfree_skb(skb); 2005 2006 if (err < 0) 2007 break; 2008 } 2009 2010 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read); 2011 if (read) { 2012 csk->copied_seq += read; 2013 csk_return_rx_credits(csk, read); 2014 conn->rxdata_octets += read; 2015 } 2016 2017 if (err < 0) { 2018 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n", 2019 csk, conn, err, read); 2020 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 2021 } 2022 } 2023 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready); 2024 2025 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt, 2026 unsigned int offset, unsigned int *off, 2027 struct scatterlist **sgp) 2028 { 2029 int i; 2030 struct scatterlist *sg; 2031 2032 for_each_sg(sgl, sg, sgcnt, i) { 2033 if (offset < sg->length) { 2034 *off = offset; 2035 *sgp = sg; 2036 return 0; 2037 } 2038 offset -= sg->length; 2039 } 2040 return -EFAULT; 2041 } 2042 2043 static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset, 2044 unsigned int dlen, struct page_frag *frags, 2045 int frag_max) 2046 { 2047 unsigned int datalen = dlen; 2048 unsigned int sglen = sg->length - sgoffset; 2049 struct page *page = sg_page(sg); 2050 int i; 2051 2052 i = 0; 2053 do { 2054 unsigned int copy; 2055 2056 if (!sglen) { 2057 sg = sg_next(sg); 2058 if (!sg) { 2059 pr_warn("sg %d NULL, len %u/%u.\n", 2060 i, datalen, dlen); 2061 return -EINVAL; 2062 } 2063 sgoffset = 0; 2064 sglen = sg->length; 2065 page = sg_page(sg); 2066 2067 } 2068 copy = min(datalen, sglen); 2069 if (i && page == frags[i - 1].page && 2070 sgoffset + sg->offset == 2071 frags[i - 1].offset + frags[i - 1].size) { 2072 frags[i - 1].size += copy; 2073 } else { 2074 if (i >= frag_max) { 2075 pr_warn("too many pages %u, dlen %u.\n", 2076 frag_max, dlen); 2077 return -EINVAL; 2078 } 2079 2080 frags[i].page = page; 2081 frags[i].offset = sg->offset + sgoffset; 2082 frags[i].size = copy; 2083 i++; 2084 } 2085 datalen -= copy; 2086 sgoffset += copy; 2087 sglen -= copy; 2088 } while (datalen); 2089 2090 return i; 2091 } 2092 2093 int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) 2094 { 2095 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 2096 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2097 struct cxgbi_device *cdev = cconn->chba->cdev; 2098 struct iscsi_conn *conn = task->conn; 2099 struct iscsi_tcp_task *tcp_task = task->dd_data; 2100 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2101 struct scsi_cmnd *sc = task->sc; 2102 int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX; 2103 2104 tcp_task->dd_data = tdata; 2105 task->hdr = NULL; 2106 2107 if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) && 2108 (opcode == ISCSI_OP_SCSI_DATA_OUT || 2109 (opcode == ISCSI_OP_SCSI_CMD && 2110 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE)))) 2111 /* data could goes into skb head */ 2112 headroom += min_t(unsigned int, 2113 SKB_MAX_HEAD(cdev->skb_tx_rsvd), 2114 conn->max_xmit_dlength); 2115 2116 tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC); 2117 if (!tdata->skb) { 2118 struct cxgbi_sock *csk = cconn->cep->csk; 2119 struct net_device *ndev = cdev->ports[csk->port_id]; 2120 ndev->stats.tx_dropped++; 2121 return -ENOMEM; 2122 } 2123 2124 skb_reserve(tdata->skb, cdev->skb_tx_rsvd); 2125 task->hdr = (struct iscsi_hdr *)tdata->skb->data; 2126 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */ 2127 2128 /* data_out uses scsi_cmd's itt */ 2129 if (opcode != ISCSI_OP_SCSI_DATA_OUT) 2130 task_reserve_itt(task, &task->hdr->itt); 2131 2132 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2133 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n", 2134 task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom, 2135 conn->max_xmit_dlength, ntohl(task->hdr->itt)); 2136 2137 return 0; 2138 } 2139 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu); 2140 2141 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc) 2142 { 2143 if (hcrc || dcrc) { 2144 u8 submode = 0; 2145 2146 if (hcrc) 2147 submode |= 1; 2148 if (dcrc) 2149 submode |= 2; 2150 cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode; 2151 } else 2152 cxgbi_skcb_ulp_mode(skb) = 0; 2153 } 2154 2155 int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset, 2156 unsigned int count) 2157 { 2158 struct iscsi_conn *conn = task->conn; 2159 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2160 struct sk_buff *skb = tdata->skb; 2161 unsigned int datalen = count; 2162 int i, padlen = iscsi_padding(count); 2163 struct page *pg; 2164 2165 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2166 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n", 2167 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK, 2168 ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count); 2169 2170 skb_put(skb, task->hdr_len); 2171 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0); 2172 if (!count) 2173 return 0; 2174 2175 if (task->sc) { 2176 struct scsi_data_buffer *sdb = scsi_out(task->sc); 2177 struct scatterlist *sg = NULL; 2178 int err; 2179 2180 tdata->offset = offset; 2181 tdata->count = count; 2182 err = sgl_seek_offset( 2183 sdb->table.sgl, sdb->table.nents, 2184 tdata->offset, &tdata->sgoffset, &sg); 2185 if (err < 0) { 2186 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n", 2187 sdb->table.nents, tdata->offset, sdb->length); 2188 return err; 2189 } 2190 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count, 2191 tdata->frags, MAX_PDU_FRAGS); 2192 if (err < 0) { 2193 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n", 2194 sdb->table.nents, tdata->offset, tdata->count); 2195 return err; 2196 } 2197 tdata->nr_frags = err; 2198 2199 if (tdata->nr_frags > MAX_SKB_FRAGS || 2200 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) { 2201 char *dst = skb->data + task->hdr_len; 2202 struct page_frag *frag = tdata->frags; 2203 2204 /* data fits in the skb's headroom */ 2205 for (i = 0; i < tdata->nr_frags; i++, frag++) { 2206 char *src = kmap_atomic(frag->page); 2207 2208 memcpy(dst, src+frag->offset, frag->size); 2209 dst += frag->size; 2210 kunmap_atomic(src); 2211 } 2212 if (padlen) { 2213 memset(dst, 0, padlen); 2214 padlen = 0; 2215 } 2216 skb_put(skb, count + padlen); 2217 } else { 2218 /* data fit into frag_list */ 2219 for (i = 0; i < tdata->nr_frags; i++) { 2220 __skb_fill_page_desc(skb, i, 2221 tdata->frags[i].page, 2222 tdata->frags[i].offset, 2223 tdata->frags[i].size); 2224 skb_frag_ref(skb, i); 2225 } 2226 skb_shinfo(skb)->nr_frags = tdata->nr_frags; 2227 skb->len += count; 2228 skb->data_len += count; 2229 skb->truesize += count; 2230 } 2231 2232 } else { 2233 pg = virt_to_page(task->data); 2234 2235 get_page(pg); 2236 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data), 2237 count); 2238 skb->len += count; 2239 skb->data_len += count; 2240 skb->truesize += count; 2241 } 2242 2243 if (padlen) { 2244 i = skb_shinfo(skb)->nr_frags; 2245 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 2246 virt_to_page(padding), offset_in_page(padding), 2247 padlen); 2248 2249 skb->data_len += padlen; 2250 skb->truesize += padlen; 2251 skb->len += padlen; 2252 } 2253 2254 return 0; 2255 } 2256 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu); 2257 2258 int cxgbi_conn_xmit_pdu(struct iscsi_task *task) 2259 { 2260 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 2261 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2262 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2263 struct sk_buff *skb = tdata->skb; 2264 unsigned int datalen; 2265 int err; 2266 2267 if (!skb) { 2268 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2269 "task 0x%p, skb NULL.\n", task); 2270 return 0; 2271 } 2272 2273 datalen = skb->data_len; 2274 tdata->skb = NULL; 2275 err = cxgbi_sock_send_pdus(cconn->cep->csk, skb); 2276 if (err > 0) { 2277 int pdulen = err; 2278 2279 log_debug(1 << CXGBI_DBG_PDU_TX, 2280 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n", 2281 task, task->sc, skb, skb->len, skb->data_len, err); 2282 2283 if (task->conn->hdrdgst_en) 2284 pdulen += ISCSI_DIGEST_SIZE; 2285 2286 if (datalen && task->conn->datadgst_en) 2287 pdulen += ISCSI_DIGEST_SIZE; 2288 2289 task->conn->txdata_octets += pdulen; 2290 return 0; 2291 } 2292 2293 if (err == -EAGAIN || err == -ENOBUFS) { 2294 log_debug(1 << CXGBI_DBG_PDU_TX, 2295 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n", 2296 task, skb, skb->len, skb->data_len, err); 2297 /* reset skb to send when we are called again */ 2298 tdata->skb = skb; 2299 return err; 2300 } 2301 2302 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2303 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n", 2304 task->itt, skb, skb->len, skb->data_len, err); 2305 2306 kfree_skb(skb); 2307 2308 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err); 2309 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED); 2310 return err; 2311 } 2312 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu); 2313 2314 void cxgbi_cleanup_task(struct iscsi_task *task) 2315 { 2316 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2317 2318 log_debug(1 << CXGBI_DBG_ISCSI, 2319 "task 0x%p, skb 0x%p, itt 0x%x.\n", 2320 task, tdata->skb, task->hdr_itt); 2321 2322 /* never reached the xmit task callout */ 2323 if (tdata->skb) 2324 __kfree_skb(tdata->skb); 2325 memset(tdata, 0, sizeof(*tdata)); 2326 2327 task_release_itt(task, task->hdr_itt); 2328 iscsi_tcp_cleanup_task(task); 2329 } 2330 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task); 2331 2332 void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn, 2333 struct iscsi_stats *stats) 2334 { 2335 struct iscsi_conn *conn = cls_conn->dd_data; 2336 2337 stats->txdata_octets = conn->txdata_octets; 2338 stats->rxdata_octets = conn->rxdata_octets; 2339 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 2340 stats->dataout_pdus = conn->dataout_pdus_cnt; 2341 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 2342 stats->datain_pdus = conn->datain_pdus_cnt; 2343 stats->r2t_pdus = conn->r2t_pdus_cnt; 2344 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 2345 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2346 stats->digest_err = 0; 2347 stats->timeout_err = 0; 2348 stats->custom_length = 1; 2349 strcpy(stats->custom[0].desc, "eh_abort_cnt"); 2350 stats->custom[0].value = conn->eh_abort_cnt; 2351 } 2352 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats); 2353 2354 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn) 2355 { 2356 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2357 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2358 struct cxgbi_device *cdev = cconn->chba->cdev; 2359 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd); 2360 unsigned int max_def = 512 * MAX_SKB_FRAGS; 2361 unsigned int max = max(max_def, headroom); 2362 2363 max = min(cconn->chba->cdev->tx_max_size, max); 2364 if (conn->max_xmit_dlength) 2365 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max); 2366 else 2367 conn->max_xmit_dlength = max; 2368 cxgbi_align_pdu_size(conn->max_xmit_dlength); 2369 2370 return 0; 2371 } 2372 2373 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn) 2374 { 2375 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2376 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2377 unsigned int max = cconn->chba->cdev->rx_max_size; 2378 2379 cxgbi_align_pdu_size(max); 2380 2381 if (conn->max_recv_dlength) { 2382 if (conn->max_recv_dlength > max) { 2383 pr_err("MaxRecvDataSegmentLength %u > %u.\n", 2384 conn->max_recv_dlength, max); 2385 return -EINVAL; 2386 } 2387 conn->max_recv_dlength = min(conn->max_recv_dlength, max); 2388 cxgbi_align_pdu_size(conn->max_recv_dlength); 2389 } else 2390 conn->max_recv_dlength = max; 2391 2392 return 0; 2393 } 2394 2395 int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn, 2396 enum iscsi_param param, char *buf, int buflen) 2397 { 2398 struct iscsi_conn *conn = cls_conn->dd_data; 2399 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2400 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2401 struct cxgbi_sock *csk = cconn->cep->csk; 2402 int err; 2403 2404 log_debug(1 << CXGBI_DBG_ISCSI, 2405 "cls_conn 0x%p, param %d, buf(%d) %s.\n", 2406 cls_conn, param, buflen, buf); 2407 2408 switch (param) { 2409 case ISCSI_PARAM_HDRDGST_EN: 2410 err = iscsi_set_param(cls_conn, param, buf, buflen); 2411 if (!err && conn->hdrdgst_en) 2412 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2413 conn->hdrdgst_en, 2414 conn->datadgst_en, 0); 2415 break; 2416 case ISCSI_PARAM_DATADGST_EN: 2417 err = iscsi_set_param(cls_conn, param, buf, buflen); 2418 if (!err && conn->datadgst_en) 2419 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2420 conn->hdrdgst_en, 2421 conn->datadgst_en, 0); 2422 break; 2423 case ISCSI_PARAM_MAX_R2T: 2424 return iscsi_tcp_set_max_r2t(conn, buf); 2425 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2426 err = iscsi_set_param(cls_conn, param, buf, buflen); 2427 if (!err) 2428 err = cxgbi_conn_max_recv_dlength(conn); 2429 break; 2430 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2431 err = iscsi_set_param(cls_conn, param, buf, buflen); 2432 if (!err) 2433 err = cxgbi_conn_max_xmit_dlength(conn); 2434 break; 2435 default: 2436 return iscsi_set_param(cls_conn, param, buf, buflen); 2437 } 2438 return err; 2439 } 2440 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param); 2441 2442 static inline int csk_print_port(struct cxgbi_sock *csk, char *buf) 2443 { 2444 int len; 2445 2446 cxgbi_sock_get(csk); 2447 len = sprintf(buf, "%hu\n", ntohs(csk->daddr.sin_port)); 2448 cxgbi_sock_put(csk); 2449 2450 return len; 2451 } 2452 2453 static inline int csk_print_ip(struct cxgbi_sock *csk, char *buf) 2454 { 2455 int len; 2456 2457 cxgbi_sock_get(csk); 2458 if (csk->csk_family == AF_INET) 2459 len = sprintf(buf, "%pI4", 2460 &csk->daddr.sin_addr.s_addr); 2461 else 2462 len = sprintf(buf, "%pI6", 2463 &csk->daddr6.sin6_addr); 2464 2465 cxgbi_sock_put(csk); 2466 2467 return len; 2468 } 2469 2470 int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param, 2471 char *buf) 2472 { 2473 struct cxgbi_endpoint *cep = ep->dd_data; 2474 struct cxgbi_sock *csk; 2475 int len; 2476 2477 log_debug(1 << CXGBI_DBG_ISCSI, 2478 "cls_conn 0x%p, param %d.\n", ep, param); 2479 2480 switch (param) { 2481 case ISCSI_PARAM_CONN_PORT: 2482 case ISCSI_PARAM_CONN_ADDRESS: 2483 if (!cep) 2484 return -ENOTCONN; 2485 2486 csk = cep->csk; 2487 if (!csk) 2488 return -ENOTCONN; 2489 2490 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 2491 &csk->daddr, param, buf); 2492 default: 2493 return -ENOSYS; 2494 } 2495 return len; 2496 } 2497 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param); 2498 2499 struct iscsi_cls_conn * 2500 cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid) 2501 { 2502 struct iscsi_cls_conn *cls_conn; 2503 struct iscsi_conn *conn; 2504 struct iscsi_tcp_conn *tcp_conn; 2505 struct cxgbi_conn *cconn; 2506 2507 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid); 2508 if (!cls_conn) 2509 return NULL; 2510 2511 conn = cls_conn->dd_data; 2512 tcp_conn = conn->dd_data; 2513 cconn = tcp_conn->dd_data; 2514 cconn->iconn = conn; 2515 2516 log_debug(1 << CXGBI_DBG_ISCSI, 2517 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n", 2518 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn); 2519 2520 return cls_conn; 2521 } 2522 EXPORT_SYMBOL_GPL(cxgbi_create_conn); 2523 2524 int cxgbi_bind_conn(struct iscsi_cls_session *cls_session, 2525 struct iscsi_cls_conn *cls_conn, 2526 u64 transport_eph, int is_leading) 2527 { 2528 struct iscsi_conn *conn = cls_conn->dd_data; 2529 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2530 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2531 struct iscsi_endpoint *ep; 2532 struct cxgbi_endpoint *cep; 2533 struct cxgbi_sock *csk; 2534 int err; 2535 2536 ep = iscsi_lookup_endpoint(transport_eph); 2537 if (!ep) 2538 return -EINVAL; 2539 2540 /* setup ddp pagesize */ 2541 cep = ep->dd_data; 2542 csk = cep->csk; 2543 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, page_idx, 0); 2544 if (err < 0) 2545 return err; 2546 2547 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 2548 if (err) 2549 return -EINVAL; 2550 2551 /* calculate the tag idx bits needed for this conn based on cmds_max */ 2552 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1; 2553 2554 write_lock_bh(&csk->callback_lock); 2555 csk->user_data = conn; 2556 cconn->chba = cep->chba; 2557 cconn->cep = cep; 2558 cep->cconn = cconn; 2559 write_unlock_bh(&csk->callback_lock); 2560 2561 cxgbi_conn_max_xmit_dlength(conn); 2562 cxgbi_conn_max_recv_dlength(conn); 2563 2564 log_debug(1 << CXGBI_DBG_ISCSI, 2565 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n", 2566 cls_session, cls_conn, ep, cconn, csk); 2567 /* init recv engine */ 2568 iscsi_tcp_hdr_recv_prep(tcp_conn); 2569 2570 return 0; 2571 } 2572 EXPORT_SYMBOL_GPL(cxgbi_bind_conn); 2573 2574 struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep, 2575 u16 cmds_max, u16 qdepth, 2576 u32 initial_cmdsn) 2577 { 2578 struct cxgbi_endpoint *cep; 2579 struct cxgbi_hba *chba; 2580 struct Scsi_Host *shost; 2581 struct iscsi_cls_session *cls_session; 2582 struct iscsi_session *session; 2583 2584 if (!ep) { 2585 pr_err("missing endpoint.\n"); 2586 return NULL; 2587 } 2588 2589 cep = ep->dd_data; 2590 chba = cep->chba; 2591 shost = chba->shost; 2592 2593 BUG_ON(chba != iscsi_host_priv(shost)); 2594 2595 cls_session = iscsi_session_setup(chba->cdev->itp, shost, 2596 cmds_max, 0, 2597 sizeof(struct iscsi_tcp_task) + 2598 sizeof(struct cxgbi_task_data), 2599 initial_cmdsn, ISCSI_MAX_TARGET); 2600 if (!cls_session) 2601 return NULL; 2602 2603 session = cls_session->dd_data; 2604 if (iscsi_tcp_r2tpool_alloc(session)) 2605 goto remove_session; 2606 2607 log_debug(1 << CXGBI_DBG_ISCSI, 2608 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session); 2609 return cls_session; 2610 2611 remove_session: 2612 iscsi_session_teardown(cls_session); 2613 return NULL; 2614 } 2615 EXPORT_SYMBOL_GPL(cxgbi_create_session); 2616 2617 void cxgbi_destroy_session(struct iscsi_cls_session *cls_session) 2618 { 2619 log_debug(1 << CXGBI_DBG_ISCSI, 2620 "cls sess 0x%p.\n", cls_session); 2621 2622 iscsi_tcp_r2tpool_free(cls_session->dd_data); 2623 iscsi_session_teardown(cls_session); 2624 } 2625 EXPORT_SYMBOL_GPL(cxgbi_destroy_session); 2626 2627 int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2628 char *buf, int buflen) 2629 { 2630 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2631 2632 if (!chba->ndev) { 2633 shost_printk(KERN_ERR, shost, "Could not get host param. " 2634 "netdev for host not set.\n"); 2635 return -ENODEV; 2636 } 2637 2638 log_debug(1 << CXGBI_DBG_ISCSI, 2639 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n", 2640 shost, chba, chba->ndev->name, param, buflen, buf); 2641 2642 switch (param) { 2643 case ISCSI_HOST_PARAM_IPADDRESS: 2644 { 2645 __be32 addr = in_aton(buf); 2646 log_debug(1 << CXGBI_DBG_ISCSI, 2647 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr); 2648 cxgbi_set_iscsi_ipv4(chba, addr); 2649 return 0; 2650 } 2651 case ISCSI_HOST_PARAM_HWADDRESS: 2652 case ISCSI_HOST_PARAM_NETDEV_NAME: 2653 return 0; 2654 default: 2655 return iscsi_host_set_param(shost, param, buf, buflen); 2656 } 2657 } 2658 EXPORT_SYMBOL_GPL(cxgbi_set_host_param); 2659 2660 int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2661 char *buf) 2662 { 2663 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2664 int len = 0; 2665 2666 if (!chba->ndev) { 2667 shost_printk(KERN_ERR, shost, "Could not get host param. " 2668 "netdev for host not set.\n"); 2669 return -ENODEV; 2670 } 2671 2672 log_debug(1 << CXGBI_DBG_ISCSI, 2673 "shost 0x%p, hba 0x%p,%s, param %d.\n", 2674 shost, chba, chba->ndev->name, param); 2675 2676 switch (param) { 2677 case ISCSI_HOST_PARAM_HWADDRESS: 2678 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6); 2679 break; 2680 case ISCSI_HOST_PARAM_NETDEV_NAME: 2681 len = sprintf(buf, "%s\n", chba->ndev->name); 2682 break; 2683 case ISCSI_HOST_PARAM_IPADDRESS: 2684 { 2685 struct cxgbi_sock *csk = find_sock_on_port(chba->cdev, 2686 chba->port_id); 2687 if (csk) { 2688 len = sprintf(buf, "%pIS", 2689 (struct sockaddr *)&csk->saddr); 2690 } 2691 log_debug(1 << CXGBI_DBG_ISCSI, 2692 "hba %s, addr %s.\n", chba->ndev->name, buf); 2693 break; 2694 } 2695 default: 2696 return iscsi_host_get_param(shost, param, buf); 2697 } 2698 2699 return len; 2700 } 2701 EXPORT_SYMBOL_GPL(cxgbi_get_host_param); 2702 2703 struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost, 2704 struct sockaddr *dst_addr, 2705 int non_blocking) 2706 { 2707 struct iscsi_endpoint *ep; 2708 struct cxgbi_endpoint *cep; 2709 struct cxgbi_hba *hba = NULL; 2710 struct cxgbi_sock *csk; 2711 int err = -EINVAL; 2712 2713 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2714 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n", 2715 shost, non_blocking, dst_addr); 2716 2717 if (shost) { 2718 hba = iscsi_host_priv(shost); 2719 if (!hba) { 2720 pr_info("shost 0x%p, priv NULL.\n", shost); 2721 goto err_out; 2722 } 2723 } 2724 2725 if (dst_addr->sa_family == AF_INET) { 2726 csk = cxgbi_check_route(dst_addr); 2727 #if IS_ENABLED(CONFIG_IPV6) 2728 } else if (dst_addr->sa_family == AF_INET6) { 2729 csk = cxgbi_check_route6(dst_addr); 2730 #endif 2731 } else { 2732 pr_info("address family 0x%x NOT supported.\n", 2733 dst_addr->sa_family); 2734 err = -EAFNOSUPPORT; 2735 return (struct iscsi_endpoint *)ERR_PTR(err); 2736 } 2737 2738 if (IS_ERR(csk)) 2739 return (struct iscsi_endpoint *)csk; 2740 cxgbi_sock_get(csk); 2741 2742 if (!hba) 2743 hba = csk->cdev->hbas[csk->port_id]; 2744 else if (hba != csk->cdev->hbas[csk->port_id]) { 2745 pr_info("Could not connect through requested host %u" 2746 "hba 0x%p != 0x%p (%u).\n", 2747 shost->host_no, hba, 2748 csk->cdev->hbas[csk->port_id], csk->port_id); 2749 err = -ENOSPC; 2750 goto release_conn; 2751 } 2752 2753 err = sock_get_port(csk); 2754 if (err) 2755 goto release_conn; 2756 2757 cxgbi_sock_set_state(csk, CTP_CONNECTING); 2758 err = csk->cdev->csk_init_act_open(csk); 2759 if (err) 2760 goto release_conn; 2761 2762 if (cxgbi_sock_is_closing(csk)) { 2763 err = -ENOSPC; 2764 pr_info("csk 0x%p is closing.\n", csk); 2765 goto release_conn; 2766 } 2767 2768 ep = iscsi_create_endpoint(sizeof(*cep)); 2769 if (!ep) { 2770 err = -ENOMEM; 2771 pr_info("iscsi alloc ep, OOM.\n"); 2772 goto release_conn; 2773 } 2774 2775 cep = ep->dd_data; 2776 cep->csk = csk; 2777 cep->chba = hba; 2778 2779 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2780 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n", 2781 ep, cep, csk, hba, hba->ndev->name); 2782 return ep; 2783 2784 release_conn: 2785 cxgbi_sock_put(csk); 2786 cxgbi_sock_closed(csk); 2787 err_out: 2788 return ERR_PTR(err); 2789 } 2790 EXPORT_SYMBOL_GPL(cxgbi_ep_connect); 2791 2792 int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 2793 { 2794 struct cxgbi_endpoint *cep = ep->dd_data; 2795 struct cxgbi_sock *csk = cep->csk; 2796 2797 if (!cxgbi_sock_is_established(csk)) 2798 return 0; 2799 return 1; 2800 } 2801 EXPORT_SYMBOL_GPL(cxgbi_ep_poll); 2802 2803 void cxgbi_ep_disconnect(struct iscsi_endpoint *ep) 2804 { 2805 struct cxgbi_endpoint *cep = ep->dd_data; 2806 struct cxgbi_conn *cconn = cep->cconn; 2807 struct cxgbi_sock *csk = cep->csk; 2808 2809 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2810 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n", 2811 ep, cep, cconn, csk, csk->state, csk->flags); 2812 2813 if (cconn && cconn->iconn) { 2814 iscsi_suspend_tx(cconn->iconn); 2815 write_lock_bh(&csk->callback_lock); 2816 cep->csk->user_data = NULL; 2817 cconn->cep = NULL; 2818 write_unlock_bh(&csk->callback_lock); 2819 } 2820 iscsi_destroy_endpoint(ep); 2821 2822 if (likely(csk->state >= CTP_ESTABLISHED)) 2823 need_active_close(csk); 2824 else 2825 cxgbi_sock_closed(csk); 2826 2827 cxgbi_sock_put(csk); 2828 } 2829 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect); 2830 2831 int cxgbi_iscsi_init(struct iscsi_transport *itp, 2832 struct scsi_transport_template **stt) 2833 { 2834 *stt = iscsi_register_transport(itp); 2835 if (*stt == NULL) { 2836 pr_err("unable to register %s transport 0x%p.\n", 2837 itp->name, itp); 2838 return -ENODEV; 2839 } 2840 log_debug(1 << CXGBI_DBG_ISCSI, 2841 "%s, registered iscsi transport 0x%p.\n", 2842 itp->name, stt); 2843 return 0; 2844 } 2845 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init); 2846 2847 void cxgbi_iscsi_cleanup(struct iscsi_transport *itp, 2848 struct scsi_transport_template **stt) 2849 { 2850 if (*stt) { 2851 log_debug(1 << CXGBI_DBG_ISCSI, 2852 "de-register transport 0x%p, %s, stt 0x%p.\n", 2853 itp, itp->name, *stt); 2854 *stt = NULL; 2855 iscsi_unregister_transport(itp); 2856 } 2857 } 2858 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup); 2859 2860 umode_t cxgbi_attr_is_visible(int param_type, int param) 2861 { 2862 switch (param_type) { 2863 case ISCSI_HOST_PARAM: 2864 switch (param) { 2865 case ISCSI_HOST_PARAM_NETDEV_NAME: 2866 case ISCSI_HOST_PARAM_HWADDRESS: 2867 case ISCSI_HOST_PARAM_IPADDRESS: 2868 case ISCSI_HOST_PARAM_INITIATOR_NAME: 2869 return S_IRUGO; 2870 default: 2871 return 0; 2872 } 2873 case ISCSI_PARAM: 2874 switch (param) { 2875 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2876 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2877 case ISCSI_PARAM_HDRDGST_EN: 2878 case ISCSI_PARAM_DATADGST_EN: 2879 case ISCSI_PARAM_CONN_ADDRESS: 2880 case ISCSI_PARAM_CONN_PORT: 2881 case ISCSI_PARAM_EXP_STATSN: 2882 case ISCSI_PARAM_PERSISTENT_ADDRESS: 2883 case ISCSI_PARAM_PERSISTENT_PORT: 2884 case ISCSI_PARAM_PING_TMO: 2885 case ISCSI_PARAM_RECV_TMO: 2886 case ISCSI_PARAM_INITIAL_R2T_EN: 2887 case ISCSI_PARAM_MAX_R2T: 2888 case ISCSI_PARAM_IMM_DATA_EN: 2889 case ISCSI_PARAM_FIRST_BURST: 2890 case ISCSI_PARAM_MAX_BURST: 2891 case ISCSI_PARAM_PDU_INORDER_EN: 2892 case ISCSI_PARAM_DATASEQ_INORDER_EN: 2893 case ISCSI_PARAM_ERL: 2894 case ISCSI_PARAM_TARGET_NAME: 2895 case ISCSI_PARAM_TPGT: 2896 case ISCSI_PARAM_USERNAME: 2897 case ISCSI_PARAM_PASSWORD: 2898 case ISCSI_PARAM_USERNAME_IN: 2899 case ISCSI_PARAM_PASSWORD_IN: 2900 case ISCSI_PARAM_FAST_ABORT: 2901 case ISCSI_PARAM_ABORT_TMO: 2902 case ISCSI_PARAM_LU_RESET_TMO: 2903 case ISCSI_PARAM_TGT_RESET_TMO: 2904 case ISCSI_PARAM_IFACE_NAME: 2905 case ISCSI_PARAM_INITIATOR_NAME: 2906 return S_IRUGO; 2907 default: 2908 return 0; 2909 } 2910 } 2911 2912 return 0; 2913 } 2914 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible); 2915 2916 static int __init libcxgbi_init_module(void) 2917 { 2918 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1; 2919 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1; 2920 2921 pr_info("%s", version); 2922 2923 pr_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n", 2924 ISCSI_ITT_MASK, sw_tag_idx_bits, 2925 ISCSI_AGE_MASK, sw_tag_age_bits); 2926 2927 ddp_setup_host_page_size(); 2928 return 0; 2929 } 2930 2931 static void __exit libcxgbi_exit_module(void) 2932 { 2933 cxgbi_device_unregister_all(0xFF); 2934 return; 2935 } 2936 2937 module_init(libcxgbi_init_module); 2938 module_exit(libcxgbi_exit_module); 2939