1 /* 2 * cxgb4i.c: Chelsio T4 iSCSI driver. 3 * 4 * Copyright (c) 2010 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/module.h> 17 #include <linux/moduleparam.h> 18 #include <scsi/scsi_host.h> 19 #include <net/tcp.h> 20 #include <net/dst.h> 21 #include <linux/netdevice.h> 22 #include <net/addrconf.h> 23 24 #include "t4_regs.h" 25 #include "t4_msg.h" 26 #include "cxgb4.h" 27 #include "cxgb4_uld.h" 28 #include "t4fw_api.h" 29 #include "l2t.h" 30 #include "cxgb4i.h" 31 32 static unsigned int dbg_level; 33 34 #include "../libcxgbi.h" 35 36 #define DRV_MODULE_NAME "cxgb4i" 37 #define DRV_MODULE_DESC "Chelsio T4/T5 iSCSI Driver" 38 #define DRV_MODULE_VERSION "0.9.4" 39 40 static char version[] = 41 DRV_MODULE_DESC " " DRV_MODULE_NAME 42 " v" DRV_MODULE_VERSION "\n"; 43 44 MODULE_AUTHOR("Chelsio Communications, Inc."); 45 MODULE_DESCRIPTION(DRV_MODULE_DESC); 46 MODULE_VERSION(DRV_MODULE_VERSION); 47 MODULE_LICENSE("GPL"); 48 49 module_param(dbg_level, uint, 0644); 50 MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)"); 51 52 static int cxgb4i_rcv_win = 256 * 1024; 53 module_param(cxgb4i_rcv_win, int, 0644); 54 MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes"); 55 56 static int cxgb4i_snd_win = 128 * 1024; 57 module_param(cxgb4i_snd_win, int, 0644); 58 MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes"); 59 60 static int cxgb4i_rx_credit_thres = 10 * 1024; 61 module_param(cxgb4i_rx_credit_thres, int, 0644); 62 MODULE_PARM_DESC(cxgb4i_rx_credit_thres, 63 "RX credits return threshold in bytes (default=10KB)"); 64 65 static unsigned int cxgb4i_max_connect = (8 * 1024); 66 module_param(cxgb4i_max_connect, uint, 0644); 67 MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections"); 68 69 static unsigned short cxgb4i_sport_base = 20000; 70 module_param(cxgb4i_sport_base, ushort, 0644); 71 MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)"); 72 73 typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *); 74 75 static void *t4_uld_add(const struct cxgb4_lld_info *); 76 static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *); 77 static int t4_uld_state_change(void *, enum cxgb4_state state); 78 79 static const struct cxgb4_uld_info cxgb4i_uld_info = { 80 .name = DRV_MODULE_NAME, 81 .add = t4_uld_add, 82 .rx_handler = t4_uld_rx_handler, 83 .state_change = t4_uld_state_change, 84 }; 85 86 static struct scsi_host_template cxgb4i_host_template = { 87 .module = THIS_MODULE, 88 .name = DRV_MODULE_NAME, 89 .proc_name = DRV_MODULE_NAME, 90 .can_queue = CXGB4I_SCSI_HOST_QDEPTH, 91 .queuecommand = iscsi_queuecommand, 92 .change_queue_depth = scsi_change_queue_depth, 93 .sg_tablesize = SG_ALL, 94 .max_sectors = 0xFFFF, 95 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 96 .eh_abort_handler = iscsi_eh_abort, 97 .eh_device_reset_handler = iscsi_eh_device_reset, 98 .eh_target_reset_handler = iscsi_eh_recover_target, 99 .target_alloc = iscsi_target_alloc, 100 .use_clustering = DISABLE_CLUSTERING, 101 .this_id = -1, 102 .track_queue_depth = 1, 103 }; 104 105 static struct iscsi_transport cxgb4i_iscsi_transport = { 106 .owner = THIS_MODULE, 107 .name = DRV_MODULE_NAME, 108 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST | 109 CAP_DATADGST | CAP_DIGEST_OFFLOAD | 110 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, 111 .attr_is_visible = cxgbi_attr_is_visible, 112 .get_host_param = cxgbi_get_host_param, 113 .set_host_param = cxgbi_set_host_param, 114 /* session management */ 115 .create_session = cxgbi_create_session, 116 .destroy_session = cxgbi_destroy_session, 117 .get_session_param = iscsi_session_get_param, 118 /* connection management */ 119 .create_conn = cxgbi_create_conn, 120 .bind_conn = cxgbi_bind_conn, 121 .destroy_conn = iscsi_tcp_conn_teardown, 122 .start_conn = iscsi_conn_start, 123 .stop_conn = iscsi_conn_stop, 124 .get_conn_param = iscsi_conn_get_param, 125 .set_param = cxgbi_set_conn_param, 126 .get_stats = cxgbi_get_conn_stats, 127 /* pdu xmit req from user space */ 128 .send_pdu = iscsi_conn_send_pdu, 129 /* task */ 130 .init_task = iscsi_tcp_task_init, 131 .xmit_task = iscsi_tcp_task_xmit, 132 .cleanup_task = cxgbi_cleanup_task, 133 /* pdu */ 134 .alloc_pdu = cxgbi_conn_alloc_pdu, 135 .init_pdu = cxgbi_conn_init_pdu, 136 .xmit_pdu = cxgbi_conn_xmit_pdu, 137 .parse_pdu_itt = cxgbi_parse_pdu_itt, 138 /* TCP connect/disconnect */ 139 .get_ep_param = cxgbi_get_ep_param, 140 .ep_connect = cxgbi_ep_connect, 141 .ep_poll = cxgbi_ep_poll, 142 .ep_disconnect = cxgbi_ep_disconnect, 143 /* Error recovery timeout call */ 144 .session_recovery_timedout = iscsi_session_recovery_timedout, 145 }; 146 147 static struct scsi_transport_template *cxgb4i_stt; 148 149 /* 150 * CPL (Chelsio Protocol Language) defines a message passing interface between 151 * the host driver and Chelsio asic. 152 * The section below implments CPLs that related to iscsi tcp connection 153 * open/close/abort and data send/receive. 154 */ 155 156 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 157 #define RCV_BUFSIZ_MASK 0x3FFU 158 #define MAX_IMM_TX_PKT_LEN 128 159 160 static inline void set_queue(struct sk_buff *skb, unsigned int queue, 161 const struct cxgbi_sock *csk) 162 { 163 skb->queue_mapping = queue; 164 } 165 166 static int push_tx_frames(struct cxgbi_sock *, int); 167 168 /* 169 * is_ofld_imm - check whether a packet can be sent as immediate data 170 * @skb: the packet 171 * 172 * Returns true if a packet can be sent as an offload WR with immediate 173 * data. We currently use the same limit as for Ethernet packets. 174 */ 175 static inline int is_ofld_imm(const struct sk_buff *skb) 176 { 177 return skb->len <= (MAX_IMM_TX_PKT_LEN - 178 sizeof(struct fw_ofld_tx_data_wr)); 179 } 180 181 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 182 struct l2t_entry *e) 183 { 184 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 185 int t4 = is_t4(lldi->adapter_type); 186 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 187 unsigned long long opt0; 188 unsigned int opt2; 189 unsigned int qid_atid = ((unsigned int)csk->atid) | 190 (((unsigned int)csk->rss_qid) << 14); 191 192 opt0 = KEEP_ALIVE_F | 193 WND_SCALE_V(wscale) | 194 MSS_IDX_V(csk->mss_idx) | 195 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 196 TX_CHAN_V(csk->tx_chan) | 197 SMAC_SEL_V(csk->smac_idx) | 198 ULP_MODE_V(ULP_MODE_ISCSI) | 199 RCV_BUFSIZ_V(cxgb4i_rcv_win >> 10); 200 opt2 = RX_CHANNEL_V(0) | 201 RSS_QUEUE_VALID_F | 202 (RX_FC_DISABLE_F) | 203 RSS_QUEUE_V(csk->rss_qid); 204 205 if (is_t4(lldi->adapter_type)) { 206 struct cpl_act_open_req *req = 207 (struct cpl_act_open_req *)skb->head; 208 209 INIT_TP_WR(req, 0); 210 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 211 qid_atid)); 212 req->local_port = csk->saddr.sin_port; 213 req->peer_port = csk->daddr.sin_port; 214 req->local_ip = csk->saddr.sin_addr.s_addr; 215 req->peer_ip = csk->daddr.sin_addr.s_addr; 216 req->opt0 = cpu_to_be64(opt0); 217 req->params = cpu_to_be32(cxgb4_select_ntuple( 218 csk->cdev->ports[csk->port_id], 219 csk->l2t)); 220 opt2 |= RX_FC_VALID_F; 221 req->opt2 = cpu_to_be32(opt2); 222 223 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 224 "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 225 csk, &req->local_ip, ntohs(req->local_port), 226 &req->peer_ip, ntohs(req->peer_port), 227 csk->atid, csk->rss_qid); 228 } else { 229 struct cpl_t5_act_open_req *req = 230 (struct cpl_t5_act_open_req *)skb->head; 231 232 INIT_TP_WR(req, 0); 233 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 234 qid_atid)); 235 req->local_port = csk->saddr.sin_port; 236 req->peer_port = csk->daddr.sin_port; 237 req->local_ip = csk->saddr.sin_addr.s_addr; 238 req->peer_ip = csk->daddr.sin_addr.s_addr; 239 req->opt0 = cpu_to_be64(opt0); 240 req->params = cpu_to_be64(FILTER_TUPLE_V( 241 cxgb4_select_ntuple( 242 csk->cdev->ports[csk->port_id], 243 csk->l2t))); 244 opt2 |= 1 << 31; 245 req->opt2 = cpu_to_be32(opt2); 246 247 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 248 "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 249 csk, &req->local_ip, ntohs(req->local_port), 250 &req->peer_ip, ntohs(req->peer_port), 251 csk->atid, csk->rss_qid); 252 } 253 254 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 255 256 pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n", 257 (&csk->saddr), (&csk->daddr), t4 ? 4 : 5, csk, 258 csk->state, csk->flags, csk->atid, csk->rss_qid); 259 260 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 261 } 262 263 #if IS_ENABLED(CONFIG_IPV6) 264 static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb, 265 struct l2t_entry *e) 266 { 267 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 268 int t4 = is_t4(lldi->adapter_type); 269 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 270 unsigned long long opt0; 271 unsigned int opt2; 272 unsigned int qid_atid = ((unsigned int)csk->atid) | 273 (((unsigned int)csk->rss_qid) << 14); 274 275 opt0 = KEEP_ALIVE_F | 276 WND_SCALE_V(wscale) | 277 MSS_IDX_V(csk->mss_idx) | 278 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 279 TX_CHAN_V(csk->tx_chan) | 280 SMAC_SEL_V(csk->smac_idx) | 281 ULP_MODE_V(ULP_MODE_ISCSI) | 282 RCV_BUFSIZ_V(cxgb4i_rcv_win >> 10); 283 284 opt2 = RX_CHANNEL_V(0) | 285 RSS_QUEUE_VALID_F | 286 RX_FC_DISABLE_F | 287 RSS_QUEUE_V(csk->rss_qid); 288 289 if (t4) { 290 struct cpl_act_open_req6 *req = 291 (struct cpl_act_open_req6 *)skb->head; 292 293 INIT_TP_WR(req, 0); 294 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 295 qid_atid)); 296 req->local_port = csk->saddr6.sin6_port; 297 req->peer_port = csk->daddr6.sin6_port; 298 299 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 300 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 301 8); 302 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 303 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 304 8); 305 306 req->opt0 = cpu_to_be64(opt0); 307 308 opt2 |= RX_FC_VALID_F; 309 req->opt2 = cpu_to_be32(opt2); 310 311 req->params = cpu_to_be32(cxgb4_select_ntuple( 312 csk->cdev->ports[csk->port_id], 313 csk->l2t)); 314 } else { 315 struct cpl_t5_act_open_req6 *req = 316 (struct cpl_t5_act_open_req6 *)skb->head; 317 318 INIT_TP_WR(req, 0); 319 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 320 qid_atid)); 321 req->local_port = csk->saddr6.sin6_port; 322 req->peer_port = csk->daddr6.sin6_port; 323 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 324 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 325 8); 326 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 327 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 328 8); 329 req->opt0 = cpu_to_be64(opt0); 330 331 opt2 |= T5_OPT_2_VALID_F; 332 req->opt2 = cpu_to_be32(opt2); 333 334 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( 335 csk->cdev->ports[csk->port_id], 336 csk->l2t))); 337 } 338 339 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 340 341 pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n", 342 t4 ? 4 : 5, csk, csk->state, csk->flags, csk->atid, 343 &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port), 344 &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port), 345 csk->rss_qid); 346 347 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 348 } 349 #endif 350 351 static void send_close_req(struct cxgbi_sock *csk) 352 { 353 struct sk_buff *skb = csk->cpl_close; 354 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 355 unsigned int tid = csk->tid; 356 357 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 358 "csk 0x%p,%u,0x%lx, tid %u.\n", 359 csk, csk->state, csk->flags, csk->tid); 360 csk->cpl_close = NULL; 361 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 362 INIT_TP_WR(req, tid); 363 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 364 req->rsvd = 0; 365 366 cxgbi_sock_skb_entail(csk, skb); 367 if (csk->state >= CTP_ESTABLISHED) 368 push_tx_frames(csk, 1); 369 } 370 371 static void abort_arp_failure(void *handle, struct sk_buff *skb) 372 { 373 struct cxgbi_sock *csk = (struct cxgbi_sock *)handle; 374 struct cpl_abort_req *req; 375 376 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 377 "csk 0x%p,%u,0x%lx, tid %u, abort.\n", 378 csk, csk->state, csk->flags, csk->tid); 379 req = (struct cpl_abort_req *)skb->data; 380 req->cmd = CPL_ABORT_NO_RST; 381 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 382 } 383 384 static void send_abort_req(struct cxgbi_sock *csk) 385 { 386 struct cpl_abort_req *req; 387 struct sk_buff *skb = csk->cpl_abort_req; 388 389 if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev) 390 return; 391 cxgbi_sock_set_state(csk, CTP_ABORTING); 392 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 393 cxgbi_sock_purge_write_queue(csk); 394 395 csk->cpl_abort_req = NULL; 396 req = (struct cpl_abort_req *)skb->head; 397 set_queue(skb, CPL_PRIORITY_DATA, csk); 398 req->cmd = CPL_ABORT_SEND_RST; 399 t4_set_arp_err_handler(skb, csk, abort_arp_failure); 400 INIT_TP_WR(req, csk->tid); 401 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 402 req->rsvd0 = htonl(csk->snd_nxt); 403 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 404 405 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 406 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 407 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 408 req->rsvd1); 409 410 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 411 } 412 413 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 414 { 415 struct sk_buff *skb = csk->cpl_abort_rpl; 416 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 417 418 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 419 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 420 csk, csk->state, csk->flags, csk->tid, rst_status); 421 422 csk->cpl_abort_rpl = NULL; 423 set_queue(skb, CPL_PRIORITY_DATA, csk); 424 INIT_TP_WR(rpl, csk->tid); 425 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 426 rpl->cmd = rst_status; 427 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 428 } 429 430 /* 431 * CPL connection rx data ack: host -> 432 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 433 * credits sent. 434 */ 435 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 436 { 437 struct sk_buff *skb; 438 struct cpl_rx_data_ack *req; 439 440 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 441 "csk 0x%p,%u,0x%lx,%u, credit %u.\n", 442 csk, csk->state, csk->flags, csk->tid, credits); 443 444 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 445 if (!skb) { 446 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 447 return 0; 448 } 449 req = (struct cpl_rx_data_ack *)skb->head; 450 451 set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); 452 INIT_TP_WR(req, csk->tid); 453 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 454 csk->tid)); 455 req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) 456 | RX_FORCE_ACK_F); 457 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 458 return credits; 459 } 460 461 /* 462 * sgl_len - calculates the size of an SGL of the given capacity 463 * @n: the number of SGL entries 464 * Calculates the number of flits needed for a scatter/gather list that 465 * can hold the given number of entries. 466 */ 467 static inline unsigned int sgl_len(unsigned int n) 468 { 469 n--; 470 return (3 * n) / 2 + (n & 1) + 2; 471 } 472 473 /* 474 * calc_tx_flits_ofld - calculate # of flits for an offload packet 475 * @skb: the packet 476 * 477 * Returns the number of flits needed for the given offload packet. 478 * These packets are already fully constructed and no additional headers 479 * will be added. 480 */ 481 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 482 { 483 unsigned int flits, cnt; 484 485 if (is_ofld_imm(skb)) 486 return DIV_ROUND_UP(skb->len, 8); 487 flits = skb_transport_offset(skb) / 8; 488 cnt = skb_shinfo(skb)->nr_frags; 489 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 490 cnt++; 491 return flits + sgl_len(cnt); 492 } 493 494 static inline void send_tx_flowc_wr(struct cxgbi_sock *csk) 495 { 496 struct sk_buff *skb; 497 struct fw_flowc_wr *flowc; 498 int flowclen, i; 499 500 flowclen = 80; 501 skb = alloc_wr(flowclen, 0, GFP_ATOMIC); 502 flowc = (struct fw_flowc_wr *)skb->head; 503 flowc->op_to_nparams = 504 htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(8)); 505 flowc->flowid_len16 = 506 htonl(FW_WR_LEN16_V(DIV_ROUND_UP(72, 16)) | 507 FW_WR_FLOWID_V(csk->tid)); 508 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 509 flowc->mnemval[0].val = htonl(csk->cdev->pfvf); 510 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 511 flowc->mnemval[1].val = htonl(csk->tx_chan); 512 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 513 flowc->mnemval[2].val = htonl(csk->tx_chan); 514 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 515 flowc->mnemval[3].val = htonl(csk->rss_qid); 516 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 517 flowc->mnemval[4].val = htonl(csk->snd_nxt); 518 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 519 flowc->mnemval[5].val = htonl(csk->rcv_nxt); 520 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 521 flowc->mnemval[6].val = htonl(cxgb4i_snd_win); 522 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 523 flowc->mnemval[7].val = htonl(csk->advmss); 524 flowc->mnemval[8].mnemonic = 0; 525 flowc->mnemval[8].val = 0; 526 for (i = 0; i < 9; i++) { 527 flowc->mnemval[i].r4[0] = 0; 528 flowc->mnemval[i].r4[1] = 0; 529 flowc->mnemval[i].r4[2] = 0; 530 } 531 set_queue(skb, CPL_PRIORITY_DATA, csk); 532 533 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 534 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n", 535 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid, 536 csk->snd_nxt, csk->rcv_nxt, cxgb4i_snd_win, 537 csk->advmss); 538 539 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 540 } 541 542 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 543 int dlen, int len, u32 credits, int compl) 544 { 545 struct fw_ofld_tx_data_wr *req; 546 unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3; 547 unsigned int wr_ulp_mode = 0, val; 548 549 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req)); 550 551 if (is_ofld_imm(skb)) { 552 req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) | 553 FW_WR_COMPL_F | 554 FW_WR_IMMDLEN_V(dlen)); 555 req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) | 556 FW_WR_LEN16_V(credits)); 557 } else { 558 req->op_to_immdlen = 559 cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) | 560 FW_WR_COMPL_F | 561 FW_WR_IMMDLEN_V(0)); 562 req->flowid_len16 = 563 cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 564 FW_WR_LEN16_V(credits)); 565 } 566 if (submode) 567 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) | 568 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 569 val = skb_peek(&csk->write_queue) ? 0 : 1; 570 req->tunnel_to_proxy = htonl(wr_ulp_mode | 571 FW_OFLD_TX_DATA_WR_SHOVE_V(val)); 572 req->plen = htonl(len); 573 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) 574 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 575 } 576 577 static void arp_failure_skb_discard(void *handle, struct sk_buff *skb) 578 { 579 kfree_skb(skb); 580 } 581 582 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 583 { 584 int total_size = 0; 585 struct sk_buff *skb; 586 587 if (unlikely(csk->state < CTP_ESTABLISHED || 588 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 589 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK | 590 1 << CXGBI_DBG_PDU_TX, 591 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 592 csk, csk->state, csk->flags, csk->tid); 593 return 0; 594 } 595 596 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { 597 int dlen = skb->len; 598 int len = skb->len; 599 unsigned int credits_needed; 600 601 skb_reset_transport_header(skb); 602 if (is_ofld_imm(skb)) 603 credits_needed = DIV_ROUND_UP(dlen + 604 sizeof(struct fw_ofld_tx_data_wr), 16); 605 else 606 credits_needed = DIV_ROUND_UP(8*calc_tx_flits_ofld(skb) 607 + sizeof(struct fw_ofld_tx_data_wr), 608 16); 609 610 if (csk->wr_cred < credits_needed) { 611 log_debug(1 << CXGBI_DBG_PDU_TX, 612 "csk 0x%p, skb %u/%u, wr %d < %u.\n", 613 csk, skb->len, skb->data_len, 614 credits_needed, csk->wr_cred); 615 break; 616 } 617 __skb_unlink(skb, &csk->write_queue); 618 set_queue(skb, CPL_PRIORITY_DATA, csk); 619 skb->csum = credits_needed; 620 csk->wr_cred -= credits_needed; 621 csk->wr_una_cred += credits_needed; 622 cxgbi_sock_enqueue_wr(csk, skb); 623 624 log_debug(1 << CXGBI_DBG_PDU_TX, 625 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 626 csk, skb->len, skb->data_len, credits_needed, 627 csk->wr_cred, csk->wr_una_cred); 628 629 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 630 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 631 send_tx_flowc_wr(csk); 632 skb->csum += 5; 633 csk->wr_cred -= 5; 634 csk->wr_una_cred += 5; 635 } 636 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 637 make_tx_data_wr(csk, skb, dlen, len, credits_needed, 638 req_completion); 639 csk->snd_nxt += len; 640 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 641 } 642 total_size += skb->truesize; 643 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard); 644 645 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 646 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n", 647 csk, csk->state, csk->flags, csk->tid, skb, len); 648 649 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 650 } 651 return total_size; 652 } 653 654 static inline void free_atid(struct cxgbi_sock *csk) 655 { 656 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 657 658 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 659 cxgb4_free_atid(lldi->tids, csk->atid); 660 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 661 cxgbi_sock_put(csk); 662 } 663 } 664 665 static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb) 666 { 667 struct cxgbi_sock *csk; 668 struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data; 669 unsigned short tcp_opt = ntohs(req->tcp_opt); 670 unsigned int tid = GET_TID(req); 671 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 672 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 673 struct tid_info *t = lldi->tids; 674 u32 rcv_isn = be32_to_cpu(req->rcv_isn); 675 676 csk = lookup_atid(t, atid); 677 if (unlikely(!csk)) { 678 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev); 679 goto rel_skb; 680 } 681 682 if (csk->atid != atid) { 683 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n", 684 atid, csk, csk->state, csk->flags, csk->tid, csk->atid); 685 goto rel_skb; 686 } 687 688 pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 689 (&csk->saddr), (&csk->daddr), 690 atid, tid, csk, csk->state, csk->flags, rcv_isn); 691 692 module_put(THIS_MODULE); 693 694 cxgbi_sock_get(csk); 695 csk->tid = tid; 696 cxgb4_insert_tid(lldi->tids, csk, tid); 697 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 698 699 free_atid(csk); 700 701 spin_lock_bh(&csk->lock); 702 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 703 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 704 csk, csk->state, csk->flags, csk->tid); 705 706 if (csk->retry_timer.function) { 707 del_timer(&csk->retry_timer); 708 csk->retry_timer.function = NULL; 709 } 710 711 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 712 /* 713 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't 714 * pass through opt0. 715 */ 716 if (cxgb4i_rcv_win > (RCV_BUFSIZ_MASK << 10)) 717 csk->rcv_wup -= cxgb4i_rcv_win - (RCV_BUFSIZ_MASK << 10); 718 719 csk->advmss = lldi->mtus[GET_TCPOPT_MSS(tcp_opt)] - 40; 720 if (GET_TCPOPT_TSTAMP(tcp_opt)) 721 csk->advmss -= 12; 722 if (csk->advmss < 128) 723 csk->advmss = 128; 724 725 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 726 "csk 0x%p, mss_idx %u, advmss %u.\n", 727 csk, GET_TCPOPT_MSS(tcp_opt), csk->advmss); 728 729 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 730 731 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 732 send_abort_req(csk); 733 else { 734 if (skb_queue_len(&csk->write_queue)) 735 push_tx_frames(csk, 0); 736 cxgbi_conn_tx_open(csk); 737 } 738 spin_unlock_bh(&csk->lock); 739 740 rel_skb: 741 __kfree_skb(skb); 742 } 743 744 static int act_open_rpl_status_to_errno(int status) 745 { 746 switch (status) { 747 case CPL_ERR_CONN_RESET: 748 return -ECONNREFUSED; 749 case CPL_ERR_ARP_MISS: 750 return -EHOSTUNREACH; 751 case CPL_ERR_CONN_TIMEDOUT: 752 return -ETIMEDOUT; 753 case CPL_ERR_TCAM_FULL: 754 return -ENOMEM; 755 case CPL_ERR_CONN_EXIST: 756 return -EADDRINUSE; 757 default: 758 return -EIO; 759 } 760 } 761 762 static void csk_act_open_retry_timer(unsigned long data) 763 { 764 struct sk_buff *skb = NULL; 765 struct cxgbi_sock *csk = (struct cxgbi_sock *)data; 766 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 767 void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *, 768 struct l2t_entry *); 769 int t4 = is_t4(lldi->adapter_type), size, size6; 770 771 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 772 "csk 0x%p,%u,0x%lx,%u.\n", 773 csk, csk->state, csk->flags, csk->tid); 774 775 cxgbi_sock_get(csk); 776 spin_lock_bh(&csk->lock); 777 778 if (t4) { 779 size = sizeof(struct cpl_act_open_req); 780 size6 = sizeof(struct cpl_act_open_req6); 781 } else { 782 size = sizeof(struct cpl_t5_act_open_req); 783 size6 = sizeof(struct cpl_t5_act_open_req6); 784 } 785 786 if (csk->csk_family == AF_INET) { 787 send_act_open_func = send_act_open_req; 788 skb = alloc_wr(size, 0, GFP_ATOMIC); 789 #if IS_ENABLED(CONFIG_IPV6) 790 } else { 791 send_act_open_func = send_act_open_req6; 792 skb = alloc_wr(size6, 0, GFP_ATOMIC); 793 #endif 794 } 795 796 if (!skb) 797 cxgbi_sock_fail_act_open(csk, -ENOMEM); 798 else { 799 skb->sk = (struct sock *)csk; 800 t4_set_arp_err_handler(skb, csk, 801 cxgbi_sock_act_open_req_arp_failure); 802 send_act_open_func(csk, skb, csk->l2t); 803 } 804 805 spin_unlock_bh(&csk->lock); 806 cxgbi_sock_put(csk); 807 808 } 809 810 static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 811 { 812 struct cxgbi_sock *csk; 813 struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data; 814 unsigned int tid = GET_TID(rpl); 815 unsigned int atid = 816 GET_TID_TID(GET_AOPEN_ATID(be32_to_cpu(rpl->atid_status))); 817 unsigned int status = GET_AOPEN_STATUS(be32_to_cpu(rpl->atid_status)); 818 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 819 struct tid_info *t = lldi->tids; 820 821 csk = lookup_atid(t, atid); 822 if (unlikely(!csk)) { 823 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid); 824 goto rel_skb; 825 } 826 827 pr_info_ipaddr("tid %u/%u, status %u.\n" 828 "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr), 829 atid, tid, status, csk, csk->state, csk->flags); 830 831 if (status == CPL_ERR_RTX_NEG_ADVICE) 832 goto rel_skb; 833 834 module_put(THIS_MODULE); 835 836 if (status && status != CPL_ERR_TCAM_FULL && 837 status != CPL_ERR_CONN_EXIST && 838 status != CPL_ERR_ARP_MISS) 839 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl)); 840 841 cxgbi_sock_get(csk); 842 spin_lock_bh(&csk->lock); 843 844 if (status == CPL_ERR_CONN_EXIST && 845 csk->retry_timer.function != csk_act_open_retry_timer) { 846 csk->retry_timer.function = csk_act_open_retry_timer; 847 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 848 } else 849 cxgbi_sock_fail_act_open(csk, 850 act_open_rpl_status_to_errno(status)); 851 852 spin_unlock_bh(&csk->lock); 853 cxgbi_sock_put(csk); 854 rel_skb: 855 __kfree_skb(skb); 856 } 857 858 static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb) 859 { 860 struct cxgbi_sock *csk; 861 struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data; 862 unsigned int tid = GET_TID(req); 863 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 864 struct tid_info *t = lldi->tids; 865 866 csk = lookup_tid(t, tid); 867 if (unlikely(!csk)) { 868 pr_err("can't find connection for tid %u.\n", tid); 869 goto rel_skb; 870 } 871 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 872 (&csk->saddr), (&csk->daddr), 873 csk, csk->state, csk->flags, csk->tid); 874 cxgbi_sock_rcv_peer_close(csk); 875 rel_skb: 876 __kfree_skb(skb); 877 } 878 879 static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 880 { 881 struct cxgbi_sock *csk; 882 struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data; 883 unsigned int tid = GET_TID(rpl); 884 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 885 struct tid_info *t = lldi->tids; 886 887 csk = lookup_tid(t, tid); 888 if (unlikely(!csk)) { 889 pr_err("can't find connection for tid %u.\n", tid); 890 goto rel_skb; 891 } 892 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 893 (&csk->saddr), (&csk->daddr), 894 csk, csk->state, csk->flags, csk->tid); 895 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 896 rel_skb: 897 __kfree_skb(skb); 898 } 899 900 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 901 int *need_rst) 902 { 903 switch (abort_reason) { 904 case CPL_ERR_BAD_SYN: /* fall through */ 905 case CPL_ERR_CONN_RESET: 906 return csk->state > CTP_ESTABLISHED ? 907 -EPIPE : -ECONNRESET; 908 case CPL_ERR_XMIT_TIMEDOUT: 909 case CPL_ERR_PERSIST_TIMEDOUT: 910 case CPL_ERR_FINWAIT2_TIMEDOUT: 911 case CPL_ERR_KEEPALIVE_TIMEDOUT: 912 return -ETIMEDOUT; 913 default: 914 return -EIO; 915 } 916 } 917 918 static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 919 { 920 struct cxgbi_sock *csk; 921 struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data; 922 unsigned int tid = GET_TID(req); 923 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 924 struct tid_info *t = lldi->tids; 925 int rst_status = CPL_ABORT_NO_RST; 926 927 csk = lookup_tid(t, tid); 928 if (unlikely(!csk)) { 929 pr_err("can't find connection for tid %u.\n", tid); 930 goto rel_skb; 931 } 932 933 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 934 (&csk->saddr), (&csk->daddr), 935 csk, csk->state, csk->flags, csk->tid, req->status); 936 937 if (req->status == CPL_ERR_RTX_NEG_ADVICE || 938 req->status == CPL_ERR_PERSIST_NEG_ADVICE) 939 goto rel_skb; 940 941 cxgbi_sock_get(csk); 942 spin_lock_bh(&csk->lock); 943 944 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 945 946 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 947 send_tx_flowc_wr(csk); 948 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 949 } 950 951 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 952 cxgbi_sock_set_state(csk, CTP_ABORTING); 953 954 send_abort_rpl(csk, rst_status); 955 956 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 957 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 958 cxgbi_sock_closed(csk); 959 } 960 961 spin_unlock_bh(&csk->lock); 962 cxgbi_sock_put(csk); 963 rel_skb: 964 __kfree_skb(skb); 965 } 966 967 static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 968 { 969 struct cxgbi_sock *csk; 970 struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data; 971 unsigned int tid = GET_TID(rpl); 972 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 973 struct tid_info *t = lldi->tids; 974 975 csk = lookup_tid(t, tid); 976 if (!csk) 977 goto rel_skb; 978 979 if (csk) 980 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 981 (&csk->saddr), (&csk->daddr), csk, 982 csk->state, csk->flags, csk->tid, rpl->status); 983 984 if (rpl->status == CPL_ERR_ABORT_FAILED) 985 goto rel_skb; 986 987 cxgbi_sock_rcv_abort_rpl(csk); 988 rel_skb: 989 __kfree_skb(skb); 990 } 991 992 static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) 993 { 994 struct cxgbi_sock *csk; 995 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; 996 unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); 997 unsigned int tid = GET_TID(cpl); 998 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 999 struct tid_info *t = lldi->tids; 1000 1001 csk = lookup_tid(t, tid); 1002 if (unlikely(!csk)) { 1003 pr_err("can't find conn. for tid %u.\n", tid); 1004 goto rel_skb; 1005 } 1006 1007 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1008 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", 1009 csk, csk->state, csk->flags, csk->tid, skb, skb->len, 1010 pdu_len_ddp); 1011 1012 spin_lock_bh(&csk->lock); 1013 1014 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1015 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1016 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1017 csk, csk->state, csk->flags, csk->tid); 1018 if (csk->state != CTP_ABORTING) 1019 goto abort_conn; 1020 else 1021 goto discard; 1022 } 1023 1024 cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq); 1025 cxgbi_skcb_flags(skb) = 0; 1026 1027 skb_reset_transport_header(skb); 1028 __skb_pull(skb, sizeof(*cpl)); 1029 __pskb_trim(skb, ntohs(cpl->len)); 1030 1031 if (!csk->skb_ulp_lhdr) { 1032 unsigned char *bhs; 1033 unsigned int hlen, dlen, plen; 1034 1035 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1036 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n", 1037 csk, csk->state, csk->flags, csk->tid, skb); 1038 csk->skb_ulp_lhdr = skb; 1039 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); 1040 1041 if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) { 1042 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n", 1043 csk->tid, cxgbi_skcb_tcp_seq(skb), 1044 csk->rcv_nxt); 1045 goto abort_conn; 1046 } 1047 1048 bhs = skb->data; 1049 hlen = ntohs(cpl->len); 1050 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF; 1051 1052 plen = ISCSI_PDU_LEN(pdu_len_ddp); 1053 if (is_t4(lldi->adapter_type)) 1054 plen -= 40; 1055 1056 if ((hlen + dlen) != plen) { 1057 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len " 1058 "mismatch %u != %u + %u, seq 0x%x.\n", 1059 csk->tid, plen, hlen, dlen, 1060 cxgbi_skcb_tcp_seq(skb)); 1061 goto abort_conn; 1062 } 1063 1064 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3); 1065 if (dlen) 1066 cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len; 1067 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb); 1068 1069 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1070 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n", 1071 csk, skb, *bhs, hlen, dlen, 1072 ntohl(*((unsigned int *)(bhs + 16))), 1073 ntohl(*((unsigned int *)(bhs + 24)))); 1074 1075 } else { 1076 struct sk_buff *lskb = csk->skb_ulp_lhdr; 1077 1078 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); 1079 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1080 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", 1081 csk, csk->state, csk->flags, skb, lskb); 1082 } 1083 1084 __skb_queue_tail(&csk->receive_queue, skb); 1085 spin_unlock_bh(&csk->lock); 1086 return; 1087 1088 abort_conn: 1089 send_abort_req(csk); 1090 discard: 1091 spin_unlock_bh(&csk->lock); 1092 rel_skb: 1093 __kfree_skb(skb); 1094 } 1095 1096 static void do_rx_data_ddp(struct cxgbi_device *cdev, 1097 struct sk_buff *skb) 1098 { 1099 struct cxgbi_sock *csk; 1100 struct sk_buff *lskb; 1101 struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data; 1102 unsigned int tid = GET_TID(rpl); 1103 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1104 struct tid_info *t = lldi->tids; 1105 unsigned int status = ntohl(rpl->ddpvld); 1106 1107 csk = lookup_tid(t, tid); 1108 if (unlikely(!csk)) { 1109 pr_err("can't find connection for tid %u.\n", tid); 1110 goto rel_skb; 1111 } 1112 1113 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1114 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n", 1115 csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr); 1116 1117 spin_lock_bh(&csk->lock); 1118 1119 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1120 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1121 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1122 csk, csk->state, csk->flags, csk->tid); 1123 if (csk->state != CTP_ABORTING) 1124 goto abort_conn; 1125 else 1126 goto discard; 1127 } 1128 1129 if (!csk->skb_ulp_lhdr) { 1130 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid); 1131 goto abort_conn; 1132 } 1133 1134 lskb = csk->skb_ulp_lhdr; 1135 csk->skb_ulp_lhdr = NULL; 1136 1137 cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc); 1138 1139 if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb)) 1140 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n", 1141 csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb)); 1142 1143 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) { 1144 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n", 1145 csk, lskb, status, cxgbi_skcb_flags(lskb)); 1146 cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR); 1147 } 1148 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) { 1149 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n", 1150 csk, lskb, status, cxgbi_skcb_flags(lskb)); 1151 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR); 1152 } 1153 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) { 1154 log_debug(1 << CXGBI_DBG_PDU_RX, 1155 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n", 1156 csk, lskb, status); 1157 cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR); 1158 } 1159 if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) && 1160 !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) { 1161 log_debug(1 << CXGBI_DBG_PDU_RX, 1162 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n", 1163 csk, lskb, status); 1164 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD); 1165 } 1166 log_debug(1 << CXGBI_DBG_PDU_RX, 1167 "csk 0x%p, lskb 0x%p, f 0x%lx.\n", 1168 csk, lskb, cxgbi_skcb_flags(lskb)); 1169 1170 cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS); 1171 cxgbi_conn_pdu_ready(csk); 1172 spin_unlock_bh(&csk->lock); 1173 goto rel_skb; 1174 1175 abort_conn: 1176 send_abort_req(csk); 1177 discard: 1178 spin_unlock_bh(&csk->lock); 1179 rel_skb: 1180 __kfree_skb(skb); 1181 } 1182 1183 static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb) 1184 { 1185 struct cxgbi_sock *csk; 1186 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data; 1187 unsigned int tid = GET_TID(rpl); 1188 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1189 struct tid_info *t = lldi->tids; 1190 1191 csk = lookup_tid(t, tid); 1192 if (unlikely(!csk)) 1193 pr_err("can't find connection for tid %u.\n", tid); 1194 else { 1195 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1196 "csk 0x%p,%u,0x%lx,%u.\n", 1197 csk, csk->state, csk->flags, csk->tid); 1198 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una), 1199 rpl->seq_vld); 1200 } 1201 __kfree_skb(skb); 1202 } 1203 1204 static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 1205 { 1206 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; 1207 unsigned int tid = GET_TID(rpl); 1208 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1209 struct tid_info *t = lldi->tids; 1210 struct cxgbi_sock *csk; 1211 1212 csk = lookup_tid(t, tid); 1213 if (!csk) 1214 pr_err("can't find conn. for tid %u.\n", tid); 1215 1216 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1217 "csk 0x%p,%u,%lx,%u, status 0x%x.\n", 1218 csk, csk->state, csk->flags, csk->tid, rpl->status); 1219 1220 if (rpl->status != CPL_ERR_NONE) 1221 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n", 1222 csk, tid, rpl->status); 1223 1224 __kfree_skb(skb); 1225 } 1226 1227 static int alloc_cpls(struct cxgbi_sock *csk) 1228 { 1229 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 1230 0, GFP_KERNEL); 1231 if (!csk->cpl_close) 1232 return -ENOMEM; 1233 1234 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 1235 0, GFP_KERNEL); 1236 if (!csk->cpl_abort_req) 1237 goto free_cpls; 1238 1239 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 1240 0, GFP_KERNEL); 1241 if (!csk->cpl_abort_rpl) 1242 goto free_cpls; 1243 return 0; 1244 1245 free_cpls: 1246 cxgbi_sock_free_cpl_skbs(csk); 1247 return -ENOMEM; 1248 } 1249 1250 static inline void l2t_put(struct cxgbi_sock *csk) 1251 { 1252 if (csk->l2t) { 1253 cxgb4_l2t_release(csk->l2t); 1254 csk->l2t = NULL; 1255 cxgbi_sock_put(csk); 1256 } 1257 } 1258 1259 static void release_offload_resources(struct cxgbi_sock *csk) 1260 { 1261 struct cxgb4_lld_info *lldi; 1262 1263 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1264 "csk 0x%p,%u,0x%lx,%u.\n", 1265 csk, csk->state, csk->flags, csk->tid); 1266 1267 cxgbi_sock_free_cpl_skbs(csk); 1268 if (csk->wr_cred != csk->wr_max_cred) { 1269 cxgbi_sock_purge_wr_queue(csk); 1270 cxgbi_sock_reset_wr_list(csk); 1271 } 1272 1273 l2t_put(csk); 1274 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 1275 free_atid(csk); 1276 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 1277 lldi = cxgbi_cdev_priv(csk->cdev); 1278 cxgb4_remove_tid(lldi->tids, 0, csk->tid); 1279 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 1280 cxgbi_sock_put(csk); 1281 } 1282 csk->dst = NULL; 1283 csk->cdev = NULL; 1284 } 1285 1286 static int init_act_open(struct cxgbi_sock *csk) 1287 { 1288 struct cxgbi_device *cdev = csk->cdev; 1289 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1290 struct net_device *ndev = cdev->ports[csk->port_id]; 1291 struct sk_buff *skb = NULL; 1292 struct neighbour *n = NULL; 1293 void *daddr; 1294 unsigned int step; 1295 unsigned int size, size6; 1296 int t4 = is_t4(lldi->adapter_type); 1297 1298 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1299 "csk 0x%p,%u,0x%lx,%u.\n", 1300 csk, csk->state, csk->flags, csk->tid); 1301 1302 if (csk->csk_family == AF_INET) 1303 daddr = &csk->daddr.sin_addr.s_addr; 1304 #if IS_ENABLED(CONFIG_IPV6) 1305 else if (csk->csk_family == AF_INET6) 1306 daddr = &csk->daddr6.sin6_addr; 1307 #endif 1308 else { 1309 pr_err("address family 0x%x not supported\n", csk->csk_family); 1310 goto rel_resource; 1311 } 1312 1313 n = dst_neigh_lookup(csk->dst, daddr); 1314 1315 if (!n) { 1316 pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name); 1317 goto rel_resource; 1318 } 1319 1320 csk->atid = cxgb4_alloc_atid(lldi->tids, csk); 1321 if (csk->atid < 0) { 1322 pr_err("%s, NO atid available.\n", ndev->name); 1323 return -EINVAL; 1324 } 1325 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 1326 cxgbi_sock_get(csk); 1327 1328 csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0); 1329 if (!csk->l2t) { 1330 pr_err("%s, cannot alloc l2t.\n", ndev->name); 1331 goto rel_resource; 1332 } 1333 cxgbi_sock_get(csk); 1334 1335 if (t4) { 1336 size = sizeof(struct cpl_act_open_req); 1337 size6 = sizeof(struct cpl_act_open_req6); 1338 } else { 1339 size = sizeof(struct cpl_t5_act_open_req); 1340 size6 = sizeof(struct cpl_t5_act_open_req6); 1341 } 1342 1343 if (csk->csk_family == AF_INET) 1344 skb = alloc_wr(size, 0, GFP_NOIO); 1345 #if IS_ENABLED(CONFIG_IPV6) 1346 else 1347 skb = alloc_wr(size6, 0, GFP_NOIO); 1348 #endif 1349 1350 if (!skb) 1351 goto rel_resource; 1352 skb->sk = (struct sock *)csk; 1353 t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure); 1354 1355 if (!csk->mtu) 1356 csk->mtu = dst_mtu(csk->dst); 1357 cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx); 1358 csk->tx_chan = cxgb4_port_chan(ndev); 1359 /* SMT two entries per row */ 1360 csk->smac_idx = ((cxgb4_port_viid(ndev) & 0x7F)) << 1; 1361 step = lldi->ntxq / lldi->nchan; 1362 csk->txq_idx = cxgb4_port_idx(ndev) * step; 1363 step = lldi->nrxq / lldi->nchan; 1364 csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step]; 1365 csk->wr_cred = lldi->wr_cred - 1366 DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); 1367 csk->wr_max_cred = csk->wr_cred; 1368 csk->wr_una_cred = 0; 1369 cxgbi_sock_reset_wr_list(csk); 1370 csk->err = 0; 1371 1372 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n", 1373 (&csk->saddr), (&csk->daddr), csk, csk->state, 1374 csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid, 1375 csk->mtu, csk->mss_idx, csk->smac_idx); 1376 1377 /* must wait for either a act_open_rpl or act_open_establish */ 1378 try_module_get(THIS_MODULE); 1379 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1380 if (csk->csk_family == AF_INET) 1381 send_act_open_req(csk, skb, csk->l2t); 1382 #if IS_ENABLED(CONFIG_IPV6) 1383 else 1384 send_act_open_req6(csk, skb, csk->l2t); 1385 #endif 1386 neigh_release(n); 1387 1388 return 0; 1389 1390 rel_resource: 1391 if (n) 1392 neigh_release(n); 1393 if (skb) 1394 __kfree_skb(skb); 1395 return -EINVAL; 1396 } 1397 1398 cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = { 1399 [CPL_ACT_ESTABLISH] = do_act_establish, 1400 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1401 [CPL_PEER_CLOSE] = do_peer_close, 1402 [CPL_ABORT_REQ_RSS] = do_abort_req_rss, 1403 [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss, 1404 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1405 [CPL_FW4_ACK] = do_fw4_ack, 1406 [CPL_ISCSI_HDR] = do_rx_iscsi_hdr, 1407 [CPL_ISCSI_DATA] = do_rx_iscsi_hdr, 1408 [CPL_SET_TCB_RPL] = do_set_tcb_rpl, 1409 [CPL_RX_DATA_DDP] = do_rx_data_ddp, 1410 [CPL_RX_ISCSI_DDP] = do_rx_data_ddp, 1411 }; 1412 1413 int cxgb4i_ofld_init(struct cxgbi_device *cdev) 1414 { 1415 int rc; 1416 1417 if (cxgb4i_max_connect > CXGB4I_MAX_CONN) 1418 cxgb4i_max_connect = CXGB4I_MAX_CONN; 1419 1420 rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base, 1421 cxgb4i_max_connect); 1422 if (rc < 0) 1423 return rc; 1424 1425 cdev->csk_release_offload_resources = release_offload_resources; 1426 cdev->csk_push_tx_frames = push_tx_frames; 1427 cdev->csk_send_abort_req = send_abort_req; 1428 cdev->csk_send_close_req = send_close_req; 1429 cdev->csk_send_rx_credits = send_rx_credits; 1430 cdev->csk_alloc_cpls = alloc_cpls; 1431 cdev->csk_init_act_open = init_act_open; 1432 1433 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1434 return 0; 1435 } 1436 1437 /* 1438 * functions to program the pagepod in h/w 1439 */ 1440 #define ULPMEM_IDATA_MAX_NPPODS 4 /* 256/PPOD_SIZE */ 1441 static inline void ulp_mem_io_set_hdr(struct cxgb4_lld_info *lldi, 1442 struct ulp_mem_io *req, 1443 unsigned int wr_len, unsigned int dlen, 1444 unsigned int pm_addr) 1445 { 1446 struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); 1447 1448 INIT_ULPTX_WR(req, wr_len, 0, 0); 1449 if (is_t4(lldi->adapter_type)) 1450 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 1451 (ULP_MEMIO_ORDER_F)); 1452 else 1453 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 1454 (T5_ULP_MEMIO_IMM_F)); 1455 req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); 1456 req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); 1457 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); 1458 1459 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 1460 idata->len = htonl(dlen); 1461 } 1462 1463 static int ddp_ppod_write_idata(struct cxgbi_device *cdev, unsigned int port_id, 1464 struct cxgbi_pagepod_hdr *hdr, unsigned int idx, 1465 unsigned int npods, 1466 struct cxgbi_gather_list *gl, 1467 unsigned int gl_pidx) 1468 { 1469 struct cxgbi_ddp_info *ddp = cdev->ddp; 1470 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1471 struct sk_buff *skb; 1472 struct ulp_mem_io *req; 1473 struct ulptx_idata *idata; 1474 struct cxgbi_pagepod *ppod; 1475 unsigned int pm_addr = idx * PPOD_SIZE + ddp->llimit; 1476 unsigned int dlen = PPOD_SIZE * npods; 1477 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + 1478 sizeof(struct ulptx_idata) + dlen, 16); 1479 unsigned int i; 1480 1481 skb = alloc_wr(wr_len, 0, GFP_ATOMIC); 1482 if (!skb) { 1483 pr_err("cdev 0x%p, idx %u, npods %u, OOM.\n", 1484 cdev, idx, npods); 1485 return -ENOMEM; 1486 } 1487 req = (struct ulp_mem_io *)skb->head; 1488 set_queue(skb, CPL_PRIORITY_CONTROL, NULL); 1489 1490 ulp_mem_io_set_hdr(lldi, req, wr_len, dlen, pm_addr); 1491 idata = (struct ulptx_idata *)(req + 1); 1492 ppod = (struct cxgbi_pagepod *)(idata + 1); 1493 1494 for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES_MAX) { 1495 if (!hdr && !gl) 1496 cxgbi_ddp_ppod_clear(ppod); 1497 else 1498 cxgbi_ddp_ppod_set(ppod, hdr, gl, gl_pidx); 1499 } 1500 1501 cxgb4_ofld_send(cdev->ports[port_id], skb); 1502 return 0; 1503 } 1504 1505 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr, 1506 unsigned int idx, unsigned int npods, 1507 struct cxgbi_gather_list *gl) 1508 { 1509 unsigned int i, cnt; 1510 int err = 0; 1511 1512 for (i = 0; i < npods; i += cnt, idx += cnt) { 1513 cnt = npods - i; 1514 if (cnt > ULPMEM_IDATA_MAX_NPPODS) 1515 cnt = ULPMEM_IDATA_MAX_NPPODS; 1516 err = ddp_ppod_write_idata(csk->cdev, csk->port_id, hdr, 1517 idx, cnt, gl, 4 * i); 1518 if (err < 0) 1519 break; 1520 } 1521 return err; 1522 } 1523 1524 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag, 1525 unsigned int idx, unsigned int npods) 1526 { 1527 unsigned int i, cnt; 1528 int err; 1529 1530 for (i = 0; i < npods; i += cnt, idx += cnt) { 1531 cnt = npods - i; 1532 if (cnt > ULPMEM_IDATA_MAX_NPPODS) 1533 cnt = ULPMEM_IDATA_MAX_NPPODS; 1534 err = ddp_ppod_write_idata(chba->cdev, chba->port_id, NULL, 1535 idx, cnt, NULL, 0); 1536 if (err < 0) 1537 break; 1538 } 1539 } 1540 1541 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid, 1542 int pg_idx, bool reply) 1543 { 1544 struct sk_buff *skb; 1545 struct cpl_set_tcb_field *req; 1546 1547 if (!pg_idx || pg_idx >= DDP_PGIDX_MAX) 1548 return 0; 1549 1550 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 1551 if (!skb) 1552 return -ENOMEM; 1553 1554 /* set up ulp page size */ 1555 req = (struct cpl_set_tcb_field *)skb->head; 1556 INIT_TP_WR(req, csk->tid); 1557 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1558 req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid)); 1559 req->word_cookie = htons(0); 1560 req->mask = cpu_to_be64(0x3 << 8); 1561 req->val = cpu_to_be64(pg_idx << 8); 1562 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 1563 1564 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1565 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx); 1566 1567 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 1568 return 0; 1569 } 1570 1571 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 1572 int hcrc, int dcrc, int reply) 1573 { 1574 struct sk_buff *skb; 1575 struct cpl_set_tcb_field *req; 1576 1577 if (!hcrc && !dcrc) 1578 return 0; 1579 1580 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 1581 if (!skb) 1582 return -ENOMEM; 1583 1584 csk->hcrc_len = (hcrc ? 4 : 0); 1585 csk->dcrc_len = (dcrc ? 4 : 0); 1586 /* set up ulp submode */ 1587 req = (struct cpl_set_tcb_field *)skb->head; 1588 INIT_TP_WR(req, tid); 1589 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1590 req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid)); 1591 req->word_cookie = htons(0); 1592 req->mask = cpu_to_be64(0x3 << 4); 1593 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | 1594 (dcrc ? ULP_CRC_DATA : 0)) << 4); 1595 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 1596 1597 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1598 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc); 1599 1600 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 1601 return 0; 1602 } 1603 1604 static int cxgb4i_ddp_init(struct cxgbi_device *cdev) 1605 { 1606 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1607 struct cxgbi_ddp_info *ddp = cdev->ddp; 1608 unsigned int tagmask, pgsz_factor[4]; 1609 int err; 1610 1611 if (ddp) { 1612 kref_get(&ddp->refcnt); 1613 pr_warn("cdev 0x%p, ddp 0x%p already set up.\n", 1614 cdev, cdev->ddp); 1615 return -EALREADY; 1616 } 1617 1618 err = cxgbi_ddp_init(cdev, lldi->vr->iscsi.start, 1619 lldi->vr->iscsi.start + lldi->vr->iscsi.size - 1, 1620 lldi->iscsi_iolen, lldi->iscsi_iolen); 1621 if (err < 0) 1622 return err; 1623 1624 ddp = cdev->ddp; 1625 1626 tagmask = ddp->idx_mask << PPOD_IDX_SHIFT; 1627 cxgbi_ddp_page_size_factor(pgsz_factor); 1628 cxgb4_iscsi_init(lldi->ports[0], tagmask, pgsz_factor); 1629 1630 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 1631 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 1632 cdev->csk_ddp_set = ddp_set_map; 1633 cdev->csk_ddp_clear = ddp_clear_map; 1634 1635 pr_info("cxgb4i 0x%p tag: sw %u, rsvd %u,%u, mask 0x%x.\n", 1636 cdev, cdev->tag_format.sw_bits, cdev->tag_format.rsvd_bits, 1637 cdev->tag_format.rsvd_shift, cdev->tag_format.rsvd_mask); 1638 pr_info("cxgb4i 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, " 1639 " %u/%u.\n", 1640 cdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask, 1641 ddp->rsvd_tag_mask, ddp->max_txsz, lldi->iscsi_iolen, 1642 ddp->max_rxsz, lldi->iscsi_iolen); 1643 pr_info("cxgb4i 0x%p max payload size: %u/%u, %u/%u.\n", 1644 cdev, cdev->tx_max_size, ddp->max_txsz, cdev->rx_max_size, 1645 ddp->max_rxsz); 1646 return 0; 1647 } 1648 1649 static void *t4_uld_add(const struct cxgb4_lld_info *lldi) 1650 { 1651 struct cxgbi_device *cdev; 1652 struct port_info *pi; 1653 int i, rc; 1654 1655 cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports); 1656 if (!cdev) { 1657 pr_info("t4 device 0x%p, register failed.\n", lldi); 1658 return NULL; 1659 } 1660 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n", 1661 cdev, lldi->adapter_type, lldi->nports, 1662 lldi->ports[0]->name, lldi->nchan, lldi->ntxq, 1663 lldi->nrxq, lldi->wr_cred); 1664 for (i = 0; i < lldi->nrxq; i++) 1665 log_debug(1 << CXGBI_DBG_DEV, 1666 "t4 0x%p, rxq id #%d: %u.\n", 1667 cdev, i, lldi->rxq_ids[i]); 1668 1669 memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi)); 1670 cdev->flags = CXGBI_FLAG_DEV_T4; 1671 cdev->pdev = lldi->pdev; 1672 cdev->ports = lldi->ports; 1673 cdev->nports = lldi->nports; 1674 cdev->mtus = lldi->mtus; 1675 cdev->nmtus = NMTUS; 1676 cdev->snd_win = cxgb4i_snd_win; 1677 cdev->rcv_win = cxgb4i_rcv_win; 1678 cdev->rx_credit_thres = cxgb4i_rx_credit_thres; 1679 cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN; 1680 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr); 1681 cdev->itp = &cxgb4i_iscsi_transport; 1682 1683 cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) 1684 << FW_VIID_PFN_S; 1685 pr_info("cdev 0x%p,%s, pfvf %u.\n", 1686 cdev, lldi->ports[0]->name, cdev->pfvf); 1687 1688 rc = cxgb4i_ddp_init(cdev); 1689 if (rc) { 1690 pr_info("t4 0x%p ddp init failed.\n", cdev); 1691 goto err_out; 1692 } 1693 rc = cxgb4i_ofld_init(cdev); 1694 if (rc) { 1695 pr_info("t4 0x%p ofld init failed.\n", cdev); 1696 goto err_out; 1697 } 1698 1699 rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN, 1700 &cxgb4i_host_template, cxgb4i_stt); 1701 if (rc) 1702 goto err_out; 1703 1704 for (i = 0; i < cdev->nports; i++) { 1705 pi = netdev_priv(lldi->ports[i]); 1706 cdev->hbas[i]->port_id = pi->port_id; 1707 } 1708 return cdev; 1709 1710 err_out: 1711 cxgbi_device_unregister(cdev); 1712 return ERR_PTR(-ENOMEM); 1713 } 1714 1715 #define RX_PULL_LEN 128 1716 static int t4_uld_rx_handler(void *handle, const __be64 *rsp, 1717 const struct pkt_gl *pgl) 1718 { 1719 const struct cpl_act_establish *rpl; 1720 struct sk_buff *skb; 1721 unsigned int opc; 1722 struct cxgbi_device *cdev = handle; 1723 1724 if (pgl == NULL) { 1725 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 1726 1727 skb = alloc_wr(len, 0, GFP_ATOMIC); 1728 if (!skb) 1729 goto nomem; 1730 skb_copy_to_linear_data(skb, &rsp[1], len); 1731 } else { 1732 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) { 1733 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", 1734 pgl->va, be64_to_cpu(*rsp), 1735 be64_to_cpu(*(u64 *)pgl->va), 1736 pgl->tot_len); 1737 return 0; 1738 } 1739 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN); 1740 if (unlikely(!skb)) 1741 goto nomem; 1742 } 1743 1744 rpl = (struct cpl_act_establish *)skb->data; 1745 opc = rpl->ot.opcode; 1746 log_debug(1 << CXGBI_DBG_TOE, 1747 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", 1748 cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb); 1749 if (cxgb4i_cplhandlers[opc]) 1750 cxgb4i_cplhandlers[opc](cdev, skb); 1751 else { 1752 pr_err("No handler for opcode 0x%x.\n", opc); 1753 __kfree_skb(skb); 1754 } 1755 return 0; 1756 nomem: 1757 log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n"); 1758 return 1; 1759 } 1760 1761 static int t4_uld_state_change(void *handle, enum cxgb4_state state) 1762 { 1763 struct cxgbi_device *cdev = handle; 1764 1765 switch (state) { 1766 case CXGB4_STATE_UP: 1767 pr_info("cdev 0x%p, UP.\n", cdev); 1768 break; 1769 case CXGB4_STATE_START_RECOVERY: 1770 pr_info("cdev 0x%p, RECOVERY.\n", cdev); 1771 /* close all connections */ 1772 break; 1773 case CXGB4_STATE_DOWN: 1774 pr_info("cdev 0x%p, DOWN.\n", cdev); 1775 break; 1776 case CXGB4_STATE_DETACH: 1777 pr_info("cdev 0x%p, DETACH.\n", cdev); 1778 cxgbi_device_unregister(cdev); 1779 break; 1780 default: 1781 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state); 1782 break; 1783 } 1784 return 0; 1785 } 1786 1787 static int __init cxgb4i_init_module(void) 1788 { 1789 int rc; 1790 1791 printk(KERN_INFO "%s", version); 1792 1793 rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt); 1794 if (rc < 0) 1795 return rc; 1796 cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info); 1797 1798 return 0; 1799 } 1800 1801 static void __exit cxgb4i_exit_module(void) 1802 { 1803 cxgb4_unregister_uld(CXGB4_ULD_ISCSI); 1804 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4); 1805 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt); 1806 } 1807 1808 module_init(cxgb4i_init_module); 1809 module_exit(cxgb4i_exit_module); 1810