1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/workqueue.h> 10 #include <linux/kthread.h> 11 #include <asm/unaligned.h> 12 #include <target/target_core_base.h> 13 #include <target/target_core_fabric.h> 14 #include "cxgbit.h" 15 16 struct sge_opaque_hdr { 17 void *dev; 18 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 19 }; 20 21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 22 23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 24 sizeof(struct fw_ofld_tx_data_wr)) 25 26 static struct sk_buff * 27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 28 { 29 struct sk_buff *skb = NULL; 30 u8 submode = 0; 31 int errcode; 32 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 33 34 if (len) { 35 skb = alloc_skb_with_frags(hdr_len, len, 36 0, &errcode, 37 GFP_KERNEL); 38 if (!skb) 39 return NULL; 40 41 skb_reserve(skb, TX_HDR_LEN); 42 skb_reset_transport_header(skb); 43 __skb_put(skb, ISCSI_HDR_LEN); 44 skb->data_len = len; 45 skb->len += len; 46 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 47 48 } else { 49 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 50 51 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 52 if (!skb) 53 return NULL; 54 55 skb_reserve(skb, TX_HDR_LEN + iso_len); 56 skb_reset_transport_header(skb); 57 __skb_put(skb, ISCSI_HDR_LEN); 58 } 59 60 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 61 cxgbit_skcb_submode(skb) = submode; 62 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 63 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 64 return skb; 65 } 66 67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 68 { 69 return __cxgbit_alloc_skb(csk, len, false); 70 } 71 72 /* 73 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 74 * @skb: the packet 75 * 76 * Returns true if a packet can be sent as an offload WR with immediate 77 * data. We currently use the same limit as for Ethernet packets. 78 */ 79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 80 { 81 int length = skb->len; 82 83 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 84 length += sizeof(struct fw_ofld_tx_data_wr); 85 86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 87 length += sizeof(struct cpl_tx_data_iso); 88 89 #define MAX_IMM_TX_PKT_LEN 256 90 return length <= MAX_IMM_TX_PKT_LEN; 91 } 92 93 /* 94 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 95 * @n: the number of SGL entries 96 * Calculates the number of flits needed for a scatter/gather list that 97 * can hold the given number of entries. 98 */ 99 static inline unsigned int cxgbit_sgl_len(unsigned int n) 100 { 101 n--; 102 return (3 * n) / 2 + (n & 1) + 2; 103 } 104 105 /* 106 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 107 * @skb: the packet 108 * 109 * Returns the number of flits needed for the given offload packet. 110 * These packets are already fully constructed and no additional headers 111 * will be added. 112 */ 113 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 114 { 115 unsigned int flits, cnt; 116 117 if (cxgbit_is_ofld_imm(skb)) 118 return DIV_ROUND_UP(skb->len, 8); 119 flits = skb_transport_offset(skb) / 8; 120 cnt = skb_shinfo(skb)->nr_frags; 121 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 122 cnt++; 123 return flits + cxgbit_sgl_len(cnt); 124 } 125 126 #define CXGBIT_ISO_FSLICE 0x1 127 #define CXGBIT_ISO_LSLICE 0x2 128 static void 129 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 130 { 131 struct cpl_tx_data_iso *cpl; 132 unsigned int submode = cxgbit_skcb_submode(skb); 133 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 134 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 135 136 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 137 138 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 139 CPL_TX_DATA_ISO_FIRST_V(fslice) | 140 CPL_TX_DATA_ISO_LAST_V(lslice) | 141 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 142 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 143 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 144 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 145 CPL_TX_DATA_ISO_SCSI_V(2)); 146 147 cpl->ahs_len = 0; 148 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 149 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 150 cpl->len = htonl(iso_info->len); 151 cpl->reserved2_seglen_offset = htonl(0); 152 cpl->datasn_offset = htonl(0); 153 cpl->buffer_offset = htonl(0); 154 cpl->reserved3 = 0; 155 156 __skb_pull(skb, sizeof(*cpl)); 157 } 158 159 static void 160 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 161 u32 len, u32 credits, u32 compl) 162 { 163 struct fw_ofld_tx_data_wr *req; 164 u32 submode = cxgbit_skcb_submode(skb); 165 u32 wr_ulp_mode = 0; 166 u32 hdr_size = sizeof(*req); 167 u32 opcode = FW_OFLD_TX_DATA_WR; 168 u32 immlen = 0; 169 u32 force = TX_FORCE_V(!submode); 170 171 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 172 opcode = FW_ISCSI_TX_DATA_WR; 173 immlen += sizeof(struct cpl_tx_data_iso); 174 hdr_size += sizeof(struct cpl_tx_data_iso); 175 submode |= 8; 176 } 177 178 if (cxgbit_is_ofld_imm(skb)) 179 immlen += dlen; 180 181 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 182 hdr_size); 183 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 184 FW_WR_COMPL_V(compl) | 185 FW_WR_IMMDLEN_V(immlen)); 186 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 187 FW_WR_LEN16_V(credits)); 188 req->plen = htonl(len); 189 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 190 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 191 192 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 193 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 194 } 195 196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 197 { 198 kfree_skb(skb); 199 } 200 201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 202 { 203 struct sk_buff *skb; 204 205 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 206 u32 dlen = skb->len; 207 u32 len = skb->len; 208 u32 credits_needed; 209 u32 compl = 0; 210 u32 flowclen16 = 0; 211 u32 iso_cpl_len = 0; 212 213 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 214 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 215 216 if (cxgbit_is_ofld_imm(skb)) 217 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 218 else 219 credits_needed = DIV_ROUND_UP((8 * 220 cxgbit_calc_tx_flits_ofld(skb)) + 221 iso_cpl_len, 16); 222 223 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 224 credits_needed += DIV_ROUND_UP( 225 sizeof(struct fw_ofld_tx_data_wr), 16); 226 /* 227 * Assumes the initial credits is large enough to support 228 * fw_flowc_wr plus largest possible first payload 229 */ 230 231 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 232 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 233 csk->wr_cred -= flowclen16; 234 csk->wr_una_cred += flowclen16; 235 } 236 237 if (csk->wr_cred < credits_needed) { 238 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 239 csk, skb->len, skb->data_len, 240 credits_needed, csk->wr_cred); 241 break; 242 } 243 __skb_unlink(skb, &csk->txq); 244 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 245 skb->csum = credits_needed + flowclen16; 246 csk->wr_cred -= credits_needed; 247 csk->wr_una_cred += credits_needed; 248 249 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 250 csk, skb->len, skb->data_len, credits_needed, 251 csk->wr_cred, csk->wr_una_cred); 252 253 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 254 len += cxgbit_skcb_tx_extralen(skb); 255 256 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 257 (!before(csk->write_seq, 258 csk->snd_una + csk->snd_win))) { 259 compl = 1; 260 csk->wr_una_cred = 0; 261 } 262 263 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 264 compl); 265 csk->snd_nxt += len; 266 267 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 268 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 269 struct cpl_close_con_req *req = 270 (struct cpl_close_con_req *)skb->data; 271 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 272 csk->wr_una_cred = 0; 273 } 274 275 cxgbit_sock_enqueue_wr(csk, skb); 276 t4_set_arp_err_handler(skb, csk, 277 cxgbit_arp_failure_skb_discard); 278 279 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 280 csk, csk->tid, skb, len); 281 282 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 283 } 284 } 285 286 static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 287 { 288 spin_lock_bh(&csk->lock); 289 290 if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 291 csk->lock_owner = true; 292 293 spin_unlock_bh(&csk->lock); 294 295 return csk->lock_owner; 296 } 297 298 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 299 { 300 struct sk_buff_head backlogq; 301 struct sk_buff *skb; 302 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 303 304 skb_queue_head_init(&backlogq); 305 306 spin_lock_bh(&csk->lock); 307 while (skb_queue_len(&csk->backlogq)) { 308 skb_queue_splice_init(&csk->backlogq, &backlogq); 309 spin_unlock_bh(&csk->lock); 310 311 while ((skb = __skb_dequeue(&backlogq))) { 312 fn = cxgbit_skcb_rx_backlog_fn(skb); 313 fn(csk, skb); 314 } 315 316 spin_lock_bh(&csk->lock); 317 } 318 319 csk->lock_owner = false; 320 spin_unlock_bh(&csk->lock); 321 } 322 323 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 324 { 325 int ret = 0; 326 327 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 328 329 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 330 signal_pending(current))) { 331 __kfree_skb(skb); 332 __skb_queue_purge(&csk->ppodq); 333 ret = -1; 334 spin_lock_bh(&csk->lock); 335 if (csk->lock_owner) { 336 spin_unlock_bh(&csk->lock); 337 goto unlock; 338 } 339 spin_unlock_bh(&csk->lock); 340 return ret; 341 } 342 343 csk->write_seq += skb->len + 344 cxgbit_skcb_tx_extralen(skb); 345 346 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 347 __skb_queue_tail(&csk->txq, skb); 348 cxgbit_push_tx_frames(csk); 349 350 unlock: 351 cxgbit_unlock_sock(csk); 352 return ret; 353 } 354 355 static int 356 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 357 u32 data_length) 358 { 359 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 360 u32 padding = ((-data_length) & 3); 361 struct scatterlist *sg; 362 struct page *page; 363 unsigned int page_off; 364 365 if (padding) 366 nr_frags--; 367 368 /* 369 * We know each entry in t_data_sg contains a page. 370 */ 371 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 372 page_off = (data_offset % PAGE_SIZE); 373 374 while (data_length && (i < nr_frags)) { 375 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 376 377 page = sg_page(sg); 378 379 get_page(page); 380 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 381 cur_len); 382 skb->data_len += cur_len; 383 skb->len += cur_len; 384 skb->truesize += cur_len; 385 386 data_length -= cur_len; 387 page_off = 0; 388 sg = sg_next(sg); 389 i++; 390 } 391 392 if (data_length) 393 return -1; 394 395 if (padding) { 396 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 397 if (!page) 398 return -1; 399 skb_fill_page_desc(skb, i, page, 0, padding); 400 skb->data_len += padding; 401 skb->len += padding; 402 skb->truesize += padding; 403 } 404 405 return 0; 406 } 407 408 static int 409 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 410 struct iscsi_datain_req *dr) 411 { 412 struct iscsi_conn *conn = csk->conn; 413 struct sk_buff *skb; 414 struct iscsi_datain datain; 415 struct cxgbit_iso_info iso_info; 416 u32 data_length = cmd->se_cmd.data_length; 417 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 418 u32 num_pdu, plen, tx_data = 0; 419 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 420 SCF_TRANSPORT_TASK_SENSE); 421 bool set_statsn = false; 422 int ret = -1; 423 424 while (data_length) { 425 num_pdu = (data_length + mrdsl - 1) / mrdsl; 426 if (num_pdu > csk->max_iso_npdu) 427 num_pdu = csk->max_iso_npdu; 428 429 plen = num_pdu * mrdsl; 430 if (plen > data_length) 431 plen = data_length; 432 433 skb = __cxgbit_alloc_skb(csk, 0, true); 434 if (unlikely(!skb)) 435 return -ENOMEM; 436 437 memset(skb->data, 0, ISCSI_HDR_LEN); 438 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 439 cxgbit_skcb_submode(skb) |= (csk->submode & 440 CXGBIT_SUBMODE_DCRC); 441 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 442 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 443 ((num_pdu - 1) * ISCSI_HDR_LEN); 444 445 memset(&datain, 0, sizeof(struct iscsi_datain)); 446 memset(&iso_info, 0, sizeof(iso_info)); 447 448 if (!tx_data) 449 iso_info.flags |= CXGBIT_ISO_FSLICE; 450 451 if (!(data_length - plen)) { 452 iso_info.flags |= CXGBIT_ISO_LSLICE; 453 if (!task_sense) { 454 datain.flags = ISCSI_FLAG_DATA_STATUS; 455 iscsit_increment_maxcmdsn(cmd, conn->sess); 456 cmd->stat_sn = conn->stat_sn++; 457 set_statsn = true; 458 } 459 } 460 461 iso_info.burst_len = num_pdu * mrdsl; 462 iso_info.mpdu = mrdsl; 463 iso_info.len = ISCSI_HDR_LEN + plen; 464 465 cxgbit_cpl_tx_data_iso(skb, &iso_info); 466 467 datain.offset = tx_data; 468 datain.data_sn = cmd->data_sn - 1; 469 470 iscsit_build_datain_pdu(cmd, conn, &datain, 471 (struct iscsi_data_rsp *)skb->data, 472 set_statsn); 473 474 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 475 if (unlikely(ret)) { 476 __kfree_skb(skb); 477 goto out; 478 } 479 480 ret = cxgbit_queue_skb(csk, skb); 481 if (unlikely(ret)) 482 goto out; 483 484 tx_data += plen; 485 data_length -= plen; 486 487 cmd->read_data_done += plen; 488 cmd->data_sn += num_pdu; 489 } 490 491 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 492 493 return 0; 494 495 out: 496 return ret; 497 } 498 499 static int 500 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 501 const struct iscsi_datain *datain) 502 { 503 struct sk_buff *skb; 504 int ret = 0; 505 506 skb = cxgbit_alloc_skb(csk, 0); 507 if (unlikely(!skb)) 508 return -ENOMEM; 509 510 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 511 512 if (datain->length) { 513 cxgbit_skcb_submode(skb) |= (csk->submode & 514 CXGBIT_SUBMODE_DCRC); 515 cxgbit_skcb_tx_extralen(skb) = 516 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 517 } 518 519 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 520 if (ret < 0) { 521 __kfree_skb(skb); 522 return ret; 523 } 524 525 return cxgbit_queue_skb(csk, skb); 526 } 527 528 static int 529 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 530 struct iscsi_datain_req *dr, 531 const struct iscsi_datain *datain) 532 { 533 struct cxgbit_sock *csk = conn->context; 534 u32 data_length = cmd->se_cmd.data_length; 535 u32 padding = ((-data_length) & 3); 536 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 537 538 if ((data_length > mrdsl) && (!dr->recovery) && 539 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 540 atomic_long_add(data_length - datain->length, 541 &conn->sess->tx_data_octets); 542 return cxgbit_tx_datain_iso(csk, cmd, dr); 543 } 544 545 return cxgbit_tx_datain(csk, cmd, datain); 546 } 547 548 static int 549 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 550 const void *data_buf, u32 data_buf_len) 551 { 552 struct cxgbit_sock *csk = conn->context; 553 struct sk_buff *skb; 554 u32 padding = ((-data_buf_len) & 3); 555 556 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 557 if (unlikely(!skb)) 558 return -ENOMEM; 559 560 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 561 562 if (data_buf_len) { 563 u32 pad_bytes = 0; 564 565 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 566 567 if (padding) 568 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 569 &pad_bytes, padding); 570 } 571 572 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 573 cxgbit_skcb_submode(skb)]; 574 575 return cxgbit_queue_skb(csk, skb); 576 } 577 578 int 579 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 580 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 581 { 582 if (dr) 583 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 584 else 585 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 586 } 587 588 int cxgbit_validate_params(struct iscsi_conn *conn) 589 { 590 struct cxgbit_sock *csk = conn->context; 591 struct cxgbit_device *cdev = csk->com.cdev; 592 struct iscsi_param *param; 593 u32 max_xmitdsl; 594 595 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 596 conn->param_list); 597 if (!param) 598 return -1; 599 600 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 601 return -1; 602 603 if (max_xmitdsl > cdev->mdsl) { 604 if (iscsi_change_param_sprintf( 605 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 606 return -1; 607 } 608 609 return 0; 610 } 611 612 static int cxgbit_set_digest(struct cxgbit_sock *csk) 613 { 614 struct iscsi_conn *conn = csk->conn; 615 struct iscsi_param *param; 616 617 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 618 if (!param) { 619 pr_err("param not found key %s\n", HEADERDIGEST); 620 return -1; 621 } 622 623 if (!strcmp(param->value, CRC32C)) 624 csk->submode |= CXGBIT_SUBMODE_HCRC; 625 626 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 627 if (!param) { 628 csk->submode = 0; 629 pr_err("param not found key %s\n", DATADIGEST); 630 return -1; 631 } 632 633 if (!strcmp(param->value, CRC32C)) 634 csk->submode |= CXGBIT_SUBMODE_DCRC; 635 636 if (cxgbit_setup_conn_digest(csk)) { 637 csk->submode = 0; 638 return -1; 639 } 640 641 return 0; 642 } 643 644 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 645 { 646 struct iscsi_conn *conn = csk->conn; 647 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 648 struct iscsi_param *param; 649 u32 mrdsl, mbl; 650 u32 max_npdu, max_iso_npdu; 651 652 if (conn->login->leading_connection) { 653 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 654 conn->param_list); 655 if (!param) { 656 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 657 return -1; 658 } 659 660 if (strcmp(param->value, YES)) 661 return 0; 662 663 param = iscsi_find_param_from_key(DATAPDUINORDER, 664 conn->param_list); 665 if (!param) { 666 pr_err("param not found key %s\n", DATAPDUINORDER); 667 return -1; 668 } 669 670 if (strcmp(param->value, YES)) 671 return 0; 672 673 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 674 conn->param_list); 675 if (!param) { 676 pr_err("param not found key %s\n", MAXBURSTLENGTH); 677 return -1; 678 } 679 680 if (kstrtou32(param->value, 0, &mbl) < 0) 681 return -1; 682 } else { 683 if (!conn->sess->sess_ops->DataSequenceInOrder) 684 return 0; 685 if (!conn->sess->sess_ops->DataPDUInOrder) 686 return 0; 687 688 mbl = conn->sess->sess_ops->MaxBurstLength; 689 } 690 691 mrdsl = conn_ops->MaxRecvDataSegmentLength; 692 max_npdu = mbl / mrdsl; 693 694 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 695 (ISCSI_HDR_LEN + mrdsl + 696 cxgbit_digest_len[csk->submode]); 697 698 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 699 700 if (csk->max_iso_npdu <= 1) 701 csk->max_iso_npdu = 0; 702 703 return 0; 704 } 705 706 static int cxgbit_set_params(struct iscsi_conn *conn) 707 { 708 struct cxgbit_sock *csk = conn->context; 709 struct cxgbit_device *cdev = csk->com.cdev; 710 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 711 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 712 struct iscsi_param *param; 713 u8 erl; 714 715 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 716 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 717 718 if (conn->login->leading_connection) { 719 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 720 conn->param_list); 721 if (!param) { 722 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 723 return -1; 724 } 725 if (kstrtou8(param->value, 0, &erl) < 0) 726 return -1; 727 } else { 728 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 729 } 730 731 if (!erl) { 732 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 733 if (cxgbit_set_iso_npdu(csk)) 734 return -1; 735 } 736 737 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 738 if (cxgbit_setup_conn_pgidx(csk, 739 ppm->tformat.pgsz_idx_dflt)) 740 return -1; 741 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 742 } 743 } 744 745 if (cxgbit_set_digest(csk)) 746 return -1; 747 748 return 0; 749 } 750 751 int 752 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 753 u32 length) 754 { 755 struct cxgbit_sock *csk = conn->context; 756 struct sk_buff *skb; 757 u32 padding_buf = 0; 758 u8 padding = ((-length) & 3); 759 760 skb = cxgbit_alloc_skb(csk, length + padding); 761 if (!skb) 762 return -ENOMEM; 763 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 764 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 765 766 if (padding) 767 skb_store_bits(skb, ISCSI_HDR_LEN + length, 768 &padding_buf, padding); 769 770 if (login->login_complete) { 771 if (cxgbit_set_params(conn)) { 772 kfree_skb(skb); 773 return -1; 774 } 775 776 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 777 } 778 779 if (cxgbit_queue_skb(csk, skb)) 780 return -1; 781 782 if ((!login->login_complete) && (!login->login_failed)) 783 schedule_delayed_work(&conn->login_work, 0); 784 785 return 0; 786 } 787 788 static void 789 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 790 unsigned int nents) 791 { 792 struct skb_seq_state st; 793 const u8 *buf; 794 unsigned int consumed = 0, buf_len; 795 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 796 797 skb_prepare_seq_read(skb, pdu_cb->doffset, 798 pdu_cb->doffset + pdu_cb->dlen, 799 &st); 800 801 while (true) { 802 buf_len = skb_seq_read(consumed, &buf, &st); 803 if (!buf_len) { 804 skb_abort_seq_read(&st); 805 break; 806 } 807 808 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 809 buf_len, consumed); 810 } 811 } 812 813 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 814 { 815 struct iscsi_conn *conn = csk->conn; 816 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 817 struct cxgbit_cmd *ccmd; 818 struct iscsi_cmd *cmd; 819 820 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 821 if (!cmd) { 822 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 823 return NULL; 824 } 825 826 ccmd = iscsit_priv_cmd(cmd); 827 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 828 ccmd->setup_ddp = true; 829 830 return cmd; 831 } 832 833 static int 834 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 835 u32 length) 836 { 837 struct iscsi_conn *conn = cmd->conn; 838 struct cxgbit_sock *csk = conn->context; 839 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 840 841 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 842 pr_err("ImmediateData CRC32C DataDigest error\n"); 843 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 844 pr_err("Unable to recover from" 845 " Immediate Data digest failure while" 846 " in ERL=0.\n"); 847 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 848 (unsigned char *)hdr); 849 return IMMEDIATE_DATA_CANNOT_RECOVER; 850 } 851 852 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 853 (unsigned char *)hdr); 854 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 855 } 856 857 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 858 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 859 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 860 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 861 862 sg_init_table(&ccmd->sg, 1); 863 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 864 dfrag->page_offset); 865 get_page(dfrag->page.p); 866 867 cmd->se_cmd.t_data_sg = &ccmd->sg; 868 cmd->se_cmd.t_data_nents = 1; 869 870 ccmd->release = true; 871 } else { 872 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 873 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 874 875 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 876 } 877 878 cmd->write_data_done += pdu_cb->dlen; 879 880 if (cmd->write_data_done == cmd->se_cmd.data_length) { 881 spin_lock_bh(&cmd->istate_lock); 882 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 883 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 884 spin_unlock_bh(&cmd->istate_lock); 885 } 886 887 return IMMEDIATE_DATA_NORMAL_OPERATION; 888 } 889 890 static int 891 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 892 bool dump_payload) 893 { 894 struct iscsi_conn *conn = cmd->conn; 895 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 896 /* 897 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 898 */ 899 if (dump_payload) 900 goto after_immediate_data; 901 902 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 903 cmd->first_burst_len); 904 after_immediate_data: 905 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 906 /* 907 * A PDU/CmdSN carrying Immediate Data passed 908 * DataCRC, check against ExpCmdSN/MaxCmdSN if 909 * Immediate Bit is not set. 910 */ 911 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 912 (unsigned char *)hdr, 913 hdr->cmdsn); 914 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 915 return -1; 916 917 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 918 target_put_sess_cmd(&cmd->se_cmd); 919 return 0; 920 } else if (cmd->unsolicited_data) { 921 iscsit_set_unsoliticed_dataout(cmd); 922 } 923 924 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 925 /* 926 * Immediate Data failed DataCRC and ERL>=1, 927 * silently drop this PDU and let the initiator 928 * plug the CmdSN gap. 929 * 930 * FIXME: Send Unsolicited NOPIN with reserved 931 * TTT here to help the initiator figure out 932 * the missing CmdSN, although they should be 933 * intelligent enough to determine the missing 934 * CmdSN and issue a retry to plug the sequence. 935 */ 936 cmd->i_state = ISTATE_REMOVE; 937 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 938 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 939 return -1; 940 941 return 0; 942 } 943 944 static int 945 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 946 { 947 struct iscsi_conn *conn = csk->conn; 948 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 949 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 950 int rc; 951 bool dump_payload = false; 952 953 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 954 if (rc < 0) 955 return rc; 956 957 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 958 (pdu_cb->nr_dfrags == 1)) 959 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 960 961 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 962 if (rc < 0) 963 return 0; 964 else if (rc > 0) 965 dump_payload = true; 966 967 if (!pdu_cb->dlen) 968 return 0; 969 970 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 971 } 972 973 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 974 { 975 struct scatterlist *sg_start; 976 struct iscsi_conn *conn = csk->conn; 977 struct iscsi_cmd *cmd = NULL; 978 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 979 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 980 u32 data_offset = be32_to_cpu(hdr->offset); 981 u32 data_len = pdu_cb->dlen; 982 int rc, sg_nents, sg_off; 983 bool dcrc_err = false; 984 985 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 986 if (rc < 0) 987 return rc; 988 else if (!cmd) 989 return 0; 990 991 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 992 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 993 " DataSN: 0x%08x\n", 994 hdr->itt, hdr->offset, data_len, 995 hdr->datasn); 996 997 dcrc_err = true; 998 goto check_payload; 999 } 1000 1001 pr_debug("DataOut data_len: %u, " 1002 "write_data_done: %u, data_length: %u\n", 1003 data_len, cmd->write_data_done, 1004 cmd->se_cmd.data_length); 1005 1006 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1007 sg_off = data_offset / PAGE_SIZE; 1008 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1009 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 1010 1011 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 1012 } 1013 1014 check_payload: 1015 1016 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1017 if (rc < 0) 1018 return rc; 1019 1020 return 0; 1021 } 1022 1023 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1024 { 1025 struct iscsi_conn *conn = csk->conn; 1026 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1027 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1028 unsigned char *ping_data = NULL; 1029 u32 payload_length = pdu_cb->dlen; 1030 int ret; 1031 1032 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1033 if (ret < 0) 1034 return 0; 1035 1036 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1037 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1038 pr_err("Unable to recover from" 1039 " NOPOUT Ping DataCRC failure while in" 1040 " ERL=0.\n"); 1041 ret = -1; 1042 goto out; 1043 } else { 1044 /* 1045 * drop this PDU and let the 1046 * initiator plug the CmdSN gap. 1047 */ 1048 pr_info("Dropping NOPOUT" 1049 " Command CmdSN: 0x%08x due to" 1050 " DataCRC error.\n", hdr->cmdsn); 1051 ret = 0; 1052 goto out; 1053 } 1054 } 1055 1056 /* 1057 * Handle NOP-OUT payload for traditional iSCSI sockets 1058 */ 1059 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1060 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1061 if (!ping_data) { 1062 pr_err("Unable to allocate memory for" 1063 " NOPOUT ping data.\n"); 1064 ret = -1; 1065 goto out; 1066 } 1067 1068 skb_copy_bits(csk->skb, pdu_cb->doffset, 1069 ping_data, payload_length); 1070 1071 ping_data[payload_length] = '\0'; 1072 /* 1073 * Attach ping data to struct iscsi_cmd->buf_ptr. 1074 */ 1075 cmd->buf_ptr = ping_data; 1076 cmd->buf_ptr_size = payload_length; 1077 1078 pr_debug("Got %u bytes of NOPOUT ping" 1079 " data.\n", payload_length); 1080 pr_debug("Ping Data: \"%s\"\n", ping_data); 1081 } 1082 1083 return iscsit_process_nop_out(conn, cmd, hdr); 1084 out: 1085 if (cmd) 1086 iscsit_free_cmd(cmd, false); 1087 return ret; 1088 } 1089 1090 static int 1091 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1092 { 1093 struct iscsi_conn *conn = csk->conn; 1094 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1095 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1096 u32 payload_length = pdu_cb->dlen; 1097 int rc; 1098 unsigned char *text_in = NULL; 1099 1100 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1101 if (rc < 0) 1102 return rc; 1103 1104 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1105 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1106 pr_err("Unable to recover from" 1107 " Text Data digest failure while in" 1108 " ERL=0.\n"); 1109 goto reject; 1110 } else { 1111 /* 1112 * drop this PDU and let the 1113 * initiator plug the CmdSN gap. 1114 */ 1115 pr_info("Dropping Text" 1116 " Command CmdSN: 0x%08x due to" 1117 " DataCRC error.\n", hdr->cmdsn); 1118 return 0; 1119 } 1120 } 1121 1122 if (payload_length) { 1123 text_in = kzalloc(payload_length, GFP_KERNEL); 1124 if (!text_in) { 1125 pr_err("Unable to allocate text_in of payload_length: %u\n", 1126 payload_length); 1127 return -ENOMEM; 1128 } 1129 skb_copy_bits(csk->skb, pdu_cb->doffset, 1130 text_in, payload_length); 1131 1132 text_in[payload_length - 1] = '\0'; 1133 1134 cmd->text_in_ptr = text_in; 1135 } 1136 1137 return iscsit_process_text_cmd(conn, cmd, hdr); 1138 1139 reject: 1140 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1141 pdu_cb->hdr); 1142 } 1143 1144 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1145 { 1146 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1147 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1148 struct iscsi_conn *conn = csk->conn; 1149 struct iscsi_cmd *cmd = NULL; 1150 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1151 int ret = -EINVAL; 1152 1153 switch (opcode) { 1154 case ISCSI_OP_SCSI_CMD: 1155 cmd = cxgbit_allocate_cmd(csk); 1156 if (!cmd) 1157 goto reject; 1158 1159 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1160 break; 1161 case ISCSI_OP_SCSI_DATA_OUT: 1162 ret = cxgbit_handle_iscsi_dataout(csk); 1163 break; 1164 case ISCSI_OP_NOOP_OUT: 1165 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1166 cmd = cxgbit_allocate_cmd(csk); 1167 if (!cmd) 1168 goto reject; 1169 } 1170 1171 ret = cxgbit_handle_nop_out(csk, cmd); 1172 break; 1173 case ISCSI_OP_SCSI_TMFUNC: 1174 cmd = cxgbit_allocate_cmd(csk); 1175 if (!cmd) 1176 goto reject; 1177 1178 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1179 (unsigned char *)hdr); 1180 break; 1181 case ISCSI_OP_TEXT: 1182 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1183 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1184 if (!cmd) 1185 goto reject; 1186 } else { 1187 cmd = cxgbit_allocate_cmd(csk); 1188 if (!cmd) 1189 goto reject; 1190 } 1191 1192 ret = cxgbit_handle_text_cmd(csk, cmd); 1193 break; 1194 case ISCSI_OP_LOGOUT: 1195 cmd = cxgbit_allocate_cmd(csk); 1196 if (!cmd) 1197 goto reject; 1198 1199 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1200 if (ret > 0) 1201 wait_for_completion_timeout(&conn->conn_logout_comp, 1202 SECONDS_FOR_LOGOUT_COMP 1203 * HZ); 1204 break; 1205 case ISCSI_OP_SNACK: 1206 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1207 break; 1208 default: 1209 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1210 dump_stack(); 1211 break; 1212 } 1213 1214 return ret; 1215 1216 reject: 1217 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1218 (unsigned char *)hdr); 1219 return ret; 1220 } 1221 1222 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1223 { 1224 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1225 struct iscsi_conn *conn = csk->conn; 1226 struct iscsi_hdr *hdr = pdu_cb->hdr; 1227 u8 opcode; 1228 1229 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1230 atomic_long_inc(&conn->sess->conn_digest_errors); 1231 goto transport_err; 1232 } 1233 1234 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1235 goto transport_err; 1236 1237 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1238 1239 if (conn->sess->sess_ops->SessionType && 1240 ((!(opcode & ISCSI_OP_TEXT)) || 1241 (!(opcode & ISCSI_OP_LOGOUT)))) { 1242 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1243 " while in Discovery Session, rejecting.\n", opcode); 1244 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1245 (unsigned char *)hdr); 1246 goto transport_err; 1247 } 1248 1249 if (cxgbit_target_rx_opcode(csk) < 0) 1250 goto transport_err; 1251 1252 return 0; 1253 1254 transport_err: 1255 return -1; 1256 } 1257 1258 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1259 { 1260 struct iscsi_conn *conn = csk->conn; 1261 struct iscsi_login *login = conn->login; 1262 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1263 struct iscsi_login_req *login_req; 1264 1265 login_req = (struct iscsi_login_req *)login->req; 1266 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1267 1268 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1269 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1270 login_req->flags, login_req->itt, login_req->cmdsn, 1271 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1272 /* 1273 * Setup the initial iscsi_login values from the leading 1274 * login request PDU. 1275 */ 1276 if (login->first_request) { 1277 login_req = (struct iscsi_login_req *)login->req; 1278 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1279 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1280 login_req->flags); 1281 login->version_min = login_req->min_version; 1282 login->version_max = login_req->max_version; 1283 memcpy(login->isid, login_req->isid, 6); 1284 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1285 login->init_task_tag = login_req->itt; 1286 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1287 login->cid = be16_to_cpu(login_req->cid); 1288 login->tsih = be16_to_cpu(login_req->tsih); 1289 } 1290 1291 if (iscsi_target_check_login_request(conn, login) < 0) 1292 return -1; 1293 1294 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1295 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1296 1297 return 0; 1298 } 1299 1300 static int 1301 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1302 { 1303 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1304 int ret; 1305 1306 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1307 1308 csk->skb = skb; 1309 1310 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1311 ret = cxgbit_rx_login_pdu(csk); 1312 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1313 } else { 1314 ret = cxgbit_rx_opcode(csk); 1315 } 1316 1317 return ret; 1318 } 1319 1320 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1321 { 1322 struct skb_shared_info *ssi = skb_shinfo(skb); 1323 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1324 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1325 u8 i; 1326 1327 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1328 skb, skb->head, skb->data, skb->len, skb->data_len, 1329 ssi->nr_frags); 1330 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1331 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1332 1333 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1334 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1335 "frags %u.\n", 1336 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1337 pdu_cb->ddigest, pdu_cb->frags); 1338 for (i = 0; i < ssi->nr_frags; i++) 1339 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1340 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1341 } 1342 1343 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1344 { 1345 struct sk_buff *skb = csk->lro_hskb; 1346 struct skb_shared_info *ssi = skb_shinfo(skb); 1347 u8 i; 1348 1349 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1350 for (i = 0; i < ssi->nr_frags; i++) 1351 put_page(skb_frag_page(&ssi->frags[i])); 1352 ssi->nr_frags = 0; 1353 } 1354 1355 static void 1356 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1357 { 1358 struct sk_buff *hskb = csk->lro_hskb; 1359 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1360 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1361 struct skb_shared_info *hssi = skb_shinfo(hskb); 1362 struct skb_shared_info *ssi = skb_shinfo(skb); 1363 unsigned int len = 0; 1364 1365 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1366 hpdu_cb->flags = pdu_cb->flags; 1367 hpdu_cb->seq = pdu_cb->seq; 1368 hpdu_cb->hdr = pdu_cb->hdr; 1369 hpdu_cb->hlen = pdu_cb->hlen; 1370 1371 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx], 1372 sizeof(skb_frag_t)); 1373 1374 get_page(skb_frag_page(&hssi->frags[0])); 1375 hssi->nr_frags = 1; 1376 hpdu_cb->frags = 1; 1377 hpdu_cb->hfrag_idx = 0; 1378 1379 len = hssi->frags[0].size; 1380 hskb->len = len; 1381 hskb->data_len = len; 1382 hskb->truesize = len; 1383 } 1384 1385 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1386 u8 hfrag_idx = 1, i; 1387 1388 hpdu_cb->flags |= pdu_cb->flags; 1389 1390 len = 0; 1391 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) { 1392 memcpy(&hssi->frags[hfrag_idx], 1393 &ssi->frags[pdu_cb->dfrag_idx + i], 1394 sizeof(skb_frag_t)); 1395 1396 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1397 1398 len += hssi->frags[hfrag_idx].size; 1399 1400 hssi->nr_frags++; 1401 hpdu_cb->frags++; 1402 } 1403 1404 hpdu_cb->dlen = pdu_cb->dlen; 1405 hpdu_cb->doffset = hpdu_cb->hlen; 1406 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1407 hpdu_cb->dfrag_idx = 1; 1408 hskb->len += len; 1409 hskb->data_len += len; 1410 hskb->truesize += len; 1411 } 1412 1413 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1414 hpdu_cb->flags |= pdu_cb->flags; 1415 1416 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1417 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1418 1419 hpdu_cb->ddigest = pdu_cb->ddigest; 1420 hpdu_cb->pdulen = pdu_cb->pdulen; 1421 } 1422 } 1423 1424 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1425 { 1426 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1427 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1428 u8 pdu_idx = 0, last_idx = 0; 1429 int ret = 0; 1430 1431 if (!pdu_cb->complete) { 1432 cxgbit_lro_skb_merge(csk, skb, 0); 1433 1434 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1435 struct sk_buff *hskb = csk->lro_hskb; 1436 1437 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1438 1439 cxgbit_lro_hskb_reset(csk); 1440 1441 if (ret < 0) 1442 goto out; 1443 } 1444 1445 pdu_idx = 1; 1446 } 1447 1448 if (lro_cb->pdu_idx) 1449 last_idx = lro_cb->pdu_idx - 1; 1450 1451 for (; pdu_idx <= last_idx; pdu_idx++) { 1452 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1453 if (ret < 0) 1454 goto out; 1455 } 1456 1457 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1458 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1459 1460 out: 1461 return ret; 1462 } 1463 1464 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1465 { 1466 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1467 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1468 int ret = -1; 1469 1470 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1471 (pdu_cb->seq != csk->rcv_nxt)) { 1472 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1473 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1474 cxgbit_lro_skb_dump(skb); 1475 return ret; 1476 } 1477 1478 csk->rcv_nxt += lro_cb->pdu_totallen; 1479 1480 ret = cxgbit_process_lro_skb(csk, skb); 1481 1482 csk->rx_credits += lro_cb->pdu_totallen; 1483 1484 if (csk->rx_credits >= (csk->rcv_win / 4)) 1485 cxgbit_rx_data_ack(csk); 1486 1487 return ret; 1488 } 1489 1490 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1491 { 1492 int ret = -1; 1493 1494 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) 1495 ret = cxgbit_rx_lro_skb(csk, skb); 1496 1497 __kfree_skb(skb); 1498 return ret; 1499 } 1500 1501 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1502 { 1503 spin_lock_bh(&csk->rxq.lock); 1504 if (skb_queue_len(&csk->rxq)) { 1505 skb_queue_splice_init(&csk->rxq, rxq); 1506 spin_unlock_bh(&csk->rxq.lock); 1507 return true; 1508 } 1509 spin_unlock_bh(&csk->rxq.lock); 1510 return false; 1511 } 1512 1513 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1514 { 1515 struct sk_buff *skb; 1516 struct sk_buff_head rxq; 1517 1518 skb_queue_head_init(&rxq); 1519 1520 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1521 1522 if (signal_pending(current)) 1523 goto out; 1524 1525 while ((skb = __skb_dequeue(&rxq))) { 1526 if (cxgbit_rx_skb(csk, skb)) 1527 goto out; 1528 } 1529 1530 return 0; 1531 out: 1532 __skb_queue_purge(&rxq); 1533 return -1; 1534 } 1535 1536 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1537 { 1538 struct cxgbit_sock *csk = conn->context; 1539 int ret = -1; 1540 1541 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1542 ret = cxgbit_wait_rxq(csk); 1543 if (ret) { 1544 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1545 break; 1546 } 1547 } 1548 1549 return ret; 1550 } 1551 1552 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1553 { 1554 struct cxgbit_sock *csk = conn->context; 1555 1556 while (!kthread_should_stop()) { 1557 iscsit_thread_check_cpumask(conn, current, 0); 1558 if (cxgbit_wait_rxq(csk)) 1559 return; 1560 } 1561 } 1562