1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2016 Chelsio Communications, Inc. 4 */ 5 6 #include <linux/workqueue.h> 7 #include <linux/kthread.h> 8 #include <linux/sched/signal.h> 9 10 #include <asm/unaligned.h> 11 #include <net/tcp.h> 12 #include <target/target_core_base.h> 13 #include <target/target_core_fabric.h> 14 #include "cxgbit.h" 15 16 struct sge_opaque_hdr { 17 void *dev; 18 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 19 }; 20 21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 22 23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 24 sizeof(struct fw_ofld_tx_data_wr)) 25 26 static struct sk_buff * 27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 28 { 29 struct sk_buff *skb = NULL; 30 u8 submode = 0; 31 int errcode; 32 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 33 34 if (len) { 35 skb = alloc_skb_with_frags(hdr_len, len, 36 0, &errcode, 37 GFP_KERNEL); 38 if (!skb) 39 return NULL; 40 41 skb_reserve(skb, TX_HDR_LEN); 42 skb_reset_transport_header(skb); 43 __skb_put(skb, ISCSI_HDR_LEN); 44 skb->data_len = len; 45 skb->len += len; 46 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 47 48 } else { 49 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 50 51 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 52 if (!skb) 53 return NULL; 54 55 skb_reserve(skb, TX_HDR_LEN + iso_len); 56 skb_reset_transport_header(skb); 57 __skb_put(skb, ISCSI_HDR_LEN); 58 } 59 60 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 61 cxgbit_skcb_submode(skb) = submode; 62 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 63 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 64 return skb; 65 } 66 67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 68 { 69 return __cxgbit_alloc_skb(csk, len, false); 70 } 71 72 /* 73 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 74 * @skb: the packet 75 * 76 * Returns true if a packet can be sent as an offload WR with immediate 77 * data. We currently use the same limit as for Ethernet packets. 78 */ 79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 80 { 81 int length = skb->len; 82 83 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 84 length += sizeof(struct fw_ofld_tx_data_wr); 85 86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 87 length += sizeof(struct cpl_tx_data_iso); 88 89 return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN; 90 } 91 92 /* 93 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 94 * @n: the number of SGL entries 95 * Calculates the number of flits needed for a scatter/gather list that 96 * can hold the given number of entries. 97 */ 98 static inline unsigned int cxgbit_sgl_len(unsigned int n) 99 { 100 n--; 101 return (3 * n) / 2 + (n & 1) + 2; 102 } 103 104 /* 105 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 106 * @skb: the packet 107 * 108 * Returns the number of flits needed for the given offload packet. 109 * These packets are already fully constructed and no additional headers 110 * will be added. 111 */ 112 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 113 { 114 unsigned int flits, cnt; 115 116 if (cxgbit_is_ofld_imm(skb)) 117 return DIV_ROUND_UP(skb->len, 8); 118 flits = skb_transport_offset(skb) / 8; 119 cnt = skb_shinfo(skb)->nr_frags; 120 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 121 cnt++; 122 return flits + cxgbit_sgl_len(cnt); 123 } 124 125 #define CXGBIT_ISO_FSLICE 0x1 126 #define CXGBIT_ISO_LSLICE 0x2 127 static void 128 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 129 { 130 struct cpl_tx_data_iso *cpl; 131 unsigned int submode = cxgbit_skcb_submode(skb); 132 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 133 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 134 135 cpl = __skb_push(skb, sizeof(*cpl)); 136 137 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 138 CPL_TX_DATA_ISO_FIRST_V(fslice) | 139 CPL_TX_DATA_ISO_LAST_V(lslice) | 140 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 141 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 142 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 143 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 144 CPL_TX_DATA_ISO_SCSI_V(2)); 145 146 cpl->ahs_len = 0; 147 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 148 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 149 cpl->len = htonl(iso_info->len); 150 cpl->reserved2_seglen_offset = htonl(0); 151 cpl->datasn_offset = htonl(0); 152 cpl->buffer_offset = htonl(0); 153 cpl->reserved3 = 0; 154 155 __skb_pull(skb, sizeof(*cpl)); 156 } 157 158 static void 159 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 160 u32 len, u32 credits, u32 compl) 161 { 162 struct fw_ofld_tx_data_wr *req; 163 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 164 u32 submode = cxgbit_skcb_submode(skb); 165 u32 wr_ulp_mode = 0; 166 u32 hdr_size = sizeof(*req); 167 u32 opcode = FW_OFLD_TX_DATA_WR; 168 u32 immlen = 0; 169 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : 170 T6_TX_FORCE_F; 171 172 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 173 opcode = FW_ISCSI_TX_DATA_WR; 174 immlen += sizeof(struct cpl_tx_data_iso); 175 hdr_size += sizeof(struct cpl_tx_data_iso); 176 submode |= 8; 177 } 178 179 if (cxgbit_is_ofld_imm(skb)) 180 immlen += dlen; 181 182 req = __skb_push(skb, hdr_size); 183 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 184 FW_WR_COMPL_V(compl) | 185 FW_WR_IMMDLEN_V(immlen)); 186 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 187 FW_WR_LEN16_V(credits)); 188 req->plen = htonl(len); 189 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 190 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 191 192 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 193 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 194 } 195 196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 197 { 198 kfree_skb(skb); 199 } 200 201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 202 { 203 struct sk_buff *skb; 204 205 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 206 u32 dlen = skb->len; 207 u32 len = skb->len; 208 u32 credits_needed; 209 u32 compl = 0; 210 u32 flowclen16 = 0; 211 u32 iso_cpl_len = 0; 212 213 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 214 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 215 216 if (cxgbit_is_ofld_imm(skb)) 217 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 218 else 219 credits_needed = DIV_ROUND_UP((8 * 220 cxgbit_calc_tx_flits_ofld(skb)) + 221 iso_cpl_len, 16); 222 223 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 224 credits_needed += DIV_ROUND_UP( 225 sizeof(struct fw_ofld_tx_data_wr), 16); 226 /* 227 * Assumes the initial credits is large enough to support 228 * fw_flowc_wr plus largest possible first payload 229 */ 230 231 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 232 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 233 csk->wr_cred -= flowclen16; 234 csk->wr_una_cred += flowclen16; 235 } 236 237 if (csk->wr_cred < credits_needed) { 238 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 239 csk, skb->len, skb->data_len, 240 credits_needed, csk->wr_cred); 241 break; 242 } 243 __skb_unlink(skb, &csk->txq); 244 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 245 skb->csum = (__force __wsum)(credits_needed + flowclen16); 246 csk->wr_cred -= credits_needed; 247 csk->wr_una_cred += credits_needed; 248 249 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 250 csk, skb->len, skb->data_len, credits_needed, 251 csk->wr_cred, csk->wr_una_cred); 252 253 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 254 len += cxgbit_skcb_tx_extralen(skb); 255 256 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 257 (!before(csk->write_seq, 258 csk->snd_una + csk->snd_win))) { 259 compl = 1; 260 csk->wr_una_cred = 0; 261 } 262 263 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 264 compl); 265 csk->snd_nxt += len; 266 267 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 268 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 269 struct cpl_close_con_req *req = 270 (struct cpl_close_con_req *)skb->data; 271 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 272 csk->wr_una_cred = 0; 273 } 274 275 cxgbit_sock_enqueue_wr(csk, skb); 276 t4_set_arp_err_handler(skb, csk, 277 cxgbit_arp_failure_skb_discard); 278 279 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 280 csk, csk->tid, skb, len); 281 282 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 283 } 284 } 285 286 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 287 { 288 struct sk_buff_head backlogq; 289 struct sk_buff *skb; 290 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 291 292 skb_queue_head_init(&backlogq); 293 294 spin_lock_bh(&csk->lock); 295 while (skb_queue_len(&csk->backlogq)) { 296 skb_queue_splice_init(&csk->backlogq, &backlogq); 297 spin_unlock_bh(&csk->lock); 298 299 while ((skb = __skb_dequeue(&backlogq))) { 300 fn = cxgbit_skcb_rx_backlog_fn(skb); 301 fn(csk, skb); 302 } 303 304 spin_lock_bh(&csk->lock); 305 } 306 307 csk->lock_owner = false; 308 spin_unlock_bh(&csk->lock); 309 } 310 311 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 312 { 313 int ret = 0; 314 315 spin_lock_bh(&csk->lock); 316 csk->lock_owner = true; 317 spin_unlock_bh(&csk->lock); 318 319 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 320 signal_pending(current))) { 321 __kfree_skb(skb); 322 __skb_queue_purge(&csk->ppodq); 323 ret = -1; 324 goto unlock; 325 } 326 327 csk->write_seq += skb->len + 328 cxgbit_skcb_tx_extralen(skb); 329 330 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 331 __skb_queue_tail(&csk->txq, skb); 332 cxgbit_push_tx_frames(csk); 333 334 unlock: 335 cxgbit_unlock_sock(csk); 336 return ret; 337 } 338 339 static int 340 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 341 u32 data_length) 342 { 343 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 344 u32 padding = ((-data_length) & 3); 345 struct scatterlist *sg; 346 struct page *page; 347 unsigned int page_off; 348 349 if (padding) 350 nr_frags--; 351 352 /* 353 * We know each entry in t_data_sg contains a page. 354 */ 355 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 356 page_off = (data_offset % PAGE_SIZE); 357 358 while (data_length && (i < nr_frags)) { 359 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 360 361 page = sg_page(sg); 362 363 get_page(page); 364 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 365 cur_len); 366 skb->data_len += cur_len; 367 skb->len += cur_len; 368 skb->truesize += cur_len; 369 370 data_length -= cur_len; 371 page_off = 0; 372 sg = sg_next(sg); 373 i++; 374 } 375 376 if (data_length) 377 return -1; 378 379 if (padding) { 380 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 381 if (!page) 382 return -1; 383 skb_fill_page_desc(skb, i, page, 0, padding); 384 skb->data_len += padding; 385 skb->len += padding; 386 skb->truesize += padding; 387 } 388 389 return 0; 390 } 391 392 static int 393 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 394 struct iscsi_datain_req *dr) 395 { 396 struct iscsi_conn *conn = csk->conn; 397 struct sk_buff *skb; 398 struct iscsi_datain datain; 399 struct cxgbit_iso_info iso_info; 400 u32 data_length = cmd->se_cmd.data_length; 401 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 402 u32 num_pdu, plen, tx_data = 0; 403 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 404 SCF_TRANSPORT_TASK_SENSE); 405 bool set_statsn = false; 406 int ret = -1; 407 408 while (data_length) { 409 num_pdu = (data_length + mrdsl - 1) / mrdsl; 410 if (num_pdu > csk->max_iso_npdu) 411 num_pdu = csk->max_iso_npdu; 412 413 plen = num_pdu * mrdsl; 414 if (plen > data_length) 415 plen = data_length; 416 417 skb = __cxgbit_alloc_skb(csk, 0, true); 418 if (unlikely(!skb)) 419 return -ENOMEM; 420 421 memset(skb->data, 0, ISCSI_HDR_LEN); 422 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 423 cxgbit_skcb_submode(skb) |= (csk->submode & 424 CXGBIT_SUBMODE_DCRC); 425 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 426 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 427 ((num_pdu - 1) * ISCSI_HDR_LEN); 428 429 memset(&datain, 0, sizeof(struct iscsi_datain)); 430 memset(&iso_info, 0, sizeof(iso_info)); 431 432 if (!tx_data) 433 iso_info.flags |= CXGBIT_ISO_FSLICE; 434 435 if (!(data_length - plen)) { 436 iso_info.flags |= CXGBIT_ISO_LSLICE; 437 if (!task_sense) { 438 datain.flags = ISCSI_FLAG_DATA_STATUS; 439 iscsit_increment_maxcmdsn(cmd, conn->sess); 440 cmd->stat_sn = conn->stat_sn++; 441 set_statsn = true; 442 } 443 } 444 445 iso_info.burst_len = num_pdu * mrdsl; 446 iso_info.mpdu = mrdsl; 447 iso_info.len = ISCSI_HDR_LEN + plen; 448 449 cxgbit_cpl_tx_data_iso(skb, &iso_info); 450 451 datain.offset = tx_data; 452 datain.data_sn = cmd->data_sn - 1; 453 454 iscsit_build_datain_pdu(cmd, conn, &datain, 455 (struct iscsi_data_rsp *)skb->data, 456 set_statsn); 457 458 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 459 if (unlikely(ret)) { 460 __kfree_skb(skb); 461 goto out; 462 } 463 464 ret = cxgbit_queue_skb(csk, skb); 465 if (unlikely(ret)) 466 goto out; 467 468 tx_data += plen; 469 data_length -= plen; 470 471 cmd->read_data_done += plen; 472 cmd->data_sn += num_pdu; 473 } 474 475 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 476 477 return 0; 478 479 out: 480 return ret; 481 } 482 483 static int 484 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 485 const struct iscsi_datain *datain) 486 { 487 struct sk_buff *skb; 488 int ret = 0; 489 490 skb = cxgbit_alloc_skb(csk, 0); 491 if (unlikely(!skb)) 492 return -ENOMEM; 493 494 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 495 496 if (datain->length) { 497 cxgbit_skcb_submode(skb) |= (csk->submode & 498 CXGBIT_SUBMODE_DCRC); 499 cxgbit_skcb_tx_extralen(skb) = 500 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 501 } 502 503 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 504 if (ret < 0) { 505 __kfree_skb(skb); 506 return ret; 507 } 508 509 return cxgbit_queue_skb(csk, skb); 510 } 511 512 static int 513 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 514 struct iscsi_datain_req *dr, 515 const struct iscsi_datain *datain) 516 { 517 struct cxgbit_sock *csk = conn->context; 518 u32 data_length = cmd->se_cmd.data_length; 519 u32 padding = ((-data_length) & 3); 520 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 521 522 if ((data_length > mrdsl) && (!dr->recovery) && 523 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 524 atomic_long_add(data_length - datain->length, 525 &conn->sess->tx_data_octets); 526 return cxgbit_tx_datain_iso(csk, cmd, dr); 527 } 528 529 return cxgbit_tx_datain(csk, cmd, datain); 530 } 531 532 static int 533 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 534 const void *data_buf, u32 data_buf_len) 535 { 536 struct cxgbit_sock *csk = conn->context; 537 struct sk_buff *skb; 538 u32 padding = ((-data_buf_len) & 3); 539 540 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 541 if (unlikely(!skb)) 542 return -ENOMEM; 543 544 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 545 546 if (data_buf_len) { 547 u32 pad_bytes = 0; 548 549 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 550 551 if (padding) 552 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 553 &pad_bytes, padding); 554 } 555 556 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 557 cxgbit_skcb_submode(skb)]; 558 559 return cxgbit_queue_skb(csk, skb); 560 } 561 562 int 563 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 564 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 565 { 566 if (dr) 567 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 568 else 569 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 570 } 571 572 int cxgbit_validate_params(struct iscsi_conn *conn) 573 { 574 struct cxgbit_sock *csk = conn->context; 575 struct cxgbit_device *cdev = csk->com.cdev; 576 struct iscsi_param *param; 577 u32 max_xmitdsl; 578 579 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 580 conn->param_list); 581 if (!param) 582 return -1; 583 584 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 585 return -1; 586 587 if (max_xmitdsl > cdev->mdsl) { 588 if (iscsi_change_param_sprintf( 589 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 590 return -1; 591 } 592 593 return 0; 594 } 595 596 static int cxgbit_set_digest(struct cxgbit_sock *csk) 597 { 598 struct iscsi_conn *conn = csk->conn; 599 struct iscsi_param *param; 600 601 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 602 if (!param) { 603 pr_err("param not found key %s\n", HEADERDIGEST); 604 return -1; 605 } 606 607 if (!strcmp(param->value, CRC32C)) 608 csk->submode |= CXGBIT_SUBMODE_HCRC; 609 610 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 611 if (!param) { 612 csk->submode = 0; 613 pr_err("param not found key %s\n", DATADIGEST); 614 return -1; 615 } 616 617 if (!strcmp(param->value, CRC32C)) 618 csk->submode |= CXGBIT_SUBMODE_DCRC; 619 620 if (cxgbit_setup_conn_digest(csk)) { 621 csk->submode = 0; 622 return -1; 623 } 624 625 return 0; 626 } 627 628 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 629 { 630 struct iscsi_conn *conn = csk->conn; 631 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 632 struct iscsi_param *param; 633 u32 mrdsl, mbl; 634 u32 max_npdu, max_iso_npdu; 635 u32 max_iso_payload; 636 637 if (conn->login->leading_connection) { 638 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 639 conn->param_list); 640 if (!param) { 641 pr_err("param not found key %s\n", MAXBURSTLENGTH); 642 return -1; 643 } 644 645 if (kstrtou32(param->value, 0, &mbl) < 0) 646 return -1; 647 } else { 648 mbl = conn->sess->sess_ops->MaxBurstLength; 649 } 650 651 mrdsl = conn_ops->MaxRecvDataSegmentLength; 652 max_npdu = mbl / mrdsl; 653 654 max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss); 655 656 max_iso_npdu = max_iso_payload / 657 (ISCSI_HDR_LEN + mrdsl + 658 cxgbit_digest_len[csk->submode]); 659 660 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 661 662 if (csk->max_iso_npdu <= 1) 663 csk->max_iso_npdu = 0; 664 665 return 0; 666 } 667 668 /* 669 * cxgbit_seq_pdu_inorder() 670 * @csk: pointer to cxgbit socket structure 671 * 672 * This function checks whether data sequence and data 673 * pdu are in order. 674 * 675 * Return: returns -1 on error, 0 if data sequence and 676 * data pdu are in order, 1 if data sequence or data pdu 677 * is not in order. 678 */ 679 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk) 680 { 681 struct iscsi_conn *conn = csk->conn; 682 struct iscsi_param *param; 683 684 if (conn->login->leading_connection) { 685 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 686 conn->param_list); 687 if (!param) { 688 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 689 return -1; 690 } 691 692 if (strcmp(param->value, YES)) 693 return 1; 694 695 param = iscsi_find_param_from_key(DATAPDUINORDER, 696 conn->param_list); 697 if (!param) { 698 pr_err("param not found key %s\n", DATAPDUINORDER); 699 return -1; 700 } 701 702 if (strcmp(param->value, YES)) 703 return 1; 704 705 } else { 706 if (!conn->sess->sess_ops->DataSequenceInOrder) 707 return 1; 708 if (!conn->sess->sess_ops->DataPDUInOrder) 709 return 1; 710 } 711 712 return 0; 713 } 714 715 static int cxgbit_set_params(struct iscsi_conn *conn) 716 { 717 struct cxgbit_sock *csk = conn->context; 718 struct cxgbit_device *cdev = csk->com.cdev; 719 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 720 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 721 struct iscsi_param *param; 722 u8 erl; 723 724 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 725 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 726 727 if (cxgbit_set_digest(csk)) 728 return -1; 729 730 if (conn->login->leading_connection) { 731 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 732 conn->param_list); 733 if (!param) { 734 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 735 return -1; 736 } 737 if (kstrtou8(param->value, 0, &erl) < 0) 738 return -1; 739 } else { 740 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 741 } 742 743 if (!erl) { 744 int ret; 745 746 ret = cxgbit_seq_pdu_inorder(csk); 747 if (ret < 0) { 748 return -1; 749 } else if (ret > 0) { 750 if (is_t5(cdev->lldi.adapter_type)) 751 goto enable_ddp; 752 else 753 return 0; 754 } 755 756 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 757 if (cxgbit_set_iso_npdu(csk)) 758 return -1; 759 } 760 761 enable_ddp: 762 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 763 if (cxgbit_setup_conn_pgidx(csk, 764 ppm->tformat.pgsz_idx_dflt)) 765 return -1; 766 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 767 } 768 } 769 770 return 0; 771 } 772 773 int 774 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 775 u32 length) 776 { 777 struct cxgbit_sock *csk = conn->context; 778 struct sk_buff *skb; 779 u32 padding_buf = 0; 780 u8 padding = ((-length) & 3); 781 782 skb = cxgbit_alloc_skb(csk, length + padding); 783 if (!skb) 784 return -ENOMEM; 785 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 786 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 787 788 if (padding) 789 skb_store_bits(skb, ISCSI_HDR_LEN + length, 790 &padding_buf, padding); 791 792 if (login->login_complete) { 793 if (cxgbit_set_params(conn)) { 794 kfree_skb(skb); 795 return -1; 796 } 797 798 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 799 } 800 801 if (cxgbit_queue_skb(csk, skb)) 802 return -1; 803 804 if ((!login->login_complete) && (!login->login_failed)) 805 schedule_delayed_work(&conn->login_work, 0); 806 807 return 0; 808 } 809 810 static void 811 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 812 unsigned int nents, u32 skip) 813 { 814 struct skb_seq_state st; 815 const u8 *buf; 816 unsigned int consumed = 0, buf_len; 817 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 818 819 skb_prepare_seq_read(skb, pdu_cb->doffset, 820 pdu_cb->doffset + pdu_cb->dlen, 821 &st); 822 823 while (true) { 824 buf_len = skb_seq_read(consumed, &buf, &st); 825 if (!buf_len) { 826 skb_abort_seq_read(&st); 827 break; 828 } 829 830 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 831 buf_len, skip + consumed); 832 } 833 } 834 835 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 836 { 837 struct iscsi_conn *conn = csk->conn; 838 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 839 struct cxgbit_cmd *ccmd; 840 struct iscsi_cmd *cmd; 841 842 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 843 if (!cmd) { 844 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 845 return NULL; 846 } 847 848 ccmd = iscsit_priv_cmd(cmd); 849 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 850 ccmd->setup_ddp = true; 851 852 return cmd; 853 } 854 855 static int 856 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 857 u32 length) 858 { 859 struct iscsi_conn *conn = cmd->conn; 860 struct cxgbit_sock *csk = conn->context; 861 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 862 863 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 864 pr_err("ImmediateData CRC32C DataDigest error\n"); 865 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 866 pr_err("Unable to recover from" 867 " Immediate Data digest failure while" 868 " in ERL=0.\n"); 869 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 870 (unsigned char *)hdr); 871 return IMMEDIATE_DATA_CANNOT_RECOVER; 872 } 873 874 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 875 (unsigned char *)hdr); 876 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 877 } 878 879 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 880 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 881 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 882 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 883 884 sg_init_table(&ccmd->sg, 1); 885 sg_set_page(&ccmd->sg, skb_frag_page(dfrag), 886 skb_frag_size(dfrag), skb_frag_off(dfrag)); 887 get_page(skb_frag_page(dfrag)); 888 889 cmd->se_cmd.t_data_sg = &ccmd->sg; 890 cmd->se_cmd.t_data_nents = 1; 891 892 ccmd->release = true; 893 } else { 894 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 895 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 896 897 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0); 898 } 899 900 cmd->write_data_done += pdu_cb->dlen; 901 902 if (cmd->write_data_done == cmd->se_cmd.data_length) { 903 spin_lock_bh(&cmd->istate_lock); 904 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 905 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 906 spin_unlock_bh(&cmd->istate_lock); 907 } 908 909 return IMMEDIATE_DATA_NORMAL_OPERATION; 910 } 911 912 static int 913 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 914 bool dump_payload) 915 { 916 struct iscsi_conn *conn = cmd->conn; 917 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 918 /* 919 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 920 */ 921 if (dump_payload) 922 goto after_immediate_data; 923 924 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 925 cmd->first_burst_len); 926 after_immediate_data: 927 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 928 /* 929 * A PDU/CmdSN carrying Immediate Data passed 930 * DataCRC, check against ExpCmdSN/MaxCmdSN if 931 * Immediate Bit is not set. 932 */ 933 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 934 (unsigned char *)hdr, 935 hdr->cmdsn); 936 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 937 return -1; 938 939 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 940 target_put_sess_cmd(&cmd->se_cmd); 941 return 0; 942 } else if (cmd->unsolicited_data) { 943 iscsit_set_unsolicited_dataout(cmd); 944 } 945 946 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 947 /* 948 * Immediate Data failed DataCRC and ERL>=1, 949 * silently drop this PDU and let the initiator 950 * plug the CmdSN gap. 951 * 952 * FIXME: Send Unsolicited NOPIN with reserved 953 * TTT here to help the initiator figure out 954 * the missing CmdSN, although they should be 955 * intelligent enough to determine the missing 956 * CmdSN and issue a retry to plug the sequence. 957 */ 958 cmd->i_state = ISTATE_REMOVE; 959 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 960 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 961 return -1; 962 963 return 0; 964 } 965 966 static int 967 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 968 { 969 struct iscsi_conn *conn = csk->conn; 970 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 971 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 972 int rc; 973 bool dump_payload = false; 974 975 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 976 if (rc < 0) 977 return rc; 978 979 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 980 (pdu_cb->nr_dfrags == 1)) 981 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 982 983 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 984 if (rc < 0) 985 return 0; 986 else if (rc > 0) 987 dump_payload = true; 988 989 if (!pdu_cb->dlen) 990 return 0; 991 992 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 993 } 994 995 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 996 { 997 struct scatterlist *sg_start; 998 struct iscsi_conn *conn = csk->conn; 999 struct iscsi_cmd *cmd = NULL; 1000 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1001 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 1002 u32 data_offset = be32_to_cpu(hdr->offset); 1003 u32 data_len = pdu_cb->dlen; 1004 int rc, sg_nents, sg_off; 1005 bool dcrc_err = false; 1006 1007 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) { 1008 u32 offset = be32_to_cpu(hdr->offset); 1009 u32 ddp_data_len; 1010 u32 payload_length = ntoh24(hdr->dlength); 1011 bool success = false; 1012 1013 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0); 1014 if (!cmd) 1015 return 0; 1016 1017 ddp_data_len = offset - cmd->write_data_done; 1018 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets); 1019 1020 cmd->write_data_done = offset; 1021 cmd->next_burst_len = ddp_data_len; 1022 cmd->data_sn = be32_to_cpu(hdr->datasn); 1023 1024 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, 1025 cmd, payload_length, &success); 1026 if (rc < 0) 1027 return rc; 1028 else if (!success) 1029 return 0; 1030 } else { 1031 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 1032 if (rc < 0) 1033 return rc; 1034 else if (!cmd) 1035 return 0; 1036 } 1037 1038 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1039 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1040 " DataSN: 0x%08x\n", 1041 hdr->itt, hdr->offset, data_len, 1042 hdr->datasn); 1043 1044 dcrc_err = true; 1045 goto check_payload; 1046 } 1047 1048 pr_debug("DataOut data_len: %u, " 1049 "write_data_done: %u, data_length: %u\n", 1050 data_len, cmd->write_data_done, 1051 cmd->se_cmd.data_length); 1052 1053 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1054 u32 skip = data_offset % PAGE_SIZE; 1055 1056 sg_off = data_offset / PAGE_SIZE; 1057 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1058 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE)); 1059 1060 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip); 1061 } 1062 1063 check_payload: 1064 1065 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1066 if (rc < 0) 1067 return rc; 1068 1069 return 0; 1070 } 1071 1072 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1073 { 1074 struct iscsi_conn *conn = csk->conn; 1075 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1076 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1077 unsigned char *ping_data = NULL; 1078 u32 payload_length = pdu_cb->dlen; 1079 int ret; 1080 1081 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1082 if (ret < 0) 1083 return 0; 1084 1085 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1086 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1087 pr_err("Unable to recover from" 1088 " NOPOUT Ping DataCRC failure while in" 1089 " ERL=0.\n"); 1090 ret = -1; 1091 goto out; 1092 } else { 1093 /* 1094 * drop this PDU and let the 1095 * initiator plug the CmdSN gap. 1096 */ 1097 pr_info("Dropping NOPOUT" 1098 " Command CmdSN: 0x%08x due to" 1099 " DataCRC error.\n", hdr->cmdsn); 1100 ret = 0; 1101 goto out; 1102 } 1103 } 1104 1105 /* 1106 * Handle NOP-OUT payload for traditional iSCSI sockets 1107 */ 1108 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1109 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1110 if (!ping_data) { 1111 pr_err("Unable to allocate memory for" 1112 " NOPOUT ping data.\n"); 1113 ret = -1; 1114 goto out; 1115 } 1116 1117 skb_copy_bits(csk->skb, pdu_cb->doffset, 1118 ping_data, payload_length); 1119 1120 ping_data[payload_length] = '\0'; 1121 /* 1122 * Attach ping data to struct iscsi_cmd->buf_ptr. 1123 */ 1124 cmd->buf_ptr = ping_data; 1125 cmd->buf_ptr_size = payload_length; 1126 1127 pr_debug("Got %u bytes of NOPOUT ping" 1128 " data.\n", payload_length); 1129 pr_debug("Ping Data: \"%s\"\n", ping_data); 1130 } 1131 1132 return iscsit_process_nop_out(conn, cmd, hdr); 1133 out: 1134 if (cmd) 1135 iscsit_free_cmd(cmd, false); 1136 return ret; 1137 } 1138 1139 static int 1140 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1141 { 1142 struct iscsi_conn *conn = csk->conn; 1143 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1144 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1145 u32 payload_length = pdu_cb->dlen; 1146 int rc; 1147 unsigned char *text_in = NULL; 1148 1149 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1150 if (rc < 0) 1151 return rc; 1152 1153 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1154 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1155 pr_err("Unable to recover from" 1156 " Text Data digest failure while in" 1157 " ERL=0.\n"); 1158 goto reject; 1159 } else { 1160 /* 1161 * drop this PDU and let the 1162 * initiator plug the CmdSN gap. 1163 */ 1164 pr_info("Dropping Text" 1165 " Command CmdSN: 0x%08x due to" 1166 " DataCRC error.\n", hdr->cmdsn); 1167 return 0; 1168 } 1169 } 1170 1171 if (payload_length) { 1172 text_in = kzalloc(payload_length, GFP_KERNEL); 1173 if (!text_in) { 1174 pr_err("Unable to allocate text_in of payload_length: %u\n", 1175 payload_length); 1176 return -ENOMEM; 1177 } 1178 skb_copy_bits(csk->skb, pdu_cb->doffset, 1179 text_in, payload_length); 1180 1181 text_in[payload_length - 1] = '\0'; 1182 1183 cmd->text_in_ptr = text_in; 1184 } 1185 1186 return iscsit_process_text_cmd(conn, cmd, hdr); 1187 1188 reject: 1189 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1190 pdu_cb->hdr); 1191 } 1192 1193 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1194 { 1195 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1196 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1197 struct iscsi_conn *conn = csk->conn; 1198 struct iscsi_cmd *cmd = NULL; 1199 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1200 int ret = -EINVAL; 1201 1202 switch (opcode) { 1203 case ISCSI_OP_SCSI_CMD: 1204 cmd = cxgbit_allocate_cmd(csk); 1205 if (!cmd) 1206 goto reject; 1207 1208 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1209 break; 1210 case ISCSI_OP_SCSI_DATA_OUT: 1211 ret = cxgbit_handle_iscsi_dataout(csk); 1212 break; 1213 case ISCSI_OP_NOOP_OUT: 1214 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1215 cmd = cxgbit_allocate_cmd(csk); 1216 if (!cmd) 1217 goto reject; 1218 } 1219 1220 ret = cxgbit_handle_nop_out(csk, cmd); 1221 break; 1222 case ISCSI_OP_SCSI_TMFUNC: 1223 cmd = cxgbit_allocate_cmd(csk); 1224 if (!cmd) 1225 goto reject; 1226 1227 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1228 (unsigned char *)hdr); 1229 break; 1230 case ISCSI_OP_TEXT: 1231 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1232 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1233 if (!cmd) 1234 goto reject; 1235 } else { 1236 cmd = cxgbit_allocate_cmd(csk); 1237 if (!cmd) 1238 goto reject; 1239 } 1240 1241 ret = cxgbit_handle_text_cmd(csk, cmd); 1242 break; 1243 case ISCSI_OP_LOGOUT: 1244 cmd = cxgbit_allocate_cmd(csk); 1245 if (!cmd) 1246 goto reject; 1247 1248 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1249 if (ret > 0) 1250 wait_for_completion_timeout(&conn->conn_logout_comp, 1251 SECONDS_FOR_LOGOUT_COMP 1252 * HZ); 1253 break; 1254 case ISCSI_OP_SNACK: 1255 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1256 break; 1257 default: 1258 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1259 dump_stack(); 1260 break; 1261 } 1262 1263 return ret; 1264 1265 reject: 1266 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1267 (unsigned char *)hdr); 1268 return ret; 1269 } 1270 1271 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1272 { 1273 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1274 struct iscsi_conn *conn = csk->conn; 1275 struct iscsi_hdr *hdr = pdu_cb->hdr; 1276 u8 opcode; 1277 1278 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1279 atomic_long_inc(&conn->sess->conn_digest_errors); 1280 goto transport_err; 1281 } 1282 1283 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1284 goto transport_err; 1285 1286 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1287 1288 if (conn->sess->sess_ops->SessionType && 1289 ((!(opcode & ISCSI_OP_TEXT)) || 1290 (!(opcode & ISCSI_OP_LOGOUT)))) { 1291 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1292 " while in Discovery Session, rejecting.\n", opcode); 1293 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1294 (unsigned char *)hdr); 1295 goto transport_err; 1296 } 1297 1298 if (cxgbit_target_rx_opcode(csk) < 0) 1299 goto transport_err; 1300 1301 return 0; 1302 1303 transport_err: 1304 return -1; 1305 } 1306 1307 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1308 { 1309 struct iscsi_conn *conn = csk->conn; 1310 struct iscsi_login *login = conn->login; 1311 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1312 struct iscsi_login_req *login_req; 1313 1314 login_req = (struct iscsi_login_req *)login->req; 1315 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1316 1317 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1318 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1319 login_req->flags, login_req->itt, login_req->cmdsn, 1320 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1321 /* 1322 * Setup the initial iscsi_login values from the leading 1323 * login request PDU. 1324 */ 1325 if (login->first_request) { 1326 login_req = (struct iscsi_login_req *)login->req; 1327 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1328 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1329 login_req->flags); 1330 login->version_min = login_req->min_version; 1331 login->version_max = login_req->max_version; 1332 memcpy(login->isid, login_req->isid, 6); 1333 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1334 login->init_task_tag = login_req->itt; 1335 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1336 login->cid = be16_to_cpu(login_req->cid); 1337 login->tsih = be16_to_cpu(login_req->tsih); 1338 } 1339 1340 if (iscsi_target_check_login_request(conn, login) < 0) 1341 return -1; 1342 1343 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1344 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1345 1346 return 0; 1347 } 1348 1349 static int 1350 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1351 { 1352 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1353 int ret; 1354 1355 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1356 1357 csk->skb = skb; 1358 1359 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1360 ret = cxgbit_rx_login_pdu(csk); 1361 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1362 } else { 1363 ret = cxgbit_rx_opcode(csk); 1364 } 1365 1366 return ret; 1367 } 1368 1369 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1370 { 1371 struct skb_shared_info *ssi = skb_shinfo(skb); 1372 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1373 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1374 u8 i; 1375 1376 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1377 skb, skb->head, skb->data, skb->len, skb->data_len, 1378 ssi->nr_frags); 1379 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1380 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1381 1382 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1383 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1384 "frags %u.\n", 1385 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1386 pdu_cb->ddigest, pdu_cb->frags); 1387 for (i = 0; i < ssi->nr_frags; i++) 1388 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1389 skb, i, skb_frag_off(&ssi->frags[i]), 1390 skb_frag_size(&ssi->frags[i])); 1391 } 1392 1393 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1394 { 1395 struct sk_buff *skb = csk->lro_hskb; 1396 struct skb_shared_info *ssi = skb_shinfo(skb); 1397 u8 i; 1398 1399 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1400 for (i = 0; i < ssi->nr_frags; i++) 1401 put_page(skb_frag_page(&ssi->frags[i])); 1402 ssi->nr_frags = 0; 1403 skb->data_len = 0; 1404 skb->truesize -= skb->len; 1405 skb->len = 0; 1406 } 1407 1408 static void 1409 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1410 { 1411 struct sk_buff *hskb = csk->lro_hskb; 1412 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1413 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1414 struct skb_shared_info *hssi = skb_shinfo(hskb); 1415 struct skb_shared_info *ssi = skb_shinfo(skb); 1416 unsigned int len = 0; 1417 1418 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1419 u8 hfrag_idx = hssi->nr_frags; 1420 1421 hpdu_cb->flags |= pdu_cb->flags; 1422 hpdu_cb->seq = pdu_cb->seq; 1423 hpdu_cb->hdr = pdu_cb->hdr; 1424 hpdu_cb->hlen = pdu_cb->hlen; 1425 1426 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx], 1427 sizeof(skb_frag_t)); 1428 1429 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1430 hssi->nr_frags++; 1431 hpdu_cb->frags++; 1432 hpdu_cb->hfrag_idx = hfrag_idx; 1433 1434 len = skb_frag_size(&hssi->frags[hfrag_idx]); 1435 hskb->len += len; 1436 hskb->data_len += len; 1437 hskb->truesize += len; 1438 } 1439 1440 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1441 u8 dfrag_idx = hssi->nr_frags, i; 1442 1443 hpdu_cb->flags |= pdu_cb->flags; 1444 hpdu_cb->dfrag_idx = dfrag_idx; 1445 1446 len = 0; 1447 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) { 1448 memcpy(&hssi->frags[dfrag_idx], 1449 &ssi->frags[pdu_cb->dfrag_idx + i], 1450 sizeof(skb_frag_t)); 1451 1452 get_page(skb_frag_page(&hssi->frags[dfrag_idx])); 1453 1454 len += skb_frag_size(&hssi->frags[dfrag_idx]); 1455 1456 hssi->nr_frags++; 1457 hpdu_cb->frags++; 1458 } 1459 1460 hpdu_cb->dlen = pdu_cb->dlen; 1461 hpdu_cb->doffset = hpdu_cb->hlen; 1462 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1463 hskb->len += len; 1464 hskb->data_len += len; 1465 hskb->truesize += len; 1466 } 1467 1468 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1469 hpdu_cb->flags |= pdu_cb->flags; 1470 1471 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1472 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1473 1474 hpdu_cb->ddigest = pdu_cb->ddigest; 1475 hpdu_cb->pdulen = pdu_cb->pdulen; 1476 } 1477 } 1478 1479 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1480 { 1481 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1482 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1483 u8 pdu_idx = 0, last_idx = 0; 1484 int ret = 0; 1485 1486 if (!pdu_cb->complete) { 1487 cxgbit_lro_skb_merge(csk, skb, 0); 1488 1489 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1490 struct sk_buff *hskb = csk->lro_hskb; 1491 1492 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1493 1494 cxgbit_lro_hskb_reset(csk); 1495 1496 if (ret < 0) 1497 goto out; 1498 } 1499 1500 pdu_idx = 1; 1501 } 1502 1503 if (lro_cb->pdu_idx) 1504 last_idx = lro_cb->pdu_idx - 1; 1505 1506 for (; pdu_idx <= last_idx; pdu_idx++) { 1507 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1508 if (ret < 0) 1509 goto out; 1510 } 1511 1512 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1513 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1514 1515 out: 1516 return ret; 1517 } 1518 1519 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1520 { 1521 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1522 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1523 int ret = -1; 1524 1525 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1526 (pdu_cb->seq != csk->rcv_nxt)) { 1527 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1528 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1529 cxgbit_lro_skb_dump(skb); 1530 return ret; 1531 } 1532 1533 csk->rcv_nxt += lro_cb->pdu_totallen; 1534 1535 ret = cxgbit_process_lro_skb(csk, skb); 1536 1537 csk->rx_credits += lro_cb->pdu_totallen; 1538 1539 if (csk->rx_credits >= (csk->rcv_win / 4)) 1540 cxgbit_rx_data_ack(csk); 1541 1542 return ret; 1543 } 1544 1545 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1546 { 1547 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 1548 int ret = -1; 1549 1550 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) { 1551 if (is_t5(lldi->adapter_type)) 1552 ret = cxgbit_rx_lro_skb(csk, skb); 1553 else 1554 ret = cxgbit_process_lro_skb(csk, skb); 1555 } 1556 1557 __kfree_skb(skb); 1558 return ret; 1559 } 1560 1561 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1562 { 1563 spin_lock_bh(&csk->rxq.lock); 1564 if (skb_queue_len(&csk->rxq)) { 1565 skb_queue_splice_init(&csk->rxq, rxq); 1566 spin_unlock_bh(&csk->rxq.lock); 1567 return true; 1568 } 1569 spin_unlock_bh(&csk->rxq.lock); 1570 return false; 1571 } 1572 1573 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1574 { 1575 struct sk_buff *skb; 1576 struct sk_buff_head rxq; 1577 1578 skb_queue_head_init(&rxq); 1579 1580 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1581 1582 if (signal_pending(current)) 1583 goto out; 1584 1585 while ((skb = __skb_dequeue(&rxq))) { 1586 if (cxgbit_rx_skb(csk, skb)) 1587 goto out; 1588 } 1589 1590 return 0; 1591 out: 1592 __skb_queue_purge(&rxq); 1593 return -1; 1594 } 1595 1596 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1597 { 1598 struct cxgbit_sock *csk = conn->context; 1599 int ret = -1; 1600 1601 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1602 ret = cxgbit_wait_rxq(csk); 1603 if (ret) { 1604 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1605 break; 1606 } 1607 } 1608 1609 return ret; 1610 } 1611 1612 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1613 { 1614 struct cxgbit_sock *csk = conn->context; 1615 1616 while (!kthread_should_stop()) { 1617 iscsit_thread_check_cpumask(conn, current, 0); 1618 if (cxgbit_wait_rxq(csk)) 1619 return; 1620 } 1621 } 1622