1 /* 2 * iSCSI Initiator over TCP/IP Data-Path 3 * 4 * Copyright (C) 2004 Dmitry Yusupov 5 * Copyright (C) 2004 Alex Aizman 6 * Copyright (C) 2005 - 2006 Mike Christie 7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 8 * maintained by open-iscsi@googlegroups.com 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published 12 * by the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * See the file COPYING included with this distribution for more details. 21 * 22 * Credits: 23 * Christoph Hellwig 24 * FUJITA Tomonori 25 * Arne Redlich 26 * Zhenyu Wang 27 */ 28 29 #include <linux/types.h> 30 #include <linux/list.h> 31 #include <linux/inet.h> 32 #include <linux/blkdev.h> 33 #include <linux/crypto.h> 34 #include <linux/delay.h> 35 #include <linux/kfifo.h> 36 #include <linux/scatterlist.h> 37 #include <linux/mutex.h> 38 #include <net/tcp.h> 39 #include <scsi/scsi_cmnd.h> 40 #include <scsi/scsi_host.h> 41 #include <scsi/scsi.h> 42 #include <scsi/scsi_transport_iscsi.h> 43 44 #include "iscsi_tcp.h" 45 46 #define ISCSI_TCP_VERSION "1.0-595" 47 48 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 49 "Alex Aizman <itn780@yahoo.com>"); 50 MODULE_DESCRIPTION("iSCSI/TCP data-path"); 51 MODULE_LICENSE("GPL"); 52 MODULE_VERSION(ISCSI_TCP_VERSION); 53 /* #define DEBUG_TCP */ 54 #define DEBUG_ASSERT 55 56 #ifdef DEBUG_TCP 57 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt) 58 #else 59 #define debug_tcp(fmt...) 60 #endif 61 62 #ifndef DEBUG_ASSERT 63 #ifdef BUG_ON 64 #undef BUG_ON 65 #endif 66 #define BUG_ON(expr) 67 #endif 68 69 static unsigned int iscsi_max_lun = 512; 70 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 71 72 static inline void 73 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size) 74 { 75 ibuf->sg.page = virt_to_page(vbuf); 76 ibuf->sg.offset = offset_in_page(vbuf); 77 ibuf->sg.length = size; 78 ibuf->sent = 0; 79 ibuf->use_sendmsg = 1; 80 } 81 82 static inline void 83 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg) 84 { 85 ibuf->sg.page = sg->page; 86 ibuf->sg.offset = sg->offset; 87 ibuf->sg.length = sg->length; 88 /* 89 * Fastpath: sg element fits into single page 90 */ 91 if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page)) 92 ibuf->use_sendmsg = 0; 93 else 94 ibuf->use_sendmsg = 1; 95 ibuf->sent = 0; 96 } 97 98 static inline int 99 iscsi_buf_left(struct iscsi_buf *ibuf) 100 { 101 int rc; 102 103 rc = ibuf->sg.length - ibuf->sent; 104 BUG_ON(rc < 0); 105 return rc; 106 } 107 108 static inline void 109 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf, 110 u8* crc) 111 { 112 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 113 114 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc); 115 buf->sg.length += sizeof(uint32_t); 116 } 117 118 static inline int 119 iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn) 120 { 121 struct sk_buff *skb = tcp_conn->in.skb; 122 123 tcp_conn->in.zero_copy_hdr = 0; 124 125 if (tcp_conn->in.copy >= tcp_conn->hdr_size && 126 tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) { 127 /* 128 * Zero-copy PDU Header: using connection context 129 * to store header pointer. 130 */ 131 if (skb_shinfo(skb)->frag_list == NULL && 132 !skb_shinfo(skb)->nr_frags) { 133 tcp_conn->in.hdr = (struct iscsi_hdr *) 134 ((char*)skb->data + tcp_conn->in.offset); 135 tcp_conn->in.zero_copy_hdr = 1; 136 } else { 137 /* ignoring return code since we checked 138 * in.copy before */ 139 skb_copy_bits(skb, tcp_conn->in.offset, 140 &tcp_conn->hdr, tcp_conn->hdr_size); 141 tcp_conn->in.hdr = &tcp_conn->hdr; 142 } 143 tcp_conn->in.offset += tcp_conn->hdr_size; 144 tcp_conn->in.copy -= tcp_conn->hdr_size; 145 } else { 146 int hdr_remains; 147 int copylen; 148 149 /* 150 * PDU header scattered across SKB's, 151 * copying it... This'll happen quite rarely. 152 */ 153 154 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) 155 tcp_conn->in.hdr_offset = 0; 156 157 hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset; 158 BUG_ON(hdr_remains <= 0); 159 160 copylen = min(tcp_conn->in.copy, hdr_remains); 161 skb_copy_bits(skb, tcp_conn->in.offset, 162 (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset, 163 copylen); 164 165 debug_tcp("PDU gather offset %d bytes %d in.offset %d " 166 "in.copy %d\n", tcp_conn->in.hdr_offset, copylen, 167 tcp_conn->in.offset, tcp_conn->in.copy); 168 169 tcp_conn->in.offset += copylen; 170 tcp_conn->in.copy -= copylen; 171 if (copylen < hdr_remains) { 172 tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER; 173 tcp_conn->in.hdr_offset += copylen; 174 return -EAGAIN; 175 } 176 tcp_conn->in.hdr = &tcp_conn->hdr; 177 tcp_conn->discontiguous_hdr_cnt++; 178 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 179 } 180 181 return 0; 182 } 183 184 /* 185 * must be called with session lock 186 */ 187 static void 188 __iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 189 { 190 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 191 struct scsi_cmnd *sc; 192 193 sc = ctask->sc; 194 if (unlikely(!sc)) 195 return; 196 197 tcp_ctask->xmstate = XMSTATE_IDLE; 198 tcp_ctask->r2t = NULL; 199 } 200 201 /** 202 * iscsi_data_rsp - SCSI Data-In Response processing 203 * @conn: iscsi connection 204 * @ctask: scsi command task 205 **/ 206 static int 207 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 208 { 209 int rc; 210 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 211 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 212 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr; 213 struct iscsi_session *session = conn->session; 214 int datasn = be32_to_cpu(rhdr->datasn); 215 216 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 217 if (rc) 218 return rc; 219 /* 220 * setup Data-In byte counter (gets decremented..) 221 */ 222 ctask->data_count = tcp_conn->in.datalen; 223 224 if (tcp_conn->in.datalen == 0) 225 return 0; 226 227 if (ctask->datasn != datasn) 228 return ISCSI_ERR_DATASN; 229 230 ctask->datasn++; 231 232 tcp_ctask->data_offset = be32_to_cpu(rhdr->offset); 233 if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length) 234 return ISCSI_ERR_DATA_OFFSET; 235 236 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) { 237 struct scsi_cmnd *sc = ctask->sc; 238 239 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 240 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) { 241 int res_count = be32_to_cpu(rhdr->residual_count); 242 243 if (res_count > 0 && 244 res_count <= sc->request_bufflen) { 245 sc->resid = res_count; 246 sc->result = (DID_OK << 16) | rhdr->cmd_status; 247 } else 248 sc->result = (DID_BAD_TARGET << 16) | 249 rhdr->cmd_status; 250 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) { 251 sc->resid = be32_to_cpu(rhdr->residual_count); 252 sc->result = (DID_OK << 16) | rhdr->cmd_status; 253 } else 254 sc->result = (DID_OK << 16) | rhdr->cmd_status; 255 } 256 257 conn->datain_pdus_cnt++; 258 return 0; 259 } 260 261 /** 262 * iscsi_solicit_data_init - initialize first Data-Out 263 * @conn: iscsi connection 264 * @ctask: scsi command task 265 * @r2t: R2T info 266 * 267 * Notes: 268 * Initialize first Data-Out within this R2T sequence and finds 269 * proper data_offset within this SCSI command. 270 * 271 * This function is called with connection lock taken. 272 **/ 273 static void 274 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 275 struct iscsi_r2t_info *r2t) 276 { 277 struct iscsi_data *hdr; 278 struct scsi_cmnd *sc = ctask->sc; 279 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 280 281 hdr = &r2t->dtask.hdr; 282 memset(hdr, 0, sizeof(struct iscsi_data)); 283 hdr->ttt = r2t->ttt; 284 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 285 r2t->solicit_datasn++; 286 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 287 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun)); 288 hdr->itt = ctask->hdr->itt; 289 hdr->exp_statsn = r2t->exp_statsn; 290 hdr->offset = cpu_to_be32(r2t->data_offset); 291 if (r2t->data_length > conn->max_xmit_dlength) { 292 hton24(hdr->dlength, conn->max_xmit_dlength); 293 r2t->data_count = conn->max_xmit_dlength; 294 hdr->flags = 0; 295 } else { 296 hton24(hdr->dlength, r2t->data_length); 297 r2t->data_count = r2t->data_length; 298 hdr->flags = ISCSI_FLAG_CMD_FINAL; 299 } 300 conn->dataout_pdus_cnt++; 301 302 r2t->sent = 0; 303 304 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr, 305 sizeof(struct iscsi_hdr)); 306 307 if (sc->use_sg) { 308 int i, sg_count = 0; 309 struct scatterlist *sg = sc->request_buffer; 310 311 r2t->sg = NULL; 312 for (i = 0; i < sc->use_sg; i++, sg += 1) { 313 /* FIXME: prefetch ? */ 314 if (sg_count + sg->length > r2t->data_offset) { 315 int page_offset; 316 317 /* sg page found! */ 318 319 /* offset within this page */ 320 page_offset = r2t->data_offset - sg_count; 321 322 /* fill in this buffer */ 323 iscsi_buf_init_sg(&r2t->sendbuf, sg); 324 r2t->sendbuf.sg.offset += page_offset; 325 r2t->sendbuf.sg.length -= page_offset; 326 327 /* xmit logic will continue with next one */ 328 r2t->sg = sg + 1; 329 break; 330 } 331 sg_count += sg->length; 332 } 333 BUG_ON(r2t->sg == NULL); 334 } else 335 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 336 (char*)sc->request_buffer + r2t->data_offset, 337 r2t->data_count); 338 } 339 340 /** 341 * iscsi_r2t_rsp - iSCSI R2T Response processing 342 * @conn: iscsi connection 343 * @ctask: scsi command task 344 **/ 345 static int 346 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 347 { 348 struct iscsi_r2t_info *r2t; 349 struct iscsi_session *session = conn->session; 350 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 351 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 352 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; 353 int r2tsn = be32_to_cpu(rhdr->r2tsn); 354 int rc; 355 356 if (tcp_conn->in.datalen) 357 return ISCSI_ERR_DATALEN; 358 359 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn) 360 return ISCSI_ERR_R2TSN; 361 362 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 363 if (rc) 364 return rc; 365 366 /* FIXME: use R2TSN to detect missing R2T */ 367 368 /* fill-in new R2T associated with the task */ 369 spin_lock(&session->lock); 370 if (!ctask->sc || ctask->mtask || 371 session->state != ISCSI_STATE_LOGGED_IN) { 372 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in " 373 "recovery...\n", ctask->itt); 374 spin_unlock(&session->lock); 375 return 0; 376 } 377 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 378 BUG_ON(!rc); 379 380 r2t->exp_statsn = rhdr->statsn; 381 r2t->data_length = be32_to_cpu(rhdr->data_length); 382 if (r2t->data_length == 0 || 383 r2t->data_length > session->max_burst) { 384 spin_unlock(&session->lock); 385 return ISCSI_ERR_DATALEN; 386 } 387 388 r2t->data_offset = be32_to_cpu(rhdr->data_offset); 389 if (r2t->data_offset + r2t->data_length > ctask->total_length) { 390 spin_unlock(&session->lock); 391 return ISCSI_ERR_DATALEN; 392 } 393 394 r2t->ttt = rhdr->ttt; /* no flip */ 395 r2t->solicit_datasn = 0; 396 397 iscsi_solicit_data_init(conn, ctask, r2t); 398 399 tcp_ctask->exp_r2tsn = r2tsn + 1; 400 tcp_ctask->xmstate |= XMSTATE_SOL_HDR; 401 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 402 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*)); 403 404 scsi_queue_work(session->host, &conn->xmitwork); 405 conn->r2t_pdus_cnt++; 406 spin_unlock(&session->lock); 407 408 return 0; 409 } 410 411 static int 412 iscsi_tcp_hdr_recv(struct iscsi_conn *conn) 413 { 414 int rc = 0, opcode, ahslen; 415 struct iscsi_hdr *hdr; 416 struct iscsi_session *session = conn->session; 417 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 418 uint32_t cdgst, rdgst = 0, itt; 419 420 hdr = tcp_conn->in.hdr; 421 422 /* verify PDU length */ 423 tcp_conn->in.datalen = ntoh24(hdr->dlength); 424 if (tcp_conn->in.datalen > conn->max_recv_dlength) { 425 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n", 426 tcp_conn->in.datalen, conn->max_recv_dlength); 427 return ISCSI_ERR_DATALEN; 428 } 429 tcp_conn->data_copied = 0; 430 431 /* read AHS */ 432 ahslen = hdr->hlength << 2; 433 tcp_conn->in.offset += ahslen; 434 tcp_conn->in.copy -= ahslen; 435 if (tcp_conn->in.copy < 0) { 436 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length " 437 "%d bytes\n", ahslen); 438 return ISCSI_ERR_AHSLEN; 439 } 440 441 /* calculate read padding */ 442 tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1); 443 if (tcp_conn->in.padding) { 444 tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding; 445 debug_scsi("read padding %d bytes\n", tcp_conn->in.padding); 446 } 447 448 if (conn->hdrdgst_en) { 449 struct scatterlist sg; 450 451 sg_init_one(&sg, (u8 *)hdr, 452 sizeof(struct iscsi_hdr) + ahslen); 453 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 454 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 455 ahslen); 456 if (cdgst != rdgst) { 457 printk(KERN_ERR "iscsi_tcp: hdrdgst error " 458 "recv 0x%x calc 0x%x\n", rdgst, cdgst); 459 return ISCSI_ERR_HDR_DGST; 460 } 461 } 462 463 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 464 /* verify itt (itt encoding: age+cid+itt) */ 465 rc = iscsi_verify_itt(conn, hdr, &itt); 466 if (rc == ISCSI_ERR_NO_SCSI_CMD) { 467 tcp_conn->in.datalen = 0; /* force drop */ 468 return 0; 469 } else if (rc) 470 return rc; 471 472 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n", 473 opcode, tcp_conn->in.offset, tcp_conn->in.copy, 474 ahslen, tcp_conn->in.datalen); 475 476 switch(opcode) { 477 case ISCSI_OP_SCSI_DATA_IN: 478 tcp_conn->in.ctask = session->cmds[itt]; 479 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask); 480 /* fall through */ 481 case ISCSI_OP_SCSI_CMD_RSP: 482 tcp_conn->in.ctask = session->cmds[itt]; 483 if (tcp_conn->in.datalen) 484 goto copy_hdr; 485 486 spin_lock(&session->lock); 487 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask); 488 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0); 489 spin_unlock(&session->lock); 490 break; 491 case ISCSI_OP_R2T: 492 tcp_conn->in.ctask = session->cmds[itt]; 493 if (ahslen) 494 rc = ISCSI_ERR_AHSLEN; 495 else if (tcp_conn->in.ctask->sc->sc_data_direction == 496 DMA_TO_DEVICE) 497 rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask); 498 else 499 rc = ISCSI_ERR_PROTO; 500 break; 501 case ISCSI_OP_LOGIN_RSP: 502 case ISCSI_OP_TEXT_RSP: 503 case ISCSI_OP_LOGOUT_RSP: 504 case ISCSI_OP_NOOP_IN: 505 case ISCSI_OP_REJECT: 506 case ISCSI_OP_ASYNC_EVENT: 507 if (tcp_conn->in.datalen) 508 goto copy_hdr; 509 /* fall through */ 510 case ISCSI_OP_SCSI_TMFUNC_RSP: 511 rc = iscsi_complete_pdu(conn, hdr, NULL, 0); 512 break; 513 default: 514 rc = ISCSI_ERR_BAD_OPCODE; 515 break; 516 } 517 518 return rc; 519 520 copy_hdr: 521 /* 522 * if we did zero copy for the header but we will need multiple 523 * skbs to complete the command then we have to copy the header 524 * for later use 525 */ 526 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy < 527 (tcp_conn->in.datalen + tcp_conn->in.padding + 528 (conn->datadgst_en ? 4 : 0))) { 529 debug_tcp("Copying header for later use. in.copy %d in.datalen" 530 " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen); 531 memcpy(&tcp_conn->hdr, tcp_conn->in.hdr, 532 sizeof(struct iscsi_hdr)); 533 tcp_conn->in.hdr = &tcp_conn->hdr; 534 tcp_conn->in.zero_copy_hdr = 0; 535 } 536 return 0; 537 } 538 539 /** 540 * iscsi_ctask_copy - copy skb bits to the destanation cmd task 541 * @conn: iscsi tcp connection 542 * @ctask: scsi command task 543 * @buf: buffer to copy to 544 * @buf_size: size of buffer 545 * @offset: offset within the buffer 546 * 547 * Notes: 548 * The function calls skb_copy_bits() and updates per-connection and 549 * per-cmd byte counters. 550 * 551 * Read counters (in bytes): 552 * 553 * conn->in.offset offset within in progress SKB 554 * conn->in.copy left to copy from in progress SKB 555 * including padding 556 * conn->in.copied copied already from in progress SKB 557 * conn->data_copied copied already from in progress buffer 558 * ctask->sent total bytes sent up to the MidLayer 559 * ctask->data_count left to copy from in progress Data-In 560 * buf_left left to copy from in progress buffer 561 **/ 562 static inline int 563 iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask, 564 void *buf, int buf_size, int offset) 565 { 566 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 567 int buf_left = buf_size - (tcp_conn->data_copied + offset); 568 int size = min(tcp_conn->in.copy, buf_left); 569 int rc; 570 571 size = min(size, ctask->data_count); 572 573 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n", 574 size, tcp_conn->in.offset, tcp_conn->in.copied); 575 576 BUG_ON(size <= 0); 577 BUG_ON(tcp_ctask->sent + size > ctask->total_length); 578 579 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 580 (char*)buf + (offset + tcp_conn->data_copied), size); 581 /* must fit into skb->len */ 582 BUG_ON(rc); 583 584 tcp_conn->in.offset += size; 585 tcp_conn->in.copy -= size; 586 tcp_conn->in.copied += size; 587 tcp_conn->data_copied += size; 588 tcp_ctask->sent += size; 589 ctask->data_count -= size; 590 591 BUG_ON(tcp_conn->in.copy < 0); 592 BUG_ON(ctask->data_count < 0); 593 594 if (buf_size != (tcp_conn->data_copied + offset)) { 595 if (!ctask->data_count) { 596 BUG_ON(buf_size - tcp_conn->data_copied < 0); 597 /* done with this PDU */ 598 return buf_size - tcp_conn->data_copied; 599 } 600 return -EAGAIN; 601 } 602 603 /* done with this buffer or with both - PDU and buffer */ 604 tcp_conn->data_copied = 0; 605 return 0; 606 } 607 608 /** 609 * iscsi_tcp_copy - copy skb bits to the destanation buffer 610 * @conn: iscsi tcp connection 611 * 612 * Notes: 613 * The function calls skb_copy_bits() and updates per-connection 614 * byte counters. 615 **/ 616 static inline int 617 iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn) 618 { 619 void *buf = tcp_conn->data; 620 int buf_size = tcp_conn->in.datalen; 621 int buf_left = buf_size - tcp_conn->data_copied; 622 int size = min(tcp_conn->in.copy, buf_left); 623 int rc; 624 625 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n", 626 size, tcp_conn->in.offset, tcp_conn->data_copied); 627 BUG_ON(size <= 0); 628 629 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 630 (char*)buf + tcp_conn->data_copied, size); 631 BUG_ON(rc); 632 633 tcp_conn->in.offset += size; 634 tcp_conn->in.copy -= size; 635 tcp_conn->in.copied += size; 636 tcp_conn->data_copied += size; 637 638 if (buf_size != tcp_conn->data_copied) 639 return -EAGAIN; 640 641 return 0; 642 } 643 644 static inline void 645 partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn, 646 struct scatterlist *sg, int offset, int length) 647 { 648 struct scatterlist temp; 649 650 memcpy(&temp, sg, sizeof(struct scatterlist)); 651 temp.offset = offset; 652 temp.length = length; 653 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1); 654 } 655 656 static void 657 iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len) 658 { 659 struct scatterlist tmp; 660 661 sg_init_one(&tmp, buf, len); 662 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1); 663 } 664 665 static int iscsi_scsi_data_in(struct iscsi_conn *conn) 666 { 667 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 668 struct iscsi_cmd_task *ctask = tcp_conn->in.ctask; 669 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 670 struct scsi_cmnd *sc = ctask->sc; 671 struct scatterlist *sg; 672 int i, offset, rc = 0; 673 674 BUG_ON((void*)ctask != sc->SCp.ptr); 675 676 /* 677 * copying Data-In into the Scsi_Cmnd 678 */ 679 if (!sc->use_sg) { 680 i = ctask->data_count; 681 rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer, 682 sc->request_bufflen, 683 tcp_ctask->data_offset); 684 if (rc == -EAGAIN) 685 return rc; 686 if (conn->datadgst_en) 687 iscsi_recv_digest_update(tcp_conn, sc->request_buffer, 688 i); 689 rc = 0; 690 goto done; 691 } 692 693 offset = tcp_ctask->data_offset; 694 sg = sc->request_buffer; 695 696 if (tcp_ctask->data_offset) 697 for (i = 0; i < tcp_ctask->sg_count; i++) 698 offset -= sg[i].length; 699 /* we've passed through partial sg*/ 700 if (offset < 0) 701 offset = 0; 702 703 for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) { 704 char *dest; 705 706 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0); 707 rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset, 708 sg[i].length, offset); 709 kunmap_atomic(dest, KM_SOFTIRQ0); 710 if (rc == -EAGAIN) 711 /* continue with the next SKB/PDU */ 712 return rc; 713 if (!rc) { 714 if (conn->datadgst_en) { 715 if (!offset) 716 crypto_digest_update( 717 tcp_conn->data_rx_tfm, 718 &sg[i], 1); 719 else 720 partial_sg_digest_update(tcp_conn, 721 &sg[i], 722 sg[i].offset + offset, 723 sg[i].length - offset); 724 } 725 offset = 0; 726 tcp_ctask->sg_count++; 727 } 728 729 if (!ctask->data_count) { 730 if (rc && conn->datadgst_en) 731 /* 732 * data-in is complete, but buffer not... 733 */ 734 partial_sg_digest_update(tcp_conn, &sg[i], 735 sg[i].offset, sg[i].length-rc); 736 rc = 0; 737 break; 738 } 739 740 if (!tcp_conn->in.copy) 741 return -EAGAIN; 742 } 743 BUG_ON(ctask->data_count); 744 745 done: 746 /* check for non-exceptional status */ 747 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) { 748 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 749 (long)sc, sc->result, ctask->itt); 750 spin_lock(&conn->session->lock); 751 __iscsi_ctask_cleanup(conn, ctask); 752 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); 753 spin_unlock(&conn->session->lock); 754 } 755 756 return rc; 757 } 758 759 static int 760 iscsi_data_recv(struct iscsi_conn *conn) 761 { 762 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 763 int rc = 0, opcode; 764 765 opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK; 766 switch (opcode) { 767 case ISCSI_OP_SCSI_DATA_IN: 768 rc = iscsi_scsi_data_in(conn); 769 break; 770 case ISCSI_OP_SCSI_CMD_RSP: 771 spin_lock(&conn->session->lock); 772 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask); 773 spin_unlock(&conn->session->lock); 774 case ISCSI_OP_TEXT_RSP: 775 case ISCSI_OP_LOGIN_RSP: 776 case ISCSI_OP_NOOP_IN: 777 case ISCSI_OP_ASYNC_EVENT: 778 case ISCSI_OP_REJECT: 779 /* 780 * Collect data segment to the connection's data 781 * placeholder 782 */ 783 if (iscsi_tcp_copy(tcp_conn)) { 784 rc = -EAGAIN; 785 goto exit; 786 } 787 788 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data, 789 tcp_conn->in.datalen); 790 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP) 791 iscsi_recv_digest_update(tcp_conn, tcp_conn->data, 792 tcp_conn->in.datalen); 793 break; 794 default: 795 BUG_ON(1); 796 } 797 exit: 798 return rc; 799 } 800 801 /** 802 * iscsi_tcp_data_recv - TCP receive in sendfile fashion 803 * @rd_desc: read descriptor 804 * @skb: socket buffer 805 * @offset: offset in skb 806 * @len: skb->len - offset 807 **/ 808 static int 809 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 810 unsigned int offset, size_t len) 811 { 812 int rc; 813 struct iscsi_conn *conn = rd_desc->arg.data; 814 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 815 int processed; 816 char pad[ISCSI_PAD_LEN]; 817 struct scatterlist sg; 818 819 /* 820 * Save current SKB and its offset in the corresponding 821 * connection context. 822 */ 823 tcp_conn->in.copy = skb->len - offset; 824 tcp_conn->in.offset = offset; 825 tcp_conn->in.skb = skb; 826 tcp_conn->in.len = tcp_conn->in.copy; 827 BUG_ON(tcp_conn->in.copy <= 0); 828 debug_tcp("in %d bytes\n", tcp_conn->in.copy); 829 830 more: 831 tcp_conn->in.copied = 0; 832 rc = 0; 833 834 if (unlikely(conn->suspend_rx)) { 835 debug_tcp("conn %d Rx suspended!\n", conn->id); 836 return 0; 837 } 838 839 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER || 840 tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) { 841 rc = iscsi_hdr_extract(tcp_conn); 842 if (rc) { 843 if (rc == -EAGAIN) 844 goto nomore; 845 else { 846 iscsi_conn_failure(conn, rc); 847 return 0; 848 } 849 } 850 851 /* 852 * Verify and process incoming PDU header. 853 */ 854 rc = iscsi_tcp_hdr_recv(conn); 855 if (!rc && tcp_conn->in.datalen) { 856 if (conn->datadgst_en) { 857 BUG_ON(!tcp_conn->data_rx_tfm); 858 crypto_digest_init(tcp_conn->data_rx_tfm); 859 } 860 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV; 861 } else if (rc) { 862 iscsi_conn_failure(conn, rc); 863 return 0; 864 } 865 } 866 867 if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) { 868 uint32_t recv_digest; 869 870 debug_tcp("extra data_recv offset %d copy %d\n", 871 tcp_conn->in.offset, tcp_conn->in.copy); 872 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset, 873 &recv_digest, 4); 874 tcp_conn->in.offset += 4; 875 tcp_conn->in.copy -= 4; 876 if (recv_digest != tcp_conn->in.datadgst) { 877 debug_tcp("iscsi_tcp: data digest error!" 878 "0x%x != 0x%x\n", recv_digest, 879 tcp_conn->in.datadgst); 880 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 881 return 0; 882 } else { 883 debug_tcp("iscsi_tcp: data digest match!" 884 "0x%x == 0x%x\n", recv_digest, 885 tcp_conn->in.datadgst); 886 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 887 } 888 } 889 890 if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV && 891 tcp_conn->in.copy) { 892 893 debug_tcp("data_recv offset %d copy %d\n", 894 tcp_conn->in.offset, tcp_conn->in.copy); 895 896 rc = iscsi_data_recv(conn); 897 if (rc) { 898 if (rc == -EAGAIN) 899 goto again; 900 iscsi_conn_failure(conn, rc); 901 return 0; 902 } 903 tcp_conn->in.copy -= tcp_conn->in.padding; 904 tcp_conn->in.offset += tcp_conn->in.padding; 905 if (conn->datadgst_en) { 906 if (tcp_conn->in.padding) { 907 debug_tcp("padding -> %d\n", 908 tcp_conn->in.padding); 909 memset(pad, 0, tcp_conn->in.padding); 910 sg_init_one(&sg, pad, tcp_conn->in.padding); 911 crypto_digest_update(tcp_conn->data_rx_tfm, 912 &sg, 1); 913 } 914 crypto_digest_final(tcp_conn->data_rx_tfm, 915 (u8 *) & tcp_conn->in.datadgst); 916 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst); 917 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 918 } else 919 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 920 } 921 922 debug_tcp("f, processed %d from out of %d padding %d\n", 923 tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding); 924 BUG_ON(tcp_conn->in.offset - offset > len); 925 926 if (tcp_conn->in.offset - offset != len) { 927 debug_tcp("continue to process %d bytes\n", 928 (int)len - (tcp_conn->in.offset - offset)); 929 goto more; 930 } 931 932 nomore: 933 processed = tcp_conn->in.offset - offset; 934 BUG_ON(processed == 0); 935 return processed; 936 937 again: 938 processed = tcp_conn->in.offset - offset; 939 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n", 940 processed, (int)len, (int)rd_desc->count); 941 BUG_ON(processed == 0); 942 BUG_ON(processed > len); 943 944 conn->rxdata_octets += processed; 945 return processed; 946 } 947 948 static void 949 iscsi_tcp_data_ready(struct sock *sk, int flag) 950 { 951 struct iscsi_conn *conn = sk->sk_user_data; 952 read_descriptor_t rd_desc; 953 954 read_lock(&sk->sk_callback_lock); 955 956 /* 957 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv. 958 * We set count to 1 because we want the network layer to 959 * hand us all the skbs that are available. iscsi_tcp_data_recv 960 * handled pdus that cross buffers or pdus that still need data. 961 */ 962 rd_desc.arg.data = conn; 963 rd_desc.count = 1; 964 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv); 965 966 read_unlock(&sk->sk_callback_lock); 967 } 968 969 static void 970 iscsi_tcp_state_change(struct sock *sk) 971 { 972 struct iscsi_tcp_conn *tcp_conn; 973 struct iscsi_conn *conn; 974 struct iscsi_session *session; 975 void (*old_state_change)(struct sock *); 976 977 read_lock(&sk->sk_callback_lock); 978 979 conn = (struct iscsi_conn*)sk->sk_user_data; 980 session = conn->session; 981 982 if ((sk->sk_state == TCP_CLOSE_WAIT || 983 sk->sk_state == TCP_CLOSE) && 984 !atomic_read(&sk->sk_rmem_alloc)) { 985 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n"); 986 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 987 } 988 989 tcp_conn = conn->dd_data; 990 old_state_change = tcp_conn->old_state_change; 991 992 read_unlock(&sk->sk_callback_lock); 993 994 old_state_change(sk); 995 } 996 997 /** 998 * iscsi_write_space - Called when more output buffer space is available 999 * @sk: socket space is available for 1000 **/ 1001 static void 1002 iscsi_write_space(struct sock *sk) 1003 { 1004 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 1005 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1006 1007 tcp_conn->old_write_space(sk); 1008 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 1009 scsi_queue_work(conn->session->host, &conn->xmitwork); 1010 } 1011 1012 static void 1013 iscsi_conn_set_callbacks(struct iscsi_conn *conn) 1014 { 1015 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1016 struct sock *sk = tcp_conn->sock->sk; 1017 1018 /* assign new callbacks */ 1019 write_lock_bh(&sk->sk_callback_lock); 1020 sk->sk_user_data = conn; 1021 tcp_conn->old_data_ready = sk->sk_data_ready; 1022 tcp_conn->old_state_change = sk->sk_state_change; 1023 tcp_conn->old_write_space = sk->sk_write_space; 1024 sk->sk_data_ready = iscsi_tcp_data_ready; 1025 sk->sk_state_change = iscsi_tcp_state_change; 1026 sk->sk_write_space = iscsi_write_space; 1027 write_unlock_bh(&sk->sk_callback_lock); 1028 } 1029 1030 static void 1031 iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1032 { 1033 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1034 struct sock *sk = tcp_conn->sock->sk; 1035 1036 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1037 write_lock_bh(&sk->sk_callback_lock); 1038 sk->sk_user_data = NULL; 1039 sk->sk_data_ready = tcp_conn->old_data_ready; 1040 sk->sk_state_change = tcp_conn->old_state_change; 1041 sk->sk_write_space = tcp_conn->old_write_space; 1042 sk->sk_no_check = 0; 1043 write_unlock_bh(&sk->sk_callback_lock); 1044 } 1045 1046 /** 1047 * iscsi_send - generic send routine 1048 * @sk: kernel's socket 1049 * @buf: buffer to write from 1050 * @size: actual size to write 1051 * @flags: socket's flags 1052 */ 1053 static inline int 1054 iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags) 1055 { 1056 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1057 struct socket *sk = tcp_conn->sock; 1058 int offset = buf->sg.offset + buf->sent, res; 1059 1060 /* 1061 * if we got use_sg=0 or are sending something we kmallocd 1062 * then we did not have to do kmap (kmap returns page_address) 1063 * 1064 * if we got use_sg > 0, but had to drop down, we do not 1065 * set clustering so this should only happen for that 1066 * slab case. 1067 */ 1068 if (buf->use_sendmsg) 1069 res = sock_no_sendpage(sk, buf->sg.page, offset, size, flags); 1070 else 1071 res = tcp_conn->sendpage(sk, buf->sg.page, offset, size, flags); 1072 1073 if (res >= 0) { 1074 conn->txdata_octets += res; 1075 buf->sent += res; 1076 return res; 1077 } 1078 1079 tcp_conn->sendpage_failures_cnt++; 1080 if (res == -EAGAIN) 1081 res = -ENOBUFS; 1082 else 1083 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1084 return res; 1085 } 1086 1087 /** 1088 * iscsi_sendhdr - send PDU Header via tcp_sendpage() 1089 * @conn: iscsi connection 1090 * @buf: buffer to write from 1091 * @datalen: lenght of data to be sent after the header 1092 * 1093 * Notes: 1094 * (Tx, Fast Path) 1095 **/ 1096 static inline int 1097 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen) 1098 { 1099 int flags = 0; /* MSG_DONTWAIT; */ 1100 int res, size; 1101 1102 size = buf->sg.length - buf->sent; 1103 BUG_ON(buf->sent + size > buf->sg.length); 1104 if (buf->sent + size != buf->sg.length || datalen) 1105 flags |= MSG_MORE; 1106 1107 res = iscsi_send(conn, buf, size, flags); 1108 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res); 1109 if (res >= 0) { 1110 if (size != res) 1111 return -EAGAIN; 1112 return 0; 1113 } 1114 1115 return res; 1116 } 1117 1118 /** 1119 * iscsi_sendpage - send one page of iSCSI Data-Out. 1120 * @conn: iscsi connection 1121 * @buf: buffer to write from 1122 * @count: remaining data 1123 * @sent: number of bytes sent 1124 * 1125 * Notes: 1126 * (Tx, Fast Path) 1127 **/ 1128 static inline int 1129 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf, 1130 int *count, int *sent) 1131 { 1132 int flags = 0; /* MSG_DONTWAIT; */ 1133 int res, size; 1134 1135 size = buf->sg.length - buf->sent; 1136 BUG_ON(buf->sent + size > buf->sg.length); 1137 if (size > *count) 1138 size = *count; 1139 if (buf->sent + size != buf->sg.length || *count != size) 1140 flags |= MSG_MORE; 1141 1142 res = iscsi_send(conn, buf, size, flags); 1143 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n", 1144 size, buf->sent, *count, *sent, res); 1145 if (res >= 0) { 1146 *count -= res; 1147 *sent += res; 1148 if (size != res) 1149 return -EAGAIN; 1150 return 0; 1151 } 1152 1153 return res; 1154 } 1155 1156 static inline void 1157 iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn, 1158 struct iscsi_cmd_task *ctask) 1159 { 1160 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1161 1162 BUG_ON(!tcp_conn->data_tx_tfm); 1163 crypto_digest_init(tcp_conn->data_tx_tfm); 1164 tcp_ctask->digest_count = 4; 1165 } 1166 1167 static int 1168 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1169 struct iscsi_buf *buf, uint32_t *digest, int final) 1170 { 1171 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1172 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1173 int rc = 0; 1174 int sent = 0; 1175 1176 if (final) 1177 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest); 1178 1179 iscsi_buf_init_iov(buf, (char*)digest, 4); 1180 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent); 1181 if (rc) { 1182 tcp_ctask->datadigest = *digest; 1183 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST; 1184 } else 1185 tcp_ctask->digest_count = 4; 1186 return rc; 1187 } 1188 1189 /** 1190 * iscsi_solicit_data_cont - initialize next Data-Out 1191 * @conn: iscsi connection 1192 * @ctask: scsi command task 1193 * @r2t: R2T info 1194 * @left: bytes left to transfer 1195 * 1196 * Notes: 1197 * Initialize next Data-Out within this R2T sequence and continue 1198 * to process next Scatter-Gather element(if any) of this SCSI command. 1199 * 1200 * Called under connection lock. 1201 **/ 1202 static void 1203 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1204 struct iscsi_r2t_info *r2t, int left) 1205 { 1206 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1207 struct iscsi_data *hdr; 1208 struct scsi_cmnd *sc = ctask->sc; 1209 int new_offset; 1210 1211 hdr = &r2t->dtask.hdr; 1212 memset(hdr, 0, sizeof(struct iscsi_data)); 1213 hdr->ttt = r2t->ttt; 1214 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 1215 r2t->solicit_datasn++; 1216 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1217 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun)); 1218 hdr->itt = ctask->hdr->itt; 1219 hdr->exp_statsn = r2t->exp_statsn; 1220 new_offset = r2t->data_offset + r2t->sent; 1221 hdr->offset = cpu_to_be32(new_offset); 1222 if (left > conn->max_xmit_dlength) { 1223 hton24(hdr->dlength, conn->max_xmit_dlength); 1224 r2t->data_count = conn->max_xmit_dlength; 1225 } else { 1226 hton24(hdr->dlength, left); 1227 r2t->data_count = left; 1228 hdr->flags = ISCSI_FLAG_CMD_FINAL; 1229 } 1230 conn->dataout_pdus_cnt++; 1231 1232 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr, 1233 sizeof(struct iscsi_hdr)); 1234 1235 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1236 BUG_ON(tcp_ctask->bad_sg == r2t->sg); 1237 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1238 r2t->sg += 1; 1239 } else 1240 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1241 (char*)sc->request_buffer + new_offset, 1242 r2t->data_count); 1243 } 1244 1245 static void 1246 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1247 { 1248 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1249 struct iscsi_data_task *dtask; 1250 1251 dtask = tcp_ctask->dtask = &tcp_ctask->unsol_dtask; 1252 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr, 1253 tcp_ctask->r2t_data_count); 1254 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr, 1255 sizeof(struct iscsi_hdr)); 1256 } 1257 1258 /** 1259 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 1260 * @conn: iscsi connection 1261 * @ctask: scsi command task 1262 * @sc: scsi command 1263 **/ 1264 static void 1265 iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask) 1266 { 1267 struct scsi_cmnd *sc = ctask->sc; 1268 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1269 1270 BUG_ON(__kfifo_len(tcp_ctask->r2tqueue)); 1271 1272 tcp_ctask->sent = 0; 1273 tcp_ctask->sg_count = 0; 1274 1275 if (sc->sc_data_direction == DMA_TO_DEVICE) { 1276 tcp_ctask->xmstate = XMSTATE_W_HDR; 1277 tcp_ctask->exp_r2tsn = 0; 1278 BUG_ON(ctask->total_length == 0); 1279 1280 if (sc->use_sg) { 1281 struct scatterlist *sg = sc->request_buffer; 1282 1283 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1284 &sg[tcp_ctask->sg_count++]); 1285 tcp_ctask->sg = sg; 1286 tcp_ctask->bad_sg = sg + sc->use_sg; 1287 } else 1288 iscsi_buf_init_iov(&tcp_ctask->sendbuf, 1289 sc->request_buffer, 1290 sc->request_bufflen); 1291 1292 if (ctask->imm_count) 1293 tcp_ctask->xmstate |= XMSTATE_IMM_DATA; 1294 1295 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1); 1296 if (tcp_ctask->pad_count) { 1297 tcp_ctask->pad_count = ISCSI_PAD_LEN - 1298 tcp_ctask->pad_count; 1299 debug_scsi("write padding %d bytes\n", 1300 tcp_ctask->pad_count); 1301 tcp_ctask->xmstate |= XMSTATE_W_PAD; 1302 } 1303 1304 if (ctask->unsol_count) 1305 tcp_ctask->xmstate |= XMSTATE_UNS_HDR | 1306 XMSTATE_UNS_INIT; 1307 tcp_ctask->r2t_data_count = ctask->total_length - 1308 ctask->imm_count - 1309 ctask->unsol_count; 1310 1311 debug_scsi("cmd [itt %x total %d imm %d imm_data %d " 1312 "r2t_data %d]\n", 1313 ctask->itt, ctask->total_length, ctask->imm_count, 1314 ctask->unsol_count, tcp_ctask->r2t_data_count); 1315 } else 1316 tcp_ctask->xmstate = XMSTATE_R_HDR; 1317 1318 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr, 1319 sizeof(struct iscsi_hdr)); 1320 } 1321 1322 /** 1323 * iscsi_tcp_mtask_xmit - xmit management(immediate) task 1324 * @conn: iscsi connection 1325 * @mtask: task management task 1326 * 1327 * Notes: 1328 * The function can return -EAGAIN in which case caller must 1329 * call it again later, or recover. '0' return code means successful 1330 * xmit. 1331 * 1332 * Management xmit state machine consists of two states: 1333 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress 1334 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress 1335 **/ 1336 static int 1337 iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask) 1338 { 1339 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data; 1340 int rc; 1341 1342 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n", 1343 conn->id, tcp_mtask->xmstate, mtask->itt); 1344 1345 if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) { 1346 tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR; 1347 if (mtask->data_count) 1348 tcp_mtask->xmstate |= XMSTATE_IMM_DATA; 1349 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 1350 conn->stop_stage != STOP_CONN_RECOVER && 1351 conn->hdrdgst_en) 1352 iscsi_hdr_digest(conn, &tcp_mtask->headbuf, 1353 (u8*)tcp_mtask->hdrext); 1354 rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf, 1355 mtask->data_count); 1356 if (rc) { 1357 tcp_mtask->xmstate |= XMSTATE_IMM_HDR; 1358 if (mtask->data_count) 1359 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA; 1360 return rc; 1361 } 1362 } 1363 1364 if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) { 1365 BUG_ON(!mtask->data_count); 1366 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA; 1367 /* FIXME: implement. 1368 * Virtual buffer could be spreaded across multiple pages... 1369 */ 1370 do { 1371 int rc; 1372 1373 rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf, 1374 &mtask->data_count, &tcp_mtask->sent); 1375 if (rc) { 1376 tcp_mtask->xmstate |= XMSTATE_IMM_DATA; 1377 return rc; 1378 } 1379 } while (mtask->data_count); 1380 } 1381 1382 BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE); 1383 if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) { 1384 struct iscsi_session *session = conn->session; 1385 1386 spin_lock_bh(&session->lock); 1387 list_del(&conn->mtask->running); 1388 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask, 1389 sizeof(void*)); 1390 spin_unlock_bh(&session->lock); 1391 } 1392 return 0; 1393 } 1394 1395 static inline int 1396 handle_xmstate_r_hdr(struct iscsi_conn *conn, 1397 struct iscsi_tcp_cmd_task *tcp_ctask) 1398 { 1399 int rc; 1400 1401 tcp_ctask->xmstate &= ~XMSTATE_R_HDR; 1402 if (conn->hdrdgst_en) 1403 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1404 (u8*)tcp_ctask->hdrext); 1405 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0); 1406 if (!rc) { 1407 BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE); 1408 return 0; /* wait for Data-In */ 1409 } 1410 tcp_ctask->xmstate |= XMSTATE_R_HDR; 1411 return rc; 1412 } 1413 1414 static inline int 1415 handle_xmstate_w_hdr(struct iscsi_conn *conn, 1416 struct iscsi_cmd_task *ctask) 1417 { 1418 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1419 int rc; 1420 1421 tcp_ctask->xmstate &= ~XMSTATE_W_HDR; 1422 if (conn->hdrdgst_en) 1423 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1424 (u8*)tcp_ctask->hdrext); 1425 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count); 1426 if (rc) 1427 tcp_ctask->xmstate |= XMSTATE_W_HDR; 1428 return rc; 1429 } 1430 1431 static inline int 1432 handle_xmstate_data_digest(struct iscsi_conn *conn, 1433 struct iscsi_cmd_task *ctask) 1434 { 1435 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1436 int rc; 1437 1438 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1439 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest); 1440 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1441 &tcp_ctask->datadigest, 0); 1442 if (rc) { 1443 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST; 1444 debug_tcp("resent data digest 0x%x fail!\n", 1445 tcp_ctask->datadigest); 1446 } 1447 1448 return rc; 1449 } 1450 1451 static inline int 1452 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1453 { 1454 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1455 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1456 int rc; 1457 1458 BUG_ON(!ctask->imm_count); 1459 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA; 1460 1461 if (conn->datadgst_en) { 1462 iscsi_data_digest_init(tcp_conn, ctask); 1463 tcp_ctask->immdigest = 0; 1464 } 1465 1466 for (;;) { 1467 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1468 &ctask->imm_count, &tcp_ctask->sent); 1469 if (rc) { 1470 tcp_ctask->xmstate |= XMSTATE_IMM_DATA; 1471 if (conn->datadgst_en) { 1472 crypto_digest_final(tcp_conn->data_tx_tfm, 1473 (u8*)&tcp_ctask->immdigest); 1474 debug_tcp("tx imm sendpage fail 0x%x\n", 1475 tcp_ctask->datadigest); 1476 } 1477 return rc; 1478 } 1479 if (conn->datadgst_en) 1480 crypto_digest_update(tcp_conn->data_tx_tfm, 1481 &tcp_ctask->sendbuf.sg, 1); 1482 1483 if (!ctask->imm_count) 1484 break; 1485 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1486 &tcp_ctask->sg[tcp_ctask->sg_count++]); 1487 } 1488 1489 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) { 1490 rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf, 1491 &tcp_ctask->immdigest, 1); 1492 if (rc) { 1493 debug_tcp("sending imm digest 0x%x fail!\n", 1494 tcp_ctask->immdigest); 1495 return rc; 1496 } 1497 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest); 1498 } 1499 1500 return 0; 1501 } 1502 1503 static inline int 1504 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1505 { 1506 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1507 struct iscsi_data_task *dtask; 1508 int rc; 1509 1510 tcp_ctask->xmstate |= XMSTATE_UNS_DATA; 1511 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) { 1512 iscsi_unsolicit_data_init(conn, ctask); 1513 dtask = tcp_ctask->dtask; 1514 if (conn->hdrdgst_en) 1515 iscsi_hdr_digest(conn, &tcp_ctask->headbuf, 1516 (u8*)dtask->hdrext); 1517 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT; 1518 } 1519 1520 rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count); 1521 if (rc) { 1522 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA; 1523 tcp_ctask->xmstate |= XMSTATE_UNS_HDR; 1524 return rc; 1525 } 1526 1527 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1528 ctask->itt, ctask->unsol_count, tcp_ctask->sent); 1529 return 0; 1530 } 1531 1532 static inline int 1533 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1534 { 1535 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1536 struct iscsi_data_task *dtask = tcp_ctask->dtask; 1537 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1538 int rc; 1539 1540 BUG_ON(!ctask->data_count); 1541 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA; 1542 1543 if (conn->datadgst_en) { 1544 iscsi_data_digest_init(tcp_conn, ctask); 1545 dtask->digest = 0; 1546 } 1547 1548 for (;;) { 1549 int start = tcp_ctask->sent; 1550 1551 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, 1552 &ctask->data_count, &tcp_ctask->sent); 1553 if (rc) { 1554 ctask->unsol_count -= tcp_ctask->sent - start; 1555 tcp_ctask->xmstate |= XMSTATE_UNS_DATA; 1556 /* will continue with this ctask later.. */ 1557 if (conn->datadgst_en) { 1558 crypto_digest_final(tcp_conn->data_tx_tfm, 1559 (u8 *)&dtask->digest); 1560 debug_tcp("tx uns data fail 0x%x\n", 1561 dtask->digest); 1562 } 1563 return rc; 1564 } 1565 1566 BUG_ON(tcp_ctask->sent > ctask->total_length); 1567 ctask->unsol_count -= tcp_ctask->sent - start; 1568 1569 /* 1570 * XXX:we may run here with un-initial sendbuf. 1571 * so pass it 1572 */ 1573 if (conn->datadgst_en && tcp_ctask->sent - start > 0) 1574 crypto_digest_update(tcp_conn->data_tx_tfm, 1575 &tcp_ctask->sendbuf.sg, 1); 1576 1577 if (!ctask->data_count) 1578 break; 1579 iscsi_buf_init_sg(&tcp_ctask->sendbuf, 1580 &tcp_ctask->sg[tcp_ctask->sg_count++]); 1581 } 1582 BUG_ON(ctask->unsol_count < 0); 1583 1584 /* 1585 * Done with the Data-Out. Next, check if we need 1586 * to send another unsolicited Data-Out. 1587 */ 1588 if (ctask->unsol_count) { 1589 if (conn->datadgst_en) { 1590 rc = iscsi_digest_final_send(conn, ctask, 1591 &dtask->digestbuf, 1592 &dtask->digest, 1); 1593 if (rc) { 1594 debug_tcp("send uns digest 0x%x fail\n", 1595 dtask->digest); 1596 return rc; 1597 } 1598 debug_tcp("sending uns digest 0x%x, more uns\n", 1599 dtask->digest); 1600 } 1601 tcp_ctask->xmstate |= XMSTATE_UNS_INIT; 1602 return 1; 1603 } 1604 1605 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) { 1606 rc = iscsi_digest_final_send(conn, ctask, 1607 &dtask->digestbuf, 1608 &dtask->digest, 1); 1609 if (rc) { 1610 debug_tcp("send last uns digest 0x%x fail\n", 1611 dtask->digest); 1612 return rc; 1613 } 1614 debug_tcp("sending uns digest 0x%x\n",dtask->digest); 1615 } 1616 1617 return 0; 1618 } 1619 1620 static inline int 1621 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1622 { 1623 struct iscsi_session *session = conn->session; 1624 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1625 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1626 struct iscsi_r2t_info *r2t = tcp_ctask->r2t; 1627 struct iscsi_data_task *dtask = &r2t->dtask; 1628 int left, rc; 1629 1630 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA; 1631 tcp_ctask->dtask = dtask; 1632 1633 if (conn->datadgst_en) { 1634 iscsi_data_digest_init(tcp_conn, ctask); 1635 dtask->digest = 0; 1636 } 1637 solicit_again: 1638 /* 1639 * send Data-Out whitnin this R2T sequence. 1640 */ 1641 if (!r2t->data_count) 1642 goto data_out_done; 1643 1644 rc = iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent); 1645 if (rc) { 1646 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1647 /* will continue with this ctask later.. */ 1648 if (conn->datadgst_en) { 1649 crypto_digest_final(tcp_conn->data_tx_tfm, 1650 (u8 *)&dtask->digest); 1651 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1652 } 1653 return rc; 1654 } 1655 1656 BUG_ON(r2t->data_count < 0); 1657 if (conn->datadgst_en) 1658 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg, 1659 1); 1660 1661 if (r2t->data_count) { 1662 BUG_ON(ctask->sc->use_sg == 0); 1663 if (!iscsi_buf_left(&r2t->sendbuf)) { 1664 BUG_ON(tcp_ctask->bad_sg == r2t->sg); 1665 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1666 r2t->sg += 1; 1667 } 1668 goto solicit_again; 1669 } 1670 1671 data_out_done: 1672 /* 1673 * Done with this Data-Out. Next, check if we have 1674 * to send another Data-Out for this R2T. 1675 */ 1676 BUG_ON(r2t->data_length - r2t->sent < 0); 1677 left = r2t->data_length - r2t->sent; 1678 if (left) { 1679 if (conn->datadgst_en) { 1680 rc = iscsi_digest_final_send(conn, ctask, 1681 &dtask->digestbuf, 1682 &dtask->digest, 1); 1683 if (rc) { 1684 debug_tcp("send r2t data digest 0x%x" 1685 "fail\n", dtask->digest); 1686 return rc; 1687 } 1688 debug_tcp("r2t data send digest 0x%x\n", 1689 dtask->digest); 1690 } 1691 iscsi_solicit_data_cont(conn, ctask, r2t, left); 1692 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1693 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR; 1694 return 1; 1695 } 1696 1697 /* 1698 * Done with this R2T. Check if there are more 1699 * outstanding R2Ts ready to be processed. 1700 */ 1701 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0); 1702 if (conn->datadgst_en) { 1703 rc = iscsi_digest_final_send(conn, ctask, &dtask->digestbuf, 1704 &dtask->digest, 1); 1705 if (rc) { 1706 debug_tcp("send last r2t data digest 0x%x" 1707 "fail\n", dtask->digest); 1708 return rc; 1709 } 1710 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest); 1711 } 1712 1713 tcp_ctask->r2t_data_count -= r2t->data_length; 1714 tcp_ctask->r2t = NULL; 1715 spin_lock_bh(&session->lock); 1716 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 1717 spin_unlock_bh(&session->lock); 1718 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) { 1719 tcp_ctask->r2t = r2t; 1720 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1721 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR; 1722 return 1; 1723 } 1724 1725 return 0; 1726 } 1727 1728 static inline int 1729 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1730 { 1731 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1732 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1733 struct iscsi_data_task *dtask = tcp_ctask->dtask; 1734 int sent, rc; 1735 1736 tcp_ctask->xmstate &= ~XMSTATE_W_PAD; 1737 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad, 1738 tcp_ctask->pad_count); 1739 rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count, 1740 &sent); 1741 if (rc) { 1742 tcp_ctask->xmstate |= XMSTATE_W_PAD; 1743 return rc; 1744 } 1745 1746 if (conn->datadgst_en) { 1747 crypto_digest_update(tcp_conn->data_tx_tfm, 1748 &tcp_ctask->sendbuf.sg, 1); 1749 /* imm data? */ 1750 if (!dtask) { 1751 rc = iscsi_digest_final_send(conn, ctask, 1752 &tcp_ctask->immbuf, 1753 &tcp_ctask->immdigest, 1); 1754 if (rc) { 1755 debug_tcp("send padding digest 0x%x" 1756 "fail!\n", tcp_ctask->immdigest); 1757 return rc; 1758 } 1759 debug_tcp("done with padding, digest 0x%x\n", 1760 tcp_ctask->datadigest); 1761 } else { 1762 rc = iscsi_digest_final_send(conn, ctask, 1763 &dtask->digestbuf, 1764 &dtask->digest, 1); 1765 if (rc) { 1766 debug_tcp("send padding digest 0x%x" 1767 "fail\n", dtask->digest); 1768 return rc; 1769 } 1770 debug_tcp("done with padding, digest 0x%x\n", 1771 dtask->digest); 1772 } 1773 } 1774 1775 return 0; 1776 } 1777 1778 static int 1779 iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1780 { 1781 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 1782 int rc = 0; 1783 1784 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n", 1785 conn->id, tcp_ctask->xmstate, ctask->itt); 1786 1787 /* 1788 * serialize with TMF AbortTask 1789 */ 1790 if (ctask->mtask) 1791 return rc; 1792 1793 if (tcp_ctask->xmstate & XMSTATE_R_HDR) 1794 return handle_xmstate_r_hdr(conn, tcp_ctask); 1795 1796 if (tcp_ctask->xmstate & XMSTATE_W_HDR) { 1797 rc = handle_xmstate_w_hdr(conn, ctask); 1798 if (rc) 1799 return rc; 1800 } 1801 1802 /* XXX: for data digest xmit recover */ 1803 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) { 1804 rc = handle_xmstate_data_digest(conn, ctask); 1805 if (rc) 1806 return rc; 1807 } 1808 1809 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) { 1810 rc = handle_xmstate_imm_data(conn, ctask); 1811 if (rc) 1812 return rc; 1813 } 1814 1815 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) { 1816 BUG_ON(!ctask->unsol_count); 1817 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR; 1818 unsolicit_head_again: 1819 rc = handle_xmstate_uns_hdr(conn, ctask); 1820 if (rc) 1821 return rc; 1822 } 1823 1824 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) { 1825 rc = handle_xmstate_uns_data(conn, ctask); 1826 if (rc == 1) 1827 goto unsolicit_head_again; 1828 else if (rc) 1829 return rc; 1830 goto done; 1831 } 1832 1833 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) { 1834 struct iscsi_r2t_info *r2t; 1835 1836 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR; 1837 tcp_ctask->xmstate |= XMSTATE_SOL_DATA; 1838 if (!tcp_ctask->r2t) 1839 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t, 1840 sizeof(void*)); 1841 solicit_head_again: 1842 r2t = tcp_ctask->r2t; 1843 if (conn->hdrdgst_en) 1844 iscsi_hdr_digest(conn, &r2t->headbuf, 1845 (u8*)r2t->dtask.hdrext); 1846 rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count); 1847 if (rc) { 1848 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA; 1849 tcp_ctask->xmstate |= XMSTATE_SOL_HDR; 1850 return rc; 1851 } 1852 1853 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n", 1854 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count, 1855 r2t->sent); 1856 } 1857 1858 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) { 1859 rc = handle_xmstate_sol_data(conn, ctask); 1860 if (rc == 1) 1861 goto solicit_head_again; 1862 if (rc) 1863 return rc; 1864 } 1865 1866 done: 1867 /* 1868 * Last thing to check is whether we need to send write 1869 * padding. Note that we check for xmstate equality, not just the bit. 1870 */ 1871 if (tcp_ctask->xmstate == XMSTATE_W_PAD) 1872 rc = handle_xmstate_w_pad(conn, ctask); 1873 1874 return rc; 1875 } 1876 1877 static struct iscsi_cls_conn * 1878 iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 1879 { 1880 struct iscsi_conn *conn; 1881 struct iscsi_cls_conn *cls_conn; 1882 struct iscsi_tcp_conn *tcp_conn; 1883 1884 cls_conn = iscsi_conn_setup(cls_session, conn_idx); 1885 if (!cls_conn) 1886 return NULL; 1887 conn = cls_conn->dd_data; 1888 /* 1889 * due to strange issues with iser these are not set 1890 * in iscsi_conn_setup 1891 */ 1892 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 1893 1894 tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL); 1895 if (!tcp_conn) 1896 goto tcp_conn_alloc_fail; 1897 1898 conn->dd_data = tcp_conn; 1899 tcp_conn->iscsi_conn = conn; 1900 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1901 /* initial operational parameters */ 1902 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 1903 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 1904 1905 /* allocate initial PDU receive place holder */ 1906 if (tcp_conn->data_size <= PAGE_SIZE) 1907 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL); 1908 else 1909 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL, 1910 get_order(tcp_conn->data_size)); 1911 if (!tcp_conn->data) 1912 goto max_recv_dlenght_alloc_fail; 1913 1914 return cls_conn; 1915 1916 max_recv_dlenght_alloc_fail: 1917 kfree(tcp_conn); 1918 tcp_conn_alloc_fail: 1919 iscsi_conn_teardown(cls_conn); 1920 return NULL; 1921 } 1922 1923 static void 1924 iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn) 1925 { 1926 struct iscsi_conn *conn = cls_conn->dd_data; 1927 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1928 int digest = 0; 1929 1930 if (conn->hdrdgst_en || conn->datadgst_en) 1931 digest = 1; 1932 1933 iscsi_conn_teardown(cls_conn); 1934 1935 /* now free tcp_conn */ 1936 if (digest) { 1937 if (tcp_conn->tx_tfm) 1938 crypto_free_tfm(tcp_conn->tx_tfm); 1939 if (tcp_conn->rx_tfm) 1940 crypto_free_tfm(tcp_conn->rx_tfm); 1941 if (tcp_conn->data_tx_tfm) 1942 crypto_free_tfm(tcp_conn->data_tx_tfm); 1943 if (tcp_conn->data_rx_tfm) 1944 crypto_free_tfm(tcp_conn->data_rx_tfm); 1945 } 1946 1947 /* free conn->data, size = MaxRecvDataSegmentLength */ 1948 if (tcp_conn->data_size <= PAGE_SIZE) 1949 kfree(tcp_conn->data); 1950 else 1951 free_pages((unsigned long)tcp_conn->data, 1952 get_order(tcp_conn->data_size)); 1953 kfree(tcp_conn); 1954 } 1955 1956 static int 1957 iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session, 1958 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 1959 int is_leading) 1960 { 1961 struct iscsi_conn *conn = cls_conn->dd_data; 1962 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1963 struct sock *sk; 1964 struct socket *sock; 1965 int err; 1966 1967 /* lookup for existing socket */ 1968 sock = sockfd_lookup((int)transport_eph, &err); 1969 if (!sock) { 1970 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err); 1971 return -EEXIST; 1972 } 1973 1974 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 1975 if (err) 1976 return err; 1977 1978 /* bind iSCSI connection and socket */ 1979 tcp_conn->sock = sock; 1980 1981 /* setup Socket parameters */ 1982 sk = sock->sk; 1983 sk->sk_reuse = 1; 1984 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 1985 sk->sk_allocation = GFP_ATOMIC; 1986 1987 /* FIXME: disable Nagle's algorithm */ 1988 1989 /* 1990 * Intercept TCP callbacks for sendfile like receive 1991 * processing. 1992 */ 1993 conn->recv_lock = &sk->sk_callback_lock; 1994 iscsi_conn_set_callbacks(conn); 1995 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage; 1996 /* 1997 * set receive state machine into initial state 1998 */ 1999 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2000 2001 return 0; 2002 } 2003 2004 static void 2005 iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 2006 { 2007 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 2008 struct iscsi_r2t_info *r2t; 2009 2010 /* flush ctask's r2t queues */ 2011 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) 2012 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, 2013 sizeof(void*)); 2014 2015 __iscsi_ctask_cleanup(conn, ctask); 2016 } 2017 2018 static void 2019 iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn) 2020 { 2021 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2022 struct sock *sk; 2023 2024 if (!tcp_conn->sock) 2025 return; 2026 2027 sk = tcp_conn->sock->sk; 2028 write_lock_bh(&sk->sk_callback_lock); 2029 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx); 2030 write_unlock_bh(&sk->sk_callback_lock); 2031 } 2032 2033 static void 2034 iscsi_tcp_terminate_conn(struct iscsi_conn *conn) 2035 { 2036 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2037 2038 if (!tcp_conn->sock) 2039 return; 2040 2041 sock_hold(tcp_conn->sock->sk); 2042 iscsi_conn_restore_callbacks(conn); 2043 sock_put(tcp_conn->sock->sk); 2044 2045 sock_release(tcp_conn->sock); 2046 tcp_conn->sock = NULL; 2047 conn->recv_lock = NULL; 2048 } 2049 2050 /* called with host lock */ 2051 static void 2052 iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask, 2053 char *data, uint32_t data_size) 2054 { 2055 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data; 2056 2057 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr, 2058 sizeof(struct iscsi_hdr)); 2059 tcp_mtask->xmstate = XMSTATE_IMM_HDR; 2060 2061 if (mtask->data_count) 2062 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data, 2063 mtask->data_count); 2064 } 2065 2066 static int 2067 iscsi_r2tpool_alloc(struct iscsi_session *session) 2068 { 2069 int i; 2070 int cmd_i; 2071 2072 /* 2073 * initialize per-task: R2T pool and xmit queue 2074 */ 2075 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 2076 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 2077 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 2078 2079 /* 2080 * pre-allocated x4 as much r2ts to handle race when 2081 * target acks DataOut faster than we data_xmit() queues 2082 * could replenish r2tqueue. 2083 */ 2084 2085 /* R2T pool */ 2086 if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4, 2087 (void***)&tcp_ctask->r2ts, 2088 sizeof(struct iscsi_r2t_info))) { 2089 goto r2t_alloc_fail; 2090 } 2091 2092 /* R2T xmit queue */ 2093 tcp_ctask->r2tqueue = kfifo_alloc( 2094 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL); 2095 if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) { 2096 iscsi_pool_free(&tcp_ctask->r2tpool, 2097 (void**)tcp_ctask->r2ts); 2098 goto r2t_alloc_fail; 2099 } 2100 } 2101 2102 return 0; 2103 2104 r2t_alloc_fail: 2105 for (i = 0; i < cmd_i; i++) { 2106 struct iscsi_cmd_task *ctask = session->cmds[i]; 2107 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 2108 2109 kfifo_free(tcp_ctask->r2tqueue); 2110 iscsi_pool_free(&tcp_ctask->r2tpool, 2111 (void**)tcp_ctask->r2ts); 2112 } 2113 return -ENOMEM; 2114 } 2115 2116 static void 2117 iscsi_r2tpool_free(struct iscsi_session *session) 2118 { 2119 int i; 2120 2121 for (i = 0; i < session->cmds_max; i++) { 2122 struct iscsi_cmd_task *ctask = session->cmds[i]; 2123 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 2124 2125 kfifo_free(tcp_ctask->r2tqueue); 2126 iscsi_pool_free(&tcp_ctask->r2tpool, 2127 (void**)tcp_ctask->r2ts); 2128 } 2129 } 2130 2131 static int 2132 iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param, 2133 uint32_t value) 2134 { 2135 struct iscsi_conn *conn = cls_conn->dd_data; 2136 struct iscsi_session *session = conn->session; 2137 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2138 2139 switch(param) { 2140 case ISCSI_PARAM_MAX_RECV_DLENGTH: { 2141 char *saveptr = tcp_conn->data; 2142 gfp_t flags = GFP_KERNEL; 2143 2144 if (tcp_conn->data_size >= value) { 2145 conn->max_recv_dlength = value; 2146 break; 2147 } 2148 2149 spin_lock_bh(&session->lock); 2150 if (conn->stop_stage == STOP_CONN_RECOVER) 2151 flags = GFP_ATOMIC; 2152 spin_unlock_bh(&session->lock); 2153 2154 if (value <= PAGE_SIZE) 2155 tcp_conn->data = kmalloc(value, flags); 2156 else 2157 tcp_conn->data = (void*)__get_free_pages(flags, 2158 get_order(value)); 2159 if (tcp_conn->data == NULL) { 2160 tcp_conn->data = saveptr; 2161 return -ENOMEM; 2162 } 2163 if (tcp_conn->data_size <= PAGE_SIZE) 2164 kfree(saveptr); 2165 else 2166 free_pages((unsigned long)saveptr, 2167 get_order(tcp_conn->data_size)); 2168 conn->max_recv_dlength = value; 2169 tcp_conn->data_size = value; 2170 } 2171 break; 2172 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2173 conn->max_xmit_dlength = value; 2174 break; 2175 case ISCSI_PARAM_HDRDGST_EN: 2176 conn->hdrdgst_en = value; 2177 tcp_conn->hdr_size = sizeof(struct iscsi_hdr); 2178 if (conn->hdrdgst_en) { 2179 tcp_conn->hdr_size += sizeof(__u32); 2180 if (!tcp_conn->tx_tfm) 2181 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c", 2182 0); 2183 if (!tcp_conn->tx_tfm) 2184 return -ENOMEM; 2185 if (!tcp_conn->rx_tfm) 2186 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c", 2187 0); 2188 if (!tcp_conn->rx_tfm) { 2189 crypto_free_tfm(tcp_conn->tx_tfm); 2190 return -ENOMEM; 2191 } 2192 } else { 2193 if (tcp_conn->tx_tfm) 2194 crypto_free_tfm(tcp_conn->tx_tfm); 2195 if (tcp_conn->rx_tfm) 2196 crypto_free_tfm(tcp_conn->rx_tfm); 2197 } 2198 break; 2199 case ISCSI_PARAM_DATADGST_EN: 2200 conn->datadgst_en = value; 2201 if (conn->datadgst_en) { 2202 if (!tcp_conn->data_tx_tfm) 2203 tcp_conn->data_tx_tfm = 2204 crypto_alloc_tfm("crc32c", 0); 2205 if (!tcp_conn->data_tx_tfm) 2206 return -ENOMEM; 2207 if (!tcp_conn->data_rx_tfm) 2208 tcp_conn->data_rx_tfm = 2209 crypto_alloc_tfm("crc32c", 0); 2210 if (!tcp_conn->data_rx_tfm) { 2211 crypto_free_tfm(tcp_conn->data_tx_tfm); 2212 return -ENOMEM; 2213 } 2214 } else { 2215 if (tcp_conn->data_tx_tfm) 2216 crypto_free_tfm(tcp_conn->data_tx_tfm); 2217 if (tcp_conn->data_rx_tfm) 2218 crypto_free_tfm(tcp_conn->data_rx_tfm); 2219 } 2220 tcp_conn->sendpage = conn->datadgst_en ? 2221 sock_no_sendpage : tcp_conn->sock->ops->sendpage; 2222 break; 2223 case ISCSI_PARAM_INITIAL_R2T_EN: 2224 session->initial_r2t_en = value; 2225 break; 2226 case ISCSI_PARAM_MAX_R2T: 2227 if (session->max_r2t == roundup_pow_of_two(value)) 2228 break; 2229 iscsi_r2tpool_free(session); 2230 session->max_r2t = value; 2231 if (session->max_r2t & (session->max_r2t - 1)) 2232 session->max_r2t = roundup_pow_of_two(session->max_r2t); 2233 if (iscsi_r2tpool_alloc(session)) 2234 return -ENOMEM; 2235 break; 2236 case ISCSI_PARAM_IMM_DATA_EN: 2237 session->imm_data_en = value; 2238 break; 2239 case ISCSI_PARAM_FIRST_BURST: 2240 session->first_burst = value; 2241 break; 2242 case ISCSI_PARAM_MAX_BURST: 2243 session->max_burst = value; 2244 break; 2245 case ISCSI_PARAM_PDU_INORDER_EN: 2246 session->pdu_inorder_en = value; 2247 break; 2248 case ISCSI_PARAM_DATASEQ_INORDER_EN: 2249 session->dataseq_inorder_en = value; 2250 break; 2251 case ISCSI_PARAM_ERL: 2252 session->erl = value; 2253 break; 2254 case ISCSI_PARAM_IFMARKER_EN: 2255 BUG_ON(value); 2256 session->ifmarker_en = value; 2257 break; 2258 case ISCSI_PARAM_OFMARKER_EN: 2259 BUG_ON(value); 2260 session->ofmarker_en = value; 2261 break; 2262 case ISCSI_PARAM_EXP_STATSN: 2263 conn->exp_statsn = value; 2264 break; 2265 default: 2266 break; 2267 } 2268 2269 return 0; 2270 } 2271 2272 static int 2273 iscsi_session_get_param(struct iscsi_cls_session *cls_session, 2274 enum iscsi_param param, uint32_t *value) 2275 { 2276 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 2277 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 2278 2279 switch(param) { 2280 case ISCSI_PARAM_INITIAL_R2T_EN: 2281 *value = session->initial_r2t_en; 2282 break; 2283 case ISCSI_PARAM_MAX_R2T: 2284 *value = session->max_r2t; 2285 break; 2286 case ISCSI_PARAM_IMM_DATA_EN: 2287 *value = session->imm_data_en; 2288 break; 2289 case ISCSI_PARAM_FIRST_BURST: 2290 *value = session->first_burst; 2291 break; 2292 case ISCSI_PARAM_MAX_BURST: 2293 *value = session->max_burst; 2294 break; 2295 case ISCSI_PARAM_PDU_INORDER_EN: 2296 *value = session->pdu_inorder_en; 2297 break; 2298 case ISCSI_PARAM_DATASEQ_INORDER_EN: 2299 *value = session->dataseq_inorder_en; 2300 break; 2301 case ISCSI_PARAM_ERL: 2302 *value = session->erl; 2303 break; 2304 case ISCSI_PARAM_IFMARKER_EN: 2305 *value = session->ifmarker_en; 2306 break; 2307 case ISCSI_PARAM_OFMARKER_EN: 2308 *value = session->ofmarker_en; 2309 break; 2310 default: 2311 return -EINVAL; 2312 } 2313 2314 return 0; 2315 } 2316 2317 static int 2318 iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn, 2319 enum iscsi_param param, uint32_t *value) 2320 { 2321 struct iscsi_conn *conn = cls_conn->dd_data; 2322 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2323 struct inet_sock *inet; 2324 2325 switch(param) { 2326 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2327 *value = conn->max_recv_dlength; 2328 break; 2329 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2330 *value = conn->max_xmit_dlength; 2331 break; 2332 case ISCSI_PARAM_HDRDGST_EN: 2333 *value = conn->hdrdgst_en; 2334 break; 2335 case ISCSI_PARAM_DATADGST_EN: 2336 *value = conn->datadgst_en; 2337 break; 2338 case ISCSI_PARAM_CONN_PORT: 2339 mutex_lock(&conn->xmitmutex); 2340 if (!tcp_conn->sock) { 2341 mutex_unlock(&conn->xmitmutex); 2342 return -EINVAL; 2343 } 2344 2345 inet = inet_sk(tcp_conn->sock->sk); 2346 *value = be16_to_cpu(inet->dport); 2347 mutex_unlock(&conn->xmitmutex); 2348 case ISCSI_PARAM_EXP_STATSN: 2349 *value = conn->exp_statsn; 2350 break; 2351 default: 2352 return -EINVAL; 2353 } 2354 2355 return 0; 2356 } 2357 2358 static int 2359 iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn, 2360 enum iscsi_param param, char *buf) 2361 { 2362 struct iscsi_conn *conn = cls_conn->dd_data; 2363 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2364 struct sock *sk; 2365 struct inet_sock *inet; 2366 struct ipv6_pinfo *np; 2367 int len = 0; 2368 2369 switch (param) { 2370 case ISCSI_PARAM_CONN_ADDRESS: 2371 mutex_lock(&conn->xmitmutex); 2372 if (!tcp_conn->sock) { 2373 mutex_unlock(&conn->xmitmutex); 2374 return -EINVAL; 2375 } 2376 2377 sk = tcp_conn->sock->sk; 2378 if (sk->sk_family == PF_INET) { 2379 inet = inet_sk(sk); 2380 len = sprintf(buf, "%u.%u.%u.%u\n", 2381 NIPQUAD(inet->daddr)); 2382 } else { 2383 np = inet6_sk(sk); 2384 len = sprintf(buf, 2385 "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n", 2386 NIP6(np->daddr)); 2387 } 2388 mutex_unlock(&conn->xmitmutex); 2389 break; 2390 default: 2391 return -EINVAL; 2392 } 2393 2394 return len; 2395 } 2396 2397 static void 2398 iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 2399 { 2400 struct iscsi_conn *conn = cls_conn->dd_data; 2401 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2402 2403 stats->txdata_octets = conn->txdata_octets; 2404 stats->rxdata_octets = conn->rxdata_octets; 2405 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 2406 stats->dataout_pdus = conn->dataout_pdus_cnt; 2407 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 2408 stats->datain_pdus = conn->datain_pdus_cnt; 2409 stats->r2t_pdus = conn->r2t_pdus_cnt; 2410 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 2411 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2412 stats->custom_length = 3; 2413 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 2414 stats->custom[0].value = tcp_conn->sendpage_failures_cnt; 2415 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 2416 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt; 2417 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 2418 stats->custom[2].value = conn->eh_abort_cnt; 2419 } 2420 2421 static struct iscsi_cls_session * 2422 iscsi_tcp_session_create(struct iscsi_transport *iscsit, 2423 struct scsi_transport_template *scsit, 2424 uint32_t initial_cmdsn, uint32_t *hostno) 2425 { 2426 struct iscsi_cls_session *cls_session; 2427 struct iscsi_session *session; 2428 uint32_t hn; 2429 int cmd_i; 2430 2431 cls_session = iscsi_session_setup(iscsit, scsit, 2432 sizeof(struct iscsi_tcp_cmd_task), 2433 sizeof(struct iscsi_tcp_mgmt_task), 2434 initial_cmdsn, &hn); 2435 if (!cls_session) 2436 return NULL; 2437 *hostno = hn; 2438 2439 session = class_to_transport_session(cls_session); 2440 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 2441 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 2442 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data; 2443 2444 ctask->hdr = &tcp_ctask->hdr; 2445 } 2446 2447 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) { 2448 struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i]; 2449 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data; 2450 2451 mtask->hdr = &tcp_mtask->hdr; 2452 } 2453 2454 if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session))) 2455 goto r2tpool_alloc_fail; 2456 2457 return cls_session; 2458 2459 r2tpool_alloc_fail: 2460 iscsi_session_teardown(cls_session); 2461 return NULL; 2462 } 2463 2464 static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session) 2465 { 2466 iscsi_r2tpool_free(class_to_transport_session(cls_session)); 2467 iscsi_session_teardown(cls_session); 2468 } 2469 2470 static struct scsi_host_template iscsi_sht = { 2471 .name = "iSCSI Initiator over TCP/IP, v" 2472 ISCSI_TCP_VERSION, 2473 .queuecommand = iscsi_queuecommand, 2474 .change_queue_depth = iscsi_change_queue_depth, 2475 .can_queue = ISCSI_XMIT_CMDS_MAX - 1, 2476 .sg_tablesize = ISCSI_SG_TABLESIZE, 2477 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 2478 .eh_abort_handler = iscsi_eh_abort, 2479 .eh_host_reset_handler = iscsi_eh_host_reset, 2480 .use_clustering = DISABLE_CLUSTERING, 2481 .proc_name = "iscsi_tcp", 2482 .this_id = -1, 2483 }; 2484 2485 static struct iscsi_transport iscsi_tcp_transport = { 2486 .owner = THIS_MODULE, 2487 .name = "tcp", 2488 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 2489 | CAP_DATADGST, 2490 .param_mask = ISCSI_MAX_RECV_DLENGTH | 2491 ISCSI_MAX_XMIT_DLENGTH | 2492 ISCSI_HDRDGST_EN | 2493 ISCSI_DATADGST_EN | 2494 ISCSI_INITIAL_R2T_EN | 2495 ISCSI_MAX_R2T | 2496 ISCSI_IMM_DATA_EN | 2497 ISCSI_FIRST_BURST | 2498 ISCSI_MAX_BURST | 2499 ISCSI_PDU_INORDER_EN | 2500 ISCSI_DATASEQ_INORDER_EN | 2501 ISCSI_ERL | 2502 ISCSI_CONN_PORT | 2503 ISCSI_CONN_ADDRESS | 2504 ISCSI_EXP_STATSN, 2505 .host_template = &iscsi_sht, 2506 .conndata_size = sizeof(struct iscsi_conn), 2507 .max_conn = 1, 2508 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN, 2509 /* session management */ 2510 .create_session = iscsi_tcp_session_create, 2511 .destroy_session = iscsi_tcp_session_destroy, 2512 /* connection management */ 2513 .create_conn = iscsi_tcp_conn_create, 2514 .bind_conn = iscsi_tcp_conn_bind, 2515 .destroy_conn = iscsi_tcp_conn_destroy, 2516 .set_param = iscsi_conn_set_param, 2517 .get_conn_param = iscsi_conn_get_param, 2518 .get_conn_str_param = iscsi_conn_get_str_param, 2519 .get_session_param = iscsi_session_get_param, 2520 .start_conn = iscsi_conn_start, 2521 .stop_conn = iscsi_conn_stop, 2522 /* these are called as part of conn recovery */ 2523 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx, 2524 .terminate_conn = iscsi_tcp_terminate_conn, 2525 /* IO */ 2526 .send_pdu = iscsi_conn_send_pdu, 2527 .get_stats = iscsi_conn_get_stats, 2528 .init_cmd_task = iscsi_tcp_cmd_init, 2529 .init_mgmt_task = iscsi_tcp_mgmt_init, 2530 .xmit_cmd_task = iscsi_tcp_ctask_xmit, 2531 .xmit_mgmt_task = iscsi_tcp_mtask_xmit, 2532 .cleanup_cmd_task = iscsi_tcp_cleanup_ctask, 2533 /* recovery */ 2534 .session_recovery_timedout = iscsi_session_recovery_timedout, 2535 }; 2536 2537 static int __init 2538 iscsi_tcp_init(void) 2539 { 2540 if (iscsi_max_lun < 1) { 2541 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n", 2542 iscsi_max_lun); 2543 return -EINVAL; 2544 } 2545 iscsi_tcp_transport.max_lun = iscsi_max_lun; 2546 2547 if (!iscsi_register_transport(&iscsi_tcp_transport)) 2548 return -ENODEV; 2549 2550 return 0; 2551 } 2552 2553 static void __exit 2554 iscsi_tcp_exit(void) 2555 { 2556 iscsi_unregister_transport(&iscsi_tcp_transport); 2557 } 2558 2559 module_init(iscsi_tcp_init); 2560 module_exit(iscsi_tcp_exit); 2561