1 /* 2 * iSCSI Initiator over TCP/IP Data-Path 3 * 4 * Copyright (C) 2004 Dmitry Yusupov 5 * Copyright (C) 2004 Alex Aizman 6 * Copyright (C) 2005 Mike Christie 7 * maintained by open-iscsi@googlegroups.com 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published 11 * by the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * See the file COPYING included with this distribution for more details. 20 * 21 * Credits: 22 * Christoph Hellwig 23 * FUJITA Tomonori 24 * Arne Redlich 25 * Zhenyu Wang 26 */ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/inet.h> 31 #include <linux/blkdev.h> 32 #include <linux/crypto.h> 33 #include <linux/delay.h> 34 #include <linux/kfifo.h> 35 #include <linux/scatterlist.h> 36 #include <linux/mutex.h> 37 #include <net/tcp.h> 38 #include <scsi/scsi_cmnd.h> 39 #include <scsi/scsi_device.h> 40 #include <scsi/scsi_eh.h> 41 #include <scsi/scsi_request.h> 42 #include <scsi/scsi_tcq.h> 43 #include <scsi/scsi_host.h> 44 #include <scsi/scsi.h> 45 #include <scsi/scsi_transport_iscsi.h> 46 47 #include "iscsi_tcp.h" 48 49 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 50 "Alex Aizman <itn780@yahoo.com>"); 51 MODULE_DESCRIPTION("iSCSI/TCP data-path"); 52 MODULE_LICENSE("GPL"); 53 MODULE_VERSION("0:4.445"); 54 /* #define DEBUG_TCP */ 55 /* #define DEBUG_SCSI */ 56 #define DEBUG_ASSERT 57 58 #ifdef DEBUG_TCP 59 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt) 60 #else 61 #define debug_tcp(fmt...) 62 #endif 63 64 #ifdef DEBUG_SCSI 65 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt) 66 #else 67 #define debug_scsi(fmt...) 68 #endif 69 70 #ifndef DEBUG_ASSERT 71 #ifdef BUG_ON 72 #undef BUG_ON 73 #endif 74 #define BUG_ON(expr) 75 #endif 76 77 #define INVALID_SN_DELTA 0xffff 78 79 static unsigned int iscsi_max_lun = 512; 80 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 81 82 /* global data */ 83 static kmem_cache_t *taskcache; 84 85 static inline void 86 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size) 87 { 88 sg_init_one(&ibuf->sg, (u8 *)vbuf, size); 89 ibuf->sent = 0; 90 ibuf->use_sendmsg = 0; 91 } 92 93 static inline void 94 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size) 95 { 96 ibuf->sg.page = virt_to_page(vbuf); 97 ibuf->sg.offset = offset_in_page(vbuf); 98 ibuf->sg.length = size; 99 ibuf->sent = 0; 100 ibuf->use_sendmsg = 1; 101 } 102 103 static inline void 104 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg) 105 { 106 ibuf->sg.page = sg->page; 107 ibuf->sg.offset = sg->offset; 108 ibuf->sg.length = sg->length; 109 /* 110 * Fastpath: sg element fits into single page 111 */ 112 if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page)) 113 ibuf->use_sendmsg = 0; 114 else 115 ibuf->use_sendmsg = 1; 116 ibuf->sent = 0; 117 } 118 119 static inline int 120 iscsi_buf_left(struct iscsi_buf *ibuf) 121 { 122 int rc; 123 124 rc = ibuf->sg.length - ibuf->sent; 125 BUG_ON(rc < 0); 126 return rc; 127 } 128 129 static inline void 130 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf, 131 u8* crc) 132 { 133 crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc); 134 buf->sg.length += sizeof(uint32_t); 135 } 136 137 static void 138 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err) 139 { 140 struct iscsi_session *session = conn->session; 141 unsigned long flags; 142 143 spin_lock_irqsave(&session->lock, flags); 144 if (session->conn_cnt == 1 || session->leadconn == conn) 145 session->state = ISCSI_STATE_FAILED; 146 spin_unlock_irqrestore(&session->lock, flags); 147 set_bit(SUSPEND_BIT, &conn->suspend_tx); 148 set_bit(SUSPEND_BIT, &conn->suspend_rx); 149 iscsi_conn_error(conn->cls_conn, err); 150 } 151 152 static inline int 153 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr) 154 { 155 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn); 156 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn); 157 158 if (max_cmdsn < exp_cmdsn -1 && 159 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA) 160 return ISCSI_ERR_MAX_CMDSN; 161 if (max_cmdsn > session->max_cmdsn || 162 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA) 163 session->max_cmdsn = max_cmdsn; 164 if (exp_cmdsn > session->exp_cmdsn || 165 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA) 166 session->exp_cmdsn = exp_cmdsn; 167 168 return 0; 169 } 170 171 static inline int 172 iscsi_hdr_extract(struct iscsi_conn *conn) 173 { 174 struct sk_buff *skb = conn->in.skb; 175 176 if (conn->in.copy >= conn->hdr_size && 177 conn->in_progress == IN_PROGRESS_WAIT_HEADER) { 178 /* 179 * Zero-copy PDU Header: using connection context 180 * to store header pointer. 181 */ 182 if (skb_shinfo(skb)->frag_list == NULL && 183 !skb_shinfo(skb)->nr_frags) 184 conn->in.hdr = (struct iscsi_hdr *) 185 ((char*)skb->data + conn->in.offset); 186 else { 187 /* ignoring return code since we checked 188 * in.copy before */ 189 skb_copy_bits(skb, conn->in.offset, 190 &conn->hdr, conn->hdr_size); 191 conn->in.hdr = &conn->hdr; 192 } 193 conn->in.offset += conn->hdr_size; 194 conn->in.copy -= conn->hdr_size; 195 } else { 196 int hdr_remains; 197 int copylen; 198 199 /* 200 * PDU header scattered across SKB's, 201 * copying it... This'll happen quite rarely. 202 */ 203 204 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER) 205 conn->in.hdr_offset = 0; 206 207 hdr_remains = conn->hdr_size - conn->in.hdr_offset; 208 BUG_ON(hdr_remains <= 0); 209 210 copylen = min(conn->in.copy, hdr_remains); 211 skb_copy_bits(skb, conn->in.offset, 212 (char*)&conn->hdr + conn->in.hdr_offset, copylen); 213 214 debug_tcp("PDU gather offset %d bytes %d in.offset %d " 215 "in.copy %d\n", conn->in.hdr_offset, copylen, 216 conn->in.offset, conn->in.copy); 217 218 conn->in.offset += copylen; 219 conn->in.copy -= copylen; 220 if (copylen < hdr_remains) { 221 conn->in_progress = IN_PROGRESS_HEADER_GATHER; 222 conn->in.hdr_offset += copylen; 223 return -EAGAIN; 224 } 225 conn->in.hdr = &conn->hdr; 226 conn->discontiguous_hdr_cnt++; 227 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 228 } 229 230 return 0; 231 } 232 233 static inline void 234 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 235 { 236 struct scsi_cmnd *sc = ctask->sc; 237 struct iscsi_session *session = conn->session; 238 239 spin_lock(&session->lock); 240 if (unlikely(!sc)) { 241 spin_unlock(&session->lock); 242 return; 243 } 244 if (sc->sc_data_direction == DMA_TO_DEVICE) { 245 struct iscsi_data_task *dtask, *n; 246 /* WRITE: cleanup Data-Out's if any */ 247 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) { 248 list_del(&dtask->item); 249 mempool_free(dtask, ctask->datapool); 250 } 251 } 252 ctask->xmstate = XMSTATE_IDLE; 253 ctask->r2t = NULL; 254 ctask->sc = NULL; 255 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*)); 256 spin_unlock(&session->lock); 257 } 258 259 /** 260 * iscsi_cmd_rsp - SCSI Command Response processing 261 * @conn: iscsi connection 262 * @ctask: scsi command task 263 **/ 264 static int 265 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 266 { 267 int rc; 268 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr; 269 struct iscsi_session *session = conn->session; 270 struct scsi_cmnd *sc = ctask->sc; 271 272 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 273 if (rc) { 274 sc->result = (DID_ERROR << 16); 275 goto out; 276 } 277 278 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 279 280 sc->result = (DID_OK << 16) | rhdr->cmd_status; 281 282 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) { 283 sc->result = (DID_ERROR << 16); 284 goto out; 285 } 286 287 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) { 288 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE); 289 290 memcpy(sc->sense_buffer, conn->data + 2, sensecopy); 291 debug_scsi("copied %d bytes of sense\n", sensecopy); 292 } 293 294 if (sc->sc_data_direction == DMA_TO_DEVICE) 295 goto out; 296 297 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) { 298 int res_count = be32_to_cpu(rhdr->residual_count); 299 300 if (res_count > 0 && res_count <= sc->request_bufflen) 301 sc->resid = res_count; 302 else 303 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; 304 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW) 305 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; 306 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) 307 sc->resid = be32_to_cpu(rhdr->residual_count); 308 309 out: 310 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 311 (long)sc, sc->result, ctask->itt); 312 conn->scsirsp_pdus_cnt++; 313 iscsi_ctask_cleanup(conn, ctask); 314 sc->scsi_done(sc); 315 return rc; 316 } 317 318 /** 319 * iscsi_data_rsp - SCSI Data-In Response processing 320 * @conn: iscsi connection 321 * @ctask: scsi command task 322 **/ 323 static int 324 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 325 { 326 int rc; 327 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr; 328 struct iscsi_session *session = conn->session; 329 int datasn = be32_to_cpu(rhdr->datasn); 330 331 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 332 if (rc) 333 return rc; 334 /* 335 * setup Data-In byte counter (gets decremented..) 336 */ 337 ctask->data_count = conn->in.datalen; 338 339 if (conn->in.datalen == 0) 340 return 0; 341 342 if (ctask->datasn != datasn) 343 return ISCSI_ERR_DATASN; 344 345 ctask->datasn++; 346 347 ctask->data_offset = be32_to_cpu(rhdr->offset); 348 if (ctask->data_offset + conn->in.datalen > ctask->total_length) 349 return ISCSI_ERR_DATA_OFFSET; 350 351 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) { 352 struct scsi_cmnd *sc = ctask->sc; 353 354 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 355 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) { 356 int res_count = be32_to_cpu(rhdr->residual_count); 357 358 if (res_count > 0 && 359 res_count <= sc->request_bufflen) { 360 sc->resid = res_count; 361 sc->result = (DID_OK << 16) | rhdr->cmd_status; 362 } else 363 sc->result = (DID_BAD_TARGET << 16) | 364 rhdr->cmd_status; 365 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) { 366 sc->resid = be32_to_cpu(rhdr->residual_count); 367 sc->result = (DID_OK << 16) | rhdr->cmd_status; 368 } else 369 sc->result = (DID_OK << 16) | rhdr->cmd_status; 370 } 371 372 conn->datain_pdus_cnt++; 373 return 0; 374 } 375 376 /** 377 * iscsi_solicit_data_init - initialize first Data-Out 378 * @conn: iscsi connection 379 * @ctask: scsi command task 380 * @r2t: R2T info 381 * 382 * Notes: 383 * Initialize first Data-Out within this R2T sequence and finds 384 * proper data_offset within this SCSI command. 385 * 386 * This function is called with connection lock taken. 387 **/ 388 static void 389 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 390 struct iscsi_r2t_info *r2t) 391 { 392 struct iscsi_data *hdr; 393 struct iscsi_data_task *dtask; 394 struct scsi_cmnd *sc = ctask->sc; 395 396 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 397 BUG_ON(!dtask); 398 hdr = &dtask->hdr; 399 memset(hdr, 0, sizeof(struct iscsi_data)); 400 hdr->ttt = r2t->ttt; 401 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 402 r2t->solicit_datasn++; 403 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 404 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 405 hdr->itt = ctask->hdr.itt; 406 hdr->exp_statsn = r2t->exp_statsn; 407 hdr->offset = cpu_to_be32(r2t->data_offset); 408 if (r2t->data_length > conn->max_xmit_dlength) { 409 hton24(hdr->dlength, conn->max_xmit_dlength); 410 r2t->data_count = conn->max_xmit_dlength; 411 hdr->flags = 0; 412 } else { 413 hton24(hdr->dlength, r2t->data_length); 414 r2t->data_count = r2t->data_length; 415 hdr->flags = ISCSI_FLAG_CMD_FINAL; 416 } 417 conn->dataout_pdus_cnt++; 418 419 r2t->sent = 0; 420 421 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 422 sizeof(struct iscsi_hdr)); 423 424 r2t->dtask = dtask; 425 426 if (sc->use_sg) { 427 int i, sg_count = 0; 428 struct scatterlist *sg = sc->request_buffer; 429 430 r2t->sg = NULL; 431 for (i = 0; i < sc->use_sg; i++, sg += 1) { 432 /* FIXME: prefetch ? */ 433 if (sg_count + sg->length > r2t->data_offset) { 434 int page_offset; 435 436 /* sg page found! */ 437 438 /* offset within this page */ 439 page_offset = r2t->data_offset - sg_count; 440 441 /* fill in this buffer */ 442 iscsi_buf_init_sg(&r2t->sendbuf, sg); 443 r2t->sendbuf.sg.offset += page_offset; 444 r2t->sendbuf.sg.length -= page_offset; 445 446 /* xmit logic will continue with next one */ 447 r2t->sg = sg + 1; 448 break; 449 } 450 sg_count += sg->length; 451 } 452 BUG_ON(r2t->sg == NULL); 453 } else 454 iscsi_buf_init_iov(&ctask->sendbuf, 455 (char*)sc->request_buffer + r2t->data_offset, 456 r2t->data_count); 457 458 list_add(&dtask->item, &ctask->dataqueue); 459 } 460 461 /** 462 * iscsi_r2t_rsp - iSCSI R2T Response processing 463 * @conn: iscsi connection 464 * @ctask: scsi command task 465 **/ 466 static int 467 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 468 { 469 struct iscsi_r2t_info *r2t; 470 struct iscsi_session *session = conn->session; 471 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr; 472 int r2tsn = be32_to_cpu(rhdr->r2tsn); 473 int rc; 474 475 if (conn->in.ahslen) 476 return ISCSI_ERR_AHSLEN; 477 478 if (conn->in.datalen) 479 return ISCSI_ERR_DATALEN; 480 481 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn) 482 return ISCSI_ERR_R2TSN; 483 484 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 485 if (rc) 486 return rc; 487 488 /* FIXME: use R2TSN to detect missing R2T */ 489 490 /* fill-in new R2T associated with the task */ 491 spin_lock(&session->lock); 492 if (!ctask->sc || ctask->mtask || 493 session->state != ISCSI_STATE_LOGGED_IN) { 494 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in " 495 "recovery...\n", ctask->itt); 496 spin_unlock(&session->lock); 497 return 0; 498 } 499 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 500 BUG_ON(!rc); 501 502 r2t->exp_statsn = rhdr->statsn; 503 r2t->data_length = be32_to_cpu(rhdr->data_length); 504 if (r2t->data_length == 0 || 505 r2t->data_length > session->max_burst) { 506 spin_unlock(&session->lock); 507 return ISCSI_ERR_DATALEN; 508 } 509 510 r2t->data_offset = be32_to_cpu(rhdr->data_offset); 511 if (r2t->data_offset + r2t->data_length > ctask->total_length) { 512 spin_unlock(&session->lock); 513 return ISCSI_ERR_DATALEN; 514 } 515 516 r2t->ttt = rhdr->ttt; /* no flip */ 517 r2t->solicit_datasn = 0; 518 519 iscsi_solicit_data_init(conn, ctask, r2t); 520 521 ctask->exp_r2tsn = r2tsn + 1; 522 ctask->xmstate |= XMSTATE_SOL_HDR; 523 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 524 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*)); 525 526 scsi_queue_work(session->host, &conn->xmitwork); 527 conn->r2t_pdus_cnt++; 528 spin_unlock(&session->lock); 529 530 return 0; 531 } 532 533 static int 534 iscsi_hdr_recv(struct iscsi_conn *conn) 535 { 536 int rc = 0; 537 struct iscsi_hdr *hdr; 538 struct iscsi_cmd_task *ctask; 539 struct iscsi_session *session = conn->session; 540 uint32_t cdgst, rdgst = 0; 541 542 hdr = conn->in.hdr; 543 544 /* verify PDU length */ 545 conn->in.datalen = ntoh24(hdr->dlength); 546 if (conn->in.datalen > conn->max_recv_dlength) { 547 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n", 548 conn->in.datalen, conn->max_recv_dlength); 549 return ISCSI_ERR_DATALEN; 550 } 551 conn->data_copied = 0; 552 553 /* read AHS */ 554 conn->in.ahslen = hdr->hlength * 4; 555 conn->in.offset += conn->in.ahslen; 556 conn->in.copy -= conn->in.ahslen; 557 if (conn->in.copy < 0) { 558 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length " 559 "%d bytes\n", conn->in.ahslen); 560 return ISCSI_ERR_AHSLEN; 561 } 562 563 /* calculate read padding */ 564 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1); 565 if (conn->in.padding) { 566 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding; 567 debug_scsi("read padding %d bytes\n", conn->in.padding); 568 } 569 570 if (conn->hdrdgst_en) { 571 struct scatterlist sg; 572 573 sg_init_one(&sg, (u8 *)hdr, 574 sizeof(struct iscsi_hdr) + conn->in.ahslen); 575 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 576 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 577 conn->in.ahslen); 578 if (cdgst != rdgst) { 579 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error " 580 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst, 581 cdgst); 582 return ISCSI_ERR_HDR_DGST; 583 } 584 } 585 586 /* save opcode for later */ 587 conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK; 588 589 /* verify itt (itt encoding: age+cid+itt) */ 590 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 591 if ((hdr->itt & AGE_MASK) != 592 (session->age << AGE_SHIFT)) { 593 printk(KERN_ERR "iscsi_tcp: received itt %x expected " 594 "session age (%x)\n", hdr->itt, 595 session->age & AGE_MASK); 596 return ISCSI_ERR_BAD_ITT; 597 } 598 599 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) { 600 printk(KERN_ERR "iscsi_tcp: received itt %x, expected " 601 "CID (%x)\n", hdr->itt, conn->id); 602 return ISCSI_ERR_BAD_ITT; 603 } 604 conn->in.itt = hdr->itt & ITT_MASK; 605 } else 606 conn->in.itt = hdr->itt; 607 608 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n", 609 hdr->opcode, conn->in.offset, conn->in.copy, 610 conn->in.ahslen, conn->in.datalen); 611 612 if (conn->in.itt < session->cmds_max) { 613 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt]; 614 615 if (!ctask->sc) { 616 printk(KERN_INFO "iscsi_tcp: dropping ctask with " 617 "itt 0x%x\n", ctask->itt); 618 conn->in.datalen = 0; /* force drop */ 619 return 0; 620 } 621 622 if (ctask->sc->SCp.phase != session->age) { 623 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, " 624 "expected %d\n", ctask->sc->SCp.phase, 625 session->age); 626 return ISCSI_ERR_SESSION_FAILED; 627 } 628 629 conn->in.ctask = ctask; 630 631 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n", 632 hdr->opcode, conn->id, (long)ctask->sc, 633 ctask->itt, conn->in.datalen); 634 635 switch(conn->in.opcode) { 636 case ISCSI_OP_SCSI_CMD_RSP: 637 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 638 if (!conn->in.datalen) 639 rc = iscsi_cmd_rsp(conn, ctask); 640 else 641 /* 642 * got sense or response data; copying PDU 643 * Header to the connection's header 644 * placeholder 645 */ 646 memcpy(&conn->hdr, hdr, 647 sizeof(struct iscsi_hdr)); 648 break; 649 case ISCSI_OP_SCSI_DATA_IN: 650 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 651 /* save flags for non-exceptional status */ 652 conn->in.flags = hdr->flags; 653 /* save cmd_status for sense data */ 654 conn->in.cmd_status = 655 ((struct iscsi_data_rsp*)hdr)->cmd_status; 656 rc = iscsi_data_rsp(conn, ctask); 657 break; 658 case ISCSI_OP_R2T: 659 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 660 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE) 661 rc = iscsi_r2t_rsp(conn, ctask); 662 else 663 rc = ISCSI_ERR_PROTO; 664 break; 665 default: 666 rc = ISCSI_ERR_BAD_OPCODE; 667 break; 668 } 669 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET && 670 conn->in.itt < ISCSI_MGMT_ITT_OFFSET + 671 session->mgmtpool_max) { 672 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *) 673 session->mgmt_cmds[conn->in.itt - 674 ISCSI_MGMT_ITT_OFFSET]; 675 676 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n", 677 conn->in.opcode, conn->id, mtask->itt, 678 conn->in.datalen); 679 680 switch(conn->in.opcode) { 681 case ISCSI_OP_LOGIN_RSP: 682 case ISCSI_OP_TEXT_RSP: 683 case ISCSI_OP_LOGOUT_RSP: 684 rc = iscsi_check_assign_cmdsn(session, 685 (struct iscsi_nopin*)hdr); 686 if (rc) 687 break; 688 689 if (!conn->in.datalen) { 690 rc = iscsi_recv_pdu(conn->cls_conn, hdr, 691 NULL, 0); 692 if (conn->login_mtask != mtask) { 693 spin_lock(&session->lock); 694 __kfifo_put(session->mgmtpool.queue, 695 (void*)&mtask, sizeof(void*)); 696 spin_unlock(&session->lock); 697 } 698 } 699 break; 700 case ISCSI_OP_SCSI_TMFUNC_RSP: 701 rc = iscsi_check_assign_cmdsn(session, 702 (struct iscsi_nopin*)hdr); 703 if (rc) 704 break; 705 706 if (conn->in.datalen || conn->in.ahslen) { 707 rc = ISCSI_ERR_PROTO; 708 break; 709 } 710 conn->tmfrsp_pdus_cnt++; 711 spin_lock(&session->lock); 712 if (conn->tmabort_state == TMABORT_INITIAL) { 713 __kfifo_put(session->mgmtpool.queue, 714 (void*)&mtask, sizeof(void*)); 715 conn->tmabort_state = 716 ((struct iscsi_tm_rsp *)hdr)-> 717 response == ISCSI_TMF_RSP_COMPLETE ? 718 TMABORT_SUCCESS:TMABORT_FAILED; 719 /* unblock eh_abort() */ 720 wake_up(&conn->ehwait); 721 } 722 spin_unlock(&session->lock); 723 break; 724 case ISCSI_OP_NOOP_IN: 725 if (hdr->ttt != ISCSI_RESERVED_TAG) { 726 rc = ISCSI_ERR_PROTO; 727 break; 728 } 729 rc = iscsi_check_assign_cmdsn(session, 730 (struct iscsi_nopin*)hdr); 731 if (rc) 732 break; 733 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1; 734 735 if (!conn->in.datalen) { 736 struct iscsi_mgmt_task *mtask; 737 738 rc = iscsi_recv_pdu(conn->cls_conn, hdr, 739 NULL, 0); 740 mtask = (struct iscsi_mgmt_task *) 741 session->mgmt_cmds[conn->in.itt - 742 ISCSI_MGMT_ITT_OFFSET]; 743 if (conn->login_mtask != mtask) { 744 spin_lock(&session->lock); 745 __kfifo_put(session->mgmtpool.queue, 746 (void*)&mtask, sizeof(void*)); 747 spin_unlock(&session->lock); 748 } 749 } 750 break; 751 default: 752 rc = ISCSI_ERR_BAD_OPCODE; 753 break; 754 } 755 } else if (conn->in.itt == ISCSI_RESERVED_TAG) { 756 switch(conn->in.opcode) { 757 case ISCSI_OP_NOOP_IN: 758 if (!conn->in.datalen) { 759 rc = iscsi_check_assign_cmdsn(session, 760 (struct iscsi_nopin*)hdr); 761 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG) 762 rc = iscsi_recv_pdu(conn->cls_conn, 763 hdr, NULL, 0); 764 } else 765 rc = ISCSI_ERR_PROTO; 766 break; 767 case ISCSI_OP_REJECT: 768 /* we need sth like iscsi_reject_rsp()*/ 769 case ISCSI_OP_ASYNC_EVENT: 770 /* we need sth like iscsi_async_event_rsp() */ 771 rc = ISCSI_ERR_BAD_OPCODE; 772 break; 773 default: 774 rc = ISCSI_ERR_BAD_OPCODE; 775 break; 776 } 777 } else 778 rc = ISCSI_ERR_BAD_ITT; 779 780 return rc; 781 } 782 783 /** 784 * iscsi_ctask_copy - copy skb bits to the destanation cmd task 785 * @conn: iscsi connection 786 * @ctask: scsi command task 787 * @buf: buffer to copy to 788 * @buf_size: size of buffer 789 * @offset: offset within the buffer 790 * 791 * Notes: 792 * The function calls skb_copy_bits() and updates per-connection and 793 * per-cmd byte counters. 794 * 795 * Read counters (in bytes): 796 * 797 * conn->in.offset offset within in progress SKB 798 * conn->in.copy left to copy from in progress SKB 799 * including padding 800 * conn->in.copied copied already from in progress SKB 801 * conn->data_copied copied already from in progress buffer 802 * ctask->sent total bytes sent up to the MidLayer 803 * ctask->data_count left to copy from in progress Data-In 804 * buf_left left to copy from in progress buffer 805 **/ 806 static inline int 807 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 808 void *buf, int buf_size, int offset) 809 { 810 int buf_left = buf_size - (conn->data_copied + offset); 811 int size = min(conn->in.copy, buf_left); 812 int rc; 813 814 size = min(size, ctask->data_count); 815 816 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n", 817 size, conn->in.offset, conn->in.copied); 818 819 BUG_ON(size <= 0); 820 BUG_ON(ctask->sent + size > ctask->total_length); 821 822 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 823 (char*)buf + (offset + conn->data_copied), size); 824 /* must fit into skb->len */ 825 BUG_ON(rc); 826 827 conn->in.offset += size; 828 conn->in.copy -= size; 829 conn->in.copied += size; 830 conn->data_copied += size; 831 ctask->sent += size; 832 ctask->data_count -= size; 833 834 BUG_ON(conn->in.copy < 0); 835 BUG_ON(ctask->data_count < 0); 836 837 if (buf_size != (conn->data_copied + offset)) { 838 if (!ctask->data_count) { 839 BUG_ON(buf_size - conn->data_copied < 0); 840 /* done with this PDU */ 841 return buf_size - conn->data_copied; 842 } 843 return -EAGAIN; 844 } 845 846 /* done with this buffer or with both - PDU and buffer */ 847 conn->data_copied = 0; 848 return 0; 849 } 850 851 /** 852 * iscsi_tcp_copy - copy skb bits to the destanation buffer 853 * @conn: iscsi connection 854 * @buf: buffer to copy to 855 * @buf_size: number of bytes to copy 856 * 857 * Notes: 858 * The function calls skb_copy_bits() and updates per-connection 859 * byte counters. 860 **/ 861 static inline int 862 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size) 863 { 864 int buf_left = buf_size - conn->data_copied; 865 int size = min(conn->in.copy, buf_left); 866 int rc; 867 868 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n", 869 size, conn->in.offset, conn->data_copied); 870 BUG_ON(size <= 0); 871 872 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 873 (char*)buf + conn->data_copied, size); 874 BUG_ON(rc); 875 876 conn->in.offset += size; 877 conn->in.copy -= size; 878 conn->in.copied += size; 879 conn->data_copied += size; 880 881 if (buf_size != conn->data_copied) 882 return -EAGAIN; 883 884 return 0; 885 } 886 887 static inline void 888 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg, 889 int offset, int length) 890 { 891 struct scatterlist temp; 892 893 memcpy(&temp, sg, sizeof(struct scatterlist)); 894 temp.offset = offset; 895 temp.length = length; 896 crypto_digest_update(conn->data_rx_tfm, &temp, 1); 897 } 898 899 static void 900 iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len) 901 { 902 struct scatterlist tmp; 903 904 sg_init_one(&tmp, buf, len); 905 crypto_digest_update(conn->data_rx_tfm, &tmp, 1); 906 } 907 908 static int iscsi_scsi_data_in(struct iscsi_conn *conn) 909 { 910 struct iscsi_cmd_task *ctask = conn->in.ctask; 911 struct scsi_cmnd *sc = ctask->sc; 912 struct scatterlist *sg; 913 int i, offset, rc = 0; 914 915 BUG_ON((void*)ctask != sc->SCp.ptr); 916 917 /* 918 * copying Data-In into the Scsi_Cmnd 919 */ 920 if (!sc->use_sg) { 921 i = ctask->data_count; 922 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer, 923 sc->request_bufflen, ctask->data_offset); 924 if (rc == -EAGAIN) 925 return rc; 926 if (conn->datadgst_en) 927 iscsi_recv_digest_update(conn, sc->request_buffer, i); 928 rc = 0; 929 goto done; 930 } 931 932 offset = ctask->data_offset; 933 sg = sc->request_buffer; 934 935 if (ctask->data_offset) 936 for (i = 0; i < ctask->sg_count; i++) 937 offset -= sg[i].length; 938 /* we've passed through partial sg*/ 939 if (offset < 0) 940 offset = 0; 941 942 for (i = ctask->sg_count; i < sc->use_sg; i++) { 943 char *dest; 944 945 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0); 946 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset, 947 sg[i].length, offset); 948 kunmap_atomic(dest, KM_SOFTIRQ0); 949 if (rc == -EAGAIN) 950 /* continue with the next SKB/PDU */ 951 return rc; 952 if (!rc) { 953 if (conn->datadgst_en) { 954 if (!offset) 955 crypto_digest_update(conn->data_rx_tfm, 956 &sg[i], 1); 957 else 958 partial_sg_digest_update(conn, &sg[i], 959 sg[i].offset + offset, 960 sg[i].length - offset); 961 } 962 offset = 0; 963 ctask->sg_count++; 964 } 965 966 if (!ctask->data_count) { 967 if (rc && conn->datadgst_en) 968 /* 969 * data-in is complete, but buffer not... 970 */ 971 partial_sg_digest_update(conn, &sg[i], 972 sg[i].offset, sg[i].length-rc); 973 rc = 0; 974 break; 975 } 976 977 if (!conn->in.copy) 978 return -EAGAIN; 979 } 980 BUG_ON(ctask->data_count); 981 982 done: 983 /* check for non-exceptional status */ 984 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) { 985 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 986 (long)sc, sc->result, ctask->itt); 987 conn->scsirsp_pdus_cnt++; 988 iscsi_ctask_cleanup(conn, ctask); 989 sc->scsi_done(sc); 990 } 991 992 return rc; 993 } 994 995 static int 996 iscsi_data_recv(struct iscsi_conn *conn) 997 { 998 struct iscsi_session *session = conn->session; 999 int rc = 0; 1000 1001 switch(conn->in.opcode) { 1002 case ISCSI_OP_SCSI_DATA_IN: 1003 rc = iscsi_scsi_data_in(conn); 1004 break; 1005 case ISCSI_OP_SCSI_CMD_RSP: { 1006 /* 1007 * SCSI Sense Data: 1008 * copying the entire Data Segment. 1009 */ 1010 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 1011 rc = -EAGAIN; 1012 goto exit; 1013 } 1014 1015 /* 1016 * check for sense 1017 */ 1018 conn->in.hdr = &conn->hdr; 1019 conn->senselen = (conn->data[0] << 8) | conn->data[1]; 1020 rc = iscsi_cmd_rsp(conn, conn->in.ctask); 1021 if (!rc && conn->datadgst_en) 1022 iscsi_recv_digest_update(conn, conn->data, 1023 conn->in.datalen); 1024 } 1025 break; 1026 case ISCSI_OP_TEXT_RSP: 1027 case ISCSI_OP_LOGIN_RSP: 1028 case ISCSI_OP_NOOP_IN: { 1029 struct iscsi_mgmt_task *mtask = NULL; 1030 1031 if (conn->in.itt != ISCSI_RESERVED_TAG) 1032 mtask = (struct iscsi_mgmt_task *) 1033 session->mgmt_cmds[conn->in.itt - 1034 ISCSI_MGMT_ITT_OFFSET]; 1035 1036 /* 1037 * Collect data segment to the connection's data 1038 * placeholder 1039 */ 1040 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 1041 rc = -EAGAIN; 1042 goto exit; 1043 } 1044 1045 rc = iscsi_recv_pdu(conn->cls_conn, conn->in.hdr, 1046 conn->data, conn->in.datalen); 1047 1048 if (!rc && conn->datadgst_en && 1049 conn->in.opcode != ISCSI_OP_LOGIN_RSP) 1050 iscsi_recv_digest_update(conn, conn->data, 1051 conn->in.datalen); 1052 1053 if (mtask && conn->login_mtask != mtask) { 1054 spin_lock(&session->lock); 1055 __kfifo_put(session->mgmtpool.queue, (void*)&mtask, 1056 sizeof(void*)); 1057 spin_unlock(&session->lock); 1058 } 1059 } 1060 break; 1061 case ISCSI_OP_ASYNC_EVENT: 1062 case ISCSI_OP_REJECT: 1063 default: 1064 BUG_ON(1); 1065 } 1066 exit: 1067 return rc; 1068 } 1069 1070 /** 1071 * iscsi_tcp_data_recv - TCP receive in sendfile fashion 1072 * @rd_desc: read descriptor 1073 * @skb: socket buffer 1074 * @offset: offset in skb 1075 * @len: skb->len - offset 1076 **/ 1077 static int 1078 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 1079 unsigned int offset, size_t len) 1080 { 1081 int rc; 1082 struct iscsi_conn *conn = rd_desc->arg.data; 1083 int processed; 1084 char pad[ISCSI_PAD_LEN]; 1085 struct scatterlist sg; 1086 1087 /* 1088 * Save current SKB and its offset in the corresponding 1089 * connection context. 1090 */ 1091 conn->in.copy = skb->len - offset; 1092 conn->in.offset = offset; 1093 conn->in.skb = skb; 1094 conn->in.len = conn->in.copy; 1095 BUG_ON(conn->in.copy <= 0); 1096 debug_tcp("in %d bytes\n", conn->in.copy); 1097 1098 more: 1099 conn->in.copied = 0; 1100 rc = 0; 1101 1102 if (unlikely(conn->suspend_rx)) { 1103 debug_tcp("conn %d Rx suspended!\n", conn->id); 1104 return 0; 1105 } 1106 1107 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER || 1108 conn->in_progress == IN_PROGRESS_HEADER_GATHER) { 1109 rc = iscsi_hdr_extract(conn); 1110 if (rc) { 1111 if (rc == -EAGAIN) 1112 goto nomore; 1113 else { 1114 iscsi_conn_failure(conn, rc); 1115 return 0; 1116 } 1117 } 1118 1119 /* 1120 * Verify and process incoming PDU header. 1121 */ 1122 rc = iscsi_hdr_recv(conn); 1123 if (!rc && conn->in.datalen) { 1124 if (conn->datadgst_en) { 1125 BUG_ON(!conn->data_rx_tfm); 1126 crypto_digest_init(conn->data_rx_tfm); 1127 } 1128 conn->in_progress = IN_PROGRESS_DATA_RECV; 1129 } else if (rc) { 1130 iscsi_conn_failure(conn, rc); 1131 return 0; 1132 } 1133 } 1134 1135 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) { 1136 uint32_t recv_digest; 1137 debug_tcp("extra data_recv offset %d copy %d\n", 1138 conn->in.offset, conn->in.copy); 1139 skb_copy_bits(conn->in.skb, conn->in.offset, 1140 &recv_digest, 4); 1141 conn->in.offset += 4; 1142 conn->in.copy -= 4; 1143 if (recv_digest != conn->in.datadgst) { 1144 debug_tcp("iscsi_tcp: data digest error!" 1145 "0x%x != 0x%x\n", recv_digest, 1146 conn->in.datadgst); 1147 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1148 return 0; 1149 } else { 1150 debug_tcp("iscsi_tcp: data digest match!" 1151 "0x%x == 0x%x\n", recv_digest, 1152 conn->in.datadgst); 1153 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1154 } 1155 } 1156 1157 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) { 1158 1159 debug_tcp("data_recv offset %d copy %d\n", 1160 conn->in.offset, conn->in.copy); 1161 1162 rc = iscsi_data_recv(conn); 1163 if (rc) { 1164 if (rc == -EAGAIN) { 1165 rd_desc->count = conn->in.datalen - 1166 conn->in.ctask->data_count; 1167 goto again; 1168 } 1169 iscsi_conn_failure(conn, rc); 1170 return 0; 1171 } 1172 conn->in.copy -= conn->in.padding; 1173 conn->in.offset += conn->in.padding; 1174 if (conn->datadgst_en) { 1175 if (conn->in.padding) { 1176 debug_tcp("padding -> %d\n", conn->in.padding); 1177 memset(pad, 0, conn->in.padding); 1178 sg_init_one(&sg, pad, conn->in.padding); 1179 crypto_digest_update(conn->data_rx_tfm, &sg, 1); 1180 } 1181 crypto_digest_final(conn->data_rx_tfm, 1182 (u8 *) & conn->in.datadgst); 1183 debug_tcp("rx digest 0x%x\n", conn->in.datadgst); 1184 conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 1185 } else 1186 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1187 } 1188 1189 debug_tcp("f, processed %d from out of %d padding %d\n", 1190 conn->in.offset - offset, (int)len, conn->in.padding); 1191 BUG_ON(conn->in.offset - offset > len); 1192 1193 if (conn->in.offset - offset != len) { 1194 debug_tcp("continue to process %d bytes\n", 1195 (int)len - (conn->in.offset - offset)); 1196 goto more; 1197 } 1198 1199 nomore: 1200 processed = conn->in.offset - offset; 1201 BUG_ON(processed == 0); 1202 return processed; 1203 1204 again: 1205 processed = conn->in.offset - offset; 1206 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n", 1207 processed, (int)len, (int)rd_desc->count); 1208 BUG_ON(processed == 0); 1209 BUG_ON(processed > len); 1210 1211 conn->rxdata_octets += processed; 1212 return processed; 1213 } 1214 1215 static void 1216 iscsi_tcp_data_ready(struct sock *sk, int flag) 1217 { 1218 struct iscsi_conn *conn = sk->sk_user_data; 1219 read_descriptor_t rd_desc; 1220 1221 read_lock(&sk->sk_callback_lock); 1222 1223 /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */ 1224 rd_desc.arg.data = conn; 1225 rd_desc.count = 0; 1226 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv); 1227 1228 read_unlock(&sk->sk_callback_lock); 1229 } 1230 1231 static void 1232 iscsi_tcp_state_change(struct sock *sk) 1233 { 1234 struct iscsi_conn *conn; 1235 struct iscsi_session *session; 1236 void (*old_state_change)(struct sock *); 1237 1238 read_lock(&sk->sk_callback_lock); 1239 1240 conn = (struct iscsi_conn*)sk->sk_user_data; 1241 session = conn->session; 1242 1243 if ((sk->sk_state == TCP_CLOSE_WAIT || 1244 sk->sk_state == TCP_CLOSE) && 1245 !atomic_read(&sk->sk_rmem_alloc)) { 1246 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n"); 1247 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1248 } 1249 1250 old_state_change = conn->old_state_change; 1251 1252 read_unlock(&sk->sk_callback_lock); 1253 1254 old_state_change(sk); 1255 } 1256 1257 /** 1258 * iscsi_write_space - Called when more output buffer space is available 1259 * @sk: socket space is available for 1260 **/ 1261 static void 1262 iscsi_write_space(struct sock *sk) 1263 { 1264 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 1265 conn->old_write_space(sk); 1266 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 1267 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 1268 scsi_queue_work(conn->session->host, &conn->xmitwork); 1269 } 1270 1271 static void 1272 iscsi_conn_set_callbacks(struct iscsi_conn *conn) 1273 { 1274 struct sock *sk = conn->sock->sk; 1275 1276 /* assign new callbacks */ 1277 write_lock_bh(&sk->sk_callback_lock); 1278 sk->sk_user_data = conn; 1279 conn->old_data_ready = sk->sk_data_ready; 1280 conn->old_state_change = sk->sk_state_change; 1281 conn->old_write_space = sk->sk_write_space; 1282 sk->sk_data_ready = iscsi_tcp_data_ready; 1283 sk->sk_state_change = iscsi_tcp_state_change; 1284 sk->sk_write_space = iscsi_write_space; 1285 write_unlock_bh(&sk->sk_callback_lock); 1286 } 1287 1288 static void 1289 iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1290 { 1291 struct sock *sk = conn->sock->sk; 1292 1293 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1294 write_lock_bh(&sk->sk_callback_lock); 1295 sk->sk_user_data = NULL; 1296 sk->sk_data_ready = conn->old_data_ready; 1297 sk->sk_state_change = conn->old_state_change; 1298 sk->sk_write_space = conn->old_write_space; 1299 sk->sk_no_check = 0; 1300 write_unlock_bh(&sk->sk_callback_lock); 1301 } 1302 1303 /** 1304 * iscsi_send - generic send routine 1305 * @sk: kernel's socket 1306 * @buf: buffer to write from 1307 * @size: actual size to write 1308 * @flags: socket's flags 1309 */ 1310 static inline int 1311 iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags) 1312 { 1313 struct socket *sk = conn->sock; 1314 int offset = buf->sg.offset + buf->sent; 1315 1316 /* 1317 * if we got use_sg=0 or are sending something we kmallocd 1318 * then we did not have to do kmap (kmap returns page_address) 1319 * 1320 * if we got use_sg > 0, but had to drop down, we do not 1321 * set clustering so this should only happen for that 1322 * slab case. 1323 */ 1324 if (buf->use_sendmsg) 1325 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags); 1326 else 1327 return conn->sendpage(sk, buf->sg.page, offset, size, flags); 1328 } 1329 1330 /** 1331 * iscsi_sendhdr - send PDU Header via tcp_sendpage() 1332 * @conn: iscsi connection 1333 * @buf: buffer to write from 1334 * @datalen: lenght of data to be sent after the header 1335 * 1336 * Notes: 1337 * (Tx, Fast Path) 1338 **/ 1339 static inline int 1340 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen) 1341 { 1342 int flags = 0; /* MSG_DONTWAIT; */ 1343 int res, size; 1344 1345 size = buf->sg.length - buf->sent; 1346 BUG_ON(buf->sent + size > buf->sg.length); 1347 if (buf->sent + size != buf->sg.length || datalen) 1348 flags |= MSG_MORE; 1349 1350 res = iscsi_send(conn, buf, size, flags); 1351 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res); 1352 if (res >= 0) { 1353 conn->txdata_octets += res; 1354 buf->sent += res; 1355 if (size != res) 1356 return -EAGAIN; 1357 return 0; 1358 } else if (res == -EAGAIN) { 1359 conn->sendpage_failures_cnt++; 1360 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1361 } else if (res == -EPIPE) 1362 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1363 1364 return res; 1365 } 1366 1367 /** 1368 * iscsi_sendpage - send one page of iSCSI Data-Out. 1369 * @conn: iscsi connection 1370 * @buf: buffer to write from 1371 * @count: remaining data 1372 * @sent: number of bytes sent 1373 * 1374 * Notes: 1375 * (Tx, Fast Path) 1376 **/ 1377 static inline int 1378 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf, 1379 int *count, int *sent) 1380 { 1381 int flags = 0; /* MSG_DONTWAIT; */ 1382 int res, size; 1383 1384 size = buf->sg.length - buf->sent; 1385 BUG_ON(buf->sent + size > buf->sg.length); 1386 if (size > *count) 1387 size = *count; 1388 if (buf->sent + size != buf->sg.length || *count != size) 1389 flags |= MSG_MORE; 1390 1391 res = iscsi_send(conn, buf, size, flags); 1392 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n", 1393 size, buf->sent, *count, *sent, res); 1394 if (res >= 0) { 1395 conn->txdata_octets += res; 1396 buf->sent += res; 1397 *count -= res; 1398 *sent += res; 1399 if (size != res) 1400 return -EAGAIN; 1401 return 0; 1402 } else if (res == -EAGAIN) { 1403 conn->sendpage_failures_cnt++; 1404 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1405 } else if (res == -EPIPE) 1406 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1407 1408 return res; 1409 } 1410 1411 static inline void 1412 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1413 { 1414 BUG_ON(!conn->data_tx_tfm); 1415 crypto_digest_init(conn->data_tx_tfm); 1416 ctask->digest_count = 4; 1417 } 1418 1419 static int 1420 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1421 struct iscsi_buf *buf, uint32_t *digest, int final) 1422 { 1423 int rc = 0; 1424 int sent = 0; 1425 1426 if (final) 1427 crypto_digest_final(conn->data_tx_tfm, (u8*)digest); 1428 1429 iscsi_buf_init_virt(buf, (char*)digest, 4); 1430 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent); 1431 if (rc) { 1432 ctask->datadigest = *digest; 1433 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1434 } else 1435 ctask->digest_count = 4; 1436 return rc; 1437 } 1438 1439 /** 1440 * iscsi_solicit_data_cont - initialize next Data-Out 1441 * @conn: iscsi connection 1442 * @ctask: scsi command task 1443 * @r2t: R2T info 1444 * @left: bytes left to transfer 1445 * 1446 * Notes: 1447 * Initialize next Data-Out within this R2T sequence and continue 1448 * to process next Scatter-Gather element(if any) of this SCSI command. 1449 * 1450 * Called under connection lock. 1451 **/ 1452 static void 1453 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1454 struct iscsi_r2t_info *r2t, int left) 1455 { 1456 struct iscsi_data *hdr; 1457 struct iscsi_data_task *dtask; 1458 struct scsi_cmnd *sc = ctask->sc; 1459 int new_offset; 1460 1461 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1462 BUG_ON(!dtask); 1463 hdr = &dtask->hdr; 1464 memset(hdr, 0, sizeof(struct iscsi_data)); 1465 hdr->ttt = r2t->ttt; 1466 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 1467 r2t->solicit_datasn++; 1468 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1469 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1470 hdr->itt = ctask->hdr.itt; 1471 hdr->exp_statsn = r2t->exp_statsn; 1472 new_offset = r2t->data_offset + r2t->sent; 1473 hdr->offset = cpu_to_be32(new_offset); 1474 if (left > conn->max_xmit_dlength) { 1475 hton24(hdr->dlength, conn->max_xmit_dlength); 1476 r2t->data_count = conn->max_xmit_dlength; 1477 } else { 1478 hton24(hdr->dlength, left); 1479 r2t->data_count = left; 1480 hdr->flags = ISCSI_FLAG_CMD_FINAL; 1481 } 1482 conn->dataout_pdus_cnt++; 1483 1484 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 1485 sizeof(struct iscsi_hdr)); 1486 1487 r2t->dtask = dtask; 1488 1489 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1490 BUG_ON(ctask->bad_sg == r2t->sg); 1491 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1492 r2t->sg += 1; 1493 } else 1494 iscsi_buf_init_iov(&ctask->sendbuf, 1495 (char*)sc->request_buffer + new_offset, 1496 r2t->data_count); 1497 1498 list_add(&dtask->item, &ctask->dataqueue); 1499 } 1500 1501 static void 1502 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1503 { 1504 struct iscsi_data *hdr; 1505 struct iscsi_data_task *dtask; 1506 1507 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1508 BUG_ON(!dtask); 1509 hdr = &dtask->hdr; 1510 memset(hdr, 0, sizeof(struct iscsi_data)); 1511 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG); 1512 hdr->datasn = cpu_to_be32(ctask->unsol_datasn); 1513 ctask->unsol_datasn++; 1514 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1515 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1516 hdr->itt = ctask->hdr.itt; 1517 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn); 1518 hdr->offset = cpu_to_be32(ctask->total_length - 1519 ctask->r2t_data_count - 1520 ctask->unsol_count); 1521 if (ctask->unsol_count > conn->max_xmit_dlength) { 1522 hton24(hdr->dlength, conn->max_xmit_dlength); 1523 ctask->data_count = conn->max_xmit_dlength; 1524 hdr->flags = 0; 1525 } else { 1526 hton24(hdr->dlength, ctask->unsol_count); 1527 ctask->data_count = ctask->unsol_count; 1528 hdr->flags = ISCSI_FLAG_CMD_FINAL; 1529 } 1530 1531 iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr, 1532 sizeof(struct iscsi_hdr)); 1533 1534 list_add(&dtask->item, &ctask->dataqueue); 1535 1536 ctask->dtask = dtask; 1537 } 1538 1539 /** 1540 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 1541 * @conn: iscsi connection 1542 * @ctask: scsi command task 1543 * @sc: scsi command 1544 **/ 1545 static void 1546 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1547 struct scsi_cmnd *sc) 1548 { 1549 struct iscsi_session *session = conn->session; 1550 1551 BUG_ON(__kfifo_len(ctask->r2tqueue)); 1552 1553 ctask->sc = sc; 1554 ctask->conn = conn; 1555 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD; 1556 ctask->hdr.flags = ISCSI_ATTR_SIMPLE; 1557 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun); 1558 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) | 1559 (session->age << AGE_SHIFT); 1560 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen); 1561 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++; 1562 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn); 1563 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len); 1564 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len); 1565 1566 ctask->mtask = NULL; 1567 ctask->sent = 0; 1568 ctask->sg_count = 0; 1569 1570 ctask->total_length = sc->request_bufflen; 1571 1572 if (sc->sc_data_direction == DMA_TO_DEVICE) { 1573 ctask->exp_r2tsn = 0; 1574 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE; 1575 BUG_ON(ctask->total_length == 0); 1576 if (sc->use_sg) { 1577 struct scatterlist *sg = sc->request_buffer; 1578 1579 iscsi_buf_init_sg(&ctask->sendbuf, 1580 &sg[ctask->sg_count++]); 1581 ctask->sg = sg; 1582 ctask->bad_sg = sg + sc->use_sg; 1583 } else { 1584 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer, 1585 sc->request_bufflen); 1586 } 1587 1588 /* 1589 * Write counters: 1590 * 1591 * imm_count bytes to be sent right after 1592 * SCSI PDU Header 1593 * 1594 * unsol_count bytes(as Data-Out) to be sent 1595 * without R2T ack right after 1596 * immediate data 1597 * 1598 * r2t_data_count bytes to be sent via R2T ack's 1599 * 1600 * pad_count bytes to be sent as zero-padding 1601 */ 1602 ctask->imm_count = 0; 1603 ctask->unsol_count = 0; 1604 ctask->unsol_datasn = 0; 1605 ctask->xmstate = XMSTATE_W_HDR; 1606 /* calculate write padding */ 1607 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1); 1608 if (ctask->pad_count) { 1609 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count; 1610 debug_scsi("write padding %d bytes\n", 1611 ctask->pad_count); 1612 ctask->xmstate |= XMSTATE_W_PAD; 1613 } 1614 if (session->imm_data_en) { 1615 if (ctask->total_length >= session->first_burst) 1616 ctask->imm_count = min(session->first_burst, 1617 conn->max_xmit_dlength); 1618 else 1619 ctask->imm_count = min(ctask->total_length, 1620 conn->max_xmit_dlength); 1621 hton24(ctask->hdr.dlength, ctask->imm_count); 1622 ctask->xmstate |= XMSTATE_IMM_DATA; 1623 } else 1624 zero_data(ctask->hdr.dlength); 1625 1626 if (!session->initial_r2t_en) 1627 ctask->unsol_count = min(session->first_burst, 1628 ctask->total_length) - ctask->imm_count; 1629 if (!ctask->unsol_count) 1630 /* No unsolicit Data-Out's */ 1631 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1632 else 1633 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT; 1634 1635 ctask->r2t_data_count = ctask->total_length - 1636 ctask->imm_count - 1637 ctask->unsol_count; 1638 1639 debug_scsi("cmd [itt %x total %d imm %d imm_data %d " 1640 "r2t_data %d]\n", 1641 ctask->itt, ctask->total_length, ctask->imm_count, 1642 ctask->unsol_count, ctask->r2t_data_count); 1643 } else { 1644 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1645 if (sc->sc_data_direction == DMA_FROM_DEVICE) 1646 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ; 1647 ctask->datasn = 0; 1648 ctask->xmstate = XMSTATE_R_HDR; 1649 zero_data(ctask->hdr.dlength); 1650 } 1651 1652 iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr, 1653 sizeof(struct iscsi_hdr)); 1654 conn->scsicmd_pdus_cnt++; 1655 } 1656 1657 /** 1658 * iscsi_mtask_xmit - xmit management(immediate) task 1659 * @conn: iscsi connection 1660 * @mtask: task management task 1661 * 1662 * Notes: 1663 * The function can return -EAGAIN in which case caller must 1664 * call it again later, or recover. '0' return code means successful 1665 * xmit. 1666 * 1667 * Management xmit state machine consists of two states: 1668 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress 1669 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress 1670 **/ 1671 static int 1672 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask) 1673 { 1674 1675 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n", 1676 conn->id, mtask->xmstate, mtask->itt); 1677 1678 if (mtask->xmstate & XMSTATE_IMM_HDR) { 1679 mtask->xmstate &= ~XMSTATE_IMM_HDR; 1680 if (mtask->data_count) 1681 mtask->xmstate |= XMSTATE_IMM_DATA; 1682 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 1683 conn->stop_stage != STOP_CONN_RECOVER && 1684 conn->hdrdgst_en) 1685 iscsi_hdr_digest(conn, &mtask->headbuf, 1686 (u8*)mtask->hdrext); 1687 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) { 1688 mtask->xmstate |= XMSTATE_IMM_HDR; 1689 if (mtask->data_count) 1690 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1691 return -EAGAIN; 1692 } 1693 } 1694 1695 if (mtask->xmstate & XMSTATE_IMM_DATA) { 1696 BUG_ON(!mtask->data_count); 1697 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1698 /* FIXME: implement. 1699 * Virtual buffer could be spreaded across multiple pages... 1700 */ 1701 do { 1702 if (iscsi_sendpage(conn, &mtask->sendbuf, 1703 &mtask->data_count, &mtask->sent)) { 1704 mtask->xmstate |= XMSTATE_IMM_DATA; 1705 return -EAGAIN; 1706 } 1707 } while (mtask->data_count); 1708 } 1709 1710 BUG_ON(mtask->xmstate != XMSTATE_IDLE); 1711 return 0; 1712 } 1713 1714 static inline int 1715 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1716 { 1717 ctask->xmstate &= ~XMSTATE_R_HDR; 1718 if (conn->hdrdgst_en) 1719 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1720 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) { 1721 BUG_ON(ctask->xmstate != XMSTATE_IDLE); 1722 return 0; /* wait for Data-In */ 1723 } 1724 ctask->xmstate |= XMSTATE_R_HDR; 1725 return -EAGAIN; 1726 } 1727 1728 static inline int 1729 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1730 { 1731 ctask->xmstate &= ~XMSTATE_W_HDR; 1732 if (conn->hdrdgst_en) 1733 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1734 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) { 1735 ctask->xmstate |= XMSTATE_W_HDR; 1736 return -EAGAIN; 1737 } 1738 return 0; 1739 } 1740 1741 static inline int 1742 handle_xmstate_data_digest(struct iscsi_conn *conn, 1743 struct iscsi_cmd_task *ctask) 1744 { 1745 ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1746 debug_tcp("resent data digest 0x%x\n", ctask->datadigest); 1747 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1748 &ctask->datadigest, 0)) { 1749 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1750 debug_tcp("resent data digest 0x%x fail!\n", 1751 ctask->datadigest); 1752 return -EAGAIN; 1753 } 1754 return 0; 1755 } 1756 1757 static inline int 1758 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1759 { 1760 BUG_ON(!ctask->imm_count); 1761 ctask->xmstate &= ~XMSTATE_IMM_DATA; 1762 1763 if (conn->datadgst_en) { 1764 iscsi_data_digest_init(conn, ctask); 1765 ctask->immdigest = 0; 1766 } 1767 1768 for (;;) { 1769 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count, 1770 &ctask->sent)) { 1771 ctask->xmstate |= XMSTATE_IMM_DATA; 1772 if (conn->datadgst_en) { 1773 crypto_digest_final(conn->data_tx_tfm, 1774 (u8*)&ctask->immdigest); 1775 debug_tcp("tx imm sendpage fail 0x%x\n", 1776 ctask->datadigest); 1777 } 1778 return -EAGAIN; 1779 } 1780 if (conn->datadgst_en) 1781 crypto_digest_update(conn->data_tx_tfm, 1782 &ctask->sendbuf.sg, 1); 1783 1784 if (!ctask->imm_count) 1785 break; 1786 iscsi_buf_init_sg(&ctask->sendbuf, 1787 &ctask->sg[ctask->sg_count++]); 1788 } 1789 1790 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1791 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1792 &ctask->immdigest, 1)) { 1793 debug_tcp("sending imm digest 0x%x fail!\n", 1794 ctask->immdigest); 1795 return -EAGAIN; 1796 } 1797 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest); 1798 } 1799 1800 return 0; 1801 } 1802 1803 static inline int 1804 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1805 { 1806 struct iscsi_data_task *dtask; 1807 1808 ctask->xmstate |= XMSTATE_UNS_DATA; 1809 if (ctask->xmstate & XMSTATE_UNS_INIT) { 1810 iscsi_unsolicit_data_init(conn, ctask); 1811 BUG_ON(!ctask->dtask); 1812 dtask = ctask->dtask; 1813 if (conn->hdrdgst_en) 1814 iscsi_hdr_digest(conn, &ctask->headbuf, 1815 (u8*)dtask->hdrext); 1816 ctask->xmstate &= ~XMSTATE_UNS_INIT; 1817 } 1818 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) { 1819 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1820 ctask->xmstate |= XMSTATE_UNS_HDR; 1821 return -EAGAIN; 1822 } 1823 1824 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1825 ctask->itt, ctask->unsol_count, ctask->sent); 1826 return 0; 1827 } 1828 1829 static inline int 1830 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1831 { 1832 struct iscsi_data_task *dtask = ctask->dtask; 1833 1834 BUG_ON(!ctask->data_count); 1835 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1836 1837 if (conn->datadgst_en) { 1838 iscsi_data_digest_init(conn, ctask); 1839 dtask->digest = 0; 1840 } 1841 1842 for (;;) { 1843 int start = ctask->sent; 1844 1845 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count, 1846 &ctask->sent)) { 1847 ctask->unsol_count -= ctask->sent - start; 1848 ctask->xmstate |= XMSTATE_UNS_DATA; 1849 /* will continue with this ctask later.. */ 1850 if (conn->datadgst_en) { 1851 crypto_digest_final(conn->data_tx_tfm, 1852 (u8 *)&dtask->digest); 1853 debug_tcp("tx uns data fail 0x%x\n", 1854 dtask->digest); 1855 } 1856 return -EAGAIN; 1857 } 1858 1859 BUG_ON(ctask->sent > ctask->total_length); 1860 ctask->unsol_count -= ctask->sent - start; 1861 1862 /* 1863 * XXX:we may run here with un-initial sendbuf. 1864 * so pass it 1865 */ 1866 if (conn->datadgst_en && ctask->sent - start > 0) 1867 crypto_digest_update(conn->data_tx_tfm, 1868 &ctask->sendbuf.sg, 1); 1869 1870 if (!ctask->data_count) 1871 break; 1872 iscsi_buf_init_sg(&ctask->sendbuf, 1873 &ctask->sg[ctask->sg_count++]); 1874 } 1875 BUG_ON(ctask->unsol_count < 0); 1876 1877 /* 1878 * Done with the Data-Out. Next, check if we need 1879 * to send another unsolicited Data-Out. 1880 */ 1881 if (ctask->unsol_count) { 1882 if (conn->datadgst_en) { 1883 if (iscsi_digest_final_send(conn, ctask, 1884 &dtask->digestbuf, 1885 &dtask->digest, 1)) { 1886 debug_tcp("send uns digest 0x%x fail\n", 1887 dtask->digest); 1888 return -EAGAIN; 1889 } 1890 debug_tcp("sending uns digest 0x%x, more uns\n", 1891 dtask->digest); 1892 } 1893 ctask->xmstate |= XMSTATE_UNS_INIT; 1894 return 1; 1895 } 1896 1897 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1898 if (iscsi_digest_final_send(conn, ctask, 1899 &dtask->digestbuf, 1900 &dtask->digest, 1)) { 1901 debug_tcp("send last uns digest 0x%x fail\n", 1902 dtask->digest); 1903 return -EAGAIN; 1904 } 1905 debug_tcp("sending uns digest 0x%x\n",dtask->digest); 1906 } 1907 1908 return 0; 1909 } 1910 1911 static inline int 1912 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1913 { 1914 struct iscsi_session *session = conn->session; 1915 struct iscsi_r2t_info *r2t = ctask->r2t; 1916 struct iscsi_data_task *dtask = r2t->dtask; 1917 int left; 1918 1919 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1920 ctask->dtask = dtask; 1921 1922 if (conn->datadgst_en) { 1923 iscsi_data_digest_init(conn, ctask); 1924 dtask->digest = 0; 1925 } 1926 solicit_again: 1927 /* 1928 * send Data-Out whitnin this R2T sequence. 1929 */ 1930 if (!r2t->data_count) 1931 goto data_out_done; 1932 1933 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) { 1934 ctask->xmstate |= XMSTATE_SOL_DATA; 1935 /* will continue with this ctask later.. */ 1936 if (conn->datadgst_en) { 1937 crypto_digest_final(conn->data_tx_tfm, 1938 (u8 *)&dtask->digest); 1939 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1940 } 1941 return -EAGAIN; 1942 } 1943 1944 BUG_ON(r2t->data_count < 0); 1945 if (conn->datadgst_en) 1946 crypto_digest_update(conn->data_tx_tfm, &r2t->sendbuf.sg, 1); 1947 1948 if (r2t->data_count) { 1949 BUG_ON(ctask->sc->use_sg == 0); 1950 if (!iscsi_buf_left(&r2t->sendbuf)) { 1951 BUG_ON(ctask->bad_sg == r2t->sg); 1952 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1953 r2t->sg += 1; 1954 } 1955 goto solicit_again; 1956 } 1957 1958 data_out_done: 1959 /* 1960 * Done with this Data-Out. Next, check if we have 1961 * to send another Data-Out for this R2T. 1962 */ 1963 BUG_ON(r2t->data_length - r2t->sent < 0); 1964 left = r2t->data_length - r2t->sent; 1965 if (left) { 1966 if (conn->datadgst_en) { 1967 if (iscsi_digest_final_send(conn, ctask, 1968 &dtask->digestbuf, 1969 &dtask->digest, 1)) { 1970 debug_tcp("send r2t data digest 0x%x" 1971 "fail\n", dtask->digest); 1972 return -EAGAIN; 1973 } 1974 debug_tcp("r2t data send digest 0x%x\n", 1975 dtask->digest); 1976 } 1977 iscsi_solicit_data_cont(conn, ctask, r2t, left); 1978 ctask->xmstate |= XMSTATE_SOL_DATA; 1979 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1980 return 1; 1981 } 1982 1983 /* 1984 * Done with this R2T. Check if there are more 1985 * outstanding R2Ts ready to be processed. 1986 */ 1987 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0); 1988 if (conn->datadgst_en) { 1989 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf, 1990 &dtask->digest, 1)) { 1991 debug_tcp("send last r2t data digest 0x%x" 1992 "fail\n", dtask->digest); 1993 return -EAGAIN; 1994 } 1995 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest); 1996 } 1997 1998 ctask->r2t_data_count -= r2t->data_length; 1999 ctask->r2t = NULL; 2000 spin_lock_bh(&session->lock); 2001 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 2002 spin_unlock_bh(&session->lock); 2003 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) { 2004 ctask->r2t = r2t; 2005 ctask->xmstate |= XMSTATE_SOL_DATA; 2006 ctask->xmstate &= ~XMSTATE_SOL_HDR; 2007 return 1; 2008 } 2009 2010 return 0; 2011 } 2012 2013 static inline int 2014 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 2015 { 2016 struct iscsi_data_task *dtask = ctask->dtask; 2017 int sent; 2018 2019 ctask->xmstate &= ~XMSTATE_W_PAD; 2020 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad, 2021 ctask->pad_count); 2022 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) { 2023 ctask->xmstate |= XMSTATE_W_PAD; 2024 return -EAGAIN; 2025 } 2026 2027 if (conn->datadgst_en) { 2028 crypto_digest_update(conn->data_tx_tfm, &ctask->sendbuf.sg, 1); 2029 /* imm data? */ 2030 if (!dtask) { 2031 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 2032 &ctask->immdigest, 1)) { 2033 debug_tcp("send padding digest 0x%x" 2034 "fail!\n", ctask->immdigest); 2035 return -EAGAIN; 2036 } 2037 debug_tcp("done with padding, digest 0x%x\n", 2038 ctask->datadigest); 2039 } else { 2040 if (iscsi_digest_final_send(conn, ctask, 2041 &dtask->digestbuf, 2042 &dtask->digest, 1)) { 2043 debug_tcp("send padding digest 0x%x" 2044 "fail\n", dtask->digest); 2045 return -EAGAIN; 2046 } 2047 debug_tcp("done with padding, digest 0x%x\n", 2048 dtask->digest); 2049 } 2050 } 2051 2052 return 0; 2053 } 2054 2055 static int 2056 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 2057 { 2058 int rc = 0; 2059 2060 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n", 2061 conn->id, ctask->xmstate, ctask->itt); 2062 2063 /* 2064 * serialize with TMF AbortTask 2065 */ 2066 if (ctask->mtask) 2067 return rc; 2068 2069 if (ctask->xmstate & XMSTATE_R_HDR) { 2070 rc = handle_xmstate_r_hdr(conn, ctask); 2071 return rc; 2072 } 2073 2074 if (ctask->xmstate & XMSTATE_W_HDR) { 2075 rc = handle_xmstate_w_hdr(conn, ctask); 2076 if (rc) 2077 return rc; 2078 } 2079 2080 /* XXX: for data digest xmit recover */ 2081 if (ctask->xmstate & XMSTATE_DATA_DIGEST) { 2082 rc = handle_xmstate_data_digest(conn, ctask); 2083 if (rc) 2084 return rc; 2085 } 2086 2087 if (ctask->xmstate & XMSTATE_IMM_DATA) { 2088 rc = handle_xmstate_imm_data(conn, ctask); 2089 if (rc) 2090 return rc; 2091 } 2092 2093 if (ctask->xmstate & XMSTATE_UNS_HDR) { 2094 BUG_ON(!ctask->unsol_count); 2095 ctask->xmstate &= ~XMSTATE_UNS_HDR; 2096 unsolicit_head_again: 2097 rc = handle_xmstate_uns_hdr(conn, ctask); 2098 if (rc) 2099 return rc; 2100 } 2101 2102 if (ctask->xmstate & XMSTATE_UNS_DATA) { 2103 rc = handle_xmstate_uns_data(conn, ctask); 2104 if (rc == 1) 2105 goto unsolicit_head_again; 2106 else if (rc) 2107 return rc; 2108 goto done; 2109 } 2110 2111 if (ctask->xmstate & XMSTATE_SOL_HDR) { 2112 struct iscsi_r2t_info *r2t; 2113 2114 ctask->xmstate &= ~XMSTATE_SOL_HDR; 2115 ctask->xmstate |= XMSTATE_SOL_DATA; 2116 if (!ctask->r2t) 2117 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t, 2118 sizeof(void*)); 2119 solicit_head_again: 2120 r2t = ctask->r2t; 2121 if (conn->hdrdgst_en) 2122 iscsi_hdr_digest(conn, &r2t->headbuf, 2123 (u8*)r2t->dtask->hdrext); 2124 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) { 2125 ctask->xmstate &= ~XMSTATE_SOL_DATA; 2126 ctask->xmstate |= XMSTATE_SOL_HDR; 2127 return -EAGAIN; 2128 } 2129 2130 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n", 2131 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count, 2132 r2t->sent); 2133 } 2134 2135 if (ctask->xmstate & XMSTATE_SOL_DATA) { 2136 rc = handle_xmstate_sol_data(conn, ctask); 2137 if (rc == 1) 2138 goto solicit_head_again; 2139 if (rc) 2140 return rc; 2141 } 2142 2143 done: 2144 /* 2145 * Last thing to check is whether we need to send write 2146 * padding. Note that we check for xmstate equality, not just the bit. 2147 */ 2148 if (ctask->xmstate == XMSTATE_W_PAD) 2149 rc = handle_xmstate_w_pad(conn, ctask); 2150 2151 return rc; 2152 } 2153 2154 /** 2155 * iscsi_data_xmit - xmit any command into the scheduled connection 2156 * @conn: iscsi connection 2157 * 2158 * Notes: 2159 * The function can return -EAGAIN in which case the caller must 2160 * re-schedule it again later or recover. '0' return code means 2161 * successful xmit. 2162 **/ 2163 static int 2164 iscsi_data_xmit(struct iscsi_conn *conn) 2165 { 2166 if (unlikely(conn->suspend_tx)) { 2167 debug_tcp("conn %d Tx suspended!\n", conn->id); 2168 return 0; 2169 } 2170 2171 /* 2172 * Transmit in the following order: 2173 * 2174 * 1) un-finished xmit (ctask or mtask) 2175 * 2) immediate control PDUs 2176 * 3) write data 2177 * 4) SCSI commands 2178 * 5) non-immediate control PDUs 2179 * 2180 * No need to lock around __kfifo_get as long as 2181 * there's one producer and one consumer. 2182 */ 2183 2184 BUG_ON(conn->ctask && conn->mtask); 2185 2186 if (conn->ctask) { 2187 if (iscsi_ctask_xmit(conn, conn->ctask)) 2188 goto again; 2189 /* done with this in-progress ctask */ 2190 conn->ctask = NULL; 2191 } 2192 if (conn->mtask) { 2193 if (iscsi_mtask_xmit(conn, conn->mtask)) 2194 goto again; 2195 /* done with this in-progress mtask */ 2196 conn->mtask = NULL; 2197 } 2198 2199 /* process immediate first */ 2200 if (unlikely(__kfifo_len(conn->immqueue))) { 2201 struct iscsi_session *session = conn->session; 2202 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask, 2203 sizeof(void*))) { 2204 if (iscsi_mtask_xmit(conn, conn->mtask)) 2205 goto again; 2206 2207 if (conn->mtask->hdr.itt == 2208 cpu_to_be32(ISCSI_RESERVED_TAG)) { 2209 spin_lock_bh(&session->lock); 2210 __kfifo_put(session->mgmtpool.queue, 2211 (void*)&conn->mtask, sizeof(void*)); 2212 spin_unlock_bh(&session->lock); 2213 } 2214 } 2215 /* done with this mtask */ 2216 conn->mtask = NULL; 2217 } 2218 2219 /* process write queue */ 2220 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask, 2221 sizeof(void*))) { 2222 if (iscsi_ctask_xmit(conn, conn->ctask)) 2223 goto again; 2224 } 2225 2226 /* process command queue */ 2227 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask, 2228 sizeof(void*))) { 2229 if (iscsi_ctask_xmit(conn, conn->ctask)) 2230 goto again; 2231 } 2232 /* done with this ctask */ 2233 conn->ctask = NULL; 2234 2235 /* process the rest control plane PDUs, if any */ 2236 if (unlikely(__kfifo_len(conn->mgmtqueue))) { 2237 struct iscsi_session *session = conn->session; 2238 2239 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask, 2240 sizeof(void*))) { 2241 if (iscsi_mtask_xmit(conn, conn->mtask)) 2242 goto again; 2243 2244 if (conn->mtask->hdr.itt == 2245 cpu_to_be32(ISCSI_RESERVED_TAG)) { 2246 spin_lock_bh(&session->lock); 2247 __kfifo_put(session->mgmtpool.queue, 2248 (void*)&conn->mtask, 2249 sizeof(void*)); 2250 spin_unlock_bh(&session->lock); 2251 } 2252 } 2253 /* done with this mtask */ 2254 conn->mtask = NULL; 2255 } 2256 2257 return 0; 2258 2259 again: 2260 if (unlikely(conn->suspend_tx)) 2261 return 0; 2262 2263 return -EAGAIN; 2264 } 2265 2266 static void 2267 iscsi_xmitworker(void *data) 2268 { 2269 struct iscsi_conn *conn = data; 2270 2271 /* 2272 * serialize Xmit worker on a per-connection basis. 2273 */ 2274 mutex_lock(&conn->xmitmutex); 2275 if (iscsi_data_xmit(conn)) 2276 scsi_queue_work(conn->session->host, &conn->xmitwork); 2277 mutex_unlock(&conn->xmitmutex); 2278 } 2279 2280 #define FAILURE_BAD_HOST 1 2281 #define FAILURE_SESSION_FAILED 2 2282 #define FAILURE_SESSION_FREED 3 2283 #define FAILURE_WINDOW_CLOSED 4 2284 #define FAILURE_SESSION_TERMINATE 5 2285 2286 static int 2287 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *)) 2288 { 2289 struct Scsi_Host *host; 2290 int reason = 0; 2291 struct iscsi_session *session; 2292 struct iscsi_conn *conn = NULL; 2293 struct iscsi_cmd_task *ctask = NULL; 2294 2295 sc->scsi_done = done; 2296 sc->result = 0; 2297 2298 host = sc->device->host; 2299 session = iscsi_hostdata(host->hostdata); 2300 BUG_ON(host != session->host); 2301 2302 spin_lock(&session->lock); 2303 2304 if (session->state != ISCSI_STATE_LOGGED_IN) { 2305 if (session->state == ISCSI_STATE_FAILED) { 2306 reason = FAILURE_SESSION_FAILED; 2307 goto reject; 2308 } else if (session->state == ISCSI_STATE_TERMINATE) { 2309 reason = FAILURE_SESSION_TERMINATE; 2310 goto fault; 2311 } 2312 reason = FAILURE_SESSION_FREED; 2313 goto fault; 2314 } 2315 2316 /* 2317 * Check for iSCSI window and take care of CmdSN wrap-around 2318 */ 2319 if ((int)(session->max_cmdsn - session->cmdsn) < 0) { 2320 reason = FAILURE_WINDOW_CLOSED; 2321 goto reject; 2322 } 2323 2324 conn = session->leadconn; 2325 2326 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*)); 2327 BUG_ON(ctask->sc); 2328 2329 sc->SCp.phase = session->age; 2330 sc->SCp.ptr = (char*)ctask; 2331 iscsi_cmd_init(conn, ctask, sc); 2332 2333 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*)); 2334 debug_scsi( 2335 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n", 2336 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read", 2337 conn->id, (long)sc, ctask->itt, sc->request_bufflen, 2338 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1); 2339 spin_unlock(&session->lock); 2340 2341 scsi_queue_work(host, &conn->xmitwork); 2342 return 0; 2343 2344 reject: 2345 spin_unlock(&session->lock); 2346 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason); 2347 return SCSI_MLQUEUE_HOST_BUSY; 2348 2349 fault: 2350 spin_unlock(&session->lock); 2351 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n", 2352 sc->cmnd[0], reason); 2353 sc->sense_buffer[0] = 0x70; 2354 sc->sense_buffer[2] = NOT_READY; 2355 sc->sense_buffer[7] = 0x6; 2356 sc->sense_buffer[12] = 0x08; 2357 sc->sense_buffer[13] = 0x00; 2358 sc->result = (DID_NO_CONNECT << 16); 2359 sc->resid = sc->request_bufflen; 2360 sc->scsi_done(sc); 2361 return 0; 2362 } 2363 2364 static int 2365 iscsi_change_queue_depth(struct scsi_device *sdev, int depth) 2366 { 2367 if (depth > ISCSI_MAX_CMD_PER_LUN) 2368 depth = ISCSI_MAX_CMD_PER_LUN; 2369 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth); 2370 return sdev->queue_depth; 2371 } 2372 2373 static int 2374 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size) 2375 { 2376 int i; 2377 2378 *items = kmalloc(max * sizeof(void*), GFP_KERNEL); 2379 if (*items == NULL) 2380 return -ENOMEM; 2381 2382 q->max = max; 2383 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL); 2384 if (q->pool == NULL) { 2385 kfree(*items); 2386 return -ENOMEM; 2387 } 2388 2389 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*), 2390 GFP_KERNEL, NULL); 2391 if (q->queue == ERR_PTR(-ENOMEM)) { 2392 kfree(q->pool); 2393 kfree(*items); 2394 return -ENOMEM; 2395 } 2396 2397 for (i = 0; i < max; i++) { 2398 q->pool[i] = kmalloc(item_size, GFP_KERNEL); 2399 if (q->pool[i] == NULL) { 2400 int j; 2401 2402 for (j = 0; j < i; j++) 2403 kfree(q->pool[j]); 2404 2405 kfifo_free(q->queue); 2406 kfree(q->pool); 2407 kfree(*items); 2408 return -ENOMEM; 2409 } 2410 memset(q->pool[i], 0, item_size); 2411 (*items)[i] = q->pool[i]; 2412 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*)); 2413 } 2414 return 0; 2415 } 2416 2417 static void 2418 iscsi_pool_free(struct iscsi_queue *q, void **items) 2419 { 2420 int i; 2421 2422 for (i = 0; i < q->max; i++) 2423 kfree(items[i]); 2424 kfree(q->pool); 2425 kfree(items); 2426 } 2427 2428 static struct iscsi_cls_conn * 2429 iscsi_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 2430 { 2431 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 2432 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 2433 struct iscsi_conn *conn; 2434 struct iscsi_cls_conn *cls_conn; 2435 2436 cls_conn = iscsi_create_conn(cls_session, conn_idx); 2437 if (!cls_conn) 2438 return NULL; 2439 conn = cls_conn->dd_data; 2440 memset(conn, 0, sizeof(*conn)); 2441 2442 conn->cls_conn = cls_conn; 2443 conn->c_stage = ISCSI_CONN_INITIAL_STAGE; 2444 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2445 conn->id = conn_idx; 2446 conn->exp_statsn = 0; 2447 conn->tmabort_state = TMABORT_INITIAL; 2448 2449 /* initial operational parameters */ 2450 conn->hdr_size = sizeof(struct iscsi_hdr); 2451 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 2452 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 2453 2454 /* initialize general xmit PDU commands queue */ 2455 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*), 2456 GFP_KERNEL, NULL); 2457 if (conn->xmitqueue == ERR_PTR(-ENOMEM)) 2458 goto xmitqueue_alloc_fail; 2459 2460 /* initialize write response PDU commands queue */ 2461 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*), 2462 GFP_KERNEL, NULL); 2463 if (conn->writequeue == ERR_PTR(-ENOMEM)) 2464 goto writequeue_alloc_fail; 2465 2466 /* initialize general immediate & non-immediate PDU commands queue */ 2467 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*), 2468 GFP_KERNEL, NULL); 2469 if (conn->immqueue == ERR_PTR(-ENOMEM)) 2470 goto immqueue_alloc_fail; 2471 2472 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*), 2473 GFP_KERNEL, NULL); 2474 if (conn->mgmtqueue == ERR_PTR(-ENOMEM)) 2475 goto mgmtqueue_alloc_fail; 2476 2477 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn); 2478 2479 /* allocate login_mtask used for the login/text sequences */ 2480 spin_lock_bh(&session->lock); 2481 if (!__kfifo_get(session->mgmtpool.queue, 2482 (void*)&conn->login_mtask, 2483 sizeof(void*))) { 2484 spin_unlock_bh(&session->lock); 2485 goto login_mtask_alloc_fail; 2486 } 2487 spin_unlock_bh(&session->lock); 2488 2489 /* allocate initial PDU receive place holder */ 2490 if (conn->data_size <= PAGE_SIZE) 2491 conn->data = kmalloc(conn->data_size, GFP_KERNEL); 2492 else 2493 conn->data = (void*)__get_free_pages(GFP_KERNEL, 2494 get_order(conn->data_size)); 2495 if (!conn->data) 2496 goto max_recv_dlenght_alloc_fail; 2497 2498 init_timer(&conn->tmabort_timer); 2499 mutex_init(&conn->xmitmutex); 2500 init_waitqueue_head(&conn->ehwait); 2501 2502 return cls_conn; 2503 2504 max_recv_dlenght_alloc_fail: 2505 spin_lock_bh(&session->lock); 2506 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 2507 sizeof(void*)); 2508 spin_unlock_bh(&session->lock); 2509 login_mtask_alloc_fail: 2510 kfifo_free(conn->mgmtqueue); 2511 mgmtqueue_alloc_fail: 2512 kfifo_free(conn->immqueue); 2513 immqueue_alloc_fail: 2514 kfifo_free(conn->writequeue); 2515 writequeue_alloc_fail: 2516 kfifo_free(conn->xmitqueue); 2517 xmitqueue_alloc_fail: 2518 iscsi_destroy_conn(cls_conn); 2519 return NULL; 2520 } 2521 2522 static void 2523 iscsi_conn_destroy(struct iscsi_cls_conn *cls_conn) 2524 { 2525 struct iscsi_conn *conn = cls_conn->dd_data; 2526 struct iscsi_session *session = conn->session; 2527 unsigned long flags; 2528 2529 mutex_lock(&conn->xmitmutex); 2530 set_bit(SUSPEND_BIT, &conn->suspend_tx); 2531 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) { 2532 struct sock *sk = conn->sock->sk; 2533 2534 /* 2535 * conn_start() has never been called! 2536 * need to cleanup the socket. 2537 */ 2538 write_lock_bh(&sk->sk_callback_lock); 2539 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2540 write_unlock_bh(&sk->sk_callback_lock); 2541 2542 sock_hold(conn->sock->sk); 2543 iscsi_conn_restore_callbacks(conn); 2544 sock_put(conn->sock->sk); 2545 sock_release(conn->sock); 2546 conn->sock = NULL; 2547 } 2548 2549 spin_lock_bh(&session->lock); 2550 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT; 2551 if (session->leadconn == conn) { 2552 /* 2553 * leading connection? then give up on recovery. 2554 */ 2555 session->state = ISCSI_STATE_TERMINATE; 2556 wake_up(&conn->ehwait); 2557 } 2558 spin_unlock_bh(&session->lock); 2559 2560 mutex_unlock(&conn->xmitmutex); 2561 2562 /* 2563 * Block until all in-progress commands for this connection 2564 * time out or fail. 2565 */ 2566 for (;;) { 2567 spin_lock_irqsave(session->host->host_lock, flags); 2568 if (!session->host->host_busy) { /* OK for ERL == 0 */ 2569 spin_unlock_irqrestore(session->host->host_lock, flags); 2570 break; 2571 } 2572 spin_unlock_irqrestore(session->host->host_lock, flags); 2573 msleep_interruptible(500); 2574 printk("conn_destroy(): host_busy %d host_failed %d\n", 2575 session->host->host_busy, session->host->host_failed); 2576 /* 2577 * force eh_abort() to unblock 2578 */ 2579 wake_up(&conn->ehwait); 2580 } 2581 2582 /* now free crypto */ 2583 if (conn->hdrdgst_en || conn->datadgst_en) { 2584 if (conn->tx_tfm) 2585 crypto_free_tfm(conn->tx_tfm); 2586 if (conn->rx_tfm) 2587 crypto_free_tfm(conn->rx_tfm); 2588 if (conn->data_tx_tfm) 2589 crypto_free_tfm(conn->data_tx_tfm); 2590 if (conn->data_rx_tfm) 2591 crypto_free_tfm(conn->data_rx_tfm); 2592 } 2593 2594 /* free conn->data, size = MaxRecvDataSegmentLength */ 2595 if (conn->data_size <= PAGE_SIZE) 2596 kfree(conn->data); 2597 else 2598 free_pages((unsigned long)conn->data, 2599 get_order(conn->data_size)); 2600 2601 spin_lock_bh(&session->lock); 2602 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 2603 sizeof(void*)); 2604 list_del(&conn->item); 2605 if (list_empty(&session->connections)) 2606 session->leadconn = NULL; 2607 if (session->leadconn && session->leadconn == conn) 2608 session->leadconn = container_of(session->connections.next, 2609 struct iscsi_conn, item); 2610 2611 if (session->leadconn == NULL) 2612 /* none connections exits.. reset sequencing */ 2613 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1; 2614 spin_unlock_bh(&session->lock); 2615 2616 kfifo_free(conn->xmitqueue); 2617 kfifo_free(conn->writequeue); 2618 kfifo_free(conn->immqueue); 2619 kfifo_free(conn->mgmtqueue); 2620 2621 iscsi_destroy_conn(cls_conn); 2622 } 2623 2624 static int 2625 iscsi_conn_bind(struct iscsi_cls_session *cls_session, 2626 struct iscsi_cls_conn *cls_conn, uint32_t transport_fd, 2627 int is_leading) 2628 { 2629 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 2630 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 2631 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = cls_conn->dd_data; 2632 struct sock *sk; 2633 struct socket *sock; 2634 int err; 2635 2636 /* lookup for existing socket */ 2637 sock = sockfd_lookup(transport_fd, &err); 2638 if (!sock) { 2639 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err); 2640 return -EEXIST; 2641 } 2642 2643 /* lookup for existing connection */ 2644 spin_lock_bh(&session->lock); 2645 list_for_each_entry(tmp, &session->connections, item) { 2646 if (tmp == conn) { 2647 if (conn->c_stage != ISCSI_CONN_STOPPED || 2648 conn->stop_stage == STOP_CONN_TERM) { 2649 printk(KERN_ERR "iscsi_tcp: can't bind " 2650 "non-stopped connection (%d:%d)\n", 2651 conn->c_stage, conn->stop_stage); 2652 spin_unlock_bh(&session->lock); 2653 return -EIO; 2654 } 2655 break; 2656 } 2657 } 2658 if (tmp != conn) { 2659 /* bind new iSCSI connection to session */ 2660 conn->session = session; 2661 2662 list_add(&conn->item, &session->connections); 2663 } 2664 spin_unlock_bh(&session->lock); 2665 2666 if (conn->stop_stage != STOP_CONN_SUSPEND) { 2667 /* bind iSCSI connection and socket */ 2668 conn->sock = sock; 2669 2670 /* setup Socket parameters */ 2671 sk = sock->sk; 2672 sk->sk_reuse = 1; 2673 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 2674 sk->sk_allocation = GFP_ATOMIC; 2675 2676 /* FIXME: disable Nagle's algorithm */ 2677 2678 /* 2679 * Intercept TCP callbacks for sendfile like receive 2680 * processing. 2681 */ 2682 iscsi_conn_set_callbacks(conn); 2683 2684 conn->sendpage = conn->sock->ops->sendpage; 2685 2686 /* 2687 * set receive state machine into initial state 2688 */ 2689 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2690 } 2691 2692 if (is_leading) 2693 session->leadconn = conn; 2694 2695 /* 2696 * Unblock xmitworker(), Login Phase will pass through. 2697 */ 2698 clear_bit(SUSPEND_BIT, &conn->suspend_rx); 2699 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 2700 2701 return 0; 2702 } 2703 2704 static int 2705 iscsi_conn_start(struct iscsi_cls_conn *cls_conn) 2706 { 2707 struct iscsi_conn *conn = cls_conn->dd_data; 2708 struct iscsi_session *session = conn->session; 2709 struct sock *sk; 2710 2711 /* FF phase warming up... */ 2712 2713 if (session == NULL) { 2714 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n"); 2715 return -EPERM; 2716 } 2717 2718 sk = conn->sock->sk; 2719 2720 write_lock_bh(&sk->sk_callback_lock); 2721 spin_lock_bh(&session->lock); 2722 conn->c_stage = ISCSI_CONN_STARTED; 2723 session->state = ISCSI_STATE_LOGGED_IN; 2724 2725 switch(conn->stop_stage) { 2726 case STOP_CONN_RECOVER: 2727 /* 2728 * unblock eh_abort() if it is blocked. re-try all 2729 * commands after successful recovery 2730 */ 2731 session->conn_cnt++; 2732 conn->stop_stage = 0; 2733 conn->tmabort_state = TMABORT_INITIAL; 2734 session->age++; 2735 wake_up(&conn->ehwait); 2736 break; 2737 case STOP_CONN_TERM: 2738 session->conn_cnt++; 2739 conn->stop_stage = 0; 2740 break; 2741 case STOP_CONN_SUSPEND: 2742 conn->stop_stage = 0; 2743 clear_bit(SUSPEND_BIT, &conn->suspend_rx); 2744 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 2745 break; 2746 default: 2747 break; 2748 } 2749 spin_unlock_bh(&session->lock); 2750 write_unlock_bh(&sk->sk_callback_lock); 2751 2752 return 0; 2753 } 2754 2755 static void 2756 iscsi_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 2757 { 2758 struct iscsi_conn *conn = cls_conn->dd_data; 2759 struct iscsi_session *session = conn->session; 2760 struct sock *sk; 2761 unsigned long flags; 2762 2763 BUG_ON(!conn->sock); 2764 sk = conn->sock->sk; 2765 write_lock_bh(&sk->sk_callback_lock); 2766 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2767 write_unlock_bh(&sk->sk_callback_lock); 2768 2769 mutex_lock(&conn->xmitmutex); 2770 2771 spin_lock_irqsave(session->host->host_lock, flags); 2772 spin_lock(&session->lock); 2773 conn->stop_stage = flag; 2774 conn->c_stage = ISCSI_CONN_STOPPED; 2775 set_bit(SUSPEND_BIT, &conn->suspend_tx); 2776 2777 if (flag != STOP_CONN_SUSPEND) 2778 session->conn_cnt--; 2779 2780 if (session->conn_cnt == 0 || session->leadconn == conn) 2781 session->state = ISCSI_STATE_FAILED; 2782 2783 spin_unlock(&session->lock); 2784 spin_unlock_irqrestore(session->host->host_lock, flags); 2785 2786 if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) { 2787 struct iscsi_cmd_task *ctask; 2788 struct iscsi_mgmt_task *mtask; 2789 2790 /* 2791 * Socket must go now. 2792 */ 2793 sock_hold(conn->sock->sk); 2794 iscsi_conn_restore_callbacks(conn); 2795 sock_put(conn->sock->sk); 2796 2797 /* 2798 * flush xmit queues. 2799 */ 2800 spin_lock_bh(&session->lock); 2801 while (__kfifo_get(conn->writequeue, (void*)&ctask, 2802 sizeof(void*)) || 2803 __kfifo_get(conn->xmitqueue, (void*)&ctask, 2804 sizeof(void*))) { 2805 struct iscsi_r2t_info *r2t; 2806 2807 /* 2808 * flush ctask's r2t queues 2809 */ 2810 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t, 2811 sizeof(void*))) 2812 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, 2813 sizeof(void*)); 2814 2815 spin_unlock_bh(&session->lock); 2816 local_bh_disable(); 2817 iscsi_ctask_cleanup(conn, ctask); 2818 local_bh_enable(); 2819 spin_lock_bh(&session->lock); 2820 } 2821 conn->ctask = NULL; 2822 while (__kfifo_get(conn->immqueue, (void*)&mtask, 2823 sizeof(void*)) || 2824 __kfifo_get(conn->mgmtqueue, (void*)&mtask, 2825 sizeof(void*))) { 2826 __kfifo_put(session->mgmtpool.queue, 2827 (void*)&mtask, sizeof(void*)); 2828 } 2829 conn->mtask = NULL; 2830 spin_unlock_bh(&session->lock); 2831 2832 /* 2833 * release socket only after we stopped data_xmit() 2834 * activity and flushed all outstandings 2835 */ 2836 sock_release(conn->sock); 2837 conn->sock = NULL; 2838 2839 /* 2840 * for connection level recovery we should not calculate 2841 * header digest. conn->hdr_size used for optimization 2842 * in hdr_extract() and will be re-negotiated at 2843 * set_param() time. 2844 */ 2845 if (flag == STOP_CONN_RECOVER) { 2846 conn->hdr_size = sizeof(struct iscsi_hdr); 2847 conn->hdrdgst_en = 0; 2848 conn->datadgst_en = 0; 2849 } 2850 } 2851 mutex_unlock(&conn->xmitmutex); 2852 } 2853 2854 static int 2855 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr, 2856 char *data, uint32_t data_size) 2857 { 2858 struct iscsi_session *session = conn->session; 2859 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr; 2860 struct iscsi_mgmt_task *mtask; 2861 2862 spin_lock_bh(&session->lock); 2863 if (session->state == ISCSI_STATE_TERMINATE) { 2864 spin_unlock_bh(&session->lock); 2865 return -EPERM; 2866 } 2867 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) || 2868 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE)) 2869 /* 2870 * Login and Text are sent serially, in 2871 * request-followed-by-response sequence. 2872 * Same mtask can be used. Same ITT must be used. 2873 * Note that login_mtask is preallocated at conn_create(). 2874 */ 2875 mtask = conn->login_mtask; 2876 else { 2877 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE); 2878 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED); 2879 2880 if (!__kfifo_get(session->mgmtpool.queue, 2881 (void*)&mtask, sizeof(void*))) { 2882 spin_unlock_bh(&session->lock); 2883 return -ENOSPC; 2884 } 2885 } 2886 2887 /* 2888 * pre-format CmdSN and ExpStatSN for outgoing PDU. 2889 */ 2890 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 2891 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) | 2892 (session->age << AGE_SHIFT); 2893 nop->cmdsn = cpu_to_be32(session->cmdsn); 2894 if (conn->c_stage == ISCSI_CONN_STARTED && 2895 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 2896 session->cmdsn++; 2897 } else 2898 /* do not advance CmdSN */ 2899 nop->cmdsn = cpu_to_be32(session->cmdsn); 2900 2901 nop->exp_statsn = cpu_to_be32(conn->exp_statsn); 2902 2903 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr)); 2904 2905 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr, 2906 sizeof(struct iscsi_hdr)); 2907 2908 spin_unlock_bh(&session->lock); 2909 2910 if (data_size) { 2911 memcpy(mtask->data, data, data_size); 2912 mtask->data_count = data_size; 2913 } else 2914 mtask->data_count = 0; 2915 2916 mtask->xmstate = XMSTATE_IMM_HDR; 2917 2918 if (mtask->data_count) { 2919 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data, 2920 mtask->data_count); 2921 } 2922 2923 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n", 2924 hdr->opcode, hdr->itt, data_size); 2925 2926 /* 2927 * since send_pdu() could be called at least from two contexts, 2928 * we need to serialize __kfifo_put, so we don't have to take 2929 * additional lock on fast data-path 2930 */ 2931 if (hdr->opcode & ISCSI_OP_IMMEDIATE) 2932 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*)); 2933 else 2934 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*)); 2935 2936 scsi_queue_work(session->host, &conn->xmitwork); 2937 return 0; 2938 } 2939 2940 static int 2941 iscsi_eh_host_reset(struct scsi_cmnd *sc) 2942 { 2943 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr; 2944 struct iscsi_conn *conn = ctask->conn; 2945 struct iscsi_session *session = conn->session; 2946 2947 spin_lock_bh(&session->lock); 2948 if (session->state == ISCSI_STATE_TERMINATE) { 2949 debug_scsi("failing host reset: session terminated " 2950 "[CID %d age %d]", conn->id, session->age); 2951 spin_unlock_bh(&session->lock); 2952 return FAILED; 2953 } 2954 spin_unlock_bh(&session->lock); 2955 2956 debug_scsi("failing connection CID %d due to SCSI host reset " 2957 "[itt 0x%x age %d]", conn->id, ctask->itt, 2958 session->age); 2959 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 2960 2961 return SUCCESS; 2962 } 2963 2964 static void 2965 iscsi_tmabort_timedout(unsigned long data) 2966 { 2967 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data; 2968 struct iscsi_conn *conn = ctask->conn; 2969 struct iscsi_session *session = conn->session; 2970 2971 spin_lock(&session->lock); 2972 if (conn->tmabort_state == TMABORT_INITIAL) { 2973 __kfifo_put(session->mgmtpool.queue, 2974 (void*)&ctask->mtask, sizeof(void*)); 2975 conn->tmabort_state = TMABORT_TIMEDOUT; 2976 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n", 2977 (long)ctask->sc, ctask->itt); 2978 /* unblock eh_abort() */ 2979 wake_up(&conn->ehwait); 2980 } 2981 spin_unlock(&session->lock); 2982 } 2983 2984 static int 2985 iscsi_eh_abort(struct scsi_cmnd *sc) 2986 { 2987 int rc; 2988 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr; 2989 struct iscsi_conn *conn = ctask->conn; 2990 struct iscsi_session *session = conn->session; 2991 2992 conn->eh_abort_cnt++; 2993 debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 2994 2995 /* 2996 * two cases for ERL=0 here: 2997 * 2998 * 1) connection-level failure; 2999 * 2) recovery due protocol error; 3000 */ 3001 mutex_lock(&conn->xmitmutex); 3002 spin_lock_bh(&session->lock); 3003 if (session->state != ISCSI_STATE_LOGGED_IN) { 3004 if (session->state == ISCSI_STATE_TERMINATE) { 3005 spin_unlock_bh(&session->lock); 3006 mutex_unlock(&conn->xmitmutex); 3007 goto failed; 3008 } 3009 spin_unlock_bh(&session->lock); 3010 } else { 3011 struct iscsi_tm *hdr = &conn->tmhdr; 3012 3013 /* 3014 * Still LOGGED_IN... 3015 */ 3016 3017 if (!ctask->sc || sc->SCp.phase != session->age) { 3018 /* 3019 * 1) ctask completed before time out. But session 3020 * is still ok => Happy Retry. 3021 * 2) session was re-open during time out of ctask. 3022 */ 3023 spin_unlock_bh(&session->lock); 3024 mutex_unlock(&conn->xmitmutex); 3025 goto success; 3026 } 3027 conn->tmabort_state = TMABORT_INITIAL; 3028 spin_unlock_bh(&session->lock); 3029 3030 /* 3031 * ctask timed out but session is OK 3032 * ERL=0 requires task mgmt abort to be issued on each 3033 * failed command. requests must be serialized. 3034 */ 3035 memset(hdr, 0, sizeof(struct iscsi_tm)); 3036 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE; 3037 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK; 3038 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3039 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 3040 hdr->rtt = ctask->hdr.itt; 3041 hdr->refcmdsn = ctask->hdr.cmdsn; 3042 3043 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr, 3044 NULL, 0); 3045 if (rc) { 3046 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3047 debug_scsi("abort sent failure [itt 0x%x]", ctask->itt); 3048 } else { 3049 struct iscsi_r2t_info *r2t; 3050 3051 /* 3052 * TMF abort vs. TMF response race logic 3053 */ 3054 spin_lock_bh(&session->lock); 3055 ctask->mtask = (struct iscsi_mgmt_task *) 3056 session->mgmt_cmds[(hdr->itt & ITT_MASK) - 3057 ISCSI_MGMT_ITT_OFFSET]; 3058 /* 3059 * have to flush r2tqueue to avoid r2t leaks 3060 */ 3061 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t, 3062 sizeof(void*))) { 3063 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, 3064 sizeof(void*)); 3065 } 3066 if (conn->tmabort_state == TMABORT_INITIAL) { 3067 conn->tmfcmd_pdus_cnt++; 3068 conn->tmabort_timer.expires = 3*HZ + jiffies; 3069 conn->tmabort_timer.function = 3070 iscsi_tmabort_timedout; 3071 conn->tmabort_timer.data = (unsigned long)ctask; 3072 add_timer(&conn->tmabort_timer); 3073 debug_scsi("abort sent [itt 0x%x]", ctask->itt); 3074 } else { 3075 if (!ctask->sc || 3076 conn->tmabort_state == TMABORT_SUCCESS) { 3077 conn->tmabort_state = TMABORT_INITIAL; 3078 spin_unlock_bh(&session->lock); 3079 mutex_unlock(&conn->xmitmutex); 3080 goto success; 3081 } 3082 conn->tmabort_state = TMABORT_INITIAL; 3083 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3084 } 3085 spin_unlock_bh(&session->lock); 3086 } 3087 } 3088 mutex_unlock(&conn->xmitmutex); 3089 3090 3091 /* 3092 * block eh thread until: 3093 * 3094 * 1) abort response; 3095 * 2) abort timeout; 3096 * 3) session re-opened; 3097 * 4) session terminated; 3098 */ 3099 for (;;) { 3100 int p_state = session->state; 3101 3102 rc = wait_event_interruptible(conn->ehwait, 3103 (p_state == ISCSI_STATE_LOGGED_IN ? 3104 (session->state == ISCSI_STATE_TERMINATE || 3105 conn->tmabort_state != TMABORT_INITIAL) : 3106 (session->state == ISCSI_STATE_TERMINATE || 3107 session->state == ISCSI_STATE_LOGGED_IN))); 3108 if (rc) { 3109 /* shutdown.. */ 3110 session->state = ISCSI_STATE_TERMINATE; 3111 goto failed; 3112 } 3113 3114 if (signal_pending(current)) 3115 flush_signals(current); 3116 3117 if (session->state == ISCSI_STATE_TERMINATE) 3118 goto failed; 3119 3120 spin_lock_bh(&session->lock); 3121 if (sc->SCp.phase == session->age && 3122 (conn->tmabort_state == TMABORT_TIMEDOUT || 3123 conn->tmabort_state == TMABORT_FAILED)) { 3124 conn->tmabort_state = TMABORT_INITIAL; 3125 if (!ctask->sc) { 3126 /* 3127 * ctask completed before tmf abort response or 3128 * time out. 3129 * But session is still ok => Happy Retry. 3130 */ 3131 spin_unlock_bh(&session->lock); 3132 break; 3133 } 3134 spin_unlock_bh(&session->lock); 3135 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3136 continue; 3137 } 3138 spin_unlock_bh(&session->lock); 3139 break; 3140 } 3141 3142 success: 3143 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 3144 rc = SUCCESS; 3145 goto exit; 3146 3147 failed: 3148 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 3149 rc = FAILED; 3150 3151 exit: 3152 del_timer_sync(&conn->tmabort_timer); 3153 3154 mutex_lock(&conn->xmitmutex); 3155 if (conn->sock) { 3156 struct sock *sk = conn->sock->sk; 3157 3158 write_lock_bh(&sk->sk_callback_lock); 3159 iscsi_ctask_cleanup(conn, ctask); 3160 write_unlock_bh(&sk->sk_callback_lock); 3161 } 3162 mutex_unlock(&conn->xmitmutex); 3163 return rc; 3164 } 3165 3166 static int 3167 iscsi_r2tpool_alloc(struct iscsi_session *session) 3168 { 3169 int i; 3170 int cmd_i; 3171 3172 /* 3173 * initialize per-task: R2T pool and xmit queue 3174 */ 3175 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 3176 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 3177 3178 /* 3179 * pre-allocated x4 as much r2ts to handle race when 3180 * target acks DataOut faster than we data_xmit() queues 3181 * could replenish r2tqueue. 3182 */ 3183 3184 /* R2T pool */ 3185 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4, 3186 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) { 3187 goto r2t_alloc_fail; 3188 } 3189 3190 /* R2T xmit queue */ 3191 ctask->r2tqueue = kfifo_alloc( 3192 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL); 3193 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) { 3194 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 3195 goto r2t_alloc_fail; 3196 } 3197 3198 /* 3199 * number of 3200 * Data-Out PDU's within R2T-sequence can be quite big; 3201 * using mempool 3202 */ 3203 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX, 3204 mempool_alloc_slab, mempool_free_slab, taskcache); 3205 if (ctask->datapool == NULL) { 3206 kfifo_free(ctask->r2tqueue); 3207 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 3208 goto r2t_alloc_fail; 3209 } 3210 INIT_LIST_HEAD(&ctask->dataqueue); 3211 } 3212 3213 return 0; 3214 3215 r2t_alloc_fail: 3216 for (i = 0; i < cmd_i; i++) { 3217 mempool_destroy(session->cmds[i]->datapool); 3218 kfifo_free(session->cmds[i]->r2tqueue); 3219 iscsi_pool_free(&session->cmds[i]->r2tpool, 3220 (void**)session->cmds[i]->r2ts); 3221 } 3222 return -ENOMEM; 3223 } 3224 3225 static void 3226 iscsi_r2tpool_free(struct iscsi_session *session) 3227 { 3228 int i; 3229 3230 for (i = 0; i < session->cmds_max; i++) { 3231 mempool_destroy(session->cmds[i]->datapool); 3232 kfifo_free(session->cmds[i]->r2tqueue); 3233 iscsi_pool_free(&session->cmds[i]->r2tpool, 3234 (void**)session->cmds[i]->r2ts); 3235 } 3236 } 3237 3238 static struct scsi_host_template iscsi_sht = { 3239 .name = "iSCSI Initiator over TCP/IP, v." 3240 ISCSI_VERSION_STR, 3241 .queuecommand = iscsi_queuecommand, 3242 .change_queue_depth = iscsi_change_queue_depth, 3243 .can_queue = ISCSI_XMIT_CMDS_MAX - 1, 3244 .sg_tablesize = ISCSI_SG_TABLESIZE, 3245 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 3246 .eh_abort_handler = iscsi_eh_abort, 3247 .eh_host_reset_handler = iscsi_eh_host_reset, 3248 .use_clustering = DISABLE_CLUSTERING, 3249 .proc_name = "iscsi_tcp", 3250 .this_id = -1, 3251 }; 3252 3253 static struct iscsi_transport iscsi_tcp_transport; 3254 3255 static struct iscsi_cls_session * 3256 iscsi_session_create(struct scsi_transport_template *scsit, 3257 uint32_t initial_cmdsn, uint32_t *sid) 3258 { 3259 struct Scsi_Host *shost; 3260 struct iscsi_session *session; 3261 int cmd_i; 3262 3263 shost = iscsi_transport_create_session(scsit, &iscsi_tcp_transport); 3264 if (!shost) 3265 return NULL; 3266 3267 session = iscsi_hostdata(shost->hostdata); 3268 memset(session, 0, sizeof(struct iscsi_session)); 3269 session->host = shost; 3270 session->state = ISCSI_STATE_FREE; 3271 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX; 3272 session->cmds_max = ISCSI_XMIT_CMDS_MAX; 3273 session->cmdsn = initial_cmdsn; 3274 session->exp_cmdsn = initial_cmdsn + 1; 3275 session->max_cmdsn = initial_cmdsn + 1; 3276 session->max_r2t = 1; 3277 *sid = shost->host_no; 3278 3279 /* initialize SCSI PDU commands pool */ 3280 if (iscsi_pool_init(&session->cmdpool, session->cmds_max, 3281 (void***)&session->cmds, sizeof(struct iscsi_cmd_task))) 3282 goto cmdpool_alloc_fail; 3283 3284 /* pre-format cmds pool with ITT */ 3285 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) 3286 session->cmds[cmd_i]->itt = cmd_i; 3287 3288 spin_lock_init(&session->lock); 3289 INIT_LIST_HEAD(&session->connections); 3290 3291 /* initialize immediate command pool */ 3292 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max, 3293 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task))) 3294 goto mgmtpool_alloc_fail; 3295 3296 3297 /* pre-format immediate cmds pool with ITT */ 3298 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) { 3299 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i; 3300 session->mgmt_cmds[cmd_i]->data = kmalloc( 3301 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL); 3302 if (!session->mgmt_cmds[cmd_i]->data) { 3303 int j; 3304 3305 for (j = 0; j < cmd_i; j++) 3306 kfree(session->mgmt_cmds[j]->data); 3307 goto immdata_alloc_fail; 3308 } 3309 } 3310 3311 if (iscsi_r2tpool_alloc(session)) 3312 goto r2tpool_alloc_fail; 3313 3314 return hostdata_session(shost->hostdata); 3315 3316 r2tpool_alloc_fail: 3317 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) 3318 kfree(session->mgmt_cmds[cmd_i]->data); 3319 immdata_alloc_fail: 3320 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds); 3321 mgmtpool_alloc_fail: 3322 iscsi_pool_free(&session->cmdpool, (void**)session->cmds); 3323 cmdpool_alloc_fail: 3324 iscsi_transport_destroy_session(shost); 3325 return NULL; 3326 } 3327 3328 static void 3329 iscsi_session_destroy(struct iscsi_cls_session *cls_session) 3330 { 3331 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 3332 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 3333 int cmd_i; 3334 struct iscsi_data_task *dtask, *n; 3335 3336 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 3337 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 3338 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) { 3339 list_del(&dtask->item); 3340 mempool_free(dtask, ctask->datapool); 3341 } 3342 } 3343 3344 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) 3345 kfree(session->mgmt_cmds[cmd_i]->data); 3346 3347 iscsi_r2tpool_free(session); 3348 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds); 3349 iscsi_pool_free(&session->cmdpool, (void**)session->cmds); 3350 3351 iscsi_transport_destroy_session(shost); 3352 } 3353 3354 static int 3355 iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param, 3356 uint32_t value) 3357 { 3358 struct iscsi_conn *conn = cls_conn->dd_data; 3359 struct iscsi_session *session = conn->session; 3360 3361 spin_lock_bh(&session->lock); 3362 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 3363 conn->stop_stage != STOP_CONN_RECOVER) { 3364 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n", 3365 param); 3366 spin_unlock_bh(&session->lock); 3367 return 0; 3368 } 3369 spin_unlock_bh(&session->lock); 3370 3371 switch(param) { 3372 case ISCSI_PARAM_MAX_RECV_DLENGTH: { 3373 char *saveptr = conn->data; 3374 gfp_t flags = GFP_KERNEL; 3375 3376 if (conn->data_size >= value) { 3377 conn->max_recv_dlength = value; 3378 break; 3379 } 3380 3381 spin_lock_bh(&session->lock); 3382 if (conn->stop_stage == STOP_CONN_RECOVER) 3383 flags = GFP_ATOMIC; 3384 spin_unlock_bh(&session->lock); 3385 3386 if (value <= PAGE_SIZE) 3387 conn->data = kmalloc(value, flags); 3388 else 3389 conn->data = (void*)__get_free_pages(flags, 3390 get_order(value)); 3391 if (conn->data == NULL) { 3392 conn->data = saveptr; 3393 return -ENOMEM; 3394 } 3395 if (conn->data_size <= PAGE_SIZE) 3396 kfree(saveptr); 3397 else 3398 free_pages((unsigned long)saveptr, 3399 get_order(conn->data_size)); 3400 conn->max_recv_dlength = value; 3401 conn->data_size = value; 3402 } 3403 break; 3404 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 3405 conn->max_xmit_dlength = value; 3406 break; 3407 case ISCSI_PARAM_HDRDGST_EN: 3408 conn->hdrdgst_en = value; 3409 conn->hdr_size = sizeof(struct iscsi_hdr); 3410 if (conn->hdrdgst_en) { 3411 conn->hdr_size += sizeof(__u32); 3412 if (!conn->tx_tfm) 3413 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0); 3414 if (!conn->tx_tfm) 3415 return -ENOMEM; 3416 if (!conn->rx_tfm) 3417 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0); 3418 if (!conn->rx_tfm) { 3419 crypto_free_tfm(conn->tx_tfm); 3420 return -ENOMEM; 3421 } 3422 } else { 3423 if (conn->tx_tfm) 3424 crypto_free_tfm(conn->tx_tfm); 3425 if (conn->rx_tfm) 3426 crypto_free_tfm(conn->rx_tfm); 3427 } 3428 break; 3429 case ISCSI_PARAM_DATADGST_EN: 3430 conn->datadgst_en = value; 3431 if (conn->datadgst_en) { 3432 if (!conn->data_tx_tfm) 3433 conn->data_tx_tfm = 3434 crypto_alloc_tfm("crc32c", 0); 3435 if (!conn->data_tx_tfm) 3436 return -ENOMEM; 3437 if (!conn->data_rx_tfm) 3438 conn->data_rx_tfm = 3439 crypto_alloc_tfm("crc32c", 0); 3440 if (!conn->data_rx_tfm) { 3441 crypto_free_tfm(conn->data_tx_tfm); 3442 return -ENOMEM; 3443 } 3444 } else { 3445 if (conn->data_tx_tfm) 3446 crypto_free_tfm(conn->data_tx_tfm); 3447 if (conn->data_rx_tfm) 3448 crypto_free_tfm(conn->data_rx_tfm); 3449 } 3450 conn->sendpage = conn->datadgst_en ? 3451 sock_no_sendpage : conn->sock->ops->sendpage; 3452 break; 3453 case ISCSI_PARAM_INITIAL_R2T_EN: 3454 session->initial_r2t_en = value; 3455 break; 3456 case ISCSI_PARAM_MAX_R2T: 3457 if (session->max_r2t == roundup_pow_of_two(value)) 3458 break; 3459 iscsi_r2tpool_free(session); 3460 session->max_r2t = value; 3461 if (session->max_r2t & (session->max_r2t - 1)) 3462 session->max_r2t = roundup_pow_of_two(session->max_r2t); 3463 if (iscsi_r2tpool_alloc(session)) 3464 return -ENOMEM; 3465 break; 3466 case ISCSI_PARAM_IMM_DATA_EN: 3467 session->imm_data_en = value; 3468 break; 3469 case ISCSI_PARAM_FIRST_BURST: 3470 session->first_burst = value; 3471 break; 3472 case ISCSI_PARAM_MAX_BURST: 3473 session->max_burst = value; 3474 break; 3475 case ISCSI_PARAM_PDU_INORDER_EN: 3476 session->pdu_inorder_en = value; 3477 break; 3478 case ISCSI_PARAM_DATASEQ_INORDER_EN: 3479 session->dataseq_inorder_en = value; 3480 break; 3481 case ISCSI_PARAM_ERL: 3482 session->erl = value; 3483 break; 3484 case ISCSI_PARAM_IFMARKER_EN: 3485 BUG_ON(value); 3486 session->ifmarker_en = value; 3487 break; 3488 case ISCSI_PARAM_OFMARKER_EN: 3489 BUG_ON(value); 3490 session->ofmarker_en = value; 3491 break; 3492 default: 3493 break; 3494 } 3495 3496 return 0; 3497 } 3498 3499 static int 3500 iscsi_session_get_param(struct iscsi_cls_session *cls_session, 3501 enum iscsi_param param, uint32_t *value) 3502 { 3503 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 3504 struct iscsi_session *session = iscsi_hostdata(shost->hostdata); 3505 3506 switch(param) { 3507 case ISCSI_PARAM_INITIAL_R2T_EN: 3508 *value = session->initial_r2t_en; 3509 break; 3510 case ISCSI_PARAM_MAX_R2T: 3511 *value = session->max_r2t; 3512 break; 3513 case ISCSI_PARAM_IMM_DATA_EN: 3514 *value = session->imm_data_en; 3515 break; 3516 case ISCSI_PARAM_FIRST_BURST: 3517 *value = session->first_burst; 3518 break; 3519 case ISCSI_PARAM_MAX_BURST: 3520 *value = session->max_burst; 3521 break; 3522 case ISCSI_PARAM_PDU_INORDER_EN: 3523 *value = session->pdu_inorder_en; 3524 break; 3525 case ISCSI_PARAM_DATASEQ_INORDER_EN: 3526 *value = session->dataseq_inorder_en; 3527 break; 3528 case ISCSI_PARAM_ERL: 3529 *value = session->erl; 3530 break; 3531 case ISCSI_PARAM_IFMARKER_EN: 3532 *value = session->ifmarker_en; 3533 break; 3534 case ISCSI_PARAM_OFMARKER_EN: 3535 *value = session->ofmarker_en; 3536 break; 3537 default: 3538 return ISCSI_ERR_PARAM_NOT_FOUND; 3539 } 3540 3541 return 0; 3542 } 3543 3544 static int 3545 iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn, 3546 enum iscsi_param param, uint32_t *value) 3547 { 3548 struct iscsi_conn *conn = cls_conn->dd_data; 3549 3550 switch(param) { 3551 case ISCSI_PARAM_MAX_RECV_DLENGTH: 3552 *value = conn->max_recv_dlength; 3553 break; 3554 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 3555 *value = conn->max_xmit_dlength; 3556 break; 3557 case ISCSI_PARAM_HDRDGST_EN: 3558 *value = conn->hdrdgst_en; 3559 break; 3560 case ISCSI_PARAM_DATADGST_EN: 3561 *value = conn->datadgst_en; 3562 break; 3563 default: 3564 return ISCSI_ERR_PARAM_NOT_FOUND; 3565 } 3566 3567 return 0; 3568 } 3569 3570 static void 3571 iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 3572 { 3573 struct iscsi_conn *conn = cls_conn->dd_data; 3574 3575 stats->txdata_octets = conn->txdata_octets; 3576 stats->rxdata_octets = conn->rxdata_octets; 3577 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 3578 stats->dataout_pdus = conn->dataout_pdus_cnt; 3579 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 3580 stats->datain_pdus = conn->datain_pdus_cnt; 3581 stats->r2t_pdus = conn->r2t_pdus_cnt; 3582 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 3583 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 3584 stats->custom_length = 3; 3585 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 3586 stats->custom[0].value = conn->sendpage_failures_cnt; 3587 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 3588 stats->custom[1].value = conn->discontiguous_hdr_cnt; 3589 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 3590 stats->custom[2].value = conn->eh_abort_cnt; 3591 } 3592 3593 static int 3594 iscsi_conn_send_pdu(struct iscsi_cls_conn *cls_conn, struct iscsi_hdr *hdr, 3595 char *data, uint32_t data_size) 3596 { 3597 struct iscsi_conn *conn = cls_conn->dd_data; 3598 int rc; 3599 3600 mutex_lock(&conn->xmitmutex); 3601 rc = iscsi_conn_send_generic(conn, hdr, data, data_size); 3602 mutex_unlock(&conn->xmitmutex); 3603 3604 return rc; 3605 } 3606 3607 static struct iscsi_transport iscsi_tcp_transport = { 3608 .owner = THIS_MODULE, 3609 .name = "tcp", 3610 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 3611 | CAP_DATADGST, 3612 .host_template = &iscsi_sht, 3613 .hostdata_size = sizeof(struct iscsi_session), 3614 .conndata_size = sizeof(struct iscsi_conn), 3615 .max_conn = 1, 3616 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN, 3617 .create_session = iscsi_session_create, 3618 .destroy_session = iscsi_session_destroy, 3619 .create_conn = iscsi_conn_create, 3620 .bind_conn = iscsi_conn_bind, 3621 .destroy_conn = iscsi_conn_destroy, 3622 .set_param = iscsi_conn_set_param, 3623 .get_conn_param = iscsi_conn_get_param, 3624 .get_session_param = iscsi_session_get_param, 3625 .start_conn = iscsi_conn_start, 3626 .stop_conn = iscsi_conn_stop, 3627 .send_pdu = iscsi_conn_send_pdu, 3628 .get_stats = iscsi_conn_get_stats, 3629 }; 3630 3631 static int __init 3632 iscsi_tcp_init(void) 3633 { 3634 if (iscsi_max_lun < 1) { 3635 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun); 3636 return -EINVAL; 3637 } 3638 iscsi_tcp_transport.max_lun = iscsi_max_lun; 3639 3640 taskcache = kmem_cache_create("iscsi_taskcache", 3641 sizeof(struct iscsi_data_task), 0, 3642 SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL); 3643 if (!taskcache) 3644 return -ENOMEM; 3645 3646 if (!iscsi_register_transport(&iscsi_tcp_transport)) 3647 kmem_cache_destroy(taskcache); 3648 3649 return 0; 3650 } 3651 3652 static void __exit 3653 iscsi_tcp_exit(void) 3654 { 3655 iscsi_unregister_transport(&iscsi_tcp_transport); 3656 kmem_cache_destroy(taskcache); 3657 } 3658 3659 module_init(iscsi_tcp_init); 3660 module_exit(iscsi_tcp_exit); 3661